xref: /linux/drivers/iommu/iommufd/selftest.c (revision 5027ec19f1049a07df5b0a37b1f462514cf2724b)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
3  *
4  * Kernel side components to support tools/testing/selftests/iommu
5  */
6 #include <linux/slab.h>
7 #include <linux/iommu.h>
8 #include <linux/xarray.h>
9 #include <linux/file.h>
10 #include <linux/anon_inodes.h>
11 #include <linux/fault-inject.h>
12 #include <linux/platform_device.h>
13 #include <uapi/linux/iommufd.h>
14 
15 #include "../iommu-priv.h"
16 #include "io_pagetable.h"
17 #include "iommufd_private.h"
18 #include "iommufd_test.h"
19 
20 static DECLARE_FAULT_ATTR(fail_iommufd);
21 static struct dentry *dbgfs_root;
22 static struct platform_device *selftest_iommu_dev;
23 static const struct iommu_ops mock_ops;
24 static struct iommu_domain_ops domain_nested_ops;
25 
26 size_t iommufd_test_memory_limit = 65536;
27 
28 enum {
29 	MOCK_DIRTY_TRACK = 1,
30 	MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2,
31 
32 	/*
33 	 * Like a real page table alignment requires the low bits of the address
34 	 * to be zero. xarray also requires the high bit to be zero, so we store
35 	 * the pfns shifted. The upper bits are used for metadata.
36 	 */
37 	MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE,
38 
39 	_MOCK_PFN_START = MOCK_PFN_MASK + 1,
40 	MOCK_PFN_START_IOVA = _MOCK_PFN_START,
41 	MOCK_PFN_LAST_IOVA = _MOCK_PFN_START,
42 	MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1,
43 };
44 
45 /*
46  * Syzkaller has trouble randomizing the correct iova to use since it is linked
47  * to the map ioctl's output, and it has no ide about that. So, simplify things.
48  * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset
49  * value. This has a much smaller randomization space and syzkaller can hit it.
50  */
51 static unsigned long iommufd_test_syz_conv_iova(struct io_pagetable *iopt,
52 						u64 *iova)
53 {
54 	struct syz_layout {
55 		__u32 nth_area;
56 		__u32 offset;
57 	};
58 	struct syz_layout *syz = (void *)iova;
59 	unsigned int nth = syz->nth_area;
60 	struct iopt_area *area;
61 
62 	down_read(&iopt->iova_rwsem);
63 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
64 	     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
65 		if (nth == 0) {
66 			up_read(&iopt->iova_rwsem);
67 			return iopt_area_iova(area) + syz->offset;
68 		}
69 		nth--;
70 	}
71 	up_read(&iopt->iova_rwsem);
72 
73 	return 0;
74 }
75 
76 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
77 				   unsigned int ioas_id, u64 *iova, u32 *flags)
78 {
79 	struct iommufd_ioas *ioas;
80 
81 	if (!(*flags & MOCK_FLAGS_ACCESS_SYZ))
82 		return;
83 	*flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ;
84 
85 	ioas = iommufd_get_ioas(ucmd->ictx, ioas_id);
86 	if (IS_ERR(ioas))
87 		return;
88 	*iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova);
89 	iommufd_put_object(&ioas->obj);
90 }
91 
92 struct mock_iommu_domain {
93 	unsigned long flags;
94 	struct iommu_domain domain;
95 	struct xarray pfns;
96 };
97 
98 struct mock_iommu_domain_nested {
99 	struct iommu_domain domain;
100 	struct mock_iommu_domain *parent;
101 	u32 iotlb[MOCK_NESTED_DOMAIN_IOTLB_NUM];
102 };
103 
104 enum selftest_obj_type {
105 	TYPE_IDEV,
106 };
107 
108 struct mock_dev {
109 	struct device dev;
110 	unsigned long flags;
111 };
112 
113 struct selftest_obj {
114 	struct iommufd_object obj;
115 	enum selftest_obj_type type;
116 
117 	union {
118 		struct {
119 			struct iommufd_device *idev;
120 			struct iommufd_ctx *ictx;
121 			struct mock_dev *mock_dev;
122 		} idev;
123 	};
124 };
125 
126 static int mock_domain_nop_attach(struct iommu_domain *domain,
127 				  struct device *dev)
128 {
129 	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
130 
131 	if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY))
132 		return -EINVAL;
133 
134 	return 0;
135 }
136 
137 static const struct iommu_domain_ops mock_blocking_ops = {
138 	.attach_dev = mock_domain_nop_attach,
139 };
140 
141 static struct iommu_domain mock_blocking_domain = {
142 	.type = IOMMU_DOMAIN_BLOCKED,
143 	.ops = &mock_blocking_ops,
144 };
145 
146 static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type)
147 {
148 	struct iommu_test_hw_info *info;
149 
150 	info = kzalloc(sizeof(*info), GFP_KERNEL);
151 	if (!info)
152 		return ERR_PTR(-ENOMEM);
153 
154 	info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL;
155 	*length = sizeof(*info);
156 	*type = IOMMU_HW_INFO_TYPE_SELFTEST;
157 
158 	return info;
159 }
160 
161 static int mock_domain_set_dirty_tracking(struct iommu_domain *domain,
162 					  bool enable)
163 {
164 	struct mock_iommu_domain *mock =
165 		container_of(domain, struct mock_iommu_domain, domain);
166 	unsigned long flags = mock->flags;
167 
168 	if (enable && !domain->dirty_ops)
169 		return -EINVAL;
170 
171 	/* No change? */
172 	if (!(enable ^ !!(flags & MOCK_DIRTY_TRACK)))
173 		return 0;
174 
175 	flags = (enable ? flags | MOCK_DIRTY_TRACK : flags & ~MOCK_DIRTY_TRACK);
176 
177 	mock->flags = flags;
178 	return 0;
179 }
180 
181 static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain,
182 					    unsigned long iova, size_t size,
183 					    unsigned long flags,
184 					    struct iommu_dirty_bitmap *dirty)
185 {
186 	struct mock_iommu_domain *mock =
187 		container_of(domain, struct mock_iommu_domain, domain);
188 	unsigned long i, max = size / MOCK_IO_PAGE_SIZE;
189 	void *ent, *old;
190 
191 	if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap)
192 		return -EINVAL;
193 
194 	for (i = 0; i < max; i++) {
195 		unsigned long cur = iova + i * MOCK_IO_PAGE_SIZE;
196 
197 		ent = xa_load(&mock->pfns, cur / MOCK_IO_PAGE_SIZE);
198 		if (ent && (xa_to_value(ent) & MOCK_PFN_DIRTY_IOVA)) {
199 			/* Clear dirty */
200 			if (!(flags & IOMMU_DIRTY_NO_CLEAR)) {
201 				unsigned long val;
202 
203 				val = xa_to_value(ent) & ~MOCK_PFN_DIRTY_IOVA;
204 				old = xa_store(&mock->pfns,
205 					       cur / MOCK_IO_PAGE_SIZE,
206 					       xa_mk_value(val), GFP_KERNEL);
207 				WARN_ON_ONCE(ent != old);
208 			}
209 			iommu_dirty_bitmap_record(dirty, cur,
210 						  MOCK_IO_PAGE_SIZE);
211 		}
212 	}
213 
214 	return 0;
215 }
216 
217 const struct iommu_dirty_ops dirty_ops = {
218 	.set_dirty_tracking = mock_domain_set_dirty_tracking,
219 	.read_and_clear_dirty = mock_domain_read_and_clear_dirty,
220 };
221 
222 static struct iommu_domain *mock_domain_alloc_paging(struct device *dev)
223 {
224 	struct mock_iommu_domain *mock;
225 
226 	mock = kzalloc(sizeof(*mock), GFP_KERNEL);
227 	if (!mock)
228 		return NULL;
229 	mock->domain.geometry.aperture_start = MOCK_APERTURE_START;
230 	mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST;
231 	mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE;
232 	mock->domain.ops = mock_ops.default_domain_ops;
233 	mock->domain.type = IOMMU_DOMAIN_UNMANAGED;
234 	xa_init(&mock->pfns);
235 	return &mock->domain;
236 }
237 
238 static struct iommu_domain *
239 __mock_domain_alloc_nested(struct mock_iommu_domain *mock_parent,
240 			   const struct iommu_hwpt_selftest *user_cfg)
241 {
242 	struct mock_iommu_domain_nested *mock_nested;
243 	int i;
244 
245 	mock_nested = kzalloc(sizeof(*mock_nested), GFP_KERNEL);
246 	if (!mock_nested)
247 		return ERR_PTR(-ENOMEM);
248 	mock_nested->parent = mock_parent;
249 	mock_nested->domain.ops = &domain_nested_ops;
250 	mock_nested->domain.type = IOMMU_DOMAIN_NESTED;
251 	for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++)
252 		mock_nested->iotlb[i] = user_cfg->iotlb;
253 	return &mock_nested->domain;
254 }
255 
256 static struct iommu_domain *
257 mock_domain_alloc_user(struct device *dev, u32 flags,
258 		       struct iommu_domain *parent,
259 		       const struct iommu_user_data *user_data)
260 {
261 	struct mock_iommu_domain *mock_parent;
262 	struct iommu_hwpt_selftest user_cfg;
263 	int rc;
264 
265 	/* must be mock_domain */
266 	if (!parent) {
267 		struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
268 		bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
269 		bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY;
270 		struct iommu_domain *domain;
271 
272 		if (flags & (~(IOMMU_HWPT_ALLOC_NEST_PARENT |
273 			       IOMMU_HWPT_ALLOC_DIRTY_TRACKING)))
274 			return ERR_PTR(-EOPNOTSUPP);
275 		if (user_data || (has_dirty_flag && no_dirty_ops))
276 			return ERR_PTR(-EOPNOTSUPP);
277 		domain = mock_domain_alloc_paging(NULL);
278 		if (!domain)
279 			return ERR_PTR(-ENOMEM);
280 		if (has_dirty_flag)
281 			container_of(domain, struct mock_iommu_domain, domain)
282 				->domain.dirty_ops = &dirty_ops;
283 		return domain;
284 	}
285 
286 	/* must be mock_domain_nested */
287 	if (user_data->type != IOMMU_HWPT_DATA_SELFTEST || flags)
288 		return ERR_PTR(-EOPNOTSUPP);
289 	if (!parent || parent->ops != mock_ops.default_domain_ops)
290 		return ERR_PTR(-EINVAL);
291 
292 	mock_parent = container_of(parent, struct mock_iommu_domain, domain);
293 	if (!mock_parent)
294 		return ERR_PTR(-EINVAL);
295 
296 	rc = iommu_copy_struct_from_user(&user_cfg, user_data,
297 					 IOMMU_HWPT_DATA_SELFTEST, iotlb);
298 	if (rc)
299 		return ERR_PTR(rc);
300 
301 	return __mock_domain_alloc_nested(mock_parent, &user_cfg);
302 }
303 
304 static void mock_domain_free(struct iommu_domain *domain)
305 {
306 	struct mock_iommu_domain *mock =
307 		container_of(domain, struct mock_iommu_domain, domain);
308 
309 	WARN_ON(!xa_empty(&mock->pfns));
310 	kfree(mock);
311 }
312 
313 static int mock_domain_map_pages(struct iommu_domain *domain,
314 				 unsigned long iova, phys_addr_t paddr,
315 				 size_t pgsize, size_t pgcount, int prot,
316 				 gfp_t gfp, size_t *mapped)
317 {
318 	struct mock_iommu_domain *mock =
319 		container_of(domain, struct mock_iommu_domain, domain);
320 	unsigned long flags = MOCK_PFN_START_IOVA;
321 	unsigned long start_iova = iova;
322 
323 	/*
324 	 * xarray does not reliably work with fault injection because it does a
325 	 * retry allocation, so put our own failure point.
326 	 */
327 	if (iommufd_should_fail())
328 		return -ENOENT;
329 
330 	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
331 	WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
332 	for (; pgcount; pgcount--) {
333 		size_t cur;
334 
335 		for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
336 			void *old;
337 
338 			if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
339 				flags = MOCK_PFN_LAST_IOVA;
340 			old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE,
341 				       xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) |
342 						   flags),
343 				       gfp);
344 			if (xa_is_err(old)) {
345 				for (; start_iova != iova;
346 				     start_iova += MOCK_IO_PAGE_SIZE)
347 					xa_erase(&mock->pfns,
348 						 start_iova /
349 							 MOCK_IO_PAGE_SIZE);
350 				return xa_err(old);
351 			}
352 			WARN_ON(old);
353 			iova += MOCK_IO_PAGE_SIZE;
354 			paddr += MOCK_IO_PAGE_SIZE;
355 			*mapped += MOCK_IO_PAGE_SIZE;
356 			flags = 0;
357 		}
358 	}
359 	return 0;
360 }
361 
362 static size_t mock_domain_unmap_pages(struct iommu_domain *domain,
363 				      unsigned long iova, size_t pgsize,
364 				      size_t pgcount,
365 				      struct iommu_iotlb_gather *iotlb_gather)
366 {
367 	struct mock_iommu_domain *mock =
368 		container_of(domain, struct mock_iommu_domain, domain);
369 	bool first = true;
370 	size_t ret = 0;
371 	void *ent;
372 
373 	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
374 	WARN_ON(pgsize % MOCK_IO_PAGE_SIZE);
375 
376 	for (; pgcount; pgcount--) {
377 		size_t cur;
378 
379 		for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) {
380 			ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
381 
382 			/*
383 			 * iommufd generates unmaps that must be a strict
384 			 * superset of the map's performend So every starting
385 			 * IOVA should have been an iova passed to map, and the
386 			 *
387 			 * First IOVA must be present and have been a first IOVA
388 			 * passed to map_pages
389 			 */
390 			if (first) {
391 				WARN_ON(ent && !(xa_to_value(ent) &
392 						 MOCK_PFN_START_IOVA));
393 				first = false;
394 			}
395 			if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize)
396 				WARN_ON(ent && !(xa_to_value(ent) &
397 						 MOCK_PFN_LAST_IOVA));
398 
399 			iova += MOCK_IO_PAGE_SIZE;
400 			ret += MOCK_IO_PAGE_SIZE;
401 		}
402 	}
403 	return ret;
404 }
405 
406 static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain,
407 					    dma_addr_t iova)
408 {
409 	struct mock_iommu_domain *mock =
410 		container_of(domain, struct mock_iommu_domain, domain);
411 	void *ent;
412 
413 	WARN_ON(iova % MOCK_IO_PAGE_SIZE);
414 	ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
415 	WARN_ON(!ent);
416 	return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE;
417 }
418 
419 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap)
420 {
421 	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
422 
423 	switch (cap) {
424 	case IOMMU_CAP_CACHE_COHERENCY:
425 		return true;
426 	case IOMMU_CAP_DIRTY_TRACKING:
427 		return !(mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY);
428 	default:
429 		break;
430 	}
431 
432 	return false;
433 }
434 
435 static struct iommu_device mock_iommu_device = {
436 };
437 
438 static struct iommu_device *mock_probe_device(struct device *dev)
439 {
440 	return &mock_iommu_device;
441 }
442 
443 static const struct iommu_ops mock_ops = {
444 	/*
445 	 * IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type()
446 	 * because it is zero.
447 	 */
448 	.default_domain = &mock_blocking_domain,
449 	.blocked_domain = &mock_blocking_domain,
450 	.owner = THIS_MODULE,
451 	.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
452 	.hw_info = mock_domain_hw_info,
453 	.domain_alloc_paging = mock_domain_alloc_paging,
454 	.domain_alloc_user = mock_domain_alloc_user,
455 	.capable = mock_domain_capable,
456 	.device_group = generic_device_group,
457 	.probe_device = mock_probe_device,
458 	.default_domain_ops =
459 		&(struct iommu_domain_ops){
460 			.free = mock_domain_free,
461 			.attach_dev = mock_domain_nop_attach,
462 			.map_pages = mock_domain_map_pages,
463 			.unmap_pages = mock_domain_unmap_pages,
464 			.iova_to_phys = mock_domain_iova_to_phys,
465 		},
466 };
467 
468 static void mock_domain_free_nested(struct iommu_domain *domain)
469 {
470 	struct mock_iommu_domain_nested *mock_nested =
471 		container_of(domain, struct mock_iommu_domain_nested, domain);
472 
473 	kfree(mock_nested);
474 }
475 
476 static struct iommu_domain_ops domain_nested_ops = {
477 	.free = mock_domain_free_nested,
478 	.attach_dev = mock_domain_nop_attach,
479 };
480 
481 static inline struct iommufd_hw_pagetable *
482 __get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, u32 hwpt_type)
483 {
484 	struct iommufd_object *obj;
485 
486 	obj = iommufd_get_object(ucmd->ictx, mockpt_id, hwpt_type);
487 	if (IS_ERR(obj))
488 		return ERR_CAST(obj);
489 	return container_of(obj, struct iommufd_hw_pagetable, obj);
490 }
491 
492 static inline struct iommufd_hw_pagetable *
493 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id,
494 		 struct mock_iommu_domain **mock)
495 {
496 	struct iommufd_hw_pagetable *hwpt;
497 
498 	hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_PAGING);
499 	if (IS_ERR(hwpt))
500 		return hwpt;
501 	if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED ||
502 	    hwpt->domain->ops != mock_ops.default_domain_ops) {
503 		iommufd_put_object(&hwpt->obj);
504 		return ERR_PTR(-EINVAL);
505 	}
506 	*mock = container_of(hwpt->domain, struct mock_iommu_domain, domain);
507 	return hwpt;
508 }
509 
510 static inline struct iommufd_hw_pagetable *
511 get_md_pagetable_nested(struct iommufd_ucmd *ucmd, u32 mockpt_id,
512 			struct mock_iommu_domain_nested **mock_nested)
513 {
514 	struct iommufd_hw_pagetable *hwpt;
515 
516 	hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_NESTED);
517 	if (IS_ERR(hwpt))
518 		return hwpt;
519 	if (hwpt->domain->type != IOMMU_DOMAIN_NESTED ||
520 	    hwpt->domain->ops != &domain_nested_ops) {
521 		iommufd_put_object(&hwpt->obj);
522 		return ERR_PTR(-EINVAL);
523 	}
524 	*mock_nested = container_of(hwpt->domain,
525 				    struct mock_iommu_domain_nested, domain);
526 	return hwpt;
527 }
528 
529 struct mock_bus_type {
530 	struct bus_type bus;
531 	struct notifier_block nb;
532 };
533 
534 static struct mock_bus_type iommufd_mock_bus_type = {
535 	.bus = {
536 		.name = "iommufd_mock",
537 	},
538 };
539 
540 static atomic_t mock_dev_num;
541 
542 static void mock_dev_release(struct device *dev)
543 {
544 	struct mock_dev *mdev = container_of(dev, struct mock_dev, dev);
545 
546 	atomic_dec(&mock_dev_num);
547 	kfree(mdev);
548 }
549 
550 static struct mock_dev *mock_dev_create(unsigned long dev_flags)
551 {
552 	struct mock_dev *mdev;
553 	int rc;
554 
555 	if (dev_flags & ~(MOCK_FLAGS_DEVICE_NO_DIRTY))
556 		return ERR_PTR(-EINVAL);
557 
558 	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
559 	if (!mdev)
560 		return ERR_PTR(-ENOMEM);
561 
562 	device_initialize(&mdev->dev);
563 	mdev->flags = dev_flags;
564 	mdev->dev.release = mock_dev_release;
565 	mdev->dev.bus = &iommufd_mock_bus_type.bus;
566 
567 	rc = dev_set_name(&mdev->dev, "iommufd_mock%u",
568 			  atomic_inc_return(&mock_dev_num));
569 	if (rc)
570 		goto err_put;
571 
572 	rc = device_add(&mdev->dev);
573 	if (rc)
574 		goto err_put;
575 	return mdev;
576 
577 err_put:
578 	put_device(&mdev->dev);
579 	return ERR_PTR(rc);
580 }
581 
582 static void mock_dev_destroy(struct mock_dev *mdev)
583 {
584 	device_unregister(&mdev->dev);
585 }
586 
587 bool iommufd_selftest_is_mock_dev(struct device *dev)
588 {
589 	return dev->release == mock_dev_release;
590 }
591 
592 /* Create an hw_pagetable with the mock domain so we can test the domain ops */
593 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
594 				    struct iommu_test_cmd *cmd)
595 {
596 	struct iommufd_device *idev;
597 	struct selftest_obj *sobj;
598 	u32 pt_id = cmd->id;
599 	u32 dev_flags = 0;
600 	u32 idev_id;
601 	int rc;
602 
603 	sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST);
604 	if (IS_ERR(sobj))
605 		return PTR_ERR(sobj);
606 
607 	sobj->idev.ictx = ucmd->ictx;
608 	sobj->type = TYPE_IDEV;
609 
610 	if (cmd->op == IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS)
611 		dev_flags = cmd->mock_domain_flags.dev_flags;
612 
613 	sobj->idev.mock_dev = mock_dev_create(dev_flags);
614 	if (IS_ERR(sobj->idev.mock_dev)) {
615 		rc = PTR_ERR(sobj->idev.mock_dev);
616 		goto out_sobj;
617 	}
618 
619 	idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev,
620 				   &idev_id);
621 	if (IS_ERR(idev)) {
622 		rc = PTR_ERR(idev);
623 		goto out_mdev;
624 	}
625 	sobj->idev.idev = idev;
626 
627 	rc = iommufd_device_attach(idev, &pt_id);
628 	if (rc)
629 		goto out_unbind;
630 
631 	/* Userspace must destroy the device_id to destroy the object */
632 	cmd->mock_domain.out_hwpt_id = pt_id;
633 	cmd->mock_domain.out_stdev_id = sobj->obj.id;
634 	cmd->mock_domain.out_idev_id = idev_id;
635 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
636 	if (rc)
637 		goto out_detach;
638 	iommufd_object_finalize(ucmd->ictx, &sobj->obj);
639 	return 0;
640 
641 out_detach:
642 	iommufd_device_detach(idev);
643 out_unbind:
644 	iommufd_device_unbind(idev);
645 out_mdev:
646 	mock_dev_destroy(sobj->idev.mock_dev);
647 out_sobj:
648 	iommufd_object_abort(ucmd->ictx, &sobj->obj);
649 	return rc;
650 }
651 
652 /* Replace the mock domain with a manually allocated hw_pagetable */
653 static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
654 					    unsigned int device_id, u32 pt_id,
655 					    struct iommu_test_cmd *cmd)
656 {
657 	struct iommufd_object *dev_obj;
658 	struct selftest_obj *sobj;
659 	int rc;
660 
661 	/*
662 	 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure
663 	 * it doesn't race with detach, which is not allowed.
664 	 */
665 	dev_obj =
666 		iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST);
667 	if (IS_ERR(dev_obj))
668 		return PTR_ERR(dev_obj);
669 
670 	sobj = container_of(dev_obj, struct selftest_obj, obj);
671 	if (sobj->type != TYPE_IDEV) {
672 		rc = -EINVAL;
673 		goto out_dev_obj;
674 	}
675 
676 	rc = iommufd_device_replace(sobj->idev.idev, &pt_id);
677 	if (rc)
678 		goto out_dev_obj;
679 
680 	cmd->mock_domain_replace.pt_id = pt_id;
681 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
682 
683 out_dev_obj:
684 	iommufd_put_object(dev_obj);
685 	return rc;
686 }
687 
688 /* Add an additional reserved IOVA to the IOAS */
689 static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
690 				     unsigned int mockpt_id,
691 				     unsigned long start, size_t length)
692 {
693 	struct iommufd_ioas *ioas;
694 	int rc;
695 
696 	ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id);
697 	if (IS_ERR(ioas))
698 		return PTR_ERR(ioas);
699 	down_write(&ioas->iopt.iova_rwsem);
700 	rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL);
701 	up_write(&ioas->iopt.iova_rwsem);
702 	iommufd_put_object(&ioas->obj);
703 	return rc;
704 }
705 
706 /* Check that every pfn under each iova matches the pfn under a user VA */
707 static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd,
708 				    unsigned int mockpt_id, unsigned long iova,
709 				    size_t length, void __user *uptr)
710 {
711 	struct iommufd_hw_pagetable *hwpt;
712 	struct mock_iommu_domain *mock;
713 	uintptr_t end;
714 	int rc;
715 
716 	if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE ||
717 	    (uintptr_t)uptr % MOCK_IO_PAGE_SIZE ||
718 	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end))
719 		return -EINVAL;
720 
721 	hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
722 	if (IS_ERR(hwpt))
723 		return PTR_ERR(hwpt);
724 
725 	for (; length; length -= MOCK_IO_PAGE_SIZE) {
726 		struct page *pages[1];
727 		unsigned long pfn;
728 		long npages;
729 		void *ent;
730 
731 		npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0,
732 					     pages);
733 		if (npages < 0) {
734 			rc = npages;
735 			goto out_put;
736 		}
737 		if (WARN_ON(npages != 1)) {
738 			rc = -EFAULT;
739 			goto out_put;
740 		}
741 		pfn = page_to_pfn(pages[0]);
742 		put_page(pages[0]);
743 
744 		ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE);
745 		if (!ent ||
746 		    (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE !=
747 			    pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) {
748 			rc = -EINVAL;
749 			goto out_put;
750 		}
751 		iova += MOCK_IO_PAGE_SIZE;
752 		uptr += MOCK_IO_PAGE_SIZE;
753 	}
754 	rc = 0;
755 
756 out_put:
757 	iommufd_put_object(&hwpt->obj);
758 	return rc;
759 }
760 
761 /* Check that the page ref count matches, to look for missing pin/unpins */
762 static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd,
763 				      void __user *uptr, size_t length,
764 				      unsigned int refs)
765 {
766 	uintptr_t end;
767 
768 	if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE ||
769 	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end))
770 		return -EINVAL;
771 
772 	for (; length; length -= PAGE_SIZE) {
773 		struct page *pages[1];
774 		long npages;
775 
776 		npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages);
777 		if (npages < 0)
778 			return npages;
779 		if (WARN_ON(npages != 1))
780 			return -EFAULT;
781 		if (!PageCompound(pages[0])) {
782 			unsigned int count;
783 
784 			count = page_ref_count(pages[0]);
785 			if (count / GUP_PIN_COUNTING_BIAS != refs) {
786 				put_page(pages[0]);
787 				return -EIO;
788 			}
789 		}
790 		put_page(pages[0]);
791 		uptr += PAGE_SIZE;
792 	}
793 	return 0;
794 }
795 
796 struct selftest_access {
797 	struct iommufd_access *access;
798 	struct file *file;
799 	struct mutex lock;
800 	struct list_head items;
801 	unsigned int next_id;
802 	bool destroying;
803 };
804 
805 struct selftest_access_item {
806 	struct list_head items_elm;
807 	unsigned long iova;
808 	size_t length;
809 	unsigned int id;
810 };
811 
812 static const struct file_operations iommfd_test_staccess_fops;
813 
814 static struct selftest_access *iommufd_access_get(int fd)
815 {
816 	struct file *file;
817 
818 	file = fget(fd);
819 	if (!file)
820 		return ERR_PTR(-EBADFD);
821 
822 	if (file->f_op != &iommfd_test_staccess_fops) {
823 		fput(file);
824 		return ERR_PTR(-EBADFD);
825 	}
826 	return file->private_data;
827 }
828 
829 static void iommufd_test_access_unmap(void *data, unsigned long iova,
830 				      unsigned long length)
831 {
832 	unsigned long iova_last = iova + length - 1;
833 	struct selftest_access *staccess = data;
834 	struct selftest_access_item *item;
835 	struct selftest_access_item *tmp;
836 
837 	mutex_lock(&staccess->lock);
838 	list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) {
839 		if (iova > item->iova + item->length - 1 ||
840 		    iova_last < item->iova)
841 			continue;
842 		list_del(&item->items_elm);
843 		iommufd_access_unpin_pages(staccess->access, item->iova,
844 					   item->length);
845 		kfree(item);
846 	}
847 	mutex_unlock(&staccess->lock);
848 }
849 
850 static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd,
851 					    unsigned int access_id,
852 					    unsigned int item_id)
853 {
854 	struct selftest_access_item *item;
855 	struct selftest_access *staccess;
856 
857 	staccess = iommufd_access_get(access_id);
858 	if (IS_ERR(staccess))
859 		return PTR_ERR(staccess);
860 
861 	mutex_lock(&staccess->lock);
862 	list_for_each_entry(item, &staccess->items, items_elm) {
863 		if (item->id == item_id) {
864 			list_del(&item->items_elm);
865 			iommufd_access_unpin_pages(staccess->access, item->iova,
866 						   item->length);
867 			mutex_unlock(&staccess->lock);
868 			kfree(item);
869 			fput(staccess->file);
870 			return 0;
871 		}
872 	}
873 	mutex_unlock(&staccess->lock);
874 	fput(staccess->file);
875 	return -ENOENT;
876 }
877 
878 static int iommufd_test_staccess_release(struct inode *inode,
879 					 struct file *filep)
880 {
881 	struct selftest_access *staccess = filep->private_data;
882 
883 	if (staccess->access) {
884 		iommufd_test_access_unmap(staccess, 0, ULONG_MAX);
885 		iommufd_access_destroy(staccess->access);
886 	}
887 	mutex_destroy(&staccess->lock);
888 	kfree(staccess);
889 	return 0;
890 }
891 
892 static const struct iommufd_access_ops selftest_access_ops_pin = {
893 	.needs_pin_pages = 1,
894 	.unmap = iommufd_test_access_unmap,
895 };
896 
897 static const struct iommufd_access_ops selftest_access_ops = {
898 	.unmap = iommufd_test_access_unmap,
899 };
900 
901 static const struct file_operations iommfd_test_staccess_fops = {
902 	.release = iommufd_test_staccess_release,
903 };
904 
905 static struct selftest_access *iommufd_test_alloc_access(void)
906 {
907 	struct selftest_access *staccess;
908 	struct file *filep;
909 
910 	staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT);
911 	if (!staccess)
912 		return ERR_PTR(-ENOMEM);
913 	INIT_LIST_HEAD(&staccess->items);
914 	mutex_init(&staccess->lock);
915 
916 	filep = anon_inode_getfile("[iommufd_test_staccess]",
917 				   &iommfd_test_staccess_fops, staccess,
918 				   O_RDWR);
919 	if (IS_ERR(filep)) {
920 		kfree(staccess);
921 		return ERR_CAST(filep);
922 	}
923 	staccess->file = filep;
924 	return staccess;
925 }
926 
927 static int iommufd_test_create_access(struct iommufd_ucmd *ucmd,
928 				      unsigned int ioas_id, unsigned int flags)
929 {
930 	struct iommu_test_cmd *cmd = ucmd->cmd;
931 	struct selftest_access *staccess;
932 	struct iommufd_access *access;
933 	u32 id;
934 	int fdno;
935 	int rc;
936 
937 	if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES)
938 		return -EOPNOTSUPP;
939 
940 	staccess = iommufd_test_alloc_access();
941 	if (IS_ERR(staccess))
942 		return PTR_ERR(staccess);
943 
944 	fdno = get_unused_fd_flags(O_CLOEXEC);
945 	if (fdno < 0) {
946 		rc = -ENOMEM;
947 		goto out_free_staccess;
948 	}
949 
950 	access = iommufd_access_create(
951 		ucmd->ictx,
952 		(flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ?
953 			&selftest_access_ops_pin :
954 			&selftest_access_ops,
955 		staccess, &id);
956 	if (IS_ERR(access)) {
957 		rc = PTR_ERR(access);
958 		goto out_put_fdno;
959 	}
960 	rc = iommufd_access_attach(access, ioas_id);
961 	if (rc)
962 		goto out_destroy;
963 	cmd->create_access.out_access_fd = fdno;
964 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
965 	if (rc)
966 		goto out_destroy;
967 
968 	staccess->access = access;
969 	fd_install(fdno, staccess->file);
970 	return 0;
971 
972 out_destroy:
973 	iommufd_access_destroy(access);
974 out_put_fdno:
975 	put_unused_fd(fdno);
976 out_free_staccess:
977 	fput(staccess->file);
978 	return rc;
979 }
980 
981 static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd,
982 					    unsigned int access_id,
983 					    unsigned int ioas_id)
984 {
985 	struct selftest_access *staccess;
986 	int rc;
987 
988 	staccess = iommufd_access_get(access_id);
989 	if (IS_ERR(staccess))
990 		return PTR_ERR(staccess);
991 
992 	rc = iommufd_access_replace(staccess->access, ioas_id);
993 	fput(staccess->file);
994 	return rc;
995 }
996 
997 /* Check that the pages in a page array match the pages in the user VA */
998 static int iommufd_test_check_pages(void __user *uptr, struct page **pages,
999 				    size_t npages)
1000 {
1001 	for (; npages; npages--) {
1002 		struct page *tmp_pages[1];
1003 		long rc;
1004 
1005 		rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages);
1006 		if (rc < 0)
1007 			return rc;
1008 		if (WARN_ON(rc != 1))
1009 			return -EFAULT;
1010 		put_page(tmp_pages[0]);
1011 		if (tmp_pages[0] != *pages)
1012 			return -EBADE;
1013 		pages++;
1014 		uptr += PAGE_SIZE;
1015 	}
1016 	return 0;
1017 }
1018 
1019 static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd,
1020 				     unsigned int access_id, unsigned long iova,
1021 				     size_t length, void __user *uptr,
1022 				     u32 flags)
1023 {
1024 	struct iommu_test_cmd *cmd = ucmd->cmd;
1025 	struct selftest_access_item *item;
1026 	struct selftest_access *staccess;
1027 	struct page **pages;
1028 	size_t npages;
1029 	int rc;
1030 
1031 	/* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
1032 	if (length > 16*1024*1024)
1033 		return -ENOMEM;
1034 
1035 	if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ))
1036 		return -EOPNOTSUPP;
1037 
1038 	staccess = iommufd_access_get(access_id);
1039 	if (IS_ERR(staccess))
1040 		return PTR_ERR(staccess);
1041 
1042 	if (staccess->access->ops != &selftest_access_ops_pin) {
1043 		rc = -EOPNOTSUPP;
1044 		goto out_put;
1045 	}
1046 
1047 	if (flags & MOCK_FLAGS_ACCESS_SYZ)
1048 		iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
1049 					&cmd->access_pages.iova);
1050 
1051 	npages = (ALIGN(iova + length, PAGE_SIZE) -
1052 		  ALIGN_DOWN(iova, PAGE_SIZE)) /
1053 		 PAGE_SIZE;
1054 	pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT);
1055 	if (!pages) {
1056 		rc = -ENOMEM;
1057 		goto out_put;
1058 	}
1059 
1060 	/*
1061 	 * Drivers will need to think very carefully about this locking. The
1062 	 * core code can do multiple unmaps instantaneously after
1063 	 * iommufd_access_pin_pages() and *all* the unmaps must not return until
1064 	 * the range is unpinned. This simple implementation puts a global lock
1065 	 * around the pin, which may not suit drivers that want this to be a
1066 	 * performance path. drivers that get this wrong will trigger WARN_ON
1067 	 * races and cause EDEADLOCK failures to userspace.
1068 	 */
1069 	mutex_lock(&staccess->lock);
1070 	rc = iommufd_access_pin_pages(staccess->access, iova, length, pages,
1071 				      flags & MOCK_FLAGS_ACCESS_WRITE);
1072 	if (rc)
1073 		goto out_unlock;
1074 
1075 	/* For syzkaller allow uptr to be NULL to skip this check */
1076 	if (uptr) {
1077 		rc = iommufd_test_check_pages(
1078 			uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages,
1079 			npages);
1080 		if (rc)
1081 			goto out_unaccess;
1082 	}
1083 
1084 	item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT);
1085 	if (!item) {
1086 		rc = -ENOMEM;
1087 		goto out_unaccess;
1088 	}
1089 
1090 	item->iova = iova;
1091 	item->length = length;
1092 	item->id = staccess->next_id++;
1093 	list_add_tail(&item->items_elm, &staccess->items);
1094 
1095 	cmd->access_pages.out_access_pages_id = item->id;
1096 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
1097 	if (rc)
1098 		goto out_free_item;
1099 	goto out_unlock;
1100 
1101 out_free_item:
1102 	list_del(&item->items_elm);
1103 	kfree(item);
1104 out_unaccess:
1105 	iommufd_access_unpin_pages(staccess->access, iova, length);
1106 out_unlock:
1107 	mutex_unlock(&staccess->lock);
1108 	kvfree(pages);
1109 out_put:
1110 	fput(staccess->file);
1111 	return rc;
1112 }
1113 
1114 static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd,
1115 				  unsigned int access_id, unsigned long iova,
1116 				  size_t length, void __user *ubuf,
1117 				  unsigned int flags)
1118 {
1119 	struct iommu_test_cmd *cmd = ucmd->cmd;
1120 	struct selftest_access *staccess;
1121 	void *tmp;
1122 	int rc;
1123 
1124 	/* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */
1125 	if (length > 16*1024*1024)
1126 		return -ENOMEM;
1127 
1128 	if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH |
1129 		      MOCK_FLAGS_ACCESS_SYZ))
1130 		return -EOPNOTSUPP;
1131 
1132 	staccess = iommufd_access_get(access_id);
1133 	if (IS_ERR(staccess))
1134 		return PTR_ERR(staccess);
1135 
1136 	tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT);
1137 	if (!tmp) {
1138 		rc = -ENOMEM;
1139 		goto out_put;
1140 	}
1141 
1142 	if (flags & MOCK_ACCESS_RW_WRITE) {
1143 		if (copy_from_user(tmp, ubuf, length)) {
1144 			rc = -EFAULT;
1145 			goto out_free;
1146 		}
1147 	}
1148 
1149 	if (flags & MOCK_FLAGS_ACCESS_SYZ)
1150 		iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt,
1151 					&cmd->access_rw.iova);
1152 
1153 	rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags);
1154 	if (rc)
1155 		goto out_free;
1156 	if (!(flags & MOCK_ACCESS_RW_WRITE)) {
1157 		if (copy_to_user(ubuf, tmp, length)) {
1158 			rc = -EFAULT;
1159 			goto out_free;
1160 		}
1161 	}
1162 
1163 out_free:
1164 	kvfree(tmp);
1165 out_put:
1166 	fput(staccess->file);
1167 	return rc;
1168 }
1169 static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE);
1170 static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH ==
1171 	      __IOMMUFD_ACCESS_RW_SLOW_PATH);
1172 
1173 static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
1174 			      unsigned long iova, size_t length,
1175 			      unsigned long page_size, void __user *uptr,
1176 			      u32 flags)
1177 {
1178 	unsigned long bitmap_size, i, max;
1179 	struct iommu_test_cmd *cmd = ucmd->cmd;
1180 	struct iommufd_hw_pagetable *hwpt;
1181 	struct mock_iommu_domain *mock;
1182 	int rc, count = 0;
1183 	void *tmp;
1184 
1185 	if (!page_size || !length || iova % page_size || length % page_size ||
1186 	    !uptr)
1187 		return -EINVAL;
1188 
1189 	hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
1190 	if (IS_ERR(hwpt))
1191 		return PTR_ERR(hwpt);
1192 
1193 	if (!(mock->flags & MOCK_DIRTY_TRACK)) {
1194 		rc = -EINVAL;
1195 		goto out_put;
1196 	}
1197 
1198 	max = length / page_size;
1199 	bitmap_size = max / BITS_PER_BYTE;
1200 
1201 	tmp = kvzalloc(bitmap_size, GFP_KERNEL_ACCOUNT);
1202 	if (!tmp) {
1203 		rc = -ENOMEM;
1204 		goto out_put;
1205 	}
1206 
1207 	if (copy_from_user(tmp, uptr, bitmap_size)) {
1208 		rc = -EFAULT;
1209 		goto out_free;
1210 	}
1211 
1212 	for (i = 0; i < max; i++) {
1213 		unsigned long cur = iova + i * page_size;
1214 		void *ent, *old;
1215 
1216 		if (!test_bit(i, (unsigned long *)tmp))
1217 			continue;
1218 
1219 		ent = xa_load(&mock->pfns, cur / page_size);
1220 		if (ent) {
1221 			unsigned long val;
1222 
1223 			val = xa_to_value(ent) | MOCK_PFN_DIRTY_IOVA;
1224 			old = xa_store(&mock->pfns, cur / page_size,
1225 				       xa_mk_value(val), GFP_KERNEL);
1226 			WARN_ON_ONCE(ent != old);
1227 			count++;
1228 		}
1229 	}
1230 
1231 	cmd->dirty.out_nr_dirty = count;
1232 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
1233 out_free:
1234 	kvfree(tmp);
1235 out_put:
1236 	iommufd_put_object(&hwpt->obj);
1237 	return rc;
1238 }
1239 
1240 void iommufd_selftest_destroy(struct iommufd_object *obj)
1241 {
1242 	struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj);
1243 
1244 	switch (sobj->type) {
1245 	case TYPE_IDEV:
1246 		iommufd_device_detach(sobj->idev.idev);
1247 		iommufd_device_unbind(sobj->idev.idev);
1248 		mock_dev_destroy(sobj->idev.mock_dev);
1249 		break;
1250 	}
1251 }
1252 
1253 int iommufd_test(struct iommufd_ucmd *ucmd)
1254 {
1255 	struct iommu_test_cmd *cmd = ucmd->cmd;
1256 
1257 	switch (cmd->op) {
1258 	case IOMMU_TEST_OP_ADD_RESERVED:
1259 		return iommufd_test_add_reserved(ucmd, cmd->id,
1260 						 cmd->add_reserved.start,
1261 						 cmd->add_reserved.length);
1262 	case IOMMU_TEST_OP_MOCK_DOMAIN:
1263 	case IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS:
1264 		return iommufd_test_mock_domain(ucmd, cmd);
1265 	case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE:
1266 		return iommufd_test_mock_domain_replace(
1267 			ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd);
1268 	case IOMMU_TEST_OP_MD_CHECK_MAP:
1269 		return iommufd_test_md_check_pa(
1270 			ucmd, cmd->id, cmd->check_map.iova,
1271 			cmd->check_map.length,
1272 			u64_to_user_ptr(cmd->check_map.uptr));
1273 	case IOMMU_TEST_OP_MD_CHECK_REFS:
1274 		return iommufd_test_md_check_refs(
1275 			ucmd, u64_to_user_ptr(cmd->check_refs.uptr),
1276 			cmd->check_refs.length, cmd->check_refs.refs);
1277 	case IOMMU_TEST_OP_CREATE_ACCESS:
1278 		return iommufd_test_create_access(ucmd, cmd->id,
1279 						  cmd->create_access.flags);
1280 	case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS:
1281 		return iommufd_test_access_replace_ioas(
1282 			ucmd, cmd->id, cmd->access_replace_ioas.ioas_id);
1283 	case IOMMU_TEST_OP_ACCESS_PAGES:
1284 		return iommufd_test_access_pages(
1285 			ucmd, cmd->id, cmd->access_pages.iova,
1286 			cmd->access_pages.length,
1287 			u64_to_user_ptr(cmd->access_pages.uptr),
1288 			cmd->access_pages.flags);
1289 	case IOMMU_TEST_OP_ACCESS_RW:
1290 		return iommufd_test_access_rw(
1291 			ucmd, cmd->id, cmd->access_rw.iova,
1292 			cmd->access_rw.length,
1293 			u64_to_user_ptr(cmd->access_rw.uptr),
1294 			cmd->access_rw.flags);
1295 	case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES:
1296 		return iommufd_test_access_item_destroy(
1297 			ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id);
1298 	case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT:
1299 		/* Protect _batch_init(), can not be less than elmsz */
1300 		if (cmd->memory_limit.limit <
1301 		    sizeof(unsigned long) + sizeof(u32))
1302 			return -EINVAL;
1303 		iommufd_test_memory_limit = cmd->memory_limit.limit;
1304 		return 0;
1305 	case IOMMU_TEST_OP_DIRTY:
1306 		return iommufd_test_dirty(ucmd, cmd->id, cmd->dirty.iova,
1307 					  cmd->dirty.length,
1308 					  cmd->dirty.page_size,
1309 					  u64_to_user_ptr(cmd->dirty.uptr),
1310 					  cmd->dirty.flags);
1311 	default:
1312 		return -EOPNOTSUPP;
1313 	}
1314 }
1315 
1316 bool iommufd_should_fail(void)
1317 {
1318 	return should_fail(&fail_iommufd, 1);
1319 }
1320 
1321 int __init iommufd_test_init(void)
1322 {
1323 	struct platform_device_info pdevinfo = {
1324 		.name = "iommufd_selftest_iommu",
1325 	};
1326 	int rc;
1327 
1328 	dbgfs_root =
1329 		fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd);
1330 
1331 	selftest_iommu_dev = platform_device_register_full(&pdevinfo);
1332 	if (IS_ERR(selftest_iommu_dev)) {
1333 		rc = PTR_ERR(selftest_iommu_dev);
1334 		goto err_dbgfs;
1335 	}
1336 
1337 	rc = bus_register(&iommufd_mock_bus_type.bus);
1338 	if (rc)
1339 		goto err_platform;
1340 
1341 	rc = iommu_device_sysfs_add(&mock_iommu_device,
1342 				    &selftest_iommu_dev->dev, NULL, "%s",
1343 				    dev_name(&selftest_iommu_dev->dev));
1344 	if (rc)
1345 		goto err_bus;
1346 
1347 	rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops,
1348 				  &iommufd_mock_bus_type.bus,
1349 				  &iommufd_mock_bus_type.nb);
1350 	if (rc)
1351 		goto err_sysfs;
1352 	return 0;
1353 
1354 err_sysfs:
1355 	iommu_device_sysfs_remove(&mock_iommu_device);
1356 err_bus:
1357 	bus_unregister(&iommufd_mock_bus_type.bus);
1358 err_platform:
1359 	platform_device_unregister(selftest_iommu_dev);
1360 err_dbgfs:
1361 	debugfs_remove_recursive(dbgfs_root);
1362 	return rc;
1363 }
1364 
1365 void iommufd_test_exit(void)
1366 {
1367 	iommu_device_sysfs_remove(&mock_iommu_device);
1368 	iommu_device_unregister_bus(&mock_iommu_device,
1369 				    &iommufd_mock_bus_type.bus,
1370 				    &iommufd_mock_bus_type.nb);
1371 	bus_unregister(&iommufd_mock_bus_type.bus);
1372 	platform_device_unregister(selftest_iommu_dev);
1373 	debugfs_remove_recursive(dbgfs_root);
1374 }
1375