1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. 3 * 4 * Kernel side components to support tools/testing/selftests/iommu 5 */ 6 #include <linux/anon_inodes.h> 7 #include <linux/debugfs.h> 8 #include <linux/fault-inject.h> 9 #include <linux/file.h> 10 #include <linux/iommu.h> 11 #include <linux/platform_device.h> 12 #include <linux/slab.h> 13 #include <linux/xarray.h> 14 #include <uapi/linux/iommufd.h> 15 16 #include "../iommu-priv.h" 17 #include "io_pagetable.h" 18 #include "iommufd_private.h" 19 #include "iommufd_test.h" 20 21 static DECLARE_FAULT_ATTR(fail_iommufd); 22 static struct dentry *dbgfs_root; 23 static struct platform_device *selftest_iommu_dev; 24 static const struct iommu_ops mock_ops; 25 static struct iommu_domain_ops domain_nested_ops; 26 27 size_t iommufd_test_memory_limit = 65536; 28 29 struct mock_bus_type { 30 struct bus_type bus; 31 struct notifier_block nb; 32 }; 33 34 static struct mock_bus_type iommufd_mock_bus_type = { 35 .bus = { 36 .name = "iommufd_mock", 37 }, 38 }; 39 40 static DEFINE_IDA(mock_dev_ida); 41 42 enum { 43 MOCK_DIRTY_TRACK = 1, 44 MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 45 MOCK_HUGE_PAGE_SIZE = 512 * MOCK_IO_PAGE_SIZE, 46 47 /* 48 * Like a real page table alignment requires the low bits of the address 49 * to be zero. xarray also requires the high bit to be zero, so we store 50 * the pfns shifted. The upper bits are used for metadata. 51 */ 52 MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 53 54 _MOCK_PFN_START = MOCK_PFN_MASK + 1, 55 MOCK_PFN_START_IOVA = _MOCK_PFN_START, 56 MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 57 MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1, 58 MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, 59 }; 60 61 /* 62 * Syzkaller has trouble randomizing the correct iova to use since it is linked 63 * to the map ioctl's output, and it has no ide about that. So, simplify things. 64 * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset 65 * value. This has a much smaller randomization space and syzkaller can hit it. 66 */ 67 static unsigned long __iommufd_test_syz_conv_iova(struct io_pagetable *iopt, 68 u64 *iova) 69 { 70 struct syz_layout { 71 __u32 nth_area; 72 __u32 offset; 73 }; 74 struct syz_layout *syz = (void *)iova; 75 unsigned int nth = syz->nth_area; 76 struct iopt_area *area; 77 78 down_read(&iopt->iova_rwsem); 79 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; 80 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { 81 if (nth == 0) { 82 up_read(&iopt->iova_rwsem); 83 return iopt_area_iova(area) + syz->offset; 84 } 85 nth--; 86 } 87 up_read(&iopt->iova_rwsem); 88 89 return 0; 90 } 91 92 static unsigned long iommufd_test_syz_conv_iova(struct iommufd_access *access, 93 u64 *iova) 94 { 95 unsigned long ret; 96 97 mutex_lock(&access->ioas_lock); 98 if (!access->ioas) { 99 mutex_unlock(&access->ioas_lock); 100 return 0; 101 } 102 ret = __iommufd_test_syz_conv_iova(&access->ioas->iopt, iova); 103 mutex_unlock(&access->ioas_lock); 104 return ret; 105 } 106 107 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 108 unsigned int ioas_id, u64 *iova, u32 *flags) 109 { 110 struct iommufd_ioas *ioas; 111 112 if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) 113 return; 114 *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; 115 116 ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); 117 if (IS_ERR(ioas)) 118 return; 119 *iova = __iommufd_test_syz_conv_iova(&ioas->iopt, iova); 120 iommufd_put_object(ucmd->ictx, &ioas->obj); 121 } 122 123 struct mock_iommu_domain { 124 unsigned long flags; 125 struct iommu_domain domain; 126 struct xarray pfns; 127 }; 128 129 struct mock_iommu_domain_nested { 130 struct iommu_domain domain; 131 struct mock_iommu_domain *parent; 132 u32 iotlb[MOCK_NESTED_DOMAIN_IOTLB_NUM]; 133 }; 134 135 enum selftest_obj_type { 136 TYPE_IDEV, 137 }; 138 139 struct mock_dev { 140 struct device dev; 141 unsigned long flags; 142 int id; 143 }; 144 145 struct selftest_obj { 146 struct iommufd_object obj; 147 enum selftest_obj_type type; 148 149 union { 150 struct { 151 struct iommufd_device *idev; 152 struct iommufd_ctx *ictx; 153 struct mock_dev *mock_dev; 154 } idev; 155 }; 156 }; 157 158 static int mock_domain_nop_attach(struct iommu_domain *domain, 159 struct device *dev) 160 { 161 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 162 163 if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY)) 164 return -EINVAL; 165 166 return 0; 167 } 168 169 static const struct iommu_domain_ops mock_blocking_ops = { 170 .attach_dev = mock_domain_nop_attach, 171 }; 172 173 static struct iommu_domain mock_blocking_domain = { 174 .type = IOMMU_DOMAIN_BLOCKED, 175 .ops = &mock_blocking_ops, 176 }; 177 178 static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type) 179 { 180 struct iommu_test_hw_info *info; 181 182 info = kzalloc(sizeof(*info), GFP_KERNEL); 183 if (!info) 184 return ERR_PTR(-ENOMEM); 185 186 info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL; 187 *length = sizeof(*info); 188 *type = IOMMU_HW_INFO_TYPE_SELFTEST; 189 190 return info; 191 } 192 193 static int mock_domain_set_dirty_tracking(struct iommu_domain *domain, 194 bool enable) 195 { 196 struct mock_iommu_domain *mock = 197 container_of(domain, struct mock_iommu_domain, domain); 198 unsigned long flags = mock->flags; 199 200 if (enable && !domain->dirty_ops) 201 return -EINVAL; 202 203 /* No change? */ 204 if (!(enable ^ !!(flags & MOCK_DIRTY_TRACK))) 205 return 0; 206 207 flags = (enable ? flags | MOCK_DIRTY_TRACK : flags & ~MOCK_DIRTY_TRACK); 208 209 mock->flags = flags; 210 return 0; 211 } 212 213 static bool mock_test_and_clear_dirty(struct mock_iommu_domain *mock, 214 unsigned long iova, size_t page_size, 215 unsigned long flags) 216 { 217 unsigned long cur, end = iova + page_size - 1; 218 bool dirty = false; 219 void *ent, *old; 220 221 for (cur = iova; cur < end; cur += MOCK_IO_PAGE_SIZE) { 222 ent = xa_load(&mock->pfns, cur / MOCK_IO_PAGE_SIZE); 223 if (!ent || !(xa_to_value(ent) & MOCK_PFN_DIRTY_IOVA)) 224 continue; 225 226 dirty = true; 227 /* Clear dirty */ 228 if (!(flags & IOMMU_DIRTY_NO_CLEAR)) { 229 unsigned long val; 230 231 val = xa_to_value(ent) & ~MOCK_PFN_DIRTY_IOVA; 232 old = xa_store(&mock->pfns, cur / MOCK_IO_PAGE_SIZE, 233 xa_mk_value(val), GFP_KERNEL); 234 WARN_ON_ONCE(ent != old); 235 } 236 } 237 238 return dirty; 239 } 240 241 static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, 242 unsigned long iova, size_t size, 243 unsigned long flags, 244 struct iommu_dirty_bitmap *dirty) 245 { 246 struct mock_iommu_domain *mock = 247 container_of(domain, struct mock_iommu_domain, domain); 248 unsigned long end = iova + size; 249 void *ent; 250 251 if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap) 252 return -EINVAL; 253 254 do { 255 unsigned long pgsize = MOCK_IO_PAGE_SIZE; 256 unsigned long head; 257 258 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 259 if (!ent) { 260 iova += pgsize; 261 continue; 262 } 263 264 if (xa_to_value(ent) & MOCK_PFN_HUGE_IOVA) 265 pgsize = MOCK_HUGE_PAGE_SIZE; 266 head = iova & ~(pgsize - 1); 267 268 /* Clear dirty */ 269 if (mock_test_and_clear_dirty(mock, head, pgsize, flags)) 270 iommu_dirty_bitmap_record(dirty, iova, pgsize); 271 iova += pgsize; 272 } while (iova < end); 273 274 return 0; 275 } 276 277 static const struct iommu_dirty_ops dirty_ops = { 278 .set_dirty_tracking = mock_domain_set_dirty_tracking, 279 .read_and_clear_dirty = mock_domain_read_and_clear_dirty, 280 }; 281 282 static struct iommu_domain *mock_domain_alloc_paging(struct device *dev) 283 { 284 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 285 struct mock_iommu_domain *mock; 286 287 mock = kzalloc(sizeof(*mock), GFP_KERNEL); 288 if (!mock) 289 return NULL; 290 mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 291 mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 292 mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 293 if (dev && mdev->flags & MOCK_FLAGS_DEVICE_HUGE_IOVA) 294 mock->domain.pgsize_bitmap |= MOCK_HUGE_PAGE_SIZE; 295 mock->domain.ops = mock_ops.default_domain_ops; 296 mock->domain.type = IOMMU_DOMAIN_UNMANAGED; 297 xa_init(&mock->pfns); 298 return &mock->domain; 299 } 300 301 static struct iommu_domain * 302 __mock_domain_alloc_nested(struct mock_iommu_domain *mock_parent, 303 const struct iommu_hwpt_selftest *user_cfg) 304 { 305 struct mock_iommu_domain_nested *mock_nested; 306 int i; 307 308 mock_nested = kzalloc(sizeof(*mock_nested), GFP_KERNEL); 309 if (!mock_nested) 310 return ERR_PTR(-ENOMEM); 311 mock_nested->parent = mock_parent; 312 mock_nested->domain.ops = &domain_nested_ops; 313 mock_nested->domain.type = IOMMU_DOMAIN_NESTED; 314 for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++) 315 mock_nested->iotlb[i] = user_cfg->iotlb; 316 return &mock_nested->domain; 317 } 318 319 static struct iommu_domain * 320 mock_domain_alloc_user(struct device *dev, u32 flags, 321 struct iommu_domain *parent, 322 const struct iommu_user_data *user_data) 323 { 324 struct mock_iommu_domain *mock_parent; 325 struct iommu_hwpt_selftest user_cfg; 326 int rc; 327 328 /* must be mock_domain */ 329 if (!parent) { 330 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 331 bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 332 bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY; 333 struct iommu_domain *domain; 334 335 if (flags & (~(IOMMU_HWPT_ALLOC_NEST_PARENT | 336 IOMMU_HWPT_ALLOC_DIRTY_TRACKING))) 337 return ERR_PTR(-EOPNOTSUPP); 338 if (user_data || (has_dirty_flag && no_dirty_ops)) 339 return ERR_PTR(-EOPNOTSUPP); 340 domain = mock_domain_alloc_paging(dev); 341 if (!domain) 342 return ERR_PTR(-ENOMEM); 343 if (has_dirty_flag) 344 container_of(domain, struct mock_iommu_domain, domain) 345 ->domain.dirty_ops = &dirty_ops; 346 return domain; 347 } 348 349 /* must be mock_domain_nested */ 350 if (user_data->type != IOMMU_HWPT_DATA_SELFTEST || flags) 351 return ERR_PTR(-EOPNOTSUPP); 352 if (!parent || parent->ops != mock_ops.default_domain_ops) 353 return ERR_PTR(-EINVAL); 354 355 mock_parent = container_of(parent, struct mock_iommu_domain, domain); 356 if (!mock_parent) 357 return ERR_PTR(-EINVAL); 358 359 rc = iommu_copy_struct_from_user(&user_cfg, user_data, 360 IOMMU_HWPT_DATA_SELFTEST, iotlb); 361 if (rc) 362 return ERR_PTR(rc); 363 364 return __mock_domain_alloc_nested(mock_parent, &user_cfg); 365 } 366 367 static void mock_domain_free(struct iommu_domain *domain) 368 { 369 struct mock_iommu_domain *mock = 370 container_of(domain, struct mock_iommu_domain, domain); 371 372 WARN_ON(!xa_empty(&mock->pfns)); 373 kfree(mock); 374 } 375 376 static int mock_domain_map_pages(struct iommu_domain *domain, 377 unsigned long iova, phys_addr_t paddr, 378 size_t pgsize, size_t pgcount, int prot, 379 gfp_t gfp, size_t *mapped) 380 { 381 struct mock_iommu_domain *mock = 382 container_of(domain, struct mock_iommu_domain, domain); 383 unsigned long flags = MOCK_PFN_START_IOVA; 384 unsigned long start_iova = iova; 385 386 /* 387 * xarray does not reliably work with fault injection because it does a 388 * retry allocation, so put our own failure point. 389 */ 390 if (iommufd_should_fail()) 391 return -ENOENT; 392 393 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 394 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 395 for (; pgcount; pgcount--) { 396 size_t cur; 397 398 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 399 void *old; 400 401 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 402 flags = MOCK_PFN_LAST_IOVA; 403 if (pgsize != MOCK_IO_PAGE_SIZE) { 404 flags |= MOCK_PFN_HUGE_IOVA; 405 } 406 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 407 xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 408 flags), 409 gfp); 410 if (xa_is_err(old)) { 411 for (; start_iova != iova; 412 start_iova += MOCK_IO_PAGE_SIZE) 413 xa_erase(&mock->pfns, 414 start_iova / 415 MOCK_IO_PAGE_SIZE); 416 return xa_err(old); 417 } 418 WARN_ON(old); 419 iova += MOCK_IO_PAGE_SIZE; 420 paddr += MOCK_IO_PAGE_SIZE; 421 *mapped += MOCK_IO_PAGE_SIZE; 422 flags = 0; 423 } 424 } 425 return 0; 426 } 427 428 static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 429 unsigned long iova, size_t pgsize, 430 size_t pgcount, 431 struct iommu_iotlb_gather *iotlb_gather) 432 { 433 struct mock_iommu_domain *mock = 434 container_of(domain, struct mock_iommu_domain, domain); 435 bool first = true; 436 size_t ret = 0; 437 void *ent; 438 439 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 440 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 441 442 for (; pgcount; pgcount--) { 443 size_t cur; 444 445 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 446 ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 447 448 /* 449 * iommufd generates unmaps that must be a strict 450 * superset of the map's performend So every 451 * starting/ending IOVA should have been an iova passed 452 * to map. 453 * 454 * This simple logic doesn't work when the HUGE_PAGE is 455 * turned on since the core code will automatically 456 * switch between the two page sizes creating a break in 457 * the unmap calls. The break can land in the middle of 458 * contiguous IOVA. 459 */ 460 if (!(domain->pgsize_bitmap & MOCK_HUGE_PAGE_SIZE)) { 461 if (first) { 462 WARN_ON(ent && !(xa_to_value(ent) & 463 MOCK_PFN_START_IOVA)); 464 first = false; 465 } 466 if (pgcount == 1 && 467 cur + MOCK_IO_PAGE_SIZE == pgsize) 468 WARN_ON(ent && !(xa_to_value(ent) & 469 MOCK_PFN_LAST_IOVA)); 470 } 471 472 iova += MOCK_IO_PAGE_SIZE; 473 ret += MOCK_IO_PAGE_SIZE; 474 } 475 } 476 return ret; 477 } 478 479 static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 480 dma_addr_t iova) 481 { 482 struct mock_iommu_domain *mock = 483 container_of(domain, struct mock_iommu_domain, domain); 484 void *ent; 485 486 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 487 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 488 WARN_ON(!ent); 489 return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 490 } 491 492 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) 493 { 494 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 495 496 switch (cap) { 497 case IOMMU_CAP_CACHE_COHERENCY: 498 return true; 499 case IOMMU_CAP_DIRTY_TRACKING: 500 return !(mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY); 501 default: 502 break; 503 } 504 505 return false; 506 } 507 508 static struct iopf_queue *mock_iommu_iopf_queue; 509 510 static struct iommu_device mock_iommu_device = { 511 }; 512 513 static struct iommu_device *mock_probe_device(struct device *dev) 514 { 515 if (dev->bus != &iommufd_mock_bus_type.bus) 516 return ERR_PTR(-ENODEV); 517 return &mock_iommu_device; 518 } 519 520 static void mock_domain_page_response(struct device *dev, struct iopf_fault *evt, 521 struct iommu_page_response *msg) 522 { 523 } 524 525 static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) 526 { 527 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 528 return -ENODEV; 529 530 return iopf_queue_add_device(mock_iommu_iopf_queue, dev); 531 } 532 533 static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 534 { 535 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 536 return -ENODEV; 537 538 iopf_queue_remove_device(mock_iommu_iopf_queue, dev); 539 540 return 0; 541 } 542 543 static const struct iommu_ops mock_ops = { 544 /* 545 * IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type() 546 * because it is zero. 547 */ 548 .default_domain = &mock_blocking_domain, 549 .blocked_domain = &mock_blocking_domain, 550 .owner = THIS_MODULE, 551 .pgsize_bitmap = MOCK_IO_PAGE_SIZE, 552 .hw_info = mock_domain_hw_info, 553 .domain_alloc_paging = mock_domain_alloc_paging, 554 .domain_alloc_user = mock_domain_alloc_user, 555 .capable = mock_domain_capable, 556 .device_group = generic_device_group, 557 .probe_device = mock_probe_device, 558 .page_response = mock_domain_page_response, 559 .dev_enable_feat = mock_dev_enable_feat, 560 .dev_disable_feat = mock_dev_disable_feat, 561 .user_pasid_table = true, 562 .default_domain_ops = 563 &(struct iommu_domain_ops){ 564 .free = mock_domain_free, 565 .attach_dev = mock_domain_nop_attach, 566 .map_pages = mock_domain_map_pages, 567 .unmap_pages = mock_domain_unmap_pages, 568 .iova_to_phys = mock_domain_iova_to_phys, 569 }, 570 }; 571 572 static void mock_domain_free_nested(struct iommu_domain *domain) 573 { 574 struct mock_iommu_domain_nested *mock_nested = 575 container_of(domain, struct mock_iommu_domain_nested, domain); 576 577 kfree(mock_nested); 578 } 579 580 static int 581 mock_domain_cache_invalidate_user(struct iommu_domain *domain, 582 struct iommu_user_data_array *array) 583 { 584 struct mock_iommu_domain_nested *mock_nested = 585 container_of(domain, struct mock_iommu_domain_nested, domain); 586 struct iommu_hwpt_invalidate_selftest inv; 587 u32 processed = 0; 588 int i = 0, j; 589 int rc = 0; 590 591 if (array->type != IOMMU_HWPT_INVALIDATE_DATA_SELFTEST) { 592 rc = -EINVAL; 593 goto out; 594 } 595 596 for ( ; i < array->entry_num; i++) { 597 rc = iommu_copy_struct_from_user_array(&inv, array, 598 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 599 i, iotlb_id); 600 if (rc) 601 break; 602 603 if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) { 604 rc = -EOPNOTSUPP; 605 break; 606 } 607 608 if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) { 609 rc = -EINVAL; 610 break; 611 } 612 613 if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) { 614 /* Invalidate all mock iotlb entries and ignore iotlb_id */ 615 for (j = 0; j < MOCK_NESTED_DOMAIN_IOTLB_NUM; j++) 616 mock_nested->iotlb[j] = 0; 617 } else { 618 mock_nested->iotlb[inv.iotlb_id] = 0; 619 } 620 621 processed++; 622 } 623 624 out: 625 array->entry_num = processed; 626 return rc; 627 } 628 629 static struct iommu_domain_ops domain_nested_ops = { 630 .free = mock_domain_free_nested, 631 .attach_dev = mock_domain_nop_attach, 632 .cache_invalidate_user = mock_domain_cache_invalidate_user, 633 }; 634 635 static inline struct iommufd_hw_pagetable * 636 __get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, u32 hwpt_type) 637 { 638 struct iommufd_object *obj; 639 640 obj = iommufd_get_object(ucmd->ictx, mockpt_id, hwpt_type); 641 if (IS_ERR(obj)) 642 return ERR_CAST(obj); 643 return container_of(obj, struct iommufd_hw_pagetable, obj); 644 } 645 646 static inline struct iommufd_hw_pagetable * 647 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, 648 struct mock_iommu_domain **mock) 649 { 650 struct iommufd_hw_pagetable *hwpt; 651 652 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_PAGING); 653 if (IS_ERR(hwpt)) 654 return hwpt; 655 if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED || 656 hwpt->domain->ops != mock_ops.default_domain_ops) { 657 iommufd_put_object(ucmd->ictx, &hwpt->obj); 658 return ERR_PTR(-EINVAL); 659 } 660 *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain); 661 return hwpt; 662 } 663 664 static inline struct iommufd_hw_pagetable * 665 get_md_pagetable_nested(struct iommufd_ucmd *ucmd, u32 mockpt_id, 666 struct mock_iommu_domain_nested **mock_nested) 667 { 668 struct iommufd_hw_pagetable *hwpt; 669 670 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_NESTED); 671 if (IS_ERR(hwpt)) 672 return hwpt; 673 if (hwpt->domain->type != IOMMU_DOMAIN_NESTED || 674 hwpt->domain->ops != &domain_nested_ops) { 675 iommufd_put_object(ucmd->ictx, &hwpt->obj); 676 return ERR_PTR(-EINVAL); 677 } 678 *mock_nested = container_of(hwpt->domain, 679 struct mock_iommu_domain_nested, domain); 680 return hwpt; 681 } 682 683 static void mock_dev_release(struct device *dev) 684 { 685 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 686 687 ida_free(&mock_dev_ida, mdev->id); 688 kfree(mdev); 689 } 690 691 static struct mock_dev *mock_dev_create(unsigned long dev_flags) 692 { 693 struct mock_dev *mdev; 694 int rc; 695 696 if (dev_flags & 697 ~(MOCK_FLAGS_DEVICE_NO_DIRTY | MOCK_FLAGS_DEVICE_HUGE_IOVA)) 698 return ERR_PTR(-EINVAL); 699 700 mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); 701 if (!mdev) 702 return ERR_PTR(-ENOMEM); 703 704 device_initialize(&mdev->dev); 705 mdev->flags = dev_flags; 706 mdev->dev.release = mock_dev_release; 707 mdev->dev.bus = &iommufd_mock_bus_type.bus; 708 709 rc = ida_alloc(&mock_dev_ida, GFP_KERNEL); 710 if (rc < 0) 711 goto err_put; 712 mdev->id = rc; 713 714 rc = dev_set_name(&mdev->dev, "iommufd_mock%u", mdev->id); 715 if (rc) 716 goto err_put; 717 718 rc = device_add(&mdev->dev); 719 if (rc) 720 goto err_put; 721 return mdev; 722 723 err_put: 724 put_device(&mdev->dev); 725 return ERR_PTR(rc); 726 } 727 728 static void mock_dev_destroy(struct mock_dev *mdev) 729 { 730 device_unregister(&mdev->dev); 731 } 732 733 bool iommufd_selftest_is_mock_dev(struct device *dev) 734 { 735 return dev->release == mock_dev_release; 736 } 737 738 /* Create an hw_pagetable with the mock domain so we can test the domain ops */ 739 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, 740 struct iommu_test_cmd *cmd) 741 { 742 struct iommufd_device *idev; 743 struct selftest_obj *sobj; 744 u32 pt_id = cmd->id; 745 u32 dev_flags = 0; 746 u32 idev_id; 747 int rc; 748 749 sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); 750 if (IS_ERR(sobj)) 751 return PTR_ERR(sobj); 752 753 sobj->idev.ictx = ucmd->ictx; 754 sobj->type = TYPE_IDEV; 755 756 if (cmd->op == IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS) 757 dev_flags = cmd->mock_domain_flags.dev_flags; 758 759 sobj->idev.mock_dev = mock_dev_create(dev_flags); 760 if (IS_ERR(sobj->idev.mock_dev)) { 761 rc = PTR_ERR(sobj->idev.mock_dev); 762 goto out_sobj; 763 } 764 765 idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, 766 &idev_id); 767 if (IS_ERR(idev)) { 768 rc = PTR_ERR(idev); 769 goto out_mdev; 770 } 771 sobj->idev.idev = idev; 772 773 rc = iommufd_device_attach(idev, &pt_id); 774 if (rc) 775 goto out_unbind; 776 777 /* Userspace must destroy the device_id to destroy the object */ 778 cmd->mock_domain.out_hwpt_id = pt_id; 779 cmd->mock_domain.out_stdev_id = sobj->obj.id; 780 cmd->mock_domain.out_idev_id = idev_id; 781 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 782 if (rc) 783 goto out_detach; 784 iommufd_object_finalize(ucmd->ictx, &sobj->obj); 785 return 0; 786 787 out_detach: 788 iommufd_device_detach(idev); 789 out_unbind: 790 iommufd_device_unbind(idev); 791 out_mdev: 792 mock_dev_destroy(sobj->idev.mock_dev); 793 out_sobj: 794 iommufd_object_abort(ucmd->ictx, &sobj->obj); 795 return rc; 796 } 797 798 /* Replace the mock domain with a manually allocated hw_pagetable */ 799 static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, 800 unsigned int device_id, u32 pt_id, 801 struct iommu_test_cmd *cmd) 802 { 803 struct iommufd_object *dev_obj; 804 struct selftest_obj *sobj; 805 int rc; 806 807 /* 808 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure 809 * it doesn't race with detach, which is not allowed. 810 */ 811 dev_obj = 812 iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST); 813 if (IS_ERR(dev_obj)) 814 return PTR_ERR(dev_obj); 815 816 sobj = container_of(dev_obj, struct selftest_obj, obj); 817 if (sobj->type != TYPE_IDEV) { 818 rc = -EINVAL; 819 goto out_dev_obj; 820 } 821 822 rc = iommufd_device_replace(sobj->idev.idev, &pt_id); 823 if (rc) 824 goto out_dev_obj; 825 826 cmd->mock_domain_replace.pt_id = pt_id; 827 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 828 829 out_dev_obj: 830 iommufd_put_object(ucmd->ictx, dev_obj); 831 return rc; 832 } 833 834 /* Add an additional reserved IOVA to the IOAS */ 835 static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, 836 unsigned int mockpt_id, 837 unsigned long start, size_t length) 838 { 839 struct iommufd_ioas *ioas; 840 int rc; 841 842 ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); 843 if (IS_ERR(ioas)) 844 return PTR_ERR(ioas); 845 down_write(&ioas->iopt.iova_rwsem); 846 rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); 847 up_write(&ioas->iopt.iova_rwsem); 848 iommufd_put_object(ucmd->ictx, &ioas->obj); 849 return rc; 850 } 851 852 /* Check that every pfn under each iova matches the pfn under a user VA */ 853 static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, 854 unsigned int mockpt_id, unsigned long iova, 855 size_t length, void __user *uptr) 856 { 857 struct iommufd_hw_pagetable *hwpt; 858 struct mock_iommu_domain *mock; 859 uintptr_t end; 860 int rc; 861 862 if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 863 (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 864 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 865 return -EINVAL; 866 867 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 868 if (IS_ERR(hwpt)) 869 return PTR_ERR(hwpt); 870 871 for (; length; length -= MOCK_IO_PAGE_SIZE) { 872 struct page *pages[1]; 873 unsigned long pfn; 874 long npages; 875 void *ent; 876 877 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 878 pages); 879 if (npages < 0) { 880 rc = npages; 881 goto out_put; 882 } 883 if (WARN_ON(npages != 1)) { 884 rc = -EFAULT; 885 goto out_put; 886 } 887 pfn = page_to_pfn(pages[0]); 888 put_page(pages[0]); 889 890 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 891 if (!ent || 892 (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 893 pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 894 rc = -EINVAL; 895 goto out_put; 896 } 897 iova += MOCK_IO_PAGE_SIZE; 898 uptr += MOCK_IO_PAGE_SIZE; 899 } 900 rc = 0; 901 902 out_put: 903 iommufd_put_object(ucmd->ictx, &hwpt->obj); 904 return rc; 905 } 906 907 /* Check that the page ref count matches, to look for missing pin/unpins */ 908 static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, 909 void __user *uptr, size_t length, 910 unsigned int refs) 911 { 912 uintptr_t end; 913 914 if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || 915 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 916 return -EINVAL; 917 918 for (; length; length -= PAGE_SIZE) { 919 struct page *pages[1]; 920 long npages; 921 922 npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages); 923 if (npages < 0) 924 return npages; 925 if (WARN_ON(npages != 1)) 926 return -EFAULT; 927 if (!PageCompound(pages[0])) { 928 unsigned int count; 929 930 count = page_ref_count(pages[0]); 931 if (count / GUP_PIN_COUNTING_BIAS != refs) { 932 put_page(pages[0]); 933 return -EIO; 934 } 935 } 936 put_page(pages[0]); 937 uptr += PAGE_SIZE; 938 } 939 return 0; 940 } 941 942 static int iommufd_test_md_check_iotlb(struct iommufd_ucmd *ucmd, 943 u32 mockpt_id, unsigned int iotlb_id, 944 u32 iotlb) 945 { 946 struct mock_iommu_domain_nested *mock_nested; 947 struct iommufd_hw_pagetable *hwpt; 948 int rc = 0; 949 950 hwpt = get_md_pagetable_nested(ucmd, mockpt_id, &mock_nested); 951 if (IS_ERR(hwpt)) 952 return PTR_ERR(hwpt); 953 954 mock_nested = container_of(hwpt->domain, 955 struct mock_iommu_domain_nested, domain); 956 957 if (iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX || 958 mock_nested->iotlb[iotlb_id] != iotlb) 959 rc = -EINVAL; 960 iommufd_put_object(ucmd->ictx, &hwpt->obj); 961 return rc; 962 } 963 964 struct selftest_access { 965 struct iommufd_access *access; 966 struct file *file; 967 struct mutex lock; 968 struct list_head items; 969 unsigned int next_id; 970 bool destroying; 971 }; 972 973 struct selftest_access_item { 974 struct list_head items_elm; 975 unsigned long iova; 976 size_t length; 977 unsigned int id; 978 }; 979 980 static const struct file_operations iommfd_test_staccess_fops; 981 982 static struct selftest_access *iommufd_access_get(int fd) 983 { 984 struct file *file; 985 986 file = fget(fd); 987 if (!file) 988 return ERR_PTR(-EBADFD); 989 990 if (file->f_op != &iommfd_test_staccess_fops) { 991 fput(file); 992 return ERR_PTR(-EBADFD); 993 } 994 return file->private_data; 995 } 996 997 static void iommufd_test_access_unmap(void *data, unsigned long iova, 998 unsigned long length) 999 { 1000 unsigned long iova_last = iova + length - 1; 1001 struct selftest_access *staccess = data; 1002 struct selftest_access_item *item; 1003 struct selftest_access_item *tmp; 1004 1005 mutex_lock(&staccess->lock); 1006 list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { 1007 if (iova > item->iova + item->length - 1 || 1008 iova_last < item->iova) 1009 continue; 1010 list_del(&item->items_elm); 1011 iommufd_access_unpin_pages(staccess->access, item->iova, 1012 item->length); 1013 kfree(item); 1014 } 1015 mutex_unlock(&staccess->lock); 1016 } 1017 1018 static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, 1019 unsigned int access_id, 1020 unsigned int item_id) 1021 { 1022 struct selftest_access_item *item; 1023 struct selftest_access *staccess; 1024 1025 staccess = iommufd_access_get(access_id); 1026 if (IS_ERR(staccess)) 1027 return PTR_ERR(staccess); 1028 1029 mutex_lock(&staccess->lock); 1030 list_for_each_entry(item, &staccess->items, items_elm) { 1031 if (item->id == item_id) { 1032 list_del(&item->items_elm); 1033 iommufd_access_unpin_pages(staccess->access, item->iova, 1034 item->length); 1035 mutex_unlock(&staccess->lock); 1036 kfree(item); 1037 fput(staccess->file); 1038 return 0; 1039 } 1040 } 1041 mutex_unlock(&staccess->lock); 1042 fput(staccess->file); 1043 return -ENOENT; 1044 } 1045 1046 static int iommufd_test_staccess_release(struct inode *inode, 1047 struct file *filep) 1048 { 1049 struct selftest_access *staccess = filep->private_data; 1050 1051 if (staccess->access) { 1052 iommufd_test_access_unmap(staccess, 0, ULONG_MAX); 1053 iommufd_access_destroy(staccess->access); 1054 } 1055 mutex_destroy(&staccess->lock); 1056 kfree(staccess); 1057 return 0; 1058 } 1059 1060 static const struct iommufd_access_ops selftest_access_ops_pin = { 1061 .needs_pin_pages = 1, 1062 .unmap = iommufd_test_access_unmap, 1063 }; 1064 1065 static const struct iommufd_access_ops selftest_access_ops = { 1066 .unmap = iommufd_test_access_unmap, 1067 }; 1068 1069 static const struct file_operations iommfd_test_staccess_fops = { 1070 .release = iommufd_test_staccess_release, 1071 }; 1072 1073 static struct selftest_access *iommufd_test_alloc_access(void) 1074 { 1075 struct selftest_access *staccess; 1076 struct file *filep; 1077 1078 staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT); 1079 if (!staccess) 1080 return ERR_PTR(-ENOMEM); 1081 INIT_LIST_HEAD(&staccess->items); 1082 mutex_init(&staccess->lock); 1083 1084 filep = anon_inode_getfile("[iommufd_test_staccess]", 1085 &iommfd_test_staccess_fops, staccess, 1086 O_RDWR); 1087 if (IS_ERR(filep)) { 1088 kfree(staccess); 1089 return ERR_CAST(filep); 1090 } 1091 staccess->file = filep; 1092 return staccess; 1093 } 1094 1095 static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, 1096 unsigned int ioas_id, unsigned int flags) 1097 { 1098 struct iommu_test_cmd *cmd = ucmd->cmd; 1099 struct selftest_access *staccess; 1100 struct iommufd_access *access; 1101 u32 id; 1102 int fdno; 1103 int rc; 1104 1105 if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) 1106 return -EOPNOTSUPP; 1107 1108 staccess = iommufd_test_alloc_access(); 1109 if (IS_ERR(staccess)) 1110 return PTR_ERR(staccess); 1111 1112 fdno = get_unused_fd_flags(O_CLOEXEC); 1113 if (fdno < 0) { 1114 rc = -ENOMEM; 1115 goto out_free_staccess; 1116 } 1117 1118 access = iommufd_access_create( 1119 ucmd->ictx, 1120 (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? 1121 &selftest_access_ops_pin : 1122 &selftest_access_ops, 1123 staccess, &id); 1124 if (IS_ERR(access)) { 1125 rc = PTR_ERR(access); 1126 goto out_put_fdno; 1127 } 1128 rc = iommufd_access_attach(access, ioas_id); 1129 if (rc) 1130 goto out_destroy; 1131 cmd->create_access.out_access_fd = fdno; 1132 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1133 if (rc) 1134 goto out_destroy; 1135 1136 staccess->access = access; 1137 fd_install(fdno, staccess->file); 1138 return 0; 1139 1140 out_destroy: 1141 iommufd_access_destroy(access); 1142 out_put_fdno: 1143 put_unused_fd(fdno); 1144 out_free_staccess: 1145 fput(staccess->file); 1146 return rc; 1147 } 1148 1149 static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, 1150 unsigned int access_id, 1151 unsigned int ioas_id) 1152 { 1153 struct selftest_access *staccess; 1154 int rc; 1155 1156 staccess = iommufd_access_get(access_id); 1157 if (IS_ERR(staccess)) 1158 return PTR_ERR(staccess); 1159 1160 rc = iommufd_access_replace(staccess->access, ioas_id); 1161 fput(staccess->file); 1162 return rc; 1163 } 1164 1165 /* Check that the pages in a page array match the pages in the user VA */ 1166 static int iommufd_test_check_pages(void __user *uptr, struct page **pages, 1167 size_t npages) 1168 { 1169 for (; npages; npages--) { 1170 struct page *tmp_pages[1]; 1171 long rc; 1172 1173 rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages); 1174 if (rc < 0) 1175 return rc; 1176 if (WARN_ON(rc != 1)) 1177 return -EFAULT; 1178 put_page(tmp_pages[0]); 1179 if (tmp_pages[0] != *pages) 1180 return -EBADE; 1181 pages++; 1182 uptr += PAGE_SIZE; 1183 } 1184 return 0; 1185 } 1186 1187 static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, 1188 unsigned int access_id, unsigned long iova, 1189 size_t length, void __user *uptr, 1190 u32 flags) 1191 { 1192 struct iommu_test_cmd *cmd = ucmd->cmd; 1193 struct selftest_access_item *item; 1194 struct selftest_access *staccess; 1195 struct page **pages; 1196 size_t npages; 1197 int rc; 1198 1199 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1200 if (length > 16*1024*1024) 1201 return -ENOMEM; 1202 1203 if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) 1204 return -EOPNOTSUPP; 1205 1206 staccess = iommufd_access_get(access_id); 1207 if (IS_ERR(staccess)) 1208 return PTR_ERR(staccess); 1209 1210 if (staccess->access->ops != &selftest_access_ops_pin) { 1211 rc = -EOPNOTSUPP; 1212 goto out_put; 1213 } 1214 1215 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1216 iova = iommufd_test_syz_conv_iova(staccess->access, 1217 &cmd->access_pages.iova); 1218 1219 npages = (ALIGN(iova + length, PAGE_SIZE) - 1220 ALIGN_DOWN(iova, PAGE_SIZE)) / 1221 PAGE_SIZE; 1222 pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT); 1223 if (!pages) { 1224 rc = -ENOMEM; 1225 goto out_put; 1226 } 1227 1228 /* 1229 * Drivers will need to think very carefully about this locking. The 1230 * core code can do multiple unmaps instantaneously after 1231 * iommufd_access_pin_pages() and *all* the unmaps must not return until 1232 * the range is unpinned. This simple implementation puts a global lock 1233 * around the pin, which may not suit drivers that want this to be a 1234 * performance path. drivers that get this wrong will trigger WARN_ON 1235 * races and cause EDEADLOCK failures to userspace. 1236 */ 1237 mutex_lock(&staccess->lock); 1238 rc = iommufd_access_pin_pages(staccess->access, iova, length, pages, 1239 flags & MOCK_FLAGS_ACCESS_WRITE); 1240 if (rc) 1241 goto out_unlock; 1242 1243 /* For syzkaller allow uptr to be NULL to skip this check */ 1244 if (uptr) { 1245 rc = iommufd_test_check_pages( 1246 uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, 1247 npages); 1248 if (rc) 1249 goto out_unaccess; 1250 } 1251 1252 item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT); 1253 if (!item) { 1254 rc = -ENOMEM; 1255 goto out_unaccess; 1256 } 1257 1258 item->iova = iova; 1259 item->length = length; 1260 item->id = staccess->next_id++; 1261 list_add_tail(&item->items_elm, &staccess->items); 1262 1263 cmd->access_pages.out_access_pages_id = item->id; 1264 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1265 if (rc) 1266 goto out_free_item; 1267 goto out_unlock; 1268 1269 out_free_item: 1270 list_del(&item->items_elm); 1271 kfree(item); 1272 out_unaccess: 1273 iommufd_access_unpin_pages(staccess->access, iova, length); 1274 out_unlock: 1275 mutex_unlock(&staccess->lock); 1276 kvfree(pages); 1277 out_put: 1278 fput(staccess->file); 1279 return rc; 1280 } 1281 1282 static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, 1283 unsigned int access_id, unsigned long iova, 1284 size_t length, void __user *ubuf, 1285 unsigned int flags) 1286 { 1287 struct iommu_test_cmd *cmd = ucmd->cmd; 1288 struct selftest_access *staccess; 1289 void *tmp; 1290 int rc; 1291 1292 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1293 if (length > 16*1024*1024) 1294 return -ENOMEM; 1295 1296 if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | 1297 MOCK_FLAGS_ACCESS_SYZ)) 1298 return -EOPNOTSUPP; 1299 1300 staccess = iommufd_access_get(access_id); 1301 if (IS_ERR(staccess)) 1302 return PTR_ERR(staccess); 1303 1304 tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT); 1305 if (!tmp) { 1306 rc = -ENOMEM; 1307 goto out_put; 1308 } 1309 1310 if (flags & MOCK_ACCESS_RW_WRITE) { 1311 if (copy_from_user(tmp, ubuf, length)) { 1312 rc = -EFAULT; 1313 goto out_free; 1314 } 1315 } 1316 1317 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1318 iova = iommufd_test_syz_conv_iova(staccess->access, 1319 &cmd->access_rw.iova); 1320 1321 rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags); 1322 if (rc) 1323 goto out_free; 1324 if (!(flags & MOCK_ACCESS_RW_WRITE)) { 1325 if (copy_to_user(ubuf, tmp, length)) { 1326 rc = -EFAULT; 1327 goto out_free; 1328 } 1329 } 1330 1331 out_free: 1332 kvfree(tmp); 1333 out_put: 1334 fput(staccess->file); 1335 return rc; 1336 } 1337 static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); 1338 static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == 1339 __IOMMUFD_ACCESS_RW_SLOW_PATH); 1340 1341 static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, 1342 unsigned long iova, size_t length, 1343 unsigned long page_size, void __user *uptr, 1344 u32 flags) 1345 { 1346 unsigned long i, max; 1347 struct iommu_test_cmd *cmd = ucmd->cmd; 1348 struct iommufd_hw_pagetable *hwpt; 1349 struct mock_iommu_domain *mock; 1350 int rc, count = 0; 1351 void *tmp; 1352 1353 if (!page_size || !length || iova % page_size || length % page_size || 1354 !uptr) 1355 return -EINVAL; 1356 1357 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 1358 if (IS_ERR(hwpt)) 1359 return PTR_ERR(hwpt); 1360 1361 if (!(mock->flags & MOCK_DIRTY_TRACK)) { 1362 rc = -EINVAL; 1363 goto out_put; 1364 } 1365 1366 max = length / page_size; 1367 tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long), 1368 GFP_KERNEL_ACCOUNT); 1369 if (!tmp) { 1370 rc = -ENOMEM; 1371 goto out_put; 1372 } 1373 1374 if (copy_from_user(tmp, uptr,DIV_ROUND_UP(max, BITS_PER_BYTE))) { 1375 rc = -EFAULT; 1376 goto out_free; 1377 } 1378 1379 for (i = 0; i < max; i++) { 1380 unsigned long cur = iova + i * page_size; 1381 void *ent, *old; 1382 1383 if (!test_bit(i, (unsigned long *)tmp)) 1384 continue; 1385 1386 ent = xa_load(&mock->pfns, cur / page_size); 1387 if (ent) { 1388 unsigned long val; 1389 1390 val = xa_to_value(ent) | MOCK_PFN_DIRTY_IOVA; 1391 old = xa_store(&mock->pfns, cur / page_size, 1392 xa_mk_value(val), GFP_KERNEL); 1393 WARN_ON_ONCE(ent != old); 1394 count++; 1395 } 1396 } 1397 1398 cmd->dirty.out_nr_dirty = count; 1399 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1400 out_free: 1401 kvfree(tmp); 1402 out_put: 1403 iommufd_put_object(ucmd->ictx, &hwpt->obj); 1404 return rc; 1405 } 1406 1407 static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, 1408 struct iommu_test_cmd *cmd) 1409 { 1410 struct iopf_fault event = { }; 1411 struct iommufd_device *idev; 1412 1413 idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id); 1414 if (IS_ERR(idev)) 1415 return PTR_ERR(idev); 1416 1417 event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1418 if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID) 1419 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1420 event.fault.type = IOMMU_FAULT_PAGE_REQ; 1421 event.fault.prm.addr = cmd->trigger_iopf.addr; 1422 event.fault.prm.pasid = cmd->trigger_iopf.pasid; 1423 event.fault.prm.grpid = cmd->trigger_iopf.grpid; 1424 event.fault.prm.perm = cmd->trigger_iopf.perm; 1425 1426 iommu_report_device_fault(idev->dev, &event); 1427 iommufd_put_object(ucmd->ictx, &idev->obj); 1428 1429 return 0; 1430 } 1431 1432 void iommufd_selftest_destroy(struct iommufd_object *obj) 1433 { 1434 struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj); 1435 1436 switch (sobj->type) { 1437 case TYPE_IDEV: 1438 iommufd_device_detach(sobj->idev.idev); 1439 iommufd_device_unbind(sobj->idev.idev); 1440 mock_dev_destroy(sobj->idev.mock_dev); 1441 break; 1442 } 1443 } 1444 1445 int iommufd_test(struct iommufd_ucmd *ucmd) 1446 { 1447 struct iommu_test_cmd *cmd = ucmd->cmd; 1448 1449 switch (cmd->op) { 1450 case IOMMU_TEST_OP_ADD_RESERVED: 1451 return iommufd_test_add_reserved(ucmd, cmd->id, 1452 cmd->add_reserved.start, 1453 cmd->add_reserved.length); 1454 case IOMMU_TEST_OP_MOCK_DOMAIN: 1455 case IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS: 1456 return iommufd_test_mock_domain(ucmd, cmd); 1457 case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: 1458 return iommufd_test_mock_domain_replace( 1459 ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd); 1460 case IOMMU_TEST_OP_MD_CHECK_MAP: 1461 return iommufd_test_md_check_pa( 1462 ucmd, cmd->id, cmd->check_map.iova, 1463 cmd->check_map.length, 1464 u64_to_user_ptr(cmd->check_map.uptr)); 1465 case IOMMU_TEST_OP_MD_CHECK_REFS: 1466 return iommufd_test_md_check_refs( 1467 ucmd, u64_to_user_ptr(cmd->check_refs.uptr), 1468 cmd->check_refs.length, cmd->check_refs.refs); 1469 case IOMMU_TEST_OP_MD_CHECK_IOTLB: 1470 return iommufd_test_md_check_iotlb(ucmd, cmd->id, 1471 cmd->check_iotlb.id, 1472 cmd->check_iotlb.iotlb); 1473 case IOMMU_TEST_OP_CREATE_ACCESS: 1474 return iommufd_test_create_access(ucmd, cmd->id, 1475 cmd->create_access.flags); 1476 case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS: 1477 return iommufd_test_access_replace_ioas( 1478 ucmd, cmd->id, cmd->access_replace_ioas.ioas_id); 1479 case IOMMU_TEST_OP_ACCESS_PAGES: 1480 return iommufd_test_access_pages( 1481 ucmd, cmd->id, cmd->access_pages.iova, 1482 cmd->access_pages.length, 1483 u64_to_user_ptr(cmd->access_pages.uptr), 1484 cmd->access_pages.flags); 1485 case IOMMU_TEST_OP_ACCESS_RW: 1486 return iommufd_test_access_rw( 1487 ucmd, cmd->id, cmd->access_rw.iova, 1488 cmd->access_rw.length, 1489 u64_to_user_ptr(cmd->access_rw.uptr), 1490 cmd->access_rw.flags); 1491 case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: 1492 return iommufd_test_access_item_destroy( 1493 ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id); 1494 case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: 1495 /* Protect _batch_init(), can not be less than elmsz */ 1496 if (cmd->memory_limit.limit < 1497 sizeof(unsigned long) + sizeof(u32)) 1498 return -EINVAL; 1499 iommufd_test_memory_limit = cmd->memory_limit.limit; 1500 return 0; 1501 case IOMMU_TEST_OP_DIRTY: 1502 return iommufd_test_dirty(ucmd, cmd->id, cmd->dirty.iova, 1503 cmd->dirty.length, 1504 cmd->dirty.page_size, 1505 u64_to_user_ptr(cmd->dirty.uptr), 1506 cmd->dirty.flags); 1507 case IOMMU_TEST_OP_TRIGGER_IOPF: 1508 return iommufd_test_trigger_iopf(ucmd, cmd); 1509 default: 1510 return -EOPNOTSUPP; 1511 } 1512 } 1513 1514 bool iommufd_should_fail(void) 1515 { 1516 return should_fail(&fail_iommufd, 1); 1517 } 1518 1519 int __init iommufd_test_init(void) 1520 { 1521 struct platform_device_info pdevinfo = { 1522 .name = "iommufd_selftest_iommu", 1523 }; 1524 int rc; 1525 1526 dbgfs_root = 1527 fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); 1528 1529 selftest_iommu_dev = platform_device_register_full(&pdevinfo); 1530 if (IS_ERR(selftest_iommu_dev)) { 1531 rc = PTR_ERR(selftest_iommu_dev); 1532 goto err_dbgfs; 1533 } 1534 1535 rc = bus_register(&iommufd_mock_bus_type.bus); 1536 if (rc) 1537 goto err_platform; 1538 1539 rc = iommu_device_sysfs_add(&mock_iommu_device, 1540 &selftest_iommu_dev->dev, NULL, "%s", 1541 dev_name(&selftest_iommu_dev->dev)); 1542 if (rc) 1543 goto err_bus; 1544 1545 rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops, 1546 &iommufd_mock_bus_type.bus, 1547 &iommufd_mock_bus_type.nb); 1548 if (rc) 1549 goto err_sysfs; 1550 1551 mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq"); 1552 1553 return 0; 1554 1555 err_sysfs: 1556 iommu_device_sysfs_remove(&mock_iommu_device); 1557 err_bus: 1558 bus_unregister(&iommufd_mock_bus_type.bus); 1559 err_platform: 1560 platform_device_unregister(selftest_iommu_dev); 1561 err_dbgfs: 1562 debugfs_remove_recursive(dbgfs_root); 1563 return rc; 1564 } 1565 1566 void iommufd_test_exit(void) 1567 { 1568 if (mock_iommu_iopf_queue) { 1569 iopf_queue_free(mock_iommu_iopf_queue); 1570 mock_iommu_iopf_queue = NULL; 1571 } 1572 1573 iommu_device_sysfs_remove(&mock_iommu_device); 1574 iommu_device_unregister_bus(&mock_iommu_device, 1575 &iommufd_mock_bus_type.bus, 1576 &iommufd_mock_bus_type.nb); 1577 bus_unregister(&iommufd_mock_bus_type.bus); 1578 platform_device_unregister(selftest_iommu_dev); 1579 debugfs_remove_recursive(dbgfs_root); 1580 } 1581