1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. 3 * 4 * Kernel side components to support tools/testing/selftests/iommu 5 */ 6 #include <linux/slab.h> 7 #include <linux/iommu.h> 8 #include <linux/xarray.h> 9 #include <linux/file.h> 10 #include <linux/anon_inodes.h> 11 #include <linux/fault-inject.h> 12 #include <uapi/linux/iommufd.h> 13 14 #include "io_pagetable.h" 15 #include "iommufd_private.h" 16 #include "iommufd_test.h" 17 18 static DECLARE_FAULT_ATTR(fail_iommufd); 19 static struct dentry *dbgfs_root; 20 21 size_t iommufd_test_memory_limit = 65536; 22 23 enum { 24 MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 25 26 /* 27 * Like a real page table alignment requires the low bits of the address 28 * to be zero. xarray also requires the high bit to be zero, so we store 29 * the pfns shifted. The upper bits are used for metadata. 30 */ 31 MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 32 33 _MOCK_PFN_START = MOCK_PFN_MASK + 1, 34 MOCK_PFN_START_IOVA = _MOCK_PFN_START, 35 MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 36 }; 37 38 /* 39 * Syzkaller has trouble randomizing the correct iova to use since it is linked 40 * to the map ioctl's output, and it has no ide about that. So, simplify things. 41 * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset 42 * value. This has a much smaller randomization space and syzkaller can hit it. 43 */ 44 static unsigned long iommufd_test_syz_conv_iova(struct io_pagetable *iopt, 45 u64 *iova) 46 { 47 struct syz_layout { 48 __u32 nth_area; 49 __u32 offset; 50 }; 51 struct syz_layout *syz = (void *)iova; 52 unsigned int nth = syz->nth_area; 53 struct iopt_area *area; 54 55 down_read(&iopt->iova_rwsem); 56 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; 57 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { 58 if (nth == 0) { 59 up_read(&iopt->iova_rwsem); 60 return iopt_area_iova(area) + syz->offset; 61 } 62 nth--; 63 } 64 up_read(&iopt->iova_rwsem); 65 66 return 0; 67 } 68 69 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 70 unsigned int ioas_id, u64 *iova, u32 *flags) 71 { 72 struct iommufd_ioas *ioas; 73 74 if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) 75 return; 76 *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; 77 78 ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); 79 if (IS_ERR(ioas)) 80 return; 81 *iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova); 82 iommufd_put_object(&ioas->obj); 83 } 84 85 struct mock_iommu_domain { 86 struct iommu_domain domain; 87 struct xarray pfns; 88 }; 89 90 enum selftest_obj_type { 91 TYPE_IDEV, 92 }; 93 94 struct mock_dev { 95 struct device dev; 96 }; 97 98 struct selftest_obj { 99 struct iommufd_object obj; 100 enum selftest_obj_type type; 101 102 union { 103 struct { 104 struct iommufd_device *idev; 105 struct iommufd_ctx *ictx; 106 struct mock_dev *mock_dev; 107 } idev; 108 }; 109 }; 110 111 static void mock_domain_blocking_free(struct iommu_domain *domain) 112 { 113 } 114 115 static int mock_domain_nop_attach(struct iommu_domain *domain, 116 struct device *dev) 117 { 118 return 0; 119 } 120 121 static const struct iommu_domain_ops mock_blocking_ops = { 122 .free = mock_domain_blocking_free, 123 .attach_dev = mock_domain_nop_attach, 124 }; 125 126 static struct iommu_domain mock_blocking_domain = { 127 .type = IOMMU_DOMAIN_BLOCKED, 128 .ops = &mock_blocking_ops, 129 }; 130 131 static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type) 132 { 133 struct mock_iommu_domain *mock; 134 135 if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED) 136 return &mock_blocking_domain; 137 138 if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED)) 139 return NULL; 140 141 mock = kzalloc(sizeof(*mock), GFP_KERNEL); 142 if (!mock) 143 return NULL; 144 mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 145 mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 146 mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 147 xa_init(&mock->pfns); 148 return &mock->domain; 149 } 150 151 static void mock_domain_free(struct iommu_domain *domain) 152 { 153 struct mock_iommu_domain *mock = 154 container_of(domain, struct mock_iommu_domain, domain); 155 156 WARN_ON(!xa_empty(&mock->pfns)); 157 kfree(mock); 158 } 159 160 static int mock_domain_map_pages(struct iommu_domain *domain, 161 unsigned long iova, phys_addr_t paddr, 162 size_t pgsize, size_t pgcount, int prot, 163 gfp_t gfp, size_t *mapped) 164 { 165 struct mock_iommu_domain *mock = 166 container_of(domain, struct mock_iommu_domain, domain); 167 unsigned long flags = MOCK_PFN_START_IOVA; 168 unsigned long start_iova = iova; 169 170 /* 171 * xarray does not reliably work with fault injection because it does a 172 * retry allocation, so put our own failure point. 173 */ 174 if (iommufd_should_fail()) 175 return -ENOENT; 176 177 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 178 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 179 for (; pgcount; pgcount--) { 180 size_t cur; 181 182 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 183 void *old; 184 185 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 186 flags = MOCK_PFN_LAST_IOVA; 187 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 188 xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 189 flags), 190 gfp); 191 if (xa_is_err(old)) { 192 for (; start_iova != iova; 193 start_iova += MOCK_IO_PAGE_SIZE) 194 xa_erase(&mock->pfns, 195 start_iova / 196 MOCK_IO_PAGE_SIZE); 197 return xa_err(old); 198 } 199 WARN_ON(old); 200 iova += MOCK_IO_PAGE_SIZE; 201 paddr += MOCK_IO_PAGE_SIZE; 202 *mapped += MOCK_IO_PAGE_SIZE; 203 flags = 0; 204 } 205 } 206 return 0; 207 } 208 209 static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 210 unsigned long iova, size_t pgsize, 211 size_t pgcount, 212 struct iommu_iotlb_gather *iotlb_gather) 213 { 214 struct mock_iommu_domain *mock = 215 container_of(domain, struct mock_iommu_domain, domain); 216 bool first = true; 217 size_t ret = 0; 218 void *ent; 219 220 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 221 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 222 223 for (; pgcount; pgcount--) { 224 size_t cur; 225 226 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 227 ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 228 WARN_ON(!ent); 229 /* 230 * iommufd generates unmaps that must be a strict 231 * superset of the map's performend So every starting 232 * IOVA should have been an iova passed to map, and the 233 * 234 * First IOVA must be present and have been a first IOVA 235 * passed to map_pages 236 */ 237 if (first) { 238 WARN_ON(!(xa_to_value(ent) & 239 MOCK_PFN_START_IOVA)); 240 first = false; 241 } 242 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 243 WARN_ON(!(xa_to_value(ent) & 244 MOCK_PFN_LAST_IOVA)); 245 246 iova += MOCK_IO_PAGE_SIZE; 247 ret += MOCK_IO_PAGE_SIZE; 248 } 249 } 250 return ret; 251 } 252 253 static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 254 dma_addr_t iova) 255 { 256 struct mock_iommu_domain *mock = 257 container_of(domain, struct mock_iommu_domain, domain); 258 void *ent; 259 260 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 261 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 262 WARN_ON(!ent); 263 return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 264 } 265 266 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) 267 { 268 return cap == IOMMU_CAP_CACHE_COHERENCY; 269 } 270 271 static void mock_domain_set_plaform_dma_ops(struct device *dev) 272 { 273 /* 274 * mock doesn't setup default domains because we can't hook into the 275 * normal probe path 276 */ 277 } 278 279 static const struct iommu_ops mock_ops = { 280 .owner = THIS_MODULE, 281 .pgsize_bitmap = MOCK_IO_PAGE_SIZE, 282 .domain_alloc = mock_domain_alloc, 283 .capable = mock_domain_capable, 284 .set_platform_dma_ops = mock_domain_set_plaform_dma_ops, 285 .default_domain_ops = 286 &(struct iommu_domain_ops){ 287 .free = mock_domain_free, 288 .attach_dev = mock_domain_nop_attach, 289 .map_pages = mock_domain_map_pages, 290 .unmap_pages = mock_domain_unmap_pages, 291 .iova_to_phys = mock_domain_iova_to_phys, 292 }, 293 }; 294 295 static struct iommu_device mock_iommu_device = { 296 .ops = &mock_ops, 297 }; 298 299 static inline struct iommufd_hw_pagetable * 300 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, 301 struct mock_iommu_domain **mock) 302 { 303 struct iommufd_hw_pagetable *hwpt; 304 struct iommufd_object *obj; 305 306 obj = iommufd_get_object(ucmd->ictx, mockpt_id, 307 IOMMUFD_OBJ_HW_PAGETABLE); 308 if (IS_ERR(obj)) 309 return ERR_CAST(obj); 310 hwpt = container_of(obj, struct iommufd_hw_pagetable, obj); 311 if (hwpt->domain->ops != mock_ops.default_domain_ops) { 312 iommufd_put_object(&hwpt->obj); 313 return ERR_PTR(-EINVAL); 314 } 315 *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain); 316 return hwpt; 317 } 318 319 static struct bus_type iommufd_mock_bus_type = { 320 .name = "iommufd_mock", 321 .iommu_ops = &mock_ops, 322 }; 323 324 static void mock_dev_release(struct device *dev) 325 { 326 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 327 328 kfree(mdev); 329 } 330 331 static struct mock_dev *mock_dev_create(void) 332 { 333 struct iommu_group *iommu_group; 334 struct dev_iommu *dev_iommu; 335 struct mock_dev *mdev; 336 int rc; 337 338 mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); 339 if (!mdev) 340 return ERR_PTR(-ENOMEM); 341 342 device_initialize(&mdev->dev); 343 mdev->dev.release = mock_dev_release; 344 mdev->dev.bus = &iommufd_mock_bus_type; 345 346 iommu_group = iommu_group_alloc(); 347 if (IS_ERR(iommu_group)) { 348 rc = PTR_ERR(iommu_group); 349 goto err_put; 350 } 351 352 rc = dev_set_name(&mdev->dev, "iommufd_mock%u", 353 iommu_group_id(iommu_group)); 354 if (rc) 355 goto err_group; 356 357 /* 358 * The iommu core has no way to associate a single device with an iommu 359 * driver (heck currently it can't even support two iommu_drivers 360 * registering). Hack it together with an open coded dev_iommu_get(). 361 * Notice that the normal notifier triggered iommu release process also 362 * does not work here because this bus is not in iommu_buses. 363 */ 364 mdev->dev.iommu = kzalloc(sizeof(*dev_iommu), GFP_KERNEL); 365 if (!mdev->dev.iommu) { 366 rc = -ENOMEM; 367 goto err_group; 368 } 369 mutex_init(&mdev->dev.iommu->lock); 370 mdev->dev.iommu->iommu_dev = &mock_iommu_device; 371 372 rc = device_add(&mdev->dev); 373 if (rc) 374 goto err_dev_iommu; 375 376 rc = iommu_group_add_device(iommu_group, &mdev->dev); 377 if (rc) 378 goto err_del; 379 iommu_group_put(iommu_group); 380 return mdev; 381 382 err_del: 383 device_del(&mdev->dev); 384 err_dev_iommu: 385 kfree(mdev->dev.iommu); 386 mdev->dev.iommu = NULL; 387 err_group: 388 iommu_group_put(iommu_group); 389 err_put: 390 put_device(&mdev->dev); 391 return ERR_PTR(rc); 392 } 393 394 static void mock_dev_destroy(struct mock_dev *mdev) 395 { 396 iommu_group_remove_device(&mdev->dev); 397 device_del(&mdev->dev); 398 kfree(mdev->dev.iommu); 399 mdev->dev.iommu = NULL; 400 put_device(&mdev->dev); 401 } 402 403 bool iommufd_selftest_is_mock_dev(struct device *dev) 404 { 405 return dev->release == mock_dev_release; 406 } 407 408 /* Create an hw_pagetable with the mock domain so we can test the domain ops */ 409 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, 410 struct iommu_test_cmd *cmd) 411 { 412 struct iommufd_device *idev; 413 struct selftest_obj *sobj; 414 u32 pt_id = cmd->id; 415 u32 idev_id; 416 int rc; 417 418 sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); 419 if (IS_ERR(sobj)) 420 return PTR_ERR(sobj); 421 422 sobj->idev.ictx = ucmd->ictx; 423 sobj->type = TYPE_IDEV; 424 425 sobj->idev.mock_dev = mock_dev_create(); 426 if (IS_ERR(sobj->idev.mock_dev)) { 427 rc = PTR_ERR(sobj->idev.mock_dev); 428 goto out_sobj; 429 } 430 431 idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, 432 &idev_id); 433 if (IS_ERR(idev)) { 434 rc = PTR_ERR(idev); 435 goto out_mdev; 436 } 437 sobj->idev.idev = idev; 438 439 rc = iommufd_device_attach(idev, &pt_id); 440 if (rc) 441 goto out_unbind; 442 443 /* Userspace must destroy the device_id to destroy the object */ 444 cmd->mock_domain.out_hwpt_id = pt_id; 445 cmd->mock_domain.out_stdev_id = sobj->obj.id; 446 cmd->mock_domain.out_idev_id = idev_id; 447 iommufd_object_finalize(ucmd->ictx, &sobj->obj); 448 return iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 449 450 out_unbind: 451 iommufd_device_unbind(idev); 452 out_mdev: 453 mock_dev_destroy(sobj->idev.mock_dev); 454 out_sobj: 455 iommufd_object_abort(ucmd->ictx, &sobj->obj); 456 return rc; 457 } 458 459 /* Replace the mock domain with a manually allocated hw_pagetable */ 460 static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, 461 unsigned int device_id, u32 pt_id, 462 struct iommu_test_cmd *cmd) 463 { 464 struct iommufd_object *dev_obj; 465 struct selftest_obj *sobj; 466 int rc; 467 468 /* 469 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure 470 * it doesn't race with detach, which is not allowed. 471 */ 472 dev_obj = 473 iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST); 474 if (IS_ERR(dev_obj)) 475 return PTR_ERR(dev_obj); 476 477 sobj = container_of(dev_obj, struct selftest_obj, obj); 478 if (sobj->type != TYPE_IDEV) { 479 rc = -EINVAL; 480 goto out_dev_obj; 481 } 482 483 rc = iommufd_device_replace(sobj->idev.idev, &pt_id); 484 if (rc) 485 goto out_dev_obj; 486 487 cmd->mock_domain_replace.pt_id = pt_id; 488 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 489 490 out_dev_obj: 491 iommufd_put_object(dev_obj); 492 return rc; 493 } 494 495 /* Add an additional reserved IOVA to the IOAS */ 496 static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, 497 unsigned int mockpt_id, 498 unsigned long start, size_t length) 499 { 500 struct iommufd_ioas *ioas; 501 int rc; 502 503 ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); 504 if (IS_ERR(ioas)) 505 return PTR_ERR(ioas); 506 down_write(&ioas->iopt.iova_rwsem); 507 rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); 508 up_write(&ioas->iopt.iova_rwsem); 509 iommufd_put_object(&ioas->obj); 510 return rc; 511 } 512 513 /* Check that every pfn under each iova matches the pfn under a user VA */ 514 static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, 515 unsigned int mockpt_id, unsigned long iova, 516 size_t length, void __user *uptr) 517 { 518 struct iommufd_hw_pagetable *hwpt; 519 struct mock_iommu_domain *mock; 520 uintptr_t end; 521 int rc; 522 523 if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 524 (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 525 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 526 return -EINVAL; 527 528 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 529 if (IS_ERR(hwpt)) 530 return PTR_ERR(hwpt); 531 532 for (; length; length -= MOCK_IO_PAGE_SIZE) { 533 struct page *pages[1]; 534 unsigned long pfn; 535 long npages; 536 void *ent; 537 538 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 539 pages); 540 if (npages < 0) { 541 rc = npages; 542 goto out_put; 543 } 544 if (WARN_ON(npages != 1)) { 545 rc = -EFAULT; 546 goto out_put; 547 } 548 pfn = page_to_pfn(pages[0]); 549 put_page(pages[0]); 550 551 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 552 if (!ent || 553 (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 554 pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 555 rc = -EINVAL; 556 goto out_put; 557 } 558 iova += MOCK_IO_PAGE_SIZE; 559 uptr += MOCK_IO_PAGE_SIZE; 560 } 561 rc = 0; 562 563 out_put: 564 iommufd_put_object(&hwpt->obj); 565 return rc; 566 } 567 568 /* Check that the page ref count matches, to look for missing pin/unpins */ 569 static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, 570 void __user *uptr, size_t length, 571 unsigned int refs) 572 { 573 uintptr_t end; 574 575 if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || 576 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 577 return -EINVAL; 578 579 for (; length; length -= PAGE_SIZE) { 580 struct page *pages[1]; 581 long npages; 582 583 npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages); 584 if (npages < 0) 585 return npages; 586 if (WARN_ON(npages != 1)) 587 return -EFAULT; 588 if (!PageCompound(pages[0])) { 589 unsigned int count; 590 591 count = page_ref_count(pages[0]); 592 if (count / GUP_PIN_COUNTING_BIAS != refs) { 593 put_page(pages[0]); 594 return -EIO; 595 } 596 } 597 put_page(pages[0]); 598 uptr += PAGE_SIZE; 599 } 600 return 0; 601 } 602 603 struct selftest_access { 604 struct iommufd_access *access; 605 struct file *file; 606 struct mutex lock; 607 struct list_head items; 608 unsigned int next_id; 609 bool destroying; 610 }; 611 612 struct selftest_access_item { 613 struct list_head items_elm; 614 unsigned long iova; 615 size_t length; 616 unsigned int id; 617 }; 618 619 static const struct file_operations iommfd_test_staccess_fops; 620 621 static struct selftest_access *iommufd_access_get(int fd) 622 { 623 struct file *file; 624 625 file = fget(fd); 626 if (!file) 627 return ERR_PTR(-EBADFD); 628 629 if (file->f_op != &iommfd_test_staccess_fops) { 630 fput(file); 631 return ERR_PTR(-EBADFD); 632 } 633 return file->private_data; 634 } 635 636 static void iommufd_test_access_unmap(void *data, unsigned long iova, 637 unsigned long length) 638 { 639 unsigned long iova_last = iova + length - 1; 640 struct selftest_access *staccess = data; 641 struct selftest_access_item *item; 642 struct selftest_access_item *tmp; 643 644 mutex_lock(&staccess->lock); 645 list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { 646 if (iova > item->iova + item->length - 1 || 647 iova_last < item->iova) 648 continue; 649 list_del(&item->items_elm); 650 iommufd_access_unpin_pages(staccess->access, item->iova, 651 item->length); 652 kfree(item); 653 } 654 mutex_unlock(&staccess->lock); 655 } 656 657 static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, 658 unsigned int access_id, 659 unsigned int item_id) 660 { 661 struct selftest_access_item *item; 662 struct selftest_access *staccess; 663 664 staccess = iommufd_access_get(access_id); 665 if (IS_ERR(staccess)) 666 return PTR_ERR(staccess); 667 668 mutex_lock(&staccess->lock); 669 list_for_each_entry(item, &staccess->items, items_elm) { 670 if (item->id == item_id) { 671 list_del(&item->items_elm); 672 iommufd_access_unpin_pages(staccess->access, item->iova, 673 item->length); 674 mutex_unlock(&staccess->lock); 675 kfree(item); 676 fput(staccess->file); 677 return 0; 678 } 679 } 680 mutex_unlock(&staccess->lock); 681 fput(staccess->file); 682 return -ENOENT; 683 } 684 685 static int iommufd_test_staccess_release(struct inode *inode, 686 struct file *filep) 687 { 688 struct selftest_access *staccess = filep->private_data; 689 690 if (staccess->access) { 691 iommufd_test_access_unmap(staccess, 0, ULONG_MAX); 692 iommufd_access_destroy(staccess->access); 693 } 694 mutex_destroy(&staccess->lock); 695 kfree(staccess); 696 return 0; 697 } 698 699 static const struct iommufd_access_ops selftest_access_ops_pin = { 700 .needs_pin_pages = 1, 701 .unmap = iommufd_test_access_unmap, 702 }; 703 704 static const struct iommufd_access_ops selftest_access_ops = { 705 .unmap = iommufd_test_access_unmap, 706 }; 707 708 static const struct file_operations iommfd_test_staccess_fops = { 709 .release = iommufd_test_staccess_release, 710 }; 711 712 static struct selftest_access *iommufd_test_alloc_access(void) 713 { 714 struct selftest_access *staccess; 715 struct file *filep; 716 717 staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT); 718 if (!staccess) 719 return ERR_PTR(-ENOMEM); 720 INIT_LIST_HEAD(&staccess->items); 721 mutex_init(&staccess->lock); 722 723 filep = anon_inode_getfile("[iommufd_test_staccess]", 724 &iommfd_test_staccess_fops, staccess, 725 O_RDWR); 726 if (IS_ERR(filep)) { 727 kfree(staccess); 728 return ERR_CAST(filep); 729 } 730 staccess->file = filep; 731 return staccess; 732 } 733 734 static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, 735 unsigned int ioas_id, unsigned int flags) 736 { 737 struct iommu_test_cmd *cmd = ucmd->cmd; 738 struct selftest_access *staccess; 739 struct iommufd_access *access; 740 u32 id; 741 int fdno; 742 int rc; 743 744 if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) 745 return -EOPNOTSUPP; 746 747 staccess = iommufd_test_alloc_access(); 748 if (IS_ERR(staccess)) 749 return PTR_ERR(staccess); 750 751 fdno = get_unused_fd_flags(O_CLOEXEC); 752 if (fdno < 0) { 753 rc = -ENOMEM; 754 goto out_free_staccess; 755 } 756 757 access = iommufd_access_create( 758 ucmd->ictx, 759 (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? 760 &selftest_access_ops_pin : 761 &selftest_access_ops, 762 staccess, &id); 763 if (IS_ERR(access)) { 764 rc = PTR_ERR(access); 765 goto out_put_fdno; 766 } 767 rc = iommufd_access_attach(access, ioas_id); 768 if (rc) 769 goto out_destroy; 770 cmd->create_access.out_access_fd = fdno; 771 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 772 if (rc) 773 goto out_destroy; 774 775 staccess->access = access; 776 fd_install(fdno, staccess->file); 777 return 0; 778 779 out_destroy: 780 iommufd_access_destroy(access); 781 out_put_fdno: 782 put_unused_fd(fdno); 783 out_free_staccess: 784 fput(staccess->file); 785 return rc; 786 } 787 788 /* Check that the pages in a page array match the pages in the user VA */ 789 static int iommufd_test_check_pages(void __user *uptr, struct page **pages, 790 size_t npages) 791 { 792 for (; npages; npages--) { 793 struct page *tmp_pages[1]; 794 long rc; 795 796 rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages); 797 if (rc < 0) 798 return rc; 799 if (WARN_ON(rc != 1)) 800 return -EFAULT; 801 put_page(tmp_pages[0]); 802 if (tmp_pages[0] != *pages) 803 return -EBADE; 804 pages++; 805 uptr += PAGE_SIZE; 806 } 807 return 0; 808 } 809 810 static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, 811 unsigned int access_id, unsigned long iova, 812 size_t length, void __user *uptr, 813 u32 flags) 814 { 815 struct iommu_test_cmd *cmd = ucmd->cmd; 816 struct selftest_access_item *item; 817 struct selftest_access *staccess; 818 struct page **pages; 819 size_t npages; 820 int rc; 821 822 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 823 if (length > 16*1024*1024) 824 return -ENOMEM; 825 826 if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) 827 return -EOPNOTSUPP; 828 829 staccess = iommufd_access_get(access_id); 830 if (IS_ERR(staccess)) 831 return PTR_ERR(staccess); 832 833 if (staccess->access->ops != &selftest_access_ops_pin) { 834 rc = -EOPNOTSUPP; 835 goto out_put; 836 } 837 838 if (flags & MOCK_FLAGS_ACCESS_SYZ) 839 iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt, 840 &cmd->access_pages.iova); 841 842 npages = (ALIGN(iova + length, PAGE_SIZE) - 843 ALIGN_DOWN(iova, PAGE_SIZE)) / 844 PAGE_SIZE; 845 pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT); 846 if (!pages) { 847 rc = -ENOMEM; 848 goto out_put; 849 } 850 851 /* 852 * Drivers will need to think very carefully about this locking. The 853 * core code can do multiple unmaps instantaneously after 854 * iommufd_access_pin_pages() and *all* the unmaps must not return until 855 * the range is unpinned. This simple implementation puts a global lock 856 * around the pin, which may not suit drivers that want this to be a 857 * performance path. drivers that get this wrong will trigger WARN_ON 858 * races and cause EDEADLOCK failures to userspace. 859 */ 860 mutex_lock(&staccess->lock); 861 rc = iommufd_access_pin_pages(staccess->access, iova, length, pages, 862 flags & MOCK_FLAGS_ACCESS_WRITE); 863 if (rc) 864 goto out_unlock; 865 866 /* For syzkaller allow uptr to be NULL to skip this check */ 867 if (uptr) { 868 rc = iommufd_test_check_pages( 869 uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, 870 npages); 871 if (rc) 872 goto out_unaccess; 873 } 874 875 item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT); 876 if (!item) { 877 rc = -ENOMEM; 878 goto out_unaccess; 879 } 880 881 item->iova = iova; 882 item->length = length; 883 item->id = staccess->next_id++; 884 list_add_tail(&item->items_elm, &staccess->items); 885 886 cmd->access_pages.out_access_pages_id = item->id; 887 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 888 if (rc) 889 goto out_free_item; 890 goto out_unlock; 891 892 out_free_item: 893 list_del(&item->items_elm); 894 kfree(item); 895 out_unaccess: 896 iommufd_access_unpin_pages(staccess->access, iova, length); 897 out_unlock: 898 mutex_unlock(&staccess->lock); 899 kvfree(pages); 900 out_put: 901 fput(staccess->file); 902 return rc; 903 } 904 905 static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, 906 unsigned int access_id, unsigned long iova, 907 size_t length, void __user *ubuf, 908 unsigned int flags) 909 { 910 struct iommu_test_cmd *cmd = ucmd->cmd; 911 struct selftest_access *staccess; 912 void *tmp; 913 int rc; 914 915 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 916 if (length > 16*1024*1024) 917 return -ENOMEM; 918 919 if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | 920 MOCK_FLAGS_ACCESS_SYZ)) 921 return -EOPNOTSUPP; 922 923 staccess = iommufd_access_get(access_id); 924 if (IS_ERR(staccess)) 925 return PTR_ERR(staccess); 926 927 tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT); 928 if (!tmp) { 929 rc = -ENOMEM; 930 goto out_put; 931 } 932 933 if (flags & MOCK_ACCESS_RW_WRITE) { 934 if (copy_from_user(tmp, ubuf, length)) { 935 rc = -EFAULT; 936 goto out_free; 937 } 938 } 939 940 if (flags & MOCK_FLAGS_ACCESS_SYZ) 941 iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt, 942 &cmd->access_rw.iova); 943 944 rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags); 945 if (rc) 946 goto out_free; 947 if (!(flags & MOCK_ACCESS_RW_WRITE)) { 948 if (copy_to_user(ubuf, tmp, length)) { 949 rc = -EFAULT; 950 goto out_free; 951 } 952 } 953 954 out_free: 955 kvfree(tmp); 956 out_put: 957 fput(staccess->file); 958 return rc; 959 } 960 static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); 961 static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == 962 __IOMMUFD_ACCESS_RW_SLOW_PATH); 963 964 void iommufd_selftest_destroy(struct iommufd_object *obj) 965 { 966 struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj); 967 968 switch (sobj->type) { 969 case TYPE_IDEV: 970 iommufd_device_detach(sobj->idev.idev); 971 iommufd_device_unbind(sobj->idev.idev); 972 mock_dev_destroy(sobj->idev.mock_dev); 973 break; 974 } 975 } 976 977 int iommufd_test(struct iommufd_ucmd *ucmd) 978 { 979 struct iommu_test_cmd *cmd = ucmd->cmd; 980 981 switch (cmd->op) { 982 case IOMMU_TEST_OP_ADD_RESERVED: 983 return iommufd_test_add_reserved(ucmd, cmd->id, 984 cmd->add_reserved.start, 985 cmd->add_reserved.length); 986 case IOMMU_TEST_OP_MOCK_DOMAIN: 987 return iommufd_test_mock_domain(ucmd, cmd); 988 case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: 989 return iommufd_test_mock_domain_replace( 990 ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd); 991 case IOMMU_TEST_OP_MD_CHECK_MAP: 992 return iommufd_test_md_check_pa( 993 ucmd, cmd->id, cmd->check_map.iova, 994 cmd->check_map.length, 995 u64_to_user_ptr(cmd->check_map.uptr)); 996 case IOMMU_TEST_OP_MD_CHECK_REFS: 997 return iommufd_test_md_check_refs( 998 ucmd, u64_to_user_ptr(cmd->check_refs.uptr), 999 cmd->check_refs.length, cmd->check_refs.refs); 1000 case IOMMU_TEST_OP_CREATE_ACCESS: 1001 return iommufd_test_create_access(ucmd, cmd->id, 1002 cmd->create_access.flags); 1003 case IOMMU_TEST_OP_ACCESS_PAGES: 1004 return iommufd_test_access_pages( 1005 ucmd, cmd->id, cmd->access_pages.iova, 1006 cmd->access_pages.length, 1007 u64_to_user_ptr(cmd->access_pages.uptr), 1008 cmd->access_pages.flags); 1009 case IOMMU_TEST_OP_ACCESS_RW: 1010 return iommufd_test_access_rw( 1011 ucmd, cmd->id, cmd->access_rw.iova, 1012 cmd->access_rw.length, 1013 u64_to_user_ptr(cmd->access_rw.uptr), 1014 cmd->access_rw.flags); 1015 case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: 1016 return iommufd_test_access_item_destroy( 1017 ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id); 1018 case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: 1019 /* Protect _batch_init(), can not be less than elmsz */ 1020 if (cmd->memory_limit.limit < 1021 sizeof(unsigned long) + sizeof(u32)) 1022 return -EINVAL; 1023 iommufd_test_memory_limit = cmd->memory_limit.limit; 1024 return 0; 1025 default: 1026 return -EOPNOTSUPP; 1027 } 1028 } 1029 1030 bool iommufd_should_fail(void) 1031 { 1032 return should_fail(&fail_iommufd, 1); 1033 } 1034 1035 void __init iommufd_test_init(void) 1036 { 1037 dbgfs_root = 1038 fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); 1039 WARN_ON(bus_register(&iommufd_mock_bus_type)); 1040 } 1041 1042 void iommufd_test_exit(void) 1043 { 1044 debugfs_remove_recursive(dbgfs_root); 1045 bus_unregister(&iommufd_mock_bus_type); 1046 } 1047