1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. 3 * 4 * Kernel side components to support tools/testing/selftests/iommu 5 */ 6 #include <linux/slab.h> 7 #include <linux/iommu.h> 8 #include <linux/xarray.h> 9 #include <linux/file.h> 10 #include <linux/anon_inodes.h> 11 #include <linux/fault-inject.h> 12 #include <linux/platform_device.h> 13 #include <uapi/linux/iommufd.h> 14 15 #include "../iommu-priv.h" 16 #include "io_pagetable.h" 17 #include "iommufd_private.h" 18 #include "iommufd_test.h" 19 20 static DECLARE_FAULT_ATTR(fail_iommufd); 21 static struct dentry *dbgfs_root; 22 static struct platform_device *selftest_iommu_dev; 23 static const struct iommu_ops mock_ops; 24 static struct iommu_domain_ops domain_nested_ops; 25 26 size_t iommufd_test_memory_limit = 65536; 27 28 struct mock_bus_type { 29 struct bus_type bus; 30 struct notifier_block nb; 31 }; 32 33 static struct mock_bus_type iommufd_mock_bus_type = { 34 .bus = { 35 .name = "iommufd_mock", 36 }, 37 }; 38 39 static DEFINE_IDA(mock_dev_ida); 40 41 enum { 42 MOCK_DIRTY_TRACK = 1, 43 MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 44 MOCK_HUGE_PAGE_SIZE = 512 * MOCK_IO_PAGE_SIZE, 45 46 /* 47 * Like a real page table alignment requires the low bits of the address 48 * to be zero. xarray also requires the high bit to be zero, so we store 49 * the pfns shifted. The upper bits are used for metadata. 50 */ 51 MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 52 53 _MOCK_PFN_START = MOCK_PFN_MASK + 1, 54 MOCK_PFN_START_IOVA = _MOCK_PFN_START, 55 MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 56 MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1, 57 MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, 58 }; 59 60 /* 61 * Syzkaller has trouble randomizing the correct iova to use since it is linked 62 * to the map ioctl's output, and it has no ide about that. So, simplify things. 63 * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset 64 * value. This has a much smaller randomization space and syzkaller can hit it. 65 */ 66 static unsigned long __iommufd_test_syz_conv_iova(struct io_pagetable *iopt, 67 u64 *iova) 68 { 69 struct syz_layout { 70 __u32 nth_area; 71 __u32 offset; 72 }; 73 struct syz_layout *syz = (void *)iova; 74 unsigned int nth = syz->nth_area; 75 struct iopt_area *area; 76 77 down_read(&iopt->iova_rwsem); 78 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; 79 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { 80 if (nth == 0) { 81 up_read(&iopt->iova_rwsem); 82 return iopt_area_iova(area) + syz->offset; 83 } 84 nth--; 85 } 86 up_read(&iopt->iova_rwsem); 87 88 return 0; 89 } 90 91 static unsigned long iommufd_test_syz_conv_iova(struct iommufd_access *access, 92 u64 *iova) 93 { 94 unsigned long ret; 95 96 mutex_lock(&access->ioas_lock); 97 if (!access->ioas) { 98 mutex_unlock(&access->ioas_lock); 99 return 0; 100 } 101 ret = __iommufd_test_syz_conv_iova(&access->ioas->iopt, iova); 102 mutex_unlock(&access->ioas_lock); 103 return ret; 104 } 105 106 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 107 unsigned int ioas_id, u64 *iova, u32 *flags) 108 { 109 struct iommufd_ioas *ioas; 110 111 if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) 112 return; 113 *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; 114 115 ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); 116 if (IS_ERR(ioas)) 117 return; 118 *iova = __iommufd_test_syz_conv_iova(&ioas->iopt, iova); 119 iommufd_put_object(ucmd->ictx, &ioas->obj); 120 } 121 122 struct mock_iommu_domain { 123 unsigned long flags; 124 struct iommu_domain domain; 125 struct xarray pfns; 126 }; 127 128 struct mock_iommu_domain_nested { 129 struct iommu_domain domain; 130 struct mock_iommu_domain *parent; 131 u32 iotlb[MOCK_NESTED_DOMAIN_IOTLB_NUM]; 132 }; 133 134 enum selftest_obj_type { 135 TYPE_IDEV, 136 }; 137 138 struct mock_dev { 139 struct device dev; 140 unsigned long flags; 141 int id; 142 }; 143 144 struct selftest_obj { 145 struct iommufd_object obj; 146 enum selftest_obj_type type; 147 148 union { 149 struct { 150 struct iommufd_device *idev; 151 struct iommufd_ctx *ictx; 152 struct mock_dev *mock_dev; 153 } idev; 154 }; 155 }; 156 157 static int mock_domain_nop_attach(struct iommu_domain *domain, 158 struct device *dev) 159 { 160 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 161 162 if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY)) 163 return -EINVAL; 164 165 return 0; 166 } 167 168 static const struct iommu_domain_ops mock_blocking_ops = { 169 .attach_dev = mock_domain_nop_attach, 170 }; 171 172 static struct iommu_domain mock_blocking_domain = { 173 .type = IOMMU_DOMAIN_BLOCKED, 174 .ops = &mock_blocking_ops, 175 }; 176 177 static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type) 178 { 179 struct iommu_test_hw_info *info; 180 181 info = kzalloc(sizeof(*info), GFP_KERNEL); 182 if (!info) 183 return ERR_PTR(-ENOMEM); 184 185 info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL; 186 *length = sizeof(*info); 187 *type = IOMMU_HW_INFO_TYPE_SELFTEST; 188 189 return info; 190 } 191 192 static int mock_domain_set_dirty_tracking(struct iommu_domain *domain, 193 bool enable) 194 { 195 struct mock_iommu_domain *mock = 196 container_of(domain, struct mock_iommu_domain, domain); 197 unsigned long flags = mock->flags; 198 199 if (enable && !domain->dirty_ops) 200 return -EINVAL; 201 202 /* No change? */ 203 if (!(enable ^ !!(flags & MOCK_DIRTY_TRACK))) 204 return 0; 205 206 flags = (enable ? flags | MOCK_DIRTY_TRACK : flags & ~MOCK_DIRTY_TRACK); 207 208 mock->flags = flags; 209 return 0; 210 } 211 212 static bool mock_test_and_clear_dirty(struct mock_iommu_domain *mock, 213 unsigned long iova, size_t page_size, 214 unsigned long flags) 215 { 216 unsigned long cur, end = iova + page_size - 1; 217 bool dirty = false; 218 void *ent, *old; 219 220 for (cur = iova; cur < end; cur += MOCK_IO_PAGE_SIZE) { 221 ent = xa_load(&mock->pfns, cur / MOCK_IO_PAGE_SIZE); 222 if (!ent || !(xa_to_value(ent) & MOCK_PFN_DIRTY_IOVA)) 223 continue; 224 225 dirty = true; 226 /* Clear dirty */ 227 if (!(flags & IOMMU_DIRTY_NO_CLEAR)) { 228 unsigned long val; 229 230 val = xa_to_value(ent) & ~MOCK_PFN_DIRTY_IOVA; 231 old = xa_store(&mock->pfns, cur / MOCK_IO_PAGE_SIZE, 232 xa_mk_value(val), GFP_KERNEL); 233 WARN_ON_ONCE(ent != old); 234 } 235 } 236 237 return dirty; 238 } 239 240 static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, 241 unsigned long iova, size_t size, 242 unsigned long flags, 243 struct iommu_dirty_bitmap *dirty) 244 { 245 struct mock_iommu_domain *mock = 246 container_of(domain, struct mock_iommu_domain, domain); 247 unsigned long end = iova + size; 248 void *ent; 249 250 if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap) 251 return -EINVAL; 252 253 do { 254 unsigned long pgsize = MOCK_IO_PAGE_SIZE; 255 unsigned long head; 256 257 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 258 if (!ent) { 259 iova += pgsize; 260 continue; 261 } 262 263 if (xa_to_value(ent) & MOCK_PFN_HUGE_IOVA) 264 pgsize = MOCK_HUGE_PAGE_SIZE; 265 head = iova & ~(pgsize - 1); 266 267 /* Clear dirty */ 268 if (mock_test_and_clear_dirty(mock, head, pgsize, flags)) 269 iommu_dirty_bitmap_record(dirty, iova, pgsize); 270 iova += pgsize; 271 } while (iova < end); 272 273 return 0; 274 } 275 276 const struct iommu_dirty_ops dirty_ops = { 277 .set_dirty_tracking = mock_domain_set_dirty_tracking, 278 .read_and_clear_dirty = mock_domain_read_and_clear_dirty, 279 }; 280 281 static struct iommu_domain *mock_domain_alloc_paging(struct device *dev) 282 { 283 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 284 struct mock_iommu_domain *mock; 285 286 mock = kzalloc(sizeof(*mock), GFP_KERNEL); 287 if (!mock) 288 return NULL; 289 mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 290 mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 291 mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 292 if (dev && mdev->flags & MOCK_FLAGS_DEVICE_HUGE_IOVA) 293 mock->domain.pgsize_bitmap |= MOCK_HUGE_PAGE_SIZE; 294 mock->domain.ops = mock_ops.default_domain_ops; 295 mock->domain.type = IOMMU_DOMAIN_UNMANAGED; 296 xa_init(&mock->pfns); 297 return &mock->domain; 298 } 299 300 static struct iommu_domain * 301 __mock_domain_alloc_nested(struct mock_iommu_domain *mock_parent, 302 const struct iommu_hwpt_selftest *user_cfg) 303 { 304 struct mock_iommu_domain_nested *mock_nested; 305 int i; 306 307 mock_nested = kzalloc(sizeof(*mock_nested), GFP_KERNEL); 308 if (!mock_nested) 309 return ERR_PTR(-ENOMEM); 310 mock_nested->parent = mock_parent; 311 mock_nested->domain.ops = &domain_nested_ops; 312 mock_nested->domain.type = IOMMU_DOMAIN_NESTED; 313 for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++) 314 mock_nested->iotlb[i] = user_cfg->iotlb; 315 return &mock_nested->domain; 316 } 317 318 static struct iommu_domain * 319 mock_domain_alloc_user(struct device *dev, u32 flags, 320 struct iommu_domain *parent, 321 const struct iommu_user_data *user_data) 322 { 323 struct mock_iommu_domain *mock_parent; 324 struct iommu_hwpt_selftest user_cfg; 325 int rc; 326 327 /* must be mock_domain */ 328 if (!parent) { 329 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 330 bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 331 bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY; 332 struct iommu_domain *domain; 333 334 if (flags & (~(IOMMU_HWPT_ALLOC_NEST_PARENT | 335 IOMMU_HWPT_ALLOC_DIRTY_TRACKING))) 336 return ERR_PTR(-EOPNOTSUPP); 337 if (user_data || (has_dirty_flag && no_dirty_ops)) 338 return ERR_PTR(-EOPNOTSUPP); 339 domain = mock_domain_alloc_paging(dev); 340 if (!domain) 341 return ERR_PTR(-ENOMEM); 342 if (has_dirty_flag) 343 container_of(domain, struct mock_iommu_domain, domain) 344 ->domain.dirty_ops = &dirty_ops; 345 return domain; 346 } 347 348 /* must be mock_domain_nested */ 349 if (user_data->type != IOMMU_HWPT_DATA_SELFTEST || flags) 350 return ERR_PTR(-EOPNOTSUPP); 351 if (!parent || parent->ops != mock_ops.default_domain_ops) 352 return ERR_PTR(-EINVAL); 353 354 mock_parent = container_of(parent, struct mock_iommu_domain, domain); 355 if (!mock_parent) 356 return ERR_PTR(-EINVAL); 357 358 rc = iommu_copy_struct_from_user(&user_cfg, user_data, 359 IOMMU_HWPT_DATA_SELFTEST, iotlb); 360 if (rc) 361 return ERR_PTR(rc); 362 363 return __mock_domain_alloc_nested(mock_parent, &user_cfg); 364 } 365 366 static void mock_domain_free(struct iommu_domain *domain) 367 { 368 struct mock_iommu_domain *mock = 369 container_of(domain, struct mock_iommu_domain, domain); 370 371 WARN_ON(!xa_empty(&mock->pfns)); 372 kfree(mock); 373 } 374 375 static int mock_domain_map_pages(struct iommu_domain *domain, 376 unsigned long iova, phys_addr_t paddr, 377 size_t pgsize, size_t pgcount, int prot, 378 gfp_t gfp, size_t *mapped) 379 { 380 struct mock_iommu_domain *mock = 381 container_of(domain, struct mock_iommu_domain, domain); 382 unsigned long flags = MOCK_PFN_START_IOVA; 383 unsigned long start_iova = iova; 384 385 /* 386 * xarray does not reliably work with fault injection because it does a 387 * retry allocation, so put our own failure point. 388 */ 389 if (iommufd_should_fail()) 390 return -ENOENT; 391 392 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 393 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 394 for (; pgcount; pgcount--) { 395 size_t cur; 396 397 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 398 void *old; 399 400 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 401 flags = MOCK_PFN_LAST_IOVA; 402 if (pgsize != MOCK_IO_PAGE_SIZE) { 403 flags |= MOCK_PFN_HUGE_IOVA; 404 } 405 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 406 xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 407 flags), 408 gfp); 409 if (xa_is_err(old)) { 410 for (; start_iova != iova; 411 start_iova += MOCK_IO_PAGE_SIZE) 412 xa_erase(&mock->pfns, 413 start_iova / 414 MOCK_IO_PAGE_SIZE); 415 return xa_err(old); 416 } 417 WARN_ON(old); 418 iova += MOCK_IO_PAGE_SIZE; 419 paddr += MOCK_IO_PAGE_SIZE; 420 *mapped += MOCK_IO_PAGE_SIZE; 421 flags = 0; 422 } 423 } 424 return 0; 425 } 426 427 static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 428 unsigned long iova, size_t pgsize, 429 size_t pgcount, 430 struct iommu_iotlb_gather *iotlb_gather) 431 { 432 struct mock_iommu_domain *mock = 433 container_of(domain, struct mock_iommu_domain, domain); 434 bool first = true; 435 size_t ret = 0; 436 void *ent; 437 438 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 439 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 440 441 for (; pgcount; pgcount--) { 442 size_t cur; 443 444 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 445 ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 446 447 /* 448 * iommufd generates unmaps that must be a strict 449 * superset of the map's performend So every 450 * starting/ending IOVA should have been an iova passed 451 * to map. 452 * 453 * This simple logic doesn't work when the HUGE_PAGE is 454 * turned on since the core code will automatically 455 * switch between the two page sizes creating a break in 456 * the unmap calls. The break can land in the middle of 457 * contiguous IOVA. 458 */ 459 if (!(domain->pgsize_bitmap & MOCK_HUGE_PAGE_SIZE)) { 460 if (first) { 461 WARN_ON(ent && !(xa_to_value(ent) & 462 MOCK_PFN_START_IOVA)); 463 first = false; 464 } 465 if (pgcount == 1 && 466 cur + MOCK_IO_PAGE_SIZE == pgsize) 467 WARN_ON(ent && !(xa_to_value(ent) & 468 MOCK_PFN_LAST_IOVA)); 469 } 470 471 iova += MOCK_IO_PAGE_SIZE; 472 ret += MOCK_IO_PAGE_SIZE; 473 } 474 } 475 return ret; 476 } 477 478 static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 479 dma_addr_t iova) 480 { 481 struct mock_iommu_domain *mock = 482 container_of(domain, struct mock_iommu_domain, domain); 483 void *ent; 484 485 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 486 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 487 WARN_ON(!ent); 488 return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 489 } 490 491 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) 492 { 493 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 494 495 switch (cap) { 496 case IOMMU_CAP_CACHE_COHERENCY: 497 return true; 498 case IOMMU_CAP_DIRTY_TRACKING: 499 return !(mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY); 500 default: 501 break; 502 } 503 504 return false; 505 } 506 507 static struct iopf_queue *mock_iommu_iopf_queue; 508 509 static struct iommu_device mock_iommu_device = { 510 }; 511 512 static struct iommu_device *mock_probe_device(struct device *dev) 513 { 514 if (dev->bus != &iommufd_mock_bus_type.bus) 515 return ERR_PTR(-ENODEV); 516 return &mock_iommu_device; 517 } 518 519 static void mock_domain_page_response(struct device *dev, struct iopf_fault *evt, 520 struct iommu_page_response *msg) 521 { 522 } 523 524 static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) 525 { 526 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 527 return -ENODEV; 528 529 return iopf_queue_add_device(mock_iommu_iopf_queue, dev); 530 } 531 532 static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 533 { 534 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 535 return -ENODEV; 536 537 iopf_queue_remove_device(mock_iommu_iopf_queue, dev); 538 539 return 0; 540 } 541 542 static const struct iommu_ops mock_ops = { 543 /* 544 * IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type() 545 * because it is zero. 546 */ 547 .default_domain = &mock_blocking_domain, 548 .blocked_domain = &mock_blocking_domain, 549 .owner = THIS_MODULE, 550 .pgsize_bitmap = MOCK_IO_PAGE_SIZE, 551 .hw_info = mock_domain_hw_info, 552 .domain_alloc_paging = mock_domain_alloc_paging, 553 .domain_alloc_user = mock_domain_alloc_user, 554 .capable = mock_domain_capable, 555 .device_group = generic_device_group, 556 .probe_device = mock_probe_device, 557 .page_response = mock_domain_page_response, 558 .dev_enable_feat = mock_dev_enable_feat, 559 .dev_disable_feat = mock_dev_disable_feat, 560 .user_pasid_table = true, 561 .default_domain_ops = 562 &(struct iommu_domain_ops){ 563 .free = mock_domain_free, 564 .attach_dev = mock_domain_nop_attach, 565 .map_pages = mock_domain_map_pages, 566 .unmap_pages = mock_domain_unmap_pages, 567 .iova_to_phys = mock_domain_iova_to_phys, 568 }, 569 }; 570 571 static void mock_domain_free_nested(struct iommu_domain *domain) 572 { 573 struct mock_iommu_domain_nested *mock_nested = 574 container_of(domain, struct mock_iommu_domain_nested, domain); 575 576 kfree(mock_nested); 577 } 578 579 static int 580 mock_domain_cache_invalidate_user(struct iommu_domain *domain, 581 struct iommu_user_data_array *array) 582 { 583 struct mock_iommu_domain_nested *mock_nested = 584 container_of(domain, struct mock_iommu_domain_nested, domain); 585 struct iommu_hwpt_invalidate_selftest inv; 586 u32 processed = 0; 587 int i = 0, j; 588 int rc = 0; 589 590 if (array->type != IOMMU_HWPT_INVALIDATE_DATA_SELFTEST) { 591 rc = -EINVAL; 592 goto out; 593 } 594 595 for ( ; i < array->entry_num; i++) { 596 rc = iommu_copy_struct_from_user_array(&inv, array, 597 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 598 i, iotlb_id); 599 if (rc) 600 break; 601 602 if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) { 603 rc = -EOPNOTSUPP; 604 break; 605 } 606 607 if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) { 608 rc = -EINVAL; 609 break; 610 } 611 612 if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) { 613 /* Invalidate all mock iotlb entries and ignore iotlb_id */ 614 for (j = 0; j < MOCK_NESTED_DOMAIN_IOTLB_NUM; j++) 615 mock_nested->iotlb[j] = 0; 616 } else { 617 mock_nested->iotlb[inv.iotlb_id] = 0; 618 } 619 620 processed++; 621 } 622 623 out: 624 array->entry_num = processed; 625 return rc; 626 } 627 628 static struct iommu_domain_ops domain_nested_ops = { 629 .free = mock_domain_free_nested, 630 .attach_dev = mock_domain_nop_attach, 631 .cache_invalidate_user = mock_domain_cache_invalidate_user, 632 }; 633 634 static inline struct iommufd_hw_pagetable * 635 __get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, u32 hwpt_type) 636 { 637 struct iommufd_object *obj; 638 639 obj = iommufd_get_object(ucmd->ictx, mockpt_id, hwpt_type); 640 if (IS_ERR(obj)) 641 return ERR_CAST(obj); 642 return container_of(obj, struct iommufd_hw_pagetable, obj); 643 } 644 645 static inline struct iommufd_hw_pagetable * 646 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, 647 struct mock_iommu_domain **mock) 648 { 649 struct iommufd_hw_pagetable *hwpt; 650 651 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_PAGING); 652 if (IS_ERR(hwpt)) 653 return hwpt; 654 if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED || 655 hwpt->domain->ops != mock_ops.default_domain_ops) { 656 iommufd_put_object(ucmd->ictx, &hwpt->obj); 657 return ERR_PTR(-EINVAL); 658 } 659 *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain); 660 return hwpt; 661 } 662 663 static inline struct iommufd_hw_pagetable * 664 get_md_pagetable_nested(struct iommufd_ucmd *ucmd, u32 mockpt_id, 665 struct mock_iommu_domain_nested **mock_nested) 666 { 667 struct iommufd_hw_pagetable *hwpt; 668 669 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_NESTED); 670 if (IS_ERR(hwpt)) 671 return hwpt; 672 if (hwpt->domain->type != IOMMU_DOMAIN_NESTED || 673 hwpt->domain->ops != &domain_nested_ops) { 674 iommufd_put_object(ucmd->ictx, &hwpt->obj); 675 return ERR_PTR(-EINVAL); 676 } 677 *mock_nested = container_of(hwpt->domain, 678 struct mock_iommu_domain_nested, domain); 679 return hwpt; 680 } 681 682 static void mock_dev_release(struct device *dev) 683 { 684 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 685 686 ida_free(&mock_dev_ida, mdev->id); 687 kfree(mdev); 688 } 689 690 static struct mock_dev *mock_dev_create(unsigned long dev_flags) 691 { 692 struct mock_dev *mdev; 693 int rc; 694 695 if (dev_flags & 696 ~(MOCK_FLAGS_DEVICE_NO_DIRTY | MOCK_FLAGS_DEVICE_HUGE_IOVA)) 697 return ERR_PTR(-EINVAL); 698 699 mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); 700 if (!mdev) 701 return ERR_PTR(-ENOMEM); 702 703 device_initialize(&mdev->dev); 704 mdev->flags = dev_flags; 705 mdev->dev.release = mock_dev_release; 706 mdev->dev.bus = &iommufd_mock_bus_type.bus; 707 708 rc = ida_alloc(&mock_dev_ida, GFP_KERNEL); 709 if (rc < 0) 710 goto err_put; 711 mdev->id = rc; 712 713 rc = dev_set_name(&mdev->dev, "iommufd_mock%u", mdev->id); 714 if (rc) 715 goto err_put; 716 717 rc = device_add(&mdev->dev); 718 if (rc) 719 goto err_put; 720 return mdev; 721 722 err_put: 723 put_device(&mdev->dev); 724 return ERR_PTR(rc); 725 } 726 727 static void mock_dev_destroy(struct mock_dev *mdev) 728 { 729 device_unregister(&mdev->dev); 730 } 731 732 bool iommufd_selftest_is_mock_dev(struct device *dev) 733 { 734 return dev->release == mock_dev_release; 735 } 736 737 /* Create an hw_pagetable with the mock domain so we can test the domain ops */ 738 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, 739 struct iommu_test_cmd *cmd) 740 { 741 struct iommufd_device *idev; 742 struct selftest_obj *sobj; 743 u32 pt_id = cmd->id; 744 u32 dev_flags = 0; 745 u32 idev_id; 746 int rc; 747 748 sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); 749 if (IS_ERR(sobj)) 750 return PTR_ERR(sobj); 751 752 sobj->idev.ictx = ucmd->ictx; 753 sobj->type = TYPE_IDEV; 754 755 if (cmd->op == IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS) 756 dev_flags = cmd->mock_domain_flags.dev_flags; 757 758 sobj->idev.mock_dev = mock_dev_create(dev_flags); 759 if (IS_ERR(sobj->idev.mock_dev)) { 760 rc = PTR_ERR(sobj->idev.mock_dev); 761 goto out_sobj; 762 } 763 764 idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, 765 &idev_id); 766 if (IS_ERR(idev)) { 767 rc = PTR_ERR(idev); 768 goto out_mdev; 769 } 770 sobj->idev.idev = idev; 771 772 rc = iommufd_device_attach(idev, &pt_id); 773 if (rc) 774 goto out_unbind; 775 776 /* Userspace must destroy the device_id to destroy the object */ 777 cmd->mock_domain.out_hwpt_id = pt_id; 778 cmd->mock_domain.out_stdev_id = sobj->obj.id; 779 cmd->mock_domain.out_idev_id = idev_id; 780 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 781 if (rc) 782 goto out_detach; 783 iommufd_object_finalize(ucmd->ictx, &sobj->obj); 784 return 0; 785 786 out_detach: 787 iommufd_device_detach(idev); 788 out_unbind: 789 iommufd_device_unbind(idev); 790 out_mdev: 791 mock_dev_destroy(sobj->idev.mock_dev); 792 out_sobj: 793 iommufd_object_abort(ucmd->ictx, &sobj->obj); 794 return rc; 795 } 796 797 /* Replace the mock domain with a manually allocated hw_pagetable */ 798 static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, 799 unsigned int device_id, u32 pt_id, 800 struct iommu_test_cmd *cmd) 801 { 802 struct iommufd_object *dev_obj; 803 struct selftest_obj *sobj; 804 int rc; 805 806 /* 807 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure 808 * it doesn't race with detach, which is not allowed. 809 */ 810 dev_obj = 811 iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST); 812 if (IS_ERR(dev_obj)) 813 return PTR_ERR(dev_obj); 814 815 sobj = container_of(dev_obj, struct selftest_obj, obj); 816 if (sobj->type != TYPE_IDEV) { 817 rc = -EINVAL; 818 goto out_dev_obj; 819 } 820 821 rc = iommufd_device_replace(sobj->idev.idev, &pt_id); 822 if (rc) 823 goto out_dev_obj; 824 825 cmd->mock_domain_replace.pt_id = pt_id; 826 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 827 828 out_dev_obj: 829 iommufd_put_object(ucmd->ictx, dev_obj); 830 return rc; 831 } 832 833 /* Add an additional reserved IOVA to the IOAS */ 834 static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, 835 unsigned int mockpt_id, 836 unsigned long start, size_t length) 837 { 838 struct iommufd_ioas *ioas; 839 int rc; 840 841 ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); 842 if (IS_ERR(ioas)) 843 return PTR_ERR(ioas); 844 down_write(&ioas->iopt.iova_rwsem); 845 rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); 846 up_write(&ioas->iopt.iova_rwsem); 847 iommufd_put_object(ucmd->ictx, &ioas->obj); 848 return rc; 849 } 850 851 /* Check that every pfn under each iova matches the pfn under a user VA */ 852 static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, 853 unsigned int mockpt_id, unsigned long iova, 854 size_t length, void __user *uptr) 855 { 856 struct iommufd_hw_pagetable *hwpt; 857 struct mock_iommu_domain *mock; 858 uintptr_t end; 859 int rc; 860 861 if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 862 (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 863 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 864 return -EINVAL; 865 866 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 867 if (IS_ERR(hwpt)) 868 return PTR_ERR(hwpt); 869 870 for (; length; length -= MOCK_IO_PAGE_SIZE) { 871 struct page *pages[1]; 872 unsigned long pfn; 873 long npages; 874 void *ent; 875 876 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 877 pages); 878 if (npages < 0) { 879 rc = npages; 880 goto out_put; 881 } 882 if (WARN_ON(npages != 1)) { 883 rc = -EFAULT; 884 goto out_put; 885 } 886 pfn = page_to_pfn(pages[0]); 887 put_page(pages[0]); 888 889 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 890 if (!ent || 891 (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 892 pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 893 rc = -EINVAL; 894 goto out_put; 895 } 896 iova += MOCK_IO_PAGE_SIZE; 897 uptr += MOCK_IO_PAGE_SIZE; 898 } 899 rc = 0; 900 901 out_put: 902 iommufd_put_object(ucmd->ictx, &hwpt->obj); 903 return rc; 904 } 905 906 /* Check that the page ref count matches, to look for missing pin/unpins */ 907 static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, 908 void __user *uptr, size_t length, 909 unsigned int refs) 910 { 911 uintptr_t end; 912 913 if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || 914 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 915 return -EINVAL; 916 917 for (; length; length -= PAGE_SIZE) { 918 struct page *pages[1]; 919 long npages; 920 921 npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages); 922 if (npages < 0) 923 return npages; 924 if (WARN_ON(npages != 1)) 925 return -EFAULT; 926 if (!PageCompound(pages[0])) { 927 unsigned int count; 928 929 count = page_ref_count(pages[0]); 930 if (count / GUP_PIN_COUNTING_BIAS != refs) { 931 put_page(pages[0]); 932 return -EIO; 933 } 934 } 935 put_page(pages[0]); 936 uptr += PAGE_SIZE; 937 } 938 return 0; 939 } 940 941 static int iommufd_test_md_check_iotlb(struct iommufd_ucmd *ucmd, 942 u32 mockpt_id, unsigned int iotlb_id, 943 u32 iotlb) 944 { 945 struct mock_iommu_domain_nested *mock_nested; 946 struct iommufd_hw_pagetable *hwpt; 947 int rc = 0; 948 949 hwpt = get_md_pagetable_nested(ucmd, mockpt_id, &mock_nested); 950 if (IS_ERR(hwpt)) 951 return PTR_ERR(hwpt); 952 953 mock_nested = container_of(hwpt->domain, 954 struct mock_iommu_domain_nested, domain); 955 956 if (iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX || 957 mock_nested->iotlb[iotlb_id] != iotlb) 958 rc = -EINVAL; 959 iommufd_put_object(ucmd->ictx, &hwpt->obj); 960 return rc; 961 } 962 963 struct selftest_access { 964 struct iommufd_access *access; 965 struct file *file; 966 struct mutex lock; 967 struct list_head items; 968 unsigned int next_id; 969 bool destroying; 970 }; 971 972 struct selftest_access_item { 973 struct list_head items_elm; 974 unsigned long iova; 975 size_t length; 976 unsigned int id; 977 }; 978 979 static const struct file_operations iommfd_test_staccess_fops; 980 981 static struct selftest_access *iommufd_access_get(int fd) 982 { 983 struct file *file; 984 985 file = fget(fd); 986 if (!file) 987 return ERR_PTR(-EBADFD); 988 989 if (file->f_op != &iommfd_test_staccess_fops) { 990 fput(file); 991 return ERR_PTR(-EBADFD); 992 } 993 return file->private_data; 994 } 995 996 static void iommufd_test_access_unmap(void *data, unsigned long iova, 997 unsigned long length) 998 { 999 unsigned long iova_last = iova + length - 1; 1000 struct selftest_access *staccess = data; 1001 struct selftest_access_item *item; 1002 struct selftest_access_item *tmp; 1003 1004 mutex_lock(&staccess->lock); 1005 list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { 1006 if (iova > item->iova + item->length - 1 || 1007 iova_last < item->iova) 1008 continue; 1009 list_del(&item->items_elm); 1010 iommufd_access_unpin_pages(staccess->access, item->iova, 1011 item->length); 1012 kfree(item); 1013 } 1014 mutex_unlock(&staccess->lock); 1015 } 1016 1017 static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, 1018 unsigned int access_id, 1019 unsigned int item_id) 1020 { 1021 struct selftest_access_item *item; 1022 struct selftest_access *staccess; 1023 1024 staccess = iommufd_access_get(access_id); 1025 if (IS_ERR(staccess)) 1026 return PTR_ERR(staccess); 1027 1028 mutex_lock(&staccess->lock); 1029 list_for_each_entry(item, &staccess->items, items_elm) { 1030 if (item->id == item_id) { 1031 list_del(&item->items_elm); 1032 iommufd_access_unpin_pages(staccess->access, item->iova, 1033 item->length); 1034 mutex_unlock(&staccess->lock); 1035 kfree(item); 1036 fput(staccess->file); 1037 return 0; 1038 } 1039 } 1040 mutex_unlock(&staccess->lock); 1041 fput(staccess->file); 1042 return -ENOENT; 1043 } 1044 1045 static int iommufd_test_staccess_release(struct inode *inode, 1046 struct file *filep) 1047 { 1048 struct selftest_access *staccess = filep->private_data; 1049 1050 if (staccess->access) { 1051 iommufd_test_access_unmap(staccess, 0, ULONG_MAX); 1052 iommufd_access_destroy(staccess->access); 1053 } 1054 mutex_destroy(&staccess->lock); 1055 kfree(staccess); 1056 return 0; 1057 } 1058 1059 static const struct iommufd_access_ops selftest_access_ops_pin = { 1060 .needs_pin_pages = 1, 1061 .unmap = iommufd_test_access_unmap, 1062 }; 1063 1064 static const struct iommufd_access_ops selftest_access_ops = { 1065 .unmap = iommufd_test_access_unmap, 1066 }; 1067 1068 static const struct file_operations iommfd_test_staccess_fops = { 1069 .release = iommufd_test_staccess_release, 1070 }; 1071 1072 static struct selftest_access *iommufd_test_alloc_access(void) 1073 { 1074 struct selftest_access *staccess; 1075 struct file *filep; 1076 1077 staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT); 1078 if (!staccess) 1079 return ERR_PTR(-ENOMEM); 1080 INIT_LIST_HEAD(&staccess->items); 1081 mutex_init(&staccess->lock); 1082 1083 filep = anon_inode_getfile("[iommufd_test_staccess]", 1084 &iommfd_test_staccess_fops, staccess, 1085 O_RDWR); 1086 if (IS_ERR(filep)) { 1087 kfree(staccess); 1088 return ERR_CAST(filep); 1089 } 1090 staccess->file = filep; 1091 return staccess; 1092 } 1093 1094 static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, 1095 unsigned int ioas_id, unsigned int flags) 1096 { 1097 struct iommu_test_cmd *cmd = ucmd->cmd; 1098 struct selftest_access *staccess; 1099 struct iommufd_access *access; 1100 u32 id; 1101 int fdno; 1102 int rc; 1103 1104 if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) 1105 return -EOPNOTSUPP; 1106 1107 staccess = iommufd_test_alloc_access(); 1108 if (IS_ERR(staccess)) 1109 return PTR_ERR(staccess); 1110 1111 fdno = get_unused_fd_flags(O_CLOEXEC); 1112 if (fdno < 0) { 1113 rc = -ENOMEM; 1114 goto out_free_staccess; 1115 } 1116 1117 access = iommufd_access_create( 1118 ucmd->ictx, 1119 (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? 1120 &selftest_access_ops_pin : 1121 &selftest_access_ops, 1122 staccess, &id); 1123 if (IS_ERR(access)) { 1124 rc = PTR_ERR(access); 1125 goto out_put_fdno; 1126 } 1127 rc = iommufd_access_attach(access, ioas_id); 1128 if (rc) 1129 goto out_destroy; 1130 cmd->create_access.out_access_fd = fdno; 1131 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1132 if (rc) 1133 goto out_destroy; 1134 1135 staccess->access = access; 1136 fd_install(fdno, staccess->file); 1137 return 0; 1138 1139 out_destroy: 1140 iommufd_access_destroy(access); 1141 out_put_fdno: 1142 put_unused_fd(fdno); 1143 out_free_staccess: 1144 fput(staccess->file); 1145 return rc; 1146 } 1147 1148 static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, 1149 unsigned int access_id, 1150 unsigned int ioas_id) 1151 { 1152 struct selftest_access *staccess; 1153 int rc; 1154 1155 staccess = iommufd_access_get(access_id); 1156 if (IS_ERR(staccess)) 1157 return PTR_ERR(staccess); 1158 1159 rc = iommufd_access_replace(staccess->access, ioas_id); 1160 fput(staccess->file); 1161 return rc; 1162 } 1163 1164 /* Check that the pages in a page array match the pages in the user VA */ 1165 static int iommufd_test_check_pages(void __user *uptr, struct page **pages, 1166 size_t npages) 1167 { 1168 for (; npages; npages--) { 1169 struct page *tmp_pages[1]; 1170 long rc; 1171 1172 rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages); 1173 if (rc < 0) 1174 return rc; 1175 if (WARN_ON(rc != 1)) 1176 return -EFAULT; 1177 put_page(tmp_pages[0]); 1178 if (tmp_pages[0] != *pages) 1179 return -EBADE; 1180 pages++; 1181 uptr += PAGE_SIZE; 1182 } 1183 return 0; 1184 } 1185 1186 static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, 1187 unsigned int access_id, unsigned long iova, 1188 size_t length, void __user *uptr, 1189 u32 flags) 1190 { 1191 struct iommu_test_cmd *cmd = ucmd->cmd; 1192 struct selftest_access_item *item; 1193 struct selftest_access *staccess; 1194 struct page **pages; 1195 size_t npages; 1196 int rc; 1197 1198 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1199 if (length > 16*1024*1024) 1200 return -ENOMEM; 1201 1202 if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) 1203 return -EOPNOTSUPP; 1204 1205 staccess = iommufd_access_get(access_id); 1206 if (IS_ERR(staccess)) 1207 return PTR_ERR(staccess); 1208 1209 if (staccess->access->ops != &selftest_access_ops_pin) { 1210 rc = -EOPNOTSUPP; 1211 goto out_put; 1212 } 1213 1214 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1215 iova = iommufd_test_syz_conv_iova(staccess->access, 1216 &cmd->access_pages.iova); 1217 1218 npages = (ALIGN(iova + length, PAGE_SIZE) - 1219 ALIGN_DOWN(iova, PAGE_SIZE)) / 1220 PAGE_SIZE; 1221 pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT); 1222 if (!pages) { 1223 rc = -ENOMEM; 1224 goto out_put; 1225 } 1226 1227 /* 1228 * Drivers will need to think very carefully about this locking. The 1229 * core code can do multiple unmaps instantaneously after 1230 * iommufd_access_pin_pages() and *all* the unmaps must not return until 1231 * the range is unpinned. This simple implementation puts a global lock 1232 * around the pin, which may not suit drivers that want this to be a 1233 * performance path. drivers that get this wrong will trigger WARN_ON 1234 * races and cause EDEADLOCK failures to userspace. 1235 */ 1236 mutex_lock(&staccess->lock); 1237 rc = iommufd_access_pin_pages(staccess->access, iova, length, pages, 1238 flags & MOCK_FLAGS_ACCESS_WRITE); 1239 if (rc) 1240 goto out_unlock; 1241 1242 /* For syzkaller allow uptr to be NULL to skip this check */ 1243 if (uptr) { 1244 rc = iommufd_test_check_pages( 1245 uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, 1246 npages); 1247 if (rc) 1248 goto out_unaccess; 1249 } 1250 1251 item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT); 1252 if (!item) { 1253 rc = -ENOMEM; 1254 goto out_unaccess; 1255 } 1256 1257 item->iova = iova; 1258 item->length = length; 1259 item->id = staccess->next_id++; 1260 list_add_tail(&item->items_elm, &staccess->items); 1261 1262 cmd->access_pages.out_access_pages_id = item->id; 1263 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1264 if (rc) 1265 goto out_free_item; 1266 goto out_unlock; 1267 1268 out_free_item: 1269 list_del(&item->items_elm); 1270 kfree(item); 1271 out_unaccess: 1272 iommufd_access_unpin_pages(staccess->access, iova, length); 1273 out_unlock: 1274 mutex_unlock(&staccess->lock); 1275 kvfree(pages); 1276 out_put: 1277 fput(staccess->file); 1278 return rc; 1279 } 1280 1281 static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, 1282 unsigned int access_id, unsigned long iova, 1283 size_t length, void __user *ubuf, 1284 unsigned int flags) 1285 { 1286 struct iommu_test_cmd *cmd = ucmd->cmd; 1287 struct selftest_access *staccess; 1288 void *tmp; 1289 int rc; 1290 1291 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1292 if (length > 16*1024*1024) 1293 return -ENOMEM; 1294 1295 if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | 1296 MOCK_FLAGS_ACCESS_SYZ)) 1297 return -EOPNOTSUPP; 1298 1299 staccess = iommufd_access_get(access_id); 1300 if (IS_ERR(staccess)) 1301 return PTR_ERR(staccess); 1302 1303 tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT); 1304 if (!tmp) { 1305 rc = -ENOMEM; 1306 goto out_put; 1307 } 1308 1309 if (flags & MOCK_ACCESS_RW_WRITE) { 1310 if (copy_from_user(tmp, ubuf, length)) { 1311 rc = -EFAULT; 1312 goto out_free; 1313 } 1314 } 1315 1316 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1317 iova = iommufd_test_syz_conv_iova(staccess->access, 1318 &cmd->access_rw.iova); 1319 1320 rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags); 1321 if (rc) 1322 goto out_free; 1323 if (!(flags & MOCK_ACCESS_RW_WRITE)) { 1324 if (copy_to_user(ubuf, tmp, length)) { 1325 rc = -EFAULT; 1326 goto out_free; 1327 } 1328 } 1329 1330 out_free: 1331 kvfree(tmp); 1332 out_put: 1333 fput(staccess->file); 1334 return rc; 1335 } 1336 static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); 1337 static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == 1338 __IOMMUFD_ACCESS_RW_SLOW_PATH); 1339 1340 static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, 1341 unsigned long iova, size_t length, 1342 unsigned long page_size, void __user *uptr, 1343 u32 flags) 1344 { 1345 unsigned long bitmap_size, i, max; 1346 struct iommu_test_cmd *cmd = ucmd->cmd; 1347 struct iommufd_hw_pagetable *hwpt; 1348 struct mock_iommu_domain *mock; 1349 int rc, count = 0; 1350 void *tmp; 1351 1352 if (!page_size || !length || iova % page_size || length % page_size || 1353 !uptr) 1354 return -EINVAL; 1355 1356 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 1357 if (IS_ERR(hwpt)) 1358 return PTR_ERR(hwpt); 1359 1360 if (!(mock->flags & MOCK_DIRTY_TRACK)) { 1361 rc = -EINVAL; 1362 goto out_put; 1363 } 1364 1365 max = length / page_size; 1366 bitmap_size = DIV_ROUND_UP(max, BITS_PER_BYTE); 1367 1368 tmp = kvzalloc(bitmap_size, GFP_KERNEL_ACCOUNT); 1369 if (!tmp) { 1370 rc = -ENOMEM; 1371 goto out_put; 1372 } 1373 1374 if (copy_from_user(tmp, uptr, bitmap_size)) { 1375 rc = -EFAULT; 1376 goto out_free; 1377 } 1378 1379 for (i = 0; i < max; i++) { 1380 unsigned long cur = iova + i * page_size; 1381 void *ent, *old; 1382 1383 if (!test_bit(i, (unsigned long *)tmp)) 1384 continue; 1385 1386 ent = xa_load(&mock->pfns, cur / page_size); 1387 if (ent) { 1388 unsigned long val; 1389 1390 val = xa_to_value(ent) | MOCK_PFN_DIRTY_IOVA; 1391 old = xa_store(&mock->pfns, cur / page_size, 1392 xa_mk_value(val), GFP_KERNEL); 1393 WARN_ON_ONCE(ent != old); 1394 count++; 1395 } 1396 } 1397 1398 cmd->dirty.out_nr_dirty = count; 1399 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1400 out_free: 1401 kvfree(tmp); 1402 out_put: 1403 iommufd_put_object(ucmd->ictx, &hwpt->obj); 1404 return rc; 1405 } 1406 1407 static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, 1408 struct iommu_test_cmd *cmd) 1409 { 1410 struct iopf_fault event = { }; 1411 struct iommufd_device *idev; 1412 1413 idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id); 1414 if (IS_ERR(idev)) 1415 return PTR_ERR(idev); 1416 1417 event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1418 if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID) 1419 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1420 event.fault.type = IOMMU_FAULT_PAGE_REQ; 1421 event.fault.prm.addr = cmd->trigger_iopf.addr; 1422 event.fault.prm.pasid = cmd->trigger_iopf.pasid; 1423 event.fault.prm.grpid = cmd->trigger_iopf.grpid; 1424 event.fault.prm.perm = cmd->trigger_iopf.perm; 1425 1426 iommu_report_device_fault(idev->dev, &event); 1427 iommufd_put_object(ucmd->ictx, &idev->obj); 1428 1429 return 0; 1430 } 1431 1432 void iommufd_selftest_destroy(struct iommufd_object *obj) 1433 { 1434 struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj); 1435 1436 switch (sobj->type) { 1437 case TYPE_IDEV: 1438 iommufd_device_detach(sobj->idev.idev); 1439 iommufd_device_unbind(sobj->idev.idev); 1440 mock_dev_destroy(sobj->idev.mock_dev); 1441 break; 1442 } 1443 } 1444 1445 int iommufd_test(struct iommufd_ucmd *ucmd) 1446 { 1447 struct iommu_test_cmd *cmd = ucmd->cmd; 1448 1449 switch (cmd->op) { 1450 case IOMMU_TEST_OP_ADD_RESERVED: 1451 return iommufd_test_add_reserved(ucmd, cmd->id, 1452 cmd->add_reserved.start, 1453 cmd->add_reserved.length); 1454 case IOMMU_TEST_OP_MOCK_DOMAIN: 1455 case IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS: 1456 return iommufd_test_mock_domain(ucmd, cmd); 1457 case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: 1458 return iommufd_test_mock_domain_replace( 1459 ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd); 1460 case IOMMU_TEST_OP_MD_CHECK_MAP: 1461 return iommufd_test_md_check_pa( 1462 ucmd, cmd->id, cmd->check_map.iova, 1463 cmd->check_map.length, 1464 u64_to_user_ptr(cmd->check_map.uptr)); 1465 case IOMMU_TEST_OP_MD_CHECK_REFS: 1466 return iommufd_test_md_check_refs( 1467 ucmd, u64_to_user_ptr(cmd->check_refs.uptr), 1468 cmd->check_refs.length, cmd->check_refs.refs); 1469 case IOMMU_TEST_OP_MD_CHECK_IOTLB: 1470 return iommufd_test_md_check_iotlb(ucmd, cmd->id, 1471 cmd->check_iotlb.id, 1472 cmd->check_iotlb.iotlb); 1473 case IOMMU_TEST_OP_CREATE_ACCESS: 1474 return iommufd_test_create_access(ucmd, cmd->id, 1475 cmd->create_access.flags); 1476 case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS: 1477 return iommufd_test_access_replace_ioas( 1478 ucmd, cmd->id, cmd->access_replace_ioas.ioas_id); 1479 case IOMMU_TEST_OP_ACCESS_PAGES: 1480 return iommufd_test_access_pages( 1481 ucmd, cmd->id, cmd->access_pages.iova, 1482 cmd->access_pages.length, 1483 u64_to_user_ptr(cmd->access_pages.uptr), 1484 cmd->access_pages.flags); 1485 case IOMMU_TEST_OP_ACCESS_RW: 1486 return iommufd_test_access_rw( 1487 ucmd, cmd->id, cmd->access_rw.iova, 1488 cmd->access_rw.length, 1489 u64_to_user_ptr(cmd->access_rw.uptr), 1490 cmd->access_rw.flags); 1491 case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: 1492 return iommufd_test_access_item_destroy( 1493 ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id); 1494 case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: 1495 /* Protect _batch_init(), can not be less than elmsz */ 1496 if (cmd->memory_limit.limit < 1497 sizeof(unsigned long) + sizeof(u32)) 1498 return -EINVAL; 1499 iommufd_test_memory_limit = cmd->memory_limit.limit; 1500 return 0; 1501 case IOMMU_TEST_OP_DIRTY: 1502 return iommufd_test_dirty(ucmd, cmd->id, cmd->dirty.iova, 1503 cmd->dirty.length, 1504 cmd->dirty.page_size, 1505 u64_to_user_ptr(cmd->dirty.uptr), 1506 cmd->dirty.flags); 1507 case IOMMU_TEST_OP_TRIGGER_IOPF: 1508 return iommufd_test_trigger_iopf(ucmd, cmd); 1509 default: 1510 return -EOPNOTSUPP; 1511 } 1512 } 1513 1514 bool iommufd_should_fail(void) 1515 { 1516 return should_fail(&fail_iommufd, 1); 1517 } 1518 1519 int __init iommufd_test_init(void) 1520 { 1521 struct platform_device_info pdevinfo = { 1522 .name = "iommufd_selftest_iommu", 1523 }; 1524 int rc; 1525 1526 dbgfs_root = 1527 fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); 1528 1529 selftest_iommu_dev = platform_device_register_full(&pdevinfo); 1530 if (IS_ERR(selftest_iommu_dev)) { 1531 rc = PTR_ERR(selftest_iommu_dev); 1532 goto err_dbgfs; 1533 } 1534 1535 rc = bus_register(&iommufd_mock_bus_type.bus); 1536 if (rc) 1537 goto err_platform; 1538 1539 rc = iommu_device_sysfs_add(&mock_iommu_device, 1540 &selftest_iommu_dev->dev, NULL, "%s", 1541 dev_name(&selftest_iommu_dev->dev)); 1542 if (rc) 1543 goto err_bus; 1544 1545 rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops, 1546 &iommufd_mock_bus_type.bus, 1547 &iommufd_mock_bus_type.nb); 1548 if (rc) 1549 goto err_sysfs; 1550 1551 mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq"); 1552 1553 return 0; 1554 1555 err_sysfs: 1556 iommu_device_sysfs_remove(&mock_iommu_device); 1557 err_bus: 1558 bus_unregister(&iommufd_mock_bus_type.bus); 1559 err_platform: 1560 platform_device_unregister(selftest_iommu_dev); 1561 err_dbgfs: 1562 debugfs_remove_recursive(dbgfs_root); 1563 return rc; 1564 } 1565 1566 void iommufd_test_exit(void) 1567 { 1568 if (mock_iommu_iopf_queue) { 1569 iopf_queue_free(mock_iommu_iopf_queue); 1570 mock_iommu_iopf_queue = NULL; 1571 } 1572 1573 iommu_device_sysfs_remove(&mock_iommu_device); 1574 iommu_device_unregister_bus(&mock_iommu_device, 1575 &iommufd_mock_bus_type.bus, 1576 &iommufd_mock_bus_type.nb); 1577 bus_unregister(&iommufd_mock_bus_type.bus); 1578 platform_device_unregister(selftest_iommu_dev); 1579 debugfs_remove_recursive(dbgfs_root); 1580 } 1581