1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. 3 * 4 * Kernel side components to support tools/testing/selftests/iommu 5 */ 6 #include <linux/anon_inodes.h> 7 #include <linux/debugfs.h> 8 #include <linux/fault-inject.h> 9 #include <linux/file.h> 10 #include <linux/iommu.h> 11 #include <linux/platform_device.h> 12 #include <linux/slab.h> 13 #include <linux/xarray.h> 14 #include <uapi/linux/iommufd.h> 15 16 #include "../iommu-priv.h" 17 #include "io_pagetable.h" 18 #include "iommufd_private.h" 19 #include "iommufd_test.h" 20 21 static DECLARE_FAULT_ATTR(fail_iommufd); 22 static struct dentry *dbgfs_root; 23 static struct platform_device *selftest_iommu_dev; 24 static const struct iommu_ops mock_ops; 25 static struct iommu_domain_ops domain_nested_ops; 26 27 size_t iommufd_test_memory_limit = 65536; 28 29 struct mock_bus_type { 30 struct bus_type bus; 31 struct notifier_block nb; 32 }; 33 34 static struct mock_bus_type iommufd_mock_bus_type = { 35 .bus = { 36 .name = "iommufd_mock", 37 }, 38 }; 39 40 static DEFINE_IDA(mock_dev_ida); 41 42 enum { 43 MOCK_DIRTY_TRACK = 1, 44 MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 45 MOCK_HUGE_PAGE_SIZE = 512 * MOCK_IO_PAGE_SIZE, 46 47 /* 48 * Like a real page table alignment requires the low bits of the address 49 * to be zero. xarray also requires the high bit to be zero, so we store 50 * the pfns shifted. The upper bits are used for metadata. 51 */ 52 MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 53 54 _MOCK_PFN_START = MOCK_PFN_MASK + 1, 55 MOCK_PFN_START_IOVA = _MOCK_PFN_START, 56 MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 57 MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1, 58 MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, 59 }; 60 61 /* 62 * Syzkaller has trouble randomizing the correct iova to use since it is linked 63 * to the map ioctl's output, and it has no ide about that. So, simplify things. 64 * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset 65 * value. This has a much smaller randomization space and syzkaller can hit it. 66 */ 67 static unsigned long __iommufd_test_syz_conv_iova(struct io_pagetable *iopt, 68 u64 *iova) 69 { 70 struct syz_layout { 71 __u32 nth_area; 72 __u32 offset; 73 }; 74 struct syz_layout *syz = (void *)iova; 75 unsigned int nth = syz->nth_area; 76 struct iopt_area *area; 77 78 down_read(&iopt->iova_rwsem); 79 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; 80 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { 81 if (nth == 0) { 82 up_read(&iopt->iova_rwsem); 83 return iopt_area_iova(area) + syz->offset; 84 } 85 nth--; 86 } 87 up_read(&iopt->iova_rwsem); 88 89 return 0; 90 } 91 92 static unsigned long iommufd_test_syz_conv_iova(struct iommufd_access *access, 93 u64 *iova) 94 { 95 unsigned long ret; 96 97 mutex_lock(&access->ioas_lock); 98 if (!access->ioas) { 99 mutex_unlock(&access->ioas_lock); 100 return 0; 101 } 102 ret = __iommufd_test_syz_conv_iova(&access->ioas->iopt, iova); 103 mutex_unlock(&access->ioas_lock); 104 return ret; 105 } 106 107 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 108 unsigned int ioas_id, u64 *iova, u32 *flags) 109 { 110 struct iommufd_ioas *ioas; 111 112 if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) 113 return; 114 *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; 115 116 ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); 117 if (IS_ERR(ioas)) 118 return; 119 *iova = __iommufd_test_syz_conv_iova(&ioas->iopt, iova); 120 iommufd_put_object(ucmd->ictx, &ioas->obj); 121 } 122 123 struct mock_iommu_domain { 124 unsigned long flags; 125 struct iommu_domain domain; 126 struct xarray pfns; 127 }; 128 129 static inline struct mock_iommu_domain * 130 to_mock_domain(struct iommu_domain *domain) 131 { 132 return container_of(domain, struct mock_iommu_domain, domain); 133 } 134 135 struct mock_iommu_domain_nested { 136 struct iommu_domain domain; 137 struct mock_viommu *mock_viommu; 138 struct mock_iommu_domain *parent; 139 u32 iotlb[MOCK_NESTED_DOMAIN_IOTLB_NUM]; 140 }; 141 142 static inline struct mock_iommu_domain_nested * 143 to_mock_nested(struct iommu_domain *domain) 144 { 145 return container_of(domain, struct mock_iommu_domain_nested, domain); 146 } 147 148 struct mock_viommu { 149 struct iommufd_viommu core; 150 struct mock_iommu_domain *s2_parent; 151 }; 152 153 static inline struct mock_viommu *to_mock_viommu(struct iommufd_viommu *viommu) 154 { 155 return container_of(viommu, struct mock_viommu, core); 156 } 157 158 enum selftest_obj_type { 159 TYPE_IDEV, 160 }; 161 162 struct mock_dev { 163 struct device dev; 164 unsigned long flags; 165 int id; 166 u32 cache[MOCK_DEV_CACHE_NUM]; 167 }; 168 169 static inline struct mock_dev *to_mock_dev(struct device *dev) 170 { 171 return container_of(dev, struct mock_dev, dev); 172 } 173 174 struct selftest_obj { 175 struct iommufd_object obj; 176 enum selftest_obj_type type; 177 178 union { 179 struct { 180 struct iommufd_device *idev; 181 struct iommufd_ctx *ictx; 182 struct mock_dev *mock_dev; 183 } idev; 184 }; 185 }; 186 187 static inline struct selftest_obj *to_selftest_obj(struct iommufd_object *obj) 188 { 189 return container_of(obj, struct selftest_obj, obj); 190 } 191 192 static int mock_domain_nop_attach(struct iommu_domain *domain, 193 struct device *dev) 194 { 195 struct mock_dev *mdev = to_mock_dev(dev); 196 197 if (domain->dirty_ops && (mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY)) 198 return -EINVAL; 199 200 return 0; 201 } 202 203 static const struct iommu_domain_ops mock_blocking_ops = { 204 .attach_dev = mock_domain_nop_attach, 205 }; 206 207 static struct iommu_domain mock_blocking_domain = { 208 .type = IOMMU_DOMAIN_BLOCKED, 209 .ops = &mock_blocking_ops, 210 }; 211 212 static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type) 213 { 214 struct iommu_test_hw_info *info; 215 216 info = kzalloc(sizeof(*info), GFP_KERNEL); 217 if (!info) 218 return ERR_PTR(-ENOMEM); 219 220 info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL; 221 *length = sizeof(*info); 222 *type = IOMMU_HW_INFO_TYPE_SELFTEST; 223 224 return info; 225 } 226 227 static int mock_domain_set_dirty_tracking(struct iommu_domain *domain, 228 bool enable) 229 { 230 struct mock_iommu_domain *mock = to_mock_domain(domain); 231 unsigned long flags = mock->flags; 232 233 if (enable && !domain->dirty_ops) 234 return -EINVAL; 235 236 /* No change? */ 237 if (!(enable ^ !!(flags & MOCK_DIRTY_TRACK))) 238 return 0; 239 240 flags = (enable ? flags | MOCK_DIRTY_TRACK : flags & ~MOCK_DIRTY_TRACK); 241 242 mock->flags = flags; 243 return 0; 244 } 245 246 static bool mock_test_and_clear_dirty(struct mock_iommu_domain *mock, 247 unsigned long iova, size_t page_size, 248 unsigned long flags) 249 { 250 unsigned long cur, end = iova + page_size - 1; 251 bool dirty = false; 252 void *ent, *old; 253 254 for (cur = iova; cur < end; cur += MOCK_IO_PAGE_SIZE) { 255 ent = xa_load(&mock->pfns, cur / MOCK_IO_PAGE_SIZE); 256 if (!ent || !(xa_to_value(ent) & MOCK_PFN_DIRTY_IOVA)) 257 continue; 258 259 dirty = true; 260 /* Clear dirty */ 261 if (!(flags & IOMMU_DIRTY_NO_CLEAR)) { 262 unsigned long val; 263 264 val = xa_to_value(ent) & ~MOCK_PFN_DIRTY_IOVA; 265 old = xa_store(&mock->pfns, cur / MOCK_IO_PAGE_SIZE, 266 xa_mk_value(val), GFP_KERNEL); 267 WARN_ON_ONCE(ent != old); 268 } 269 } 270 271 return dirty; 272 } 273 274 static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, 275 unsigned long iova, size_t size, 276 unsigned long flags, 277 struct iommu_dirty_bitmap *dirty) 278 { 279 struct mock_iommu_domain *mock = to_mock_domain(domain); 280 unsigned long end = iova + size; 281 void *ent; 282 283 if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap) 284 return -EINVAL; 285 286 do { 287 unsigned long pgsize = MOCK_IO_PAGE_SIZE; 288 unsigned long head; 289 290 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 291 if (!ent) { 292 iova += pgsize; 293 continue; 294 } 295 296 if (xa_to_value(ent) & MOCK_PFN_HUGE_IOVA) 297 pgsize = MOCK_HUGE_PAGE_SIZE; 298 head = iova & ~(pgsize - 1); 299 300 /* Clear dirty */ 301 if (mock_test_and_clear_dirty(mock, head, pgsize, flags)) 302 iommu_dirty_bitmap_record(dirty, iova, pgsize); 303 iova += pgsize; 304 } while (iova < end); 305 306 return 0; 307 } 308 309 static const struct iommu_dirty_ops dirty_ops = { 310 .set_dirty_tracking = mock_domain_set_dirty_tracking, 311 .read_and_clear_dirty = mock_domain_read_and_clear_dirty, 312 }; 313 314 static struct mock_iommu_domain_nested * 315 __mock_domain_alloc_nested(const struct iommu_user_data *user_data) 316 { 317 struct mock_iommu_domain_nested *mock_nested; 318 struct iommu_hwpt_selftest user_cfg; 319 int rc, i; 320 321 if (user_data->type != IOMMU_HWPT_DATA_SELFTEST) 322 return ERR_PTR(-EOPNOTSUPP); 323 324 rc = iommu_copy_struct_from_user(&user_cfg, user_data, 325 IOMMU_HWPT_DATA_SELFTEST, iotlb); 326 if (rc) 327 return ERR_PTR(rc); 328 329 mock_nested = kzalloc(sizeof(*mock_nested), GFP_KERNEL); 330 if (!mock_nested) 331 return ERR_PTR(-ENOMEM); 332 mock_nested->domain.ops = &domain_nested_ops; 333 mock_nested->domain.type = IOMMU_DOMAIN_NESTED; 334 for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++) 335 mock_nested->iotlb[i] = user_cfg.iotlb; 336 return mock_nested; 337 } 338 339 static struct iommu_domain * 340 mock_domain_alloc_nested(struct device *dev, struct iommu_domain *parent, 341 u32 flags, const struct iommu_user_data *user_data) 342 { 343 struct mock_iommu_domain_nested *mock_nested; 344 struct mock_iommu_domain *mock_parent; 345 346 if (flags) 347 return ERR_PTR(-EOPNOTSUPP); 348 if (!parent || parent->ops != mock_ops.default_domain_ops) 349 return ERR_PTR(-EINVAL); 350 351 mock_parent = to_mock_domain(parent); 352 if (!mock_parent) 353 return ERR_PTR(-EINVAL); 354 355 mock_nested = __mock_domain_alloc_nested(user_data); 356 if (IS_ERR(mock_nested)) 357 return ERR_CAST(mock_nested); 358 mock_nested->parent = mock_parent; 359 return &mock_nested->domain; 360 } 361 362 static struct iommu_domain * 363 mock_domain_alloc_paging_flags(struct device *dev, u32 flags, 364 const struct iommu_user_data *user_data) 365 { 366 bool has_dirty_flag = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 367 const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING | 368 IOMMU_HWPT_ALLOC_NEST_PARENT; 369 struct mock_dev *mdev = to_mock_dev(dev); 370 bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY; 371 struct mock_iommu_domain *mock; 372 373 if (user_data) 374 return ERR_PTR(-EOPNOTSUPP); 375 if ((flags & ~PAGING_FLAGS) || (has_dirty_flag && no_dirty_ops)) 376 return ERR_PTR(-EOPNOTSUPP); 377 378 mock = kzalloc(sizeof(*mock), GFP_KERNEL); 379 if (!mock) 380 return ERR_PTR(-ENOMEM); 381 mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 382 mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 383 mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 384 if (dev && mdev->flags & MOCK_FLAGS_DEVICE_HUGE_IOVA) 385 mock->domain.pgsize_bitmap |= MOCK_HUGE_PAGE_SIZE; 386 mock->domain.ops = mock_ops.default_domain_ops; 387 mock->domain.type = IOMMU_DOMAIN_UNMANAGED; 388 xa_init(&mock->pfns); 389 390 if (has_dirty_flag) 391 mock->domain.dirty_ops = &dirty_ops; 392 return &mock->domain; 393 } 394 395 static void mock_domain_free(struct iommu_domain *domain) 396 { 397 struct mock_iommu_domain *mock = to_mock_domain(domain); 398 399 WARN_ON(!xa_empty(&mock->pfns)); 400 kfree(mock); 401 } 402 403 static int mock_domain_map_pages(struct iommu_domain *domain, 404 unsigned long iova, phys_addr_t paddr, 405 size_t pgsize, size_t pgcount, int prot, 406 gfp_t gfp, size_t *mapped) 407 { 408 struct mock_iommu_domain *mock = to_mock_domain(domain); 409 unsigned long flags = MOCK_PFN_START_IOVA; 410 unsigned long start_iova = iova; 411 412 /* 413 * xarray does not reliably work with fault injection because it does a 414 * retry allocation, so put our own failure point. 415 */ 416 if (iommufd_should_fail()) 417 return -ENOENT; 418 419 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 420 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 421 for (; pgcount; pgcount--) { 422 size_t cur; 423 424 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 425 void *old; 426 427 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 428 flags = MOCK_PFN_LAST_IOVA; 429 if (pgsize != MOCK_IO_PAGE_SIZE) { 430 flags |= MOCK_PFN_HUGE_IOVA; 431 } 432 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 433 xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 434 flags), 435 gfp); 436 if (xa_is_err(old)) { 437 for (; start_iova != iova; 438 start_iova += MOCK_IO_PAGE_SIZE) 439 xa_erase(&mock->pfns, 440 start_iova / 441 MOCK_IO_PAGE_SIZE); 442 return xa_err(old); 443 } 444 WARN_ON(old); 445 iova += MOCK_IO_PAGE_SIZE; 446 paddr += MOCK_IO_PAGE_SIZE; 447 *mapped += MOCK_IO_PAGE_SIZE; 448 flags = 0; 449 } 450 } 451 return 0; 452 } 453 454 static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 455 unsigned long iova, size_t pgsize, 456 size_t pgcount, 457 struct iommu_iotlb_gather *iotlb_gather) 458 { 459 struct mock_iommu_domain *mock = to_mock_domain(domain); 460 bool first = true; 461 size_t ret = 0; 462 void *ent; 463 464 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 465 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 466 467 for (; pgcount; pgcount--) { 468 size_t cur; 469 470 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 471 ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 472 473 /* 474 * iommufd generates unmaps that must be a strict 475 * superset of the map's performend So every 476 * starting/ending IOVA should have been an iova passed 477 * to map. 478 * 479 * This simple logic doesn't work when the HUGE_PAGE is 480 * turned on since the core code will automatically 481 * switch between the two page sizes creating a break in 482 * the unmap calls. The break can land in the middle of 483 * contiguous IOVA. 484 */ 485 if (!(domain->pgsize_bitmap & MOCK_HUGE_PAGE_SIZE)) { 486 if (first) { 487 WARN_ON(ent && !(xa_to_value(ent) & 488 MOCK_PFN_START_IOVA)); 489 first = false; 490 } 491 if (pgcount == 1 && 492 cur + MOCK_IO_PAGE_SIZE == pgsize) 493 WARN_ON(ent && !(xa_to_value(ent) & 494 MOCK_PFN_LAST_IOVA)); 495 } 496 497 iova += MOCK_IO_PAGE_SIZE; 498 ret += MOCK_IO_PAGE_SIZE; 499 } 500 } 501 return ret; 502 } 503 504 static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 505 dma_addr_t iova) 506 { 507 struct mock_iommu_domain *mock = to_mock_domain(domain); 508 void *ent; 509 510 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 511 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 512 WARN_ON(!ent); 513 return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 514 } 515 516 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) 517 { 518 struct mock_dev *mdev = to_mock_dev(dev); 519 520 switch (cap) { 521 case IOMMU_CAP_CACHE_COHERENCY: 522 return true; 523 case IOMMU_CAP_DIRTY_TRACKING: 524 return !(mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY); 525 default: 526 break; 527 } 528 529 return false; 530 } 531 532 static struct iopf_queue *mock_iommu_iopf_queue; 533 534 static struct mock_iommu_device { 535 struct iommu_device iommu_dev; 536 struct completion complete; 537 refcount_t users; 538 } mock_iommu; 539 540 static struct iommu_device *mock_probe_device(struct device *dev) 541 { 542 if (dev->bus != &iommufd_mock_bus_type.bus) 543 return ERR_PTR(-ENODEV); 544 return &mock_iommu.iommu_dev; 545 } 546 547 static void mock_domain_page_response(struct device *dev, struct iopf_fault *evt, 548 struct iommu_page_response *msg) 549 { 550 } 551 552 static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) 553 { 554 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 555 return -ENODEV; 556 557 return iopf_queue_add_device(mock_iommu_iopf_queue, dev); 558 } 559 560 static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 561 { 562 if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 563 return -ENODEV; 564 565 iopf_queue_remove_device(mock_iommu_iopf_queue, dev); 566 567 return 0; 568 } 569 570 static void mock_viommu_destroy(struct iommufd_viommu *viommu) 571 { 572 struct mock_iommu_device *mock_iommu = container_of( 573 viommu->iommu_dev, struct mock_iommu_device, iommu_dev); 574 575 if (refcount_dec_and_test(&mock_iommu->users)) 576 complete(&mock_iommu->complete); 577 578 /* iommufd core frees mock_viommu and viommu */ 579 } 580 581 static struct iommu_domain * 582 mock_viommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags, 583 const struct iommu_user_data *user_data) 584 { 585 struct mock_viommu *mock_viommu = to_mock_viommu(viommu); 586 struct mock_iommu_domain_nested *mock_nested; 587 588 if (flags) 589 return ERR_PTR(-EOPNOTSUPP); 590 591 mock_nested = __mock_domain_alloc_nested(user_data); 592 if (IS_ERR(mock_nested)) 593 return ERR_CAST(mock_nested); 594 mock_nested->mock_viommu = mock_viommu; 595 mock_nested->parent = mock_viommu->s2_parent; 596 return &mock_nested->domain; 597 } 598 599 static int mock_viommu_cache_invalidate(struct iommufd_viommu *viommu, 600 struct iommu_user_data_array *array) 601 { 602 struct iommu_viommu_invalidate_selftest *cmds; 603 struct iommu_viommu_invalidate_selftest *cur; 604 struct iommu_viommu_invalidate_selftest *end; 605 int rc; 606 607 /* A zero-length array is allowed to validate the array type */ 608 if (array->entry_num == 0 && 609 array->type == IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST) { 610 array->entry_num = 0; 611 return 0; 612 } 613 614 cmds = kcalloc(array->entry_num, sizeof(*cmds), GFP_KERNEL); 615 if (!cmds) 616 return -ENOMEM; 617 cur = cmds; 618 end = cmds + array->entry_num; 619 620 static_assert(sizeof(*cmds) == 3 * sizeof(u32)); 621 rc = iommu_copy_struct_from_full_user_array( 622 cmds, sizeof(*cmds), array, 623 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST); 624 if (rc) 625 goto out; 626 627 while (cur != end) { 628 struct mock_dev *mdev; 629 struct device *dev; 630 int i; 631 632 if (cur->flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) { 633 rc = -EOPNOTSUPP; 634 goto out; 635 } 636 637 if (cur->cache_id > MOCK_DEV_CACHE_ID_MAX) { 638 rc = -EINVAL; 639 goto out; 640 } 641 642 xa_lock(&viommu->vdevs); 643 dev = iommufd_viommu_find_dev(viommu, 644 (unsigned long)cur->vdev_id); 645 if (!dev) { 646 xa_unlock(&viommu->vdevs); 647 rc = -EINVAL; 648 goto out; 649 } 650 mdev = container_of(dev, struct mock_dev, dev); 651 652 if (cur->flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) { 653 /* Invalidate all cache entries and ignore cache_id */ 654 for (i = 0; i < MOCK_DEV_CACHE_NUM; i++) 655 mdev->cache[i] = 0; 656 } else { 657 mdev->cache[cur->cache_id] = 0; 658 } 659 xa_unlock(&viommu->vdevs); 660 661 cur++; 662 } 663 out: 664 array->entry_num = cur - cmds; 665 kfree(cmds); 666 return rc; 667 } 668 669 static struct iommufd_viommu_ops mock_viommu_ops = { 670 .destroy = mock_viommu_destroy, 671 .alloc_domain_nested = mock_viommu_alloc_domain_nested, 672 .cache_invalidate = mock_viommu_cache_invalidate, 673 }; 674 675 static struct iommufd_viommu *mock_viommu_alloc(struct device *dev, 676 struct iommu_domain *domain, 677 struct iommufd_ctx *ictx, 678 unsigned int viommu_type) 679 { 680 struct mock_iommu_device *mock_iommu = 681 iommu_get_iommu_dev(dev, struct mock_iommu_device, iommu_dev); 682 struct mock_viommu *mock_viommu; 683 684 if (viommu_type != IOMMU_VIOMMU_TYPE_SELFTEST) 685 return ERR_PTR(-EOPNOTSUPP); 686 687 mock_viommu = iommufd_viommu_alloc(ictx, struct mock_viommu, core, 688 &mock_viommu_ops); 689 if (IS_ERR(mock_viommu)) 690 return ERR_CAST(mock_viommu); 691 692 refcount_inc(&mock_iommu->users); 693 return &mock_viommu->core; 694 } 695 696 static const struct iommu_ops mock_ops = { 697 /* 698 * IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type() 699 * because it is zero. 700 */ 701 .default_domain = &mock_blocking_domain, 702 .blocked_domain = &mock_blocking_domain, 703 .owner = THIS_MODULE, 704 .pgsize_bitmap = MOCK_IO_PAGE_SIZE, 705 .hw_info = mock_domain_hw_info, 706 .domain_alloc_paging_flags = mock_domain_alloc_paging_flags, 707 .domain_alloc_nested = mock_domain_alloc_nested, 708 .capable = mock_domain_capable, 709 .device_group = generic_device_group, 710 .probe_device = mock_probe_device, 711 .page_response = mock_domain_page_response, 712 .dev_enable_feat = mock_dev_enable_feat, 713 .dev_disable_feat = mock_dev_disable_feat, 714 .user_pasid_table = true, 715 .viommu_alloc = mock_viommu_alloc, 716 .default_domain_ops = 717 &(struct iommu_domain_ops){ 718 .free = mock_domain_free, 719 .attach_dev = mock_domain_nop_attach, 720 .map_pages = mock_domain_map_pages, 721 .unmap_pages = mock_domain_unmap_pages, 722 .iova_to_phys = mock_domain_iova_to_phys, 723 }, 724 }; 725 726 static void mock_domain_free_nested(struct iommu_domain *domain) 727 { 728 kfree(to_mock_nested(domain)); 729 } 730 731 static int 732 mock_domain_cache_invalidate_user(struct iommu_domain *domain, 733 struct iommu_user_data_array *array) 734 { 735 struct mock_iommu_domain_nested *mock_nested = to_mock_nested(domain); 736 struct iommu_hwpt_invalidate_selftest inv; 737 u32 processed = 0; 738 int i = 0, j; 739 int rc = 0; 740 741 if (array->type != IOMMU_HWPT_INVALIDATE_DATA_SELFTEST) { 742 rc = -EINVAL; 743 goto out; 744 } 745 746 for ( ; i < array->entry_num; i++) { 747 rc = iommu_copy_struct_from_user_array(&inv, array, 748 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 749 i, iotlb_id); 750 if (rc) 751 break; 752 753 if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) { 754 rc = -EOPNOTSUPP; 755 break; 756 } 757 758 if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) { 759 rc = -EINVAL; 760 break; 761 } 762 763 if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) { 764 /* Invalidate all mock iotlb entries and ignore iotlb_id */ 765 for (j = 0; j < MOCK_NESTED_DOMAIN_IOTLB_NUM; j++) 766 mock_nested->iotlb[j] = 0; 767 } else { 768 mock_nested->iotlb[inv.iotlb_id] = 0; 769 } 770 771 processed++; 772 } 773 774 out: 775 array->entry_num = processed; 776 return rc; 777 } 778 779 static struct iommu_domain_ops domain_nested_ops = { 780 .free = mock_domain_free_nested, 781 .attach_dev = mock_domain_nop_attach, 782 .cache_invalidate_user = mock_domain_cache_invalidate_user, 783 }; 784 785 static inline struct iommufd_hw_pagetable * 786 __get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, u32 hwpt_type) 787 { 788 struct iommufd_object *obj; 789 790 obj = iommufd_get_object(ucmd->ictx, mockpt_id, hwpt_type); 791 if (IS_ERR(obj)) 792 return ERR_CAST(obj); 793 return container_of(obj, struct iommufd_hw_pagetable, obj); 794 } 795 796 static inline struct iommufd_hw_pagetable * 797 get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, 798 struct mock_iommu_domain **mock) 799 { 800 struct iommufd_hw_pagetable *hwpt; 801 802 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_PAGING); 803 if (IS_ERR(hwpt)) 804 return hwpt; 805 if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED || 806 hwpt->domain->ops != mock_ops.default_domain_ops) { 807 iommufd_put_object(ucmd->ictx, &hwpt->obj); 808 return ERR_PTR(-EINVAL); 809 } 810 *mock = to_mock_domain(hwpt->domain); 811 return hwpt; 812 } 813 814 static inline struct iommufd_hw_pagetable * 815 get_md_pagetable_nested(struct iommufd_ucmd *ucmd, u32 mockpt_id, 816 struct mock_iommu_domain_nested **mock_nested) 817 { 818 struct iommufd_hw_pagetable *hwpt; 819 820 hwpt = __get_md_pagetable(ucmd, mockpt_id, IOMMUFD_OBJ_HWPT_NESTED); 821 if (IS_ERR(hwpt)) 822 return hwpt; 823 if (hwpt->domain->type != IOMMU_DOMAIN_NESTED || 824 hwpt->domain->ops != &domain_nested_ops) { 825 iommufd_put_object(ucmd->ictx, &hwpt->obj); 826 return ERR_PTR(-EINVAL); 827 } 828 *mock_nested = to_mock_nested(hwpt->domain); 829 return hwpt; 830 } 831 832 static void mock_dev_release(struct device *dev) 833 { 834 struct mock_dev *mdev = to_mock_dev(dev); 835 836 ida_free(&mock_dev_ida, mdev->id); 837 kfree(mdev); 838 } 839 840 static struct mock_dev *mock_dev_create(unsigned long dev_flags) 841 { 842 struct mock_dev *mdev; 843 int rc, i; 844 845 if (dev_flags & 846 ~(MOCK_FLAGS_DEVICE_NO_DIRTY | MOCK_FLAGS_DEVICE_HUGE_IOVA)) 847 return ERR_PTR(-EINVAL); 848 849 mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); 850 if (!mdev) 851 return ERR_PTR(-ENOMEM); 852 853 device_initialize(&mdev->dev); 854 mdev->flags = dev_flags; 855 mdev->dev.release = mock_dev_release; 856 mdev->dev.bus = &iommufd_mock_bus_type.bus; 857 for (i = 0; i < MOCK_DEV_CACHE_NUM; i++) 858 mdev->cache[i] = IOMMU_TEST_DEV_CACHE_DEFAULT; 859 860 rc = ida_alloc(&mock_dev_ida, GFP_KERNEL); 861 if (rc < 0) 862 goto err_put; 863 mdev->id = rc; 864 865 rc = dev_set_name(&mdev->dev, "iommufd_mock%u", mdev->id); 866 if (rc) 867 goto err_put; 868 869 rc = device_add(&mdev->dev); 870 if (rc) 871 goto err_put; 872 return mdev; 873 874 err_put: 875 put_device(&mdev->dev); 876 return ERR_PTR(rc); 877 } 878 879 static void mock_dev_destroy(struct mock_dev *mdev) 880 { 881 device_unregister(&mdev->dev); 882 } 883 884 bool iommufd_selftest_is_mock_dev(struct device *dev) 885 { 886 return dev->release == mock_dev_release; 887 } 888 889 /* Create an hw_pagetable with the mock domain so we can test the domain ops */ 890 static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, 891 struct iommu_test_cmd *cmd) 892 { 893 struct iommufd_device *idev; 894 struct selftest_obj *sobj; 895 u32 pt_id = cmd->id; 896 u32 dev_flags = 0; 897 u32 idev_id; 898 int rc; 899 900 sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); 901 if (IS_ERR(sobj)) 902 return PTR_ERR(sobj); 903 904 sobj->idev.ictx = ucmd->ictx; 905 sobj->type = TYPE_IDEV; 906 907 if (cmd->op == IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS) 908 dev_flags = cmd->mock_domain_flags.dev_flags; 909 910 sobj->idev.mock_dev = mock_dev_create(dev_flags); 911 if (IS_ERR(sobj->idev.mock_dev)) { 912 rc = PTR_ERR(sobj->idev.mock_dev); 913 goto out_sobj; 914 } 915 916 idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, 917 &idev_id); 918 if (IS_ERR(idev)) { 919 rc = PTR_ERR(idev); 920 goto out_mdev; 921 } 922 sobj->idev.idev = idev; 923 924 rc = iommufd_device_attach(idev, &pt_id); 925 if (rc) 926 goto out_unbind; 927 928 /* Userspace must destroy the device_id to destroy the object */ 929 cmd->mock_domain.out_hwpt_id = pt_id; 930 cmd->mock_domain.out_stdev_id = sobj->obj.id; 931 cmd->mock_domain.out_idev_id = idev_id; 932 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 933 if (rc) 934 goto out_detach; 935 iommufd_object_finalize(ucmd->ictx, &sobj->obj); 936 return 0; 937 938 out_detach: 939 iommufd_device_detach(idev); 940 out_unbind: 941 iommufd_device_unbind(idev); 942 out_mdev: 943 mock_dev_destroy(sobj->idev.mock_dev); 944 out_sobj: 945 iommufd_object_abort(ucmd->ictx, &sobj->obj); 946 return rc; 947 } 948 949 /* Replace the mock domain with a manually allocated hw_pagetable */ 950 static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, 951 unsigned int device_id, u32 pt_id, 952 struct iommu_test_cmd *cmd) 953 { 954 struct iommufd_object *dev_obj; 955 struct selftest_obj *sobj; 956 int rc; 957 958 /* 959 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure 960 * it doesn't race with detach, which is not allowed. 961 */ 962 dev_obj = 963 iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST); 964 if (IS_ERR(dev_obj)) 965 return PTR_ERR(dev_obj); 966 967 sobj = to_selftest_obj(dev_obj); 968 if (sobj->type != TYPE_IDEV) { 969 rc = -EINVAL; 970 goto out_dev_obj; 971 } 972 973 rc = iommufd_device_replace(sobj->idev.idev, &pt_id); 974 if (rc) 975 goto out_dev_obj; 976 977 cmd->mock_domain_replace.pt_id = pt_id; 978 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 979 980 out_dev_obj: 981 iommufd_put_object(ucmd->ictx, dev_obj); 982 return rc; 983 } 984 985 /* Add an additional reserved IOVA to the IOAS */ 986 static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, 987 unsigned int mockpt_id, 988 unsigned long start, size_t length) 989 { 990 struct iommufd_ioas *ioas; 991 int rc; 992 993 ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); 994 if (IS_ERR(ioas)) 995 return PTR_ERR(ioas); 996 down_write(&ioas->iopt.iova_rwsem); 997 rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); 998 up_write(&ioas->iopt.iova_rwsem); 999 iommufd_put_object(ucmd->ictx, &ioas->obj); 1000 return rc; 1001 } 1002 1003 /* Check that every pfn under each iova matches the pfn under a user VA */ 1004 static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, 1005 unsigned int mockpt_id, unsigned long iova, 1006 size_t length, void __user *uptr) 1007 { 1008 struct iommufd_hw_pagetable *hwpt; 1009 struct mock_iommu_domain *mock; 1010 uintptr_t end; 1011 int rc; 1012 1013 if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 1014 (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 1015 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 1016 return -EINVAL; 1017 1018 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 1019 if (IS_ERR(hwpt)) 1020 return PTR_ERR(hwpt); 1021 1022 for (; length; length -= MOCK_IO_PAGE_SIZE) { 1023 struct page *pages[1]; 1024 unsigned long pfn; 1025 long npages; 1026 void *ent; 1027 1028 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 1029 pages); 1030 if (npages < 0) { 1031 rc = npages; 1032 goto out_put; 1033 } 1034 if (WARN_ON(npages != 1)) { 1035 rc = -EFAULT; 1036 goto out_put; 1037 } 1038 pfn = page_to_pfn(pages[0]); 1039 put_page(pages[0]); 1040 1041 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 1042 if (!ent || 1043 (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 1044 pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 1045 rc = -EINVAL; 1046 goto out_put; 1047 } 1048 iova += MOCK_IO_PAGE_SIZE; 1049 uptr += MOCK_IO_PAGE_SIZE; 1050 } 1051 rc = 0; 1052 1053 out_put: 1054 iommufd_put_object(ucmd->ictx, &hwpt->obj); 1055 return rc; 1056 } 1057 1058 /* Check that the page ref count matches, to look for missing pin/unpins */ 1059 static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, 1060 void __user *uptr, size_t length, 1061 unsigned int refs) 1062 { 1063 uintptr_t end; 1064 1065 if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || 1066 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 1067 return -EINVAL; 1068 1069 for (; length; length -= PAGE_SIZE) { 1070 struct page *pages[1]; 1071 long npages; 1072 1073 npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages); 1074 if (npages < 0) 1075 return npages; 1076 if (WARN_ON(npages != 1)) 1077 return -EFAULT; 1078 if (!PageCompound(pages[0])) { 1079 unsigned int count; 1080 1081 count = page_ref_count(pages[0]); 1082 if (count / GUP_PIN_COUNTING_BIAS != refs) { 1083 put_page(pages[0]); 1084 return -EIO; 1085 } 1086 } 1087 put_page(pages[0]); 1088 uptr += PAGE_SIZE; 1089 } 1090 return 0; 1091 } 1092 1093 static int iommufd_test_md_check_iotlb(struct iommufd_ucmd *ucmd, 1094 u32 mockpt_id, unsigned int iotlb_id, 1095 u32 iotlb) 1096 { 1097 struct mock_iommu_domain_nested *mock_nested; 1098 struct iommufd_hw_pagetable *hwpt; 1099 int rc = 0; 1100 1101 hwpt = get_md_pagetable_nested(ucmd, mockpt_id, &mock_nested); 1102 if (IS_ERR(hwpt)) 1103 return PTR_ERR(hwpt); 1104 1105 mock_nested = to_mock_nested(hwpt->domain); 1106 1107 if (iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX || 1108 mock_nested->iotlb[iotlb_id] != iotlb) 1109 rc = -EINVAL; 1110 iommufd_put_object(ucmd->ictx, &hwpt->obj); 1111 return rc; 1112 } 1113 1114 static int iommufd_test_dev_check_cache(struct iommufd_ucmd *ucmd, u32 idev_id, 1115 unsigned int cache_id, u32 cache) 1116 { 1117 struct iommufd_device *idev; 1118 struct mock_dev *mdev; 1119 int rc = 0; 1120 1121 idev = iommufd_get_device(ucmd, idev_id); 1122 if (IS_ERR(idev)) 1123 return PTR_ERR(idev); 1124 mdev = container_of(idev->dev, struct mock_dev, dev); 1125 1126 if (cache_id > MOCK_DEV_CACHE_ID_MAX || mdev->cache[cache_id] != cache) 1127 rc = -EINVAL; 1128 iommufd_put_object(ucmd->ictx, &idev->obj); 1129 return rc; 1130 } 1131 1132 struct selftest_access { 1133 struct iommufd_access *access; 1134 struct file *file; 1135 struct mutex lock; 1136 struct list_head items; 1137 unsigned int next_id; 1138 bool destroying; 1139 }; 1140 1141 struct selftest_access_item { 1142 struct list_head items_elm; 1143 unsigned long iova; 1144 size_t length; 1145 unsigned int id; 1146 }; 1147 1148 static const struct file_operations iommfd_test_staccess_fops; 1149 1150 static struct selftest_access *iommufd_access_get(int fd) 1151 { 1152 struct file *file; 1153 1154 file = fget(fd); 1155 if (!file) 1156 return ERR_PTR(-EBADFD); 1157 1158 if (file->f_op != &iommfd_test_staccess_fops) { 1159 fput(file); 1160 return ERR_PTR(-EBADFD); 1161 } 1162 return file->private_data; 1163 } 1164 1165 static void iommufd_test_access_unmap(void *data, unsigned long iova, 1166 unsigned long length) 1167 { 1168 unsigned long iova_last = iova + length - 1; 1169 struct selftest_access *staccess = data; 1170 struct selftest_access_item *item; 1171 struct selftest_access_item *tmp; 1172 1173 mutex_lock(&staccess->lock); 1174 list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { 1175 if (iova > item->iova + item->length - 1 || 1176 iova_last < item->iova) 1177 continue; 1178 list_del(&item->items_elm); 1179 iommufd_access_unpin_pages(staccess->access, item->iova, 1180 item->length); 1181 kfree(item); 1182 } 1183 mutex_unlock(&staccess->lock); 1184 } 1185 1186 static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, 1187 unsigned int access_id, 1188 unsigned int item_id) 1189 { 1190 struct selftest_access_item *item; 1191 struct selftest_access *staccess; 1192 1193 staccess = iommufd_access_get(access_id); 1194 if (IS_ERR(staccess)) 1195 return PTR_ERR(staccess); 1196 1197 mutex_lock(&staccess->lock); 1198 list_for_each_entry(item, &staccess->items, items_elm) { 1199 if (item->id == item_id) { 1200 list_del(&item->items_elm); 1201 iommufd_access_unpin_pages(staccess->access, item->iova, 1202 item->length); 1203 mutex_unlock(&staccess->lock); 1204 kfree(item); 1205 fput(staccess->file); 1206 return 0; 1207 } 1208 } 1209 mutex_unlock(&staccess->lock); 1210 fput(staccess->file); 1211 return -ENOENT; 1212 } 1213 1214 static int iommufd_test_staccess_release(struct inode *inode, 1215 struct file *filep) 1216 { 1217 struct selftest_access *staccess = filep->private_data; 1218 1219 if (staccess->access) { 1220 iommufd_test_access_unmap(staccess, 0, ULONG_MAX); 1221 iommufd_access_destroy(staccess->access); 1222 } 1223 mutex_destroy(&staccess->lock); 1224 kfree(staccess); 1225 return 0; 1226 } 1227 1228 static const struct iommufd_access_ops selftest_access_ops_pin = { 1229 .needs_pin_pages = 1, 1230 .unmap = iommufd_test_access_unmap, 1231 }; 1232 1233 static const struct iommufd_access_ops selftest_access_ops = { 1234 .unmap = iommufd_test_access_unmap, 1235 }; 1236 1237 static const struct file_operations iommfd_test_staccess_fops = { 1238 .release = iommufd_test_staccess_release, 1239 }; 1240 1241 static struct selftest_access *iommufd_test_alloc_access(void) 1242 { 1243 struct selftest_access *staccess; 1244 struct file *filep; 1245 1246 staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT); 1247 if (!staccess) 1248 return ERR_PTR(-ENOMEM); 1249 INIT_LIST_HEAD(&staccess->items); 1250 mutex_init(&staccess->lock); 1251 1252 filep = anon_inode_getfile("[iommufd_test_staccess]", 1253 &iommfd_test_staccess_fops, staccess, 1254 O_RDWR); 1255 if (IS_ERR(filep)) { 1256 kfree(staccess); 1257 return ERR_CAST(filep); 1258 } 1259 staccess->file = filep; 1260 return staccess; 1261 } 1262 1263 static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, 1264 unsigned int ioas_id, unsigned int flags) 1265 { 1266 struct iommu_test_cmd *cmd = ucmd->cmd; 1267 struct selftest_access *staccess; 1268 struct iommufd_access *access; 1269 u32 id; 1270 int fdno; 1271 int rc; 1272 1273 if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) 1274 return -EOPNOTSUPP; 1275 1276 staccess = iommufd_test_alloc_access(); 1277 if (IS_ERR(staccess)) 1278 return PTR_ERR(staccess); 1279 1280 fdno = get_unused_fd_flags(O_CLOEXEC); 1281 if (fdno < 0) { 1282 rc = -ENOMEM; 1283 goto out_free_staccess; 1284 } 1285 1286 access = iommufd_access_create( 1287 ucmd->ictx, 1288 (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? 1289 &selftest_access_ops_pin : 1290 &selftest_access_ops, 1291 staccess, &id); 1292 if (IS_ERR(access)) { 1293 rc = PTR_ERR(access); 1294 goto out_put_fdno; 1295 } 1296 rc = iommufd_access_attach(access, ioas_id); 1297 if (rc) 1298 goto out_destroy; 1299 cmd->create_access.out_access_fd = fdno; 1300 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1301 if (rc) 1302 goto out_destroy; 1303 1304 staccess->access = access; 1305 fd_install(fdno, staccess->file); 1306 return 0; 1307 1308 out_destroy: 1309 iommufd_access_destroy(access); 1310 out_put_fdno: 1311 put_unused_fd(fdno); 1312 out_free_staccess: 1313 fput(staccess->file); 1314 return rc; 1315 } 1316 1317 static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, 1318 unsigned int access_id, 1319 unsigned int ioas_id) 1320 { 1321 struct selftest_access *staccess; 1322 int rc; 1323 1324 staccess = iommufd_access_get(access_id); 1325 if (IS_ERR(staccess)) 1326 return PTR_ERR(staccess); 1327 1328 rc = iommufd_access_replace(staccess->access, ioas_id); 1329 fput(staccess->file); 1330 return rc; 1331 } 1332 1333 /* Check that the pages in a page array match the pages in the user VA */ 1334 static int iommufd_test_check_pages(void __user *uptr, struct page **pages, 1335 size_t npages) 1336 { 1337 for (; npages; npages--) { 1338 struct page *tmp_pages[1]; 1339 long rc; 1340 1341 rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages); 1342 if (rc < 0) 1343 return rc; 1344 if (WARN_ON(rc != 1)) 1345 return -EFAULT; 1346 put_page(tmp_pages[0]); 1347 if (tmp_pages[0] != *pages) 1348 return -EBADE; 1349 pages++; 1350 uptr += PAGE_SIZE; 1351 } 1352 return 0; 1353 } 1354 1355 static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, 1356 unsigned int access_id, unsigned long iova, 1357 size_t length, void __user *uptr, 1358 u32 flags) 1359 { 1360 struct iommu_test_cmd *cmd = ucmd->cmd; 1361 struct selftest_access_item *item; 1362 struct selftest_access *staccess; 1363 struct page **pages; 1364 size_t npages; 1365 int rc; 1366 1367 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1368 if (length > 16*1024*1024) 1369 return -ENOMEM; 1370 1371 if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) 1372 return -EOPNOTSUPP; 1373 1374 staccess = iommufd_access_get(access_id); 1375 if (IS_ERR(staccess)) 1376 return PTR_ERR(staccess); 1377 1378 if (staccess->access->ops != &selftest_access_ops_pin) { 1379 rc = -EOPNOTSUPP; 1380 goto out_put; 1381 } 1382 1383 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1384 iova = iommufd_test_syz_conv_iova(staccess->access, 1385 &cmd->access_pages.iova); 1386 1387 npages = (ALIGN(iova + length, PAGE_SIZE) - 1388 ALIGN_DOWN(iova, PAGE_SIZE)) / 1389 PAGE_SIZE; 1390 pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT); 1391 if (!pages) { 1392 rc = -ENOMEM; 1393 goto out_put; 1394 } 1395 1396 /* 1397 * Drivers will need to think very carefully about this locking. The 1398 * core code can do multiple unmaps instantaneously after 1399 * iommufd_access_pin_pages() and *all* the unmaps must not return until 1400 * the range is unpinned. This simple implementation puts a global lock 1401 * around the pin, which may not suit drivers that want this to be a 1402 * performance path. drivers that get this wrong will trigger WARN_ON 1403 * races and cause EDEADLOCK failures to userspace. 1404 */ 1405 mutex_lock(&staccess->lock); 1406 rc = iommufd_access_pin_pages(staccess->access, iova, length, pages, 1407 flags & MOCK_FLAGS_ACCESS_WRITE); 1408 if (rc) 1409 goto out_unlock; 1410 1411 /* For syzkaller allow uptr to be NULL to skip this check */ 1412 if (uptr) { 1413 rc = iommufd_test_check_pages( 1414 uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, 1415 npages); 1416 if (rc) 1417 goto out_unaccess; 1418 } 1419 1420 item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT); 1421 if (!item) { 1422 rc = -ENOMEM; 1423 goto out_unaccess; 1424 } 1425 1426 item->iova = iova; 1427 item->length = length; 1428 item->id = staccess->next_id++; 1429 list_add_tail(&item->items_elm, &staccess->items); 1430 1431 cmd->access_pages.out_access_pages_id = item->id; 1432 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1433 if (rc) 1434 goto out_free_item; 1435 goto out_unlock; 1436 1437 out_free_item: 1438 list_del(&item->items_elm); 1439 kfree(item); 1440 out_unaccess: 1441 iommufd_access_unpin_pages(staccess->access, iova, length); 1442 out_unlock: 1443 mutex_unlock(&staccess->lock); 1444 kvfree(pages); 1445 out_put: 1446 fput(staccess->file); 1447 return rc; 1448 } 1449 1450 static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, 1451 unsigned int access_id, unsigned long iova, 1452 size_t length, void __user *ubuf, 1453 unsigned int flags) 1454 { 1455 struct iommu_test_cmd *cmd = ucmd->cmd; 1456 struct selftest_access *staccess; 1457 void *tmp; 1458 int rc; 1459 1460 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 1461 if (length > 16*1024*1024) 1462 return -ENOMEM; 1463 1464 if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | 1465 MOCK_FLAGS_ACCESS_SYZ)) 1466 return -EOPNOTSUPP; 1467 1468 staccess = iommufd_access_get(access_id); 1469 if (IS_ERR(staccess)) 1470 return PTR_ERR(staccess); 1471 1472 tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT); 1473 if (!tmp) { 1474 rc = -ENOMEM; 1475 goto out_put; 1476 } 1477 1478 if (flags & MOCK_ACCESS_RW_WRITE) { 1479 if (copy_from_user(tmp, ubuf, length)) { 1480 rc = -EFAULT; 1481 goto out_free; 1482 } 1483 } 1484 1485 if (flags & MOCK_FLAGS_ACCESS_SYZ) 1486 iova = iommufd_test_syz_conv_iova(staccess->access, 1487 &cmd->access_rw.iova); 1488 1489 rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags); 1490 if (rc) 1491 goto out_free; 1492 if (!(flags & MOCK_ACCESS_RW_WRITE)) { 1493 if (copy_to_user(ubuf, tmp, length)) { 1494 rc = -EFAULT; 1495 goto out_free; 1496 } 1497 } 1498 1499 out_free: 1500 kvfree(tmp); 1501 out_put: 1502 fput(staccess->file); 1503 return rc; 1504 } 1505 static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); 1506 static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == 1507 __IOMMUFD_ACCESS_RW_SLOW_PATH); 1508 1509 static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, 1510 unsigned long iova, size_t length, 1511 unsigned long page_size, void __user *uptr, 1512 u32 flags) 1513 { 1514 unsigned long i, max; 1515 struct iommu_test_cmd *cmd = ucmd->cmd; 1516 struct iommufd_hw_pagetable *hwpt; 1517 struct mock_iommu_domain *mock; 1518 int rc, count = 0; 1519 void *tmp; 1520 1521 if (!page_size || !length || iova % page_size || length % page_size || 1522 !uptr) 1523 return -EINVAL; 1524 1525 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 1526 if (IS_ERR(hwpt)) 1527 return PTR_ERR(hwpt); 1528 1529 if (!(mock->flags & MOCK_DIRTY_TRACK)) { 1530 rc = -EINVAL; 1531 goto out_put; 1532 } 1533 1534 max = length / page_size; 1535 tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long), 1536 GFP_KERNEL_ACCOUNT); 1537 if (!tmp) { 1538 rc = -ENOMEM; 1539 goto out_put; 1540 } 1541 1542 if (copy_from_user(tmp, uptr,DIV_ROUND_UP(max, BITS_PER_BYTE))) { 1543 rc = -EFAULT; 1544 goto out_free; 1545 } 1546 1547 for (i = 0; i < max; i++) { 1548 unsigned long cur = iova + i * page_size; 1549 void *ent, *old; 1550 1551 if (!test_bit(i, (unsigned long *)tmp)) 1552 continue; 1553 1554 ent = xa_load(&mock->pfns, cur / page_size); 1555 if (ent) { 1556 unsigned long val; 1557 1558 val = xa_to_value(ent) | MOCK_PFN_DIRTY_IOVA; 1559 old = xa_store(&mock->pfns, cur / page_size, 1560 xa_mk_value(val), GFP_KERNEL); 1561 WARN_ON_ONCE(ent != old); 1562 count++; 1563 } 1564 } 1565 1566 cmd->dirty.out_nr_dirty = count; 1567 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 1568 out_free: 1569 kvfree(tmp); 1570 out_put: 1571 iommufd_put_object(ucmd->ictx, &hwpt->obj); 1572 return rc; 1573 } 1574 1575 static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, 1576 struct iommu_test_cmd *cmd) 1577 { 1578 struct iopf_fault event = { }; 1579 struct iommufd_device *idev; 1580 1581 idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id); 1582 if (IS_ERR(idev)) 1583 return PTR_ERR(idev); 1584 1585 event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1586 if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID) 1587 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1588 event.fault.type = IOMMU_FAULT_PAGE_REQ; 1589 event.fault.prm.addr = cmd->trigger_iopf.addr; 1590 event.fault.prm.pasid = cmd->trigger_iopf.pasid; 1591 event.fault.prm.grpid = cmd->trigger_iopf.grpid; 1592 event.fault.prm.perm = cmd->trigger_iopf.perm; 1593 1594 iommu_report_device_fault(idev->dev, &event); 1595 iommufd_put_object(ucmd->ictx, &idev->obj); 1596 1597 return 0; 1598 } 1599 1600 void iommufd_selftest_destroy(struct iommufd_object *obj) 1601 { 1602 struct selftest_obj *sobj = to_selftest_obj(obj); 1603 1604 switch (sobj->type) { 1605 case TYPE_IDEV: 1606 iommufd_device_detach(sobj->idev.idev); 1607 iommufd_device_unbind(sobj->idev.idev); 1608 mock_dev_destroy(sobj->idev.mock_dev); 1609 break; 1610 } 1611 } 1612 1613 int iommufd_test(struct iommufd_ucmd *ucmd) 1614 { 1615 struct iommu_test_cmd *cmd = ucmd->cmd; 1616 1617 switch (cmd->op) { 1618 case IOMMU_TEST_OP_ADD_RESERVED: 1619 return iommufd_test_add_reserved(ucmd, cmd->id, 1620 cmd->add_reserved.start, 1621 cmd->add_reserved.length); 1622 case IOMMU_TEST_OP_MOCK_DOMAIN: 1623 case IOMMU_TEST_OP_MOCK_DOMAIN_FLAGS: 1624 return iommufd_test_mock_domain(ucmd, cmd); 1625 case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: 1626 return iommufd_test_mock_domain_replace( 1627 ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd); 1628 case IOMMU_TEST_OP_MD_CHECK_MAP: 1629 return iommufd_test_md_check_pa( 1630 ucmd, cmd->id, cmd->check_map.iova, 1631 cmd->check_map.length, 1632 u64_to_user_ptr(cmd->check_map.uptr)); 1633 case IOMMU_TEST_OP_MD_CHECK_REFS: 1634 return iommufd_test_md_check_refs( 1635 ucmd, u64_to_user_ptr(cmd->check_refs.uptr), 1636 cmd->check_refs.length, cmd->check_refs.refs); 1637 case IOMMU_TEST_OP_MD_CHECK_IOTLB: 1638 return iommufd_test_md_check_iotlb(ucmd, cmd->id, 1639 cmd->check_iotlb.id, 1640 cmd->check_iotlb.iotlb); 1641 case IOMMU_TEST_OP_DEV_CHECK_CACHE: 1642 return iommufd_test_dev_check_cache(ucmd, cmd->id, 1643 cmd->check_dev_cache.id, 1644 cmd->check_dev_cache.cache); 1645 case IOMMU_TEST_OP_CREATE_ACCESS: 1646 return iommufd_test_create_access(ucmd, cmd->id, 1647 cmd->create_access.flags); 1648 case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS: 1649 return iommufd_test_access_replace_ioas( 1650 ucmd, cmd->id, cmd->access_replace_ioas.ioas_id); 1651 case IOMMU_TEST_OP_ACCESS_PAGES: 1652 return iommufd_test_access_pages( 1653 ucmd, cmd->id, cmd->access_pages.iova, 1654 cmd->access_pages.length, 1655 u64_to_user_ptr(cmd->access_pages.uptr), 1656 cmd->access_pages.flags); 1657 case IOMMU_TEST_OP_ACCESS_RW: 1658 return iommufd_test_access_rw( 1659 ucmd, cmd->id, cmd->access_rw.iova, 1660 cmd->access_rw.length, 1661 u64_to_user_ptr(cmd->access_rw.uptr), 1662 cmd->access_rw.flags); 1663 case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: 1664 return iommufd_test_access_item_destroy( 1665 ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id); 1666 case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: 1667 /* Protect _batch_init(), can not be less than elmsz */ 1668 if (cmd->memory_limit.limit < 1669 sizeof(unsigned long) + sizeof(u32)) 1670 return -EINVAL; 1671 iommufd_test_memory_limit = cmd->memory_limit.limit; 1672 return 0; 1673 case IOMMU_TEST_OP_DIRTY: 1674 return iommufd_test_dirty(ucmd, cmd->id, cmd->dirty.iova, 1675 cmd->dirty.length, 1676 cmd->dirty.page_size, 1677 u64_to_user_ptr(cmd->dirty.uptr), 1678 cmd->dirty.flags); 1679 case IOMMU_TEST_OP_TRIGGER_IOPF: 1680 return iommufd_test_trigger_iopf(ucmd, cmd); 1681 default: 1682 return -EOPNOTSUPP; 1683 } 1684 } 1685 1686 bool iommufd_should_fail(void) 1687 { 1688 return should_fail(&fail_iommufd, 1); 1689 } 1690 1691 int __init iommufd_test_init(void) 1692 { 1693 struct platform_device_info pdevinfo = { 1694 .name = "iommufd_selftest_iommu", 1695 }; 1696 int rc; 1697 1698 dbgfs_root = 1699 fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); 1700 1701 selftest_iommu_dev = platform_device_register_full(&pdevinfo); 1702 if (IS_ERR(selftest_iommu_dev)) { 1703 rc = PTR_ERR(selftest_iommu_dev); 1704 goto err_dbgfs; 1705 } 1706 1707 rc = bus_register(&iommufd_mock_bus_type.bus); 1708 if (rc) 1709 goto err_platform; 1710 1711 rc = iommu_device_sysfs_add(&mock_iommu.iommu_dev, 1712 &selftest_iommu_dev->dev, NULL, "%s", 1713 dev_name(&selftest_iommu_dev->dev)); 1714 if (rc) 1715 goto err_bus; 1716 1717 rc = iommu_device_register_bus(&mock_iommu.iommu_dev, &mock_ops, 1718 &iommufd_mock_bus_type.bus, 1719 &iommufd_mock_bus_type.nb); 1720 if (rc) 1721 goto err_sysfs; 1722 1723 refcount_set(&mock_iommu.users, 1); 1724 init_completion(&mock_iommu.complete); 1725 1726 mock_iommu_iopf_queue = iopf_queue_alloc("mock-iopfq"); 1727 1728 return 0; 1729 1730 err_sysfs: 1731 iommu_device_sysfs_remove(&mock_iommu.iommu_dev); 1732 err_bus: 1733 bus_unregister(&iommufd_mock_bus_type.bus); 1734 err_platform: 1735 platform_device_unregister(selftest_iommu_dev); 1736 err_dbgfs: 1737 debugfs_remove_recursive(dbgfs_root); 1738 return rc; 1739 } 1740 1741 static void iommufd_test_wait_for_users(void) 1742 { 1743 if (refcount_dec_and_test(&mock_iommu.users)) 1744 return; 1745 /* 1746 * Time out waiting for iommu device user count to become 0. 1747 * 1748 * Note that this is just making an example here, since the selftest is 1749 * built into the iommufd module, i.e. it only unplugs the iommu device 1750 * when unloading the module. So, it is expected that this WARN_ON will 1751 * not trigger, as long as any iommufd FDs are open. 1752 */ 1753 WARN_ON(!wait_for_completion_timeout(&mock_iommu.complete, 1754 msecs_to_jiffies(10000))); 1755 } 1756 1757 void iommufd_test_exit(void) 1758 { 1759 if (mock_iommu_iopf_queue) { 1760 iopf_queue_free(mock_iommu_iopf_queue); 1761 mock_iommu_iopf_queue = NULL; 1762 } 1763 1764 iommufd_test_wait_for_users(); 1765 iommu_device_sysfs_remove(&mock_iommu.iommu_dev); 1766 iommu_device_unregister_bus(&mock_iommu.iommu_dev, 1767 &iommufd_mock_bus_type.bus, 1768 &iommufd_mock_bus_type.nb); 1769 bus_unregister(&iommufd_mock_bus_type.bus); 1770 platform_device_unregister(selftest_iommu_dev); 1771 debugfs_remove_recursive(dbgfs_root); 1772 } 1773