1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */ 3 #include <asm/unistd.h> 4 #include <stdlib.h> 5 #include <sys/capability.h> 6 #include <sys/mman.h> 7 #include <sys/eventfd.h> 8 9 #define __EXPORTED_HEADERS__ 10 #include <linux/vfio.h> 11 12 #include "iommufd_utils.h" 13 14 static unsigned long HUGEPAGE_SIZE; 15 16 static unsigned long get_huge_page_size(void) 17 { 18 char buf[80]; 19 int ret; 20 int fd; 21 22 fd = open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", 23 O_RDONLY); 24 if (fd < 0) 25 return 2 * 1024 * 1024; 26 27 ret = read(fd, buf, sizeof(buf)); 28 close(fd); 29 if (ret <= 0 || ret == sizeof(buf)) 30 return 2 * 1024 * 1024; 31 buf[ret] = 0; 32 return strtoul(buf, NULL, 10); 33 } 34 35 static __attribute__((constructor)) void setup_sizes(void) 36 { 37 void *vrc; 38 int rc; 39 40 PAGE_SIZE = sysconf(_SC_PAGE_SIZE); 41 HUGEPAGE_SIZE = get_huge_page_size(); 42 43 BUFFER_SIZE = PAGE_SIZE * 16; 44 rc = posix_memalign(&buffer, HUGEPAGE_SIZE, BUFFER_SIZE); 45 assert(!rc); 46 assert(buffer); 47 assert((uintptr_t)buffer % HUGEPAGE_SIZE == 0); 48 vrc = mmap(buffer, BUFFER_SIZE, PROT_READ | PROT_WRITE, 49 MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0); 50 assert(vrc == buffer); 51 52 mfd_buffer = memfd_mmap(BUFFER_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, 53 &mfd); 54 assert(mfd_buffer != MAP_FAILED); 55 assert(mfd > 0); 56 } 57 58 FIXTURE(iommufd) 59 { 60 int fd; 61 }; 62 63 FIXTURE_SETUP(iommufd) 64 { 65 self->fd = open("/dev/iommu", O_RDWR); 66 ASSERT_NE(-1, self->fd); 67 } 68 69 FIXTURE_TEARDOWN(iommufd) 70 { 71 teardown_iommufd(self->fd, _metadata); 72 } 73 74 TEST_F(iommufd, simple_close) 75 { 76 } 77 78 TEST_F(iommufd, cmd_fail) 79 { 80 struct iommu_destroy cmd = { .size = sizeof(cmd), .id = 0 }; 81 82 /* object id is invalid */ 83 EXPECT_ERRNO(ENOENT, _test_ioctl_destroy(self->fd, 0)); 84 /* Bad pointer */ 85 EXPECT_ERRNO(EFAULT, ioctl(self->fd, IOMMU_DESTROY, NULL)); 86 /* Unknown ioctl */ 87 EXPECT_ERRNO(ENOTTY, 88 ioctl(self->fd, _IO(IOMMUFD_TYPE, IOMMUFD_CMD_BASE - 1), 89 &cmd)); 90 } 91 92 TEST_F(iommufd, cmd_length) 93 { 94 #define TEST_LENGTH(_struct, _ioctl, _last) \ 95 { \ 96 size_t min_size = offsetofend(struct _struct, _last); \ 97 struct { \ 98 struct _struct cmd; \ 99 uint8_t extra; \ 100 } cmd = { .cmd = { .size = min_size - 1 }, \ 101 .extra = UINT8_MAX }; \ 102 int old_errno; \ 103 int rc; \ 104 \ 105 EXPECT_ERRNO(EINVAL, ioctl(self->fd, _ioctl, &cmd)); \ 106 cmd.cmd.size = sizeof(struct _struct) + 1; \ 107 EXPECT_ERRNO(E2BIG, ioctl(self->fd, _ioctl, &cmd)); \ 108 cmd.cmd.size = sizeof(struct _struct); \ 109 rc = ioctl(self->fd, _ioctl, &cmd); \ 110 old_errno = errno; \ 111 cmd.cmd.size = sizeof(struct _struct) + 1; \ 112 cmd.extra = 0; \ 113 if (rc) { \ 114 EXPECT_ERRNO(old_errno, \ 115 ioctl(self->fd, _ioctl, &cmd)); \ 116 } else { \ 117 ASSERT_EQ(0, ioctl(self->fd, _ioctl, &cmd)); \ 118 } \ 119 } 120 121 TEST_LENGTH(iommu_destroy, IOMMU_DESTROY, id); 122 TEST_LENGTH(iommu_hw_info, IOMMU_GET_HW_INFO, __reserved); 123 TEST_LENGTH(iommu_hwpt_alloc, IOMMU_HWPT_ALLOC, __reserved); 124 TEST_LENGTH(iommu_hwpt_invalidate, IOMMU_HWPT_INVALIDATE, __reserved); 125 TEST_LENGTH(iommu_ioas_alloc, IOMMU_IOAS_ALLOC, out_ioas_id); 126 TEST_LENGTH(iommu_ioas_iova_ranges, IOMMU_IOAS_IOVA_RANGES, 127 out_iova_alignment); 128 TEST_LENGTH(iommu_ioas_allow_iovas, IOMMU_IOAS_ALLOW_IOVAS, 129 allowed_iovas); 130 TEST_LENGTH(iommu_ioas_map, IOMMU_IOAS_MAP, iova); 131 TEST_LENGTH(iommu_ioas_copy, IOMMU_IOAS_COPY, src_iova); 132 TEST_LENGTH(iommu_ioas_unmap, IOMMU_IOAS_UNMAP, length); 133 TEST_LENGTH(iommu_option, IOMMU_OPTION, val64); 134 TEST_LENGTH(iommu_vfio_ioas, IOMMU_VFIO_IOAS, __reserved); 135 TEST_LENGTH(iommu_ioas_map_file, IOMMU_IOAS_MAP_FILE, iova); 136 TEST_LENGTH(iommu_viommu_alloc, IOMMU_VIOMMU_ALLOC, out_viommu_id); 137 TEST_LENGTH(iommu_vdevice_alloc, IOMMU_VDEVICE_ALLOC, virt_id); 138 TEST_LENGTH(iommu_ioas_change_process, IOMMU_IOAS_CHANGE_PROCESS, 139 __reserved); 140 #undef TEST_LENGTH 141 } 142 143 TEST_F(iommufd, cmd_ex_fail) 144 { 145 struct { 146 struct iommu_destroy cmd; 147 __u64 future; 148 } cmd = { .cmd = { .size = sizeof(cmd), .id = 0 } }; 149 150 /* object id is invalid and command is longer */ 151 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd)); 152 /* future area is non-zero */ 153 cmd.future = 1; 154 EXPECT_ERRNO(E2BIG, ioctl(self->fd, IOMMU_DESTROY, &cmd)); 155 /* Original command "works" */ 156 cmd.cmd.size = sizeof(cmd.cmd); 157 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_DESTROY, &cmd)); 158 /* Short command fails */ 159 cmd.cmd.size = sizeof(cmd.cmd) - 1; 160 EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_DESTROY, &cmd)); 161 } 162 163 TEST_F(iommufd, global_options) 164 { 165 struct iommu_option cmd = { 166 .size = sizeof(cmd), 167 .option_id = IOMMU_OPTION_RLIMIT_MODE, 168 .op = IOMMU_OPTION_OP_GET, 169 .val64 = 1, 170 }; 171 172 cmd.option_id = IOMMU_OPTION_RLIMIT_MODE; 173 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 174 ASSERT_EQ(0, cmd.val64); 175 176 /* This requires root */ 177 cmd.op = IOMMU_OPTION_OP_SET; 178 cmd.val64 = 1; 179 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 180 cmd.val64 = 2; 181 EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd)); 182 183 cmd.op = IOMMU_OPTION_OP_GET; 184 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 185 ASSERT_EQ(1, cmd.val64); 186 187 cmd.op = IOMMU_OPTION_OP_SET; 188 cmd.val64 = 0; 189 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 190 191 cmd.op = IOMMU_OPTION_OP_GET; 192 cmd.option_id = IOMMU_OPTION_HUGE_PAGES; 193 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd)); 194 cmd.op = IOMMU_OPTION_OP_SET; 195 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_OPTION, &cmd)); 196 } 197 198 static void drop_cap_ipc_lock(struct __test_metadata *_metadata) 199 { 200 cap_t caps; 201 cap_value_t cap_list[1] = { CAP_IPC_LOCK }; 202 203 caps = cap_get_proc(); 204 ASSERT_NE(caps, NULL); 205 ASSERT_NE(-1, 206 cap_set_flag(caps, CAP_EFFECTIVE, 1, cap_list, CAP_CLEAR)); 207 ASSERT_NE(-1, cap_set_proc(caps)); 208 cap_free(caps); 209 } 210 211 static long get_proc_status_value(pid_t pid, const char *var) 212 { 213 FILE *fp; 214 char buf[80], tag[80]; 215 long val = -1; 216 217 snprintf(buf, sizeof(buf), "/proc/%d/status", pid); 218 fp = fopen(buf, "r"); 219 if (!fp) 220 return val; 221 222 while (fgets(buf, sizeof(buf), fp)) 223 if (fscanf(fp, "%s %ld\n", tag, &val) == 2 && !strcmp(tag, var)) 224 break; 225 226 fclose(fp); 227 return val; 228 } 229 230 static long get_vm_pinned(pid_t pid) 231 { 232 return get_proc_status_value(pid, "VmPin:"); 233 } 234 235 static long get_vm_locked(pid_t pid) 236 { 237 return get_proc_status_value(pid, "VmLck:"); 238 } 239 240 FIXTURE(change_process) 241 { 242 int fd; 243 uint32_t ioas_id; 244 }; 245 246 FIXTURE_VARIANT(change_process) 247 { 248 int accounting; 249 }; 250 251 FIXTURE_SETUP(change_process) 252 { 253 self->fd = open("/dev/iommu", O_RDWR); 254 ASSERT_NE(-1, self->fd); 255 256 drop_cap_ipc_lock(_metadata); 257 if (variant->accounting != IOPT_PAGES_ACCOUNT_NONE) { 258 struct iommu_option set_limit_cmd = { 259 .size = sizeof(set_limit_cmd), 260 .option_id = IOMMU_OPTION_RLIMIT_MODE, 261 .op = IOMMU_OPTION_OP_SET, 262 .val64 = (variant->accounting == IOPT_PAGES_ACCOUNT_MM), 263 }; 264 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &set_limit_cmd)); 265 } 266 267 test_ioctl_ioas_alloc(&self->ioas_id); 268 test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL); 269 } 270 271 FIXTURE_TEARDOWN(change_process) 272 { 273 teardown_iommufd(self->fd, _metadata); 274 } 275 276 FIXTURE_VARIANT_ADD(change_process, account_none) 277 { 278 .accounting = IOPT_PAGES_ACCOUNT_NONE, 279 }; 280 281 FIXTURE_VARIANT_ADD(change_process, account_user) 282 { 283 .accounting = IOPT_PAGES_ACCOUNT_USER, 284 }; 285 286 FIXTURE_VARIANT_ADD(change_process, account_mm) 287 { 288 .accounting = IOPT_PAGES_ACCOUNT_MM, 289 }; 290 291 TEST_F(change_process, basic) 292 { 293 pid_t parent = getpid(); 294 pid_t child; 295 __u64 iova; 296 struct iommu_ioas_change_process cmd = { 297 .size = sizeof(cmd), 298 }; 299 300 /* Expect failure if non-file maps exist */ 301 test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova); 302 EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_IOAS_CHANGE_PROCESS, &cmd)); 303 test_ioctl_ioas_unmap(iova, PAGE_SIZE); 304 305 /* Change process works in current process. */ 306 test_ioctl_ioas_map_file(mfd, 0, PAGE_SIZE, &iova); 307 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_CHANGE_PROCESS, &cmd)); 308 309 /* Change process works in another process */ 310 child = fork(); 311 if (!child) { 312 int nlock = PAGE_SIZE / 1024; 313 314 /* Parent accounts for locked memory before */ 315 ASSERT_EQ(nlock, get_vm_pinned(parent)); 316 if (variant->accounting == IOPT_PAGES_ACCOUNT_MM) 317 ASSERT_EQ(nlock, get_vm_locked(parent)); 318 ASSERT_EQ(0, get_vm_pinned(getpid())); 319 ASSERT_EQ(0, get_vm_locked(getpid())); 320 321 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_CHANGE_PROCESS, &cmd)); 322 323 /* Child accounts for locked memory after */ 324 ASSERT_EQ(0, get_vm_pinned(parent)); 325 ASSERT_EQ(0, get_vm_locked(parent)); 326 ASSERT_EQ(nlock, get_vm_pinned(getpid())); 327 if (variant->accounting == IOPT_PAGES_ACCOUNT_MM) 328 ASSERT_EQ(nlock, get_vm_locked(getpid())); 329 330 exit(0); 331 } 332 ASSERT_NE(-1, child); 333 ASSERT_EQ(child, waitpid(child, NULL, 0)); 334 } 335 336 FIXTURE(iommufd_ioas) 337 { 338 int fd; 339 uint32_t ioas_id; 340 uint32_t stdev_id; 341 uint32_t hwpt_id; 342 uint32_t device_id; 343 uint64_t base_iova; 344 uint32_t device_pasid_id; 345 }; 346 347 FIXTURE_VARIANT(iommufd_ioas) 348 { 349 unsigned int mock_domains; 350 unsigned int memory_limit; 351 bool pasid_capable; 352 }; 353 354 FIXTURE_SETUP(iommufd_ioas) 355 { 356 unsigned int i; 357 358 359 self->fd = open("/dev/iommu", O_RDWR); 360 ASSERT_NE(-1, self->fd); 361 test_ioctl_ioas_alloc(&self->ioas_id); 362 363 if (!variant->memory_limit) { 364 test_ioctl_set_default_memory_limit(); 365 } else { 366 test_ioctl_set_temp_memory_limit(variant->memory_limit); 367 } 368 369 for (i = 0; i != variant->mock_domains; i++) { 370 test_cmd_mock_domain(self->ioas_id, &self->stdev_id, 371 &self->hwpt_id, &self->device_id); 372 test_cmd_dev_check_cache_all(self->device_id, 373 IOMMU_TEST_DEV_CACHE_DEFAULT); 374 self->base_iova = MOCK_APERTURE_START; 375 } 376 377 if (variant->pasid_capable) 378 test_cmd_mock_domain_flags(self->ioas_id, 379 MOCK_FLAGS_DEVICE_PASID, 380 NULL, NULL, 381 &self->device_pasid_id); 382 } 383 384 FIXTURE_TEARDOWN(iommufd_ioas) 385 { 386 test_ioctl_set_default_memory_limit(); 387 teardown_iommufd(self->fd, _metadata); 388 } 389 390 FIXTURE_VARIANT_ADD(iommufd_ioas, no_domain) 391 { 392 }; 393 394 FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain) 395 { 396 .mock_domains = 1, 397 .pasid_capable = true, 398 }; 399 400 FIXTURE_VARIANT_ADD(iommufd_ioas, two_mock_domain) 401 { 402 .mock_domains = 2, 403 }; 404 405 FIXTURE_VARIANT_ADD(iommufd_ioas, mock_domain_limit) 406 { 407 .mock_domains = 1, 408 .memory_limit = 16, 409 }; 410 411 TEST_F(iommufd_ioas, ioas_auto_destroy) 412 { 413 } 414 415 TEST_F(iommufd_ioas, ioas_destroy) 416 { 417 if (self->stdev_id) { 418 /* IOAS cannot be freed while a device has a HWPT using it */ 419 EXPECT_ERRNO(EBUSY, 420 _test_ioctl_destroy(self->fd, self->ioas_id)); 421 } else { 422 /* Can allocate and manually free an IOAS table */ 423 test_ioctl_destroy(self->ioas_id); 424 } 425 } 426 427 TEST_F(iommufd_ioas, alloc_hwpt_nested) 428 { 429 const uint32_t min_data_len = 430 offsetofend(struct iommu_hwpt_selftest, iotlb); 431 struct iommu_hwpt_selftest data = { 432 .iotlb = IOMMU_TEST_IOTLB_DEFAULT, 433 }; 434 struct iommu_hwpt_invalidate_selftest inv_reqs[2] = {}; 435 uint32_t nested_hwpt_id[2] = {}; 436 uint32_t num_inv; 437 uint32_t parent_hwpt_id = 0; 438 uint32_t parent_hwpt_id_not_work = 0; 439 uint32_t test_hwpt_id = 0; 440 uint32_t iopf_hwpt_id; 441 uint32_t fault_id; 442 uint32_t fault_fd; 443 444 if (self->device_id) { 445 /* Negative tests */ 446 test_err_hwpt_alloc(ENOENT, self->ioas_id, self->device_id, 0, 447 &test_hwpt_id); 448 test_err_hwpt_alloc(EINVAL, self->device_id, self->device_id, 0, 449 &test_hwpt_id); 450 test_err_hwpt_alloc(EOPNOTSUPP, self->device_id, self->ioas_id, 451 IOMMU_HWPT_ALLOC_NEST_PARENT | 452 IOMMU_HWPT_FAULT_ID_VALID, 453 &test_hwpt_id); 454 455 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 456 IOMMU_HWPT_ALLOC_NEST_PARENT, 457 &parent_hwpt_id); 458 459 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 0, 460 &parent_hwpt_id_not_work); 461 462 /* Negative nested tests */ 463 test_err_hwpt_alloc_nested(EINVAL, self->device_id, 464 parent_hwpt_id, 0, 465 &nested_hwpt_id[0], 466 IOMMU_HWPT_DATA_NONE, &data, 467 sizeof(data)); 468 test_err_hwpt_alloc_nested(EOPNOTSUPP, self->device_id, 469 parent_hwpt_id, 0, 470 &nested_hwpt_id[0], 471 IOMMU_HWPT_DATA_SELFTEST + 1, &data, 472 sizeof(data)); 473 test_err_hwpt_alloc_nested(EINVAL, self->device_id, 474 parent_hwpt_id, 0, 475 &nested_hwpt_id[0], 476 IOMMU_HWPT_DATA_SELFTEST, &data, 477 min_data_len - 1); 478 test_err_hwpt_alloc_nested(EFAULT, self->device_id, 479 parent_hwpt_id, 0, 480 &nested_hwpt_id[0], 481 IOMMU_HWPT_DATA_SELFTEST, NULL, 482 sizeof(data)); 483 test_err_hwpt_alloc_nested( 484 EOPNOTSUPP, self->device_id, parent_hwpt_id, 485 IOMMU_HWPT_ALLOC_NEST_PARENT, &nested_hwpt_id[0], 486 IOMMU_HWPT_DATA_SELFTEST, &data, sizeof(data)); 487 test_err_hwpt_alloc_nested(EINVAL, self->device_id, 488 parent_hwpt_id_not_work, 0, 489 &nested_hwpt_id[0], 490 IOMMU_HWPT_DATA_SELFTEST, &data, 491 sizeof(data)); 492 493 /* Allocate two nested hwpts sharing one common parent hwpt */ 494 test_ioctl_fault_alloc(&fault_id, &fault_fd); 495 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0, 496 &nested_hwpt_id[0], 497 IOMMU_HWPT_DATA_SELFTEST, &data, 498 sizeof(data)); 499 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 0, 500 &nested_hwpt_id[1], 501 IOMMU_HWPT_DATA_SELFTEST, &data, 502 sizeof(data)); 503 test_err_hwpt_alloc_iopf(ENOENT, self->device_id, parent_hwpt_id, 504 UINT32_MAX, IOMMU_HWPT_FAULT_ID_VALID, 505 &iopf_hwpt_id, IOMMU_HWPT_DATA_SELFTEST, 506 &data, sizeof(data)); 507 test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id, 508 IOMMU_HWPT_FAULT_ID_VALID, &iopf_hwpt_id, 509 IOMMU_HWPT_DATA_SELFTEST, &data, 510 sizeof(data)); 511 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0], 512 IOMMU_TEST_IOTLB_DEFAULT); 513 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1], 514 IOMMU_TEST_IOTLB_DEFAULT); 515 516 /* Negative test: a nested hwpt on top of a nested hwpt */ 517 test_err_hwpt_alloc_nested(EINVAL, self->device_id, 518 nested_hwpt_id[0], 0, &test_hwpt_id, 519 IOMMU_HWPT_DATA_SELFTEST, &data, 520 sizeof(data)); 521 /* Negative test: parent hwpt now cannot be freed */ 522 EXPECT_ERRNO(EBUSY, 523 _test_ioctl_destroy(self->fd, parent_hwpt_id)); 524 525 /* hwpt_invalidate does not support a parent hwpt */ 526 num_inv = 1; 527 test_err_hwpt_invalidate(EINVAL, parent_hwpt_id, inv_reqs, 528 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 529 sizeof(*inv_reqs), &num_inv); 530 assert(!num_inv); 531 532 /* Check data_type by passing zero-length array */ 533 num_inv = 0; 534 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs, 535 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 536 sizeof(*inv_reqs), &num_inv); 537 assert(!num_inv); 538 539 /* Negative test: Invalid data_type */ 540 num_inv = 1; 541 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 542 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST_INVALID, 543 sizeof(*inv_reqs), &num_inv); 544 assert(!num_inv); 545 546 /* Negative test: structure size sanity */ 547 num_inv = 1; 548 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 549 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 550 sizeof(*inv_reqs) + 1, &num_inv); 551 assert(!num_inv); 552 553 num_inv = 1; 554 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 555 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 556 1, &num_inv); 557 assert(!num_inv); 558 559 /* Negative test: entry_len is bounded by PAGE_SIZE */ 560 num_inv = 1; 561 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 562 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 563 PAGE_SIZE + 1, &num_inv); 564 assert(!num_inv); 565 566 /* Negative test: entry_num is bounded */ 567 #define IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX (1U << 19) 568 num_inv = IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX + 1; 569 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 570 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 571 sizeof(*inv_reqs), &num_inv); 572 assert(!num_inv); 573 574 /* Negative test: invalid flag is passed */ 575 num_inv = 1; 576 inv_reqs[0].flags = 0xffffffff; 577 test_err_hwpt_invalidate(EOPNOTSUPP, nested_hwpt_id[0], inv_reqs, 578 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 579 sizeof(*inv_reqs), &num_inv); 580 assert(!num_inv); 581 582 /* Negative test: invalid data_uptr when array is not empty */ 583 num_inv = 1; 584 inv_reqs[0].flags = 0; 585 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], NULL, 586 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 587 sizeof(*inv_reqs), &num_inv); 588 assert(!num_inv); 589 590 /* Negative test: invalid entry_len when array is not empty */ 591 num_inv = 1; 592 inv_reqs[0].flags = 0; 593 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 594 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 595 0, &num_inv); 596 assert(!num_inv); 597 598 /* Negative test: invalid iotlb_id */ 599 num_inv = 1; 600 inv_reqs[0].flags = 0; 601 inv_reqs[0].iotlb_id = MOCK_NESTED_DOMAIN_IOTLB_ID_MAX + 1; 602 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 603 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 604 sizeof(*inv_reqs), &num_inv); 605 assert(!num_inv); 606 607 /* 608 * Invalidate the 1st iotlb entry but fail the 2nd request 609 * due to invalid flags configuration in the 2nd request. 610 */ 611 num_inv = 2; 612 inv_reqs[0].flags = 0; 613 inv_reqs[0].iotlb_id = 0; 614 inv_reqs[1].flags = 0xffffffff; 615 inv_reqs[1].iotlb_id = 1; 616 test_err_hwpt_invalidate(EOPNOTSUPP, nested_hwpt_id[0], inv_reqs, 617 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 618 sizeof(*inv_reqs), &num_inv); 619 assert(num_inv == 1); 620 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0); 621 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1, 622 IOMMU_TEST_IOTLB_DEFAULT); 623 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2, 624 IOMMU_TEST_IOTLB_DEFAULT); 625 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3, 626 IOMMU_TEST_IOTLB_DEFAULT); 627 628 /* 629 * Invalidate the 1st iotlb entry but fail the 2nd request 630 * due to invalid iotlb_id configuration in the 2nd request. 631 */ 632 num_inv = 2; 633 inv_reqs[0].flags = 0; 634 inv_reqs[0].iotlb_id = 0; 635 inv_reqs[1].flags = 0; 636 inv_reqs[1].iotlb_id = MOCK_NESTED_DOMAIN_IOTLB_ID_MAX + 1; 637 test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, 638 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 639 sizeof(*inv_reqs), &num_inv); 640 assert(num_inv == 1); 641 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0); 642 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1, 643 IOMMU_TEST_IOTLB_DEFAULT); 644 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2, 645 IOMMU_TEST_IOTLB_DEFAULT); 646 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3, 647 IOMMU_TEST_IOTLB_DEFAULT); 648 649 /* Invalidate the 2nd iotlb entry and verify */ 650 num_inv = 1; 651 inv_reqs[0].flags = 0; 652 inv_reqs[0].iotlb_id = 1; 653 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs, 654 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 655 sizeof(*inv_reqs), &num_inv); 656 assert(num_inv == 1); 657 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 0, 0); 658 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 1, 0); 659 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 2, 660 IOMMU_TEST_IOTLB_DEFAULT); 661 test_cmd_hwpt_check_iotlb(nested_hwpt_id[0], 3, 662 IOMMU_TEST_IOTLB_DEFAULT); 663 664 /* Invalidate the 3rd and 4th iotlb entries and verify */ 665 num_inv = 2; 666 inv_reqs[0].flags = 0; 667 inv_reqs[0].iotlb_id = 2; 668 inv_reqs[1].flags = 0; 669 inv_reqs[1].iotlb_id = 3; 670 test_cmd_hwpt_invalidate(nested_hwpt_id[0], inv_reqs, 671 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 672 sizeof(*inv_reqs), &num_inv); 673 assert(num_inv == 2); 674 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[0], 0); 675 676 /* Invalidate all iotlb entries for nested_hwpt_id[1] and verify */ 677 num_inv = 1; 678 inv_reqs[0].flags = IOMMU_TEST_INVALIDATE_FLAG_ALL; 679 test_cmd_hwpt_invalidate(nested_hwpt_id[1], inv_reqs, 680 IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 681 sizeof(*inv_reqs), &num_inv); 682 assert(num_inv == 1); 683 test_cmd_hwpt_check_iotlb_all(nested_hwpt_id[1], 0); 684 685 /* Attach device to nested_hwpt_id[0] that then will be busy */ 686 test_cmd_mock_domain_replace(self->stdev_id, nested_hwpt_id[0]); 687 EXPECT_ERRNO(EBUSY, 688 _test_ioctl_destroy(self->fd, nested_hwpt_id[0])); 689 690 /* Switch from nested_hwpt_id[0] to nested_hwpt_id[1] */ 691 test_cmd_mock_domain_replace(self->stdev_id, nested_hwpt_id[1]); 692 EXPECT_ERRNO(EBUSY, 693 _test_ioctl_destroy(self->fd, nested_hwpt_id[1])); 694 test_ioctl_destroy(nested_hwpt_id[0]); 695 696 /* Switch from nested_hwpt_id[1] to iopf_hwpt_id */ 697 test_cmd_mock_domain_replace(self->stdev_id, iopf_hwpt_id); 698 EXPECT_ERRNO(EBUSY, 699 _test_ioctl_destroy(self->fd, iopf_hwpt_id)); 700 /* Trigger an IOPF on the device */ 701 test_cmd_trigger_iopf(self->device_id, fault_fd); 702 703 /* Detach from nested_hwpt_id[1] and destroy it */ 704 test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id); 705 test_ioctl_destroy(nested_hwpt_id[1]); 706 test_ioctl_destroy(iopf_hwpt_id); 707 708 /* Detach from the parent hw_pagetable and destroy it */ 709 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id); 710 test_ioctl_destroy(parent_hwpt_id); 711 test_ioctl_destroy(parent_hwpt_id_not_work); 712 close(fault_fd); 713 test_ioctl_destroy(fault_id); 714 } else { 715 test_err_hwpt_alloc(ENOENT, self->device_id, self->ioas_id, 0, 716 &parent_hwpt_id); 717 test_err_hwpt_alloc_nested(ENOENT, self->device_id, 718 parent_hwpt_id, 0, 719 &nested_hwpt_id[0], 720 IOMMU_HWPT_DATA_SELFTEST, &data, 721 sizeof(data)); 722 test_err_hwpt_alloc_nested(ENOENT, self->device_id, 723 parent_hwpt_id, 0, 724 &nested_hwpt_id[1], 725 IOMMU_HWPT_DATA_SELFTEST, &data, 726 sizeof(data)); 727 test_err_mock_domain_replace(ENOENT, self->stdev_id, 728 nested_hwpt_id[0]); 729 test_err_mock_domain_replace(ENOENT, self->stdev_id, 730 nested_hwpt_id[1]); 731 } 732 } 733 734 TEST_F(iommufd_ioas, hwpt_attach) 735 { 736 /* Create a device attached directly to a hwpt */ 737 if (self->stdev_id) { 738 test_cmd_mock_domain(self->hwpt_id, NULL, NULL, NULL); 739 } else { 740 test_err_mock_domain(ENOENT, self->hwpt_id, NULL, NULL); 741 } 742 } 743 744 TEST_F(iommufd_ioas, ioas_area_destroy) 745 { 746 /* Adding an area does not change ability to destroy */ 747 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova); 748 if (self->stdev_id) 749 EXPECT_ERRNO(EBUSY, 750 _test_ioctl_destroy(self->fd, self->ioas_id)); 751 else 752 test_ioctl_destroy(self->ioas_id); 753 } 754 755 TEST_F(iommufd_ioas, ioas_area_auto_destroy) 756 { 757 int i; 758 759 /* Can allocate and automatically free an IOAS table with many areas */ 760 for (i = 0; i != 10; i++) { 761 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, 762 self->base_iova + i * PAGE_SIZE); 763 } 764 } 765 766 TEST_F(iommufd_ioas, get_hw_info) 767 { 768 struct iommu_test_hw_info buffer_exact; 769 struct iommu_test_hw_info_buffer_larger { 770 struct iommu_test_hw_info info; 771 uint64_t trailing_bytes; 772 } buffer_larger; 773 774 if (self->device_id) { 775 uint8_t max_pasid = 0; 776 777 /* Provide a zero-size user_buffer */ 778 test_cmd_get_hw_info(self->device_id, 779 IOMMU_HW_INFO_TYPE_DEFAULT, NULL, 0); 780 /* Provide a user_buffer with exact size */ 781 test_cmd_get_hw_info(self->device_id, 782 IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_exact, 783 sizeof(buffer_exact)); 784 785 /* Request for a wrong data_type, and a correct one */ 786 test_err_get_hw_info(EOPNOTSUPP, self->device_id, 787 IOMMU_HW_INFO_TYPE_SELFTEST + 1, 788 &buffer_exact, sizeof(buffer_exact)); 789 test_cmd_get_hw_info(self->device_id, 790 IOMMU_HW_INFO_TYPE_SELFTEST, &buffer_exact, 791 sizeof(buffer_exact)); 792 /* 793 * Provide a user_buffer with size larger than the exact size to check if 794 * kernel zero the trailing bytes. 795 */ 796 test_cmd_get_hw_info(self->device_id, 797 IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_larger, 798 sizeof(buffer_larger)); 799 /* 800 * Provide a user_buffer with size smaller than the exact size to check if 801 * the fields within the size range still gets updated. 802 */ 803 test_cmd_get_hw_info(self->device_id, 804 IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_exact, 805 offsetofend(struct iommu_test_hw_info, 806 flags)); 807 test_cmd_get_hw_info_pasid(self->device_id, &max_pasid); 808 ASSERT_EQ(0, max_pasid); 809 if (variant->pasid_capable) { 810 test_cmd_get_hw_info_pasid(self->device_pasid_id, 811 &max_pasid); 812 ASSERT_EQ(MOCK_PASID_WIDTH, max_pasid); 813 } 814 } else { 815 test_err_get_hw_info(ENOENT, self->device_id, 816 IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_exact, 817 sizeof(buffer_exact)); 818 test_err_get_hw_info(ENOENT, self->device_id, 819 IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_larger, 820 sizeof(buffer_larger)); 821 } 822 } 823 824 TEST_F(iommufd_ioas, area) 825 { 826 int i; 827 828 /* Unmap fails if nothing is mapped */ 829 for (i = 0; i != 10; i++) 830 test_err_ioctl_ioas_unmap(ENOENT, i * PAGE_SIZE, PAGE_SIZE); 831 832 /* Unmap works */ 833 for (i = 0; i != 10; i++) 834 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, 835 self->base_iova + i * PAGE_SIZE); 836 for (i = 0; i != 10; i++) 837 test_ioctl_ioas_unmap(self->base_iova + i * PAGE_SIZE, 838 PAGE_SIZE); 839 840 /* Split fails */ 841 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE * 2, 842 self->base_iova + 16 * PAGE_SIZE); 843 test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 16 * PAGE_SIZE, 844 PAGE_SIZE); 845 test_err_ioctl_ioas_unmap(ENOENT, self->base_iova + 17 * PAGE_SIZE, 846 PAGE_SIZE); 847 848 /* Over map fails */ 849 test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2, 850 self->base_iova + 16 * PAGE_SIZE); 851 test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE, 852 self->base_iova + 16 * PAGE_SIZE); 853 test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE, 854 self->base_iova + 17 * PAGE_SIZE); 855 test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 2, 856 self->base_iova + 15 * PAGE_SIZE); 857 test_err_ioctl_ioas_map_fixed(EEXIST, buffer, PAGE_SIZE * 3, 858 self->base_iova + 15 * PAGE_SIZE); 859 860 /* unmap all works */ 861 test_ioctl_ioas_unmap(0, UINT64_MAX); 862 863 /* Unmap all succeeds on an empty IOAS */ 864 test_ioctl_ioas_unmap(0, UINT64_MAX); 865 } 866 867 TEST_F(iommufd_ioas, unmap_fully_contained_areas) 868 { 869 uint64_t unmap_len; 870 int i; 871 872 /* Give no_domain some space to rewind base_iova */ 873 self->base_iova += 4 * PAGE_SIZE; 874 875 for (i = 0; i != 4; i++) 876 test_ioctl_ioas_map_fixed(buffer, 8 * PAGE_SIZE, 877 self->base_iova + i * 16 * PAGE_SIZE); 878 879 /* Unmap not fully contained area doesn't work */ 880 test_err_ioctl_ioas_unmap(ENOENT, self->base_iova - 4 * PAGE_SIZE, 881 8 * PAGE_SIZE); 882 test_err_ioctl_ioas_unmap(ENOENT, 883 self->base_iova + 3 * 16 * PAGE_SIZE + 884 8 * PAGE_SIZE - 4 * PAGE_SIZE, 885 8 * PAGE_SIZE); 886 887 /* Unmap fully contained areas works */ 888 ASSERT_EQ(0, _test_ioctl_ioas_unmap(self->fd, self->ioas_id, 889 self->base_iova - 4 * PAGE_SIZE, 890 3 * 16 * PAGE_SIZE + 8 * PAGE_SIZE + 891 4 * PAGE_SIZE, 892 &unmap_len)); 893 ASSERT_EQ(32 * PAGE_SIZE, unmap_len); 894 } 895 896 TEST_F(iommufd_ioas, area_auto_iova) 897 { 898 struct iommu_test_cmd test_cmd = { 899 .size = sizeof(test_cmd), 900 .op = IOMMU_TEST_OP_ADD_RESERVED, 901 .id = self->ioas_id, 902 .add_reserved = { .start = PAGE_SIZE * 4, 903 .length = PAGE_SIZE * 100 }, 904 }; 905 struct iommu_iova_range ranges[1] = {}; 906 struct iommu_ioas_allow_iovas allow_cmd = { 907 .size = sizeof(allow_cmd), 908 .ioas_id = self->ioas_id, 909 .num_iovas = 1, 910 .allowed_iovas = (uintptr_t)ranges, 911 }; 912 __u64 iovas[10]; 913 int i; 914 915 /* Simple 4k pages */ 916 for (i = 0; i != 10; i++) 917 test_ioctl_ioas_map(buffer, PAGE_SIZE, &iovas[i]); 918 for (i = 0; i != 10; i++) 919 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE); 920 921 /* Kernel automatically aligns IOVAs properly */ 922 for (i = 0; i != 10; i++) { 923 size_t length = PAGE_SIZE * (i + 1); 924 925 if (self->stdev_id) { 926 test_ioctl_ioas_map(buffer, length, &iovas[i]); 927 } else { 928 test_ioctl_ioas_map((void *)(1UL << 31), length, 929 &iovas[i]); 930 } 931 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1))); 932 } 933 for (i = 0; i != 10; i++) 934 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1)); 935 936 /* Avoids a reserved region */ 937 ASSERT_EQ(0, 938 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED), 939 &test_cmd)); 940 for (i = 0; i != 10; i++) { 941 size_t length = PAGE_SIZE * (i + 1); 942 943 test_ioctl_ioas_map(buffer, length, &iovas[i]); 944 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1))); 945 EXPECT_EQ(false, 946 iovas[i] > test_cmd.add_reserved.start && 947 iovas[i] < 948 test_cmd.add_reserved.start + 949 test_cmd.add_reserved.length); 950 } 951 for (i = 0; i != 10; i++) 952 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1)); 953 954 /* Allowed region intersects with a reserved region */ 955 ranges[0].start = PAGE_SIZE; 956 ranges[0].last = PAGE_SIZE * 600; 957 EXPECT_ERRNO(EADDRINUSE, 958 ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd)); 959 960 /* Allocate from an allowed region */ 961 if (self->stdev_id) { 962 ranges[0].start = MOCK_APERTURE_START + PAGE_SIZE; 963 ranges[0].last = MOCK_APERTURE_START + PAGE_SIZE * 600 - 1; 964 } else { 965 ranges[0].start = PAGE_SIZE * 200; 966 ranges[0].last = PAGE_SIZE * 600 - 1; 967 } 968 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd)); 969 for (i = 0; i != 10; i++) { 970 size_t length = PAGE_SIZE * (i + 1); 971 972 test_ioctl_ioas_map(buffer, length, &iovas[i]); 973 EXPECT_EQ(0, iovas[i] % (1UL << (ffs(length) - 1))); 974 EXPECT_EQ(true, iovas[i] >= ranges[0].start); 975 EXPECT_EQ(true, iovas[i] <= ranges[0].last); 976 EXPECT_EQ(true, iovas[i] + length > ranges[0].start); 977 EXPECT_EQ(true, iovas[i] + length <= ranges[0].last + 1); 978 } 979 for (i = 0; i != 10; i++) 980 test_ioctl_ioas_unmap(iovas[i], PAGE_SIZE * (i + 1)); 981 } 982 983 /* https://lore.kernel.org/r/685af644.a00a0220.2e5631.0094.GAE@google.com */ 984 TEST_F(iommufd_ioas, reserved_overflow) 985 { 986 struct iommu_test_cmd test_cmd = { 987 .size = sizeof(test_cmd), 988 .op = IOMMU_TEST_OP_ADD_RESERVED, 989 .id = self->ioas_id, 990 .add_reserved.start = 6, 991 }; 992 unsigned int map_len; 993 __u64 iova; 994 995 if (PAGE_SIZE == 4096) { 996 test_cmd.add_reserved.length = 0xffffffffffff8001; 997 map_len = 0x5000; 998 } else { 999 test_cmd.add_reserved.length = 1000 0xffffffffffffffff - MOCK_PAGE_SIZE * 16; 1001 map_len = MOCK_PAGE_SIZE * 10; 1002 } 1003 1004 ASSERT_EQ(0, 1005 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED), 1006 &test_cmd)); 1007 test_err_ioctl_ioas_map(ENOSPC, buffer, map_len, &iova); 1008 } 1009 1010 TEST_F(iommufd_ioas, area_allowed) 1011 { 1012 struct iommu_test_cmd test_cmd = { 1013 .size = sizeof(test_cmd), 1014 .op = IOMMU_TEST_OP_ADD_RESERVED, 1015 .id = self->ioas_id, 1016 .add_reserved = { .start = PAGE_SIZE * 4, 1017 .length = PAGE_SIZE * 100 }, 1018 }; 1019 struct iommu_iova_range ranges[1] = {}; 1020 struct iommu_ioas_allow_iovas allow_cmd = { 1021 .size = sizeof(allow_cmd), 1022 .ioas_id = self->ioas_id, 1023 .num_iovas = 1, 1024 .allowed_iovas = (uintptr_t)ranges, 1025 }; 1026 1027 /* Reserved intersects an allowed */ 1028 allow_cmd.num_iovas = 1; 1029 ranges[0].start = self->base_iova; 1030 ranges[0].last = ranges[0].start + PAGE_SIZE * 600; 1031 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd)); 1032 test_cmd.add_reserved.start = ranges[0].start + PAGE_SIZE; 1033 test_cmd.add_reserved.length = PAGE_SIZE; 1034 EXPECT_ERRNO(EADDRINUSE, 1035 ioctl(self->fd, 1036 _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED), 1037 &test_cmd)); 1038 allow_cmd.num_iovas = 0; 1039 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd)); 1040 1041 /* Allowed intersects a reserved */ 1042 ASSERT_EQ(0, 1043 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED), 1044 &test_cmd)); 1045 allow_cmd.num_iovas = 1; 1046 ranges[0].start = self->base_iova; 1047 ranges[0].last = ranges[0].start + PAGE_SIZE * 600; 1048 EXPECT_ERRNO(EADDRINUSE, 1049 ioctl(self->fd, IOMMU_IOAS_ALLOW_IOVAS, &allow_cmd)); 1050 } 1051 1052 TEST_F(iommufd_ioas, copy_area) 1053 { 1054 struct iommu_ioas_copy copy_cmd = { 1055 .size = sizeof(copy_cmd), 1056 .flags = IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE, 1057 .dst_ioas_id = self->ioas_id, 1058 .src_ioas_id = self->ioas_id, 1059 .length = PAGE_SIZE, 1060 }; 1061 1062 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, self->base_iova); 1063 1064 /* Copy inside a single IOAS */ 1065 copy_cmd.src_iova = self->base_iova; 1066 copy_cmd.dst_iova = self->base_iova + PAGE_SIZE; 1067 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, ©_cmd)); 1068 1069 /* Copy between IOAS's */ 1070 copy_cmd.src_iova = self->base_iova; 1071 copy_cmd.dst_iova = 0; 1072 test_ioctl_ioas_alloc(©_cmd.dst_ioas_id); 1073 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, ©_cmd)); 1074 } 1075 1076 TEST_F(iommufd_ioas, iova_ranges) 1077 { 1078 struct iommu_test_cmd test_cmd = { 1079 .size = sizeof(test_cmd), 1080 .op = IOMMU_TEST_OP_ADD_RESERVED, 1081 .id = self->ioas_id, 1082 .add_reserved = { .start = PAGE_SIZE, .length = PAGE_SIZE }, 1083 }; 1084 struct iommu_iova_range *ranges = buffer; 1085 struct iommu_ioas_iova_ranges ranges_cmd = { 1086 .size = sizeof(ranges_cmd), 1087 .ioas_id = self->ioas_id, 1088 .num_iovas = BUFFER_SIZE / sizeof(*ranges), 1089 .allowed_iovas = (uintptr_t)ranges, 1090 }; 1091 1092 /* Range can be read */ 1093 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd)); 1094 EXPECT_EQ(1, ranges_cmd.num_iovas); 1095 if (!self->stdev_id) { 1096 EXPECT_EQ(0, ranges[0].start); 1097 EXPECT_EQ(SIZE_MAX, ranges[0].last); 1098 EXPECT_EQ(1, ranges_cmd.out_iova_alignment); 1099 } else { 1100 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start); 1101 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last); 1102 EXPECT_EQ(MOCK_PAGE_SIZE, ranges_cmd.out_iova_alignment); 1103 } 1104 1105 /* Buffer too small */ 1106 memset(ranges, 0, BUFFER_SIZE); 1107 ranges_cmd.num_iovas = 0; 1108 EXPECT_ERRNO(EMSGSIZE, 1109 ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd)); 1110 EXPECT_EQ(1, ranges_cmd.num_iovas); 1111 EXPECT_EQ(0, ranges[0].start); 1112 EXPECT_EQ(0, ranges[0].last); 1113 1114 /* 2 ranges */ 1115 ASSERT_EQ(0, 1116 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ADD_RESERVED), 1117 &test_cmd)); 1118 ranges_cmd.num_iovas = BUFFER_SIZE / sizeof(*ranges); 1119 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd)); 1120 if (!self->stdev_id) { 1121 EXPECT_EQ(2, ranges_cmd.num_iovas); 1122 EXPECT_EQ(0, ranges[0].start); 1123 EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last); 1124 EXPECT_EQ(PAGE_SIZE * 2, ranges[1].start); 1125 EXPECT_EQ(SIZE_MAX, ranges[1].last); 1126 } else { 1127 EXPECT_EQ(1, ranges_cmd.num_iovas); 1128 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start); 1129 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last); 1130 } 1131 1132 /* Buffer too small */ 1133 memset(ranges, 0, BUFFER_SIZE); 1134 ranges_cmd.num_iovas = 1; 1135 if (!self->stdev_id) { 1136 EXPECT_ERRNO(EMSGSIZE, ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, 1137 &ranges_cmd)); 1138 EXPECT_EQ(2, ranges_cmd.num_iovas); 1139 EXPECT_EQ(0, ranges[0].start); 1140 EXPECT_EQ(PAGE_SIZE - 1, ranges[0].last); 1141 } else { 1142 ASSERT_EQ(0, 1143 ioctl(self->fd, IOMMU_IOAS_IOVA_RANGES, &ranges_cmd)); 1144 EXPECT_EQ(1, ranges_cmd.num_iovas); 1145 EXPECT_EQ(MOCK_APERTURE_START, ranges[0].start); 1146 EXPECT_EQ(MOCK_APERTURE_LAST, ranges[0].last); 1147 } 1148 EXPECT_EQ(0, ranges[1].start); 1149 EXPECT_EQ(0, ranges[1].last); 1150 } 1151 1152 TEST_F(iommufd_ioas, access_domain_destory) 1153 { 1154 struct iommu_test_cmd access_cmd = { 1155 .size = sizeof(access_cmd), 1156 .op = IOMMU_TEST_OP_ACCESS_PAGES, 1157 .access_pages = { .iova = self->base_iova + PAGE_SIZE, 1158 .length = PAGE_SIZE}, 1159 }; 1160 size_t buf_size = 2 * HUGEPAGE_SIZE; 1161 uint8_t *buf; 1162 1163 buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, 1164 MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1, 1165 0); 1166 ASSERT_NE(MAP_FAILED, buf); 1167 test_ioctl_ioas_map_fixed(buf, buf_size, self->base_iova); 1168 1169 test_cmd_create_access(self->ioas_id, &access_cmd.id, 1170 MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES); 1171 access_cmd.access_pages.uptr = (uintptr_t)buf + PAGE_SIZE; 1172 ASSERT_EQ(0, 1173 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1174 &access_cmd)); 1175 1176 /* Causes a complicated unpin across a huge page boundary */ 1177 if (self->stdev_id) 1178 test_ioctl_destroy(self->stdev_id); 1179 1180 test_cmd_destroy_access_pages( 1181 access_cmd.id, access_cmd.access_pages.out_access_pages_id); 1182 test_cmd_destroy_access(access_cmd.id); 1183 ASSERT_EQ(0, munmap(buf, buf_size)); 1184 } 1185 1186 TEST_F(iommufd_ioas, access_pin) 1187 { 1188 struct iommu_test_cmd access_cmd = { 1189 .size = sizeof(access_cmd), 1190 .op = IOMMU_TEST_OP_ACCESS_PAGES, 1191 .access_pages = { .iova = MOCK_APERTURE_START, 1192 .length = BUFFER_SIZE, 1193 .uptr = (uintptr_t)buffer }, 1194 }; 1195 struct iommu_test_cmd check_map_cmd = { 1196 .size = sizeof(check_map_cmd), 1197 .op = IOMMU_TEST_OP_MD_CHECK_MAP, 1198 .check_map = { .iova = MOCK_APERTURE_START, 1199 .length = BUFFER_SIZE, 1200 .uptr = (uintptr_t)buffer }, 1201 }; 1202 uint32_t access_pages_id; 1203 unsigned int npages; 1204 1205 test_cmd_create_access(self->ioas_id, &access_cmd.id, 1206 MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES); 1207 1208 for (npages = 1; npages < BUFFER_SIZE / PAGE_SIZE; npages++) { 1209 uint32_t mock_stdev_id; 1210 uint32_t mock_hwpt_id; 1211 1212 access_cmd.access_pages.length = npages * PAGE_SIZE; 1213 1214 /* Single map/unmap */ 1215 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, 1216 MOCK_APERTURE_START); 1217 ASSERT_EQ(0, ioctl(self->fd, 1218 _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1219 &access_cmd)); 1220 test_cmd_destroy_access_pages( 1221 access_cmd.id, 1222 access_cmd.access_pages.out_access_pages_id); 1223 1224 /* Double user */ 1225 ASSERT_EQ(0, ioctl(self->fd, 1226 _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1227 &access_cmd)); 1228 access_pages_id = access_cmd.access_pages.out_access_pages_id; 1229 ASSERT_EQ(0, ioctl(self->fd, 1230 _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1231 &access_cmd)); 1232 test_cmd_destroy_access_pages( 1233 access_cmd.id, 1234 access_cmd.access_pages.out_access_pages_id); 1235 test_cmd_destroy_access_pages(access_cmd.id, access_pages_id); 1236 1237 /* Add/remove a domain with a user */ 1238 ASSERT_EQ(0, ioctl(self->fd, 1239 _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1240 &access_cmd)); 1241 test_cmd_mock_domain(self->ioas_id, &mock_stdev_id, 1242 &mock_hwpt_id, NULL); 1243 check_map_cmd.id = mock_hwpt_id; 1244 ASSERT_EQ(0, ioctl(self->fd, 1245 _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP), 1246 &check_map_cmd)); 1247 1248 test_ioctl_destroy(mock_stdev_id); 1249 test_cmd_destroy_access_pages( 1250 access_cmd.id, 1251 access_cmd.access_pages.out_access_pages_id); 1252 1253 test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE); 1254 } 1255 test_cmd_destroy_access(access_cmd.id); 1256 } 1257 1258 TEST_F(iommufd_ioas, access_pin_unmap) 1259 { 1260 struct iommu_test_cmd access_pages_cmd = { 1261 .size = sizeof(access_pages_cmd), 1262 .op = IOMMU_TEST_OP_ACCESS_PAGES, 1263 .access_pages = { .iova = MOCK_APERTURE_START, 1264 .length = BUFFER_SIZE, 1265 .uptr = (uintptr_t)buffer }, 1266 }; 1267 1268 test_cmd_create_access(self->ioas_id, &access_pages_cmd.id, 1269 MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES); 1270 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, MOCK_APERTURE_START); 1271 ASSERT_EQ(0, 1272 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1273 &access_pages_cmd)); 1274 1275 /* Trigger the unmap op */ 1276 test_ioctl_ioas_unmap(MOCK_APERTURE_START, BUFFER_SIZE); 1277 1278 /* kernel removed the item for us */ 1279 test_err_destroy_access_pages( 1280 ENOENT, access_pages_cmd.id, 1281 access_pages_cmd.access_pages.out_access_pages_id); 1282 } 1283 1284 static void check_access_rw(struct __test_metadata *_metadata, int fd, 1285 unsigned int access_id, uint64_t iova, 1286 unsigned int def_flags) 1287 { 1288 uint16_t tmp[32]; 1289 struct iommu_test_cmd access_cmd = { 1290 .size = sizeof(access_cmd), 1291 .op = IOMMU_TEST_OP_ACCESS_RW, 1292 .id = access_id, 1293 .access_rw = { .uptr = (uintptr_t)tmp }, 1294 }; 1295 uint16_t *buffer16 = buffer; 1296 unsigned int i; 1297 void *tmp2; 1298 1299 for (i = 0; i != BUFFER_SIZE / sizeof(*buffer16); i++) 1300 buffer16[i] = rand(); 1301 1302 for (access_cmd.access_rw.iova = iova + PAGE_SIZE - 50; 1303 access_cmd.access_rw.iova < iova + PAGE_SIZE + 50; 1304 access_cmd.access_rw.iova++) { 1305 for (access_cmd.access_rw.length = 1; 1306 access_cmd.access_rw.length < sizeof(tmp); 1307 access_cmd.access_rw.length++) { 1308 access_cmd.access_rw.flags = def_flags; 1309 ASSERT_EQ(0, ioctl(fd, 1310 _IOMMU_TEST_CMD( 1311 IOMMU_TEST_OP_ACCESS_RW), 1312 &access_cmd)); 1313 ASSERT_EQ(0, 1314 memcmp(buffer + (access_cmd.access_rw.iova - 1315 iova), 1316 tmp, access_cmd.access_rw.length)); 1317 1318 for (i = 0; i != ARRAY_SIZE(tmp); i++) 1319 tmp[i] = rand(); 1320 access_cmd.access_rw.flags = def_flags | 1321 MOCK_ACCESS_RW_WRITE; 1322 ASSERT_EQ(0, ioctl(fd, 1323 _IOMMU_TEST_CMD( 1324 IOMMU_TEST_OP_ACCESS_RW), 1325 &access_cmd)); 1326 ASSERT_EQ(0, 1327 memcmp(buffer + (access_cmd.access_rw.iova - 1328 iova), 1329 tmp, access_cmd.access_rw.length)); 1330 } 1331 } 1332 1333 /* Multi-page test */ 1334 tmp2 = malloc(BUFFER_SIZE); 1335 ASSERT_NE(NULL, tmp2); 1336 access_cmd.access_rw.iova = iova; 1337 access_cmd.access_rw.length = BUFFER_SIZE; 1338 access_cmd.access_rw.flags = def_flags; 1339 access_cmd.access_rw.uptr = (uintptr_t)tmp2; 1340 ASSERT_EQ(0, ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_RW), 1341 &access_cmd)); 1342 ASSERT_EQ(0, memcmp(buffer, tmp2, access_cmd.access_rw.length)); 1343 free(tmp2); 1344 } 1345 1346 TEST_F(iommufd_ioas, access_rw) 1347 { 1348 __u32 access_id; 1349 __u64 iova; 1350 1351 test_cmd_create_access(self->ioas_id, &access_id, 0); 1352 test_ioctl_ioas_map(buffer, BUFFER_SIZE, &iova); 1353 check_access_rw(_metadata, self->fd, access_id, iova, 0); 1354 check_access_rw(_metadata, self->fd, access_id, iova, 1355 MOCK_ACCESS_RW_SLOW_PATH); 1356 test_ioctl_ioas_unmap(iova, BUFFER_SIZE); 1357 test_cmd_destroy_access(access_id); 1358 } 1359 1360 TEST_F(iommufd_ioas, access_rw_unaligned) 1361 { 1362 __u32 access_id; 1363 __u64 iova; 1364 1365 test_cmd_create_access(self->ioas_id, &access_id, 0); 1366 1367 /* Unaligned pages */ 1368 iova = self->base_iova + MOCK_PAGE_SIZE; 1369 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, iova); 1370 check_access_rw(_metadata, self->fd, access_id, iova, 0); 1371 test_ioctl_ioas_unmap(iova, BUFFER_SIZE); 1372 test_cmd_destroy_access(access_id); 1373 } 1374 1375 TEST_F(iommufd_ioas, fork_gone) 1376 { 1377 __u32 access_id; 1378 pid_t child; 1379 1380 test_cmd_create_access(self->ioas_id, &access_id, 0); 1381 1382 /* Create a mapping with a different mm */ 1383 child = fork(); 1384 if (!child) { 1385 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, 1386 MOCK_APERTURE_START); 1387 exit(0); 1388 } 1389 ASSERT_NE(-1, child); 1390 ASSERT_EQ(child, waitpid(child, NULL, 0)); 1391 1392 if (self->stdev_id) { 1393 /* 1394 * If a domain already existed then everything was pinned within 1395 * the fork, so this copies from one domain to another. 1396 */ 1397 test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL); 1398 check_access_rw(_metadata, self->fd, access_id, 1399 MOCK_APERTURE_START, 0); 1400 1401 } else { 1402 /* 1403 * Otherwise we need to actually pin pages which can't happen 1404 * since the fork is gone. 1405 */ 1406 test_err_mock_domain(EFAULT, self->ioas_id, NULL, NULL); 1407 } 1408 1409 test_cmd_destroy_access(access_id); 1410 } 1411 1412 TEST_F(iommufd_ioas, fork_present) 1413 { 1414 __u32 access_id; 1415 int pipefds[2]; 1416 uint64_t tmp; 1417 pid_t child; 1418 int efd; 1419 1420 test_cmd_create_access(self->ioas_id, &access_id, 0); 1421 1422 ASSERT_EQ(0, pipe2(pipefds, O_CLOEXEC)); 1423 efd = eventfd(0, EFD_CLOEXEC); 1424 ASSERT_NE(-1, efd); 1425 1426 /* Create a mapping with a different mm */ 1427 child = fork(); 1428 if (!child) { 1429 __u64 iova; 1430 uint64_t one = 1; 1431 1432 close(pipefds[1]); 1433 test_ioctl_ioas_map_fixed(buffer, BUFFER_SIZE, 1434 MOCK_APERTURE_START); 1435 if (write(efd, &one, sizeof(one)) != sizeof(one)) 1436 exit(100); 1437 if (read(pipefds[0], &iova, 1) != 1) 1438 exit(100); 1439 exit(0); 1440 } 1441 close(pipefds[0]); 1442 ASSERT_NE(-1, child); 1443 ASSERT_EQ(8, read(efd, &tmp, sizeof(tmp))); 1444 1445 /* Read pages from the remote process */ 1446 test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL); 1447 check_access_rw(_metadata, self->fd, access_id, MOCK_APERTURE_START, 0); 1448 1449 ASSERT_EQ(0, close(pipefds[1])); 1450 ASSERT_EQ(child, waitpid(child, NULL, 0)); 1451 1452 test_cmd_destroy_access(access_id); 1453 } 1454 1455 TEST_F(iommufd_ioas, ioas_option_huge_pages) 1456 { 1457 struct iommu_option cmd = { 1458 .size = sizeof(cmd), 1459 .option_id = IOMMU_OPTION_HUGE_PAGES, 1460 .op = IOMMU_OPTION_OP_GET, 1461 .val64 = 3, 1462 .object_id = self->ioas_id, 1463 }; 1464 1465 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1466 ASSERT_EQ(1, cmd.val64); 1467 1468 cmd.op = IOMMU_OPTION_OP_SET; 1469 cmd.val64 = 0; 1470 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1471 1472 cmd.op = IOMMU_OPTION_OP_GET; 1473 cmd.val64 = 3; 1474 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1475 ASSERT_EQ(0, cmd.val64); 1476 1477 cmd.op = IOMMU_OPTION_OP_SET; 1478 cmd.val64 = 2; 1479 EXPECT_ERRNO(EINVAL, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1480 1481 cmd.op = IOMMU_OPTION_OP_SET; 1482 cmd.val64 = 1; 1483 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1484 } 1485 1486 TEST_F(iommufd_ioas, ioas_iova_alloc) 1487 { 1488 unsigned int length; 1489 __u64 iova; 1490 1491 for (length = 1; length != PAGE_SIZE * 2; length++) { 1492 if (variant->mock_domains && (length % MOCK_PAGE_SIZE)) { 1493 test_err_ioctl_ioas_map(EINVAL, buffer, length, &iova); 1494 } else { 1495 test_ioctl_ioas_map(buffer, length, &iova); 1496 test_ioctl_ioas_unmap(iova, length); 1497 } 1498 } 1499 } 1500 1501 TEST_F(iommufd_ioas, ioas_align_change) 1502 { 1503 struct iommu_option cmd = { 1504 .size = sizeof(cmd), 1505 .option_id = IOMMU_OPTION_HUGE_PAGES, 1506 .op = IOMMU_OPTION_OP_SET, 1507 .object_id = self->ioas_id, 1508 /* 0 means everything must be aligned to PAGE_SIZE */ 1509 .val64 = 0, 1510 }; 1511 1512 /* 1513 * We cannot upgrade the alignment using OPTION_HUGE_PAGES when a domain 1514 * and map are present. 1515 */ 1516 if (variant->mock_domains) 1517 return; 1518 1519 /* 1520 * We can upgrade to PAGE_SIZE alignment when things are aligned right 1521 */ 1522 test_ioctl_ioas_map_fixed(buffer, PAGE_SIZE, MOCK_APERTURE_START); 1523 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1524 1525 /* Misalignment is rejected at map time */ 1526 test_err_ioctl_ioas_map_fixed(EINVAL, buffer + MOCK_PAGE_SIZE, 1527 PAGE_SIZE, 1528 MOCK_APERTURE_START + PAGE_SIZE); 1529 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1530 1531 /* Reduce alignment */ 1532 cmd.val64 = 1; 1533 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1534 1535 /* Confirm misalignment is rejected during alignment upgrade */ 1536 test_ioctl_ioas_map_fixed(buffer + MOCK_PAGE_SIZE, PAGE_SIZE, 1537 MOCK_APERTURE_START + PAGE_SIZE); 1538 cmd.val64 = 0; 1539 EXPECT_ERRNO(EADDRINUSE, ioctl(self->fd, IOMMU_OPTION, &cmd)); 1540 1541 test_ioctl_ioas_unmap(MOCK_APERTURE_START + PAGE_SIZE, PAGE_SIZE); 1542 test_ioctl_ioas_unmap(MOCK_APERTURE_START, PAGE_SIZE); 1543 } 1544 1545 TEST_F(iommufd_ioas, copy_sweep) 1546 { 1547 struct iommu_ioas_copy copy_cmd = { 1548 .size = sizeof(copy_cmd), 1549 .flags = IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE, 1550 .src_ioas_id = self->ioas_id, 1551 .dst_iova = MOCK_APERTURE_START, 1552 .length = MOCK_PAGE_SIZE, 1553 }; 1554 unsigned int dst_ioas_id; 1555 uint64_t last_iova; 1556 uint64_t iova; 1557 1558 test_ioctl_ioas_alloc(&dst_ioas_id); 1559 copy_cmd.dst_ioas_id = dst_ioas_id; 1560 1561 if (variant->mock_domains) 1562 last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 1; 1563 else 1564 last_iova = MOCK_APERTURE_START + BUFFER_SIZE - 2; 1565 1566 test_ioctl_ioas_map_fixed(buffer, last_iova - MOCK_APERTURE_START + 1, 1567 MOCK_APERTURE_START); 1568 1569 for (iova = MOCK_APERTURE_START - PAGE_SIZE; iova <= last_iova; 1570 iova += 511) { 1571 copy_cmd.src_iova = iova; 1572 if (iova < MOCK_APERTURE_START || 1573 iova + copy_cmd.length - 1 > last_iova) { 1574 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_IOAS_COPY, 1575 ©_cmd)); 1576 } else { 1577 ASSERT_EQ(0, 1578 ioctl(self->fd, IOMMU_IOAS_COPY, ©_cmd)); 1579 test_ioctl_ioas_unmap_id(dst_ioas_id, copy_cmd.dst_iova, 1580 copy_cmd.length); 1581 } 1582 } 1583 1584 test_ioctl_destroy(dst_ioas_id); 1585 } 1586 1587 TEST_F(iommufd_ioas, dmabuf_simple) 1588 { 1589 size_t buf_size = PAGE_SIZE*4; 1590 __u64 iova; 1591 int dfd; 1592 1593 test_cmd_get_dmabuf(buf_size, &dfd); 1594 test_err_ioctl_ioas_map_file(EINVAL, dfd, 0, 0, &iova); 1595 test_err_ioctl_ioas_map_file(EINVAL, dfd, buf_size, buf_size, &iova); 1596 test_err_ioctl_ioas_map_file(EINVAL, dfd, 0, buf_size + 1, &iova); 1597 test_ioctl_ioas_map_file(dfd, 0, buf_size, &iova); 1598 1599 close(dfd); 1600 } 1601 1602 TEST_F(iommufd_ioas, dmabuf_revoke) 1603 { 1604 size_t buf_size = PAGE_SIZE*4; 1605 __u32 hwpt_id; 1606 __u64 iova; 1607 __u64 iova2; 1608 int dfd; 1609 1610 test_cmd_get_dmabuf(buf_size, &dfd); 1611 test_ioctl_ioas_map_file(dfd, 0, buf_size, &iova); 1612 test_cmd_revoke_dmabuf(dfd, true); 1613 1614 if (variant->mock_domains) 1615 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 0, 1616 &hwpt_id); 1617 1618 test_err_ioctl_ioas_map_file(ENODEV, dfd, 0, buf_size, &iova2); 1619 1620 test_cmd_revoke_dmabuf(dfd, false); 1621 test_ioctl_ioas_map_file(dfd, 0, buf_size, &iova2); 1622 1623 /* Restore the iova back */ 1624 test_ioctl_ioas_unmap(iova, buf_size); 1625 test_ioctl_ioas_map_fixed_file(dfd, 0, buf_size, iova); 1626 1627 close(dfd); 1628 } 1629 1630 FIXTURE(iommufd_mock_domain) 1631 { 1632 int fd; 1633 uint32_t ioas_id; 1634 uint32_t hwpt_id; 1635 uint32_t hwpt_ids[2]; 1636 uint32_t stdev_ids[2]; 1637 uint32_t idev_ids[2]; 1638 int mmap_flags; 1639 size_t mmap_buf_size; 1640 }; 1641 1642 FIXTURE_VARIANT(iommufd_mock_domain) 1643 { 1644 unsigned int mock_domains; 1645 bool hugepages; 1646 bool file; 1647 }; 1648 1649 FIXTURE_SETUP(iommufd_mock_domain) 1650 { 1651 unsigned int i; 1652 1653 self->fd = open("/dev/iommu", O_RDWR); 1654 ASSERT_NE(-1, self->fd); 1655 test_ioctl_ioas_alloc(&self->ioas_id); 1656 1657 ASSERT_GE(ARRAY_SIZE(self->hwpt_ids), variant->mock_domains); 1658 1659 for (i = 0; i != variant->mock_domains; i++) { 1660 test_cmd_mock_domain(self->ioas_id, &self->stdev_ids[i], 1661 &self->hwpt_ids[i], &self->idev_ids[i]); 1662 test_cmd_dev_check_cache_all(self->idev_ids[0], 1663 IOMMU_TEST_DEV_CACHE_DEFAULT); 1664 } 1665 self->hwpt_id = self->hwpt_ids[0]; 1666 1667 self->mmap_flags = MAP_SHARED | MAP_ANONYMOUS; 1668 self->mmap_buf_size = PAGE_SIZE * 8; 1669 if (variant->hugepages) { 1670 /* 1671 * MAP_POPULATE will cause the kernel to fail mmap if THPs are 1672 * not available. 1673 */ 1674 self->mmap_flags |= MAP_HUGETLB | MAP_POPULATE; 1675 self->mmap_buf_size = HUGEPAGE_SIZE * 2; 1676 } 1677 } 1678 1679 FIXTURE_TEARDOWN(iommufd_mock_domain) 1680 { 1681 teardown_iommufd(self->fd, _metadata); 1682 } 1683 1684 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain) 1685 { 1686 .mock_domains = 1, 1687 .hugepages = false, 1688 .file = false, 1689 }; 1690 1691 FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains) 1692 { 1693 .mock_domains = 2, 1694 .hugepages = false, 1695 .file = false, 1696 }; 1697 1698 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_hugepage) 1699 { 1700 .mock_domains = 1, 1701 .hugepages = true, 1702 .file = false, 1703 }; 1704 1705 FIXTURE_VARIANT_ADD(iommufd_mock_domain, two_domains_hugepage) 1706 { 1707 .mock_domains = 2, 1708 .hugepages = true, 1709 .file = false, 1710 }; 1711 1712 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_file) 1713 { 1714 .mock_domains = 1, 1715 .hugepages = false, 1716 .file = true, 1717 }; 1718 1719 FIXTURE_VARIANT_ADD(iommufd_mock_domain, one_domain_file_hugepage) 1720 { 1721 .mock_domains = 1, 1722 .hugepages = true, 1723 .file = true, 1724 }; 1725 1726 1727 /* Have the kernel check that the user pages made it to the iommu_domain */ 1728 #define check_mock_iova(_ptr, _iova, _length) \ 1729 ({ \ 1730 struct iommu_test_cmd check_map_cmd = { \ 1731 .size = sizeof(check_map_cmd), \ 1732 .op = IOMMU_TEST_OP_MD_CHECK_MAP, \ 1733 .id = self->hwpt_id, \ 1734 .check_map = { .iova = _iova, \ 1735 .length = _length, \ 1736 .uptr = (uintptr_t)(_ptr) }, \ 1737 }; \ 1738 ASSERT_EQ(0, \ 1739 ioctl(self->fd, \ 1740 _IOMMU_TEST_CMD(IOMMU_TEST_OP_MD_CHECK_MAP), \ 1741 &check_map_cmd)); \ 1742 if (self->hwpt_ids[1]) { \ 1743 check_map_cmd.id = self->hwpt_ids[1]; \ 1744 ASSERT_EQ(0, \ 1745 ioctl(self->fd, \ 1746 _IOMMU_TEST_CMD( \ 1747 IOMMU_TEST_OP_MD_CHECK_MAP), \ 1748 &check_map_cmd)); \ 1749 } \ 1750 }) 1751 1752 static void 1753 test_basic_mmap(struct __test_metadata *_metadata, 1754 struct _test_data_iommufd_mock_domain *self, 1755 const struct _fixture_variant_iommufd_mock_domain *variant) 1756 { 1757 size_t buf_size = self->mmap_buf_size; 1758 uint8_t *buf; 1759 __u64 iova; 1760 1761 /* Simple one page map */ 1762 test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova); 1763 check_mock_iova(buffer, iova, PAGE_SIZE); 1764 1765 buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, self->mmap_flags, -1, 1766 0); 1767 ASSERT_NE(MAP_FAILED, buf); 1768 1769 /* EFAULT half way through mapping */ 1770 ASSERT_EQ(0, munmap(buf + buf_size / 2, buf_size / 2)); 1771 test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova); 1772 1773 /* EFAULT on first page */ 1774 ASSERT_EQ(0, munmap(buf, buf_size / 2)); 1775 test_err_ioctl_ioas_map(EFAULT, buf, buf_size, &iova); 1776 } 1777 1778 static void 1779 test_basic_file(struct __test_metadata *_metadata, 1780 struct _test_data_iommufd_mock_domain *self, 1781 const struct _fixture_variant_iommufd_mock_domain *variant) 1782 { 1783 size_t buf_size = self->mmap_buf_size; 1784 uint8_t *buf; 1785 __u64 iova; 1786 int mfd_tmp; 1787 int prot = PROT_READ | PROT_WRITE; 1788 1789 /* Simple one page map */ 1790 test_ioctl_ioas_map_file(mfd, 0, PAGE_SIZE, &iova); 1791 check_mock_iova(mfd_buffer, iova, PAGE_SIZE); 1792 1793 buf = memfd_mmap(buf_size, prot, MAP_SHARED, &mfd_tmp); 1794 ASSERT_NE(MAP_FAILED, buf); 1795 1796 test_err_ioctl_ioas_map_file(EINVAL, mfd_tmp, 0, buf_size + 1, &iova); 1797 1798 ASSERT_EQ(0, ftruncate(mfd_tmp, 0)); 1799 test_err_ioctl_ioas_map_file(EINVAL, mfd_tmp, 0, buf_size, &iova); 1800 1801 close(mfd_tmp); 1802 } 1803 1804 TEST_F(iommufd_mock_domain, basic) 1805 { 1806 if (variant->file) 1807 test_basic_file(_metadata, self, variant); 1808 else 1809 test_basic_mmap(_metadata, self, variant); 1810 } 1811 1812 TEST_F(iommufd_mock_domain, ro_unshare) 1813 { 1814 uint8_t *buf; 1815 __u64 iova; 1816 int fd; 1817 1818 fd = open("/proc/self/exe", O_RDONLY); 1819 ASSERT_NE(-1, fd); 1820 1821 buf = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); 1822 ASSERT_NE(MAP_FAILED, buf); 1823 close(fd); 1824 1825 /* 1826 * There have been lots of changes to the "unshare" mechanism in 1827 * get_user_pages(), make sure it works right. The write to the page 1828 * after we map it for reading should not change the assigned PFN. 1829 */ 1830 ASSERT_EQ(0, 1831 _test_ioctl_ioas_map(self->fd, self->ioas_id, buf, PAGE_SIZE, 1832 &iova, IOMMU_IOAS_MAP_READABLE)); 1833 check_mock_iova(buf, iova, PAGE_SIZE); 1834 memset(buf, 1, PAGE_SIZE); 1835 check_mock_iova(buf, iova, PAGE_SIZE); 1836 ASSERT_EQ(0, munmap(buf, PAGE_SIZE)); 1837 } 1838 1839 TEST_F(iommufd_mock_domain, all_aligns) 1840 { 1841 size_t test_step = variant->hugepages ? (self->mmap_buf_size / 16) : 1842 MOCK_PAGE_SIZE; 1843 size_t buf_size = self->mmap_buf_size; 1844 unsigned int start; 1845 unsigned int end; 1846 uint8_t *buf; 1847 int prot = PROT_READ | PROT_WRITE; 1848 int mfd = -1; 1849 1850 if (variant->file) 1851 buf = memfd_mmap(buf_size, prot, MAP_SHARED, &mfd); 1852 else 1853 buf = mmap(0, buf_size, prot, self->mmap_flags, -1, 0); 1854 ASSERT_NE(MAP_FAILED, buf); 1855 if (variant->file) 1856 ASSERT_GT(mfd, 0); 1857 check_refs(buf, buf_size, 0); 1858 1859 /* 1860 * Map every combination of page size and alignment within a big region, 1861 * less for hugepage case as it takes so long to finish. 1862 */ 1863 for (start = 0; start < buf_size; start += test_step) { 1864 if (variant->hugepages) 1865 end = buf_size; 1866 else 1867 end = start + MOCK_PAGE_SIZE; 1868 for (; end < buf_size; end += MOCK_PAGE_SIZE) { 1869 size_t length = end - start; 1870 __u64 iova; 1871 1872 if (variant->file) { 1873 test_ioctl_ioas_map_file(mfd, start, length, 1874 &iova); 1875 } else { 1876 test_ioctl_ioas_map(buf + start, length, &iova); 1877 } 1878 check_mock_iova(buf + start, iova, length); 1879 check_refs(buf + start / PAGE_SIZE * PAGE_SIZE, 1880 end / PAGE_SIZE * PAGE_SIZE - 1881 start / PAGE_SIZE * PAGE_SIZE, 1882 1); 1883 1884 test_ioctl_ioas_unmap(iova, length); 1885 } 1886 } 1887 check_refs(buf, buf_size, 0); 1888 ASSERT_EQ(0, munmap(buf, buf_size)); 1889 if (variant->file) 1890 close(mfd); 1891 } 1892 1893 TEST_F(iommufd_mock_domain, all_aligns_copy) 1894 { 1895 size_t test_step = variant->hugepages ? self->mmap_buf_size / 16 : 1896 MOCK_PAGE_SIZE; 1897 size_t buf_size = self->mmap_buf_size; 1898 unsigned int start; 1899 unsigned int end; 1900 uint8_t *buf; 1901 int prot = PROT_READ | PROT_WRITE; 1902 int mfd = -1; 1903 1904 if (variant->file) 1905 buf = memfd_mmap(buf_size, prot, MAP_SHARED, &mfd); 1906 else 1907 buf = mmap(0, buf_size, prot, self->mmap_flags, -1, 0); 1908 ASSERT_NE(MAP_FAILED, buf); 1909 if (variant->file) 1910 ASSERT_GT(mfd, 0); 1911 check_refs(buf, buf_size, 0); 1912 1913 /* 1914 * Map every combination of page size and alignment within a big region, 1915 * less for hugepage case as it takes so long to finish. 1916 */ 1917 for (start = 0; start < buf_size; start += test_step) { 1918 if (variant->hugepages) 1919 end = buf_size; 1920 else 1921 end = start + MOCK_PAGE_SIZE; 1922 for (; end < buf_size; end += MOCK_PAGE_SIZE) { 1923 size_t length = end - start; 1924 unsigned int old_id; 1925 uint32_t mock_stdev_id; 1926 __u64 iova; 1927 1928 if (variant->file) { 1929 test_ioctl_ioas_map_file(mfd, start, length, 1930 &iova); 1931 } else { 1932 test_ioctl_ioas_map(buf + start, length, &iova); 1933 } 1934 1935 /* Add and destroy a domain while the area exists */ 1936 old_id = self->hwpt_ids[1]; 1937 test_cmd_mock_domain(self->ioas_id, &mock_stdev_id, 1938 &self->hwpt_ids[1], NULL); 1939 1940 check_mock_iova(buf + start, iova, length); 1941 check_refs(buf + start / PAGE_SIZE * PAGE_SIZE, 1942 end / PAGE_SIZE * PAGE_SIZE - 1943 start / PAGE_SIZE * PAGE_SIZE, 1944 1); 1945 1946 test_ioctl_destroy(mock_stdev_id); 1947 self->hwpt_ids[1] = old_id; 1948 1949 test_ioctl_ioas_unmap(iova, length); 1950 } 1951 } 1952 check_refs(buf, buf_size, 0); 1953 ASSERT_EQ(0, munmap(buf, buf_size)); 1954 if (variant->file) 1955 close(mfd); 1956 } 1957 1958 TEST_F(iommufd_mock_domain, user_copy) 1959 { 1960 void *buf = variant->file ? mfd_buffer : buffer; 1961 struct iommu_test_cmd access_cmd = { 1962 .size = sizeof(access_cmd), 1963 .op = IOMMU_TEST_OP_ACCESS_PAGES, 1964 .access_pages = { .length = BUFFER_SIZE, 1965 .uptr = (uintptr_t)buf }, 1966 }; 1967 struct iommu_ioas_copy copy_cmd = { 1968 .size = sizeof(copy_cmd), 1969 .flags = IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE, 1970 .dst_ioas_id = self->ioas_id, 1971 .dst_iova = MOCK_APERTURE_START, 1972 .length = BUFFER_SIZE, 1973 }; 1974 struct iommu_ioas_unmap unmap_cmd = { 1975 .size = sizeof(unmap_cmd), 1976 .ioas_id = self->ioas_id, 1977 .iova = MOCK_APERTURE_START, 1978 .length = BUFFER_SIZE, 1979 }; 1980 unsigned int new_ioas_id, ioas_id; 1981 1982 /* Pin the pages in an IOAS with no domains then copy to an IOAS with domains */ 1983 test_ioctl_ioas_alloc(&ioas_id); 1984 if (variant->file) { 1985 test_ioctl_ioas_map_id_file(ioas_id, mfd, 0, BUFFER_SIZE, 1986 ©_cmd.src_iova); 1987 } else { 1988 test_ioctl_ioas_map_id(ioas_id, buf, BUFFER_SIZE, 1989 ©_cmd.src_iova); 1990 } 1991 test_cmd_create_access(ioas_id, &access_cmd.id, 1992 MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES); 1993 1994 access_cmd.access_pages.iova = copy_cmd.src_iova; 1995 ASSERT_EQ(0, 1996 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 1997 &access_cmd)); 1998 copy_cmd.src_ioas_id = ioas_id; 1999 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, ©_cmd)); 2000 check_mock_iova(buf, MOCK_APERTURE_START, BUFFER_SIZE); 2001 2002 /* Now replace the ioas with a new one */ 2003 test_ioctl_ioas_alloc(&new_ioas_id); 2004 if (variant->file) { 2005 test_ioctl_ioas_map_id_file(new_ioas_id, mfd, 0, BUFFER_SIZE, 2006 ©_cmd.src_iova); 2007 } else { 2008 test_ioctl_ioas_map_id(new_ioas_id, buf, BUFFER_SIZE, 2009 ©_cmd.src_iova); 2010 } 2011 test_cmd_access_replace_ioas(access_cmd.id, new_ioas_id); 2012 2013 /* Destroy the old ioas and cleanup copied mapping */ 2014 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_UNMAP, &unmap_cmd)); 2015 test_ioctl_destroy(ioas_id); 2016 2017 /* Then run the same test again with the new ioas */ 2018 access_cmd.access_pages.iova = copy_cmd.src_iova; 2019 ASSERT_EQ(0, 2020 ioctl(self->fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_ACCESS_PAGES), 2021 &access_cmd)); 2022 copy_cmd.src_ioas_id = new_ioas_id; 2023 ASSERT_EQ(0, ioctl(self->fd, IOMMU_IOAS_COPY, ©_cmd)); 2024 check_mock_iova(buf, MOCK_APERTURE_START, BUFFER_SIZE); 2025 2026 test_cmd_destroy_access_pages( 2027 access_cmd.id, access_cmd.access_pages.out_access_pages_id); 2028 test_cmd_destroy_access(access_cmd.id); 2029 2030 test_ioctl_destroy(new_ioas_id); 2031 } 2032 2033 TEST_F(iommufd_mock_domain, replace) 2034 { 2035 uint32_t ioas_id; 2036 2037 test_ioctl_ioas_alloc(&ioas_id); 2038 2039 test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id); 2040 2041 /* 2042 * Replacing the IOAS causes the prior HWPT to be deallocated, thus we 2043 * should get enoent when we try to use it. 2044 */ 2045 if (variant->mock_domains == 1) 2046 test_err_mock_domain_replace(ENOENT, self->stdev_ids[0], 2047 self->hwpt_ids[0]); 2048 2049 test_cmd_mock_domain_replace(self->stdev_ids[0], ioas_id); 2050 if (variant->mock_domains >= 2) { 2051 test_cmd_mock_domain_replace(self->stdev_ids[0], 2052 self->hwpt_ids[1]); 2053 test_cmd_mock_domain_replace(self->stdev_ids[0], 2054 self->hwpt_ids[1]); 2055 test_cmd_mock_domain_replace(self->stdev_ids[0], 2056 self->hwpt_ids[0]); 2057 } 2058 2059 test_cmd_mock_domain_replace(self->stdev_ids[0], self->ioas_id); 2060 test_ioctl_destroy(ioas_id); 2061 } 2062 2063 TEST_F(iommufd_mock_domain, alloc_hwpt) 2064 { 2065 int i; 2066 2067 for (i = 0; i != variant->mock_domains; i++) { 2068 uint32_t hwpt_id[2]; 2069 uint32_t stddev_id; 2070 2071 test_err_hwpt_alloc(EOPNOTSUPP, 2072 self->idev_ids[i], self->ioas_id, 2073 ~IOMMU_HWPT_ALLOC_NEST_PARENT, &hwpt_id[0]); 2074 test_cmd_hwpt_alloc(self->idev_ids[i], self->ioas_id, 2075 0, &hwpt_id[0]); 2076 test_cmd_hwpt_alloc(self->idev_ids[i], self->ioas_id, 2077 IOMMU_HWPT_ALLOC_NEST_PARENT, &hwpt_id[1]); 2078 2079 /* Do a hw_pagetable rotation test */ 2080 test_cmd_mock_domain_replace(self->stdev_ids[i], hwpt_id[0]); 2081 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, hwpt_id[0])); 2082 test_cmd_mock_domain_replace(self->stdev_ids[i], hwpt_id[1]); 2083 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, hwpt_id[1])); 2084 test_cmd_mock_domain_replace(self->stdev_ids[i], self->ioas_id); 2085 test_ioctl_destroy(hwpt_id[1]); 2086 2087 test_cmd_mock_domain(hwpt_id[0], &stddev_id, NULL, NULL); 2088 test_ioctl_destroy(stddev_id); 2089 test_ioctl_destroy(hwpt_id[0]); 2090 } 2091 } 2092 2093 FIXTURE(iommufd_dirty_tracking) 2094 { 2095 int fd; 2096 uint32_t ioas_id; 2097 uint32_t hwpt_id; 2098 uint32_t stdev_id; 2099 uint32_t idev_id; 2100 unsigned long page_size; 2101 unsigned long bitmap_size; 2102 void *bitmap; 2103 void *buffer; 2104 }; 2105 2106 FIXTURE_VARIANT(iommufd_dirty_tracking) 2107 { 2108 unsigned long buffer_size; 2109 bool hugepages; 2110 }; 2111 2112 FIXTURE_SETUP(iommufd_dirty_tracking) 2113 { 2114 struct iommu_option cmd = { 2115 .size = sizeof(cmd), 2116 .option_id = IOMMU_OPTION_HUGE_PAGES, 2117 .op = IOMMU_OPTION_OP_SET, 2118 .val64 = 0, 2119 }; 2120 size_t mmap_buffer_size; 2121 unsigned long size; 2122 int mmap_flags; 2123 void *vrc; 2124 int rc; 2125 2126 if (variant->buffer_size < MOCK_PAGE_SIZE) { 2127 SKIP(return, 2128 "Skipping buffer_size=%lu, less than MOCK_PAGE_SIZE=%u", 2129 variant->buffer_size, MOCK_PAGE_SIZE); 2130 } 2131 2132 self->fd = open("/dev/iommu", O_RDWR); 2133 ASSERT_NE(-1, self->fd); 2134 2135 mmap_flags = MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED; 2136 mmap_buffer_size = variant->buffer_size; 2137 if (variant->hugepages) { 2138 /* 2139 * MAP_POPULATE will cause the kernel to fail mmap if THPs are 2140 * not available. 2141 */ 2142 mmap_flags |= MAP_HUGETLB | MAP_POPULATE; 2143 2144 /* 2145 * Allocation must be aligned to the HUGEPAGE_SIZE, because the 2146 * following mmap() will automatically align the length to be a 2147 * multiple of the underlying huge page size. Failing to do the 2148 * same at this allocation will result in a memory overwrite by 2149 * the mmap(). 2150 */ 2151 if (mmap_buffer_size < HUGEPAGE_SIZE) 2152 mmap_buffer_size = HUGEPAGE_SIZE; 2153 } 2154 2155 rc = posix_memalign(&self->buffer, HUGEPAGE_SIZE, mmap_buffer_size); 2156 if (rc || !self->buffer) { 2157 SKIP(return, "Skipping buffer_size=%lu due to errno=%d", 2158 mmap_buffer_size, rc); 2159 } 2160 assert((uintptr_t)self->buffer % HUGEPAGE_SIZE == 0); 2161 vrc = mmap(self->buffer, mmap_buffer_size, PROT_READ | PROT_WRITE, 2162 mmap_flags, -1, 0); 2163 assert(vrc == self->buffer); 2164 2165 self->page_size = MOCK_PAGE_SIZE; 2166 self->bitmap_size = variant->buffer_size / self->page_size; 2167 2168 /* Provision with an extra (PAGE_SIZE) for the unaligned case */ 2169 size = DIV_ROUND_UP(self->bitmap_size, BITS_PER_BYTE); 2170 rc = posix_memalign(&self->bitmap, PAGE_SIZE, size + PAGE_SIZE); 2171 assert(!rc); 2172 assert(self->bitmap); 2173 assert((uintptr_t)self->bitmap % PAGE_SIZE == 0); 2174 2175 test_ioctl_ioas_alloc(&self->ioas_id); 2176 2177 /* 2178 * For dirty testing it is important that the page size fed into 2179 * the iommu page tables matches the size the dirty logic 2180 * expects, or set_dirty can touch too much stuff. 2181 */ 2182 cmd.object_id = self->ioas_id; 2183 if (!variant->hugepages) 2184 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 2185 2186 test_cmd_mock_domain(self->ioas_id, &self->stdev_id, &self->hwpt_id, 2187 &self->idev_id); 2188 } 2189 2190 FIXTURE_TEARDOWN(iommufd_dirty_tracking) 2191 { 2192 free(self->buffer); 2193 free(self->bitmap); 2194 teardown_iommufd(self->fd, _metadata); 2195 } 2196 2197 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty8k) 2198 { 2199 /* half of an u8 index bitmap */ 2200 .buffer_size = 8UL * 1024UL, 2201 }; 2202 2203 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty16k) 2204 { 2205 /* one u8 index bitmap */ 2206 .buffer_size = 16UL * 1024UL, 2207 }; 2208 2209 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64k) 2210 { 2211 /* one u32 index bitmap */ 2212 .buffer_size = 64UL * 1024UL, 2213 }; 2214 2215 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128k) 2216 { 2217 /* one u64 index bitmap */ 2218 .buffer_size = 128UL * 1024UL, 2219 }; 2220 2221 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty320k) 2222 { 2223 /* two u64 index and trailing end bitmap */ 2224 .buffer_size = 320UL * 1024UL, 2225 }; 2226 2227 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M) 2228 { 2229 /* 4K bitmap (64M IOVA range) */ 2230 .buffer_size = 64UL * 1024UL * 1024UL, 2231 }; 2232 2233 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty64M_huge) 2234 { 2235 /* 4K bitmap (64M IOVA range) */ 2236 .buffer_size = 64UL * 1024UL * 1024UL, 2237 .hugepages = true, 2238 }; 2239 2240 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M) 2241 { 2242 /* 8K bitmap (128M IOVA range) */ 2243 .buffer_size = 128UL * 1024UL * 1024UL, 2244 }; 2245 2246 FIXTURE_VARIANT_ADD(iommufd_dirty_tracking, domain_dirty128M_huge) 2247 { 2248 /* 8K bitmap (128M IOVA range) */ 2249 .buffer_size = 128UL * 1024UL * 1024UL, 2250 .hugepages = true, 2251 }; 2252 2253 TEST_F(iommufd_dirty_tracking, enforce_dirty) 2254 { 2255 uint32_t ioas_id, stddev_id, idev_id; 2256 uint32_t hwpt_id, _hwpt_id; 2257 uint32_t dev_flags; 2258 2259 /* Regular case */ 2260 dev_flags = MOCK_FLAGS_DEVICE_NO_DIRTY; 2261 test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, 2262 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id); 2263 test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL); 2264 test_err_mock_domain_flags(EINVAL, hwpt_id, dev_flags, &stddev_id, 2265 NULL); 2266 test_ioctl_destroy(stddev_id); 2267 test_ioctl_destroy(hwpt_id); 2268 2269 /* IOMMU device does not support dirty tracking */ 2270 test_ioctl_ioas_alloc(&ioas_id); 2271 test_cmd_mock_domain_flags(ioas_id, dev_flags, &stddev_id, &_hwpt_id, 2272 &idev_id); 2273 test_err_hwpt_alloc(EOPNOTSUPP, idev_id, ioas_id, 2274 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id); 2275 test_ioctl_destroy(stddev_id); 2276 } 2277 2278 TEST_F(iommufd_dirty_tracking, set_dirty_tracking) 2279 { 2280 uint32_t stddev_id; 2281 uint32_t hwpt_id; 2282 2283 test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, 2284 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id); 2285 test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL); 2286 test_cmd_set_dirty_tracking(hwpt_id, true); 2287 test_cmd_set_dirty_tracking(hwpt_id, false); 2288 2289 test_ioctl_destroy(stddev_id); 2290 test_ioctl_destroy(hwpt_id); 2291 } 2292 2293 TEST_F(iommufd_dirty_tracking, pasid_set_dirty_tracking) 2294 { 2295 uint32_t stddev_id, ioas_id, hwpt_id, pasid = 100; 2296 uint32_t dev_flags = MOCK_FLAGS_DEVICE_PASID; 2297 2298 /* Regular case */ 2299 test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, 2300 IOMMU_HWPT_ALLOC_PASID | IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2301 &hwpt_id); 2302 test_cmd_mock_domain_flags(hwpt_id, dev_flags, &stddev_id, NULL, NULL); 2303 ASSERT_EQ(0, _test_cmd_pasid_attach(self->fd, stddev_id, pasid, hwpt_id)); 2304 test_cmd_set_dirty_tracking(hwpt_id, true); 2305 test_cmd_set_dirty_tracking(hwpt_id, false); 2306 ASSERT_EQ(0, _test_cmd_pasid_detach(self->fd, stddev_id, pasid)); 2307 2308 test_ioctl_destroy(stddev_id); 2309 2310 /* IOMMU device does not support dirty tracking */ 2311 dev_flags |= MOCK_FLAGS_DEVICE_NO_DIRTY; 2312 test_ioctl_ioas_alloc(&ioas_id); 2313 test_cmd_mock_domain_flags(ioas_id, dev_flags, &stddev_id, NULL, NULL); 2314 EXPECT_ERRNO(EINVAL, _test_cmd_pasid_attach(self->fd, stddev_id, pasid, hwpt_id)); 2315 2316 test_ioctl_destroy(stddev_id); 2317 test_ioctl_destroy(hwpt_id); 2318 } 2319 2320 TEST_F(iommufd_dirty_tracking, device_dirty_capability) 2321 { 2322 uint32_t caps = 0; 2323 uint32_t stddev_id; 2324 uint32_t hwpt_id; 2325 2326 test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, 0, &hwpt_id); 2327 test_cmd_mock_domain(hwpt_id, &stddev_id, NULL, NULL); 2328 test_cmd_get_hw_capabilities(self->idev_id, caps); 2329 ASSERT_EQ(IOMMU_HW_CAP_DIRTY_TRACKING, 2330 caps & IOMMU_HW_CAP_DIRTY_TRACKING); 2331 2332 test_ioctl_destroy(stddev_id); 2333 test_ioctl_destroy(hwpt_id); 2334 } 2335 2336 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap) 2337 { 2338 uint32_t page_size = MOCK_PAGE_SIZE; 2339 uint32_t ioas_id = self->ioas_id; 2340 uint32_t hwpt_id; 2341 2342 if (variant->hugepages) 2343 page_size = MOCK_HUGE_PAGE_SIZE; 2344 2345 test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer, 2346 variant->buffer_size, MOCK_APERTURE_START); 2347 2348 if (variant->hugepages) 2349 test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2350 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2351 MOCK_IOMMUPT_HUGE, &hwpt_id); 2352 else 2353 test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2354 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2355 MOCK_IOMMUPT_DEFAULT, &hwpt_id); 2356 2357 test_cmd_set_dirty_tracking(hwpt_id, true); 2358 2359 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2360 MOCK_APERTURE_START, self->page_size, page_size, 2361 self->bitmap, self->bitmap_size, 0, _metadata); 2362 2363 /* PAGE_SIZE unaligned bitmap */ 2364 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2365 MOCK_APERTURE_START, self->page_size, page_size, 2366 self->bitmap + MOCK_PAGE_SIZE, 2367 self->bitmap_size, 0, _metadata); 2368 2369 /* u64 unaligned bitmap */ 2370 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2371 MOCK_APERTURE_START, self->page_size, page_size, 2372 self->bitmap + 0xff1, self->bitmap_size, 0, 2373 _metadata); 2374 2375 test_ioctl_destroy(hwpt_id); 2376 } 2377 2378 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap_no_clear) 2379 { 2380 uint32_t page_size = MOCK_PAGE_SIZE; 2381 uint32_t ioas_id = self->ioas_id; 2382 uint32_t hwpt_id; 2383 2384 if (variant->hugepages) 2385 page_size = MOCK_HUGE_PAGE_SIZE; 2386 2387 test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer, 2388 variant->buffer_size, MOCK_APERTURE_START); 2389 2390 2391 if (variant->hugepages) 2392 test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2393 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2394 MOCK_IOMMUPT_HUGE, &hwpt_id); 2395 else 2396 test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2397 IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2398 MOCK_IOMMUPT_DEFAULT, &hwpt_id); 2399 2400 test_cmd_set_dirty_tracking(hwpt_id, true); 2401 2402 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2403 MOCK_APERTURE_START, self->page_size, page_size, 2404 self->bitmap, self->bitmap_size, 2405 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR, 2406 _metadata); 2407 2408 /* Unaligned bitmap */ 2409 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2410 MOCK_APERTURE_START, self->page_size, page_size, 2411 self->bitmap + MOCK_PAGE_SIZE, 2412 self->bitmap_size, 2413 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR, 2414 _metadata); 2415 2416 /* u64 unaligned bitmap */ 2417 test_mock_dirty_bitmaps(hwpt_id, variant->buffer_size, 2418 MOCK_APERTURE_START, self->page_size, page_size, 2419 self->bitmap + 0xff1, self->bitmap_size, 2420 IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR, 2421 _metadata); 2422 2423 test_ioctl_destroy(hwpt_id); 2424 } 2425 2426 /* VFIO compatibility IOCTLs */ 2427 2428 TEST_F(iommufd, simple_ioctls) 2429 { 2430 ASSERT_EQ(VFIO_API_VERSION, ioctl(self->fd, VFIO_GET_API_VERSION)); 2431 ASSERT_EQ(1, ioctl(self->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)); 2432 } 2433 2434 TEST_F(iommufd, unmap_cmd) 2435 { 2436 struct vfio_iommu_type1_dma_unmap unmap_cmd = { 2437 .iova = MOCK_APERTURE_START, 2438 .size = PAGE_SIZE, 2439 }; 2440 2441 unmap_cmd.argsz = 1; 2442 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2443 2444 unmap_cmd.argsz = sizeof(unmap_cmd); 2445 unmap_cmd.flags = 1 << 31; 2446 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2447 2448 unmap_cmd.flags = 0; 2449 EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2450 } 2451 2452 TEST_F(iommufd, map_cmd) 2453 { 2454 struct vfio_iommu_type1_dma_map map_cmd = { 2455 .iova = MOCK_APERTURE_START, 2456 .size = PAGE_SIZE, 2457 .vaddr = (__u64)buffer, 2458 }; 2459 2460 map_cmd.argsz = 1; 2461 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2462 2463 map_cmd.argsz = sizeof(map_cmd); 2464 map_cmd.flags = 1 << 31; 2465 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2466 2467 /* Requires a domain to be attached */ 2468 map_cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; 2469 EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2470 } 2471 2472 TEST_F(iommufd, info_cmd) 2473 { 2474 struct vfio_iommu_type1_info info_cmd = {}; 2475 2476 /* Invalid argsz */ 2477 info_cmd.argsz = 1; 2478 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd)); 2479 2480 info_cmd.argsz = sizeof(info_cmd); 2481 EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_IOMMU_GET_INFO, &info_cmd)); 2482 } 2483 2484 TEST_F(iommufd, set_iommu_cmd) 2485 { 2486 /* Requires a domain to be attached */ 2487 EXPECT_ERRNO(ENODEV, 2488 ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU)); 2489 EXPECT_ERRNO(ENODEV, ioctl(self->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)); 2490 } 2491 2492 TEST_F(iommufd, vfio_ioas) 2493 { 2494 struct iommu_vfio_ioas vfio_ioas_cmd = { 2495 .size = sizeof(vfio_ioas_cmd), 2496 .op = IOMMU_VFIO_IOAS_GET, 2497 }; 2498 __u32 ioas_id; 2499 2500 /* ENODEV if there is no compat ioas */ 2501 EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2502 2503 /* Invalid id for set */ 2504 vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_SET; 2505 EXPECT_ERRNO(ENOENT, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2506 2507 /* Valid id for set*/ 2508 test_ioctl_ioas_alloc(&ioas_id); 2509 vfio_ioas_cmd.ioas_id = ioas_id; 2510 ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2511 2512 /* Same id comes back from get */ 2513 vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET; 2514 ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2515 ASSERT_EQ(ioas_id, vfio_ioas_cmd.ioas_id); 2516 2517 /* Clear works */ 2518 vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_CLEAR; 2519 ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2520 vfio_ioas_cmd.op = IOMMU_VFIO_IOAS_GET; 2521 EXPECT_ERRNO(ENODEV, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2522 } 2523 2524 FIXTURE(vfio_compat_mock_domain) 2525 { 2526 int fd; 2527 uint32_t ioas_id; 2528 }; 2529 2530 FIXTURE_VARIANT(vfio_compat_mock_domain) 2531 { 2532 unsigned int version; 2533 }; 2534 2535 FIXTURE_SETUP(vfio_compat_mock_domain) 2536 { 2537 struct iommu_vfio_ioas vfio_ioas_cmd = { 2538 .size = sizeof(vfio_ioas_cmd), 2539 .op = IOMMU_VFIO_IOAS_SET, 2540 }; 2541 2542 self->fd = open("/dev/iommu", O_RDWR); 2543 ASSERT_NE(-1, self->fd); 2544 2545 /* Create what VFIO would consider a group */ 2546 test_ioctl_ioas_alloc(&self->ioas_id); 2547 test_cmd_mock_domain(self->ioas_id, NULL, NULL, NULL); 2548 2549 /* Attach it to the vfio compat */ 2550 vfio_ioas_cmd.ioas_id = self->ioas_id; 2551 ASSERT_EQ(0, ioctl(self->fd, IOMMU_VFIO_IOAS, &vfio_ioas_cmd)); 2552 ASSERT_EQ(0, ioctl(self->fd, VFIO_SET_IOMMU, variant->version)); 2553 } 2554 2555 FIXTURE_TEARDOWN(vfio_compat_mock_domain) 2556 { 2557 teardown_iommufd(self->fd, _metadata); 2558 } 2559 2560 FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v2) 2561 { 2562 .version = VFIO_TYPE1v2_IOMMU, 2563 }; 2564 2565 FIXTURE_VARIANT_ADD(vfio_compat_mock_domain, Ver1v0) 2566 { 2567 .version = VFIO_TYPE1_IOMMU, 2568 }; 2569 2570 TEST_F(vfio_compat_mock_domain, simple_close) 2571 { 2572 } 2573 2574 TEST_F(vfio_compat_mock_domain, option_huge_pages) 2575 { 2576 struct iommu_option cmd = { 2577 .size = sizeof(cmd), 2578 .option_id = IOMMU_OPTION_HUGE_PAGES, 2579 .op = IOMMU_OPTION_OP_GET, 2580 .val64 = 3, 2581 .object_id = self->ioas_id, 2582 }; 2583 2584 ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 2585 if (variant->version == VFIO_TYPE1_IOMMU) { 2586 ASSERT_EQ(0, cmd.val64); 2587 } else { 2588 ASSERT_EQ(1, cmd.val64); 2589 } 2590 } 2591 2592 /* 2593 * Execute an ioctl command stored in buffer and check that the result does not 2594 * overflow memory. 2595 */ 2596 static bool is_filled(const void *buf, uint8_t c, size_t len) 2597 { 2598 const uint8_t *cbuf = buf; 2599 2600 for (; len; cbuf++, len--) 2601 if (*cbuf != c) 2602 return false; 2603 return true; 2604 } 2605 2606 #define ioctl_check_buf(fd, cmd) \ 2607 ({ \ 2608 size_t _cmd_len = *(__u32 *)buffer; \ 2609 \ 2610 memset(buffer + _cmd_len, 0xAA, BUFFER_SIZE - _cmd_len); \ 2611 ASSERT_EQ(0, ioctl(fd, cmd, buffer)); \ 2612 ASSERT_EQ(true, is_filled(buffer + _cmd_len, 0xAA, \ 2613 BUFFER_SIZE - _cmd_len)); \ 2614 }) 2615 2616 static void check_vfio_info_cap_chain(struct __test_metadata *_metadata, 2617 struct vfio_iommu_type1_info *info_cmd) 2618 { 2619 const struct vfio_info_cap_header *cap; 2620 2621 ASSERT_GE(info_cmd->argsz, info_cmd->cap_offset + sizeof(*cap)); 2622 cap = buffer + info_cmd->cap_offset; 2623 while (true) { 2624 size_t cap_size; 2625 2626 if (cap->next) 2627 cap_size = (buffer + cap->next) - (void *)cap; 2628 else 2629 cap_size = (buffer + info_cmd->argsz) - (void *)cap; 2630 2631 switch (cap->id) { 2632 case VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE: { 2633 struct vfio_iommu_type1_info_cap_iova_range *data = 2634 (void *)cap; 2635 2636 ASSERT_EQ(1, data->header.version); 2637 ASSERT_EQ(1, data->nr_iovas); 2638 EXPECT_EQ(MOCK_APERTURE_START, 2639 data->iova_ranges[0].start); 2640 EXPECT_EQ(MOCK_APERTURE_LAST, data->iova_ranges[0].end); 2641 break; 2642 } 2643 case VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL: { 2644 struct vfio_iommu_type1_info_dma_avail *data = 2645 (void *)cap; 2646 2647 ASSERT_EQ(1, data->header.version); 2648 ASSERT_EQ(sizeof(*data), cap_size); 2649 break; 2650 } 2651 default: 2652 ASSERT_EQ(false, true); 2653 break; 2654 } 2655 if (!cap->next) 2656 break; 2657 2658 ASSERT_GE(info_cmd->argsz, cap->next + sizeof(*cap)); 2659 ASSERT_GE(buffer + cap->next, (void *)cap); 2660 cap = buffer + cap->next; 2661 } 2662 } 2663 2664 TEST_F(vfio_compat_mock_domain, get_info) 2665 { 2666 struct vfio_iommu_type1_info *info_cmd = buffer; 2667 unsigned int i; 2668 size_t caplen; 2669 2670 /* Pre-cap ABI */ 2671 *info_cmd = (struct vfio_iommu_type1_info){ 2672 .argsz = offsetof(struct vfio_iommu_type1_info, cap_offset), 2673 }; 2674 ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO); 2675 ASSERT_NE(0, info_cmd->iova_pgsizes); 2676 ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS, 2677 info_cmd->flags); 2678 2679 /* Read the cap chain size */ 2680 *info_cmd = (struct vfio_iommu_type1_info){ 2681 .argsz = sizeof(*info_cmd), 2682 }; 2683 ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO); 2684 ASSERT_NE(0, info_cmd->iova_pgsizes); 2685 ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS, 2686 info_cmd->flags); 2687 ASSERT_EQ(0, info_cmd->cap_offset); 2688 ASSERT_LT(sizeof(*info_cmd), info_cmd->argsz); 2689 2690 /* Read the caps, kernel should never create a corrupted caps */ 2691 caplen = info_cmd->argsz; 2692 for (i = sizeof(*info_cmd); i < caplen; i++) { 2693 *info_cmd = (struct vfio_iommu_type1_info){ 2694 .argsz = i, 2695 }; 2696 ioctl_check_buf(self->fd, VFIO_IOMMU_GET_INFO); 2697 ASSERT_EQ(VFIO_IOMMU_INFO_PGSIZES | VFIO_IOMMU_INFO_CAPS, 2698 info_cmd->flags); 2699 if (!info_cmd->cap_offset) 2700 continue; 2701 check_vfio_info_cap_chain(_metadata, info_cmd); 2702 } 2703 } 2704 2705 static void shuffle_array(unsigned long *array, size_t nelms) 2706 { 2707 unsigned int i; 2708 2709 /* Shuffle */ 2710 for (i = 0; i != nelms; i++) { 2711 unsigned long tmp = array[i]; 2712 unsigned int other = rand() % (nelms - i); 2713 2714 array[i] = array[other]; 2715 array[other] = tmp; 2716 } 2717 } 2718 2719 TEST_F(vfio_compat_mock_domain, map) 2720 { 2721 struct vfio_iommu_type1_dma_map map_cmd = { 2722 .argsz = sizeof(map_cmd), 2723 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 2724 .vaddr = (uintptr_t)buffer, 2725 .size = BUFFER_SIZE, 2726 .iova = MOCK_APERTURE_START, 2727 }; 2728 struct vfio_iommu_type1_dma_unmap unmap_cmd = { 2729 .argsz = sizeof(unmap_cmd), 2730 .size = BUFFER_SIZE, 2731 .iova = MOCK_APERTURE_START, 2732 }; 2733 unsigned long pages_iova[BUFFER_SIZE / PAGE_SIZE]; 2734 unsigned int i; 2735 2736 /* Simple map/unmap */ 2737 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2738 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2739 ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size); 2740 /* Unmap of empty is success */ 2741 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2742 2743 /* UNMAP_FLAG_ALL requires 0 iova/size */ 2744 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2745 unmap_cmd.flags = VFIO_DMA_UNMAP_FLAG_ALL; 2746 EXPECT_ERRNO(EINVAL, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2747 2748 unmap_cmd.iova = 0; 2749 unmap_cmd.size = 0; 2750 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2751 ASSERT_EQ(BUFFER_SIZE, unmap_cmd.size); 2752 2753 /* Small pages */ 2754 for (i = 0; i != ARRAY_SIZE(pages_iova); i++) { 2755 map_cmd.iova = pages_iova[i] = 2756 MOCK_APERTURE_START + i * PAGE_SIZE; 2757 map_cmd.vaddr = (uintptr_t)buffer + i * PAGE_SIZE; 2758 map_cmd.size = PAGE_SIZE; 2759 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2760 } 2761 shuffle_array(pages_iova, ARRAY_SIZE(pages_iova)); 2762 2763 unmap_cmd.flags = 0; 2764 unmap_cmd.size = PAGE_SIZE; 2765 for (i = 0; i != ARRAY_SIZE(pages_iova); i++) { 2766 unmap_cmd.iova = pages_iova[i]; 2767 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, &unmap_cmd)); 2768 } 2769 } 2770 2771 TEST_F(vfio_compat_mock_domain, huge_map) 2772 { 2773 size_t buf_size = HUGEPAGE_SIZE * 2; 2774 struct vfio_iommu_type1_dma_map map_cmd = { 2775 .argsz = sizeof(map_cmd), 2776 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 2777 .size = buf_size, 2778 .iova = MOCK_APERTURE_START, 2779 }; 2780 struct vfio_iommu_type1_dma_unmap unmap_cmd = { 2781 .argsz = sizeof(unmap_cmd), 2782 }; 2783 unsigned long pages_iova[16]; 2784 unsigned int i; 2785 void *buf; 2786 2787 /* Test huge pages and splitting */ 2788 buf = mmap(0, buf_size, PROT_READ | PROT_WRITE, 2789 MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1, 2790 0); 2791 ASSERT_NE(MAP_FAILED, buf); 2792 map_cmd.vaddr = (uintptr_t)buf; 2793 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_MAP_DMA, &map_cmd)); 2794 2795 unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova); 2796 for (i = 0; i != ARRAY_SIZE(pages_iova); i++) 2797 pages_iova[i] = MOCK_APERTURE_START + (i * unmap_cmd.size); 2798 shuffle_array(pages_iova, ARRAY_SIZE(pages_iova)); 2799 2800 /* type1 mode can cut up larger mappings, type1v2 always fails */ 2801 for (i = 0; i != ARRAY_SIZE(pages_iova); i++) { 2802 unmap_cmd.iova = pages_iova[i]; 2803 unmap_cmd.size = buf_size / ARRAY_SIZE(pages_iova); 2804 if (variant->version == VFIO_TYPE1_IOMMU) { 2805 ASSERT_EQ(0, ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, 2806 &unmap_cmd)); 2807 } else { 2808 EXPECT_ERRNO(ENOENT, 2809 ioctl(self->fd, VFIO_IOMMU_UNMAP_DMA, 2810 &unmap_cmd)); 2811 } 2812 } 2813 } 2814 2815 FIXTURE(iommufd_viommu) 2816 { 2817 int fd; 2818 uint32_t ioas_id; 2819 uint32_t stdev_id; 2820 uint32_t hwpt_id; 2821 uint32_t nested_hwpt_id; 2822 uint32_t device_id; 2823 uint32_t viommu_id; 2824 }; 2825 2826 FIXTURE_VARIANT(iommufd_viommu) 2827 { 2828 unsigned int viommu; 2829 }; 2830 2831 FIXTURE_SETUP(iommufd_viommu) 2832 { 2833 self->fd = open("/dev/iommu", O_RDWR); 2834 ASSERT_NE(-1, self->fd); 2835 test_ioctl_ioas_alloc(&self->ioas_id); 2836 test_ioctl_set_default_memory_limit(); 2837 2838 if (variant->viommu) { 2839 struct iommu_hwpt_selftest data = { 2840 .iotlb = IOMMU_TEST_IOTLB_DEFAULT, 2841 }; 2842 2843 test_cmd_mock_domain(self->ioas_id, &self->stdev_id, NULL, 2844 &self->device_id); 2845 2846 /* Allocate a nesting parent hwpt */ 2847 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 2848 IOMMU_HWPT_ALLOC_NEST_PARENT, 2849 &self->hwpt_id); 2850 2851 /* Allocate a vIOMMU taking refcount of the parent hwpt */ 2852 test_cmd_viommu_alloc(self->device_id, self->hwpt_id, 2853 IOMMU_VIOMMU_TYPE_SELFTEST, NULL, 0, 2854 &self->viommu_id); 2855 2856 /* Allocate a regular nested hwpt */ 2857 test_cmd_hwpt_alloc_nested(self->device_id, self->viommu_id, 0, 2858 &self->nested_hwpt_id, 2859 IOMMU_HWPT_DATA_SELFTEST, &data, 2860 sizeof(data)); 2861 } 2862 } 2863 2864 FIXTURE_TEARDOWN(iommufd_viommu) 2865 { 2866 teardown_iommufd(self->fd, _metadata); 2867 } 2868 2869 FIXTURE_VARIANT_ADD(iommufd_viommu, no_viommu) 2870 { 2871 .viommu = 0, 2872 }; 2873 2874 FIXTURE_VARIANT_ADD(iommufd_viommu, mock_viommu) 2875 { 2876 .viommu = 1, 2877 }; 2878 2879 TEST_F(iommufd_viommu, viommu_auto_destroy) 2880 { 2881 } 2882 2883 TEST_F(iommufd_viommu, viommu_negative_tests) 2884 { 2885 uint32_t device_id = self->device_id; 2886 uint32_t ioas_id = self->ioas_id; 2887 uint32_t hwpt_id; 2888 2889 if (self->device_id) { 2890 /* Negative test -- invalid hwpt (hwpt_id=0) */ 2891 test_err_viommu_alloc(ENOENT, device_id, 0, 2892 IOMMU_VIOMMU_TYPE_SELFTEST, NULL, 0, 2893 NULL); 2894 2895 /* Negative test -- not a nesting parent hwpt */ 2896 test_cmd_hwpt_alloc(device_id, ioas_id, 0, &hwpt_id); 2897 test_err_viommu_alloc(EINVAL, device_id, hwpt_id, 2898 IOMMU_VIOMMU_TYPE_SELFTEST, NULL, 0, 2899 NULL); 2900 test_ioctl_destroy(hwpt_id); 2901 2902 /* Negative test -- unsupported viommu type */ 2903 test_err_viommu_alloc(EOPNOTSUPP, device_id, self->hwpt_id, 2904 0xdead, NULL, 0, NULL); 2905 EXPECT_ERRNO(EBUSY, 2906 _test_ioctl_destroy(self->fd, self->hwpt_id)); 2907 EXPECT_ERRNO(EBUSY, 2908 _test_ioctl_destroy(self->fd, self->viommu_id)); 2909 } else { 2910 test_err_viommu_alloc(ENOENT, self->device_id, self->hwpt_id, 2911 IOMMU_VIOMMU_TYPE_SELFTEST, NULL, 0, 2912 NULL); 2913 } 2914 } 2915 2916 TEST_F(iommufd_viommu, viommu_alloc_nested_iopf) 2917 { 2918 struct iommu_hwpt_selftest data = { 2919 .iotlb = IOMMU_TEST_IOTLB_DEFAULT, 2920 }; 2921 uint32_t viommu_id = self->viommu_id; 2922 uint32_t dev_id = self->device_id; 2923 uint32_t iopf_hwpt_id; 2924 uint32_t fault_id; 2925 uint32_t fault_fd; 2926 uint32_t vdev_id; 2927 2928 if (!dev_id) 2929 SKIP(return, "Skipping test for variant no_viommu"); 2930 2931 test_ioctl_fault_alloc(&fault_id, &fault_fd); 2932 test_err_hwpt_alloc_iopf(ENOENT, dev_id, viommu_id, UINT32_MAX, 2933 IOMMU_HWPT_FAULT_ID_VALID, &iopf_hwpt_id, 2934 IOMMU_HWPT_DATA_SELFTEST, &data, sizeof(data)); 2935 test_err_hwpt_alloc_iopf(EOPNOTSUPP, dev_id, viommu_id, fault_id, 2936 IOMMU_HWPT_FAULT_ID_VALID | (1 << 31), 2937 &iopf_hwpt_id, IOMMU_HWPT_DATA_SELFTEST, &data, 2938 sizeof(data)); 2939 test_cmd_hwpt_alloc_iopf(dev_id, viommu_id, fault_id, 2940 IOMMU_HWPT_FAULT_ID_VALID, &iopf_hwpt_id, 2941 IOMMU_HWPT_DATA_SELFTEST, &data, sizeof(data)); 2942 2943 /* Must allocate vdevice before attaching to a nested hwpt */ 2944 test_err_mock_domain_replace(ENOENT, self->stdev_id, iopf_hwpt_id); 2945 test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); 2946 test_cmd_mock_domain_replace(self->stdev_id, iopf_hwpt_id); 2947 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, iopf_hwpt_id)); 2948 test_cmd_trigger_iopf(dev_id, fault_fd); 2949 2950 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id); 2951 test_ioctl_destroy(iopf_hwpt_id); 2952 close(fault_fd); 2953 test_ioctl_destroy(fault_id); 2954 } 2955 2956 TEST_F(iommufd_viommu, viommu_alloc_with_data) 2957 { 2958 struct iommu_viommu_selftest data = { 2959 .in_data = 0xbeef, 2960 }; 2961 uint32_t *test; 2962 2963 if (!self->device_id) 2964 SKIP(return, "Skipping test for variant no_viommu"); 2965 2966 test_cmd_viommu_alloc(self->device_id, self->hwpt_id, 2967 IOMMU_VIOMMU_TYPE_SELFTEST, &data, sizeof(data), 2968 &self->viommu_id); 2969 ASSERT_EQ(data.out_data, data.in_data); 2970 2971 /* Negative mmap tests -- offset and length cannot be changed */ 2972 test_err_mmap(ENXIO, data.out_mmap_length, 2973 data.out_mmap_offset + PAGE_SIZE); 2974 test_err_mmap(ENXIO, data.out_mmap_length, 2975 data.out_mmap_offset + PAGE_SIZE * 2); 2976 test_err_mmap(ENXIO, data.out_mmap_length / 2, data.out_mmap_offset); 2977 test_err_mmap(ENXIO, data.out_mmap_length * 2, data.out_mmap_offset); 2978 2979 /* Now do a correct mmap for a loopback test */ 2980 test = mmap(NULL, data.out_mmap_length, PROT_READ | PROT_WRITE, 2981 MAP_SHARED, self->fd, data.out_mmap_offset); 2982 ASSERT_NE(MAP_FAILED, test); 2983 ASSERT_EQ(data.in_data, *test); 2984 2985 /* The owner of the mmap region should be blocked */ 2986 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, self->viommu_id)); 2987 munmap(test, data.out_mmap_length); 2988 } 2989 2990 TEST_F(iommufd_viommu, vdevice_alloc) 2991 { 2992 uint32_t viommu_id = self->viommu_id; 2993 uint32_t dev_id = self->device_id; 2994 uint32_t vdev_id = 0; 2995 uint32_t veventq_id; 2996 uint32_t veventq_fd; 2997 int prev_seq = -1; 2998 size_t hdr_size = sizeof(struct iommufd_vevent_header); 2999 char vbuf[64]; 3000 3001 if (dev_id) { 3002 /* Must allocate vdevice before attaching to a nested hwpt */ 3003 test_err_mock_domain_replace(ENOENT, self->stdev_id, 3004 self->nested_hwpt_id); 3005 3006 /* Test depth lower and upper bounds (mirrors kernel cap) */ 3007 #define VEVENTQ_MAX_DEPTH (1U << 19) 3008 test_err_veventq_alloc(EINVAL, viommu_id, 3009 IOMMU_VEVENTQ_TYPE_SELFTEST, 0, NULL, 3010 NULL); 3011 test_err_veventq_alloc(EINVAL, viommu_id, 3012 IOMMU_VEVENTQ_TYPE_SELFTEST, 3013 VEVENTQ_MAX_DEPTH + 1, NULL, NULL); 3014 test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST, 3015 VEVENTQ_MAX_DEPTH, &veventq_id, 3016 &veventq_fd); 3017 close(veventq_fd); 3018 test_ioctl_destroy(veventq_id); 3019 3020 /* Allocate a vEVENTQ with veventq_depth=2 */ 3021 test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST, 3022 2, &veventq_id, &veventq_fd); 3023 test_err_veventq_alloc(EEXIST, viommu_id, 3024 IOMMU_VEVENTQ_TYPE_SELFTEST, 2, NULL, 3025 NULL); 3026 3027 /* Invalid read counts on an empty vEVENTQ */ 3028 ASSERT_EQ(-1, read(veventq_fd, vbuf, 0)); 3029 ASSERT_EQ(EINVAL, errno); 3030 ASSERT_EQ(-1, read(veventq_fd, vbuf, hdr_size - 1)); 3031 ASSERT_EQ(EINVAL, errno); 3032 3033 /* Set vdev_id to 0x99, unset it, and set to 0x88 */ 3034 test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); 3035 test_cmd_mock_domain_replace(self->stdev_id, 3036 self->nested_hwpt_id); 3037 test_cmd_trigger_vevents(dev_id, 1); 3038 3039 /* Invalid read counts on a non-empty vEVENTQ */ 3040 ASSERT_EQ(-1, read(veventq_fd, vbuf, 0)); 3041 ASSERT_EQ(EINVAL, errno); 3042 /* header fits but the event's payload doesn't */ 3043 ASSERT_EQ(-1, read(veventq_fd, vbuf, hdr_size)); 3044 ASSERT_EQ(EINVAL, errno); 3045 3046 test_cmd_read_vevents(veventq_fd, 1, 0x99, &prev_seq); 3047 test_err_vdevice_alloc(EEXIST, viommu_id, dev_id, 0x99, 3048 &vdev_id); 3049 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id); 3050 test_ioctl_destroy(vdev_id); 3051 3052 /* Try again with 0x88 */ 3053 test_cmd_vdevice_alloc(viommu_id, dev_id, 0x88, &vdev_id); 3054 test_cmd_mock_domain_replace(self->stdev_id, 3055 self->nested_hwpt_id); 3056 /* Trigger an overflow with three events */ 3057 test_cmd_trigger_vevents(dev_id, 3); 3058 test_err_read_vevents(EOVERFLOW, veventq_fd, 3, 0x88, 3059 &prev_seq); 3060 /* Overflow must be gone after the previous reads */ 3061 test_cmd_trigger_vevents(dev_id, 1); 3062 test_cmd_read_vevents(veventq_fd, 1, 0x88, &prev_seq); 3063 close(veventq_fd); 3064 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id); 3065 test_ioctl_destroy(vdev_id); 3066 test_ioctl_destroy(veventq_id); 3067 } else { 3068 test_err_vdevice_alloc(ENOENT, viommu_id, dev_id, 0x99, NULL); 3069 } 3070 } 3071 3072 TEST_F(iommufd_viommu, vdevice_cache) 3073 { 3074 struct iommu_viommu_invalidate_selftest inv_reqs[2] = {}; 3075 uint32_t viommu_id = self->viommu_id; 3076 uint32_t dev_id = self->device_id; 3077 uint32_t vdev_id = 0; 3078 uint32_t num_inv; 3079 3080 if (!dev_id) 3081 SKIP(return, "Skipping test for variant no_viommu"); 3082 3083 test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); 3084 3085 test_cmd_dev_check_cache_all(dev_id, IOMMU_TEST_DEV_CACHE_DEFAULT); 3086 3087 /* Check data_type by passing zero-length array */ 3088 num_inv = 0; 3089 test_cmd_viommu_invalidate(viommu_id, inv_reqs, sizeof(*inv_reqs), 3090 &num_inv); 3091 assert(!num_inv); 3092 3093 /* Negative test: Invalid data_type */ 3094 num_inv = 1; 3095 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3096 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST_INVALID, 3097 sizeof(*inv_reqs), &num_inv); 3098 assert(!num_inv); 3099 3100 /* Negative test: structure size sanity */ 3101 num_inv = 1; 3102 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3103 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3104 sizeof(*inv_reqs) + 1, &num_inv); 3105 assert(!num_inv); 3106 3107 num_inv = 1; 3108 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3109 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 1, 3110 &num_inv); 3111 assert(!num_inv); 3112 3113 /* Negative test: invalid flag is passed */ 3114 num_inv = 1; 3115 inv_reqs[0].flags = 0xffffffff; 3116 inv_reqs[0].vdev_id = 0x99; 3117 test_err_viommu_invalidate(EOPNOTSUPP, viommu_id, inv_reqs, 3118 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3119 sizeof(*inv_reqs), &num_inv); 3120 assert(!num_inv); 3121 3122 /* Negative test: invalid data_uptr when array is not empty */ 3123 num_inv = 1; 3124 inv_reqs[0].flags = 0; 3125 inv_reqs[0].vdev_id = 0x99; 3126 test_err_viommu_invalidate(EINVAL, viommu_id, NULL, 3127 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3128 sizeof(*inv_reqs), &num_inv); 3129 assert(!num_inv); 3130 3131 /* Negative test: invalid entry_len when array is not empty */ 3132 num_inv = 1; 3133 inv_reqs[0].flags = 0; 3134 inv_reqs[0].vdev_id = 0x99; 3135 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3136 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 0, 3137 &num_inv); 3138 assert(!num_inv); 3139 3140 /* Negative test: invalid cache_id */ 3141 num_inv = 1; 3142 inv_reqs[0].flags = 0; 3143 inv_reqs[0].vdev_id = 0x99; 3144 inv_reqs[0].cache_id = MOCK_DEV_CACHE_ID_MAX + 1; 3145 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3146 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3147 sizeof(*inv_reqs), &num_inv); 3148 assert(!num_inv); 3149 3150 /* Negative test: invalid vdev_id */ 3151 num_inv = 1; 3152 inv_reqs[0].flags = 0; 3153 inv_reqs[0].vdev_id = 0x9; 3154 inv_reqs[0].cache_id = 0; 3155 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3156 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3157 sizeof(*inv_reqs), &num_inv); 3158 assert(!num_inv); 3159 3160 /* 3161 * Invalidate the 1st cache entry but fail the 2nd request 3162 * due to invalid flags configuration in the 2nd request. 3163 */ 3164 num_inv = 2; 3165 inv_reqs[0].flags = 0; 3166 inv_reqs[0].vdev_id = 0x99; 3167 inv_reqs[0].cache_id = 0; 3168 inv_reqs[1].flags = 0xffffffff; 3169 inv_reqs[1].vdev_id = 0x99; 3170 inv_reqs[1].cache_id = 1; 3171 test_err_viommu_invalidate(EOPNOTSUPP, viommu_id, inv_reqs, 3172 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3173 sizeof(*inv_reqs), &num_inv); 3174 assert(num_inv == 1); 3175 test_cmd_dev_check_cache(dev_id, 0, 0); 3176 test_cmd_dev_check_cache(dev_id, 1, IOMMU_TEST_DEV_CACHE_DEFAULT); 3177 test_cmd_dev_check_cache(dev_id, 2, IOMMU_TEST_DEV_CACHE_DEFAULT); 3178 test_cmd_dev_check_cache(dev_id, 3, IOMMU_TEST_DEV_CACHE_DEFAULT); 3179 3180 /* 3181 * Invalidate the 1st cache entry but fail the 2nd request 3182 * due to invalid cache_id configuration in the 2nd request. 3183 */ 3184 num_inv = 2; 3185 inv_reqs[0].flags = 0; 3186 inv_reqs[0].vdev_id = 0x99; 3187 inv_reqs[0].cache_id = 0; 3188 inv_reqs[1].flags = 0; 3189 inv_reqs[1].vdev_id = 0x99; 3190 inv_reqs[1].cache_id = MOCK_DEV_CACHE_ID_MAX + 1; 3191 test_err_viommu_invalidate(EINVAL, viommu_id, inv_reqs, 3192 IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 3193 sizeof(*inv_reqs), &num_inv); 3194 assert(num_inv == 1); 3195 test_cmd_dev_check_cache(dev_id, 0, 0); 3196 test_cmd_dev_check_cache(dev_id, 1, IOMMU_TEST_DEV_CACHE_DEFAULT); 3197 test_cmd_dev_check_cache(dev_id, 2, IOMMU_TEST_DEV_CACHE_DEFAULT); 3198 test_cmd_dev_check_cache(dev_id, 3, IOMMU_TEST_DEV_CACHE_DEFAULT); 3199 3200 /* Invalidate the 2nd cache entry and verify */ 3201 num_inv = 1; 3202 inv_reqs[0].flags = 0; 3203 inv_reqs[0].vdev_id = 0x99; 3204 inv_reqs[0].cache_id = 1; 3205 test_cmd_viommu_invalidate(viommu_id, inv_reqs, sizeof(*inv_reqs), 3206 &num_inv); 3207 assert(num_inv == 1); 3208 test_cmd_dev_check_cache(dev_id, 0, 0); 3209 test_cmd_dev_check_cache(dev_id, 1, 0); 3210 test_cmd_dev_check_cache(dev_id, 2, IOMMU_TEST_DEV_CACHE_DEFAULT); 3211 test_cmd_dev_check_cache(dev_id, 3, IOMMU_TEST_DEV_CACHE_DEFAULT); 3212 3213 /* Invalidate the 3rd and 4th cache entries and verify */ 3214 num_inv = 2; 3215 inv_reqs[0].flags = 0; 3216 inv_reqs[0].vdev_id = 0x99; 3217 inv_reqs[0].cache_id = 2; 3218 inv_reqs[1].flags = 0; 3219 inv_reqs[1].vdev_id = 0x99; 3220 inv_reqs[1].cache_id = 3; 3221 test_cmd_viommu_invalidate(viommu_id, inv_reqs, sizeof(*inv_reqs), 3222 &num_inv); 3223 assert(num_inv == 2); 3224 test_cmd_dev_check_cache_all(dev_id, 0); 3225 3226 /* Invalidate all cache entries for nested_dev_id[1] and verify */ 3227 num_inv = 1; 3228 inv_reqs[0].vdev_id = 0x99; 3229 inv_reqs[0].flags = IOMMU_TEST_INVALIDATE_FLAG_ALL; 3230 test_cmd_viommu_invalidate(viommu_id, inv_reqs, sizeof(*inv_reqs), 3231 &num_inv); 3232 assert(num_inv == 1); 3233 test_cmd_dev_check_cache_all(dev_id, 0); 3234 test_ioctl_destroy(vdev_id); 3235 } 3236 3237 TEST_F(iommufd_viommu, hw_queue) 3238 { 3239 __u64 iova = MOCK_APERTURE_START, iova2; 3240 uint32_t viommu_id = self->viommu_id; 3241 uint32_t hw_queue_id[2]; 3242 3243 if (!viommu_id) 3244 SKIP(return, "Skipping test for variant no_viommu"); 3245 3246 /* Fail IOMMU_HW_QUEUE_TYPE_DEFAULT */ 3247 test_err_hw_queue_alloc(EOPNOTSUPP, viommu_id, 3248 IOMMU_HW_QUEUE_TYPE_DEFAULT, 0, iova, PAGE_SIZE, 3249 &hw_queue_id[0]); 3250 /* Fail queue addr and length */ 3251 test_err_hw_queue_alloc(EINVAL, viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 3252 0, iova, 0, &hw_queue_id[0]); 3253 test_err_hw_queue_alloc(EOVERFLOW, viommu_id, 3254 IOMMU_HW_QUEUE_TYPE_SELFTEST, 0, ~(uint64_t)0, 3255 PAGE_SIZE, &hw_queue_id[0]); 3256 /* Fail missing iova */ 3257 test_err_hw_queue_alloc(ENOENT, viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 3258 0, iova, PAGE_SIZE, &hw_queue_id[0]); 3259 3260 /* Map iova */ 3261 test_ioctl_ioas_map(buffer, PAGE_SIZE, &iova); 3262 test_ioctl_ioas_map(buffer + PAGE_SIZE, PAGE_SIZE, &iova2); 3263 3264 /* Fail index=1 and =MAX; must start from index=0 */ 3265 test_err_hw_queue_alloc(EIO, viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 1, 3266 iova, PAGE_SIZE, &hw_queue_id[0]); 3267 test_err_hw_queue_alloc(EINVAL, viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 3268 IOMMU_TEST_HW_QUEUE_MAX, iova, PAGE_SIZE, 3269 &hw_queue_id[0]); 3270 3271 /* Allocate index=0, declare ownership of the iova */ 3272 test_cmd_hw_queue_alloc(viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 0, 3273 iova, PAGE_SIZE, &hw_queue_id[0]); 3274 /* Fail duplicated index */ 3275 test_err_hw_queue_alloc(EEXIST, viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 3276 0, iova, PAGE_SIZE, &hw_queue_id[0]); 3277 /* Fail unmap, due to iova ownership */ 3278 test_err_ioctl_ioas_unmap(EBUSY, iova, PAGE_SIZE); 3279 /* The 2nd page is not pinned, so it can be unmmap */ 3280 test_ioctl_ioas_unmap(iova2, PAGE_SIZE); 3281 3282 /* Allocate index=1, with an unaligned case */ 3283 test_cmd_hw_queue_alloc(viommu_id, IOMMU_HW_QUEUE_TYPE_SELFTEST, 1, 3284 iova + PAGE_SIZE / 2, PAGE_SIZE / 2, 3285 &hw_queue_id[1]); 3286 /* Fail to destroy, due to dependency */ 3287 EXPECT_ERRNO(EBUSY, _test_ioctl_destroy(self->fd, hw_queue_id[0])); 3288 3289 /* Destroy in descending order */ 3290 test_ioctl_destroy(hw_queue_id[1]); 3291 test_ioctl_destroy(hw_queue_id[0]); 3292 /* Now it can unmap the first page */ 3293 test_ioctl_ioas_unmap(iova, PAGE_SIZE); 3294 } 3295 3296 TEST_F(iommufd_viommu, vdevice_tombstone) 3297 { 3298 uint32_t viommu_id = self->viommu_id; 3299 uint32_t dev_id = self->device_id; 3300 uint32_t vdev_id = 0; 3301 3302 if (!dev_id) 3303 SKIP(return, "Skipping test for variant no_viommu"); 3304 3305 test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); 3306 test_ioctl_destroy(self->stdev_id); 3307 EXPECT_ERRNO(ENOENT, _test_ioctl_destroy(self->fd, vdev_id)); 3308 } 3309 3310 FIXTURE(iommufd_device_pasid) 3311 { 3312 int fd; 3313 uint32_t ioas_id; 3314 uint32_t hwpt_id; 3315 uint32_t stdev_id; 3316 uint32_t device_id; 3317 uint32_t no_pasid_stdev_id; 3318 uint32_t no_pasid_device_id; 3319 }; 3320 3321 FIXTURE_VARIANT(iommufd_device_pasid) 3322 { 3323 bool pasid_capable; 3324 }; 3325 3326 FIXTURE_SETUP(iommufd_device_pasid) 3327 { 3328 self->fd = open("/dev/iommu", O_RDWR); 3329 ASSERT_NE(-1, self->fd); 3330 test_ioctl_ioas_alloc(&self->ioas_id); 3331 3332 test_cmd_mock_domain_flags(self->ioas_id, 3333 MOCK_FLAGS_DEVICE_PASID, 3334 &self->stdev_id, &self->hwpt_id, 3335 &self->device_id); 3336 if (!variant->pasid_capable) 3337 test_cmd_mock_domain_flags(self->ioas_id, 0, 3338 &self->no_pasid_stdev_id, NULL, 3339 &self->no_pasid_device_id); 3340 } 3341 3342 FIXTURE_TEARDOWN(iommufd_device_pasid) 3343 { 3344 teardown_iommufd(self->fd, _metadata); 3345 } 3346 3347 FIXTURE_VARIANT_ADD(iommufd_device_pasid, no_pasid) 3348 { 3349 .pasid_capable = false, 3350 }; 3351 3352 FIXTURE_VARIANT_ADD(iommufd_device_pasid, has_pasid) 3353 { 3354 .pasid_capable = true, 3355 }; 3356 3357 TEST_F(iommufd_device_pasid, pasid_attach) 3358 { 3359 struct iommu_hwpt_selftest data = { 3360 .iotlb = IOMMU_TEST_IOTLB_DEFAULT, 3361 }; 3362 uint32_t nested_hwpt_id[3] = {}; 3363 uint32_t parent_hwpt_id = 0; 3364 uint32_t fault_id, fault_fd; 3365 uint32_t s2_hwpt_id = 0; 3366 uint32_t iopf_hwpt_id; 3367 uint32_t pasid = 100; 3368 uint32_t viommu_id; 3369 3370 /* 3371 * Negative, detach pasid without attaching, this is not expected. 3372 * But it should not result in failure anyway. 3373 */ 3374 test_cmd_pasid_detach(pasid); 3375 3376 /* Allocate two nested hwpts sharing one common parent hwpt */ 3377 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 3378 IOMMU_HWPT_ALLOC_NEST_PARENT, 3379 &parent_hwpt_id); 3380 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 3381 IOMMU_HWPT_ALLOC_PASID, 3382 &nested_hwpt_id[0], 3383 IOMMU_HWPT_DATA_SELFTEST, 3384 &data, sizeof(data)); 3385 test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id, 3386 IOMMU_HWPT_ALLOC_PASID, 3387 &nested_hwpt_id[1], 3388 IOMMU_HWPT_DATA_SELFTEST, 3389 &data, sizeof(data)); 3390 3391 /* Fault related preparation */ 3392 test_ioctl_fault_alloc(&fault_id, &fault_fd); 3393 test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id, 3394 IOMMU_HWPT_FAULT_ID_VALID | IOMMU_HWPT_ALLOC_PASID, 3395 &iopf_hwpt_id, 3396 IOMMU_HWPT_DATA_SELFTEST, &data, 3397 sizeof(data)); 3398 3399 /* Allocate a regular nested hwpt based on viommu */ 3400 test_cmd_viommu_alloc(self->device_id, parent_hwpt_id, 3401 IOMMU_VIOMMU_TYPE_SELFTEST, NULL, 0, &viommu_id); 3402 test_cmd_hwpt_alloc_nested(self->device_id, viommu_id, 3403 IOMMU_HWPT_ALLOC_PASID, 3404 &nested_hwpt_id[2], 3405 IOMMU_HWPT_DATA_SELFTEST, &data, 3406 sizeof(data)); 3407 3408 test_cmd_hwpt_alloc(self->device_id, self->ioas_id, 3409 IOMMU_HWPT_ALLOC_PASID, 3410 &s2_hwpt_id); 3411 3412 /* Attach RID to non-pasid compat domain, */ 3413 test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id); 3414 /* then attach to pasid should fail */ 3415 test_err_pasid_attach(EINVAL, pasid, s2_hwpt_id); 3416 3417 /* Attach RID to pasid compat domain, */ 3418 test_cmd_mock_domain_replace(self->stdev_id, s2_hwpt_id); 3419 /* then attach to pasid should succeed, */ 3420 test_cmd_pasid_attach(pasid, nested_hwpt_id[0]); 3421 /* but attach RID to non-pasid compat domain should fail now. */ 3422 test_err_mock_domain_replace(EINVAL, self->stdev_id, parent_hwpt_id); 3423 /* 3424 * Detach hwpt from pasid 100, and check if the pasid 100 3425 * has null domain. 3426 */ 3427 test_cmd_pasid_detach(pasid); 3428 ASSERT_EQ(0, 3429 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3430 pasid, 0)); 3431 /* RID is attached to pasid-comapt domain, pasid path is not used */ 3432 3433 if (!variant->pasid_capable) { 3434 /* 3435 * PASID-compatible domain can be used by non-PASID-capable 3436 * device. 3437 */ 3438 test_cmd_mock_domain_replace(self->no_pasid_stdev_id, nested_hwpt_id[0]); 3439 test_cmd_mock_domain_replace(self->no_pasid_stdev_id, self->ioas_id); 3440 /* 3441 * Attach hwpt to pasid 100 of non-PASID-capable device, 3442 * should fail, no matter domain is pasid-comapt or not. 3443 */ 3444 EXPECT_ERRNO(EINVAL, 3445 _test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id, 3446 pasid, parent_hwpt_id)); 3447 EXPECT_ERRNO(EINVAL, 3448 _test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id, 3449 pasid, s2_hwpt_id)); 3450 } 3451 3452 /* 3453 * Attach non pasid compat hwpt to pasid-capable device, should 3454 * fail, and have null domain. 3455 */ 3456 test_err_pasid_attach(EINVAL, pasid, parent_hwpt_id); 3457 ASSERT_EQ(0, 3458 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3459 pasid, 0)); 3460 3461 /* 3462 * Attach ioas to pasid 100, should fail, domain should 3463 * be null. 3464 */ 3465 test_err_pasid_attach(EINVAL, pasid, self->ioas_id); 3466 ASSERT_EQ(0, 3467 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3468 pasid, 0)); 3469 3470 /* 3471 * Attach the s2_hwpt to pasid 100, should succeed, domain should 3472 * be valid. 3473 */ 3474 test_cmd_pasid_attach(pasid, s2_hwpt_id); 3475 ASSERT_EQ(0, 3476 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3477 pasid, s2_hwpt_id)); 3478 3479 /* 3480 * Try attach pasid 100 with another hwpt, should FAIL 3481 * as attach does not allow overwrite, use REPLACE instead. 3482 */ 3483 test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]); 3484 3485 /* 3486 * Detach hwpt from pasid 100 for next test, should succeed, 3487 * and have null domain. 3488 */ 3489 test_cmd_pasid_detach(pasid); 3490 ASSERT_EQ(0, 3491 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3492 pasid, 0)); 3493 3494 /* 3495 * Attach nested hwpt to pasid 100, should succeed, domain 3496 * should be valid. 3497 */ 3498 test_cmd_pasid_attach(pasid, nested_hwpt_id[0]); 3499 ASSERT_EQ(0, 3500 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3501 pasid, nested_hwpt_id[0])); 3502 3503 /* Attach to pasid 100 which has been attached, should fail. */ 3504 test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]); 3505 3506 /* cleanup pasid 100 */ 3507 test_cmd_pasid_detach(pasid); 3508 3509 /* Replace tests */ 3510 3511 pasid = 200; 3512 /* 3513 * Replace pasid 200 without attaching it, should fail 3514 * with -EINVAL. 3515 */ 3516 test_err_pasid_replace(EINVAL, pasid, s2_hwpt_id); 3517 3518 /* 3519 * Attach the s2 hwpt to pasid 200, should succeed, domain should 3520 * be valid. 3521 */ 3522 test_cmd_pasid_attach(pasid, s2_hwpt_id); 3523 ASSERT_EQ(0, 3524 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3525 pasid, s2_hwpt_id)); 3526 3527 /* 3528 * Replace pasid 200 with self->ioas_id, should fail 3529 * and domain should be the prior s2 hwpt. 3530 */ 3531 test_err_pasid_replace(EINVAL, pasid, self->ioas_id); 3532 ASSERT_EQ(0, 3533 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3534 pasid, s2_hwpt_id)); 3535 3536 /* 3537 * Replace a nested hwpt for pasid 200, should succeed, 3538 * and have valid domain. 3539 */ 3540 test_cmd_pasid_replace(pasid, nested_hwpt_id[0]); 3541 ASSERT_EQ(0, 3542 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3543 pasid, nested_hwpt_id[0])); 3544 3545 /* 3546 * Replace with another nested hwpt for pasid 200, should 3547 * succeed, and have valid domain. 3548 */ 3549 test_cmd_pasid_replace(pasid, nested_hwpt_id[1]); 3550 ASSERT_EQ(0, 3551 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3552 pasid, nested_hwpt_id[1])); 3553 3554 /* cleanup pasid 200 */ 3555 test_cmd_pasid_detach(pasid); 3556 3557 /* Negative Tests for pasid replace, use pasid 1024 */ 3558 3559 /* 3560 * Attach the s2 hwpt to pasid 1024, should succeed, domain should 3561 * be valid. 3562 */ 3563 pasid = 1024; 3564 test_cmd_pasid_attach(pasid, s2_hwpt_id); 3565 ASSERT_EQ(0, 3566 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3567 pasid, s2_hwpt_id)); 3568 3569 /* 3570 * Replace pasid 1024 with nested_hwpt_id[0], should fail, 3571 * but have the old valid domain. This is a designed 3572 * negative case. Normally, this shall succeed. 3573 */ 3574 test_err_pasid_replace(ENOMEM, pasid, nested_hwpt_id[0]); 3575 ASSERT_EQ(0, 3576 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3577 pasid, s2_hwpt_id)); 3578 3579 /* cleanup pasid 1024 */ 3580 test_cmd_pasid_detach(pasid); 3581 3582 /* Attach to iopf-capable hwpt */ 3583 3584 /* 3585 * Attach an iopf hwpt to pasid 2048, should succeed, domain should 3586 * be valid. 3587 */ 3588 pasid = 2048; 3589 test_cmd_pasid_attach(pasid, iopf_hwpt_id); 3590 ASSERT_EQ(0, 3591 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3592 pasid, iopf_hwpt_id)); 3593 3594 test_cmd_trigger_iopf_pasid(self->device_id, pasid, fault_fd); 3595 3596 /* 3597 * Replace with s2_hwpt_id for pasid 2048, should 3598 * succeed, and have valid domain. 3599 */ 3600 test_cmd_pasid_replace(pasid, s2_hwpt_id); 3601 ASSERT_EQ(0, 3602 test_cmd_pasid_check_hwpt(self->fd, self->stdev_id, 3603 pasid, s2_hwpt_id)); 3604 3605 /* cleanup pasid 2048 */ 3606 test_cmd_pasid_detach(pasid); 3607 3608 test_ioctl_destroy(iopf_hwpt_id); 3609 close(fault_fd); 3610 test_ioctl_destroy(fault_id); 3611 3612 /* Detach the s2_hwpt_id from RID */ 3613 test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id); 3614 } 3615 3616 TEST_HARNESS_MAIN 3617