1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <dirent.h> 3 #include <fcntl.h> 4 #include <libgen.h> 5 #include <stdint.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 10 #include <sys/eventfd.h> 11 #include <sys/ioctl.h> 12 #include <sys/mman.h> 13 14 #include <uapi/linux/types.h> 15 #include <linux/limits.h> 16 #include <linux/mman.h> 17 #include <linux/types.h> 18 #include <linux/vfio.h> 19 #include <linux/iommufd.h> 20 21 #include "../../../kselftest.h" 22 #include <vfio_util.h> 23 24 #define PCI_SYSFS_PATH "/sys/bus/pci/devices" 25 26 #define ioctl_assert(_fd, _op, _arg) do { \ 27 void *__arg = (_arg); \ 28 int __ret = ioctl((_fd), (_op), (__arg)); \ 29 VFIO_ASSERT_EQ(__ret, 0, "ioctl(%s, %s, %s) returned %d\n", #_fd, #_op, #_arg, __ret); \ 30 } while (0) 31 32 iova_t __to_iova(struct vfio_pci_device *device, void *vaddr) 33 { 34 struct vfio_dma_region *region; 35 36 list_for_each_entry(region, &device->dma_regions, link) { 37 if (vaddr < region->vaddr) 38 continue; 39 40 if (vaddr >= region->vaddr + region->size) 41 continue; 42 43 return region->iova + (vaddr - region->vaddr); 44 } 45 46 return INVALID_IOVA; 47 } 48 49 iova_t to_iova(struct vfio_pci_device *device, void *vaddr) 50 { 51 iova_t iova; 52 53 iova = __to_iova(device, vaddr); 54 VFIO_ASSERT_NE(iova, INVALID_IOVA, "%p is not mapped into device.\n", vaddr); 55 56 return iova; 57 } 58 59 static void vfio_pci_irq_set(struct vfio_pci_device *device, 60 u32 index, u32 vector, u32 count, int *fds) 61 { 62 u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count] = {}; 63 struct vfio_irq_set *irq = (void *)&buf; 64 int *irq_fds = (void *)&irq->data; 65 66 irq->argsz = sizeof(buf); 67 irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; 68 irq->index = index; 69 irq->start = vector; 70 irq->count = count; 71 72 if (count) { 73 irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; 74 memcpy(irq_fds, fds, sizeof(int) * count); 75 } else { 76 irq->flags |= VFIO_IRQ_SET_DATA_NONE; 77 } 78 79 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); 80 } 81 82 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) 83 { 84 struct vfio_irq_set irq = { 85 .argsz = sizeof(irq), 86 .flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE, 87 .index = index, 88 .start = vector, 89 .count = 1, 90 }; 91 92 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq); 93 } 94 95 static void check_supported_irq_index(u32 index) 96 { 97 /* VFIO selftests only supports MSI and MSI-x for now. */ 98 VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX || 99 index == VFIO_PCI_MSIX_IRQ_INDEX, 100 "Unsupported IRQ index: %u\n", index); 101 } 102 103 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector, 104 int count) 105 { 106 int i; 107 108 check_supported_irq_index(index); 109 110 for (i = vector; i < vector + count; i++) { 111 VFIO_ASSERT_LT(device->msi_eventfds[i], 0); 112 device->msi_eventfds[i] = eventfd(0, 0); 113 VFIO_ASSERT_GE(device->msi_eventfds[i], 0); 114 } 115 116 vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector); 117 } 118 119 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index) 120 { 121 int i; 122 123 check_supported_irq_index(index); 124 125 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 126 if (device->msi_eventfds[i] < 0) 127 continue; 128 129 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 130 device->msi_eventfds[i] = -1; 131 } 132 133 vfio_pci_irq_set(device, index, 0, 0, NULL); 134 } 135 136 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, 137 struct vfio_irq_info *irq_info) 138 { 139 irq_info->argsz = sizeof(*irq_info); 140 irq_info->index = index; 141 142 ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info); 143 } 144 145 static int vfio_iommu_dma_map(struct vfio_pci_device *device, 146 struct vfio_dma_region *region) 147 { 148 struct vfio_iommu_type1_dma_map args = { 149 .argsz = sizeof(args), 150 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 151 .vaddr = (u64)region->vaddr, 152 .iova = region->iova, 153 .size = region->size, 154 }; 155 156 if (ioctl(device->container_fd, VFIO_IOMMU_MAP_DMA, &args)) 157 return -errno; 158 159 return 0; 160 } 161 162 static int iommufd_dma_map(struct vfio_pci_device *device, 163 struct vfio_dma_region *region) 164 { 165 struct iommu_ioas_map args = { 166 .size = sizeof(args), 167 .flags = IOMMU_IOAS_MAP_READABLE | 168 IOMMU_IOAS_MAP_WRITEABLE | 169 IOMMU_IOAS_MAP_FIXED_IOVA, 170 .user_va = (u64)region->vaddr, 171 .iova = region->iova, 172 .length = region->size, 173 .ioas_id = device->ioas_id, 174 }; 175 176 if (ioctl(device->iommufd, IOMMU_IOAS_MAP, &args)) 177 return -errno; 178 179 return 0; 180 } 181 182 int __vfio_pci_dma_map(struct vfio_pci_device *device, 183 struct vfio_dma_region *region) 184 { 185 int ret; 186 187 if (device->iommufd) 188 ret = iommufd_dma_map(device, region); 189 else 190 ret = vfio_iommu_dma_map(device, region); 191 192 if (ret) 193 return ret; 194 195 list_add(®ion->link, &device->dma_regions); 196 197 return 0; 198 } 199 200 static int vfio_iommu_dma_unmap(int fd, u64 iova, u64 size, u32 flags, 201 u64 *unmapped) 202 { 203 struct vfio_iommu_type1_dma_unmap args = { 204 .argsz = sizeof(args), 205 .iova = iova, 206 .size = size, 207 .flags = flags, 208 }; 209 210 if (ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &args)) 211 return -errno; 212 213 if (unmapped) 214 *unmapped = args.size; 215 216 return 0; 217 } 218 219 static int iommufd_dma_unmap(int fd, u64 iova, u64 length, u32 ioas_id, 220 u64 *unmapped) 221 { 222 struct iommu_ioas_unmap args = { 223 .size = sizeof(args), 224 .iova = iova, 225 .length = length, 226 .ioas_id = ioas_id, 227 }; 228 229 if (ioctl(fd, IOMMU_IOAS_UNMAP, &args)) 230 return -errno; 231 232 if (unmapped) 233 *unmapped = args.length; 234 235 return 0; 236 } 237 238 int __vfio_pci_dma_unmap(struct vfio_pci_device *device, 239 struct vfio_dma_region *region, u64 *unmapped) 240 { 241 int ret; 242 243 if (device->iommufd) 244 ret = iommufd_dma_unmap(device->iommufd, region->iova, 245 region->size, device->ioas_id, 246 unmapped); 247 else 248 ret = vfio_iommu_dma_unmap(device->container_fd, region->iova, 249 region->size, 0, unmapped); 250 251 if (ret) 252 return ret; 253 254 list_del_init(®ion->link); 255 256 return 0; 257 } 258 259 int __vfio_pci_dma_unmap_all(struct vfio_pci_device *device, u64 *unmapped) 260 { 261 int ret; 262 struct vfio_dma_region *curr, *next; 263 264 if (device->iommufd) 265 ret = iommufd_dma_unmap(device->iommufd, 0, UINT64_MAX, 266 device->ioas_id, unmapped); 267 else 268 ret = vfio_iommu_dma_unmap(device->container_fd, 0, 0, 269 VFIO_DMA_UNMAP_FLAG_ALL, unmapped); 270 271 if (ret) 272 return ret; 273 274 list_for_each_entry_safe(curr, next, &device->dma_regions, link) 275 list_del_init(&curr->link); 276 277 return 0; 278 } 279 280 static void vfio_pci_region_get(struct vfio_pci_device *device, int index, 281 struct vfio_region_info *info) 282 { 283 memset(info, 0, sizeof(*info)); 284 285 info->argsz = sizeof(*info); 286 info->index = index; 287 288 ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info); 289 } 290 291 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index) 292 { 293 struct vfio_pci_bar *bar = &device->bars[index]; 294 int prot = 0; 295 296 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 297 VFIO_ASSERT_NULL(bar->vaddr); 298 VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP); 299 300 if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ) 301 prot |= PROT_READ; 302 if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE) 303 prot |= PROT_WRITE; 304 305 bar->vaddr = mmap(NULL, bar->info.size, prot, MAP_FILE | MAP_SHARED, 306 device->fd, bar->info.offset); 307 VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED); 308 } 309 310 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index) 311 { 312 struct vfio_pci_bar *bar = &device->bars[index]; 313 314 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 315 VFIO_ASSERT_NOT_NULL(bar->vaddr); 316 317 VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0); 318 bar->vaddr = NULL; 319 } 320 321 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device) 322 { 323 int i; 324 325 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 326 if (device->bars[i].vaddr) 327 vfio_pci_bar_unmap(device, i); 328 } 329 } 330 331 void vfio_pci_config_access(struct vfio_pci_device *device, bool write, 332 size_t config, size_t size, void *data) 333 { 334 struct vfio_region_info *config_space = &device->config_space; 335 int ret; 336 337 if (write) 338 ret = pwrite(device->fd, data, size, config_space->offset + config); 339 else 340 ret = pread(device->fd, data, size, config_space->offset + config); 341 342 VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n", 343 write ? "write to" : "read from", config); 344 } 345 346 void vfio_pci_device_reset(struct vfio_pci_device *device) 347 { 348 ioctl_assert(device->fd, VFIO_DEVICE_RESET, NULL); 349 } 350 351 static unsigned int vfio_pci_get_group_from_dev(const char *bdf) 352 { 353 char dev_iommu_group_path[PATH_MAX] = {0}; 354 char sysfs_path[PATH_MAX] = {0}; 355 unsigned int group; 356 int ret; 357 358 snprintf(sysfs_path, PATH_MAX, "%s/%s/iommu_group", PCI_SYSFS_PATH, bdf); 359 360 ret = readlink(sysfs_path, dev_iommu_group_path, sizeof(dev_iommu_group_path)); 361 VFIO_ASSERT_NE(ret, -1, "Failed to get the IOMMU group for device: %s\n", bdf); 362 363 ret = sscanf(basename(dev_iommu_group_path), "%u", &group); 364 VFIO_ASSERT_EQ(ret, 1, "Failed to get the IOMMU group for device: %s\n", bdf); 365 366 return group; 367 } 368 369 static void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf) 370 { 371 struct vfio_group_status group_status = { 372 .argsz = sizeof(group_status), 373 }; 374 char group_path[32]; 375 int group; 376 377 group = vfio_pci_get_group_from_dev(bdf); 378 snprintf(group_path, sizeof(group_path), "/dev/vfio/%d", group); 379 380 device->group_fd = open(group_path, O_RDWR); 381 VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path); 382 383 ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status); 384 VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE); 385 386 ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->container_fd); 387 } 388 389 static void vfio_pci_container_setup(struct vfio_pci_device *device, const char *bdf) 390 { 391 unsigned long iommu_type = device->iommu_mode->iommu_type; 392 const char *path = device->iommu_mode->container_path; 393 int version; 394 int ret; 395 396 device->container_fd = open(path, O_RDWR); 397 VFIO_ASSERT_GE(device->container_fd, 0, "open(%s) failed\n", path); 398 399 version = ioctl(device->container_fd, VFIO_GET_API_VERSION); 400 VFIO_ASSERT_EQ(version, VFIO_API_VERSION, "Unsupported version: %d\n", version); 401 402 vfio_pci_group_setup(device, bdf); 403 404 ret = ioctl(device->container_fd, VFIO_CHECK_EXTENSION, iommu_type); 405 VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type); 406 407 ioctl_assert(device->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); 408 409 device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf); 410 VFIO_ASSERT_GE(device->fd, 0); 411 } 412 413 static void vfio_pci_device_setup(struct vfio_pci_device *device) 414 { 415 int i; 416 417 device->info.argsz = sizeof(device->info); 418 ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info); 419 420 vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space); 421 422 /* Sanity check VFIO does not advertise mmap for config space */ 423 VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP), 424 "PCI config space should not support mmap()\n"); 425 426 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 427 struct vfio_pci_bar *bar = device->bars + i; 428 429 vfio_pci_region_get(device, i, &bar->info); 430 if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP) 431 vfio_pci_bar_map(device, i); 432 } 433 434 vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info); 435 vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info); 436 437 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) 438 device->msi_eventfds[i] = -1; 439 } 440 441 const char *vfio_pci_get_cdev_path(const char *bdf) 442 { 443 char dir_path[PATH_MAX]; 444 struct dirent *entry; 445 char *cdev_path; 446 DIR *dir; 447 448 cdev_path = calloc(PATH_MAX, 1); 449 VFIO_ASSERT_NOT_NULL(cdev_path); 450 451 snprintf(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf); 452 453 dir = opendir(dir_path); 454 VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path); 455 456 while ((entry = readdir(dir)) != NULL) { 457 /* Find the file that starts with "vfio" */ 458 if (strncmp("vfio", entry->d_name, 4)) 459 continue; 460 461 snprintf(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name); 462 break; 463 } 464 465 VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n"); 466 VFIO_ASSERT_EQ(closedir(dir), 0); 467 468 return cdev_path; 469 } 470 471 /* Reminder: Keep in sync with FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(). */ 472 static const struct vfio_iommu_mode iommu_modes[] = { 473 { 474 .name = "vfio_type1_iommu", 475 .container_path = "/dev/vfio/vfio", 476 .iommu_type = VFIO_TYPE1_IOMMU, 477 }, 478 { 479 .name = "vfio_type1v2_iommu", 480 .container_path = "/dev/vfio/vfio", 481 .iommu_type = VFIO_TYPE1v2_IOMMU, 482 }, 483 { 484 .name = "iommufd_compat_type1", 485 .container_path = "/dev/iommu", 486 .iommu_type = VFIO_TYPE1_IOMMU, 487 }, 488 { 489 .name = "iommufd_compat_type1v2", 490 .container_path = "/dev/iommu", 491 .iommu_type = VFIO_TYPE1v2_IOMMU, 492 }, 493 { 494 .name = "iommufd", 495 }, 496 }; 497 498 const char *default_iommu_mode = "iommufd"; 499 500 static const struct vfio_iommu_mode *lookup_iommu_mode(const char *iommu_mode) 501 { 502 int i; 503 504 if (!iommu_mode) 505 iommu_mode = default_iommu_mode; 506 507 for (i = 0; i < ARRAY_SIZE(iommu_modes); i++) { 508 if (strcmp(iommu_mode, iommu_modes[i].name)) 509 continue; 510 511 return &iommu_modes[i]; 512 } 513 514 VFIO_FAIL("Unrecognized IOMMU mode: %s\n", iommu_mode); 515 } 516 517 static void vfio_device_bind_iommufd(int device_fd, int iommufd) 518 { 519 struct vfio_device_bind_iommufd args = { 520 .argsz = sizeof(args), 521 .iommufd = iommufd, 522 }; 523 524 ioctl_assert(device_fd, VFIO_DEVICE_BIND_IOMMUFD, &args); 525 } 526 527 static u32 iommufd_ioas_alloc(int iommufd) 528 { 529 struct iommu_ioas_alloc args = { 530 .size = sizeof(args), 531 }; 532 533 ioctl_assert(iommufd, IOMMU_IOAS_ALLOC, &args); 534 return args.out_ioas_id; 535 } 536 537 static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id) 538 { 539 struct vfio_device_attach_iommufd_pt args = { 540 .argsz = sizeof(args), 541 .pt_id = pt_id, 542 }; 543 544 ioctl_assert(device_fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &args); 545 } 546 547 static void vfio_pci_iommufd_setup(struct vfio_pci_device *device, const char *bdf) 548 { 549 const char *cdev_path = vfio_pci_get_cdev_path(bdf); 550 551 device->fd = open(cdev_path, O_RDWR); 552 VFIO_ASSERT_GE(device->fd, 0); 553 free((void *)cdev_path); 554 555 /* 556 * Require device->iommufd to be >0 so that a simple non-0 check can be 557 * used to check if iommufd is enabled. In practice open() will never 558 * return 0 unless stdin is closed. 559 */ 560 device->iommufd = open("/dev/iommu", O_RDWR); 561 VFIO_ASSERT_GT(device->iommufd, 0); 562 563 vfio_device_bind_iommufd(device->fd, device->iommufd); 564 device->ioas_id = iommufd_ioas_alloc(device->iommufd); 565 vfio_device_attach_iommufd_pt(device->fd, device->ioas_id); 566 } 567 568 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode) 569 { 570 struct vfio_pci_device *device; 571 572 device = calloc(1, sizeof(*device)); 573 VFIO_ASSERT_NOT_NULL(device); 574 575 INIT_LIST_HEAD(&device->dma_regions); 576 577 device->iommu_mode = lookup_iommu_mode(iommu_mode); 578 579 if (device->iommu_mode->container_path) 580 vfio_pci_container_setup(device, bdf); 581 else 582 vfio_pci_iommufd_setup(device, bdf); 583 584 vfio_pci_device_setup(device); 585 vfio_pci_driver_probe(device); 586 587 return device; 588 } 589 590 void vfio_pci_device_cleanup(struct vfio_pci_device *device) 591 { 592 int i; 593 594 if (device->driver.initialized) 595 vfio_pci_driver_remove(device); 596 597 vfio_pci_bar_unmap_all(device); 598 599 VFIO_ASSERT_EQ(close(device->fd), 0); 600 601 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 602 if (device->msi_eventfds[i] < 0) 603 continue; 604 605 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 606 } 607 608 if (device->iommufd) { 609 VFIO_ASSERT_EQ(close(device->iommufd), 0); 610 } else { 611 VFIO_ASSERT_EQ(close(device->group_fd), 0); 612 VFIO_ASSERT_EQ(close(device->container_fd), 0); 613 } 614 615 free(device); 616 } 617 618 static bool is_bdf(const char *str) 619 { 620 unsigned int s, b, d, f; 621 int length, count; 622 623 count = sscanf(str, "%4x:%2x:%2x.%2x%n", &s, &b, &d, &f, &length); 624 return count == 4 && length == strlen(str); 625 } 626 627 const char *vfio_selftests_get_bdf(int *argc, char *argv[]) 628 { 629 char *bdf; 630 631 if (*argc > 1 && is_bdf(argv[*argc - 1])) 632 return argv[--(*argc)]; 633 634 bdf = getenv("VFIO_SELFTESTS_BDF"); 635 if (bdf) { 636 VFIO_ASSERT_TRUE(is_bdf(bdf), "Invalid BDF: %s\n", bdf); 637 return bdf; 638 } 639 640 fprintf(stderr, "Unable to determine which device to use, skipping test.\n"); 641 fprintf(stderr, "\n"); 642 fprintf(stderr, "To pass the device address via environment variable:\n"); 643 fprintf(stderr, "\n"); 644 fprintf(stderr, " export VFIO_SELFTESTS_BDF=segment:bus:device.function\n"); 645 fprintf(stderr, " %s [options]\n", argv[0]); 646 fprintf(stderr, "\n"); 647 fprintf(stderr, "To pass the device address via argv:\n"); 648 fprintf(stderr, "\n"); 649 fprintf(stderr, " %s [options] segment:bus:device.function\n", argv[0]); 650 fprintf(stderr, "\n"); 651 exit(KSFT_SKIP); 652 } 653