1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <dirent.h> 3 #include <fcntl.h> 4 #include <libgen.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <unistd.h> 8 9 #include <sys/eventfd.h> 10 #include <sys/ioctl.h> 11 #include <sys/mman.h> 12 13 #include <uapi/linux/types.h> 14 #include <linux/limits.h> 15 #include <linux/mman.h> 16 #include <linux/types.h> 17 #include <linux/vfio.h> 18 #include <linux/iommufd.h> 19 20 #include "../../../kselftest.h" 21 #include <vfio_util.h> 22 23 #define PCI_SYSFS_PATH "/sys/bus/pci/devices" 24 25 #define ioctl_assert(_fd, _op, _arg) do { \ 26 void *__arg = (_arg); \ 27 int __ret = ioctl((_fd), (_op), (__arg)); \ 28 VFIO_ASSERT_EQ(__ret, 0, "ioctl(%s, %s, %s) returned %d\n", #_fd, #_op, #_arg, __ret); \ 29 } while (0) 30 31 iova_t __to_iova(struct vfio_pci_device *device, void *vaddr) 32 { 33 struct vfio_dma_region *region; 34 35 list_for_each_entry(region, &device->dma_regions, link) { 36 if (vaddr < region->vaddr) 37 continue; 38 39 if (vaddr >= region->vaddr + region->size) 40 continue; 41 42 return region->iova + (vaddr - region->vaddr); 43 } 44 45 return INVALID_IOVA; 46 } 47 48 iova_t to_iova(struct vfio_pci_device *device, void *vaddr) 49 { 50 iova_t iova; 51 52 iova = __to_iova(device, vaddr); 53 VFIO_ASSERT_NE(iova, INVALID_IOVA, "%p is not mapped into device.\n", vaddr); 54 55 return iova; 56 } 57 58 static void vfio_pci_irq_set(struct vfio_pci_device *device, 59 u32 index, u32 vector, u32 count, int *fds) 60 { 61 u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count] = {}; 62 struct vfio_irq_set *irq = (void *)&buf; 63 int *irq_fds = (void *)&irq->data; 64 65 irq->argsz = sizeof(buf); 66 irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; 67 irq->index = index; 68 irq->start = vector; 69 irq->count = count; 70 71 if (count) { 72 irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; 73 memcpy(irq_fds, fds, sizeof(int) * count); 74 } else { 75 irq->flags |= VFIO_IRQ_SET_DATA_NONE; 76 } 77 78 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); 79 } 80 81 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) 82 { 83 struct vfio_irq_set irq = { 84 .argsz = sizeof(irq), 85 .flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE, 86 .index = index, 87 .start = vector, 88 .count = 1, 89 }; 90 91 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq); 92 } 93 94 static void check_supported_irq_index(u32 index) 95 { 96 /* VFIO selftests only supports MSI and MSI-x for now. */ 97 VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX || 98 index == VFIO_PCI_MSIX_IRQ_INDEX, 99 "Unsupported IRQ index: %u\n", index); 100 } 101 102 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector, 103 int count) 104 { 105 int i; 106 107 check_supported_irq_index(index); 108 109 for (i = vector; i < vector + count; i++) { 110 VFIO_ASSERT_LT(device->msi_eventfds[i], 0); 111 device->msi_eventfds[i] = eventfd(0, 0); 112 VFIO_ASSERT_GE(device->msi_eventfds[i], 0); 113 } 114 115 vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector); 116 } 117 118 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index) 119 { 120 int i; 121 122 check_supported_irq_index(index); 123 124 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 125 if (device->msi_eventfds[i] < 0) 126 continue; 127 128 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 129 device->msi_eventfds[i] = -1; 130 } 131 132 vfio_pci_irq_set(device, index, 0, 0, NULL); 133 } 134 135 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, 136 struct vfio_irq_info *irq_info) 137 { 138 irq_info->argsz = sizeof(*irq_info); 139 irq_info->index = index; 140 141 ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info); 142 } 143 144 static void vfio_iommu_dma_map(struct vfio_pci_device *device, 145 struct vfio_dma_region *region) 146 { 147 struct vfio_iommu_type1_dma_map args = { 148 .argsz = sizeof(args), 149 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 150 .vaddr = (u64)region->vaddr, 151 .iova = region->iova, 152 .size = region->size, 153 }; 154 155 ioctl_assert(device->container_fd, VFIO_IOMMU_MAP_DMA, &args); 156 } 157 158 static void iommufd_dma_map(struct vfio_pci_device *device, 159 struct vfio_dma_region *region) 160 { 161 struct iommu_ioas_map args = { 162 .size = sizeof(args), 163 .flags = IOMMU_IOAS_MAP_READABLE | 164 IOMMU_IOAS_MAP_WRITEABLE | 165 IOMMU_IOAS_MAP_FIXED_IOVA, 166 .user_va = (u64)region->vaddr, 167 .iova = region->iova, 168 .length = region->size, 169 .ioas_id = device->ioas_id, 170 }; 171 172 ioctl_assert(device->iommufd, IOMMU_IOAS_MAP, &args); 173 } 174 175 void vfio_pci_dma_map(struct vfio_pci_device *device, 176 struct vfio_dma_region *region) 177 { 178 if (device->iommufd) 179 iommufd_dma_map(device, region); 180 else 181 vfio_iommu_dma_map(device, region); 182 183 list_add(®ion->link, &device->dma_regions); 184 } 185 186 static void vfio_iommu_dma_unmap(struct vfio_pci_device *device, 187 struct vfio_dma_region *region) 188 { 189 struct vfio_iommu_type1_dma_unmap args = { 190 .argsz = sizeof(args), 191 .iova = region->iova, 192 .size = region->size, 193 }; 194 195 ioctl_assert(device->container_fd, VFIO_IOMMU_UNMAP_DMA, &args); 196 } 197 198 static void iommufd_dma_unmap(struct vfio_pci_device *device, 199 struct vfio_dma_region *region) 200 { 201 struct iommu_ioas_unmap args = { 202 .size = sizeof(args), 203 .iova = region->iova, 204 .length = region->size, 205 .ioas_id = device->ioas_id, 206 }; 207 208 ioctl_assert(device->iommufd, IOMMU_IOAS_UNMAP, &args); 209 } 210 211 void vfio_pci_dma_unmap(struct vfio_pci_device *device, 212 struct vfio_dma_region *region) 213 { 214 if (device->iommufd) 215 iommufd_dma_unmap(device, region); 216 else 217 vfio_iommu_dma_unmap(device, region); 218 219 list_del(®ion->link); 220 } 221 222 static void vfio_pci_region_get(struct vfio_pci_device *device, int index, 223 struct vfio_region_info *info) 224 { 225 memset(info, 0, sizeof(*info)); 226 227 info->argsz = sizeof(*info); 228 info->index = index; 229 230 ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info); 231 } 232 233 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index) 234 { 235 struct vfio_pci_bar *bar = &device->bars[index]; 236 int prot = 0; 237 238 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 239 VFIO_ASSERT_NULL(bar->vaddr); 240 VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP); 241 242 if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ) 243 prot |= PROT_READ; 244 if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE) 245 prot |= PROT_WRITE; 246 247 bar->vaddr = mmap(NULL, bar->info.size, prot, MAP_FILE | MAP_SHARED, 248 device->fd, bar->info.offset); 249 VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED); 250 } 251 252 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index) 253 { 254 struct vfio_pci_bar *bar = &device->bars[index]; 255 256 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 257 VFIO_ASSERT_NOT_NULL(bar->vaddr); 258 259 VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0); 260 bar->vaddr = NULL; 261 } 262 263 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device) 264 { 265 int i; 266 267 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 268 if (device->bars[i].vaddr) 269 vfio_pci_bar_unmap(device, i); 270 } 271 } 272 273 void vfio_pci_config_access(struct vfio_pci_device *device, bool write, 274 size_t config, size_t size, void *data) 275 { 276 struct vfio_region_info *config_space = &device->config_space; 277 int ret; 278 279 if (write) 280 ret = pwrite(device->fd, data, size, config_space->offset + config); 281 else 282 ret = pread(device->fd, data, size, config_space->offset + config); 283 284 VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n", 285 write ? "write to" : "read from", config); 286 } 287 288 void vfio_pci_device_reset(struct vfio_pci_device *device) 289 { 290 ioctl_assert(device->fd, VFIO_DEVICE_RESET, NULL); 291 } 292 293 static unsigned int vfio_pci_get_group_from_dev(const char *bdf) 294 { 295 char dev_iommu_group_path[PATH_MAX] = {0}; 296 char sysfs_path[PATH_MAX] = {0}; 297 unsigned int group; 298 int ret; 299 300 snprintf(sysfs_path, PATH_MAX, "%s/%s/iommu_group", PCI_SYSFS_PATH, bdf); 301 302 ret = readlink(sysfs_path, dev_iommu_group_path, sizeof(dev_iommu_group_path)); 303 VFIO_ASSERT_NE(ret, -1, "Failed to get the IOMMU group for device: %s\n", bdf); 304 305 ret = sscanf(basename(dev_iommu_group_path), "%u", &group); 306 VFIO_ASSERT_EQ(ret, 1, "Failed to get the IOMMU group for device: %s\n", bdf); 307 308 return group; 309 } 310 311 static void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf) 312 { 313 struct vfio_group_status group_status = { 314 .argsz = sizeof(group_status), 315 }; 316 char group_path[32]; 317 int group; 318 319 group = vfio_pci_get_group_from_dev(bdf); 320 snprintf(group_path, sizeof(group_path), "/dev/vfio/%d", group); 321 322 device->group_fd = open(group_path, O_RDWR); 323 VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path); 324 325 ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status); 326 VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE); 327 328 ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->container_fd); 329 } 330 331 static void vfio_pci_container_setup(struct vfio_pci_device *device, const char *bdf) 332 { 333 unsigned long iommu_type = device->iommu_mode->iommu_type; 334 const char *path = device->iommu_mode->container_path; 335 int version; 336 int ret; 337 338 device->container_fd = open(path, O_RDWR); 339 VFIO_ASSERT_GE(device->container_fd, 0, "open(%s) failed\n", path); 340 341 version = ioctl(device->container_fd, VFIO_GET_API_VERSION); 342 VFIO_ASSERT_EQ(version, VFIO_API_VERSION, "Unsupported version: %d\n", version); 343 344 vfio_pci_group_setup(device, bdf); 345 346 ret = ioctl(device->container_fd, VFIO_CHECK_EXTENSION, iommu_type); 347 VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type); 348 349 ioctl_assert(device->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); 350 351 device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf); 352 VFIO_ASSERT_GE(device->fd, 0); 353 } 354 355 static void vfio_pci_device_setup(struct vfio_pci_device *device) 356 { 357 int i; 358 359 device->info.argsz = sizeof(device->info); 360 ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info); 361 362 vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space); 363 364 /* Sanity check VFIO does not advertise mmap for config space */ 365 VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP), 366 "PCI config space should not support mmap()\n"); 367 368 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 369 struct vfio_pci_bar *bar = device->bars + i; 370 371 vfio_pci_region_get(device, i, &bar->info); 372 if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP) 373 vfio_pci_bar_map(device, i); 374 } 375 376 vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info); 377 vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info); 378 379 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) 380 device->msi_eventfds[i] = -1; 381 } 382 383 const char *vfio_pci_get_cdev_path(const char *bdf) 384 { 385 char dir_path[PATH_MAX]; 386 struct dirent *entry; 387 char *cdev_path; 388 DIR *dir; 389 390 cdev_path = calloc(PATH_MAX, 1); 391 VFIO_ASSERT_NOT_NULL(cdev_path); 392 393 snprintf(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf); 394 395 dir = opendir(dir_path); 396 VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path); 397 398 while ((entry = readdir(dir)) != NULL) { 399 /* Find the file that starts with "vfio" */ 400 if (strncmp("vfio", entry->d_name, 4)) 401 continue; 402 403 snprintf(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name); 404 break; 405 } 406 407 VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n"); 408 VFIO_ASSERT_EQ(closedir(dir), 0); 409 410 return cdev_path; 411 } 412 413 /* Reminder: Keep in sync with FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(). */ 414 static const struct vfio_iommu_mode iommu_modes[] = { 415 { 416 .name = "vfio_type1_iommu", 417 .container_path = "/dev/vfio/vfio", 418 .iommu_type = VFIO_TYPE1_IOMMU, 419 }, 420 { 421 .name = "vfio_type1v2_iommu", 422 .container_path = "/dev/vfio/vfio", 423 .iommu_type = VFIO_TYPE1v2_IOMMU, 424 }, 425 { 426 .name = "iommufd_compat_type1", 427 .container_path = "/dev/iommu", 428 .iommu_type = VFIO_TYPE1_IOMMU, 429 }, 430 { 431 .name = "iommufd_compat_type1v2", 432 .container_path = "/dev/iommu", 433 .iommu_type = VFIO_TYPE1v2_IOMMU, 434 }, 435 { 436 .name = "iommufd", 437 }, 438 }; 439 440 const char *default_iommu_mode = "iommufd"; 441 442 static const struct vfio_iommu_mode *lookup_iommu_mode(const char *iommu_mode) 443 { 444 int i; 445 446 if (!iommu_mode) 447 iommu_mode = default_iommu_mode; 448 449 for (i = 0; i < ARRAY_SIZE(iommu_modes); i++) { 450 if (strcmp(iommu_mode, iommu_modes[i].name)) 451 continue; 452 453 return &iommu_modes[i]; 454 } 455 456 VFIO_FAIL("Unrecognized IOMMU mode: %s\n", iommu_mode); 457 } 458 459 static void vfio_device_bind_iommufd(int device_fd, int iommufd) 460 { 461 struct vfio_device_bind_iommufd args = { 462 .argsz = sizeof(args), 463 .iommufd = iommufd, 464 }; 465 466 ioctl_assert(device_fd, VFIO_DEVICE_BIND_IOMMUFD, &args); 467 } 468 469 static u32 iommufd_ioas_alloc(int iommufd) 470 { 471 struct iommu_ioas_alloc args = { 472 .size = sizeof(args), 473 }; 474 475 ioctl_assert(iommufd, IOMMU_IOAS_ALLOC, &args); 476 return args.out_ioas_id; 477 } 478 479 static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id) 480 { 481 struct vfio_device_attach_iommufd_pt args = { 482 .argsz = sizeof(args), 483 .pt_id = pt_id, 484 }; 485 486 ioctl_assert(device_fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &args); 487 } 488 489 static void vfio_pci_iommufd_setup(struct vfio_pci_device *device, const char *bdf) 490 { 491 const char *cdev_path = vfio_pci_get_cdev_path(bdf); 492 493 device->fd = open(cdev_path, O_RDWR); 494 VFIO_ASSERT_GE(device->fd, 0); 495 free((void *)cdev_path); 496 497 /* 498 * Require device->iommufd to be >0 so that a simple non-0 check can be 499 * used to check if iommufd is enabled. In practice open() will never 500 * return 0 unless stdin is closed. 501 */ 502 device->iommufd = open("/dev/iommu", O_RDWR); 503 VFIO_ASSERT_GT(device->iommufd, 0); 504 505 vfio_device_bind_iommufd(device->fd, device->iommufd); 506 device->ioas_id = iommufd_ioas_alloc(device->iommufd); 507 vfio_device_attach_iommufd_pt(device->fd, device->ioas_id); 508 } 509 510 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode) 511 { 512 struct vfio_pci_device *device; 513 514 device = calloc(1, sizeof(*device)); 515 VFIO_ASSERT_NOT_NULL(device); 516 517 INIT_LIST_HEAD(&device->dma_regions); 518 519 device->iommu_mode = lookup_iommu_mode(iommu_mode); 520 521 if (device->iommu_mode->container_path) 522 vfio_pci_container_setup(device, bdf); 523 else 524 vfio_pci_iommufd_setup(device, bdf); 525 526 vfio_pci_device_setup(device); 527 vfio_pci_driver_probe(device); 528 529 return device; 530 } 531 532 void vfio_pci_device_cleanup(struct vfio_pci_device *device) 533 { 534 int i; 535 536 if (device->driver.initialized) 537 vfio_pci_driver_remove(device); 538 539 vfio_pci_bar_unmap_all(device); 540 541 VFIO_ASSERT_EQ(close(device->fd), 0); 542 543 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 544 if (device->msi_eventfds[i] < 0) 545 continue; 546 547 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 548 } 549 550 if (device->iommufd) { 551 VFIO_ASSERT_EQ(close(device->iommufd), 0); 552 } else { 553 VFIO_ASSERT_EQ(close(device->group_fd), 0); 554 VFIO_ASSERT_EQ(close(device->container_fd), 0); 555 } 556 557 free(device); 558 } 559 560 static bool is_bdf(const char *str) 561 { 562 unsigned int s, b, d, f; 563 int length, count; 564 565 count = sscanf(str, "%4x:%2x:%2x.%2x%n", &s, &b, &d, &f, &length); 566 return count == 4 && length == strlen(str); 567 } 568 569 const char *vfio_selftests_get_bdf(int *argc, char *argv[]) 570 { 571 char *bdf; 572 573 if (*argc > 1 && is_bdf(argv[*argc - 1])) 574 return argv[--(*argc)]; 575 576 bdf = getenv("VFIO_SELFTESTS_BDF"); 577 if (bdf) { 578 VFIO_ASSERT_TRUE(is_bdf(bdf), "Invalid BDF: %s\n", bdf); 579 return bdf; 580 } 581 582 fprintf(stderr, "Unable to determine which device to use, skipping test.\n"); 583 fprintf(stderr, "\n"); 584 fprintf(stderr, "To pass the device address via environment variable:\n"); 585 fprintf(stderr, "\n"); 586 fprintf(stderr, " export VFIO_SELFTESTS_BDF=segment:bus:device.function\n"); 587 fprintf(stderr, " %s [options]\n", argv[0]); 588 fprintf(stderr, "\n"); 589 fprintf(stderr, "To pass the device address via argv:\n"); 590 fprintf(stderr, "\n"); 591 fprintf(stderr, " %s [options] segment:bus:device.function\n", argv[0]); 592 fprintf(stderr, "\n"); 593 exit(KSFT_SKIP); 594 } 595