1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <dirent.h> 3 #include <fcntl.h> 4 #include <libgen.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <unistd.h> 8 9 #include <sys/eventfd.h> 10 #include <sys/ioctl.h> 11 #include <sys/mman.h> 12 13 #include <linux/limits.h> 14 #include <linux/mman.h> 15 #include <linux/types.h> 16 #include <linux/vfio.h> 17 18 #include "../../../kselftest.h" 19 #include <vfio_util.h> 20 21 #define VFIO_DEV_PATH "/dev/vfio/vfio" 22 #define PCI_SYSFS_PATH "/sys/bus/pci/devices" 23 24 #define ioctl_assert(_fd, _op, _arg) do { \ 25 void *__arg = (_arg); \ 26 int __ret = ioctl((_fd), (_op), (__arg)); \ 27 VFIO_ASSERT_EQ(__ret, 0, "ioctl(%s, %s, %s) returned %d\n", #_fd, #_op, #_arg, __ret); \ 28 } while (0) 29 30 iova_t __to_iova(struct vfio_pci_device *device, void *vaddr) 31 { 32 struct vfio_dma_region *region; 33 34 list_for_each_entry(region, &device->dma_regions, link) { 35 if (vaddr < region->vaddr) 36 continue; 37 38 if (vaddr >= region->vaddr + region->size) 39 continue; 40 41 return region->iova + (vaddr - region->vaddr); 42 } 43 44 return INVALID_IOVA; 45 } 46 47 iova_t to_iova(struct vfio_pci_device *device, void *vaddr) 48 { 49 iova_t iova; 50 51 iova = __to_iova(device, vaddr); 52 VFIO_ASSERT_NE(iova, INVALID_IOVA, "%p is not mapped into device.\n", vaddr); 53 54 return iova; 55 } 56 57 static void vfio_pci_irq_set(struct vfio_pci_device *device, 58 u32 index, u32 vector, u32 count, int *fds) 59 { 60 u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count] = {}; 61 struct vfio_irq_set *irq = (void *)&buf; 62 int *irq_fds = (void *)&irq->data; 63 64 irq->argsz = sizeof(buf); 65 irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; 66 irq->index = index; 67 irq->start = vector; 68 irq->count = count; 69 70 if (count) { 71 irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; 72 memcpy(irq_fds, fds, sizeof(int) * count); 73 } else { 74 irq->flags |= VFIO_IRQ_SET_DATA_NONE; 75 } 76 77 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); 78 } 79 80 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) 81 { 82 struct vfio_irq_set irq = { 83 .argsz = sizeof(irq), 84 .flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE, 85 .index = index, 86 .start = vector, 87 .count = 1, 88 }; 89 90 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq); 91 } 92 93 static void check_supported_irq_index(u32 index) 94 { 95 /* VFIO selftests only supports MSI and MSI-x for now. */ 96 VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX || 97 index == VFIO_PCI_MSIX_IRQ_INDEX, 98 "Unsupported IRQ index: %u\n", index); 99 } 100 101 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector, 102 int count) 103 { 104 int i; 105 106 check_supported_irq_index(index); 107 108 for (i = vector; i < vector + count; i++) { 109 VFIO_ASSERT_LT(device->msi_eventfds[i], 0); 110 device->msi_eventfds[i] = eventfd(0, 0); 111 VFIO_ASSERT_GE(device->msi_eventfds[i], 0); 112 } 113 114 vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector); 115 } 116 117 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index) 118 { 119 int i; 120 121 check_supported_irq_index(index); 122 123 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 124 if (device->msi_eventfds[i] < 0) 125 continue; 126 127 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 128 device->msi_eventfds[i] = -1; 129 } 130 131 vfio_pci_irq_set(device, index, 0, 0, NULL); 132 } 133 134 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, 135 struct vfio_irq_info *irq_info) 136 { 137 irq_info->argsz = sizeof(*irq_info); 138 irq_info->index = index; 139 140 ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info); 141 } 142 143 void vfio_pci_dma_map(struct vfio_pci_device *device, 144 struct vfio_dma_region *region) 145 { 146 struct vfio_iommu_type1_dma_map map = { 147 .argsz = sizeof(map), 148 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 149 .vaddr = (u64)region->vaddr, 150 .iova = region->iova, 151 .size = region->size, 152 }; 153 154 ioctl_assert(device->container_fd, VFIO_IOMMU_MAP_DMA, &map); 155 156 list_add(®ion->link, &device->dma_regions); 157 } 158 159 void vfio_pci_dma_unmap(struct vfio_pci_device *device, 160 struct vfio_dma_region *region) 161 { 162 struct vfio_iommu_type1_dma_unmap unmap = { 163 .argsz = sizeof(unmap), 164 .iova = region->iova, 165 .size = region->size, 166 }; 167 168 ioctl_assert(device->container_fd, VFIO_IOMMU_UNMAP_DMA, &unmap); 169 170 list_del(®ion->link); 171 } 172 173 static void vfio_pci_region_get(struct vfio_pci_device *device, int index, 174 struct vfio_region_info *info) 175 { 176 memset(info, 0, sizeof(*info)); 177 178 info->argsz = sizeof(*info); 179 info->index = index; 180 181 ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info); 182 } 183 184 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index) 185 { 186 struct vfio_pci_bar *bar = &device->bars[index]; 187 int prot = 0; 188 189 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 190 VFIO_ASSERT_NULL(bar->vaddr); 191 VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP); 192 193 if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ) 194 prot |= PROT_READ; 195 if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE) 196 prot |= PROT_WRITE; 197 198 bar->vaddr = mmap(NULL, bar->info.size, prot, MAP_FILE | MAP_SHARED, 199 device->fd, bar->info.offset); 200 VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED); 201 } 202 203 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index) 204 { 205 struct vfio_pci_bar *bar = &device->bars[index]; 206 207 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 208 VFIO_ASSERT_NOT_NULL(bar->vaddr); 209 210 VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0); 211 bar->vaddr = NULL; 212 } 213 214 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device) 215 { 216 int i; 217 218 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 219 if (device->bars[i].vaddr) 220 vfio_pci_bar_unmap(device, i); 221 } 222 } 223 224 void vfio_pci_config_access(struct vfio_pci_device *device, bool write, 225 size_t config, size_t size, void *data) 226 { 227 struct vfio_region_info *config_space = &device->config_space; 228 int ret; 229 230 if (write) 231 ret = pwrite(device->fd, data, size, config_space->offset + config); 232 else 233 ret = pread(device->fd, data, size, config_space->offset + config); 234 235 VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n", 236 write ? "write to" : "read from", config); 237 } 238 239 void vfio_pci_device_reset(struct vfio_pci_device *device) 240 { 241 ioctl_assert(device->fd, VFIO_DEVICE_RESET, NULL); 242 } 243 244 static unsigned int vfio_pci_get_group_from_dev(const char *bdf) 245 { 246 char dev_iommu_group_path[PATH_MAX] = {0}; 247 char sysfs_path[PATH_MAX] = {0}; 248 unsigned int group; 249 int ret; 250 251 snprintf(sysfs_path, PATH_MAX, "%s/%s/iommu_group", PCI_SYSFS_PATH, bdf); 252 253 ret = readlink(sysfs_path, dev_iommu_group_path, sizeof(dev_iommu_group_path)); 254 VFIO_ASSERT_NE(ret, -1, "Failed to get the IOMMU group for device: %s\n", bdf); 255 256 ret = sscanf(basename(dev_iommu_group_path), "%u", &group); 257 VFIO_ASSERT_EQ(ret, 1, "Failed to get the IOMMU group for device: %s\n", bdf); 258 259 return group; 260 } 261 262 static void vfio_pci_container_setup(struct vfio_pci_device *device) 263 { 264 int version; 265 266 device->container_fd = open(VFIO_DEV_PATH, O_RDWR); 267 VFIO_ASSERT_GE(device->container_fd, 0, "open(%s) failed\n", VFIO_DEV_PATH); 268 269 version = ioctl(device->container_fd, VFIO_GET_API_VERSION); 270 VFIO_ASSERT_EQ(version, VFIO_API_VERSION); 271 } 272 273 static void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf) 274 { 275 struct vfio_group_status group_status = { 276 .argsz = sizeof(group_status), 277 }; 278 char group_path[32]; 279 int group; 280 281 group = vfio_pci_get_group_from_dev(bdf); 282 snprintf(group_path, sizeof(group_path), "/dev/vfio/%d", group); 283 284 device->group_fd = open(group_path, O_RDWR); 285 VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path); 286 287 ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status); 288 VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE); 289 290 ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->container_fd); 291 } 292 293 static void vfio_pci_iommu_setup(struct vfio_pci_device *device, unsigned long iommu_type) 294 { 295 int ret; 296 297 INIT_LIST_HEAD(&device->dma_regions); 298 299 ret = ioctl(device->container_fd, VFIO_CHECK_EXTENSION, iommu_type); 300 VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type); 301 302 ioctl_assert(device->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); 303 } 304 305 static void vfio_pci_device_setup(struct vfio_pci_device *device, const char *bdf) 306 { 307 int i; 308 309 device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf); 310 VFIO_ASSERT_GE(device->fd, 0); 311 312 device->info.argsz = sizeof(device->info); 313 ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info); 314 315 vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space); 316 317 /* Sanity check VFIO does not advertise mmap for config space */ 318 VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP), 319 "PCI config space should not support mmap()\n"); 320 321 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 322 struct vfio_pci_bar *bar = device->bars + i; 323 324 vfio_pci_region_get(device, i, &bar->info); 325 if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP) 326 vfio_pci_bar_map(device, i); 327 } 328 329 vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info); 330 vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info); 331 332 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) 333 device->msi_eventfds[i] = -1; 334 } 335 336 const char *vfio_pci_get_cdev_path(const char *bdf) 337 { 338 char dir_path[PATH_MAX]; 339 struct dirent *entry; 340 char *cdev_path; 341 DIR *dir; 342 343 cdev_path = calloc(PATH_MAX, 1); 344 VFIO_ASSERT_NOT_NULL(cdev_path); 345 346 snprintf(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf); 347 348 dir = opendir(dir_path); 349 VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path); 350 351 while ((entry = readdir(dir)) != NULL) { 352 /* Find the file that starts with "vfio" */ 353 if (strncmp("vfio", entry->d_name, 4)) 354 continue; 355 356 snprintf(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name); 357 break; 358 } 359 360 VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n"); 361 VFIO_ASSERT_EQ(closedir(dir), 0); 362 363 return cdev_path; 364 } 365 366 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, int iommu_type) 367 { 368 struct vfio_pci_device *device; 369 370 device = calloc(1, sizeof(*device)); 371 VFIO_ASSERT_NOT_NULL(device); 372 373 vfio_pci_container_setup(device); 374 vfio_pci_group_setup(device, bdf); 375 vfio_pci_iommu_setup(device, iommu_type); 376 vfio_pci_device_setup(device, bdf); 377 378 vfio_pci_driver_probe(device); 379 380 return device; 381 } 382 383 void vfio_pci_device_cleanup(struct vfio_pci_device *device) 384 { 385 int i; 386 387 if (device->driver.initialized) 388 vfio_pci_driver_remove(device); 389 390 vfio_pci_bar_unmap_all(device); 391 392 VFIO_ASSERT_EQ(close(device->fd), 0); 393 394 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 395 if (device->msi_eventfds[i] < 0) 396 continue; 397 398 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 399 } 400 401 VFIO_ASSERT_EQ(close(device->group_fd), 0); 402 VFIO_ASSERT_EQ(close(device->container_fd), 0); 403 404 free(device); 405 } 406 407 static bool is_bdf(const char *str) 408 { 409 unsigned int s, b, d, f; 410 int length, count; 411 412 count = sscanf(str, "%4x:%2x:%2x.%2x%n", &s, &b, &d, &f, &length); 413 return count == 4 && length == strlen(str); 414 } 415 416 const char *vfio_selftests_get_bdf(int *argc, char *argv[]) 417 { 418 char *bdf; 419 420 if (*argc > 1 && is_bdf(argv[*argc - 1])) 421 return argv[--(*argc)]; 422 423 bdf = getenv("VFIO_SELFTESTS_BDF"); 424 if (bdf) { 425 VFIO_ASSERT_TRUE(is_bdf(bdf), "Invalid BDF: %s\n", bdf); 426 return bdf; 427 } 428 429 fprintf(stderr, "Unable to determine which device to use, skipping test.\n"); 430 fprintf(stderr, "\n"); 431 fprintf(stderr, "To pass the device address via environment variable:\n"); 432 fprintf(stderr, "\n"); 433 fprintf(stderr, " export VFIO_SELFTESTS_BDF=segment:bus:device.function\n"); 434 fprintf(stderr, " %s [options]\n", argv[0]); 435 fprintf(stderr, "\n"); 436 fprintf(stderr, "To pass the device address via argv:\n"); 437 fprintf(stderr, "\n"); 438 fprintf(stderr, " %s [options] segment:bus:device.function\n", argv[0]); 439 fprintf(stderr, "\n"); 440 exit(KSFT_SKIP); 441 } 442