1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <dirent.h> 3 #include <fcntl.h> 4 #include <libgen.h> 5 #include <stdint.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 10 #include <sys/eventfd.h> 11 #include <sys/ioctl.h> 12 #include <sys/mman.h> 13 14 #include <linux/align.h> 15 #include <linux/iommufd.h> 16 #include <linux/kernel.h> 17 #include <linux/limits.h> 18 #include <linux/log2.h> 19 #include <linux/mman.h> 20 #include <linux/overflow.h> 21 #include <linux/sizes.h> 22 #include <linux/types.h> 23 #include <linux/vfio.h> 24 25 #include <uuid/uuid.h> 26 27 #include "kselftest.h" 28 #include <libvfio.h> 29 30 static void vfio_pci_irq_set(struct vfio_pci_device *device, 31 u32 index, u32 vector, u32 count, int *fds) 32 { 33 u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count]; 34 struct vfio_irq_set *irq = (void *)&buf; 35 int *irq_fds = (void *)&irq->data; 36 37 memset(buf, 0, sizeof(buf)); 38 39 irq->argsz = sizeof(buf); 40 irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; 41 irq->index = index; 42 irq->start = vector; 43 irq->count = count; 44 45 if (count) { 46 irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD; 47 memcpy(irq_fds, fds, sizeof(int) * count); 48 } else { 49 irq->flags |= VFIO_IRQ_SET_DATA_NONE; 50 } 51 52 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq); 53 } 54 55 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector) 56 { 57 struct vfio_irq_set irq = { 58 .argsz = sizeof(irq), 59 .flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE, 60 .index = index, 61 .start = vector, 62 .count = 1, 63 }; 64 65 ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq); 66 } 67 68 static void check_supported_irq_index(u32 index) 69 { 70 /* VFIO selftests only supports MSI and MSI-x for now. */ 71 VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX || 72 index == VFIO_PCI_MSIX_IRQ_INDEX, 73 "Unsupported IRQ index: %u\n", index); 74 } 75 76 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector, 77 int count) 78 { 79 int i; 80 81 check_supported_irq_index(index); 82 83 for (i = vector; i < vector + count; i++) { 84 VFIO_ASSERT_LT(device->msi_eventfds[i], 0); 85 device->msi_eventfds[i] = eventfd(0, 0); 86 VFIO_ASSERT_GE(device->msi_eventfds[i], 0); 87 } 88 89 vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector); 90 } 91 92 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index) 93 { 94 int i; 95 96 check_supported_irq_index(index); 97 98 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 99 if (device->msi_eventfds[i] < 0) 100 continue; 101 102 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 103 device->msi_eventfds[i] = -1; 104 } 105 106 vfio_pci_irq_set(device, index, 0, 0, NULL); 107 } 108 109 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index, 110 struct vfio_irq_info *irq_info) 111 { 112 irq_info->argsz = sizeof(*irq_info); 113 irq_info->index = index; 114 115 ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info); 116 } 117 118 static void vfio_pci_region_get(struct vfio_pci_device *device, int index, 119 struct vfio_region_info *info) 120 { 121 memset(info, 0, sizeof(*info)); 122 123 info->argsz = sizeof(*info); 124 info->index = index; 125 126 ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info); 127 } 128 129 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index) 130 { 131 struct vfio_pci_bar *bar = &device->bars[index]; 132 size_t align, size; 133 int prot = 0; 134 void *vaddr; 135 136 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 137 VFIO_ASSERT_NULL(bar->vaddr); 138 VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP); 139 VFIO_ASSERT_TRUE(is_power_of_2(bar->info.size)); 140 141 if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ) 142 prot |= PROT_READ; 143 if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE) 144 prot |= PROT_WRITE; 145 146 size = bar->info.size; 147 148 /* 149 * Align BAR mmaps to improve page fault granularity during potential 150 * subsequent IOMMU mapping of these BAR vaddr. 1G for x86 is the 151 * largest hugepage size across any architecture, so no benefit from 152 * larger alignment. BARs smaller than 1G will be aligned by their 153 * power-of-two size, guaranteeing sufficient alignment for smaller 154 * hugepages, if present. 155 */ 156 align = min_t(size_t, size, SZ_1G); 157 158 vaddr = mmap_reserve(size, align, 0); 159 bar->vaddr = mmap(vaddr, size, prot, MAP_SHARED | MAP_FIXED, 160 device->fd, bar->info.offset); 161 VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED); 162 163 madvise(bar->vaddr, size, MADV_HUGEPAGE); 164 } 165 166 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index) 167 { 168 struct vfio_pci_bar *bar = &device->bars[index]; 169 170 VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS); 171 VFIO_ASSERT_NOT_NULL(bar->vaddr); 172 173 VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0); 174 bar->vaddr = NULL; 175 } 176 177 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device) 178 { 179 int i; 180 181 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 182 if (device->bars[i].vaddr) 183 vfio_pci_bar_unmap(device, i); 184 } 185 } 186 187 void vfio_pci_config_access(struct vfio_pci_device *device, bool write, 188 size_t config, size_t size, void *data) 189 { 190 struct vfio_region_info *config_space = &device->config_space; 191 int ret; 192 193 if (write) 194 ret = pwrite(device->fd, data, size, config_space->offset + config); 195 else 196 ret = pread(device->fd, data, size, config_space->offset + config); 197 198 VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n", 199 write ? "write to" : "read from", config); 200 } 201 202 void vfio_pci_device_reset(struct vfio_pci_device *device) 203 { 204 ioctl_assert(device->fd, VFIO_DEVICE_RESET, NULL); 205 } 206 207 static void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf) 208 { 209 struct vfio_group_status group_status = { 210 .argsz = sizeof(group_status), 211 }; 212 char group_path[32]; 213 int group; 214 215 group = sysfs_iommu_group_get(bdf); 216 snprintf_assert(group_path, sizeof(group_path), "/dev/vfio/%d", group); 217 218 device->group_fd = open(group_path, O_RDWR); 219 VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path); 220 221 ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status); 222 VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE); 223 224 ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->iommu->container_fd); 225 } 226 227 static void vfio_pci_group_get_device_fd(struct vfio_pci_device *device, 228 const char *bdf, const char *vf_token) 229 { 230 char arg[64]; 231 232 /* 233 * If a vf_token exists, argument to VFIO_GROUP_GET_DEVICE_FD 234 * will be in the form of the following example: 235 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 236 */ 237 if (vf_token) 238 snprintf_assert(arg, ARRAY_SIZE(arg), "%s vf_token=%s", bdf, vf_token); 239 else 240 snprintf_assert(arg, ARRAY_SIZE(arg), "%s", bdf); 241 242 device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, arg); 243 VFIO_ASSERT_GE(device->fd, 0); 244 } 245 246 static void vfio_pci_container_setup(struct vfio_pci_device *device, 247 const char *bdf, const char *vf_token) 248 { 249 struct iommu *iommu = device->iommu; 250 unsigned long iommu_type = iommu->mode->iommu_type; 251 int ret; 252 253 vfio_pci_group_setup(device, bdf); 254 255 ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type); 256 VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type); 257 258 /* 259 * Allow multiple threads to race to set the IOMMU type on the 260 * container. The first will succeed and the rest should fail 261 * because the IOMMU type is already set. 262 */ 263 (void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); 264 265 vfio_pci_group_get_device_fd(device, bdf, vf_token); 266 } 267 268 static void vfio_pci_device_setup(struct vfio_pci_device *device) 269 { 270 int i; 271 272 device->info.argsz = sizeof(device->info); 273 ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info); 274 275 vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space); 276 277 /* Sanity check VFIO does not advertise mmap for config space */ 278 VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP), 279 "PCI config space should not support mmap()\n"); 280 281 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 282 struct vfio_pci_bar *bar = device->bars + i; 283 284 vfio_pci_region_get(device, i, &bar->info); 285 if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP) 286 vfio_pci_bar_map(device, i); 287 } 288 289 vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info); 290 vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info); 291 292 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) 293 device->msi_eventfds[i] = -1; 294 } 295 296 const char *vfio_pci_get_cdev_path(const char *bdf) 297 { 298 char dir_path[PATH_MAX]; 299 struct dirent *entry; 300 char *cdev_path; 301 DIR *dir; 302 303 cdev_path = calloc(PATH_MAX, 1); 304 VFIO_ASSERT_NOT_NULL(cdev_path); 305 306 snprintf_assert(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf); 307 308 dir = opendir(dir_path); 309 VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path); 310 311 while ((entry = readdir(dir)) != NULL) { 312 /* Find the file that starts with "vfio" */ 313 if (strncmp("vfio", entry->d_name, 4)) 314 continue; 315 316 snprintf_assert(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name); 317 break; 318 } 319 320 VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n"); 321 VFIO_ASSERT_EQ(closedir(dir), 0); 322 323 return cdev_path; 324 } 325 326 static void vfio_device_bind_iommufd(int device_fd, int iommufd, 327 const char *vf_token) 328 { 329 struct vfio_device_bind_iommufd args = { 330 .argsz = sizeof(args), 331 .iommufd = iommufd, 332 }; 333 uuid_t token_uuid; 334 335 if (vf_token) { 336 VFIO_ASSERT_EQ(uuid_parse(vf_token, token_uuid), 0); 337 args.flags |= VFIO_DEVICE_BIND_FLAG_TOKEN; 338 args.token_uuid_ptr = (u64)token_uuid; 339 } 340 341 ioctl_assert(device_fd, VFIO_DEVICE_BIND_IOMMUFD, &args); 342 } 343 344 static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id) 345 { 346 struct vfio_device_attach_iommufd_pt args = { 347 .argsz = sizeof(args), 348 .pt_id = pt_id, 349 }; 350 351 ioctl_assert(device_fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &args); 352 } 353 354 static void vfio_pci_iommufd_setup(struct vfio_pci_device *device, 355 const char *bdf, const char *vf_token) 356 { 357 const char *cdev_path = vfio_pci_get_cdev_path(bdf); 358 359 device->fd = open(cdev_path, O_RDWR); 360 VFIO_ASSERT_GE(device->fd, 0); 361 free((void *)cdev_path); 362 363 vfio_device_bind_iommufd(device->fd, device->iommu->iommufd, vf_token); 364 vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id); 365 } 366 367 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu) 368 { 369 struct vfio_pci_device *device; 370 371 device = calloc(1, sizeof(*device)); 372 VFIO_ASSERT_NOT_NULL(device); 373 374 VFIO_ASSERT_NOT_NULL(iommu); 375 device->iommu = iommu; 376 device->bdf = bdf; 377 378 if (iommu->mode->container_path) 379 vfio_pci_container_setup(device, bdf, NULL); 380 else 381 vfio_pci_iommufd_setup(device, bdf, NULL); 382 383 vfio_pci_device_setup(device); 384 vfio_pci_driver_probe(device); 385 386 return device; 387 } 388 389 void vfio_pci_device_cleanup(struct vfio_pci_device *device) 390 { 391 int i; 392 393 if (device->driver.initialized) 394 vfio_pci_driver_remove(device); 395 396 vfio_pci_bar_unmap_all(device); 397 398 VFIO_ASSERT_EQ(close(device->fd), 0); 399 400 for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) { 401 if (device->msi_eventfds[i] < 0) 402 continue; 403 404 VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); 405 } 406 407 if (device->group_fd) 408 VFIO_ASSERT_EQ(close(device->group_fd), 0); 409 410 free(device); 411 } 412