xref: /linux/tools/testing/selftests/vfio/lib/vfio_pci_device.c (revision 28ca5f67e1a79f62ed7988674fb07a8fc9eab8d2) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <dirent.h>
3 #include <errno.h>
4 #include <fcntl.h>
5 #include <libgen.h>
6 #include <stdint.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <unistd.h>
10 
11 #include <sys/eventfd.h>
12 #include <sys/ioctl.h>
13 #include <sys/mman.h>
14 
15 #include <linux/align.h>
16 #include <linux/iommufd.h>
17 #include <linux/kernel.h>
18 #include <linux/limits.h>
19 #include <linux/log2.h>
20 #include <linux/mman.h>
21 #include <linux/overflow.h>
22 #include <linux/sizes.h>
23 #include <linux/types.h>
24 #include <linux/vfio.h>
25 
26 #include <uuid/uuid.h>
27 
28 #include "kselftest.h"
29 #include <libvfio.h>
30 
31 static void vfio_pci_irq_set(struct vfio_pci_device *device,
32 			     u32 index, u32 vector, u32 count, int *fds)
33 {
34 	size_t argsz = sizeof(struct vfio_irq_set) + sizeof(int) * count;
35 	struct vfio_irq_set *irq;
36 
37 	irq = calloc_assert(1, argsz);
38 	irq->argsz = argsz;
39 	irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER;
40 	irq->index = index;
41 	irq->start = vector;
42 	irq->count = count;
43 
44 	if (count) {
45 		irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD;
46 		memcpy(irq->data, fds, sizeof(int) * count);
47 	} else {
48 		irq->flags |= VFIO_IRQ_SET_DATA_NONE;
49 	}
50 
51 	ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq);
52 	free(irq);
53 }
54 
55 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector)
56 {
57 	struct vfio_irq_set irq = {
58 		.argsz = sizeof(irq),
59 		.flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE,
60 		.index = index,
61 		.start = vector,
62 		.count = 1,
63 	};
64 
65 	ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq);
66 }
67 
68 static void check_supported_irq_index(u32 index)
69 {
70 	/* VFIO selftests only supports MSI and MSI-x for now. */
71 	VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX ||
72 			 index == VFIO_PCI_MSIX_IRQ_INDEX,
73 			 "Unsupported IRQ index: %u\n", index);
74 }
75 
76 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector,
77 			 int count)
78 {
79 	int i;
80 
81 	check_supported_irq_index(index);
82 
83 	for (i = vector; i < vector + count; i++) {
84 		VFIO_ASSERT_LT(device->msi_eventfds[i], 0);
85 		device->msi_eventfds[i] = eventfd(0, 0);
86 		VFIO_ASSERT_GE(device->msi_eventfds[i], 0);
87 	}
88 
89 	vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector);
90 }
91 
92 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index)
93 {
94 	int i;
95 
96 	check_supported_irq_index(index);
97 
98 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) {
99 		if (device->msi_eventfds[i] < 0)
100 			continue;
101 
102 		VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0);
103 		device->msi_eventfds[i] = -1;
104 	}
105 
106 	vfio_pci_irq_set(device, index, 0, 0, NULL);
107 }
108 
109 /*
110  * Re-issue VFIO_DEVICE_SET_IRQS for an already-enabled vector range using
111  * the existing eventfds.  Intended for drivers that need to re-arm device
112  * interrupts after a VFIO_DEVICE_RESET, which tears down the kernel-side
113  * IRQ trigger but leaves user-side eventfds intact.  Recreating the
114  * eventfds would invalidate any test-fixture cache of the fd, so this
115  * helper deliberately preserves them.
116  */
117 void vfio_pci_irq_reenable(struct vfio_pci_device *device, u32 index,
118 			   u32 vector, int count)
119 {
120 	int i;
121 
122 	check_supported_irq_index(index);
123 
124 	for (i = vector; i < vector + count; i++)
125 		VFIO_ASSERT_GE(device->msi_eventfds[i], 0,
126 			       "vector %d eventfd not allocated\n", i);
127 
128 	vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector);
129 }
130 
131 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index,
132 			     struct vfio_irq_info *irq_info)
133 {
134 	irq_info->argsz = sizeof(*irq_info);
135 	irq_info->index = index;
136 
137 	ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info);
138 }
139 
140 static int vfio_device_feature_ioctl(int fd, u32 flags, void *data,
141 				     size_t data_size)
142 {
143 	size_t argsz = sizeof(struct vfio_device_feature) + data_size;
144 	struct vfio_device_feature *feature;
145 	int ret;
146 
147 	feature = calloc_assert(1, argsz);
148 	memcpy(feature->data, data, data_size);
149 
150 	feature->argsz = argsz;
151 	feature->flags = flags;
152 
153 	ret = ioctl(fd, VFIO_DEVICE_FEATURE, feature);
154 	free(feature);
155 
156 	return ret;
157 }
158 
159 static void vfio_device_feature_set(int fd, u16 feature, void *data, size_t data_size)
160 {
161 	u32 flags = VFIO_DEVICE_FEATURE_SET | feature;
162 	int ret;
163 
164 	ret = vfio_device_feature_ioctl(fd, flags, data, data_size);
165 	VFIO_ASSERT_EQ(ret, 0, "Failed to set feature %u\n", feature);
166 }
167 
168 void vfio_device_set_vf_token(int fd, const char *vf_token)
169 {
170 	uuid_t token_uuid = {0};
171 
172 	VFIO_ASSERT_NOT_NULL(vf_token, "vf_token is NULL");
173 	VFIO_ASSERT_EQ(uuid_parse(vf_token, token_uuid), 0);
174 
175 	vfio_device_feature_set(fd, VFIO_DEVICE_FEATURE_PCI_VF_TOKEN,
176 				token_uuid, sizeof(uuid_t));
177 }
178 
179 static void vfio_pci_region_get(struct vfio_pci_device *device, int index,
180 				struct vfio_region_info *info)
181 {
182 	memset(info, 0, sizeof(*info));
183 
184 	info->argsz = sizeof(*info);
185 	info->index = index;
186 
187 	ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info);
188 }
189 
190 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index)
191 {
192 	struct vfio_pci_bar *bar = &device->bars[index];
193 	size_t align, size;
194 	int prot = 0;
195 	void *vaddr;
196 
197 	VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS);
198 	VFIO_ASSERT_NULL(bar->vaddr);
199 	VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP);
200 	VFIO_ASSERT_TRUE(is_power_of_2(bar->info.size));
201 
202 	if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ)
203 		prot |= PROT_READ;
204 	if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE)
205 		prot |= PROT_WRITE;
206 
207 	size = bar->info.size;
208 
209 	/*
210 	 * Align BAR mmaps to improve page fault granularity during potential
211 	 * subsequent IOMMU mapping of these BAR vaddr. 1G for x86 is the
212 	 * largest hugepage size across any architecture, so no benefit from
213 	 * larger alignment. BARs smaller than 1G will be aligned by their
214 	 * power-of-two size, guaranteeing sufficient alignment for smaller
215 	 * hugepages, if present.
216 	 */
217 	align = min_t(size_t, size, SZ_1G);
218 
219 	vaddr = mmap_reserve(size, align, 0);
220 	bar->vaddr = mmap(vaddr, size, prot, MAP_SHARED | MAP_FIXED,
221 			  device->fd, bar->info.offset);
222 	VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED);
223 
224 	madvise(bar->vaddr, size, MADV_HUGEPAGE);
225 }
226 
227 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index)
228 {
229 	struct vfio_pci_bar *bar = &device->bars[index];
230 
231 	VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS);
232 	VFIO_ASSERT_NOT_NULL(bar->vaddr);
233 
234 	VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0);
235 	bar->vaddr = NULL;
236 }
237 
238 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device)
239 {
240 	int i;
241 
242 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
243 		if (device->bars[i].vaddr)
244 			vfio_pci_bar_unmap(device, i);
245 	}
246 }
247 
248 void vfio_pci_config_access(struct vfio_pci_device *device, bool write,
249 			    size_t config, size_t size, void *data)
250 {
251 	struct vfio_region_info *config_space = &device->config_space;
252 	int ret;
253 
254 	if (write)
255 		ret = pwrite(device->fd, data, size, config_space->offset + config);
256 	else
257 		ret = pread(device->fd, data, size, config_space->offset + config);
258 
259 	VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n",
260 		       write ? "write to" : "read from", config);
261 }
262 
263 int __vfio_pci_device_reset(struct vfio_pci_device *device)
264 {
265 	if (ioctl(device->fd, VFIO_DEVICE_RESET, NULL))
266 		return -errno;
267 
268 	return 0;
269 }
270 
271 void vfio_pci_device_reset(struct vfio_pci_device *device)
272 {
273 	int retries = 20;
274 	int r;
275 
276 	do {
277 		r = __vfio_pci_device_reset(device);
278 		if (r == -EAGAIN)
279 			usleep(10000);
280 	} while (r == -EAGAIN && retries-- > 0);
281 
282 	VFIO_ASSERT_EQ(r, 0, "ioctl(device->fd, VFIO_DEVICE_RESET) failed\n");
283 }
284 
285 void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf)
286 {
287 	struct vfio_group_status group_status = {
288 		.argsz = sizeof(group_status),
289 	};
290 	char group_path[32];
291 	int group;
292 
293 	group = sysfs_iommu_group_get(bdf);
294 	snprintf_assert(group_path, sizeof(group_path), "/dev/vfio/%d", group);
295 
296 	device->group_fd = open(group_path, O_RDWR);
297 	VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path);
298 
299 	ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status);
300 	VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE);
301 
302 	ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->iommu->container_fd);
303 }
304 
305 void __vfio_pci_group_get_device_fd(struct vfio_pci_device *device,
306 				    const char *bdf, const char *vf_token)
307 {
308 	char arg[64];
309 
310 	/*
311 	 * If a vf_token exists, argument to VFIO_GROUP_GET_DEVICE_FD
312 	 * will be in the form of the following example:
313 	 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
314 	 */
315 	if (vf_token)
316 		snprintf_assert(arg, ARRAY_SIZE(arg), "%s vf_token=%s", bdf, vf_token);
317 	else
318 		snprintf_assert(arg, ARRAY_SIZE(arg), "%s", bdf);
319 
320 	device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, arg);
321 }
322 
323 static void vfio_pci_group_get_device_fd(struct vfio_pci_device *device,
324 					 const char *bdf, const char *vf_token)
325 {
326 	__vfio_pci_group_get_device_fd(device, bdf, vf_token);
327 	VFIO_ASSERT_GE(device->fd, 0);
328 }
329 
330 void vfio_container_set_iommu(struct vfio_pci_device *device)
331 {
332 	struct iommu *iommu = device->iommu;
333 	unsigned long iommu_type = iommu->mode->iommu_type;
334 	int ret;
335 
336 	ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type);
337 	VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type);
338 
339 	/*
340 	 * Allow multiple threads to race to set the IOMMU type on the
341 	 * container. The first will succeed and the rest should fail
342 	 * because the IOMMU type is already set.
343 	 */
344 	(void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type);
345 }
346 
347 static void vfio_pci_container_setup(struct vfio_pci_device *device,
348 				     const char *bdf, const char *vf_token)
349 {
350 	vfio_pci_group_setup(device, bdf);
351 	vfio_container_set_iommu(device);
352 	vfio_pci_group_get_device_fd(device, bdf, vf_token);
353 }
354 
355 static void vfio_pci_device_setup(struct vfio_pci_device *device)
356 {
357 	int i;
358 
359 	device->info.argsz = sizeof(device->info);
360 	ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info);
361 
362 	vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space);
363 
364 	/* Sanity check VFIO does not advertise mmap for config space */
365 	VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP),
366 			 "PCI config space should not support mmap()\n");
367 
368 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
369 		struct vfio_pci_bar *bar = device->bars + i;
370 
371 		vfio_pci_region_get(device, i, &bar->info);
372 		if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP)
373 			vfio_pci_bar_map(device, i);
374 	}
375 
376 	vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info);
377 	vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info);
378 
379 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++)
380 		device->msi_eventfds[i] = -1;
381 }
382 
383 const char *vfio_pci_get_cdev_path(const char *bdf)
384 {
385 	char dir_path[PATH_MAX];
386 	struct dirent *entry;
387 	char *cdev_path;
388 	DIR *dir;
389 
390 	cdev_path = calloc_assert(PATH_MAX, 1);
391 
392 	snprintf_assert(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf);
393 
394 	dir = opendir(dir_path);
395 	VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path);
396 
397 	while ((entry = readdir(dir)) != NULL) {
398 		/* Find the file that starts with "vfio" */
399 		if (strncmp("vfio", entry->d_name, 4))
400 			continue;
401 
402 		snprintf_assert(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name);
403 		break;
404 	}
405 
406 	VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n");
407 	VFIO_ASSERT_EQ(closedir(dir), 0);
408 
409 	return cdev_path;
410 }
411 
412 int __vfio_device_bind_iommufd(int device_fd, int iommufd, const char *vf_token)
413 {
414 	struct vfio_device_bind_iommufd args = {
415 		.argsz = sizeof(args),
416 		.iommufd = iommufd,
417 	};
418 	uuid_t token_uuid;
419 
420 	if (vf_token) {
421 		VFIO_ASSERT_EQ(uuid_parse(vf_token, token_uuid), 0);
422 		args.flags |= VFIO_DEVICE_BIND_FLAG_TOKEN;
423 		args.token_uuid_ptr = (u64)token_uuid;
424 	}
425 
426 	if (ioctl(device_fd, VFIO_DEVICE_BIND_IOMMUFD, &args))
427 		return -errno;
428 
429 	return 0;
430 }
431 
432 static void vfio_device_bind_iommufd(int device_fd, int iommufd,
433 				     const char *vf_token)
434 {
435 	int ret = __vfio_device_bind_iommufd(device_fd, iommufd, vf_token);
436 
437 	VFIO_ASSERT_EQ(ret, 0, "Failed VFIO_DEVICE_BIND_IOMMUFD ioctl\n");
438 }
439 
440 static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id)
441 {
442 	struct vfio_device_attach_iommufd_pt args = {
443 		.argsz = sizeof(args),
444 		.pt_id = pt_id,
445 	};
446 
447 	ioctl_assert(device_fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &args);
448 }
449 
450 void vfio_pci_cdev_open(struct vfio_pci_device *device, const char *bdf)
451 {
452 	const char *cdev_path = vfio_pci_get_cdev_path(bdf);
453 
454 	device->fd = open(cdev_path, O_RDWR);
455 	VFIO_ASSERT_GE(device->fd, 0);
456 	free((void *)cdev_path);
457 }
458 
459 static void vfio_pci_iommufd_setup(struct vfio_pci_device *device,
460 				   const char *bdf, const char *vf_token)
461 {
462 	vfio_pci_cdev_open(device, bdf);
463 	vfio_device_bind_iommufd(device->fd, device->iommu->iommufd, vf_token);
464 	vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id);
465 }
466 
467 struct vfio_pci_device *vfio_pci_device_alloc(const char *bdf, struct iommu *iommu)
468 {
469 	struct vfio_pci_device *device;
470 
471 	device = calloc_assert(1, sizeof(*device));
472 
473 	VFIO_ASSERT_NOT_NULL(iommu);
474 	device->iommu = iommu;
475 	device->bdf = bdf;
476 
477 	return device;
478 }
479 
480 void vfio_pci_device_free(struct vfio_pci_device *device)
481 {
482 	free(device);
483 }
484 
485 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu)
486 {
487 	struct vfio_pci_device *device;
488 
489 	device = vfio_pci_device_alloc(bdf, iommu);
490 
491 	if (iommu->mode->container_path)
492 		vfio_pci_container_setup(device, bdf, NULL);
493 	else
494 		vfio_pci_iommufd_setup(device, bdf, NULL);
495 
496 	vfio_pci_device_setup(device);
497 	vfio_pci_driver_probe(device);
498 
499 	return device;
500 }
501 
502 void vfio_pci_device_cleanup(struct vfio_pci_device *device)
503 {
504 	int i;
505 
506 	if (device->driver.initialized)
507 		vfio_pci_driver_remove(device);
508 
509 	vfio_pci_bar_unmap_all(device);
510 
511 	VFIO_ASSERT_EQ(close(device->fd), 0);
512 
513 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) {
514 		if (device->msi_eventfds[i] < 0)
515 			continue;
516 
517 		VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0);
518 	}
519 
520 	if (device->group_fd)
521 		VFIO_ASSERT_EQ(close(device->group_fd), 0);
522 
523 	vfio_pci_device_free(device);
524 }
525