xref: /linux/tools/testing/selftests/vfio/lib/vfio_pci_device.c (revision 60ab36e5d02a8e50e1edce24bff58507b514b94e) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <dirent.h>
3 #include <fcntl.h>
4 #include <libgen.h>
5 #include <stdint.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 
10 #include <sys/eventfd.h>
11 #include <sys/ioctl.h>
12 #include <sys/mman.h>
13 
14 #include <linux/align.h>
15 #include <linux/iommufd.h>
16 #include <linux/kernel.h>
17 #include <linux/limits.h>
18 #include <linux/log2.h>
19 #include <linux/mman.h>
20 #include <linux/overflow.h>
21 #include <linux/sizes.h>
22 #include <linux/types.h>
23 #include <linux/vfio.h>
24 
25 #include <uuid/uuid.h>
26 
27 #include "kselftest.h"
28 #include <libvfio.h>
29 
30 static void vfio_pci_irq_set(struct vfio_pci_device *device,
31 			     u32 index, u32 vector, u32 count, int *fds)
32 {
33 	u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count];
34 	struct vfio_irq_set *irq = (void *)&buf;
35 	int *irq_fds = (void *)&irq->data;
36 
37 	memset(buf, 0, sizeof(buf));
38 
39 	irq->argsz = sizeof(buf);
40 	irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER;
41 	irq->index = index;
42 	irq->start = vector;
43 	irq->count = count;
44 
45 	if (count) {
46 		irq->flags |= VFIO_IRQ_SET_DATA_EVENTFD;
47 		memcpy(irq_fds, fds, sizeof(int) * count);
48 	} else {
49 		irq->flags |= VFIO_IRQ_SET_DATA_NONE;
50 	}
51 
52 	ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, irq);
53 }
54 
55 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector)
56 {
57 	struct vfio_irq_set irq = {
58 		.argsz = sizeof(irq),
59 		.flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE,
60 		.index = index,
61 		.start = vector,
62 		.count = 1,
63 	};
64 
65 	ioctl_assert(device->fd, VFIO_DEVICE_SET_IRQS, &irq);
66 }
67 
68 static void check_supported_irq_index(u32 index)
69 {
70 	/* VFIO selftests only supports MSI and MSI-x for now. */
71 	VFIO_ASSERT_TRUE(index == VFIO_PCI_MSI_IRQ_INDEX ||
72 			 index == VFIO_PCI_MSIX_IRQ_INDEX,
73 			 "Unsupported IRQ index: %u\n", index);
74 }
75 
76 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, u32 vector,
77 			 int count)
78 {
79 	int i;
80 
81 	check_supported_irq_index(index);
82 
83 	for (i = vector; i < vector + count; i++) {
84 		VFIO_ASSERT_LT(device->msi_eventfds[i], 0);
85 		device->msi_eventfds[i] = eventfd(0, 0);
86 		VFIO_ASSERT_GE(device->msi_eventfds[i], 0);
87 	}
88 
89 	vfio_pci_irq_set(device, index, vector, count, device->msi_eventfds + vector);
90 }
91 
92 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index)
93 {
94 	int i;
95 
96 	check_supported_irq_index(index);
97 
98 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) {
99 		if (device->msi_eventfds[i] < 0)
100 			continue;
101 
102 		VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0);
103 		device->msi_eventfds[i] = -1;
104 	}
105 
106 	vfio_pci_irq_set(device, index, 0, 0, NULL);
107 }
108 
109 static void vfio_pci_irq_get(struct vfio_pci_device *device, u32 index,
110 			     struct vfio_irq_info *irq_info)
111 {
112 	irq_info->argsz = sizeof(*irq_info);
113 	irq_info->index = index;
114 
115 	ioctl_assert(device->fd, VFIO_DEVICE_GET_IRQ_INFO, irq_info);
116 }
117 
118 static void vfio_pci_region_get(struct vfio_pci_device *device, int index,
119 				struct vfio_region_info *info)
120 {
121 	memset(info, 0, sizeof(*info));
122 
123 	info->argsz = sizeof(*info);
124 	info->index = index;
125 
126 	ioctl_assert(device->fd, VFIO_DEVICE_GET_REGION_INFO, info);
127 }
128 
129 static void vfio_pci_bar_map(struct vfio_pci_device *device, int index)
130 {
131 	struct vfio_pci_bar *bar = &device->bars[index];
132 	size_t align, size;
133 	int prot = 0;
134 	void *vaddr;
135 
136 	VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS);
137 	VFIO_ASSERT_NULL(bar->vaddr);
138 	VFIO_ASSERT_TRUE(bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP);
139 	VFIO_ASSERT_TRUE(is_power_of_2(bar->info.size));
140 
141 	if (bar->info.flags & VFIO_REGION_INFO_FLAG_READ)
142 		prot |= PROT_READ;
143 	if (bar->info.flags & VFIO_REGION_INFO_FLAG_WRITE)
144 		prot |= PROT_WRITE;
145 
146 	size = bar->info.size;
147 
148 	/*
149 	 * Align BAR mmaps to improve page fault granularity during potential
150 	 * subsequent IOMMU mapping of these BAR vaddr. 1G for x86 is the
151 	 * largest hugepage size across any architecture, so no benefit from
152 	 * larger alignment. BARs smaller than 1G will be aligned by their
153 	 * power-of-two size, guaranteeing sufficient alignment for smaller
154 	 * hugepages, if present.
155 	 */
156 	align = min_t(size_t, size, SZ_1G);
157 
158 	vaddr = mmap_reserve(size, align, 0);
159 	bar->vaddr = mmap(vaddr, size, prot, MAP_SHARED | MAP_FIXED,
160 			  device->fd, bar->info.offset);
161 	VFIO_ASSERT_NE(bar->vaddr, MAP_FAILED);
162 
163 	madvise(bar->vaddr, size, MADV_HUGEPAGE);
164 }
165 
166 static void vfio_pci_bar_unmap(struct vfio_pci_device *device, int index)
167 {
168 	struct vfio_pci_bar *bar = &device->bars[index];
169 
170 	VFIO_ASSERT_LT(index, PCI_STD_NUM_BARS);
171 	VFIO_ASSERT_NOT_NULL(bar->vaddr);
172 
173 	VFIO_ASSERT_EQ(munmap(bar->vaddr, bar->info.size), 0);
174 	bar->vaddr = NULL;
175 }
176 
177 static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device)
178 {
179 	int i;
180 
181 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
182 		if (device->bars[i].vaddr)
183 			vfio_pci_bar_unmap(device, i);
184 	}
185 }
186 
187 void vfio_pci_config_access(struct vfio_pci_device *device, bool write,
188 			    size_t config, size_t size, void *data)
189 {
190 	struct vfio_region_info *config_space = &device->config_space;
191 	int ret;
192 
193 	if (write)
194 		ret = pwrite(device->fd, data, size, config_space->offset + config);
195 	else
196 		ret = pread(device->fd, data, size, config_space->offset + config);
197 
198 	VFIO_ASSERT_EQ(ret, size, "Failed to %s PCI config space: 0x%lx\n",
199 		       write ? "write to" : "read from", config);
200 }
201 
202 void vfio_pci_device_reset(struct vfio_pci_device *device)
203 {
204 	ioctl_assert(device->fd, VFIO_DEVICE_RESET, NULL);
205 }
206 
207 void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf)
208 {
209 	struct vfio_group_status group_status = {
210 		.argsz = sizeof(group_status),
211 	};
212 	char group_path[32];
213 	int group;
214 
215 	group = sysfs_iommu_group_get(bdf);
216 	snprintf_assert(group_path, sizeof(group_path), "/dev/vfio/%d", group);
217 
218 	device->group_fd = open(group_path, O_RDWR);
219 	VFIO_ASSERT_GE(device->group_fd, 0, "open(%s) failed\n", group_path);
220 
221 	ioctl_assert(device->group_fd, VFIO_GROUP_GET_STATUS, &group_status);
222 	VFIO_ASSERT_TRUE(group_status.flags & VFIO_GROUP_FLAGS_VIABLE);
223 
224 	ioctl_assert(device->group_fd, VFIO_GROUP_SET_CONTAINER, &device->iommu->container_fd);
225 }
226 
227 void __vfio_pci_group_get_device_fd(struct vfio_pci_device *device,
228 				    const char *bdf, const char *vf_token)
229 {
230 	char arg[64];
231 
232 	/*
233 	 * If a vf_token exists, argument to VFIO_GROUP_GET_DEVICE_FD
234 	 * will be in the form of the following example:
235 	 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
236 	 */
237 	if (vf_token)
238 		snprintf_assert(arg, ARRAY_SIZE(arg), "%s vf_token=%s", bdf, vf_token);
239 	else
240 		snprintf_assert(arg, ARRAY_SIZE(arg), "%s", bdf);
241 
242 	device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, arg);
243 }
244 
245 static void vfio_pci_group_get_device_fd(struct vfio_pci_device *device,
246 					 const char *bdf, const char *vf_token)
247 {
248 	__vfio_pci_group_get_device_fd(device, bdf, vf_token);
249 	VFIO_ASSERT_GE(device->fd, 0);
250 }
251 
252 void vfio_container_set_iommu(struct vfio_pci_device *device)
253 {
254 	struct iommu *iommu = device->iommu;
255 	unsigned long iommu_type = iommu->mode->iommu_type;
256 	int ret;
257 
258 	ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type);
259 	VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type);
260 
261 	/*
262 	 * Allow multiple threads to race to set the IOMMU type on the
263 	 * container. The first will succeed and the rest should fail
264 	 * because the IOMMU type is already set.
265 	 */
266 	(void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type);
267 }
268 
269 static void vfio_pci_container_setup(struct vfio_pci_device *device,
270 				     const char *bdf, const char *vf_token)
271 {
272 	vfio_pci_group_setup(device, bdf);
273 	vfio_container_set_iommu(device);
274 	vfio_pci_group_get_device_fd(device, bdf, vf_token);
275 }
276 
277 static void vfio_pci_device_setup(struct vfio_pci_device *device)
278 {
279 	int i;
280 
281 	device->info.argsz = sizeof(device->info);
282 	ioctl_assert(device->fd, VFIO_DEVICE_GET_INFO, &device->info);
283 
284 	vfio_pci_region_get(device, VFIO_PCI_CONFIG_REGION_INDEX, &device->config_space);
285 
286 	/* Sanity check VFIO does not advertise mmap for config space */
287 	VFIO_ASSERT_TRUE(!(device->config_space.flags & VFIO_REGION_INFO_FLAG_MMAP),
288 			 "PCI config space should not support mmap()\n");
289 
290 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
291 		struct vfio_pci_bar *bar = device->bars + i;
292 
293 		vfio_pci_region_get(device, i, &bar->info);
294 		if (bar->info.flags & VFIO_REGION_INFO_FLAG_MMAP)
295 			vfio_pci_bar_map(device, i);
296 	}
297 
298 	vfio_pci_irq_get(device, VFIO_PCI_MSI_IRQ_INDEX, &device->msi_info);
299 	vfio_pci_irq_get(device, VFIO_PCI_MSIX_IRQ_INDEX, &device->msix_info);
300 
301 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++)
302 		device->msi_eventfds[i] = -1;
303 }
304 
305 const char *vfio_pci_get_cdev_path(const char *bdf)
306 {
307 	char dir_path[PATH_MAX];
308 	struct dirent *entry;
309 	char *cdev_path;
310 	DIR *dir;
311 
312 	cdev_path = calloc(PATH_MAX, 1);
313 	VFIO_ASSERT_NOT_NULL(cdev_path);
314 
315 	snprintf_assert(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf);
316 
317 	dir = opendir(dir_path);
318 	VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path);
319 
320 	while ((entry = readdir(dir)) != NULL) {
321 		/* Find the file that starts with "vfio" */
322 		if (strncmp("vfio", entry->d_name, 4))
323 			continue;
324 
325 		snprintf_assert(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name);
326 		break;
327 	}
328 
329 	VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n");
330 	VFIO_ASSERT_EQ(closedir(dir), 0);
331 
332 	return cdev_path;
333 }
334 
335 int __vfio_device_bind_iommufd(int device_fd, int iommufd, const char *vf_token)
336 {
337 	struct vfio_device_bind_iommufd args = {
338 		.argsz = sizeof(args),
339 		.iommufd = iommufd,
340 	};
341 	uuid_t token_uuid;
342 
343 	if (vf_token) {
344 		VFIO_ASSERT_EQ(uuid_parse(vf_token, token_uuid), 0);
345 		args.flags |= VFIO_DEVICE_BIND_FLAG_TOKEN;
346 		args.token_uuid_ptr = (u64)token_uuid;
347 	}
348 
349 	if (ioctl(device_fd, VFIO_DEVICE_BIND_IOMMUFD, &args))
350 		return -errno;
351 
352 	return 0;
353 }
354 
355 static void vfio_device_bind_iommufd(int device_fd, int iommufd,
356 				     const char *vf_token)
357 {
358 	int ret = __vfio_device_bind_iommufd(device_fd, iommufd, vf_token);
359 
360 	VFIO_ASSERT_EQ(ret, 0, "Failed VFIO_DEVICE_BIND_IOMMUFD ioctl\n");
361 }
362 
363 static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id)
364 {
365 	struct vfio_device_attach_iommufd_pt args = {
366 		.argsz = sizeof(args),
367 		.pt_id = pt_id,
368 	};
369 
370 	ioctl_assert(device_fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &args);
371 }
372 
373 void vfio_pci_cdev_open(struct vfio_pci_device *device, const char *bdf)
374 {
375 	const char *cdev_path = vfio_pci_get_cdev_path(bdf);
376 
377 	device->fd = open(cdev_path, O_RDWR);
378 	VFIO_ASSERT_GE(device->fd, 0);
379 	free((void *)cdev_path);
380 }
381 
382 static void vfio_pci_iommufd_setup(struct vfio_pci_device *device,
383 				   const char *bdf, const char *vf_token)
384 {
385 	vfio_pci_cdev_open(device, bdf);
386 	vfio_device_bind_iommufd(device->fd, device->iommu->iommufd, vf_token);
387 	vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id);
388 }
389 
390 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, struct iommu *iommu)
391 {
392 	struct vfio_pci_device *device;
393 
394 	device = calloc(1, sizeof(*device));
395 	VFIO_ASSERT_NOT_NULL(device);
396 
397 	VFIO_ASSERT_NOT_NULL(iommu);
398 	device->iommu = iommu;
399 	device->bdf = bdf;
400 
401 	if (iommu->mode->container_path)
402 		vfio_pci_container_setup(device, bdf, NULL);
403 	else
404 		vfio_pci_iommufd_setup(device, bdf, NULL);
405 
406 	vfio_pci_device_setup(device);
407 	vfio_pci_driver_probe(device);
408 
409 	return device;
410 }
411 
412 void vfio_pci_device_cleanup(struct vfio_pci_device *device)
413 {
414 	int i;
415 
416 	if (device->driver.initialized)
417 		vfio_pci_driver_remove(device);
418 
419 	vfio_pci_bar_unmap_all(device);
420 
421 	VFIO_ASSERT_EQ(close(device->fd), 0);
422 
423 	for (i = 0; i < ARRAY_SIZE(device->msi_eventfds); i++) {
424 		if (device->msi_eventfds[i] < 0)
425 			continue;
426 
427 		VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0);
428 	}
429 
430 	if (device->group_fd)
431 		VFIO_ASSERT_EQ(close(device->group_fd), 0);
432 
433 	free(device);
434 }
435