1 // SPDX-License-Identifier: GPL-2.0-only
2 #include "kselftest.h"
3 #include <libvfio.h>
4
5 #ifdef __x86_64__
6 extern struct vfio_pci_driver_ops dsa_ops;
7 extern struct vfio_pci_driver_ops ioat_ops;
8 #endif
9 extern struct vfio_pci_driver_ops nv_falcon_ops;
10 extern struct vfio_pci_driver_ops igb_ops;
11
12 static struct vfio_pci_driver_ops *driver_ops[] = {
13 #ifdef __x86_64__
14 &dsa_ops,
15 &ioat_ops,
16 #endif
17 &nv_falcon_ops,
18 &igb_ops,
19 };
20
vfio_pci_driver_probe(struct vfio_pci_device * device)21 void vfio_pci_driver_probe(struct vfio_pci_device *device)
22 {
23 struct vfio_pci_driver_ops *ops;
24 int i;
25
26 VFIO_ASSERT_NULL(device->driver.ops);
27
28 for (i = 0; i < ARRAY_SIZE(driver_ops); i++) {
29 ops = driver_ops[i];
30
31 if (ops->probe(device))
32 continue;
33
34 device->driver.ops = ops;
35 }
36 }
37
vfio_check_driver_op(struct vfio_pci_driver * driver,void * op,const char * op_name)38 static void vfio_check_driver_op(struct vfio_pci_driver *driver, void *op,
39 const char *op_name)
40 {
41 VFIO_ASSERT_NOT_NULL(driver->ops);
42 VFIO_ASSERT_NOT_NULL(op, "Driver has no %s()\n", op_name);
43 VFIO_ASSERT_EQ(driver->initialized, op != driver->ops->init);
44 VFIO_ASSERT_EQ(driver->memcpy_in_progress, op == driver->ops->memcpy_wait);
45 }
46
47 #define VFIO_CHECK_DRIVER_OP(_driver, _op) do { \
48 struct vfio_pci_driver *__driver = (_driver); \
49 vfio_check_driver_op(__driver, __driver->ops->_op, #_op); \
50 } while (0)
51
vfio_pci_driver_init(struct vfio_pci_device * device)52 void vfio_pci_driver_init(struct vfio_pci_device *device)
53 {
54 struct vfio_pci_driver *driver = &device->driver;
55
56 VFIO_ASSERT_NOT_NULL(driver->region.vaddr);
57 VFIO_CHECK_DRIVER_OP(driver, init);
58
59 driver->ops->init(device);
60
61 driver->initialized = true;
62 }
63
vfio_pci_driver_remove(struct vfio_pci_device * device)64 void vfio_pci_driver_remove(struct vfio_pci_device *device)
65 {
66 struct vfio_pci_driver *driver = &device->driver;
67
68 VFIO_CHECK_DRIVER_OP(driver, remove);
69
70 driver->ops->remove(device);
71 driver->initialized = false;
72 }
73
vfio_pci_driver_send_msi(struct vfio_pci_device * device)74 void vfio_pci_driver_send_msi(struct vfio_pci_device *device)
75 {
76 struct vfio_pci_driver *driver = &device->driver;
77
78 VFIO_CHECK_DRIVER_OP(driver, send_msi);
79
80 driver->ops->send_msi(device);
81 }
82
vfio_pci_driver_memcpy_start(struct vfio_pci_device * device,iova_t src,iova_t dst,u64 size,u64 count)83 void vfio_pci_driver_memcpy_start(struct vfio_pci_device *device,
84 iova_t src, iova_t dst, u64 size,
85 u64 count)
86 {
87 struct vfio_pci_driver *driver = &device->driver;
88
89 VFIO_ASSERT_LE(size, driver->max_memcpy_size);
90 VFIO_ASSERT_LE(count, driver->max_memcpy_count);
91 VFIO_CHECK_DRIVER_OP(driver, memcpy_start);
92
93 driver->ops->memcpy_start(device, src, dst, size, count);
94 driver->memcpy_in_progress = true;
95 }
96
vfio_pci_driver_memcpy_wait(struct vfio_pci_device * device)97 int vfio_pci_driver_memcpy_wait(struct vfio_pci_device *device)
98 {
99 struct vfio_pci_driver *driver = &device->driver;
100 int r;
101
102 VFIO_CHECK_DRIVER_OP(driver, memcpy_wait);
103
104 r = driver->ops->memcpy_wait(device);
105 driver->memcpy_in_progress = false;
106
107 return r;
108 }
109
vfio_pci_driver_memcpy(struct vfio_pci_device * device,iova_t src,iova_t dst,u64 size)110 int vfio_pci_driver_memcpy(struct vfio_pci_device *device,
111 iova_t src, iova_t dst, u64 size)
112 {
113 struct vfio_pci_driver *driver = &device->driver;
114 u64 offset = 0;
115
116 while (offset < size) {
117 u64 chunk = min(size - offset, driver->max_memcpy_size);
118 int ret;
119
120 vfio_pci_driver_memcpy_start(device, src + offset,
121 dst + offset, chunk, 1);
122 ret = vfio_pci_driver_memcpy_wait(device);
123 if (ret)
124 return ret;
125
126 offset += chunk;
127 }
128
129 return 0;
130 }
131