1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <sys/ioctl.h>
3 #include <sys/mman.h>
4
5 #include <linux/sizes.h>
6 #include <linux/vfio.h>
7
8 #include <libvfio.h>
9
10 #include "kselftest_harness.h"
11
12 static const char *device_bdf;
13
14 #define fcntl_set_msi_nonblock(_self) do { \
15 if (_self->device->driver.ops->send_msi) \
16 fcntl_set_nonblock(_self->msi_fd); \
17 } while (0)
18
19 #define ASSERT_NO_MSI(_self) do { \
20 u64 __value; \
21 \
22 if (!_self->device->driver.ops->send_msi) \
23 break; \
24 ASSERT_EQ(-1, read(_self->msi_fd, &__value, 8)); \
25 ASSERT_EQ(EAGAIN, errno); \
26 } while (0)
27
region_setup(struct iommu * iommu,struct iova_allocator * iova_allocator,struct dma_region * region,u64 size)28 static void region_setup(struct iommu *iommu,
29 struct iova_allocator *iova_allocator,
30 struct dma_region *region, u64 size)
31 {
32 const int flags = MAP_SHARED | MAP_ANONYMOUS;
33 const int prot = PROT_READ | PROT_WRITE;
34 void *vaddr;
35
36 vaddr = mmap(NULL, size, prot, flags, -1, 0);
37 VFIO_ASSERT_NE(vaddr, MAP_FAILED);
38
39 region->vaddr = vaddr;
40 region->iova = iova_allocator_alloc(iova_allocator, size);
41 region->size = size;
42
43 iommu_map(iommu, region);
44 }
45
region_teardown(struct iommu * iommu,struct dma_region * region)46 static void region_teardown(struct iommu *iommu, struct dma_region *region)
47 {
48 iommu_unmap(iommu, region);
49 VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0);
50 }
51
FIXTURE(vfio_pci_driver_test)52 FIXTURE(vfio_pci_driver_test) {
53 struct iommu *iommu;
54 struct vfio_pci_device *device;
55 struct iova_allocator *iova_allocator;
56 struct dma_region memcpy_region;
57 void *vaddr;
58 int msi_fd;
59
60 u64 size;
61 void *src;
62 void *dst;
63 iova_t src_iova;
64 iova_t dst_iova;
65 iova_t unmapped_iova;
66 };
67
FIXTURE_VARIANT(vfio_pci_driver_test)68 FIXTURE_VARIANT(vfio_pci_driver_test) {
69 const char *iommu_mode;
70 };
71
72 #define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \
73 FIXTURE_VARIANT_ADD(vfio_pci_driver_test, _iommu_mode) { \
74 .iommu_mode = #_iommu_mode, \
75 }
76
77 FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES();
78
FIXTURE_SETUP(vfio_pci_driver_test)79 FIXTURE_SETUP(vfio_pci_driver_test)
80 {
81 struct vfio_pci_driver *driver;
82
83 self->iommu = iommu_init(variant->iommu_mode);
84 self->device = vfio_pci_device_init(device_bdf, self->iommu);
85 self->iova_allocator = iova_allocator_init(self->iommu);
86
87 driver = &self->device->driver;
88
89 region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G);
90 region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M);
91
92 /* Any IOVA that doesn't overlap memcpy_region and driver->region. */
93 self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G);
94
95 vfio_pci_driver_init(self->device);
96 self->msi_fd = self->device->msi_eventfds[driver->msi];
97
98 /*
99 * Use 4x the driver's max_memcpy_size to exercise the chunking
100 * logic in vfio_pci_driver_memcpy(). Cap to half the memcpy
101 * region so src and dst do not overlap.
102 */
103 self->size = min_t(u64, driver->max_memcpy_size * 4,
104 self->memcpy_region.size / 2);
105
106 self->src = self->memcpy_region.vaddr;
107 self->dst = self->src + self->size;
108
109 self->src_iova = to_iova(self->device, self->src);
110 self->dst_iova = to_iova(self->device, self->dst);
111 }
112
FIXTURE_TEARDOWN(vfio_pci_driver_test)113 FIXTURE_TEARDOWN(vfio_pci_driver_test)
114 {
115 struct vfio_pci_driver *driver = &self->device->driver;
116
117 vfio_pci_driver_remove(self->device);
118
119 region_teardown(self->iommu, &self->memcpy_region);
120 region_teardown(self->iommu, &driver->region);
121
122 iova_allocator_cleanup(self->iova_allocator);
123 vfio_pci_device_cleanup(self->device);
124 iommu_cleanup(self->iommu);
125 }
126
TEST_F(vfio_pci_driver_test,init_remove)127 TEST_F(vfio_pci_driver_test, init_remove)
128 {
129 int i;
130
131 for (i = 0; i < 10; i++) {
132 vfio_pci_driver_remove(self->device);
133 vfio_pci_driver_init(self->device);
134 }
135 }
136
TEST_F(vfio_pci_driver_test,memcpy_success)137 TEST_F(vfio_pci_driver_test, memcpy_success)
138 {
139 fcntl_set_msi_nonblock(self);
140
141 memset(self->src, 'x', self->size);
142 memset(self->dst, 'y', self->size);
143
144 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device,
145 self->src_iova,
146 self->dst_iova,
147 self->size));
148
149 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size));
150 ASSERT_NO_MSI(self);
151 }
152
TEST_F(vfio_pci_driver_test,memcpy_from_unmapped_iova)153 TEST_F(vfio_pci_driver_test, memcpy_from_unmapped_iova)
154 {
155 fcntl_set_msi_nonblock(self);
156
157 /*
158 * Ignore the return value since not all devices will detect and report
159 * accesses to unmapped IOVAs as errors.
160 */
161 vfio_pci_driver_memcpy(self->device, self->unmapped_iova,
162 self->dst_iova, self->size);
163
164 ASSERT_NO_MSI(self);
165 }
166
TEST_F(vfio_pci_driver_test,memcpy_to_unmapped_iova)167 TEST_F(vfio_pci_driver_test, memcpy_to_unmapped_iova)
168 {
169 fcntl_set_msi_nonblock(self);
170
171 /*
172 * Ignore the return value since not all devices will detect and report
173 * accesses to unmapped IOVAs as errors.
174 */
175 vfio_pci_driver_memcpy(self->device, self->src_iova,
176 self->unmapped_iova, self->size);
177
178 ASSERT_NO_MSI(self);
179 }
180
TEST_F(vfio_pci_driver_test,send_msi)181 TEST_F(vfio_pci_driver_test, send_msi)
182 {
183 u64 value;
184
185 if (!self->device->driver.ops->send_msi)
186 SKIP(return, "Driver does not support send_msi()\n");
187
188 vfio_pci_driver_send_msi(self->device);
189 ASSERT_EQ(8, read(self->msi_fd, &value, 8));
190 ASSERT_EQ(1, value);
191 }
192
TEST_F(vfio_pci_driver_test,mix_and_match)193 TEST_F(vfio_pci_driver_test, mix_and_match)
194 {
195 u64 value;
196 int i;
197
198 for (i = 0; i < 10; i++) {
199 memset(self->src, 'x', self->size);
200 memset(self->dst, 'y', self->size);
201
202 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device,
203 self->src_iova,
204 self->dst_iova,
205 self->size));
206
207 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size));
208
209 vfio_pci_driver_memcpy(self->device,
210 self->unmapped_iova,
211 self->dst_iova,
212 self->size);
213
214 if (!self->device->driver.ops->send_msi)
215 continue;
216
217 vfio_pci_driver_send_msi(self->device);
218 ASSERT_EQ(8, read(self->msi_fd, &value, 8));
219 ASSERT_EQ(1, value);
220 }
221 }
222
223 TEST_F_TIMEOUT(vfio_pci_driver_test, memcpy_storm, 60)
224 {
225 struct vfio_pci_driver *driver = &self->device->driver;
226 u64 total_size;
227 u64 size;
228 u64 count;
229
230 fcntl_set_msi_nonblock(self);
231
232 /*
233 * Perform up to 250GiB worth of DMA reads and writes across several
234 * memcpy operations. Some devices can support even more but the test
235 * will take too long.
236 */
237 total_size = 250UL * SZ_1G;
238 size = min(driver->max_memcpy_size, self->memcpy_region.size / 2);
239 count = min(total_size / size, driver->max_memcpy_count);
240
241 printf("Kicking off %lu memcpys of size 0x%lx\n", count, size);
242 vfio_pci_driver_memcpy_start(self->device,
243 self->src_iova,
244 self->dst_iova,
245 size, count);
246
247 ASSERT_EQ(0, vfio_pci_driver_memcpy_wait(self->device));
248 ASSERT_NO_MSI(self);
249 }
250
device_has_selftests_driver(const char * bdf)251 static bool device_has_selftests_driver(const char *bdf)
252 {
253 struct vfio_pci_device *device;
254 struct iommu *iommu;
255 bool has_driver;
256
257 iommu = iommu_init(default_iommu_mode);
258 device = vfio_pci_device_init(device_bdf, iommu);
259
260 has_driver = !!device->driver.ops;
261
262 vfio_pci_device_cleanup(device);
263 iommu_cleanup(iommu);
264
265 return has_driver;
266 }
267
main(int argc,char * argv[])268 int main(int argc, char *argv[])
269 {
270 device_bdf = vfio_selftests_get_bdf(&argc, argv);
271
272 if (!device_has_selftests_driver(device_bdf)) {
273 fprintf(stderr, "No driver found for device %s\n", device_bdf);
274 return KSFT_SKIP;
275 }
276
277 return test_harness_run(argc, argv);
278 }
279