1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <sys/ioctl.h>
3 #include <sys/mman.h>
4
5 #include <linux/sizes.h>
6 #include <linux/vfio.h>
7
8 #include <libvfio.h>
9
10 #include "kselftest_harness.h"
11
12 static const char *device_bdf;
13
14 #define ASSERT_NO_MSI(_eventfd) do { \
15 u64 __value; \
16 \
17 ASSERT_EQ(-1, read(_eventfd, &__value, 8)); \
18 ASSERT_EQ(EAGAIN, errno); \
19 } while (0)
20
region_setup(struct iommu * iommu,struct iova_allocator * iova_allocator,struct dma_region * region,u64 size)21 static void region_setup(struct iommu *iommu,
22 struct iova_allocator *iova_allocator,
23 struct dma_region *region, u64 size)
24 {
25 const int flags = MAP_SHARED | MAP_ANONYMOUS;
26 const int prot = PROT_READ | PROT_WRITE;
27 void *vaddr;
28
29 vaddr = mmap(NULL, size, prot, flags, -1, 0);
30 VFIO_ASSERT_NE(vaddr, MAP_FAILED);
31
32 region->vaddr = vaddr;
33 region->iova = iova_allocator_alloc(iova_allocator, size);
34 region->size = size;
35
36 iommu_map(iommu, region);
37 }
38
region_teardown(struct iommu * iommu,struct dma_region * region)39 static void region_teardown(struct iommu *iommu, struct dma_region *region)
40 {
41 iommu_unmap(iommu, region);
42 VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0);
43 }
44
FIXTURE(vfio_pci_driver_test)45 FIXTURE(vfio_pci_driver_test) {
46 struct iommu *iommu;
47 struct vfio_pci_device *device;
48 struct iova_allocator *iova_allocator;
49 struct dma_region memcpy_region;
50 void *vaddr;
51 int msi_fd;
52
53 u64 size;
54 void *src;
55 void *dst;
56 iova_t src_iova;
57 iova_t dst_iova;
58 iova_t unmapped_iova;
59 };
60
FIXTURE_VARIANT(vfio_pci_driver_test)61 FIXTURE_VARIANT(vfio_pci_driver_test) {
62 const char *iommu_mode;
63 };
64
65 #define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \
66 FIXTURE_VARIANT_ADD(vfio_pci_driver_test, _iommu_mode) { \
67 .iommu_mode = #_iommu_mode, \
68 }
69
70 FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES();
71
FIXTURE_SETUP(vfio_pci_driver_test)72 FIXTURE_SETUP(vfio_pci_driver_test)
73 {
74 struct vfio_pci_driver *driver;
75
76 self->iommu = iommu_init(variant->iommu_mode);
77 self->device = vfio_pci_device_init(device_bdf, self->iommu);
78 self->iova_allocator = iova_allocator_init(self->iommu);
79
80 driver = &self->device->driver;
81
82 region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G);
83 region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M);
84
85 /* Any IOVA that doesn't overlap memcpy_region and driver->region. */
86 self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G);
87
88 vfio_pci_driver_init(self->device);
89 self->msi_fd = self->device->msi_eventfds[driver->msi];
90
91 /*
92 * Use the maximum size supported by the device for memcpy operations,
93 * slimmed down to fit into the memcpy region (divided by 2 so src and
94 * dst regions do not overlap).
95 */
96 self->size = self->device->driver.max_memcpy_size;
97 self->size = min(self->size, self->memcpy_region.size / 2);
98
99 self->src = self->memcpy_region.vaddr;
100 self->dst = self->src + self->size;
101
102 self->src_iova = to_iova(self->device, self->src);
103 self->dst_iova = to_iova(self->device, self->dst);
104 }
105
FIXTURE_TEARDOWN(vfio_pci_driver_test)106 FIXTURE_TEARDOWN(vfio_pci_driver_test)
107 {
108 struct vfio_pci_driver *driver = &self->device->driver;
109
110 vfio_pci_driver_remove(self->device);
111
112 region_teardown(self->iommu, &self->memcpy_region);
113 region_teardown(self->iommu, &driver->region);
114
115 iova_allocator_cleanup(self->iova_allocator);
116 vfio_pci_device_cleanup(self->device);
117 iommu_cleanup(self->iommu);
118 }
119
TEST_F(vfio_pci_driver_test,init_remove)120 TEST_F(vfio_pci_driver_test, init_remove)
121 {
122 int i;
123
124 for (i = 0; i < 10; i++) {
125 vfio_pci_driver_remove(self->device);
126 vfio_pci_driver_init(self->device);
127 }
128 }
129
TEST_F(vfio_pci_driver_test,memcpy_success)130 TEST_F(vfio_pci_driver_test, memcpy_success)
131 {
132 fcntl_set_nonblock(self->msi_fd);
133
134 memset(self->src, 'x', self->size);
135 memset(self->dst, 'y', self->size);
136
137 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device,
138 self->src_iova,
139 self->dst_iova,
140 self->size));
141
142 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size));
143 ASSERT_NO_MSI(self->msi_fd);
144 }
145
TEST_F(vfio_pci_driver_test,memcpy_from_unmapped_iova)146 TEST_F(vfio_pci_driver_test, memcpy_from_unmapped_iova)
147 {
148 fcntl_set_nonblock(self->msi_fd);
149
150 /*
151 * Ignore the return value since not all devices will detect and report
152 * accesses to unmapped IOVAs as errors.
153 */
154 vfio_pci_driver_memcpy(self->device, self->unmapped_iova,
155 self->dst_iova, self->size);
156
157 ASSERT_NO_MSI(self->msi_fd);
158 }
159
TEST_F(vfio_pci_driver_test,memcpy_to_unmapped_iova)160 TEST_F(vfio_pci_driver_test, memcpy_to_unmapped_iova)
161 {
162 fcntl_set_nonblock(self->msi_fd);
163
164 /*
165 * Ignore the return value since not all devices will detect and report
166 * accesses to unmapped IOVAs as errors.
167 */
168 vfio_pci_driver_memcpy(self->device, self->src_iova,
169 self->unmapped_iova, self->size);
170
171 ASSERT_NO_MSI(self->msi_fd);
172 }
173
TEST_F(vfio_pci_driver_test,send_msi)174 TEST_F(vfio_pci_driver_test, send_msi)
175 {
176 u64 value;
177
178 vfio_pci_driver_send_msi(self->device);
179 ASSERT_EQ(8, read(self->msi_fd, &value, 8));
180 ASSERT_EQ(1, value);
181 }
182
TEST_F(vfio_pci_driver_test,mix_and_match)183 TEST_F(vfio_pci_driver_test, mix_and_match)
184 {
185 u64 value;
186 int i;
187
188 for (i = 0; i < 10; i++) {
189 memset(self->src, 'x', self->size);
190 memset(self->dst, 'y', self->size);
191
192 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device,
193 self->src_iova,
194 self->dst_iova,
195 self->size));
196
197 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size));
198
199 vfio_pci_driver_memcpy(self->device,
200 self->unmapped_iova,
201 self->dst_iova,
202 self->size);
203
204 vfio_pci_driver_send_msi(self->device);
205 ASSERT_EQ(8, read(self->msi_fd, &value, 8));
206 ASSERT_EQ(1, value);
207 }
208 }
209
210 TEST_F_TIMEOUT(vfio_pci_driver_test, memcpy_storm, 60)
211 {
212 struct vfio_pci_driver *driver = &self->device->driver;
213 u64 total_size;
214 u64 count;
215
216 fcntl_set_nonblock(self->msi_fd);
217
218 /*
219 * Perform up to 250GiB worth of DMA reads and writes across several
220 * memcpy operations. Some devices can support even more but the test
221 * will take too long.
222 */
223 total_size = 250UL * SZ_1G;
224 count = min(total_size / self->size, driver->max_memcpy_count);
225
226 printf("Kicking off %lu memcpys of size 0x%lx\n", count, self->size);
227 vfio_pci_driver_memcpy_start(self->device,
228 self->src_iova,
229 self->dst_iova,
230 self->size, count);
231
232 ASSERT_EQ(0, vfio_pci_driver_memcpy_wait(self->device));
233 ASSERT_NO_MSI(self->msi_fd);
234 }
235
device_has_selftests_driver(const char * bdf)236 static bool device_has_selftests_driver(const char *bdf)
237 {
238 struct vfio_pci_device *device;
239 struct iommu *iommu;
240 bool has_driver;
241
242 iommu = iommu_init(default_iommu_mode);
243 device = vfio_pci_device_init(device_bdf, iommu);
244
245 has_driver = !!device->driver.ops;
246
247 vfio_pci_device_cleanup(device);
248 iommu_cleanup(iommu);
249
250 return has_driver;
251 }
252
main(int argc,char * argv[])253 int main(int argc, char *argv[])
254 {
255 device_bdf = vfio_selftests_get_bdf(&argc, argv);
256
257 if (!device_has_selftests_driver(device_bdf)) {
258 fprintf(stderr, "No driver found for device %s\n", device_bdf);
259 return KSFT_SKIP;
260 }
261
262 return test_harness_run(argc, argv);
263 }
264