1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <sys/ioctl.h> 3 #include <sys/mman.h> 4 5 #include <linux/sizes.h> 6 #include <linux/vfio.h> 7 8 #include <libvfio.h> 9 10 #include "kselftest_harness.h" 11 12 static const char *device_bdf; 13 14 #define fcntl_set_msi_nonblock(_self) do { \ 15 if (_self->device->driver.ops->send_msi) \ 16 fcntl_set_nonblock(_self->msi_fd); \ 17 } while (0) 18 19 #define ASSERT_NO_MSI(_self) do { \ 20 u64 __value; \ 21 \ 22 if (!_self->device->driver.ops->send_msi) \ 23 break; \ 24 ASSERT_EQ(-1, read(_self->msi_fd, &__value, 8)); \ 25 ASSERT_EQ(EAGAIN, errno); \ 26 } while (0) 27 28 static void region_setup(struct iommu *iommu, 29 struct iova_allocator *iova_allocator, 30 struct dma_region *region, u64 size) 31 { 32 const int flags = MAP_SHARED | MAP_ANONYMOUS; 33 const int prot = PROT_READ | PROT_WRITE; 34 void *vaddr; 35 36 vaddr = mmap(NULL, size, prot, flags, -1, 0); 37 VFIO_ASSERT_NE(vaddr, MAP_FAILED); 38 39 region->vaddr = vaddr; 40 region->iova = iova_allocator_alloc(iova_allocator, size); 41 region->size = size; 42 43 iommu_map(iommu, region); 44 } 45 46 static void region_teardown(struct iommu *iommu, struct dma_region *region) 47 { 48 iommu_unmap(iommu, region); 49 VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0); 50 } 51 52 FIXTURE(vfio_pci_driver_test) { 53 struct iommu *iommu; 54 struct vfio_pci_device *device; 55 struct iova_allocator *iova_allocator; 56 struct dma_region memcpy_region; 57 void *vaddr; 58 int msi_fd; 59 60 u64 size; 61 void *src; 62 void *dst; 63 iova_t src_iova; 64 iova_t dst_iova; 65 iova_t unmapped_iova; 66 }; 67 68 FIXTURE_VARIANT(vfio_pci_driver_test) { 69 const char *iommu_mode; 70 }; 71 72 #define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \ 73 FIXTURE_VARIANT_ADD(vfio_pci_driver_test, _iommu_mode) { \ 74 .iommu_mode = #_iommu_mode, \ 75 } 76 77 FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(); 78 79 FIXTURE_SETUP(vfio_pci_driver_test) 80 { 81 struct vfio_pci_driver *driver; 82 83 self->iommu = iommu_init(variant->iommu_mode); 84 self->device = vfio_pci_device_init(device_bdf, self->iommu); 85 self->iova_allocator = iova_allocator_init(self->iommu); 86 87 driver = &self->device->driver; 88 89 region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G); 90 region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M); 91 92 /* Any IOVA that doesn't overlap memcpy_region and driver->region. */ 93 self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G); 94 95 vfio_pci_driver_init(self->device); 96 self->msi_fd = self->device->msi_eventfds[driver->msi]; 97 98 /* 99 * Use 4x the driver's max_memcpy_size to exercise the chunking 100 * logic in vfio_pci_driver_memcpy(). Cap to half the memcpy 101 * region so src and dst do not overlap. 102 */ 103 self->size = min_t(u64, driver->max_memcpy_size * 4, 104 self->memcpy_region.size / 2); 105 106 self->src = self->memcpy_region.vaddr; 107 self->dst = self->src + self->size; 108 109 self->src_iova = to_iova(self->device, self->src); 110 self->dst_iova = to_iova(self->device, self->dst); 111 } 112 113 FIXTURE_TEARDOWN(vfio_pci_driver_test) 114 { 115 struct vfio_pci_driver *driver = &self->device->driver; 116 117 vfio_pci_driver_remove(self->device); 118 119 region_teardown(self->iommu, &self->memcpy_region); 120 region_teardown(self->iommu, &driver->region); 121 122 iova_allocator_cleanup(self->iova_allocator); 123 vfio_pci_device_cleanup(self->device); 124 iommu_cleanup(self->iommu); 125 } 126 127 TEST_F(vfio_pci_driver_test, init_remove) 128 { 129 int i; 130 131 for (i = 0; i < 10; i++) { 132 vfio_pci_driver_remove(self->device); 133 vfio_pci_driver_init(self->device); 134 } 135 } 136 137 TEST_F(vfio_pci_driver_test, memcpy_success) 138 { 139 fcntl_set_msi_nonblock(self); 140 141 memset(self->src, 'x', self->size); 142 memset(self->dst, 'y', self->size); 143 144 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 145 self->src_iova, 146 self->dst_iova, 147 self->size)); 148 149 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 150 ASSERT_NO_MSI(self); 151 } 152 153 TEST_F(vfio_pci_driver_test, memcpy_from_unmapped_iova) 154 { 155 fcntl_set_msi_nonblock(self); 156 157 /* 158 * Ignore the return value since not all devices will detect and report 159 * accesses to unmapped IOVAs as errors. 160 */ 161 vfio_pci_driver_memcpy(self->device, self->unmapped_iova, 162 self->dst_iova, self->size); 163 164 ASSERT_NO_MSI(self); 165 } 166 167 TEST_F(vfio_pci_driver_test, memcpy_to_unmapped_iova) 168 { 169 fcntl_set_msi_nonblock(self); 170 171 /* 172 * Ignore the return value since not all devices will detect and report 173 * accesses to unmapped IOVAs as errors. 174 */ 175 vfio_pci_driver_memcpy(self->device, self->src_iova, 176 self->unmapped_iova, self->size); 177 178 ASSERT_NO_MSI(self); 179 } 180 181 TEST_F(vfio_pci_driver_test, send_msi) 182 { 183 u64 value; 184 185 if (!self->device->driver.ops->send_msi) 186 SKIP(return, "Driver does not support send_msi()\n"); 187 188 vfio_pci_driver_send_msi(self->device); 189 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 190 ASSERT_EQ(1, value); 191 } 192 193 TEST_F(vfio_pci_driver_test, mix_and_match) 194 { 195 u64 value; 196 int i; 197 198 for (i = 0; i < 10; i++) { 199 memset(self->src, 'x', self->size); 200 memset(self->dst, 'y', self->size); 201 202 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 203 self->src_iova, 204 self->dst_iova, 205 self->size)); 206 207 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 208 209 vfio_pci_driver_memcpy(self->device, 210 self->unmapped_iova, 211 self->dst_iova, 212 self->size); 213 214 if (!self->device->driver.ops->send_msi) 215 continue; 216 217 vfio_pci_driver_send_msi(self->device); 218 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 219 ASSERT_EQ(1, value); 220 } 221 } 222 223 TEST_F_TIMEOUT(vfio_pci_driver_test, memcpy_storm, 60) 224 { 225 struct vfio_pci_driver *driver = &self->device->driver; 226 u64 total_size; 227 u64 size; 228 u64 count; 229 230 fcntl_set_msi_nonblock(self); 231 232 /* 233 * Perform up to 250GiB worth of DMA reads and writes across several 234 * memcpy operations. Some devices can support even more but the test 235 * will take too long. 236 */ 237 total_size = 250UL * SZ_1G; 238 size = min(driver->max_memcpy_size, self->memcpy_region.size / 2); 239 count = min(total_size / size, driver->max_memcpy_count); 240 241 printf("Kicking off %lu memcpys of size 0x%lx\n", count, size); 242 vfio_pci_driver_memcpy_start(self->device, 243 self->src_iova, 244 self->dst_iova, 245 size, count); 246 247 ASSERT_EQ(0, vfio_pci_driver_memcpy_wait(self->device)); 248 ASSERT_NO_MSI(self); 249 } 250 251 static bool device_has_selftests_driver(const char *bdf) 252 { 253 struct vfio_pci_device *device; 254 struct iommu *iommu; 255 bool has_driver; 256 257 iommu = iommu_init(default_iommu_mode); 258 device = vfio_pci_device_init(device_bdf, iommu); 259 260 has_driver = !!device->driver.ops; 261 262 vfio_pci_device_cleanup(device); 263 iommu_cleanup(iommu); 264 265 return has_driver; 266 } 267 268 int main(int argc, char *argv[]) 269 { 270 device_bdf = vfio_selftests_get_bdf(&argc, argv); 271 272 if (!device_has_selftests_driver(device_bdf)) { 273 fprintf(stderr, "No driver found for device %s\n", device_bdf); 274 return KSFT_SKIP; 275 } 276 277 return test_harness_run(argc, argv); 278 } 279