1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <sys/ioctl.h> 3 #include <sys/mman.h> 4 5 #include <linux/sizes.h> 6 #include <linux/vfio.h> 7 8 #include <libvfio.h> 9 10 #include "kselftest_harness.h" 11 12 static const char *device_bdf; 13 14 #define ASSERT_NO_MSI(_eventfd) do { \ 15 u64 __value; \ 16 \ 17 ASSERT_EQ(-1, read(_eventfd, &__value, 8)); \ 18 ASSERT_EQ(EAGAIN, errno); \ 19 } while (0) 20 21 static void region_setup(struct iommu *iommu, 22 struct iova_allocator *iova_allocator, 23 struct dma_region *region, u64 size) 24 { 25 const int flags = MAP_SHARED | MAP_ANONYMOUS; 26 const int prot = PROT_READ | PROT_WRITE; 27 void *vaddr; 28 29 vaddr = mmap(NULL, size, prot, flags, -1, 0); 30 VFIO_ASSERT_NE(vaddr, MAP_FAILED); 31 32 region->vaddr = vaddr; 33 region->iova = iova_allocator_alloc(iova_allocator, size); 34 region->size = size; 35 36 iommu_map(iommu, region); 37 } 38 39 static void region_teardown(struct iommu *iommu, struct dma_region *region) 40 { 41 iommu_unmap(iommu, region); 42 VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0); 43 } 44 45 FIXTURE(vfio_pci_driver_test) { 46 struct iommu *iommu; 47 struct vfio_pci_device *device; 48 struct iova_allocator *iova_allocator; 49 struct dma_region memcpy_region; 50 void *vaddr; 51 int msi_fd; 52 53 u64 size; 54 void *src; 55 void *dst; 56 iova_t src_iova; 57 iova_t dst_iova; 58 iova_t unmapped_iova; 59 }; 60 61 FIXTURE_VARIANT(vfio_pci_driver_test) { 62 const char *iommu_mode; 63 }; 64 65 #define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \ 66 FIXTURE_VARIANT_ADD(vfio_pci_driver_test, _iommu_mode) { \ 67 .iommu_mode = #_iommu_mode, \ 68 } 69 70 FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(); 71 72 FIXTURE_SETUP(vfio_pci_driver_test) 73 { 74 struct vfio_pci_driver *driver; 75 76 self->iommu = iommu_init(variant->iommu_mode); 77 self->device = vfio_pci_device_init(device_bdf, self->iommu); 78 self->iova_allocator = iova_allocator_init(self->iommu); 79 80 driver = &self->device->driver; 81 82 region_setup(self->iommu, self->iova_allocator, &self->memcpy_region, SZ_1G); 83 region_setup(self->iommu, self->iova_allocator, &driver->region, SZ_2M); 84 85 /* Any IOVA that doesn't overlap memcpy_region and driver->region. */ 86 self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G); 87 88 vfio_pci_driver_init(self->device); 89 self->msi_fd = self->device->msi_eventfds[driver->msi]; 90 91 /* 92 * Use the maximum size supported by the device for memcpy operations, 93 * slimmed down to fit into the memcpy region (divided by 2 so src and 94 * dst regions do not overlap). 95 */ 96 self->size = self->device->driver.max_memcpy_size; 97 self->size = min(self->size, self->memcpy_region.size / 2); 98 99 self->src = self->memcpy_region.vaddr; 100 self->dst = self->src + self->size; 101 102 self->src_iova = to_iova(self->device, self->src); 103 self->dst_iova = to_iova(self->device, self->dst); 104 } 105 106 FIXTURE_TEARDOWN(vfio_pci_driver_test) 107 { 108 struct vfio_pci_driver *driver = &self->device->driver; 109 110 vfio_pci_driver_remove(self->device); 111 112 region_teardown(self->iommu, &self->memcpy_region); 113 region_teardown(self->iommu, &driver->region); 114 115 iova_allocator_cleanup(self->iova_allocator); 116 vfio_pci_device_cleanup(self->device); 117 iommu_cleanup(self->iommu); 118 } 119 120 TEST_F(vfio_pci_driver_test, init_remove) 121 { 122 int i; 123 124 for (i = 0; i < 10; i++) { 125 vfio_pci_driver_remove(self->device); 126 vfio_pci_driver_init(self->device); 127 } 128 } 129 130 TEST_F(vfio_pci_driver_test, memcpy_success) 131 { 132 fcntl_set_nonblock(self->msi_fd); 133 134 memset(self->src, 'x', self->size); 135 memset(self->dst, 'y', self->size); 136 137 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 138 self->src_iova, 139 self->dst_iova, 140 self->size)); 141 142 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 143 ASSERT_NO_MSI(self->msi_fd); 144 } 145 146 TEST_F(vfio_pci_driver_test, memcpy_from_unmapped_iova) 147 { 148 fcntl_set_nonblock(self->msi_fd); 149 150 /* 151 * Ignore the return value since not all devices will detect and report 152 * accesses to unmapped IOVAs as errors. 153 */ 154 vfio_pci_driver_memcpy(self->device, self->unmapped_iova, 155 self->dst_iova, self->size); 156 157 ASSERT_NO_MSI(self->msi_fd); 158 } 159 160 TEST_F(vfio_pci_driver_test, memcpy_to_unmapped_iova) 161 { 162 fcntl_set_nonblock(self->msi_fd); 163 164 /* 165 * Ignore the return value since not all devices will detect and report 166 * accesses to unmapped IOVAs as errors. 167 */ 168 vfio_pci_driver_memcpy(self->device, self->src_iova, 169 self->unmapped_iova, self->size); 170 171 ASSERT_NO_MSI(self->msi_fd); 172 } 173 174 TEST_F(vfio_pci_driver_test, send_msi) 175 { 176 u64 value; 177 178 vfio_pci_driver_send_msi(self->device); 179 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 180 ASSERT_EQ(1, value); 181 } 182 183 TEST_F(vfio_pci_driver_test, mix_and_match) 184 { 185 u64 value; 186 int i; 187 188 for (i = 0; i < 10; i++) { 189 memset(self->src, 'x', self->size); 190 memset(self->dst, 'y', self->size); 191 192 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 193 self->src_iova, 194 self->dst_iova, 195 self->size)); 196 197 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 198 199 vfio_pci_driver_memcpy(self->device, 200 self->unmapped_iova, 201 self->dst_iova, 202 self->size); 203 204 vfio_pci_driver_send_msi(self->device); 205 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 206 ASSERT_EQ(1, value); 207 } 208 } 209 210 TEST_F_TIMEOUT(vfio_pci_driver_test, memcpy_storm, 60) 211 { 212 struct vfio_pci_driver *driver = &self->device->driver; 213 u64 total_size; 214 u64 count; 215 216 fcntl_set_nonblock(self->msi_fd); 217 218 /* 219 * Perform up to 250GiB worth of DMA reads and writes across several 220 * memcpy operations. Some devices can support even more but the test 221 * will take too long. 222 */ 223 total_size = 250UL * SZ_1G; 224 count = min(total_size / self->size, driver->max_memcpy_count); 225 226 printf("Kicking off %lu memcpys of size 0x%lx\n", count, self->size); 227 vfio_pci_driver_memcpy_start(self->device, 228 self->src_iova, 229 self->dst_iova, 230 self->size, count); 231 232 ASSERT_EQ(0, vfio_pci_driver_memcpy_wait(self->device)); 233 ASSERT_NO_MSI(self->msi_fd); 234 } 235 236 static bool device_has_selftests_driver(const char *bdf) 237 { 238 struct vfio_pci_device *device; 239 struct iommu *iommu; 240 bool has_driver; 241 242 iommu = iommu_init(default_iommu_mode); 243 device = vfio_pci_device_init(device_bdf, iommu); 244 245 has_driver = !!device->driver.ops; 246 247 vfio_pci_device_cleanup(device); 248 iommu_cleanup(iommu); 249 250 return has_driver; 251 } 252 253 int main(int argc, char *argv[]) 254 { 255 device_bdf = vfio_selftests_get_bdf(&argc, argv); 256 257 if (!device_has_selftests_driver(device_bdf)) { 258 fprintf(stderr, "No driver found for device %s\n", device_bdf); 259 return KSFT_SKIP; 260 } 261 262 return test_harness_run(argc, argv); 263 } 264