1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <sys/ioctl.h> 3 #include <sys/mman.h> 4 5 #include <linux/sizes.h> 6 #include <linux/vfio.h> 7 8 #include <vfio_util.h> 9 10 #include "../kselftest_harness.h" 11 12 static const char *device_bdf; 13 14 #define ASSERT_NO_MSI(_eventfd) do { \ 15 u64 __value; \ 16 \ 17 ASSERT_EQ(-1, read(_eventfd, &__value, 8)); \ 18 ASSERT_EQ(EAGAIN, errno); \ 19 } while (0) 20 21 static void region_setup(struct vfio_pci_device *device, 22 struct iova_allocator *iova_allocator, 23 struct dma_region *region, u64 size) 24 { 25 const int flags = MAP_SHARED | MAP_ANONYMOUS; 26 const int prot = PROT_READ | PROT_WRITE; 27 void *vaddr; 28 29 vaddr = mmap(NULL, size, prot, flags, -1, 0); 30 VFIO_ASSERT_NE(vaddr, MAP_FAILED); 31 32 region->vaddr = vaddr; 33 region->iova = iova_allocator_alloc(iova_allocator, size); 34 region->size = size; 35 36 vfio_pci_dma_map(device, region); 37 } 38 39 static void region_teardown(struct vfio_pci_device *device, 40 struct dma_region *region) 41 { 42 vfio_pci_dma_unmap(device, region); 43 VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0); 44 } 45 46 FIXTURE(vfio_pci_driver_test) { 47 struct iommu *iommu; 48 struct vfio_pci_device *device; 49 struct iova_allocator *iova_allocator; 50 struct dma_region memcpy_region; 51 void *vaddr; 52 int msi_fd; 53 54 u64 size; 55 void *src; 56 void *dst; 57 iova_t src_iova; 58 iova_t dst_iova; 59 iova_t unmapped_iova; 60 }; 61 62 FIXTURE_VARIANT(vfio_pci_driver_test) { 63 const char *iommu_mode; 64 }; 65 66 #define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \ 67 FIXTURE_VARIANT_ADD(vfio_pci_driver_test, _iommu_mode) { \ 68 .iommu_mode = #_iommu_mode, \ 69 } 70 71 FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(); 72 73 FIXTURE_SETUP(vfio_pci_driver_test) 74 { 75 struct vfio_pci_driver *driver; 76 77 self->iommu = iommu_init(variant->iommu_mode); 78 self->device = vfio_pci_device_init(device_bdf, self->iommu); 79 self->iova_allocator = iova_allocator_init(self->device); 80 81 driver = &self->device->driver; 82 83 region_setup(self->device, self->iova_allocator, &self->memcpy_region, SZ_1G); 84 region_setup(self->device, self->iova_allocator, &driver->region, SZ_2M); 85 86 /* Any IOVA that doesn't overlap memcpy_region and driver->region. */ 87 self->unmapped_iova = iova_allocator_alloc(self->iova_allocator, SZ_1G); 88 89 vfio_pci_driver_init(self->device); 90 self->msi_fd = self->device->msi_eventfds[driver->msi]; 91 92 /* 93 * Use the maximum size supported by the device for memcpy operations, 94 * slimmed down to fit into the memcpy region (divided by 2 so src and 95 * dst regions do not overlap). 96 */ 97 self->size = self->device->driver.max_memcpy_size; 98 self->size = min(self->size, self->memcpy_region.size / 2); 99 100 self->src = self->memcpy_region.vaddr; 101 self->dst = self->src + self->size; 102 103 self->src_iova = to_iova(self->device, self->src); 104 self->dst_iova = to_iova(self->device, self->dst); 105 } 106 107 FIXTURE_TEARDOWN(vfio_pci_driver_test) 108 { 109 struct vfio_pci_driver *driver = &self->device->driver; 110 111 vfio_pci_driver_remove(self->device); 112 113 region_teardown(self->device, &self->memcpy_region); 114 region_teardown(self->device, &driver->region); 115 116 iova_allocator_cleanup(self->iova_allocator); 117 vfio_pci_device_cleanup(self->device); 118 iommu_cleanup(self->iommu); 119 } 120 121 TEST_F(vfio_pci_driver_test, init_remove) 122 { 123 int i; 124 125 for (i = 0; i < 10; i++) { 126 vfio_pci_driver_remove(self->device); 127 vfio_pci_driver_init(self->device); 128 } 129 } 130 131 TEST_F(vfio_pci_driver_test, memcpy_success) 132 { 133 fcntl_set_nonblock(self->msi_fd); 134 135 memset(self->src, 'x', self->size); 136 memset(self->dst, 'y', self->size); 137 138 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 139 self->src_iova, 140 self->dst_iova, 141 self->size)); 142 143 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 144 ASSERT_NO_MSI(self->msi_fd); 145 } 146 147 TEST_F(vfio_pci_driver_test, memcpy_from_unmapped_iova) 148 { 149 fcntl_set_nonblock(self->msi_fd); 150 151 /* 152 * Ignore the return value since not all devices will detect and report 153 * accesses to unmapped IOVAs as errors. 154 */ 155 vfio_pci_driver_memcpy(self->device, self->unmapped_iova, 156 self->dst_iova, self->size); 157 158 ASSERT_NO_MSI(self->msi_fd); 159 } 160 161 TEST_F(vfio_pci_driver_test, memcpy_to_unmapped_iova) 162 { 163 fcntl_set_nonblock(self->msi_fd); 164 165 /* 166 * Ignore the return value since not all devices will detect and report 167 * accesses to unmapped IOVAs as errors. 168 */ 169 vfio_pci_driver_memcpy(self->device, self->src_iova, 170 self->unmapped_iova, self->size); 171 172 ASSERT_NO_MSI(self->msi_fd); 173 } 174 175 TEST_F(vfio_pci_driver_test, send_msi) 176 { 177 u64 value; 178 179 vfio_pci_driver_send_msi(self->device); 180 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 181 ASSERT_EQ(1, value); 182 } 183 184 TEST_F(vfio_pci_driver_test, mix_and_match) 185 { 186 u64 value; 187 int i; 188 189 for (i = 0; i < 10; i++) { 190 memset(self->src, 'x', self->size); 191 memset(self->dst, 'y', self->size); 192 193 ASSERT_EQ(0, vfio_pci_driver_memcpy(self->device, 194 self->src_iova, 195 self->dst_iova, 196 self->size)); 197 198 ASSERT_EQ(0, memcmp(self->src, self->dst, self->size)); 199 200 vfio_pci_driver_memcpy(self->device, 201 self->unmapped_iova, 202 self->dst_iova, 203 self->size); 204 205 vfio_pci_driver_send_msi(self->device); 206 ASSERT_EQ(8, read(self->msi_fd, &value, 8)); 207 ASSERT_EQ(1, value); 208 } 209 } 210 211 TEST_F_TIMEOUT(vfio_pci_driver_test, memcpy_storm, 60) 212 { 213 struct vfio_pci_driver *driver = &self->device->driver; 214 u64 total_size; 215 u64 count; 216 217 fcntl_set_nonblock(self->msi_fd); 218 219 /* 220 * Perform up to 250GiB worth of DMA reads and writes across several 221 * memcpy operations. Some devices can support even more but the test 222 * will take too long. 223 */ 224 total_size = 250UL * SZ_1G; 225 count = min(total_size / self->size, driver->max_memcpy_count); 226 227 printf("Kicking off %lu memcpys of size 0x%lx\n", count, self->size); 228 vfio_pci_driver_memcpy_start(self->device, 229 self->src_iova, 230 self->dst_iova, 231 self->size, count); 232 233 ASSERT_EQ(0, vfio_pci_driver_memcpy_wait(self->device)); 234 ASSERT_NO_MSI(self->msi_fd); 235 } 236 237 static bool device_has_selftests_driver(const char *bdf) 238 { 239 struct vfio_pci_device *device; 240 struct iommu *iommu; 241 bool has_driver; 242 243 iommu = iommu_init(default_iommu_mode); 244 device = vfio_pci_device_init(device_bdf, iommu); 245 246 has_driver = !!device->driver.ops; 247 248 vfio_pci_device_cleanup(device); 249 iommu_cleanup(iommu); 250 251 return has_driver; 252 } 253 254 int main(int argc, char *argv[]) 255 { 256 device_bdf = vfio_selftests_get_bdf(&argc, argv); 257 258 if (!device_has_selftests_driver(device_bdf)) { 259 fprintf(stderr, "No driver found for device %s\n", device_bdf); 260 return KSFT_SKIP; 261 } 262 263 return test_harness_run(argc, argv); 264 } 265