1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 #ifndef SELFTESTS_VFIO_LIB_INCLUDE_VFIO_UTIL_H 3 #define SELFTESTS_VFIO_LIB_INCLUDE_VFIO_UTIL_H 4 5 #include <fcntl.h> 6 #include <string.h> 7 8 #include <uapi/linux/types.h> 9 #include <linux/iommufd.h> 10 #include <linux/list.h> 11 #include <linux/pci_regs.h> 12 #include <linux/vfio.h> 13 14 #include "../../../kselftest.h" 15 16 #define VFIO_LOG_AND_EXIT(...) do { \ 17 fprintf(stderr, " " __VA_ARGS__); \ 18 fprintf(stderr, "\n"); \ 19 exit(KSFT_FAIL); \ 20 } while (0) 21 22 #define VFIO_ASSERT_OP(_lhs, _rhs, _op, ...) do { \ 23 typeof(_lhs) __lhs = (_lhs); \ 24 typeof(_rhs) __rhs = (_rhs); \ 25 \ 26 if (__lhs _op __rhs) \ 27 break; \ 28 \ 29 fprintf(stderr, "%s:%u: Assertion Failure\n\n", __FILE__, __LINE__); \ 30 fprintf(stderr, " Expression: " #_lhs " " #_op " " #_rhs "\n"); \ 31 fprintf(stderr, " Observed: %#lx %s %#lx\n", \ 32 (u64)__lhs, #_op, (u64)__rhs); \ 33 fprintf(stderr, " [errno: %d - %s]\n", errno, strerror(errno)); \ 34 VFIO_LOG_AND_EXIT(__VA_ARGS__); \ 35 } while (0) 36 37 #define VFIO_ASSERT_EQ(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, ==, ##__VA_ARGS__) 38 #define VFIO_ASSERT_NE(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, !=, ##__VA_ARGS__) 39 #define VFIO_ASSERT_LT(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, <, ##__VA_ARGS__) 40 #define VFIO_ASSERT_LE(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, <=, ##__VA_ARGS__) 41 #define VFIO_ASSERT_GT(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, >, ##__VA_ARGS__) 42 #define VFIO_ASSERT_GE(_a, _b, ...) VFIO_ASSERT_OP(_a, _b, >=, ##__VA_ARGS__) 43 #define VFIO_ASSERT_TRUE(_a, ...) VFIO_ASSERT_NE(false, (_a), ##__VA_ARGS__) 44 #define VFIO_ASSERT_FALSE(_a, ...) VFIO_ASSERT_EQ(false, (_a), ##__VA_ARGS__) 45 #define VFIO_ASSERT_NULL(_a, ...) VFIO_ASSERT_EQ(NULL, _a, ##__VA_ARGS__) 46 #define VFIO_ASSERT_NOT_NULL(_a, ...) VFIO_ASSERT_NE(NULL, _a, ##__VA_ARGS__) 47 48 #define VFIO_FAIL(_fmt, ...) do { \ 49 fprintf(stderr, "%s:%u: FAIL\n\n", __FILE__, __LINE__); \ 50 VFIO_LOG_AND_EXIT(_fmt, ##__VA_ARGS__); \ 51 } while (0) 52 53 struct vfio_iommu_mode { 54 const char *name; 55 const char *container_path; 56 unsigned long iommu_type; 57 }; 58 59 /* 60 * Generator for VFIO selftests fixture variants that replicate across all 61 * possible IOMMU modes. Tests must define FIXTURE_VARIANT_ADD_IOMMU_MODE() 62 * which should then use FIXTURE_VARIANT_ADD() to create the variant. 63 */ 64 #define FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(...) \ 65 FIXTURE_VARIANT_ADD_IOMMU_MODE(vfio_type1_iommu, ##__VA_ARGS__); \ 66 FIXTURE_VARIANT_ADD_IOMMU_MODE(vfio_type1v2_iommu, ##__VA_ARGS__); \ 67 FIXTURE_VARIANT_ADD_IOMMU_MODE(iommufd_compat_type1, ##__VA_ARGS__); \ 68 FIXTURE_VARIANT_ADD_IOMMU_MODE(iommufd_compat_type1v2, ##__VA_ARGS__); \ 69 FIXTURE_VARIANT_ADD_IOMMU_MODE(iommufd, ##__VA_ARGS__) 70 71 struct vfio_pci_bar { 72 struct vfio_region_info info; 73 void *vaddr; 74 }; 75 76 typedef u64 iova_t; 77 78 #define INVALID_IOVA UINT64_MAX 79 80 struct vfio_dma_region { 81 struct list_head link; 82 void *vaddr; 83 iova_t iova; 84 u64 size; 85 }; 86 87 struct vfio_pci_device; 88 89 struct vfio_pci_driver_ops { 90 const char *name; 91 92 /** 93 * @probe() - Check if the driver supports the given device. 94 * 95 * Return: 0 on success, non-0 on failure. 96 */ 97 int (*probe)(struct vfio_pci_device *device); 98 99 /** 100 * @init() - Initialize the driver for @device. 101 * 102 * Must be called after device->driver.region has been initialized. 103 */ 104 void (*init)(struct vfio_pci_device *device); 105 106 /** 107 * remove() - Deinitialize the driver for @device. 108 */ 109 void (*remove)(struct vfio_pci_device *device); 110 111 /** 112 * memcpy_start() - Kick off @count repeated memcpy operations from 113 * [@src, @src + @size) to [@dst, @dst + @size). 114 * 115 * Guarantees: 116 * - The device will attempt DMA reads on [src, src + size). 117 * - The device will attempt DMA writes on [dst, dst + size). 118 * - The device will not generate any interrupts. 119 * 120 * memcpy_start() returns immediately, it does not wait for the 121 * copies to complete. 122 */ 123 void (*memcpy_start)(struct vfio_pci_device *device, 124 iova_t src, iova_t dst, u64 size, u64 count); 125 126 /** 127 * memcpy_wait() - Wait until the memcpy operations started by 128 * memcpy_start() have finished. 129 * 130 * Guarantees: 131 * - All in-flight DMAs initiated by memcpy_start() are fully complete 132 * before memcpy_wait() returns. 133 * 134 * Returns non-0 if the driver detects that an error occurred during the 135 * memcpy, 0 otherwise. 136 */ 137 int (*memcpy_wait)(struct vfio_pci_device *device); 138 139 /** 140 * send_msi() - Make the device send the MSI device->driver.msi. 141 * 142 * Guarantees: 143 * - The device will send the MSI once. 144 */ 145 void (*send_msi)(struct vfio_pci_device *device); 146 }; 147 148 struct vfio_pci_driver { 149 const struct vfio_pci_driver_ops *ops; 150 bool initialized; 151 bool memcpy_in_progress; 152 153 /* Region to be used by the driver (e.g. for in-memory descriptors) */ 154 struct vfio_dma_region region; 155 156 /* The maximum size that can be passed to memcpy_start(). */ 157 u64 max_memcpy_size; 158 159 /* The maximum count that can be passed to memcpy_start(). */ 160 u64 max_memcpy_count; 161 162 /* The MSI vector the device will signal in ops->send_msi(). */ 163 int msi; 164 }; 165 166 struct vfio_pci_device { 167 int fd; 168 169 const struct vfio_iommu_mode *iommu_mode; 170 int group_fd; 171 int container_fd; 172 173 int iommufd; 174 u32 ioas_id; 175 176 struct vfio_device_info info; 177 struct vfio_region_info config_space; 178 struct vfio_pci_bar bars[PCI_STD_NUM_BARS]; 179 180 struct vfio_irq_info msi_info; 181 struct vfio_irq_info msix_info; 182 183 struct list_head dma_regions; 184 185 /* eventfds for MSI and MSI-x interrupts */ 186 int msi_eventfds[PCI_MSIX_FLAGS_QSIZE + 1]; 187 188 struct vfio_pci_driver driver; 189 }; 190 191 struct iova_allocator { 192 struct iommu_iova_range *ranges; 193 u32 nranges; 194 u32 range_idx; 195 u64 range_offset; 196 }; 197 198 /* 199 * Return the BDF string of the device that the test should use. 200 * 201 * If a BDF string is provided by the user on the command line (as the last 202 * element of argv[]), then this function will return that and decrement argc 203 * by 1. 204 * 205 * Otherwise this function will attempt to use the environment variable 206 * $VFIO_SELFTESTS_BDF. 207 * 208 * If BDF cannot be determined then the test will exit with KSFT_SKIP. 209 */ 210 const char *vfio_selftests_get_bdf(int *argc, char *argv[]); 211 const char *vfio_pci_get_cdev_path(const char *bdf); 212 213 extern const char *default_iommu_mode; 214 215 struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode); 216 void vfio_pci_device_cleanup(struct vfio_pci_device *device); 217 void vfio_pci_device_reset(struct vfio_pci_device *device); 218 219 struct iommu_iova_range *vfio_pci_iova_ranges(struct vfio_pci_device *device, 220 u32 *nranges); 221 222 struct iova_allocator *iova_allocator_init(struct vfio_pci_device *device); 223 void iova_allocator_cleanup(struct iova_allocator *allocator); 224 iova_t iova_allocator_alloc(struct iova_allocator *allocator, size_t size); 225 226 int __vfio_pci_dma_map(struct vfio_pci_device *device, 227 struct vfio_dma_region *region); 228 int __vfio_pci_dma_unmap(struct vfio_pci_device *device, 229 struct vfio_dma_region *region, 230 u64 *unmapped); 231 int __vfio_pci_dma_unmap_all(struct vfio_pci_device *device, u64 *unmapped); 232 233 static inline void vfio_pci_dma_map(struct vfio_pci_device *device, 234 struct vfio_dma_region *region) 235 { 236 VFIO_ASSERT_EQ(__vfio_pci_dma_map(device, region), 0); 237 } 238 239 static inline void vfio_pci_dma_unmap(struct vfio_pci_device *device, 240 struct vfio_dma_region *region) 241 { 242 VFIO_ASSERT_EQ(__vfio_pci_dma_unmap(device, region, NULL), 0); 243 } 244 245 static inline void vfio_pci_dma_unmap_all(struct vfio_pci_device *device) 246 { 247 VFIO_ASSERT_EQ(__vfio_pci_dma_unmap_all(device, NULL), 0); 248 } 249 250 void vfio_pci_config_access(struct vfio_pci_device *device, bool write, 251 size_t config, size_t size, void *data); 252 253 #define vfio_pci_config_read(_device, _offset, _type) ({ \ 254 _type __data; \ 255 vfio_pci_config_access((_device), false, _offset, sizeof(__data), &__data); \ 256 __data; \ 257 }) 258 259 #define vfio_pci_config_readb(_d, _o) vfio_pci_config_read(_d, _o, u8) 260 #define vfio_pci_config_readw(_d, _o) vfio_pci_config_read(_d, _o, u16) 261 #define vfio_pci_config_readl(_d, _o) vfio_pci_config_read(_d, _o, u32) 262 263 #define vfio_pci_config_write(_device, _offset, _value, _type) do { \ 264 _type __data = (_value); \ 265 vfio_pci_config_access((_device), true, _offset, sizeof(_type), &__data); \ 266 } while (0) 267 268 #define vfio_pci_config_writeb(_d, _o, _v) vfio_pci_config_write(_d, _o, _v, u8) 269 #define vfio_pci_config_writew(_d, _o, _v) vfio_pci_config_write(_d, _o, _v, u16) 270 #define vfio_pci_config_writel(_d, _o, _v) vfio_pci_config_write(_d, _o, _v, u32) 271 272 void vfio_pci_irq_enable(struct vfio_pci_device *device, u32 index, 273 u32 vector, int count); 274 void vfio_pci_irq_disable(struct vfio_pci_device *device, u32 index); 275 void vfio_pci_irq_trigger(struct vfio_pci_device *device, u32 index, u32 vector); 276 277 static inline void fcntl_set_nonblock(int fd) 278 { 279 int r; 280 281 r = fcntl(fd, F_GETFL, 0); 282 VFIO_ASSERT_NE(r, -1, "F_GETFL failed for fd %d\n", fd); 283 284 r = fcntl(fd, F_SETFL, r | O_NONBLOCK); 285 VFIO_ASSERT_NE(r, -1, "F_SETFL O_NONBLOCK failed for fd %d\n", fd); 286 } 287 288 static inline void vfio_pci_msi_enable(struct vfio_pci_device *device, 289 u32 vector, int count) 290 { 291 vfio_pci_irq_enable(device, VFIO_PCI_MSI_IRQ_INDEX, vector, count); 292 } 293 294 static inline void vfio_pci_msi_disable(struct vfio_pci_device *device) 295 { 296 vfio_pci_irq_disable(device, VFIO_PCI_MSI_IRQ_INDEX); 297 } 298 299 static inline void vfio_pci_msix_enable(struct vfio_pci_device *device, 300 u32 vector, int count) 301 { 302 vfio_pci_irq_enable(device, VFIO_PCI_MSIX_IRQ_INDEX, vector, count); 303 } 304 305 static inline void vfio_pci_msix_disable(struct vfio_pci_device *device) 306 { 307 vfio_pci_irq_disable(device, VFIO_PCI_MSIX_IRQ_INDEX); 308 } 309 310 iova_t __to_iova(struct vfio_pci_device *device, void *vaddr); 311 iova_t to_iova(struct vfio_pci_device *device, void *vaddr); 312 313 static inline bool vfio_pci_device_match(struct vfio_pci_device *device, 314 u16 vendor_id, u16 device_id) 315 { 316 return (vendor_id == vfio_pci_config_readw(device, PCI_VENDOR_ID)) && 317 (device_id == vfio_pci_config_readw(device, PCI_DEVICE_ID)); 318 } 319 320 void vfio_pci_driver_probe(struct vfio_pci_device *device); 321 void vfio_pci_driver_init(struct vfio_pci_device *device); 322 void vfio_pci_driver_remove(struct vfio_pci_device *device); 323 int vfio_pci_driver_memcpy(struct vfio_pci_device *device, 324 iova_t src, iova_t dst, u64 size); 325 void vfio_pci_driver_memcpy_start(struct vfio_pci_device *device, 326 iova_t src, iova_t dst, u64 size, 327 u64 count); 328 int vfio_pci_driver_memcpy_wait(struct vfio_pci_device *device); 329 void vfio_pci_driver_send_msi(struct vfio_pci_device *device); 330 331 #endif /* SELFTESTS_VFIO_LIB_INCLUDE_VFIO_UTIL_H */ 332