1 // SPDX-License-Identifier: GPL-2.0-only 2 #include "kselftest.h" 3 #include <libvfio.h> 4 5 #ifdef __x86_64__ 6 extern struct vfio_pci_driver_ops dsa_ops; 7 extern struct vfio_pci_driver_ops ioat_ops; 8 #endif 9 10 static struct vfio_pci_driver_ops *driver_ops[] = { 11 #ifdef __x86_64__ 12 &dsa_ops, 13 &ioat_ops, 14 #endif 15 }; 16 17 void vfio_pci_driver_probe(struct vfio_pci_device *device) 18 { 19 struct vfio_pci_driver_ops *ops; 20 int i; 21 22 VFIO_ASSERT_NULL(device->driver.ops); 23 24 for (i = 0; i < ARRAY_SIZE(driver_ops); i++) { 25 ops = driver_ops[i]; 26 27 if (ops->probe(device)) 28 continue; 29 30 device->driver.ops = ops; 31 } 32 } 33 34 static void vfio_check_driver_op(struct vfio_pci_driver *driver, void *op, 35 const char *op_name) 36 { 37 VFIO_ASSERT_NOT_NULL(driver->ops); 38 VFIO_ASSERT_NOT_NULL(op, "Driver has no %s()\n", op_name); 39 VFIO_ASSERT_EQ(driver->initialized, op != driver->ops->init); 40 VFIO_ASSERT_EQ(driver->memcpy_in_progress, op == driver->ops->memcpy_wait); 41 } 42 43 #define VFIO_CHECK_DRIVER_OP(_driver, _op) do { \ 44 struct vfio_pci_driver *__driver = (_driver); \ 45 vfio_check_driver_op(__driver, __driver->ops->_op, #_op); \ 46 } while (0) 47 48 void vfio_pci_driver_init(struct vfio_pci_device *device) 49 { 50 struct vfio_pci_driver *driver = &device->driver; 51 52 VFIO_ASSERT_NOT_NULL(driver->region.vaddr); 53 VFIO_CHECK_DRIVER_OP(driver, init); 54 55 driver->ops->init(device); 56 57 driver->initialized = true; 58 } 59 60 void vfio_pci_driver_remove(struct vfio_pci_device *device) 61 { 62 struct vfio_pci_driver *driver = &device->driver; 63 64 VFIO_CHECK_DRIVER_OP(driver, remove); 65 66 driver->ops->remove(device); 67 driver->initialized = false; 68 } 69 70 void vfio_pci_driver_send_msi(struct vfio_pci_device *device) 71 { 72 struct vfio_pci_driver *driver = &device->driver; 73 74 VFIO_CHECK_DRIVER_OP(driver, send_msi); 75 76 driver->ops->send_msi(device); 77 } 78 79 void vfio_pci_driver_memcpy_start(struct vfio_pci_device *device, 80 iova_t src, iova_t dst, u64 size, 81 u64 count) 82 { 83 struct vfio_pci_driver *driver = &device->driver; 84 85 VFIO_ASSERT_LE(size, driver->max_memcpy_size); 86 VFIO_ASSERT_LE(count, driver->max_memcpy_count); 87 VFIO_CHECK_DRIVER_OP(driver, memcpy_start); 88 89 driver->ops->memcpy_start(device, src, dst, size, count); 90 driver->memcpy_in_progress = true; 91 } 92 93 int vfio_pci_driver_memcpy_wait(struct vfio_pci_device *device) 94 { 95 struct vfio_pci_driver *driver = &device->driver; 96 int r; 97 98 VFIO_CHECK_DRIVER_OP(driver, memcpy_wait); 99 100 r = driver->ops->memcpy_wait(device); 101 driver->memcpy_in_progress = false; 102 103 return r; 104 } 105 106 int vfio_pci_driver_memcpy(struct vfio_pci_device *device, 107 iova_t src, iova_t dst, u64 size) 108 { 109 vfio_pci_driver_memcpy_start(device, src, dst, size, 1); 110 111 return vfio_pci_driver_memcpy_wait(device); 112 } 113