1 // SPDX-License-Identifier: GPL-2.0-only 2 #include "kselftest.h" 3 #include <libvfio.h> 4 5 #ifdef __x86_64__ 6 extern struct vfio_pci_driver_ops dsa_ops; 7 extern struct vfio_pci_driver_ops ioat_ops; 8 #endif 9 extern struct vfio_pci_driver_ops nv_falcon_ops; 10 extern struct vfio_pci_driver_ops igb_ops; 11 12 static struct vfio_pci_driver_ops *driver_ops[] = { 13 #ifdef __x86_64__ 14 &dsa_ops, 15 &ioat_ops, 16 #endif 17 &nv_falcon_ops, 18 &igb_ops, 19 }; 20 21 void vfio_pci_driver_probe(struct vfio_pci_device *device) 22 { 23 struct vfio_pci_driver_ops *ops; 24 int i; 25 26 VFIO_ASSERT_NULL(device->driver.ops); 27 28 for (i = 0; i < ARRAY_SIZE(driver_ops); i++) { 29 ops = driver_ops[i]; 30 31 if (ops->probe(device)) 32 continue; 33 34 device->driver.ops = ops; 35 } 36 } 37 38 static void vfio_check_driver_op(struct vfio_pci_driver *driver, void *op, 39 const char *op_name) 40 { 41 VFIO_ASSERT_NOT_NULL(driver->ops); 42 VFIO_ASSERT_NOT_NULL(op, "Driver has no %s()\n", op_name); 43 VFIO_ASSERT_EQ(driver->initialized, op != driver->ops->init); 44 VFIO_ASSERT_EQ(driver->memcpy_in_progress, op == driver->ops->memcpy_wait); 45 } 46 47 #define VFIO_CHECK_DRIVER_OP(_driver, _op) do { \ 48 struct vfio_pci_driver *__driver = (_driver); \ 49 vfio_check_driver_op(__driver, __driver->ops->_op, #_op); \ 50 } while (0) 51 52 void vfio_pci_driver_init(struct vfio_pci_device *device) 53 { 54 struct vfio_pci_driver *driver = &device->driver; 55 56 VFIO_ASSERT_NOT_NULL(driver->region.vaddr); 57 VFIO_CHECK_DRIVER_OP(driver, init); 58 59 driver->ops->init(device); 60 61 driver->initialized = true; 62 } 63 64 void vfio_pci_driver_remove(struct vfio_pci_device *device) 65 { 66 struct vfio_pci_driver *driver = &device->driver; 67 68 VFIO_CHECK_DRIVER_OP(driver, remove); 69 70 driver->ops->remove(device); 71 driver->initialized = false; 72 } 73 74 void vfio_pci_driver_send_msi(struct vfio_pci_device *device) 75 { 76 struct vfio_pci_driver *driver = &device->driver; 77 78 VFIO_CHECK_DRIVER_OP(driver, send_msi); 79 80 driver->ops->send_msi(device); 81 } 82 83 void vfio_pci_driver_memcpy_start(struct vfio_pci_device *device, 84 iova_t src, iova_t dst, u64 size, 85 u64 count) 86 { 87 struct vfio_pci_driver *driver = &device->driver; 88 89 VFIO_ASSERT_LE(size, driver->max_memcpy_size); 90 VFIO_ASSERT_LE(count, driver->max_memcpy_count); 91 VFIO_CHECK_DRIVER_OP(driver, memcpy_start); 92 93 driver->ops->memcpy_start(device, src, dst, size, count); 94 driver->memcpy_in_progress = true; 95 } 96 97 int vfio_pci_driver_memcpy_wait(struct vfio_pci_device *device) 98 { 99 struct vfio_pci_driver *driver = &device->driver; 100 int r; 101 102 VFIO_CHECK_DRIVER_OP(driver, memcpy_wait); 103 104 r = driver->ops->memcpy_wait(device); 105 driver->memcpy_in_progress = false; 106 107 return r; 108 } 109 110 int vfio_pci_driver_memcpy(struct vfio_pci_device *device, 111 iova_t src, iova_t dst, u64 size) 112 { 113 struct vfio_pci_driver *driver = &device->driver; 114 u64 offset = 0; 115 116 while (offset < size) { 117 u64 chunk = min(size - offset, driver->max_memcpy_size); 118 int ret; 119 120 vfio_pci_driver_memcpy_start(device, src + offset, 121 dst + offset, chunk, 1); 122 ret = vfio_pci_driver_memcpy_wait(device); 123 if (ret) 124 return ret; 125 126 offset += chunk; 127 } 128 129 return 0; 130 } 131