1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11 #include <linux/mutex.h> 12 #include <linux/pci.h> 13 #include <linux/vfio.h> 14 #include <linux/irqbypass.h> 15 #include <linux/rcupdate.h> 16 #include <linux/types.h> 17 #include <linux/uuid.h> 18 #include <linux/notifier.h> 19 20 #ifndef VFIO_PCI_CORE_H 21 #define VFIO_PCI_CORE_H 22 23 #define VFIO_PCI_OFFSET_SHIFT 40 24 #define VFIO_PCI_OFFSET_TO_INDEX(off) ((u64)(off) >> VFIO_PCI_OFFSET_SHIFT) 25 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 26 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 27 28 struct vfio_pci_core_device; 29 struct vfio_pci_region; 30 struct p2pdma_provider; 31 struct dma_buf_attachment; 32 33 struct vfio_pci_eventfd { 34 struct eventfd_ctx *ctx; 35 struct rcu_head rcu; 36 }; 37 38 struct vfio_pci_regops { 39 ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, 40 size_t count, loff_t *ppos, bool iswrite); 41 void (*release)(struct vfio_pci_core_device *vdev, 42 struct vfio_pci_region *region); 43 int (*mmap)(struct vfio_pci_core_device *vdev, 44 struct vfio_pci_region *region, 45 struct vm_area_struct *vma); 46 int (*add_capability)(struct vfio_pci_core_device *vdev, 47 struct vfio_pci_region *region, 48 struct vfio_info_cap *caps); 49 }; 50 51 struct vfio_pci_region { 52 u32 type; 53 u32 subtype; 54 const struct vfio_pci_regops *ops; 55 void *data; 56 size_t size; 57 u32 flags; 58 }; 59 60 struct vfio_pci_device_ops { 61 int (*get_dmabuf_phys)(struct vfio_pci_core_device *vdev, 62 struct p2pdma_provider **provider, 63 unsigned int region_index, 64 struct phys_vec *phys_vec, 65 struct vfio_region_dma_range *dma_ranges, 66 size_t nr_ranges); 67 }; 68 69 #if IS_ENABLED(CONFIG_VFIO_PCI_DMABUF) 70 int vfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec, 71 struct vfio_region_dma_range *dma_ranges, 72 size_t nr_ranges, phys_addr_t start, 73 phys_addr_t len); 74 int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev, 75 struct p2pdma_provider **provider, 76 unsigned int region_index, 77 struct phys_vec *phys_vec, 78 struct vfio_region_dma_range *dma_ranges, 79 size_t nr_ranges); 80 #else 81 static inline int 82 vfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec, 83 struct vfio_region_dma_range *dma_ranges, 84 size_t nr_ranges, phys_addr_t start, 85 phys_addr_t len) 86 { 87 return -EINVAL; 88 } 89 static inline int vfio_pci_core_get_dmabuf_phys( 90 struct vfio_pci_core_device *vdev, struct p2pdma_provider **provider, 91 unsigned int region_index, struct phys_vec *phys_vec, 92 struct vfio_region_dma_range *dma_ranges, size_t nr_ranges) 93 { 94 return -EOPNOTSUPP; 95 } 96 #endif 97 98 struct vfio_pci_core_device { 99 struct vfio_device vdev; 100 struct pci_dev *pdev; 101 const struct vfio_pci_device_ops *pci_ops; 102 void __iomem *barmap[PCI_STD_NUM_BARS]; 103 bool bar_mmap_supported[PCI_STD_NUM_BARS]; 104 /* Flags modified at runtime - dedicated storage unit */ 105 bool virq_disabled; 106 bool bardirty; 107 u8 *pci_config_map; 108 u8 *vconfig; 109 struct perm_bits *msi_perm; 110 spinlock_t irqlock; 111 struct mutex igate; 112 struct xarray ctx; 113 int irq_type; 114 int num_regions; 115 struct vfio_pci_region *region; 116 u8 msi_qmax; 117 u8 msix_bar; 118 u16 msix_size; 119 u32 msix_offset; 120 u32 rbar[7]; 121 /* Flags only modified on setup/release - bitfield ok */ 122 bool has_dyn_msix:1; 123 bool pci_2_3:1; 124 bool reset_works:1; 125 bool extended_caps:1; 126 bool has_vga:1; 127 bool nointx:1; 128 bool needs_pm_restore:1; 129 bool disable_idle_d3:1; 130 bool nointxmask:1; 131 bool disable_vga:1; 132 /* Flags modified at runtime - dedicated storage unit */ 133 bool needs_reset; 134 bool pm_intx_masked; 135 bool pm_runtime_engaged; 136 bool sriov_active; 137 struct pci_saved_state *pci_saved_state; 138 struct pci_saved_state *pm_save; 139 int ioeventfds_nr; 140 struct vfio_pci_eventfd __rcu *err_trigger; 141 struct vfio_pci_eventfd __rcu *req_trigger; 142 struct eventfd_ctx *pm_wake_eventfd_ctx; 143 struct list_head dummy_resources_list; 144 struct mutex ioeventfds_lock; 145 struct list_head ioeventfds_list; 146 struct vfio_pci_vf_token *vf_token; 147 struct list_head sriov_pfs_item; 148 struct vfio_pci_core_device *sriov_pf_core_dev; 149 struct notifier_block nb; 150 struct rw_semaphore memory_lock; 151 struct list_head dmabufs; 152 }; 153 154 enum vfio_pci_io_width { 155 VFIO_PCI_IO_WIDTH_1 = 1, 156 VFIO_PCI_IO_WIDTH_2 = 2, 157 VFIO_PCI_IO_WIDTH_4 = 4, 158 VFIO_PCI_IO_WIDTH_8 = 8, 159 }; 160 161 /* Will be exported for vfio pci drivers usage */ 162 int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, 163 unsigned int type, unsigned int subtype, 164 const struct vfio_pci_regops *ops, 165 size_t size, u32 flags, void *data); 166 void vfio_pci_core_close_device(struct vfio_device *core_vdev); 167 int vfio_pci_core_init_dev(struct vfio_device *core_vdev); 168 void vfio_pci_core_release_dev(struct vfio_device *core_vdev); 169 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); 170 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); 171 extern const struct pci_error_handlers vfio_pci_core_err_handlers; 172 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, 173 int nr_virtfn); 174 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 175 unsigned long arg); 176 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, 177 void __user *arg, size_t argsz); 178 int vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev, 179 struct vfio_region_info *info, 180 struct vfio_info_cap *caps); 181 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, 182 size_t count, loff_t *ppos); 183 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, 184 size_t count, loff_t *ppos); 185 vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, 186 struct vm_fault *vmf, unsigned long pfn, 187 unsigned int order); 188 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma); 189 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count); 190 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf); 191 int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev, 192 const uuid_t *uuid); 193 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); 194 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); 195 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); 196 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, 197 pci_channel_state_t state); 198 ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, 199 void __iomem *io, char __user *buf, 200 loff_t off, size_t count, size_t x_start, 201 size_t x_end, bool iswrite, 202 enum vfio_pci_io_width max_width); 203 bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); 204 bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, 205 loff_t reg_start, size_t reg_cnt, 206 loff_t *buf_offset, 207 size_t *intersect_count, 208 size_t *register_offset); 209 #define VFIO_IOWRITE_DECLARATION(size) \ 210 int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ 211 bool test_mem, u##size val, void __iomem *io); 212 213 VFIO_IOWRITE_DECLARATION(8) 214 VFIO_IOWRITE_DECLARATION(16) 215 VFIO_IOWRITE_DECLARATION(32) 216 #ifdef iowrite64 217 VFIO_IOWRITE_DECLARATION(64) 218 #endif 219 220 #define VFIO_IOREAD_DECLARATION(size) \ 221 int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \ 222 bool test_mem, u##size *val, void __iomem *io); 223 224 VFIO_IOREAD_DECLARATION(8) 225 VFIO_IOREAD_DECLARATION(16) 226 VFIO_IOREAD_DECLARATION(32) 227 #ifdef ioread64 228 VFIO_IOREAD_DECLARATION(64) 229 #endif 230 231 static inline bool is_aligned_for_order(struct vm_area_struct *vma, 232 unsigned long addr, 233 unsigned long pfn, 234 unsigned int order) 235 { 236 return !(order && (addr < vma->vm_start || 237 addr + (PAGE_SIZE << order) > vma->vm_end || 238 !IS_ALIGNED(pfn, 1 << order))); 239 } 240 241 /* 242 * Returns a BAR's iomap base or an ERR_PTR() if, for example, the 243 * BAR isn't valid, its resource wasn't acquired, or its iomap 244 * failed. This shall only be used after vfio_pci_core_enable() 245 * has set up the BAR maps and before vfio_pci_core_disable() 246 * tears them down. 247 */ 248 static inline void __iomem __must_check * 249 vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar) 250 { 251 if (WARN_ON_ONCE(bar >= PCI_STD_NUM_BARS)) 252 return IOMEM_ERR_PTR(-EINVAL); 253 254 if (WARN_ON_ONCE(!vdev->barmap[bar])) 255 return IOMEM_ERR_PTR(-ENODEV); 256 257 return vdev->barmap[bar]; 258 } 259 260 int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, 261 struct phys_vec *phys); 262 263 #endif /* VFIO_PCI_CORE_H */ 264