1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * VFIO API definition 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 */ 8 #ifndef VFIO_H 9 #define VFIO_H 10 11 12 #include <linux/iommu.h> 13 #include <linux/mm.h> 14 #include <linux/workqueue.h> 15 #include <linux/poll.h> 16 #include <linux/cdev.h> 17 #include <uapi/linux/vfio.h> 18 #include <linux/iova_bitmap.h> 19 #include <linux/uaccess.h> 20 21 struct kvm; 22 struct iommufd_ctx; 23 struct iommufd_device; 24 struct iommufd_access; 25 struct vfio_info_cap; 26 27 /* 28 * VFIO devices can be placed in a set, this allows all devices to share this 29 * structure and the VFIO core will provide a lock that is held around 30 * open_device()/close_device() for all devices in the set. 31 */ 32 struct vfio_device_set { 33 void *set_id; 34 struct mutex lock; 35 struct list_head device_list; 36 unsigned int device_count; 37 }; 38 39 struct vfio_device { 40 struct device *dev; 41 const struct vfio_device_ops *ops; 42 /* 43 * mig_ops/log_ops is a static property of the vfio_device which must 44 * be set prior to registering the vfio_device. 45 */ 46 const struct vfio_migration_ops *mig_ops; 47 const struct vfio_log_ops *log_ops; 48 #if IS_ENABLED(CONFIG_VFIO_GROUP) 49 struct vfio_group *group; 50 struct list_head group_next; 51 struct list_head iommu_entry; 52 #endif 53 struct vfio_device_set *dev_set; 54 struct list_head dev_set_list; 55 unsigned int migration_flags; 56 u8 precopy_info_v2; 57 struct kvm *kvm; 58 59 /* Members below here are private, not for driver use */ 60 unsigned int index; 61 struct device device; /* device.kref covers object life circle */ 62 #if IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) 63 struct cdev cdev; 64 #endif 65 refcount_t refcount; /* user count on registered device*/ 66 unsigned int open_count; 67 struct completion comp; 68 struct iommufd_access *iommufd_access; 69 void (*put_kvm)(struct kvm *kvm); 70 struct inode *inode; 71 #if IS_ENABLED(CONFIG_IOMMUFD) 72 struct iommufd_device *iommufd_device; 73 struct ida pasids; 74 u8 iommufd_attached:1; 75 #endif 76 u8 cdev_opened:1; 77 /* 78 * debug_root is a static property of the vfio_device 79 * which must be set prior to registering the vfio_device. 80 */ 81 struct dentry *debug_root; 82 }; 83 84 /** 85 * struct vfio_device_ops - VFIO bus driver device callbacks 86 * 87 * @name: Name of the device driver. 88 * @init: initialize private fields in device structure 89 * @release: Reclaim private fields in device structure 90 * @bind_iommufd: Called when binding the device to an iommufd 91 * @unbind_iommufd: Opposite of bind_iommufd 92 * @attach_ioas: Called when attaching device to an IOAS/HWPT managed by the 93 * bound iommufd. Undo in unbind_iommufd if @detach_ioas is not 94 * called. 95 * @detach_ioas: Opposite of attach_ioas 96 * @pasid_attach_ioas: The pasid variation of attach_ioas 97 * @pasid_detach_ioas: Opposite of pasid_attach_ioas 98 * @open_device: Called when the first file descriptor is opened for this device 99 * @close_device: Opposite of open_device 100 * @read: Perform read(2) on device file descriptor 101 * @write: Perform write(2) on device file descriptor 102 * @ioctl: Perform ioctl(2) on device file descriptor, supporting VFIO_DEVICE_* 103 * operations documented below 104 * @mmap: Perform mmap(2) on a region of the device file descriptor 105 * @request: Request for the bus driver to release the device 106 * @match: Optional device name match callback (return: 0 for no-match, >0 for 107 * match, -errno for abort (ex. match with insufficient or incorrect 108 * additional args) 109 * @match_token_uuid: Optional device token match/validation. Return 0 110 * if the uuid is valid for the device, -errno otherwise. uuid is NULL 111 * if none was provided. 112 * @dma_unmap: Called when userspace unmaps IOVA from the container 113 * this device is attached to. 114 * @device_feature: Optional, fill in the VFIO_DEVICE_FEATURE ioctl 115 */ 116 struct vfio_device_ops { 117 char *name; 118 int (*init)(struct vfio_device *vdev); 119 void (*release)(struct vfio_device *vdev); 120 int (*bind_iommufd)(struct vfio_device *vdev, 121 struct iommufd_ctx *ictx, u32 *out_device_id); 122 void (*unbind_iommufd)(struct vfio_device *vdev); 123 int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id); 124 void (*detach_ioas)(struct vfio_device *vdev); 125 int (*pasid_attach_ioas)(struct vfio_device *vdev, u32 pasid, 126 u32 *pt_id); 127 void (*pasid_detach_ioas)(struct vfio_device *vdev, u32 pasid); 128 int (*open_device)(struct vfio_device *vdev); 129 void (*close_device)(struct vfio_device *vdev); 130 ssize_t (*read)(struct vfio_device *vdev, char __user *buf, 131 size_t count, loff_t *ppos); 132 ssize_t (*write)(struct vfio_device *vdev, const char __user *buf, 133 size_t count, loff_t *size); 134 long (*ioctl)(struct vfio_device *vdev, unsigned int cmd, 135 unsigned long arg); 136 int (*get_region_info_caps)(struct vfio_device *vdev, 137 struct vfio_region_info *info, 138 struct vfio_info_cap *caps); 139 int (*mmap)(struct vfio_device *vdev, struct vm_area_struct *vma); 140 void (*request)(struct vfio_device *vdev, unsigned int count); 141 int (*match)(struct vfio_device *vdev, char *buf); 142 int (*match_token_uuid)(struct vfio_device *vdev, const uuid_t *uuid); 143 void (*dma_unmap)(struct vfio_device *vdev, u64 iova, u64 length); 144 int (*device_feature)(struct vfio_device *device, u32 flags, 145 void __user *arg, size_t argsz); 146 }; 147 148 #if IS_ENABLED(CONFIG_IOMMUFD) 149 struct iommufd_ctx *vfio_iommufd_device_ictx(struct vfio_device *vdev); 150 int vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx); 151 int vfio_iommufd_physical_bind(struct vfio_device *vdev, 152 struct iommufd_ctx *ictx, u32 *out_device_id); 153 void vfio_iommufd_physical_unbind(struct vfio_device *vdev); 154 int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id); 155 void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev); 156 int vfio_iommufd_physical_pasid_attach_ioas(struct vfio_device *vdev, 157 u32 pasid, u32 *pt_id); 158 void vfio_iommufd_physical_pasid_detach_ioas(struct vfio_device *vdev, 159 u32 pasid); 160 int vfio_iommufd_emulated_bind(struct vfio_device *vdev, 161 struct iommufd_ctx *ictx, u32 *out_device_id); 162 void vfio_iommufd_emulated_unbind(struct vfio_device *vdev); 163 int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id); 164 void vfio_iommufd_emulated_detach_ioas(struct vfio_device *vdev); 165 #else 166 static inline struct iommufd_ctx * 167 vfio_iommufd_device_ictx(struct vfio_device *vdev) 168 { 169 return NULL; 170 } 171 172 static inline int 173 vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx) 174 { 175 return VFIO_PCI_DEVID_NOT_OWNED; 176 } 177 178 #define vfio_iommufd_physical_bind \ 179 ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \ 180 u32 *out_device_id)) NULL) 181 #define vfio_iommufd_physical_unbind \ 182 ((void (*)(struct vfio_device *vdev)) NULL) 183 #define vfio_iommufd_physical_attach_ioas \ 184 ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL) 185 #define vfio_iommufd_physical_detach_ioas \ 186 ((void (*)(struct vfio_device *vdev)) NULL) 187 #define vfio_iommufd_physical_pasid_attach_ioas \ 188 ((int (*)(struct vfio_device *vdev, u32 pasid, u32 *pt_id)) NULL) 189 #define vfio_iommufd_physical_pasid_detach_ioas \ 190 ((void (*)(struct vfio_device *vdev, u32 pasid)) NULL) 191 #define vfio_iommufd_emulated_bind \ 192 ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \ 193 u32 *out_device_id)) NULL) 194 #define vfio_iommufd_emulated_unbind \ 195 ((void (*)(struct vfio_device *vdev)) NULL) 196 #define vfio_iommufd_emulated_attach_ioas \ 197 ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL) 198 #define vfio_iommufd_emulated_detach_ioas \ 199 ((void (*)(struct vfio_device *vdev)) NULL) 200 #endif 201 202 static inline bool vfio_device_cdev_opened(struct vfio_device *device) 203 { 204 return device->cdev_opened; 205 } 206 207 /** 208 * struct vfio_migration_ops - VFIO bus device driver migration callbacks 209 * 210 * @migration_set_state: Optional callback to change the migration state for 211 * devices that support migration. It's mandatory for 212 * VFIO_DEVICE_FEATURE_MIGRATION migration support. 213 * The returned FD is used for data transfer according to the FSM 214 * definition. The driver is responsible to ensure that FD reaches end 215 * of stream or error whenever the migration FSM leaves a data transfer 216 * state or before close_device() returns. 217 * @migration_get_state: Optional callback to get the migration state for 218 * devices that support migration. It's mandatory for 219 * VFIO_DEVICE_FEATURE_MIGRATION migration support. 220 * @migration_get_data_size: Optional callback to get the estimated data 221 * length that will be required to complete stop copy. It's mandatory for 222 * VFIO_DEVICE_FEATURE_MIGRATION migration support. 223 */ 224 struct vfio_migration_ops { 225 struct file *(*migration_set_state)( 226 struct vfio_device *device, 227 enum vfio_device_mig_state new_state); 228 int (*migration_get_state)(struct vfio_device *device, 229 enum vfio_device_mig_state *curr_state); 230 int (*migration_get_data_size)(struct vfio_device *device, 231 unsigned long *stop_copy_length); 232 }; 233 234 /** 235 * struct vfio_log_ops - VFIO bus device driver logging callbacks 236 * 237 * @log_start: Optional callback to ask the device start DMA logging. 238 * @log_stop: Optional callback to ask the device stop DMA logging. 239 * @log_read_and_clear: Optional callback to ask the device read 240 * and clear the dirty DMAs in some given range. 241 * 242 * The vfio core implementation of the DEVICE_FEATURE_DMA_LOGGING_ set 243 * of features does not track logging state relative to the device, 244 * therefore the device implementation of vfio_log_ops must handle 245 * arbitrary user requests. This includes rejecting subsequent calls 246 * to log_start without an intervening log_stop, as well as graceful 247 * handling of log_stop and log_read_and_clear from invalid states. 248 */ 249 struct vfio_log_ops { 250 int (*log_start)(struct vfio_device *device, 251 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); 252 int (*log_stop)(struct vfio_device *device); 253 int (*log_read_and_clear)(struct vfio_device *device, 254 unsigned long iova, unsigned long length, 255 struct iova_bitmap *dirty); 256 }; 257 258 /** 259 * vfio_check_feature - Validate user input for the VFIO_DEVICE_FEATURE ioctl 260 * @flags: Arg from the device_feature op 261 * @argsz: Arg from the device_feature op 262 * @supported_ops: Combination of VFIO_DEVICE_FEATURE_GET and SET the driver 263 * supports 264 * @minsz: Minimum data size the driver accepts 265 * 266 * For use in a driver's device_feature op. Checks that the inputs to the 267 * VFIO_DEVICE_FEATURE ioctl are correct for the driver's feature. Returns 1 if 268 * the driver should execute the get or set, otherwise the relevant 269 * value should be returned. 270 */ 271 static inline int vfio_check_feature(u32 flags, size_t argsz, u32 supported_ops, 272 size_t minsz) 273 { 274 if ((flags & (VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_SET)) & 275 ~supported_ops) 276 return -EINVAL; 277 if (flags & VFIO_DEVICE_FEATURE_PROBE) 278 return 0; 279 /* Without PROBE one of GET or SET must be requested */ 280 if (!(flags & (VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_SET))) 281 return -EINVAL; 282 if (argsz < minsz) 283 return -EINVAL; 284 return 1; 285 } 286 287 /** 288 * vfio_check_precopy_ioctl - Validate user input for the VFIO_MIG_GET_PRECOPY_INFO ioctl 289 * @vdev: The vfio device 290 * @cmd: Cmd from the ioctl 291 * @arg: Arg from the ioctl 292 * @info: Driver pointer to hold the userspace input to the ioctl 293 * 294 * For use in a driver's get_precopy_info. Checks that the inputs to the 295 * VFIO_MIG_GET_PRECOPY_INFO ioctl are correct. 296 297 * Returns 0 on success, otherwise errno. 298 */ 299 300 static inline int 301 vfio_check_precopy_ioctl(struct vfio_device *vdev, unsigned int cmd, 302 unsigned long arg, struct vfio_precopy_info *info) 303 { 304 unsigned long minsz; 305 306 if (cmd != VFIO_MIG_GET_PRECOPY_INFO) 307 return -ENOTTY; 308 309 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); 310 311 if (copy_from_user(info, (void __user *)arg, minsz)) 312 return -EFAULT; 313 314 if (info->argsz < minsz) 315 return -EINVAL; 316 317 /* keep v1 behaviour as is for compatibility reasons */ 318 if (vdev->precopy_info_v2) 319 /* flags are output, set its initial value to 0 */ 320 info->flags = 0; 321 322 return 0; 323 } 324 325 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, 326 const struct vfio_device_ops *ops); 327 #define vfio_alloc_device(dev_struct, member, dev, ops) \ 328 container_of(_vfio_alloc_device(sizeof(struct dev_struct) + \ 329 BUILD_BUG_ON_ZERO(offsetof( \ 330 struct dev_struct, member)), \ 331 dev, ops), \ 332 struct dev_struct, member) 333 334 static inline void vfio_put_device(struct vfio_device *device) 335 { 336 put_device(&device->device); 337 } 338 339 int vfio_register_group_dev(struct vfio_device *device); 340 int vfio_register_emulated_iommu_dev(struct vfio_device *device); 341 void vfio_unregister_group_dev(struct vfio_device *device); 342 bool vfio_device_try_get_registration(struct vfio_device *device); 343 void vfio_device_put_registration(struct vfio_device *device); 344 345 int vfio_assign_device_set(struct vfio_device *device, void *set_id); 346 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set); 347 struct vfio_device * 348 vfio_find_device_in_devset(struct vfio_device_set *dev_set, 349 struct device *dev); 350 351 int vfio_mig_get_next_state(struct vfio_device *device, 352 enum vfio_device_mig_state cur_fsm, 353 enum vfio_device_mig_state new_fsm, 354 enum vfio_device_mig_state *next_fsm); 355 356 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, 357 u32 req_nodes); 358 359 /* 360 * External user API 361 */ 362 struct iommu_group *vfio_file_iommu_group(struct file *file); 363 364 #if IS_ENABLED(CONFIG_VFIO_GROUP) 365 bool vfio_file_is_group(struct file *file); 366 bool vfio_file_has_dev(struct file *file, struct vfio_device *device); 367 #else 368 static inline bool vfio_file_is_group(struct file *file) 369 { 370 return false; 371 } 372 373 static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *device) 374 { 375 return false; 376 } 377 #endif 378 bool vfio_file_is_valid(struct file *file); 379 bool vfio_file_enforced_coherent(struct file *file); 380 void vfio_file_set_kvm(struct file *file, struct kvm *kvm); 381 382 #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) 383 384 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, 385 int npage, int prot, struct page **pages); 386 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); 387 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, 388 void *data, size_t len, bool write); 389 390 /* 391 * Sub-module helpers 392 */ 393 struct vfio_info_cap { 394 struct vfio_info_cap_header *buf; 395 size_t size; 396 }; 397 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, 398 size_t size, u16 id, 399 u16 version); 400 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset); 401 402 int vfio_info_add_capability(struct vfio_info_cap *caps, 403 struct vfio_info_cap_header *cap, size_t size); 404 405 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, 406 int num_irqs, int max_irq_type, 407 size_t *data_size); 408 409 /* 410 * IRQfd - generic 411 */ 412 struct virqfd { 413 void *opaque; 414 struct eventfd_ctx *eventfd; 415 int (*handler)(void *, void *); 416 void (*thread)(void *, void *); 417 void *data; 418 struct work_struct inject; 419 wait_queue_entry_t wait; 420 poll_table pt; 421 struct work_struct shutdown; 422 struct work_struct flush_inject; 423 struct virqfd **pvirqfd; 424 }; 425 426 int vfio_virqfd_enable(void *opaque, int (*handler)(void *, void *), 427 void (*thread)(void *, void *), void *data, 428 struct virqfd **pvirqfd, int fd); 429 void vfio_virqfd_disable(struct virqfd **pvirqfd); 430 void vfio_virqfd_flush_thread(struct virqfd **pvirqfd); 431 432 #endif /* VFIO_H */ 433