13a4d5c94SMichael S. Tsirkin #ifndef _VHOST_H 23a4d5c94SMichael S. Tsirkin #define _VHOST_H 33a4d5c94SMichael S. Tsirkin 43a4d5c94SMichael S. Tsirkin #include <linux/eventfd.h> 53a4d5c94SMichael S. Tsirkin #include <linux/vhost.h> 63a4d5c94SMichael S. Tsirkin #include <linux/mm.h> 73a4d5c94SMichael S. Tsirkin #include <linux/mutex.h> 83a4d5c94SMichael S. Tsirkin #include <linux/poll.h> 93a4d5c94SMichael S. Tsirkin #include <linux/file.h> 103a4d5c94SMichael S. Tsirkin #include <linux/uio.h> 113a4d5c94SMichael S. Tsirkin #include <linux/virtio_config.h> 123a4d5c94SMichael S. Tsirkin #include <linux/virtio_ring.h> 1360063497SArun Sharma #include <linux/atomic.h> 143a4d5c94SMichael S. Tsirkin 153a4d5c94SMichael S. Tsirkin struct vhost_device; 163a4d5c94SMichael S. Tsirkin 17c23f3445STejun Heo struct vhost_work; 18c23f3445STejun Heo typedef void (*vhost_work_fn_t)(struct vhost_work *work); 19c23f3445STejun Heo 20c23f3445STejun Heo struct vhost_work { 21c23f3445STejun Heo struct list_head node; 22c23f3445STejun Heo vhost_work_fn_t fn; 23c23f3445STejun Heo wait_queue_head_t done; 24c23f3445STejun Heo int flushing; 25c23f3445STejun Heo unsigned queue_seq; 26c23f3445STejun Heo unsigned done_seq; 27c23f3445STejun Heo }; 28c23f3445STejun Heo 293a4d5c94SMichael S. Tsirkin /* Poll a file (eventfd or socket) */ 303a4d5c94SMichael S. Tsirkin /* Note: there's nothing vhost specific about this structure. */ 313a4d5c94SMichael S. Tsirkin struct vhost_poll { 323a4d5c94SMichael S. Tsirkin poll_table table; 333a4d5c94SMichael S. Tsirkin wait_queue_head_t *wqh; 343a4d5c94SMichael S. Tsirkin wait_queue_t wait; 35c23f3445STejun Heo struct vhost_work work; 363a4d5c94SMichael S. Tsirkin unsigned long mask; 37c23f3445STejun Heo struct vhost_dev *dev; 383a4d5c94SMichael S. Tsirkin }; 393a4d5c94SMichael S. Tsirkin 40163049aeSStefan Hajnoczi void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 41163049aeSStefan Hajnoczi void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); 42163049aeSStefan Hajnoczi 43c23f3445STejun Heo void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 44c23f3445STejun Heo unsigned long mask, struct vhost_dev *dev); 452b8b328bSJason Wang int vhost_poll_start(struct vhost_poll *poll, struct file *file); 463a4d5c94SMichael S. Tsirkin void vhost_poll_stop(struct vhost_poll *poll); 473a4d5c94SMichael S. Tsirkin void vhost_poll_flush(struct vhost_poll *poll); 483a4d5c94SMichael S. Tsirkin void vhost_poll_queue(struct vhost_poll *poll); 496ac1afbfSAsias He void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work); 506ac1afbfSAsias He long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); 513a4d5c94SMichael S. Tsirkin 523a4d5c94SMichael S. Tsirkin struct vhost_log { 533a4d5c94SMichael S. Tsirkin u64 addr; 543a4d5c94SMichael S. Tsirkin u64 len; 553a4d5c94SMichael S. Tsirkin }; 563a4d5c94SMichael S. Tsirkin 57bab632d6SMichael S. Tsirkin struct vhost_virtqueue; 58bab632d6SMichael S. Tsirkin 593a4d5c94SMichael S. Tsirkin /* The virtqueue structure describes a queue attached to a device. */ 603a4d5c94SMichael S. Tsirkin struct vhost_virtqueue { 613a4d5c94SMichael S. Tsirkin struct vhost_dev *dev; 623a4d5c94SMichael S. Tsirkin 633a4d5c94SMichael S. Tsirkin /* The actual ring of buffers. */ 643a4d5c94SMichael S. Tsirkin struct mutex mutex; 653a4d5c94SMichael S. Tsirkin unsigned int num; 663a4d5c94SMichael S. Tsirkin struct vring_desc __user *desc; 673a4d5c94SMichael S. Tsirkin struct vring_avail __user *avail; 683a4d5c94SMichael S. Tsirkin struct vring_used __user *used; 693a4d5c94SMichael S. Tsirkin struct file *kick; 703a4d5c94SMichael S. Tsirkin struct file *call; 713a4d5c94SMichael S. Tsirkin struct file *error; 723a4d5c94SMichael S. Tsirkin struct eventfd_ctx *call_ctx; 733a4d5c94SMichael S. Tsirkin struct eventfd_ctx *error_ctx; 743a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 753a4d5c94SMichael S. Tsirkin 763a4d5c94SMichael S. Tsirkin struct vhost_poll poll; 773a4d5c94SMichael S. Tsirkin 783a4d5c94SMichael S. Tsirkin /* The routine to call when the Guest pings us, or timeout. */ 79c23f3445STejun Heo vhost_work_fn_t handle_kick; 803a4d5c94SMichael S. Tsirkin 813a4d5c94SMichael S. Tsirkin /* Last available index we saw. */ 823a4d5c94SMichael S. Tsirkin u16 last_avail_idx; 833a4d5c94SMichael S. Tsirkin 843a4d5c94SMichael S. Tsirkin /* Caches available index value from user. */ 853a4d5c94SMichael S. Tsirkin u16 avail_idx; 863a4d5c94SMichael S. Tsirkin 873a4d5c94SMichael S. Tsirkin /* Last index we used. */ 883a4d5c94SMichael S. Tsirkin u16 last_used_idx; 893a4d5c94SMichael S. Tsirkin 903a4d5c94SMichael S. Tsirkin /* Used flags */ 913a4d5c94SMichael S. Tsirkin u16 used_flags; 923a4d5c94SMichael S. Tsirkin 938ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 948ea8cf89SMichael S. Tsirkin u16 signalled_used; 958ea8cf89SMichael S. Tsirkin 968ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 978ea8cf89SMichael S. Tsirkin bool signalled_used_valid; 988ea8cf89SMichael S. Tsirkin 993a4d5c94SMichael S. Tsirkin /* Log writes to used structure. */ 1003a4d5c94SMichael S. Tsirkin bool log_used; 1013a4d5c94SMichael S. Tsirkin u64 log_addr; 1023a4d5c94SMichael S. Tsirkin 103e0e9b406SJason Wang struct iovec iov[UIO_MAXIOV]; 104e0e9b406SJason Wang struct iovec *indirect; 105e0e9b406SJason Wang struct vring_used_elem *heads; 10622fa90c7SAsias He /* Protected by virtqueue mutex. */ 10747283befSMichael S. Tsirkin struct vhost_memory *memory; 10822fa90c7SAsias He void *private_data; 109bd82752aSMichael S. Tsirkin u64 acked_features; 1103a4d5c94SMichael S. Tsirkin /* Log write descriptors */ 1113a4d5c94SMichael S. Tsirkin void __user *log_base; 112e0e9b406SJason Wang struct vhost_log *log; 1133a4d5c94SMichael S. Tsirkin }; 1143a4d5c94SMichael S. Tsirkin 1153a4d5c94SMichael S. Tsirkin struct vhost_dev { 11647283befSMichael S. Tsirkin struct vhost_memory *memory; 1173a4d5c94SMichael S. Tsirkin struct mm_struct *mm; 1183a4d5c94SMichael S. Tsirkin struct mutex mutex; 1193ab2e420SAsias He struct vhost_virtqueue **vqs; 1203a4d5c94SMichael S. Tsirkin int nvqs; 1213a4d5c94SMichael S. Tsirkin struct file *log_file; 1223a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 123c23f3445STejun Heo spinlock_t work_lock; 124c23f3445STejun Heo struct list_head work_list; 125c23f3445STejun Heo struct task_struct *worker; 1263a4d5c94SMichael S. Tsirkin }; 1273a4d5c94SMichael S. Tsirkin 12859566b6eSZhi Yong Wu void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs); 12954db63c2SAsias He long vhost_dev_set_owner(struct vhost_dev *dev); 13005c05351SMichael S. Tsirkin bool vhost_dev_has_owner(struct vhost_dev *dev); 1313a4d5c94SMichael S. Tsirkin long vhost_dev_check_owner(struct vhost_dev *); 132150b9e51SMichael S. Tsirkin struct vhost_memory *vhost_dev_reset_owner_prepare(void); 133150b9e51SMichael S. Tsirkin void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_memory *); 134ea5d4046SMichael S. Tsirkin void vhost_dev_cleanup(struct vhost_dev *, bool locked); 135b211616dSMichael S. Tsirkin void vhost_dev_stop(struct vhost_dev *); 136935cdee7SMichael S. Tsirkin long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); 137935cdee7SMichael S. Tsirkin long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); 1383a4d5c94SMichael S. Tsirkin int vhost_vq_access_ok(struct vhost_virtqueue *vq); 1393a4d5c94SMichael S. Tsirkin int vhost_log_access_ok(struct vhost_dev *); 1403a4d5c94SMichael S. Tsirkin 14147283befSMichael S. Tsirkin int vhost_get_vq_desc(struct vhost_virtqueue *, 1423a4d5c94SMichael S. Tsirkin struct iovec iov[], unsigned int iov_count, 1433a4d5c94SMichael S. Tsirkin unsigned int *out_num, unsigned int *in_num, 1443a4d5c94SMichael S. Tsirkin struct vhost_log *log, unsigned int *log_num); 1458dd014adSDavid Stevens void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); 1463a4d5c94SMichael S. Tsirkin 147f59281daSJason Wang int vhost_init_used(struct vhost_virtqueue *); 1483a4d5c94SMichael S. Tsirkin int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len); 1498dd014adSDavid Stevens int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads, 1508dd014adSDavid Stevens unsigned count); 1513a4d5c94SMichael S. Tsirkin void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *, 1528dd014adSDavid Stevens unsigned int id, int len); 1538dd014adSDavid Stevens void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, 1548dd014adSDavid Stevens struct vring_used_elem *heads, unsigned count); 1558dd014adSDavid Stevens void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); 1568ea8cf89SMichael S. Tsirkin void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1578ea8cf89SMichael S. Tsirkin bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1583a4d5c94SMichael S. Tsirkin 1593a4d5c94SMichael S. Tsirkin int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1603a4d5c94SMichael S. Tsirkin unsigned int log_num, u64 len); 1613a4d5c94SMichael S. Tsirkin 1623a4d5c94SMichael S. Tsirkin #define vq_err(vq, fmt, ...) do { \ 1633a4d5c94SMichael S. Tsirkin pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ 1643a4d5c94SMichael S. Tsirkin if ((vq)->error_ctx) \ 1653a4d5c94SMichael S. Tsirkin eventfd_signal((vq)->error_ctx, 1);\ 1663a4d5c94SMichael S. Tsirkin } while (0) 1673a4d5c94SMichael S. Tsirkin 1683a4d5c94SMichael S. Tsirkin enum { 1698ea8cf89SMichael S. Tsirkin VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | 1708ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 1718ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_EVENT_IDX) | 1720dd05a3bSStefan Hajnoczi (1ULL << VHOST_F_LOG_ALL), 1733a4d5c94SMichael S. Tsirkin }; 1743a4d5c94SMichael S. Tsirkin 175bd82752aSMichael S. Tsirkin static inline bool vhost_has_feature(struct vhost_virtqueue *vq, int bit) 1763a4d5c94SMichael S. Tsirkin { 177bd82752aSMichael S. Tsirkin return vq->acked_features & (1ULL << bit); 1783a4d5c94SMichael S. Tsirkin } 179*e05fd12bSMichael S. Tsirkin 180*e05fd12bSMichael S. Tsirkin /* Memory accessors */ 181*e05fd12bSMichael S. Tsirkin static inline u16 vhost16_to_cpu(struct vhost_virtqueue *vq, __virtio16 val) 182*e05fd12bSMichael S. Tsirkin { 183*e05fd12bSMichael S. Tsirkin return __virtio16_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 184*e05fd12bSMichael S. Tsirkin } 185*e05fd12bSMichael S. Tsirkin 186*e05fd12bSMichael S. Tsirkin static inline __virtio16 cpu_to_vhost16(struct vhost_virtqueue *vq, u16 val) 187*e05fd12bSMichael S. Tsirkin { 188*e05fd12bSMichael S. Tsirkin return __cpu_to_virtio16(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 189*e05fd12bSMichael S. Tsirkin } 190*e05fd12bSMichael S. Tsirkin 191*e05fd12bSMichael S. Tsirkin static inline u32 vhost32_to_cpu(struct vhost_virtqueue *vq, __virtio32 val) 192*e05fd12bSMichael S. Tsirkin { 193*e05fd12bSMichael S. Tsirkin return __virtio32_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 194*e05fd12bSMichael S. Tsirkin } 195*e05fd12bSMichael S. Tsirkin 196*e05fd12bSMichael S. Tsirkin static inline __virtio32 cpu_to_vhost32(struct vhost_virtqueue *vq, u32 val) 197*e05fd12bSMichael S. Tsirkin { 198*e05fd12bSMichael S. Tsirkin return __cpu_to_virtio32(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 199*e05fd12bSMichael S. Tsirkin } 200*e05fd12bSMichael S. Tsirkin 201*e05fd12bSMichael S. Tsirkin static inline u64 vhost64_to_cpu(struct vhost_virtqueue *vq, __virtio64 val) 202*e05fd12bSMichael S. Tsirkin { 203*e05fd12bSMichael S. Tsirkin return __virtio64_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 204*e05fd12bSMichael S. Tsirkin } 205*e05fd12bSMichael S. Tsirkin 206*e05fd12bSMichael S. Tsirkin static inline __virtio64 cpu_to_vhost64(struct vhost_virtqueue *vq, u64 val) 207*e05fd12bSMichael S. Tsirkin { 208*e05fd12bSMichael S. Tsirkin return __cpu_to_virtio64(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val); 209*e05fd12bSMichael S. Tsirkin } 2103a4d5c94SMichael S. Tsirkin #endif 211