13a4d5c94SMichael S. Tsirkin #ifndef _VHOST_H 23a4d5c94SMichael S. Tsirkin #define _VHOST_H 33a4d5c94SMichael S. Tsirkin 43a4d5c94SMichael S. Tsirkin #include <linux/eventfd.h> 53a4d5c94SMichael S. Tsirkin #include <linux/vhost.h> 63a4d5c94SMichael S. Tsirkin #include <linux/mm.h> 73a4d5c94SMichael S. Tsirkin #include <linux/mutex.h> 83a4d5c94SMichael S. Tsirkin #include <linux/poll.h> 93a4d5c94SMichael S. Tsirkin #include <linux/file.h> 103a4d5c94SMichael S. Tsirkin #include <linux/uio.h> 113a4d5c94SMichael S. Tsirkin #include <linux/virtio_config.h> 123a4d5c94SMichael S. Tsirkin #include <linux/virtio_ring.h> 1360063497SArun Sharma #include <linux/atomic.h> 143a4d5c94SMichael S. Tsirkin 153a4d5c94SMichael S. Tsirkin struct vhost_device; 163a4d5c94SMichael S. Tsirkin 17c23f3445STejun Heo struct vhost_work; 18c23f3445STejun Heo typedef void (*vhost_work_fn_t)(struct vhost_work *work); 19c23f3445STejun Heo 20c23f3445STejun Heo struct vhost_work { 21c23f3445STejun Heo struct list_head node; 22c23f3445STejun Heo vhost_work_fn_t fn; 23c23f3445STejun Heo wait_queue_head_t done; 24c23f3445STejun Heo int flushing; 25c23f3445STejun Heo unsigned queue_seq; 26c23f3445STejun Heo unsigned done_seq; 27c23f3445STejun Heo }; 28c23f3445STejun Heo 293a4d5c94SMichael S. Tsirkin /* Poll a file (eventfd or socket) */ 303a4d5c94SMichael S. Tsirkin /* Note: there's nothing vhost specific about this structure. */ 313a4d5c94SMichael S. Tsirkin struct vhost_poll { 323a4d5c94SMichael S. Tsirkin poll_table table; 333a4d5c94SMichael S. Tsirkin wait_queue_head_t *wqh; 343a4d5c94SMichael S. Tsirkin wait_queue_t wait; 35c23f3445STejun Heo struct vhost_work work; 363a4d5c94SMichael S. Tsirkin unsigned long mask; 37c23f3445STejun Heo struct vhost_dev *dev; 383a4d5c94SMichael S. Tsirkin }; 393a4d5c94SMichael S. Tsirkin 40163049aeSStefan Hajnoczi void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 41163049aeSStefan Hajnoczi void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); 42163049aeSStefan Hajnoczi 43c23f3445STejun Heo void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 44c23f3445STejun Heo unsigned long mask, struct vhost_dev *dev); 452b8b328bSJason Wang int vhost_poll_start(struct vhost_poll *poll, struct file *file); 463a4d5c94SMichael S. Tsirkin void vhost_poll_stop(struct vhost_poll *poll); 473a4d5c94SMichael S. Tsirkin void vhost_poll_flush(struct vhost_poll *poll); 483a4d5c94SMichael S. Tsirkin void vhost_poll_queue(struct vhost_poll *poll); 493a4d5c94SMichael S. Tsirkin 503a4d5c94SMichael S. Tsirkin struct vhost_log { 513a4d5c94SMichael S. Tsirkin u64 addr; 523a4d5c94SMichael S. Tsirkin u64 len; 533a4d5c94SMichael S. Tsirkin }; 543a4d5c94SMichael S. Tsirkin 55bab632d6SMichael S. Tsirkin struct vhost_virtqueue; 56bab632d6SMichael S. Tsirkin 573a4d5c94SMichael S. Tsirkin /* The virtqueue structure describes a queue attached to a device. */ 583a4d5c94SMichael S. Tsirkin struct vhost_virtqueue { 593a4d5c94SMichael S. Tsirkin struct vhost_dev *dev; 603a4d5c94SMichael S. Tsirkin 613a4d5c94SMichael S. Tsirkin /* The actual ring of buffers. */ 623a4d5c94SMichael S. Tsirkin struct mutex mutex; 633a4d5c94SMichael S. Tsirkin unsigned int num; 643a4d5c94SMichael S. Tsirkin struct vring_desc __user *desc; 653a4d5c94SMichael S. Tsirkin struct vring_avail __user *avail; 663a4d5c94SMichael S. Tsirkin struct vring_used __user *used; 673a4d5c94SMichael S. Tsirkin struct file *kick; 683a4d5c94SMichael S. Tsirkin struct file *call; 693a4d5c94SMichael S. Tsirkin struct file *error; 703a4d5c94SMichael S. Tsirkin struct eventfd_ctx *call_ctx; 713a4d5c94SMichael S. Tsirkin struct eventfd_ctx *error_ctx; 723a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 733a4d5c94SMichael S. Tsirkin 743a4d5c94SMichael S. Tsirkin struct vhost_poll poll; 753a4d5c94SMichael S. Tsirkin 763a4d5c94SMichael S. Tsirkin /* The routine to call when the Guest pings us, or timeout. */ 77c23f3445STejun Heo vhost_work_fn_t handle_kick; 783a4d5c94SMichael S. Tsirkin 793a4d5c94SMichael S. Tsirkin /* Last available index we saw. */ 803a4d5c94SMichael S. Tsirkin u16 last_avail_idx; 813a4d5c94SMichael S. Tsirkin 823a4d5c94SMichael S. Tsirkin /* Caches available index value from user. */ 833a4d5c94SMichael S. Tsirkin u16 avail_idx; 843a4d5c94SMichael S. Tsirkin 853a4d5c94SMichael S. Tsirkin /* Last index we used. */ 863a4d5c94SMichael S. Tsirkin u16 last_used_idx; 873a4d5c94SMichael S. Tsirkin 883a4d5c94SMichael S. Tsirkin /* Used flags */ 893a4d5c94SMichael S. Tsirkin u16 used_flags; 903a4d5c94SMichael S. Tsirkin 918ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 928ea8cf89SMichael S. Tsirkin u16 signalled_used; 938ea8cf89SMichael S. Tsirkin 948ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 958ea8cf89SMichael S. Tsirkin bool signalled_used_valid; 968ea8cf89SMichael S. Tsirkin 973a4d5c94SMichael S. Tsirkin /* Log writes to used structure. */ 983a4d5c94SMichael S. Tsirkin bool log_used; 993a4d5c94SMichael S. Tsirkin u64 log_addr; 1003a4d5c94SMichael S. Tsirkin 101e0e9b406SJason Wang struct iovec iov[UIO_MAXIOV]; 102e0e9b406SJason Wang struct iovec *indirect; 103e0e9b406SJason Wang struct vring_used_elem *heads; 1043a4d5c94SMichael S. Tsirkin /* We use a kind of RCU to access private pointer. 105c23f3445STejun Heo * All readers access it from worker, which makes it possible to 106c23f3445STejun Heo * flush the vhost_work instead of synchronize_rcu. Therefore readers do 1073a4d5c94SMichael S. Tsirkin * not need to call rcu_read_lock/rcu_read_unlock: the beginning of 108c23f3445STejun Heo * vhost_work execution acts instead of rcu_read_lock() and the end of 109a290aec8SJason Wang * vhost_work execution acts instead of rcu_read_unlock(). 1103a4d5c94SMichael S. Tsirkin * Writers use virtqueue mutex. */ 11128457ee6SArnd Bergmann void __rcu *private_data; 1123a4d5c94SMichael S. Tsirkin /* Log write descriptors */ 1133a4d5c94SMichael S. Tsirkin void __user *log_base; 114e0e9b406SJason Wang struct vhost_log *log; 1153a4d5c94SMichael S. Tsirkin }; 1163a4d5c94SMichael S. Tsirkin 1173a4d5c94SMichael S. Tsirkin struct vhost_dev { 1183a4d5c94SMichael S. Tsirkin /* Readers use RCU to access memory table pointer 1193a4d5c94SMichael S. Tsirkin * log base pointer and features. 1203a4d5c94SMichael S. Tsirkin * Writers use mutex below.*/ 12128457ee6SArnd Bergmann struct vhost_memory __rcu *memory; 1223a4d5c94SMichael S. Tsirkin struct mm_struct *mm; 1233a4d5c94SMichael S. Tsirkin struct mutex mutex; 1243a4d5c94SMichael S. Tsirkin unsigned acked_features; 1253ab2e420SAsias He struct vhost_virtqueue **vqs; 1263a4d5c94SMichael S. Tsirkin int nvqs; 1273a4d5c94SMichael S. Tsirkin struct file *log_file; 1283a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 129c23f3445STejun Heo spinlock_t work_lock; 130c23f3445STejun Heo struct list_head work_list; 131c23f3445STejun Heo struct task_struct *worker; 1323a4d5c94SMichael S. Tsirkin }; 1333a4d5c94SMichael S. Tsirkin 1343ab2e420SAsias He long vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs); 13554db63c2SAsias He long vhost_dev_set_owner(struct vhost_dev *dev); 136*05c05351SMichael S. Tsirkin bool vhost_dev_has_owner(struct vhost_dev *dev); 1373a4d5c94SMichael S. Tsirkin long vhost_dev_check_owner(struct vhost_dev *); 138150b9e51SMichael S. Tsirkin struct vhost_memory *vhost_dev_reset_owner_prepare(void); 139150b9e51SMichael S. Tsirkin void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_memory *); 140ea5d4046SMichael S. Tsirkin void vhost_dev_cleanup(struct vhost_dev *, bool locked); 141b211616dSMichael S. Tsirkin void vhost_dev_stop(struct vhost_dev *); 142935cdee7SMichael S. Tsirkin long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); 143935cdee7SMichael S. Tsirkin long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); 1443a4d5c94SMichael S. Tsirkin int vhost_vq_access_ok(struct vhost_virtqueue *vq); 1453a4d5c94SMichael S. Tsirkin int vhost_log_access_ok(struct vhost_dev *); 1463a4d5c94SMichael S. Tsirkin 147d5675bd2SMichael S. Tsirkin int vhost_get_vq_desc(struct vhost_dev *, struct vhost_virtqueue *, 1483a4d5c94SMichael S. Tsirkin struct iovec iov[], unsigned int iov_count, 1493a4d5c94SMichael S. Tsirkin unsigned int *out_num, unsigned int *in_num, 1503a4d5c94SMichael S. Tsirkin struct vhost_log *log, unsigned int *log_num); 1518dd014adSDavid Stevens void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); 1523a4d5c94SMichael S. Tsirkin 153f59281daSJason Wang int vhost_init_used(struct vhost_virtqueue *); 1543a4d5c94SMichael S. Tsirkin int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len); 1558dd014adSDavid Stevens int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads, 1568dd014adSDavid Stevens unsigned count); 1573a4d5c94SMichael S. Tsirkin void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *, 1588dd014adSDavid Stevens unsigned int id, int len); 1598dd014adSDavid Stevens void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, 1608dd014adSDavid Stevens struct vring_used_elem *heads, unsigned count); 1618dd014adSDavid Stevens void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); 1628ea8cf89SMichael S. Tsirkin void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1638ea8cf89SMichael S. Tsirkin bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1643a4d5c94SMichael S. Tsirkin 1653a4d5c94SMichael S. Tsirkin int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1663a4d5c94SMichael S. Tsirkin unsigned int log_num, u64 len); 1673a4d5c94SMichael S. Tsirkin 1683a4d5c94SMichael S. Tsirkin #define vq_err(vq, fmt, ...) do { \ 1693a4d5c94SMichael S. Tsirkin pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ 1703a4d5c94SMichael S. Tsirkin if ((vq)->error_ctx) \ 1713a4d5c94SMichael S. Tsirkin eventfd_signal((vq)->error_ctx, 1);\ 1723a4d5c94SMichael S. Tsirkin } while (0) 1733a4d5c94SMichael S. Tsirkin 1743a4d5c94SMichael S. Tsirkin enum { 1758ea8cf89SMichael S. Tsirkin VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | 1768ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 1778ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_EVENT_IDX) | 1780dd05a3bSStefan Hajnoczi (1ULL << VHOST_F_LOG_ALL), 1793a4d5c94SMichael S. Tsirkin }; 1803a4d5c94SMichael S. Tsirkin 1813a4d5c94SMichael S. Tsirkin static inline int vhost_has_feature(struct vhost_dev *dev, int bit) 1823a4d5c94SMichael S. Tsirkin { 18328457ee6SArnd Bergmann unsigned acked_features; 18428457ee6SArnd Bergmann 1855e18247bSMichael S. Tsirkin /* TODO: check that we are running from vhost_worker or dev mutex is 1865e18247bSMichael S. Tsirkin * held? */ 1875e18247bSMichael S. Tsirkin acked_features = rcu_dereference_index_check(dev->acked_features, 1); 1883a4d5c94SMichael S. Tsirkin return acked_features & (1 << bit); 1893a4d5c94SMichael S. Tsirkin } 1903a4d5c94SMichael S. Tsirkin #endif 191