13a4d5c94SMichael S. Tsirkin #ifndef _VHOST_H 23a4d5c94SMichael S. Tsirkin #define _VHOST_H 33a4d5c94SMichael S. Tsirkin 43a4d5c94SMichael S. Tsirkin #include <linux/eventfd.h> 53a4d5c94SMichael S. Tsirkin #include <linux/vhost.h> 63a4d5c94SMichael S. Tsirkin #include <linux/mm.h> 73a4d5c94SMichael S. Tsirkin #include <linux/mutex.h> 83a4d5c94SMichael S. Tsirkin #include <linux/poll.h> 93a4d5c94SMichael S. Tsirkin #include <linux/file.h> 103a4d5c94SMichael S. Tsirkin #include <linux/uio.h> 113a4d5c94SMichael S. Tsirkin #include <linux/virtio_config.h> 123a4d5c94SMichael S. Tsirkin #include <linux/virtio_ring.h> 1360063497SArun Sharma #include <linux/atomic.h> 143a4d5c94SMichael S. Tsirkin 153a4d5c94SMichael S. Tsirkin struct vhost_device; 163a4d5c94SMichael S. Tsirkin 17c23f3445STejun Heo struct vhost_work; 18c23f3445STejun Heo typedef void (*vhost_work_fn_t)(struct vhost_work *work); 19c23f3445STejun Heo 20c23f3445STejun Heo struct vhost_work { 21c23f3445STejun Heo struct list_head node; 22c23f3445STejun Heo vhost_work_fn_t fn; 23c23f3445STejun Heo wait_queue_head_t done; 24c23f3445STejun Heo int flushing; 25c23f3445STejun Heo unsigned queue_seq; 26c23f3445STejun Heo unsigned done_seq; 27c23f3445STejun Heo }; 28c23f3445STejun Heo 293a4d5c94SMichael S. Tsirkin /* Poll a file (eventfd or socket) */ 303a4d5c94SMichael S. Tsirkin /* Note: there's nothing vhost specific about this structure. */ 313a4d5c94SMichael S. Tsirkin struct vhost_poll { 323a4d5c94SMichael S. Tsirkin poll_table table; 333a4d5c94SMichael S. Tsirkin wait_queue_head_t *wqh; 343a4d5c94SMichael S. Tsirkin wait_queue_t wait; 35c23f3445STejun Heo struct vhost_work work; 363a4d5c94SMichael S. Tsirkin unsigned long mask; 37c23f3445STejun Heo struct vhost_dev *dev; 383a4d5c94SMichael S. Tsirkin }; 393a4d5c94SMichael S. Tsirkin 40163049aeSStefan Hajnoczi void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 41163049aeSStefan Hajnoczi void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); 42163049aeSStefan Hajnoczi 43c23f3445STejun Heo void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 44c23f3445STejun Heo unsigned long mask, struct vhost_dev *dev); 45*2b8b328bSJason Wang int vhost_poll_start(struct vhost_poll *poll, struct file *file); 463a4d5c94SMichael S. Tsirkin void vhost_poll_stop(struct vhost_poll *poll); 473a4d5c94SMichael S. Tsirkin void vhost_poll_flush(struct vhost_poll *poll); 483a4d5c94SMichael S. Tsirkin void vhost_poll_queue(struct vhost_poll *poll); 493a4d5c94SMichael S. Tsirkin 503a4d5c94SMichael S. Tsirkin struct vhost_log { 513a4d5c94SMichael S. Tsirkin u64 addr; 523a4d5c94SMichael S. Tsirkin u64 len; 533a4d5c94SMichael S. Tsirkin }; 543a4d5c94SMichael S. Tsirkin 55bab632d6SMichael S. Tsirkin struct vhost_virtqueue; 56bab632d6SMichael S. Tsirkin 57bab632d6SMichael S. Tsirkin struct vhost_ubuf_ref { 58bab632d6SMichael S. Tsirkin struct kref kref; 59bab632d6SMichael S. Tsirkin wait_queue_head_t wait; 60bab632d6SMichael S. Tsirkin struct vhost_virtqueue *vq; 61bab632d6SMichael S. Tsirkin }; 62bab632d6SMichael S. Tsirkin 63bab632d6SMichael S. Tsirkin struct vhost_ubuf_ref *vhost_ubuf_alloc(struct vhost_virtqueue *, bool zcopy); 64bab632d6SMichael S. Tsirkin void vhost_ubuf_put(struct vhost_ubuf_ref *); 65bab632d6SMichael S. Tsirkin void vhost_ubuf_put_and_wait(struct vhost_ubuf_ref *); 66bab632d6SMichael S. Tsirkin 67b211616dSMichael S. Tsirkin struct ubuf_info; 68b211616dSMichael S. Tsirkin 693a4d5c94SMichael S. Tsirkin /* The virtqueue structure describes a queue attached to a device. */ 703a4d5c94SMichael S. Tsirkin struct vhost_virtqueue { 713a4d5c94SMichael S. Tsirkin struct vhost_dev *dev; 723a4d5c94SMichael S. Tsirkin 733a4d5c94SMichael S. Tsirkin /* The actual ring of buffers. */ 743a4d5c94SMichael S. Tsirkin struct mutex mutex; 753a4d5c94SMichael S. Tsirkin unsigned int num; 763a4d5c94SMichael S. Tsirkin struct vring_desc __user *desc; 773a4d5c94SMichael S. Tsirkin struct vring_avail __user *avail; 783a4d5c94SMichael S. Tsirkin struct vring_used __user *used; 793a4d5c94SMichael S. Tsirkin struct file *kick; 803a4d5c94SMichael S. Tsirkin struct file *call; 813a4d5c94SMichael S. Tsirkin struct file *error; 823a4d5c94SMichael S. Tsirkin struct eventfd_ctx *call_ctx; 833a4d5c94SMichael S. Tsirkin struct eventfd_ctx *error_ctx; 843a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 853a4d5c94SMichael S. Tsirkin 863a4d5c94SMichael S. Tsirkin struct vhost_poll poll; 873a4d5c94SMichael S. Tsirkin 883a4d5c94SMichael S. Tsirkin /* The routine to call when the Guest pings us, or timeout. */ 89c23f3445STejun Heo vhost_work_fn_t handle_kick; 903a4d5c94SMichael S. Tsirkin 913a4d5c94SMichael S. Tsirkin /* Last available index we saw. */ 923a4d5c94SMichael S. Tsirkin u16 last_avail_idx; 933a4d5c94SMichael S. Tsirkin 943a4d5c94SMichael S. Tsirkin /* Caches available index value from user. */ 953a4d5c94SMichael S. Tsirkin u16 avail_idx; 963a4d5c94SMichael S. Tsirkin 973a4d5c94SMichael S. Tsirkin /* Last index we used. */ 983a4d5c94SMichael S. Tsirkin u16 last_used_idx; 993a4d5c94SMichael S. Tsirkin 1003a4d5c94SMichael S. Tsirkin /* Used flags */ 1013a4d5c94SMichael S. Tsirkin u16 used_flags; 1023a4d5c94SMichael S. Tsirkin 1038ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 1048ea8cf89SMichael S. Tsirkin u16 signalled_used; 1058ea8cf89SMichael S. Tsirkin 1068ea8cf89SMichael S. Tsirkin /* Last used index value we have signalled on */ 1078ea8cf89SMichael S. Tsirkin bool signalled_used_valid; 1088ea8cf89SMichael S. Tsirkin 1093a4d5c94SMichael S. Tsirkin /* Log writes to used structure. */ 1103a4d5c94SMichael S. Tsirkin bool log_used; 1113a4d5c94SMichael S. Tsirkin u64 log_addr; 1123a4d5c94SMichael S. Tsirkin 113e0e9b406SJason Wang struct iovec iov[UIO_MAXIOV]; 114e0e9b406SJason Wang /* hdr is used to store the virtio header. 115e0e9b406SJason Wang * Since each iovec has >= 1 byte length, we never need more than 116e0e9b406SJason Wang * header length entries to store the header. */ 117e0e9b406SJason Wang struct iovec hdr[sizeof(struct virtio_net_hdr_mrg_rxbuf)]; 118e0e9b406SJason Wang struct iovec *indirect; 1198dd014adSDavid Stevens size_t vhost_hlen; 1208dd014adSDavid Stevens size_t sock_hlen; 121e0e9b406SJason Wang struct vring_used_elem *heads; 1223a4d5c94SMichael S. Tsirkin /* We use a kind of RCU to access private pointer. 123c23f3445STejun Heo * All readers access it from worker, which makes it possible to 124c23f3445STejun Heo * flush the vhost_work instead of synchronize_rcu. Therefore readers do 1253a4d5c94SMichael S. Tsirkin * not need to call rcu_read_lock/rcu_read_unlock: the beginning of 126c23f3445STejun Heo * vhost_work execution acts instead of rcu_read_lock() and the end of 127a290aec8SJason Wang * vhost_work execution acts instead of rcu_read_unlock(). 1283a4d5c94SMichael S. Tsirkin * Writers use virtqueue mutex. */ 12928457ee6SArnd Bergmann void __rcu *private_data; 1303a4d5c94SMichael S. Tsirkin /* Log write descriptors */ 1313a4d5c94SMichael S. Tsirkin void __user *log_base; 132e0e9b406SJason Wang struct vhost_log *log; 133bab632d6SMichael S. Tsirkin /* vhost zerocopy support fields below: */ 134bab632d6SMichael S. Tsirkin /* last used idx for outstanding DMA zerocopy buffers */ 135bab632d6SMichael S. Tsirkin int upend_idx; 136bab632d6SMichael S. Tsirkin /* first used idx for DMA done zerocopy buffers */ 137bab632d6SMichael S. Tsirkin int done_idx; 138bab632d6SMichael S. Tsirkin /* an array of userspace buffers info */ 139bab632d6SMichael S. Tsirkin struct ubuf_info *ubuf_info; 140bab632d6SMichael S. Tsirkin /* Reference counting for outstanding ubufs. 141bab632d6SMichael S. Tsirkin * Protected by vq mutex. Writers must also take device mutex. */ 142bab632d6SMichael S. Tsirkin struct vhost_ubuf_ref *ubufs; 1433a4d5c94SMichael S. Tsirkin }; 1443a4d5c94SMichael S. Tsirkin 1453a4d5c94SMichael S. Tsirkin struct vhost_dev { 1463a4d5c94SMichael S. Tsirkin /* Readers use RCU to access memory table pointer 1473a4d5c94SMichael S. Tsirkin * log base pointer and features. 1483a4d5c94SMichael S. Tsirkin * Writers use mutex below.*/ 14928457ee6SArnd Bergmann struct vhost_memory __rcu *memory; 1503a4d5c94SMichael S. Tsirkin struct mm_struct *mm; 1513a4d5c94SMichael S. Tsirkin struct mutex mutex; 1523a4d5c94SMichael S. Tsirkin unsigned acked_features; 1533a4d5c94SMichael S. Tsirkin struct vhost_virtqueue *vqs; 1543a4d5c94SMichael S. Tsirkin int nvqs; 1553a4d5c94SMichael S. Tsirkin struct file *log_file; 1563a4d5c94SMichael S. Tsirkin struct eventfd_ctx *log_ctx; 157c23f3445STejun Heo spinlock_t work_lock; 158c23f3445STejun Heo struct list_head work_list; 159c23f3445STejun Heo struct task_struct *worker; 1603a4d5c94SMichael S. Tsirkin }; 1613a4d5c94SMichael S. Tsirkin 1623a4d5c94SMichael S. Tsirkin long vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue *vqs, int nvqs); 1633a4d5c94SMichael S. Tsirkin long vhost_dev_check_owner(struct vhost_dev *); 1643a4d5c94SMichael S. Tsirkin long vhost_dev_reset_owner(struct vhost_dev *); 165ea5d4046SMichael S. Tsirkin void vhost_dev_cleanup(struct vhost_dev *, bool locked); 166b211616dSMichael S. Tsirkin void vhost_dev_stop(struct vhost_dev *); 167935cdee7SMichael S. Tsirkin long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); 168935cdee7SMichael S. Tsirkin long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); 1693a4d5c94SMichael S. Tsirkin int vhost_vq_access_ok(struct vhost_virtqueue *vq); 1703a4d5c94SMichael S. Tsirkin int vhost_log_access_ok(struct vhost_dev *); 1713a4d5c94SMichael S. Tsirkin 172d5675bd2SMichael S. Tsirkin int vhost_get_vq_desc(struct vhost_dev *, struct vhost_virtqueue *, 1733a4d5c94SMichael S. Tsirkin struct iovec iov[], unsigned int iov_count, 1743a4d5c94SMichael S. Tsirkin unsigned int *out_num, unsigned int *in_num, 1753a4d5c94SMichael S. Tsirkin struct vhost_log *log, unsigned int *log_num); 1768dd014adSDavid Stevens void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); 1773a4d5c94SMichael S. Tsirkin 178f59281daSJason Wang int vhost_init_used(struct vhost_virtqueue *); 1793a4d5c94SMichael S. Tsirkin int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len); 1808dd014adSDavid Stevens int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads, 1818dd014adSDavid Stevens unsigned count); 1823a4d5c94SMichael S. Tsirkin void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *, 1838dd014adSDavid Stevens unsigned int id, int len); 1848dd014adSDavid Stevens void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *, 1858dd014adSDavid Stevens struct vring_used_elem *heads, unsigned count); 1868dd014adSDavid Stevens void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); 1878ea8cf89SMichael S. Tsirkin void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1888ea8cf89SMichael S. Tsirkin bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); 1893a4d5c94SMichael S. Tsirkin 1903a4d5c94SMichael S. Tsirkin int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1913a4d5c94SMichael S. Tsirkin unsigned int log_num, u64 len); 1923a4d5c94SMichael S. Tsirkin 1933a4d5c94SMichael S. Tsirkin #define vq_err(vq, fmt, ...) do { \ 1943a4d5c94SMichael S. Tsirkin pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ 1953a4d5c94SMichael S. Tsirkin if ((vq)->error_ctx) \ 1963a4d5c94SMichael S. Tsirkin eventfd_signal((vq)->error_ctx, 1);\ 1973a4d5c94SMichael S. Tsirkin } while (0) 1983a4d5c94SMichael S. Tsirkin 1993a4d5c94SMichael S. Tsirkin enum { 2008ea8cf89SMichael S. Tsirkin VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | 2018ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 2028ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_RING_F_EVENT_IDX) | 2030dd05a3bSStefan Hajnoczi (1ULL << VHOST_F_LOG_ALL), 2040dd05a3bSStefan Hajnoczi VHOST_NET_FEATURES = VHOST_FEATURES | 2058ea8cf89SMichael S. Tsirkin (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | 2068ea8cf89SMichael S. Tsirkin (1ULL << VIRTIO_NET_F_MRG_RXBUF), 2073a4d5c94SMichael S. Tsirkin }; 2083a4d5c94SMichael S. Tsirkin 2093a4d5c94SMichael S. Tsirkin static inline int vhost_has_feature(struct vhost_dev *dev, int bit) 2103a4d5c94SMichael S. Tsirkin { 21128457ee6SArnd Bergmann unsigned acked_features; 21228457ee6SArnd Bergmann 2135e18247bSMichael S. Tsirkin /* TODO: check that we are running from vhost_worker or dev mutex is 2145e18247bSMichael S. Tsirkin * held? */ 2155e18247bSMichael S. Tsirkin acked_features = rcu_dereference_index_check(dev->acked_features, 1); 2163a4d5c94SMichael S. Tsirkin return acked_features & (1 << bit); 2173a4d5c94SMichael S. Tsirkin } 2183a4d5c94SMichael S. Tsirkin 219bab632d6SMichael S. Tsirkin void vhost_enable_zcopy(int vq); 220bab632d6SMichael S. Tsirkin 2213a4d5c94SMichael S. Tsirkin #endif 222