1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV) 12 #define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE |\ 13 ZCRX_FEATURE_EVENT) 14 #define ZCRX_EVENT_TYPE_MASK ((1U << ZCRX_EVENT_ALLOC_FAIL) |\ 15 (1U << ZCRX_EVENT_COPY)) 16 17 struct io_zcrx_mem { 18 unsigned long size; 19 bool is_dmabuf; 20 21 struct page **pages; 22 unsigned long nr_folios; 23 struct sg_table page_sg_table; 24 unsigned long account_pages; 25 struct sg_table *sgt; 26 27 struct dma_buf_attachment *attach; 28 struct dma_buf *dmabuf; 29 }; 30 31 struct io_zcrx_area { 32 struct net_iov_area nia; 33 struct io_zcrx_ifq *ifq; 34 atomic_t *user_refs; 35 36 bool is_mapped; 37 u16 area_id; 38 39 /* freelist */ 40 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 41 u32 free_count; 42 u32 *freelist; 43 44 struct io_zcrx_mem mem; 45 }; 46 47 struct zcrx_rq { 48 spinlock_t lock; 49 struct io_uring *ring; 50 struct io_uring_zcrx_rqe *rqes; 51 u32 cached_head; 52 u32 nr_entries; 53 }; 54 55 struct io_zcrx_ifq { 56 struct io_zcrx_area *area; 57 unsigned niov_shift; 58 struct user_struct *user; 59 struct mm_struct *mm_account; 60 bool kern_readable; 61 62 struct zcrx_rq rq ____cacheline_aligned_in_smp; 63 64 u32 if_rxq; 65 struct device *dev; 66 struct net_device *netdev; 67 netdevice_tracker netdev_tracker; 68 refcount_t refs; 69 /* counts userspace facing users like io_uring */ 70 refcount_t user_refs; 71 72 /* 73 * Page pool and net configuration lock, can be taken deeper in the 74 * net stack. 75 */ 76 struct mutex pp_lock; 77 struct io_mapped_region rq_region; 78 79 spinlock_t ctx_lock; 80 struct io_ring_ctx *master_ctx; 81 u32 allowed_notif_mask; 82 u32 fired_notifs; 83 u64 notif_data; 84 struct zcrx_stats *notif_stats; 85 }; 86 87 #if defined(CONFIG_IO_URING_ZCRX) 88 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg); 89 int io_register_zcrx(struct io_ring_ctx *ctx, 90 struct io_uring_zcrx_ifq_reg __user *arg); 91 void io_unregister_zcrx(struct io_ring_ctx *ctx); 92 void io_terminate_zcrx(struct io_ring_ctx *ctx); 93 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 94 struct socket *sock, unsigned int flags, 95 unsigned issue_flags, unsigned int *len); 96 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 97 unsigned int id); 98 #else 99 static inline int io_register_zcrx(struct io_ring_ctx *ctx, 100 struct io_uring_zcrx_ifq_reg __user *arg) 101 { 102 return -EOPNOTSUPP; 103 } 104 static inline void io_unregister_zcrx(struct io_ring_ctx *ctx) 105 { 106 } 107 static inline void io_terminate_zcrx(struct io_ring_ctx *ctx) 108 { 109 } 110 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 111 struct socket *sock, unsigned int flags, 112 unsigned issue_flags, unsigned int *len) 113 { 114 return -EOPNOTSUPP; 115 } 116 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 117 unsigned int id) 118 { 119 return NULL; 120 } 121 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx, 122 void __user *arg, unsigned nr_arg) 123 { 124 return -EOPNOTSUPP; 125 } 126 #endif 127 128 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 129 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 130 131 #endif 132