1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV) 12 #define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE) 13 14 struct io_zcrx_mem { 15 unsigned long size; 16 bool is_dmabuf; 17 18 struct page **pages; 19 unsigned long nr_folios; 20 struct sg_table page_sg_table; 21 unsigned long account_pages; 22 struct sg_table *sgt; 23 24 struct dma_buf_attachment *attach; 25 struct dma_buf *dmabuf; 26 }; 27 28 struct io_zcrx_area { 29 struct net_iov_area nia; 30 struct io_zcrx_ifq *ifq; 31 atomic_t *user_refs; 32 33 bool is_mapped; 34 u16 area_id; 35 36 /* freelist */ 37 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 38 u32 free_count; 39 u32 *freelist; 40 41 struct io_zcrx_mem mem; 42 }; 43 44 struct zcrx_rq { 45 spinlock_t lock; 46 struct io_uring *ring; 47 struct io_uring_zcrx_rqe *rqes; 48 u32 cached_head; 49 u32 nr_entries; 50 }; 51 52 struct io_zcrx_ifq { 53 struct io_zcrx_area *area; 54 unsigned niov_shift; 55 struct user_struct *user; 56 struct mm_struct *mm_account; 57 bool kern_readable; 58 59 struct zcrx_rq rq ____cacheline_aligned_in_smp; 60 61 u32 if_rxq; 62 struct device *dev; 63 struct net_device *netdev; 64 netdevice_tracker netdev_tracker; 65 refcount_t refs; 66 /* counts userspace facing users like io_uring */ 67 refcount_t user_refs; 68 69 /* 70 * Page pool and net configuration lock, can be taken deeper in the 71 * net stack. 72 */ 73 struct mutex pp_lock; 74 struct io_mapped_region rq_region; 75 }; 76 77 #if defined(CONFIG_IO_URING_ZCRX) 78 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg); 79 int io_register_zcrx(struct io_ring_ctx *ctx, 80 struct io_uring_zcrx_ifq_reg __user *arg); 81 void io_unregister_zcrx(struct io_ring_ctx *ctx); 82 void io_terminate_zcrx(struct io_ring_ctx *ctx); 83 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 84 struct socket *sock, unsigned int flags, 85 unsigned issue_flags, unsigned int *len); 86 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 87 unsigned int id); 88 #else 89 static inline int io_register_zcrx(struct io_ring_ctx *ctx, 90 struct io_uring_zcrx_ifq_reg __user *arg) 91 { 92 return -EOPNOTSUPP; 93 } 94 static inline void io_unregister_zcrx(struct io_ring_ctx *ctx) 95 { 96 } 97 static inline void io_terminate_zcrx(struct io_ring_ctx *ctx) 98 { 99 } 100 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 101 struct socket *sock, unsigned int flags, 102 unsigned issue_flags, unsigned int *len) 103 { 104 return -EOPNOTSUPP; 105 } 106 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 107 unsigned int id) 108 { 109 return NULL; 110 } 111 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx, 112 void __user *arg, unsigned nr_arg) 113 { 114 return -EOPNOTSUPP; 115 } 116 #endif 117 118 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 119 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 120 121 #endif 122