1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 struct io_zcrx_mem { 12 unsigned long size; 13 bool is_dmabuf; 14 15 struct page **pages; 16 unsigned long nr_folios; 17 struct sg_table page_sg_table; 18 unsigned long account_pages; 19 struct sg_table *sgt; 20 21 struct dma_buf_attachment *attach; 22 struct dma_buf *dmabuf; 23 }; 24 25 struct io_zcrx_area { 26 struct net_iov_area nia; 27 struct io_zcrx_ifq *ifq; 28 atomic_t *user_refs; 29 30 bool is_mapped; 31 u16 area_id; 32 33 /* freelist */ 34 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 35 u32 free_count; 36 u32 *freelist; 37 38 struct io_zcrx_mem mem; 39 }; 40 41 struct io_zcrx_ifq { 42 struct io_zcrx_area *area; 43 unsigned niov_shift; 44 struct user_struct *user; 45 struct mm_struct *mm_account; 46 47 spinlock_t rq_lock ____cacheline_aligned_in_smp; 48 struct io_uring *rq_ring; 49 struct io_uring_zcrx_rqe *rqes; 50 u32 cached_rq_head; 51 u32 rq_entries; 52 53 u32 if_rxq; 54 struct device *dev; 55 struct net_device *netdev; 56 netdevice_tracker netdev_tracker; 57 refcount_t refs; 58 /* counts userspace facing users like io_uring */ 59 refcount_t user_refs; 60 61 /* 62 * Page pool and net configuration lock, can be taken deeper in the 63 * net stack. 64 */ 65 struct mutex pp_lock; 66 struct io_mapped_region region; 67 }; 68 69 #if defined(CONFIG_IO_URING_ZCRX) 70 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg); 71 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 72 struct io_uring_zcrx_ifq_reg __user *arg); 73 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); 74 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 75 struct socket *sock, unsigned int flags, 76 unsigned issue_flags, unsigned int *len); 77 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 78 unsigned int id); 79 #else 80 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 81 struct io_uring_zcrx_ifq_reg __user *arg) 82 { 83 return -EOPNOTSUPP; 84 } 85 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 86 { 87 } 88 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 89 struct socket *sock, unsigned int flags, 90 unsigned issue_flags, unsigned int *len) 91 { 92 return -EOPNOTSUPP; 93 } 94 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 95 unsigned int id) 96 { 97 return NULL; 98 } 99 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx, 100 void __user *arg, unsigned nr_arg) 101 { 102 return -EOPNOTSUPP; 103 } 104 #endif 105 106 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 107 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 108 109 #endif 110