1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 struct io_zcrx_mem { 12 unsigned long size; 13 bool is_dmabuf; 14 15 struct page **pages; 16 unsigned long nr_folios; 17 struct sg_table page_sg_table; 18 unsigned long account_pages; 19 struct sg_table *sgt; 20 21 struct dma_buf_attachment *attach; 22 struct dma_buf *dmabuf; 23 }; 24 25 struct io_zcrx_area { 26 struct net_iov_area nia; 27 struct io_zcrx_ifq *ifq; 28 atomic_t *user_refs; 29 30 bool is_mapped; 31 u16 area_id; 32 33 /* freelist */ 34 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 35 u32 free_count; 36 u32 *freelist; 37 38 struct io_zcrx_mem mem; 39 }; 40 41 struct io_zcrx_ifq { 42 struct io_ring_ctx *ctx; 43 struct io_zcrx_area *area; 44 unsigned niov_shift; 45 46 spinlock_t rq_lock ____cacheline_aligned_in_smp; 47 struct io_uring *rq_ring; 48 struct io_uring_zcrx_rqe *rqes; 49 u32 cached_rq_head; 50 u32 rq_entries; 51 52 u32 if_rxq; 53 struct device *dev; 54 struct net_device *netdev; 55 netdevice_tracker netdev_tracker; 56 57 /* 58 * Page pool and net configuration lock, can be taken deeper in the 59 * net stack. 60 */ 61 struct mutex pp_lock; 62 struct io_mapped_region region; 63 }; 64 65 #if defined(CONFIG_IO_URING_ZCRX) 66 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 67 struct io_uring_zcrx_ifq_reg __user *arg); 68 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); 69 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx); 70 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 71 struct socket *sock, unsigned int flags, 72 unsigned issue_flags, unsigned int *len); 73 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 74 unsigned int id); 75 #else 76 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 77 struct io_uring_zcrx_ifq_reg __user *arg) 78 { 79 return -EOPNOTSUPP; 80 } 81 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 82 { 83 } 84 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx) 85 { 86 } 87 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 88 struct socket *sock, unsigned int flags, 89 unsigned issue_flags, unsigned int *len) 90 { 91 return -EOPNOTSUPP; 92 } 93 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 94 unsigned int id) 95 { 96 return NULL; 97 } 98 #endif 99 100 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 101 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 102 103 #endif 104