1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 struct io_zcrx_mem { 12 unsigned long size; 13 bool is_dmabuf; 14 15 struct page **pages; 16 unsigned long nr_folios; 17 struct sg_table page_sg_table; 18 unsigned long account_pages; 19 struct sg_table *sgt; 20 21 struct dma_buf_attachment *attach; 22 struct dma_buf *dmabuf; 23 }; 24 25 struct io_zcrx_area { 26 struct net_iov_area nia; 27 struct io_zcrx_ifq *ifq; 28 atomic_t *user_refs; 29 30 bool is_mapped; 31 u16 area_id; 32 33 /* freelist */ 34 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 35 u32 free_count; 36 u32 *freelist; 37 38 struct io_zcrx_mem mem; 39 }; 40 41 struct io_zcrx_ifq { 42 struct io_ring_ctx *ctx; 43 struct io_zcrx_area *area; 44 unsigned niov_shift; 45 46 spinlock_t rq_lock ____cacheline_aligned_in_smp; 47 struct io_uring *rq_ring; 48 struct io_uring_zcrx_rqe *rqes; 49 u32 cached_rq_head; 50 u32 rq_entries; 51 52 u32 if_rxq; 53 struct device *dev; 54 struct net_device *netdev; 55 netdevice_tracker netdev_tracker; 56 57 /* 58 * Page pool and net configuration lock, can be taken deeper in the 59 * net stack. 60 */ 61 struct mutex pp_lock; 62 struct io_mapped_region region; 63 }; 64 65 #if defined(CONFIG_IO_URING_ZCRX) 66 int io_zcrx_return_bufs(struct io_ring_ctx *ctx, 67 void __user *arg, unsigned nr_arg); 68 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 69 struct io_uring_zcrx_ifq_reg __user *arg); 70 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); 71 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx); 72 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 73 struct socket *sock, unsigned int flags, 74 unsigned issue_flags, unsigned int *len); 75 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 76 unsigned int id); 77 #else 78 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 79 struct io_uring_zcrx_ifq_reg __user *arg) 80 { 81 return -EOPNOTSUPP; 82 } 83 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 84 { 85 } 86 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx) 87 { 88 } 89 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 90 struct socket *sock, unsigned int flags, 91 unsigned issue_flags, unsigned int *len) 92 { 93 return -EOPNOTSUPP; 94 } 95 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 96 unsigned int id) 97 { 98 return NULL; 99 } 100 static inline int io_zcrx_return_bufs(struct io_ring_ctx *ctx, 101 void __user *arg, unsigned nr_arg) 102 { 103 return -EOPNOTSUPP; 104 } 105 #endif 106 107 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 108 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 109 110 #endif 111