1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef IOU_ZC_RX_H 3 #define IOU_ZC_RX_H 4 5 #include <linux/io_uring_types.h> 6 #include <linux/dma-buf.h> 7 #include <linux/socket.h> 8 #include <net/page_pool/types.h> 9 #include <net/net_trackers.h> 10 11 struct io_zcrx_mem { 12 unsigned long size; 13 bool is_dmabuf; 14 15 struct page **pages; 16 unsigned long nr_folios; 17 struct sg_table page_sg_table; 18 unsigned long account_pages; 19 20 struct dma_buf_attachment *attach; 21 struct dma_buf *dmabuf; 22 struct sg_table *sgt; 23 unsigned long dmabuf_offset; 24 }; 25 26 struct io_zcrx_area { 27 struct net_iov_area nia; 28 struct io_zcrx_ifq *ifq; 29 atomic_t *user_refs; 30 31 bool is_mapped; 32 u16 area_id; 33 34 /* freelist */ 35 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 36 u32 free_count; 37 u32 *freelist; 38 39 struct io_zcrx_mem mem; 40 }; 41 42 struct io_zcrx_ifq { 43 struct io_ring_ctx *ctx; 44 struct io_zcrx_area *area; 45 46 spinlock_t rq_lock ____cacheline_aligned_in_smp; 47 struct io_uring *rq_ring; 48 struct io_uring_zcrx_rqe *rqes; 49 u32 cached_rq_head; 50 u32 rq_entries; 51 52 u32 if_rxq; 53 struct device *dev; 54 struct net_device *netdev; 55 netdevice_tracker netdev_tracker; 56 spinlock_t lock; 57 struct mutex dma_lock; 58 struct io_mapped_region region; 59 }; 60 61 #if defined(CONFIG_IO_URING_ZCRX) 62 int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 63 struct io_uring_zcrx_ifq_reg __user *arg); 64 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); 65 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx); 66 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 67 struct socket *sock, unsigned int flags, 68 unsigned issue_flags, unsigned int *len); 69 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 70 unsigned int id); 71 #else 72 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, 73 struct io_uring_zcrx_ifq_reg __user *arg) 74 { 75 return -EOPNOTSUPP; 76 } 77 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) 78 { 79 } 80 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx) 81 { 82 } 83 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 84 struct socket *sock, unsigned int flags, 85 unsigned issue_flags, unsigned int *len) 86 { 87 return -EOPNOTSUPP; 88 } 89 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 90 unsigned int id) 91 { 92 return NULL; 93 } 94 #endif 95 96 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags); 97 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 98 99 #endif 100