1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/dma-buf.h>
7 #include <linux/socket.h>
8 #include <net/page_pool/types.h>
9 #include <net/net_trackers.h>
10
11 struct io_zcrx_mem {
12 unsigned long size;
13 bool is_dmabuf;
14
15 struct page **pages;
16 unsigned long nr_folios;
17
18 struct dma_buf_attachment *attach;
19 struct dma_buf *dmabuf;
20 struct sg_table *sgt;
21 unsigned long dmabuf_offset;
22 };
23
24 struct io_zcrx_area {
25 struct net_iov_area nia;
26 struct io_zcrx_ifq *ifq;
27 atomic_t *user_refs;
28
29 bool is_mapped;
30 u16 area_id;
31
32 /* freelist */
33 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
34 u32 free_count;
35 u32 *freelist;
36
37 struct io_zcrx_mem mem;
38 };
39
40 struct io_zcrx_ifq {
41 struct io_ring_ctx *ctx;
42 struct io_zcrx_area *area;
43
44 spinlock_t rq_lock ____cacheline_aligned_in_smp;
45 struct io_uring *rq_ring;
46 struct io_uring_zcrx_rqe *rqes;
47 u32 cached_rq_head;
48 u32 rq_entries;
49
50 u32 if_rxq;
51 struct device *dev;
52 struct net_device *netdev;
53 netdevice_tracker netdev_tracker;
54 spinlock_t lock;
55 struct mutex dma_lock;
56 struct io_mapped_region region;
57 };
58
59 #if defined(CONFIG_IO_URING_ZCRX)
60 int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
61 struct io_uring_zcrx_ifq_reg __user *arg);
62 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
63 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);
64 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
65 struct socket *sock, unsigned int flags,
66 unsigned issue_flags, unsigned int *len);
67 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
68 unsigned int id);
69 #else
io_register_zcrx_ifq(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)70 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
71 struct io_uring_zcrx_ifq_reg __user *arg)
72 {
73 return -EOPNOTSUPP;
74 }
io_unregister_zcrx_ifqs(struct io_ring_ctx * ctx)75 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
76 {
77 }
io_shutdown_zcrx_ifqs(struct io_ring_ctx * ctx)78 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
79 {
80 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)81 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
82 struct socket *sock, unsigned int flags,
83 unsigned issue_flags, unsigned int *len)
84 {
85 return -EOPNOTSUPP;
86 }
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)87 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
88 unsigned int id)
89 {
90 return NULL;
91 }
92 #endif
93
94 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
95 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
96
97 #endif
98