1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/dma-buf.h>
7 #include <linux/socket.h>
8 #include <net/page_pool/types.h>
9 #include <net/net_trackers.h>
10
11 struct io_zcrx_mem {
12 unsigned long size;
13 bool is_dmabuf;
14
15 struct page **pages;
16 unsigned long nr_folios;
17 struct sg_table page_sg_table;
18 unsigned long account_pages;
19 struct sg_table *sgt;
20
21 struct dma_buf_attachment *attach;
22 struct dma_buf *dmabuf;
23 };
24
25 struct io_zcrx_area {
26 struct net_iov_area nia;
27 struct io_zcrx_ifq *ifq;
28 atomic_t *user_refs;
29
30 bool is_mapped;
31 u16 area_id;
32
33 /* freelist */
34 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
35 u32 free_count;
36 u32 *freelist;
37
38 struct io_zcrx_mem mem;
39 };
40
41 struct io_zcrx_ifq {
42 struct io_zcrx_area *area;
43 unsigned niov_shift;
44 struct user_struct *user;
45 struct mm_struct *mm_account;
46
47 spinlock_t rq_lock ____cacheline_aligned_in_smp;
48 struct io_uring *rq_ring;
49 struct io_uring_zcrx_rqe *rqes;
50 u32 cached_rq_head;
51 u32 rq_entries;
52
53 u32 if_rxq;
54 struct device *dev;
55 struct net_device *netdev;
56 netdevice_tracker netdev_tracker;
57 refcount_t refs;
58 /* counts userspace facing users like io_uring */
59 refcount_t user_refs;
60
61 /*
62 * Page pool and net configuration lock, can be taken deeper in the
63 * net stack.
64 */
65 struct mutex pp_lock;
66 struct io_mapped_region region;
67 };
68
69 #if defined(CONFIG_IO_URING_ZCRX)
70 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
71 int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
72 struct io_uring_zcrx_ifq_reg __user *arg);
73 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
74 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
75 struct socket *sock, unsigned int flags,
76 unsigned issue_flags, unsigned int *len);
77 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
78 unsigned int id);
79 #else
io_register_zcrx_ifq(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)80 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
81 struct io_uring_zcrx_ifq_reg __user *arg)
82 {
83 return -EOPNOTSUPP;
84 }
io_unregister_zcrx_ifqs(struct io_ring_ctx * ctx)85 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
86 {
87 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)88 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
89 struct socket *sock, unsigned int flags,
90 unsigned issue_flags, unsigned int *len)
91 {
92 return -EOPNOTSUPP;
93 }
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)94 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
95 unsigned int id)
96 {
97 return NULL;
98 }
io_zcrx_ctrl(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_arg)99 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
100 void __user *arg, unsigned nr_arg)
101 {
102 return -EOPNOTSUPP;
103 }
104 #endif
105
106 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
107 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
108
109 #endif
110