1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/dma-buf.h>
7 #include <linux/socket.h>
8 #include <net/page_pool/types.h>
9 #include <net/net_trackers.h>
10
11 #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
12 #define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE |\
13 ZCRX_FEATURE_EVENT)
14 #define ZCRX_EVENT_TYPE_MASK ((1U << ZCRX_EVENT_ALLOC_FAIL) |\
15 (1U << ZCRX_EVENT_COPY))
16
17 struct io_zcrx_mem {
18 unsigned long size;
19 bool is_dmabuf;
20
21 struct page **pages;
22 unsigned long nr_folios;
23 struct sg_table page_sg_table;
24 unsigned long account_pages;
25 struct sg_table *sgt;
26
27 struct dma_buf_attachment *attach;
28 struct dma_buf *dmabuf;
29 };
30
31 struct io_zcrx_area {
32 struct net_iov_area nia;
33 struct io_zcrx_ifq *ifq;
34 atomic_t *user_refs;
35
36 bool is_mapped;
37 u16 area_id;
38
39 /* freelist */
40 u32 free_count;
41 u32 *freelist;
42
43 struct io_zcrx_mem mem;
44 };
45
46 struct zcrx_rq_hdr {
47 u32 head ____cacheline_aligned_in_smp;
48 u32 tail ____cacheline_aligned_in_smp;
49 };
50
51 struct zcrx_rq {
52 spinlock_t lock;
53 struct zcrx_rq_hdr *ring;
54 struct io_uring_zcrx_rqe *rqes;
55 u32 cached_head;
56 u32 cached_tail;
57 u32 nr_entries;
58 };
59
60 struct io_zcrx_ifq {
61 /* read-protected by any of: ->pp_lock, ->alloc_lock, ->rq.lock */
62 struct io_zcrx_area **areas;
63 unsigned nr_areas;
64
65 unsigned niov_shift;
66 struct user_struct *user;
67 struct mm_struct *mm_account;
68 bool kern_readable;
69
70 struct zcrx_rq rq ____cacheline_aligned_in_smp;
71 spinlock_t alloc_lock ____cacheline_aligned_in_smp;
72
73 u32 if_rxq;
74 struct device *dev;
75 struct net_device *netdev;
76 netdevice_tracker netdev_tracker;
77 refcount_t refs;
78 /* counts userspace facing users like io_uring */
79 refcount_t user_refs;
80
81 /*
82 * Page pool and net configuration lock, can be taken deeper in the
83 * net stack.
84 */
85 struct mutex pp_lock;
86 struct io_mapped_region rq_region;
87
88 spinlock_t ctx_lock;
89 struct io_ring_ctx *master_ctx;
90 u32 allowed_notif_mask;
91 u32 fired_notifs;
92 u64 notif_data;
93 struct zcrx_stats *notif_stats;
94 };
95
96 #if defined(CONFIG_IO_URING_ZCRX)
97 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
98 int io_register_zcrx(struct io_ring_ctx *ctx,
99 struct io_uring_zcrx_ifq_reg __user *arg);
100 void io_unregister_zcrx(struct io_ring_ctx *ctx);
101 void io_terminate_zcrx(struct io_ring_ctx *ctx);
102 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
103 struct socket *sock, unsigned int flags,
104 unsigned issue_flags, unsigned int *len);
105 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
106 unsigned int id);
107 #else
io_register_zcrx(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)108 static inline int io_register_zcrx(struct io_ring_ctx *ctx,
109 struct io_uring_zcrx_ifq_reg __user *arg)
110 {
111 return -EOPNOTSUPP;
112 }
io_unregister_zcrx(struct io_ring_ctx * ctx)113 static inline void io_unregister_zcrx(struct io_ring_ctx *ctx)
114 {
115 }
io_terminate_zcrx(struct io_ring_ctx * ctx)116 static inline void io_terminate_zcrx(struct io_ring_ctx *ctx)
117 {
118 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)119 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
120 struct socket *sock, unsigned int flags,
121 unsigned issue_flags, unsigned int *len)
122 {
123 return -EOPNOTSUPP;
124 }
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)125 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
126 unsigned int id)
127 {
128 return NULL;
129 }
io_zcrx_ctrl(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_arg)130 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
131 void __user *arg, unsigned nr_arg)
132 {
133 return -EOPNOTSUPP;
134 }
135 #endif
136
137 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
138 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
139
140 #endif
141