xref: /linux/include/net/xdp_sock.h (revision 1a9239bb4253f9076b5b4b2a1a4e8d7defd77a95)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* AF_XDP internal functions
3  * Copyright(c) 2018 Intel Corporation.
4  */
5 
6 #ifndef _LINUX_XDP_SOCK_H
7 #define _LINUX_XDP_SOCK_H
8 
9 #include <linux/bpf.h>
10 #include <linux/workqueue.h>
11 #include <linux/if_xdp.h>
12 #include <linux/mutex.h>
13 #include <linux/spinlock.h>
14 #include <linux/mm.h>
15 #include <net/sock.h>
16 
17 #define XDP_UMEM_SG_FLAG (1 << 1)
18 
19 struct net_device;
20 struct xsk_queue;
21 struct xdp_buff;
22 
23 struct xdp_umem {
24 	void *addrs;
25 	u64 size;
26 	u32 headroom;
27 	u32 chunk_size;
28 	u32 chunks;
29 	u32 npgs;
30 	struct user_struct *user;
31 	refcount_t users;
32 	u8 flags;
33 	u8 tx_metadata_len;
34 	bool zc;
35 	struct page **pgs;
36 	int id;
37 	struct list_head xsk_dma_list;
38 	struct work_struct work;
39 };
40 
41 struct xsk_map {
42 	struct bpf_map map;
43 	spinlock_t lock; /* Synchronize map updates */
44 	atomic_t count;
45 	struct xdp_sock __rcu *xsk_map[];
46 };
47 
48 struct xdp_sock {
49 	/* struct sock must be the first member of struct xdp_sock */
50 	struct sock sk;
51 	struct xsk_queue *rx ____cacheline_aligned_in_smp;
52 	struct net_device *dev;
53 	struct xdp_umem *umem;
54 	struct list_head flush_node;
55 	struct xsk_buff_pool *pool;
56 	u16 queue_id;
57 	bool zc;
58 	bool sg;
59 	enum {
60 		XSK_READY = 0,
61 		XSK_BOUND,
62 		XSK_UNBOUND,
63 	} state;
64 
65 	struct xsk_queue *tx ____cacheline_aligned_in_smp;
66 	struct list_head tx_list;
67 	/* record the number of tx descriptors sent by this xsk and
68 	 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
69 	 * to be given to other xsks for sending tx descriptors, thereby
70 	 * preventing other XSKs from being starved.
71 	 */
72 	u32 tx_budget_spent;
73 
74 	/* Protects generic receive. */
75 	spinlock_t rx_lock;
76 
77 	/* Statistics */
78 	u64 rx_dropped;
79 	u64 rx_queue_full;
80 
81 	/* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
82 	 * packet, the partially built skb is saved here so that packet building can resume in next
83 	 * call of __xsk_generic_xmit().
84 	 */
85 	struct sk_buff *skb;
86 
87 	struct list_head map_list;
88 	/* Protects map_list */
89 	spinlock_t map_list_lock;
90 	/* Protects multiple processes in the control path */
91 	struct mutex mutex;
92 	struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
93 	struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94 };
95 
96 /*
97  * AF_XDP TX metadata hooks for network devices.
98  * The following hooks can be defined; unless noted otherwise, they are
99  * optional and can be filled with a null pointer.
100  *
101  * void (*tmo_request_timestamp)(void *priv)
102  *     Called when AF_XDP frame requested egress timestamp.
103  *
104  * u64 (*tmo_fill_timestamp)(void *priv)
105  *     Called when AF_XDP frame, that had requested egress timestamp,
106  *     received a completion. The hook needs to return the actual HW timestamp.
107  *
108  * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109  *     Called when AF_XDP frame requested HW checksum offload. csum_start
110  *     indicates position where checksumming should start.
111  *     csum_offset indicates position where checksum should be stored.
112  *
113  * void (*tmo_request_launch_time)(u64 launch_time, void *priv)
114  *     Called when AF_XDP frame requested launch time HW offload support.
115  *     launch_time indicates the PTP time at which the device can schedule the
116  *     packet for transmission.
117  */
118 struct xsk_tx_metadata_ops {
119 	void	(*tmo_request_timestamp)(void *priv);
120 	u64	(*tmo_fill_timestamp)(void *priv);
121 	void	(*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
122 	void	(*tmo_request_launch_time)(u64 launch_time, void *priv);
123 };
124 
125 #ifdef CONFIG_XDP_SOCKETS
126 
127 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
128 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
129 void __xsk_map_flush(struct list_head *flush_list);
130 
131 /**
132  *  xsk_tx_metadata_to_compl - Save enough relevant metadata information
133  *  to perform tx completion in the future.
134  *  @meta: pointer to AF_XDP metadata area
135  *  @compl: pointer to output struct xsk_tx_metadata_to_compl
136  *
137  *  This function should be called by the networking device when
138  *  it prepares AF_XDP egress packet. The value of @compl should be stored
139  *  and passed to xsk_tx_metadata_complete upon TX completion.
140  */
xsk_tx_metadata_to_compl(struct xsk_tx_metadata * meta,struct xsk_tx_metadata_compl * compl)141 static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
142 					    struct xsk_tx_metadata_compl *compl)
143 {
144 	if (!meta)
145 		return;
146 
147 	if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
148 		compl->tx_timestamp = &meta->completion.tx_timestamp;
149 	else
150 		compl->tx_timestamp = NULL;
151 }
152 
153 /**
154  *  xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
155  *  and call appropriate xsk_tx_metadata_ops operation.
156  *  @meta: pointer to AF_XDP metadata area
157  *  @ops: pointer to struct xsk_tx_metadata_ops
158  *  @priv: pointer to driver-private aread
159  *
160  *  This function should be called by the networking device when
161  *  it prepares AF_XDP egress packet.
162  */
xsk_tx_metadata_request(const struct xsk_tx_metadata * meta,const struct xsk_tx_metadata_ops * ops,void * priv)163 static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
164 					   const struct xsk_tx_metadata_ops *ops,
165 					   void *priv)
166 {
167 	if (!meta)
168 		return;
169 
170 	if (ops->tmo_request_launch_time)
171 		if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
172 			ops->tmo_request_launch_time(meta->request.launch_time,
173 						     priv);
174 
175 	if (ops->tmo_request_timestamp)
176 		if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
177 			ops->tmo_request_timestamp(priv);
178 
179 	if (ops->tmo_request_checksum)
180 		if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
181 			ops->tmo_request_checksum(meta->request.csum_start,
182 						  meta->request.csum_offset, priv);
183 }
184 
185 /**
186  *  xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
187  *  and call appropriate xsk_tx_metadata_ops operation.
188  *  @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
189  *  @ops: pointer to struct xsk_tx_metadata_ops
190  *  @priv: pointer to driver-private aread
191  *
192  *  This function should be called by the networking device upon
193  *  AF_XDP egress completion.
194  */
xsk_tx_metadata_complete(struct xsk_tx_metadata_compl * compl,const struct xsk_tx_metadata_ops * ops,void * priv)195 static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
196 					    const struct xsk_tx_metadata_ops *ops,
197 					    void *priv)
198 {
199 	if (!compl)
200 		return;
201 	if (!compl->tx_timestamp)
202 		return;
203 
204 	*compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
205 }
206 
207 #else
208 
xsk_generic_rcv(struct xdp_sock * xs,struct xdp_buff * xdp)209 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
210 {
211 	return -ENOTSUPP;
212 }
213 
__xsk_map_redirect(struct xdp_sock * xs,struct xdp_buff * xdp)214 static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
215 {
216 	return -EOPNOTSUPP;
217 }
218 
__xsk_map_flush(struct list_head * flush_list)219 static inline void __xsk_map_flush(struct list_head *flush_list)
220 {
221 }
222 
xsk_tx_metadata_to_compl(struct xsk_tx_metadata * meta,struct xsk_tx_metadata_compl * compl)223 static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
224 					    struct xsk_tx_metadata_compl *compl)
225 {
226 }
227 
xsk_tx_metadata_request(struct xsk_tx_metadata * meta,const struct xsk_tx_metadata_ops * ops,void * priv)228 static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
229 					   const struct xsk_tx_metadata_ops *ops,
230 					   void *priv)
231 {
232 }
233 
xsk_tx_metadata_complete(struct xsk_tx_metadata_compl * compl,const struct xsk_tx_metadata_ops * ops,void * priv)234 static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
235 					    const struct xsk_tx_metadata_ops *ops,
236 					    void *priv)
237 {
238 }
239 
240 #endif /* CONFIG_XDP_SOCKETS */
241 #endif /* _LINUX_XDP_SOCK_H */
242