1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (C) 2025 Intel Corporation */
3
4 #ifndef __LIBETH_XDP_H
5 #define __LIBETH_XDP_H
6
7 #include <linux/bpf_trace.h>
8 #include <linux/unroll.h>
9
10 #include <net/libeth/rx.h>
11 #include <net/libeth/tx.h>
12 #include <net/xsk_buff_pool.h>
13
14 /*
15 * Defined as bits to be able to use them as a mask on Rx.
16 * Also used as internal return values on Tx.
17 */
18 enum {
19 LIBETH_XDP_PASS = 0U,
20 LIBETH_XDP_DROP = BIT(0),
21 LIBETH_XDP_ABORTED = BIT(1),
22 LIBETH_XDP_TX = BIT(2),
23 LIBETH_XDP_REDIRECT = BIT(3),
24 };
25
26 /*
27 * &xdp_buff_xsk is the largest structure &libeth_xdp_buff gets casted to,
28 * pick maximum pointer-compatible alignment.
29 */
30 #define __LIBETH_XDP_BUFF_ALIGN \
31 (IS_ALIGNED(sizeof(struct xdp_buff_xsk), 16) ? 16 : \
32 IS_ALIGNED(sizeof(struct xdp_buff_xsk), 8) ? 8 : \
33 sizeof(long))
34
35 /**
36 * struct libeth_xdp_buff - libeth extension over &xdp_buff
37 * @base: main &xdp_buff
38 * @data: shortcut for @base.data
39 * @desc: RQ descriptor containing metadata for this buffer
40 * @priv: driver-private scratchspace
41 *
42 * The main reason for this is to have a pointer to the descriptor to be able
43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
44 * (as well as bigger alignment).
45 * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk.
46 */
47 struct libeth_xdp_buff {
48 union {
49 struct xdp_buff base;
50 void *data;
51 };
52
53 const void *desc;
54 unsigned long priv[]
55 __aligned(__LIBETH_XDP_BUFF_ALIGN);
56 } __aligned(__LIBETH_XDP_BUFF_ALIGN);
57 static_assert(offsetof(struct libeth_xdp_buff, data) ==
58 offsetof(struct xdp_buff_xsk, xdp.data));
59 static_assert(offsetof(struct libeth_xdp_buff, desc) ==
60 offsetof(struct xdp_buff_xsk, cb));
61 static_assert(IS_ALIGNED(sizeof(struct xdp_buff_xsk),
62 __alignof(struct libeth_xdp_buff)));
63
64 /**
65 * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
66 * @name: name of the variable to declare
67 * @...: sizeof() of the driver-private data
68 */
69 #define __LIBETH_XDP_ONSTACK_BUFF(name, ...) \
70 ___LIBETH_XDP_ONSTACK_BUFF(name, ##__VA_ARGS__)
71 /**
72 * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
73 * @name: name of the variable to declare
74 * @...: type or variable name of the driver-private data
75 */
76 #define LIBETH_XDP_ONSTACK_BUFF(name, ...) \
77 __LIBETH_XDP_ONSTACK_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__))
78
79 #define ___LIBETH_XDP_ONSTACK_BUFF(name, ...) \
80 __DEFINE_FLEX(struct libeth_xdp_buff, name, priv, \
81 LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0), \
82 __uninitialized); \
83 LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0)
84
85 #define __libeth_xdp_priv_sz(...) \
86 CONCATENATE(__libeth_xdp_psz, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)
87
88 #define __libeth_xdp_psz0(...)
89 #define __libeth_xdp_psz1(...) sizeof(__VA_ARGS__)
90
91 #define LIBETH_XDP_PRIV_SZ(sz) \
92 (ALIGN(sz, __alignof(struct libeth_xdp_buff)) / sizeof(long))
93
94 /* Performs XSK_CHECK_PRIV_TYPE() */
95 #define LIBETH_XDP_ASSERT_PRIV_SZ(sz) \
96 static_assert(offsetofend(struct xdp_buff_xsk, cb) >= \
97 struct_size_t(struct libeth_xdp_buff, priv, \
98 LIBETH_XDP_PRIV_SZ(sz)))
99
100 /* XDPSQ sharing */
101
102 DECLARE_STATIC_KEY_FALSE(libeth_xdpsq_share);
103
104 /**
105 * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys
106 * @rxq: current number of active Rx queues
107 * @txq: current number of active Tx queues
108 * @max: maximum number of Tx queues
109 *
110 * Each RQ must have its own XDPSQ for XSk pairs, each CPU must have own XDPSQ
111 * for lockless sending (``XDP_TX``, .ndo_xdp_xmit()). Cap the maximum of these
112 * two with the number of SQs the device can have (minus used ones).
113 *
114 * Return: number of XDP Tx queues the device needs to use.
115 */
libeth_xdpsq_num(u32 rxq,u32 txq,u32 max)116 static inline u32 libeth_xdpsq_num(u32 rxq, u32 txq, u32 max)
117 {
118 return min(max(nr_cpu_ids, rxq), max - txq);
119 }
120
121 /**
122 * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs
123 * @num: number of active XDPSQs
124 *
125 * Return: true if there's no 1:1 XDPSQ/CPU association, false otherwise.
126 */
libeth_xdpsq_shared(u32 num)127 static inline bool libeth_xdpsq_shared(u32 num)
128 {
129 return num < nr_cpu_ids;
130 }
131
132 /**
133 * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
134 * @num: number of active XDPSQs
135 *
136 * Helper for libeth_xdp routines, do not use in drivers directly.
137 *
138 * Return: XDPSQ index needs to be used on this CPU.
139 */
libeth_xdpsq_id(u32 num)140 static inline u32 libeth_xdpsq_id(u32 num)
141 {
142 u32 ret = raw_smp_processor_id();
143
144 if (static_branch_unlikely(&libeth_xdpsq_share) &&
145 libeth_xdpsq_shared(num))
146 ret %= num;
147
148 return ret;
149 }
150
151 void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
152 const struct net_device *dev);
153 void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
154 const struct net_device *dev);
155
156 /**
157 * libeth_xdpsq_get - initialize &libeth_xdpsq_lock
158 * @lock: lock to initialize
159 * @dev: netdev which this lock belongs to
160 * @share: whether XDPSQs can be shared
161 *
162 * Tracks the current XDPSQ association and enables the static lock
163 * if needed.
164 */
libeth_xdpsq_get(struct libeth_xdpsq_lock * lock,const struct net_device * dev,bool share)165 static inline void libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
166 const struct net_device *dev,
167 bool share)
168 {
169 if (unlikely(share))
170 __libeth_xdpsq_get(lock, dev);
171 }
172
173 /**
174 * libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock
175 * @lock: lock to deinitialize
176 * @dev: netdev which this lock belongs to
177 *
178 * Tracks the current XDPSQ association and disables the static lock
179 * if needed.
180 */
libeth_xdpsq_put(struct libeth_xdpsq_lock * lock,const struct net_device * dev)181 static inline void libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
182 const struct net_device *dev)
183 {
184 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
185 __libeth_xdpsq_put(lock, dev);
186 }
187
188 void __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock);
189 void __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock);
190
191 /**
192 * libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed
193 * @lock: lock to take
194 *
195 * Touches the underlying spinlock only if the static key is enabled
196 * and the queue itself is marked as shareable.
197 */
libeth_xdpsq_lock(struct libeth_xdpsq_lock * lock)198 static inline void libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock)
199 {
200 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
201 __libeth_xdpsq_lock(lock);
202 }
203
204 /**
205 * libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed
206 * @lock: lock to free
207 *
208 * Touches the underlying spinlock only if the static key is enabled
209 * and the queue itself is marked as shareable.
210 */
libeth_xdpsq_unlock(struct libeth_xdpsq_lock * lock)211 static inline void libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock)
212 {
213 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
214 __libeth_xdpsq_unlock(lock);
215 }
216
217 /* XDPSQ clean-up timers */
218
219 void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq,
220 struct libeth_xdpsq_lock *lock,
221 void (*poll)(struct work_struct *work));
222
223 /**
224 * libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer
225 * @timer: timer to deinitialize
226 *
227 * Flush and disable the underlying workqueue.
228 */
libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer * timer)229 static inline void libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer *timer)
230 {
231 cancel_delayed_work_sync(&timer->dwork);
232 }
233
234 /**
235 * libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer
236 * @timer: timer to queue
237 *
238 * Should be called after the queue was filled and the transmission was run
239 * to complete the pending buffers if no further sending will be done in a
240 * second (-> lazy cleaning won't happen).
241 * If the timer was already run, it will be requeued back to one second
242 * timeout again.
243 */
libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer * timer)244 static inline void libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer *timer)
245 {
246 mod_delayed_work_on(raw_smp_processor_id(), system_bh_highpri_wq,
247 &timer->dwork, HZ);
248 }
249
250 /**
251 * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event
252 * @work: workqueue belonging to the corresponding timer
253 * @poll: driver-specific completion queue poll function
254 *
255 * Run the polling function on the locked queue and requeue the timer if
256 * there's more work to do.
257 * Designed to be used via LIBETH_XDP_DEFINE_TIMER() below.
258 */
259 static __always_inline void
libeth_xdpsq_run_timer(struct work_struct * work,u32 (* poll)(void * xdpsq,u32 budget))260 libeth_xdpsq_run_timer(struct work_struct *work,
261 u32 (*poll)(void *xdpsq, u32 budget))
262 {
263 struct libeth_xdpsq_timer *timer = container_of(work, typeof(*timer),
264 dwork.work);
265
266 libeth_xdpsq_lock(timer->lock);
267
268 if (poll(timer->xdpsq, U32_MAX))
269 libeth_xdpsq_queue_timer(timer);
270
271 libeth_xdpsq_unlock(timer->lock);
272 }
273
274 /* Common Tx bits */
275
276 /**
277 * enum - libeth_xdp internal Tx flags
278 * @LIBETH_XDP_TX_BULK: one bulk size at which it will be flushed to the queue
279 * @LIBETH_XDP_TX_BATCH: batch size for which the queue fill loop is unrolled
280 * @LIBETH_XDP_TX_DROP: indicates the send function must drop frames not sent
281 * @LIBETH_XDP_TX_NDO: whether the send function is called from .ndo_xdp_xmit()
282 * @LIBETH_XDP_TX_XSK: whether the function is called for ``XDP_TX`` for XSk
283 */
284 enum {
285 LIBETH_XDP_TX_BULK = DEV_MAP_BULK_SIZE,
286 LIBETH_XDP_TX_BATCH = 8,
287
288 LIBETH_XDP_TX_DROP = BIT(0),
289 LIBETH_XDP_TX_NDO = BIT(1),
290 LIBETH_XDP_TX_XSK = BIT(2),
291 };
292
293 /**
294 * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags
295 * @LIBETH_XDP_TX_LEN: only for ``XDP_TX``, [15:0] of ::len_fl is actual length
296 * @LIBETH_XDP_TX_CSUM: for XSk xmit, enable checksum offload
297 * @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits
298 * @LIBETH_XDP_TX_FIRST: indicates the frag is the first one of the frame
299 * @LIBETH_XDP_TX_LAST: whether the frag is the last one of the frame
300 * @LIBETH_XDP_TX_MULTI: whether the frame contains several frags
301 * @LIBETH_XDP_TX_FLAGS: only for ``XDP_TX``, [31:16] of ::len_fl is flags
302 */
303 enum {
304 LIBETH_XDP_TX_LEN = GENMASK(15, 0),
305
306 LIBETH_XDP_TX_CSUM = XDP_TXMD_FLAGS_CHECKSUM,
307 LIBETH_XDP_TX_XSKMD = LIBETH_XDP_TX_LEN,
308
309 LIBETH_XDP_TX_FIRST = BIT(16),
310 LIBETH_XDP_TX_LAST = BIT(17),
311 LIBETH_XDP_TX_MULTI = BIT(18),
312
313 LIBETH_XDP_TX_FLAGS = GENMASK(31, 16),
314 };
315
316 /**
317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
318 * @data: frame start pointer for ``XDP_TX``
319 * @len_fl: ``XDP_TX``, combined flags [31:16] and len [15:0] field for speed
320 * @soff: ``XDP_TX``, offset from @data to the start of &skb_shared_info
321 * @frag: one (non-head) frag for ``XDP_TX``
322 * @xdpf: &xdp_frame for the head frag for .ndo_xdp_xmit()
323 * @dma: DMA address of the non-head frag for .ndo_xdp_xmit()
324 * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
325 * @len: frag length for XSk ``XDP_TX`` and .ndo_xdp_xmit()
326 * @flags: Tx flags for the above
327 * @opts: combined @len + @flags for the above for speed
328 * @desc: XSk xmit descriptor for direct casting
329 */
330 struct libeth_xdp_tx_frame {
331 union {
332 /* ``XDP_TX`` */
333 struct {
334 void *data;
335 u32 len_fl;
336 u32 soff;
337 };
338
339 /* ``XDP_TX`` frag */
340 skb_frag_t frag;
341
342 /* .ndo_xdp_xmit(), XSk ``XDP_TX`` */
343 struct {
344 union {
345 struct xdp_frame *xdpf;
346 dma_addr_t dma;
347
348 struct libeth_xdp_buff *xsk;
349 };
350 union {
351 struct {
352 u32 len;
353 u32 flags;
354 };
355 aligned_u64 opts;
356 };
357 };
358
359 /* XSk xmit */
360 struct xdp_desc desc;
361 };
362 } __aligned(sizeof(struct xdp_desc));
363 static_assert(offsetof(struct libeth_xdp_tx_frame, frag.len) ==
364 offsetof(struct libeth_xdp_tx_frame, len_fl));
365 static_assert(sizeof(struct libeth_xdp_tx_frame) == sizeof(struct xdp_desc));
366
367 /**
368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
369 * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
370 * @dev: &net_device which the frames are transmitted on
371 * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure
372 * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
373 * @count: current number of frames in @bulk
374 * @bulk: array of queued frames for bulk Tx
375 *
376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
377 * and flush it when @count reaches the array end. Bulk is always placed on
378 * the stack for performance. One bulk element contains all the data necessary
379 * for sending a frame and then freeing it on completion.
380 * For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly
381 * to &libeth_xdp_tx_frame as they are compatible and the bulk structure is
382 * not used.
383 */
384 struct libeth_xdp_tx_bulk {
385 const struct bpf_prog *prog;
386 struct net_device *dev;
387 void *xdpsq;
388
389 u32 act_mask;
390 u32 count;
391 struct libeth_xdp_tx_frame bulk[LIBETH_XDP_TX_BULK];
392 } __aligned(sizeof(struct libeth_xdp_tx_frame));
393
394 /**
395 * LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack
396 * @bq: name of the variable to declare
397 *
398 * Helper to declare a bulk on the stack with a compiler hint that it should
399 * not be initialized automatically (with `CONFIG_INIT_STACK_ALL_*`) for
400 * performance reasons.
401 */
402 #define LIBETH_XDP_ONSTACK_BULK(bq) \
403 struct libeth_xdp_tx_bulk bq __uninitialized
404
405 /**
406 * struct libeth_xdpsq - abstraction for an XDPSQ
407 * @pool: XSk buffer pool for XSk ``XDP_TX`` and xmit
408 * @sqes: array of Tx buffers from the actual queue struct
409 * @descs: opaque pointer to the HW descriptor array
410 * @ntu: pointer to the next free descriptor index
411 * @count: number of descriptors on that queue
412 * @pending: pointer to the number of sent-not-completed descs on that queue
413 * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames
414 * @lock: corresponding XDPSQ lock
415 *
416 * Abstraction for driver-independent implementation of Tx. Placed on the stack
417 * and filled by the driver before the transmission, so that the generic
418 * functions can access and modify driver-specific resources.
419 */
420 struct libeth_xdpsq {
421 struct xsk_buff_pool *pool;
422 struct libeth_sqe *sqes;
423 void *descs;
424
425 u32 *ntu;
426 u32 count;
427
428 u32 *pending;
429 u32 *xdp_tx;
430 struct libeth_xdpsq_lock *lock;
431 };
432
433 /**
434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
435 * @addr: DMA address of the frame
436 * @len: length of the frame
437 * @flags: XDP Tx flags
438 * @opts: combined @len + @flags for speed
439 *
440 * Filled by the generic functions and then passed to driver-specific functions
441 * to fill a HW Tx descriptor, always placed on the [function] stack.
442 */
443 struct libeth_xdp_tx_desc {
444 dma_addr_t addr;
445 union {
446 struct {
447 u32 len;
448 u32 flags;
449 };
450 aligned_u64 opts;
451 };
452 } __aligned_largest;
453
454 /**
455 * libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv
456 * @ptr: pointer to convert
457 *
458 * The main sending function passes private data as the largest scalar, u64.
459 * Use this helper when you want to pass a pointer there.
460 */
461 #define libeth_xdp_ptr_to_priv(ptr) ({ \
462 typecheck_pointer(ptr); \
463 ((u64)(uintptr_t)(ptr)); \
464 })
465 /**
466 * libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer
467 * @priv: private data to convert
468 *
469 * The main sending function passes private data as the largest scalar, u64.
470 * Use this helper when your callback takes this u64 and you want to convert
471 * it back to a pointer.
472 */
473 #define libeth_xdp_priv_to_ptr(priv) ({ \
474 static_assert(__same_type(priv, u64)); \
475 ((const void *)(uintptr_t)(priv)); \
476 })
477
478 /*
479 * On 64-bit systems, assigning one u64 is faster than two u32s. When ::len
480 * occupies lowest 32 bits (LE), whole ::opts can be assigned directly instead.
481 */
482 #ifdef __LITTLE_ENDIAN
483 #define __LIBETH_WORD_ACCESS 1
484 #endif
485 #ifdef __LIBETH_WORD_ACCESS
486 #define __libeth_xdp_tx_len(flen, ...) \
487 .opts = ((flen) | FIELD_PREP(GENMASK_ULL(63, 32), (__VA_ARGS__ + 0)))
488 #else
489 #define __libeth_xdp_tx_len(flen, ...) \
490 .len = (flen), .flags = (__VA_ARGS__ + 0)
491 #endif
492
493 /**
494 * libeth_xdp_tx_xmit_bulk - main XDP Tx function
495 * @bulk: array of frames to send
496 * @xdpsq: pointer to the driver-specific XDPSQ struct
497 * @n: number of frames to send
498 * @unroll: whether to unroll the queue filling loop for speed
499 * @priv: driver-specific private data
500 * @prep: callback for cleaning the queue and filling abstract &libeth_xdpsq
501 * @fill: internal callback for filling &libeth_sqe and &libeth_xdp_tx_desc
502 * @xmit: callback for filling a HW descriptor with the frame info
503 *
504 * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
505 * all types of frames: ``XDP_TX``, .ndo_xdp_xmit(), XSk ``XDP_TX``, and XSk
506 * xmit.
507 * @prep must lock the queue as this function releases it at the end. @unroll
508 * greatly increases the object code size, but also greatly increases XSk xmit
509 * performance; for other types of frames, it's not enabled.
510 * The compilers inline all those onstack abstractions to direct data accesses.
511 *
512 * Return: number of frames actually placed on the queue, <= @n. The function
513 * can't fail, but can send less frames if there's no enough free descriptors
514 * available. The actual free space is returned by @prep from the driver.
515 */
516 static __always_inline u32
libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame * bulk,void * xdpsq,u32 n,bool unroll,u64 priv,u32 (* prep)(void * xdpsq,struct libeth_xdpsq * sq),struct libeth_xdp_tx_desc (* fill)(struct libeth_xdp_tx_frame frm,u32 i,const struct libeth_xdpsq * sq,u64 priv),void (* xmit)(struct libeth_xdp_tx_desc desc,u32 i,const struct libeth_xdpsq * sq,u64 priv))517 libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq,
518 u32 n, bool unroll, u64 priv,
519 u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
520 struct libeth_xdp_tx_desc
521 (*fill)(struct libeth_xdp_tx_frame frm, u32 i,
522 const struct libeth_xdpsq *sq, u64 priv),
523 void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
524 const struct libeth_xdpsq *sq, u64 priv))
525 {
526 struct libeth_xdpsq sq __uninitialized;
527 u32 this, batched, off = 0;
528 u32 ntu, i = 0;
529
530 n = min(n, prep(xdpsq, &sq));
531 if (unlikely(!n))
532 goto unlock;
533
534 ntu = *sq.ntu;
535
536 this = sq.count - ntu;
537 if (likely(this > n))
538 this = n;
539
540 again:
541 if (!unroll)
542 goto linear;
543
544 batched = ALIGN_DOWN(this, LIBETH_XDP_TX_BATCH);
545
546 for ( ; i < off + batched; i += LIBETH_XDP_TX_BATCH) {
547 u32 base = ntu + i - off;
548
549 unrolled_count(LIBETH_XDP_TX_BATCH)
550 for (u32 j = 0; j < LIBETH_XDP_TX_BATCH; j++)
551 xmit(fill(bulk[i + j], base + j, &sq, priv),
552 base + j, &sq, priv);
553 }
554
555 if (batched < this) {
556 linear:
557 for ( ; i < off + this; i++)
558 xmit(fill(bulk[i], ntu + i - off, &sq, priv),
559 ntu + i - off, &sq, priv);
560 }
561
562 ntu += this;
563 if (likely(ntu < sq.count))
564 goto out;
565
566 ntu = 0;
567
568 if (i < n) {
569 this = n - i;
570 off = i;
571
572 goto again;
573 }
574
575 out:
576 *sq.ntu = ntu;
577 *sq.pending += n;
578 if (sq.xdp_tx)
579 *sq.xdp_tx += n;
580
581 unlock:
582 libeth_xdpsq_unlock(sq.lock);
583
584 return n;
585 }
586
587 /* ``XDP_TX`` bulking */
588
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
590
591 /**
592 * libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head
593 * @bq: XDP Tx bulk to queue the head frag to
594 * @xdp: XDP buffer with the head to queue
595 *
596 * Return: false if it's the only frag of the frame, true if it's an S/G frame.
597 */
libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk * bq,const struct libeth_xdp_buff * xdp)598 static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq,
599 const struct libeth_xdp_buff *xdp)
600 {
601 const struct xdp_buff *base = &xdp->base;
602
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
604 .data = xdp->data,
605 .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,
606 .soff = xdp_data_hard_end(base) - xdp->data,
607 };
608
609 if (!xdp_buff_has_frags(base))
610 return false;
611
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI;
613
614 return true;
615 }
616
617 /**
618 * libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag
619 * @bq: XDP Tx bulk to queue the frag to
620 * @frag: frag to queue
621 */
libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk * bq,const skb_frag_t * frag)622 static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq,
623 const skb_frag_t *frag)
624 {
625 bq->bulk[bq->count++].frag = *frag;
626 }
627
628 /**
629 * libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame
630 * @bq: XDP Tx bulk to queue the frame to
631 * @xdp: XDP buffer to queue
632 * @flush_bulk: driver callback to flush the bulk to the HW queue
633 *
634 * Return: true on success, false on flush error.
635 */
636 static __always_inline bool
libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk * bq,struct libeth_xdp_buff * xdp,bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags))637 libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq,
638 struct libeth_xdp_buff *xdp,
639 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
640 u32 flags))
641 {
642 const struct skb_shared_info *sinfo;
643 bool ret = true;
644 u32 nr_frags;
645
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
647 unlikely(!flush_bulk(bq, 0))) {
648 libeth_xdp_return_buff_slow(xdp);
649 return false;
650 }
651
652 if (!libeth_xdp_tx_queue_head(bq, xdp))
653 goto out;
654
655 sinfo = xdp_get_shared_info_from_buff(&xdp->base);
656 nr_frags = sinfo->nr_frags;
657
658 for (u32 i = 0; i < nr_frags; i++) {
659 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
660 unlikely(!flush_bulk(bq, 0))) {
661 ret = false;
662 break;
663 }
664
665 libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]);
666 }
667
668 out:
669 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST;
670 xdp->data = NULL;
671
672 return ret;
673 }
674
675 /**
676 * libeth_xdp_tx_fill_stats - fill &libeth_sqe with ``XDP_TX`` frame stats
677 * @sqe: SQ element to fill
678 * @desc: libeth_xdp Tx descriptor
679 * @sinfo: &skb_shared_info for this frame
680 *
681 * Internal helper for filling an SQE with the frame stats, do not use in
682 * drivers. Fills the number of frags and bytes for this frame.
683 */
684 #define libeth_xdp_tx_fill_stats(sqe, desc, sinfo) \
685 __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, __UNIQUE_ID(sqe_), \
686 __UNIQUE_ID(desc_), __UNIQUE_ID(sinfo_))
687
688 #define __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, ue, ud, us) do { \
689 const struct libeth_xdp_tx_desc *ud = (desc); \
690 const struct skb_shared_info *us; \
691 struct libeth_sqe *ue = (sqe); \
692 \
693 ue->nr_frags = 1; \
694 ue->bytes = ud->len; \
695 \
696 if (ud->flags & LIBETH_XDP_TX_MULTI) { \
697 us = (sinfo); \
698 ue->nr_frags += us->nr_frags; \
699 ue->bytes += us->xdp_frags_size; \
700 } \
701 } while (0)
702
703 /**
704 * libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe
705 * @frm: XDP Tx frame from the bulk
706 * @i: index on the HW queue
707 * @sq: XDPSQ abstraction for the queue
708 * @priv: private data
709 *
710 * Return: XDP Tx descriptor with the synced DMA and other info to pass to
711 * the driver callback.
712 */
713 static inline struct libeth_xdp_tx_desc
libeth_xdp_tx_fill_buf(struct libeth_xdp_tx_frame frm,u32 i,const struct libeth_xdpsq * sq,u64 priv)714 libeth_xdp_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
715 const struct libeth_xdpsq *sq, u64 priv)
716 {
717 struct libeth_xdp_tx_desc desc;
718 struct skb_shared_info *sinfo;
719 skb_frag_t *frag = &frm.frag;
720 struct libeth_sqe *sqe;
721 netmem_ref netmem;
722
723 if (frm.len_fl & LIBETH_XDP_TX_FIRST) {
724 sinfo = frm.data + frm.soff;
725 skb_frag_fill_netmem_desc(frag, virt_to_netmem(frm.data),
726 offset_in_page(frm.data),
727 frm.len_fl);
728 } else {
729 sinfo = NULL;
730 }
731
732 netmem = skb_frag_netmem(frag);
733 desc = (typeof(desc)){
734 .addr = page_pool_get_dma_addr_netmem(netmem) +
735 skb_frag_off(frag),
736 .len = skb_frag_size(frag) & LIBETH_XDP_TX_LEN,
737 .flags = skb_frag_size(frag) & LIBETH_XDP_TX_FLAGS,
738 };
739
740 dma_sync_single_for_device(__netmem_get_pp(netmem)->p.dev, desc.addr,
741 desc.len, DMA_BIDIRECTIONAL);
742
743 if (!sinfo)
744 return desc;
745
746 sqe = &sq->sqes[i];
747 sqe->type = LIBETH_SQE_XDP_TX;
748 sqe->sinfo = sinfo;
749 libeth_xdp_tx_fill_stats(sqe, &desc, sinfo);
750
751 return desc;
752 }
753
754 void libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent,
755 u32 flags);
756
757 /**
758 * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
759 * @bq: bulk to flush
760 * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
761 * @prep: driver-specific callback to prepare the queue for sending
762 * @fill: libeth_xdp callback to fill &libeth_sqe and &libeth_xdp_tx_desc
763 * @xmit: driver callback to fill a HW descriptor
764 *
765 * Internal abstraction to create bulk flush functions for drivers. Used for
766 * everything except XSk xmit.
767 *
768 * Return: true if anything was sent, false otherwise.
769 */
770 static __always_inline bool
__libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk * bq,u32 flags,u32 (* prep)(void * xdpsq,struct libeth_xdpsq * sq),struct libeth_xdp_tx_desc (* fill)(struct libeth_xdp_tx_frame frm,u32 i,const struct libeth_xdpsq * sq,u64 priv),void (* xmit)(struct libeth_xdp_tx_desc desc,u32 i,const struct libeth_xdpsq * sq,u64 priv))771 __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
772 u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
773 struct libeth_xdp_tx_desc
774 (*fill)(struct libeth_xdp_tx_frame frm, u32 i,
775 const struct libeth_xdpsq *sq, u64 priv),
776 void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
777 const struct libeth_xdpsq *sq,
778 u64 priv))
779 {
780 u32 sent, drops;
781 int err = 0;
782
783 sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
784 min(bq->count, LIBETH_XDP_TX_BULK),
785 false, 0, prep, fill, xmit);
786 drops = bq->count - sent;
787
788 if (unlikely(drops)) {
789 libeth_xdp_tx_exception(bq, sent, flags);
790 err = -ENXIO;
791 } else {
792 bq->count = 0;
793 }
794
795 trace_xdp_bulk_tx(bq->dev, sent, drops, err);
796
797 return likely(sent);
798 }
799
800 /**
801 * libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk
802 * @bq: bulk to flush
803 * @flags: Tx flags, see above
804 * @prep: driver callback to prepare the queue
805 * @xmit: driver callback to fill a HW descriptor
806 *
807 * Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver
808 * callback.
809 */
810 #define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit) \
811 __libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf, \
812 xmit)
813
814 /* .ndo_xdp_xmit() implementation */
815
816 /**
817 * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
818 * @bq: bulk to initialize
819 * @dev: target &net_device
820 * @xdpsqs: array of driver-specific XDPSQ structs
821 * @num: number of active XDPSQs (the above array length)
822 */
823 #define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \
824 __libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
825
__libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk * bq,struct net_device * dev,void * xdpsq)826 static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq,
827 struct net_device *dev,
828 void *xdpsq)
829 {
830 bq->dev = dev;
831 bq->xdpsq = xdpsq;
832 bq->count = 0;
833 }
834
835 /**
836 * libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame
837 * @xf: pointer to the XDP frame
838 *
839 * There's no place in &libeth_xdp_tx_frame to store DMA address for an
840 * &xdp_frame head. The headroom is used then, the address is placed right
841 * after the frame struct, naturally aligned.
842 *
843 * Return: pointer to the DMA address to use.
844 */
845 #define libeth_xdp_xmit_frame_dma(xf) \
846 _Generic((xf), \
847 const struct xdp_frame *: \
848 (const dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf), \
849 struct xdp_frame *: \
850 (dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf) \
851 )
852
__libeth_xdp_xmit_frame_dma(const struct xdp_frame * xdpf)853 static inline void *__libeth_xdp_xmit_frame_dma(const struct xdp_frame *xdpf)
854 {
855 void *addr = (void *)(xdpf + 1);
856
857 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
858 __alignof(*xdpf) < sizeof(dma_addr_t))
859 addr = PTR_ALIGN(addr, sizeof(dma_addr_t));
860
861 return addr;
862 }
863
864 /**
865 * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
866 * @bq: XDP Tx bulk to queue the head frag to
867 * @xdpf: XDP frame with the head to queue
868 * @dev: device to perform DMA mapping
869 *
870 * Return: ``LIBETH_XDP_DROP`` on DMA mapping error,
871 * ``LIBETH_XDP_PASS`` if it's the only frag in the frame,
872 * ``LIBETH_XDP_TX`` if it's an S/G frame.
873 */
libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk * bq,struct xdp_frame * xdpf,struct device * dev)874 static inline u32 libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk *bq,
875 struct xdp_frame *xdpf,
876 struct device *dev)
877 {
878 dma_addr_t dma;
879
880 dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
881 if (dma_mapping_error(dev, dma))
882 return LIBETH_XDP_DROP;
883
884 *libeth_xdp_xmit_frame_dma(xdpf) = dma;
885
886 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
887 .xdpf = xdpf,
888 __libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST),
889 };
890
891 if (!xdp_frame_has_frags(xdpf))
892 return LIBETH_XDP_PASS;
893
894 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI;
895
896 return LIBETH_XDP_TX;
897 }
898
899 /**
900 * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
901 * @bq: XDP Tx bulk to queue the frag to
902 * @frag: frag to queue
903 * @dev: device to perform DMA mapping
904 *
905 * Return: true on success, false on DMA mapping error.
906 */
libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk * bq,const skb_frag_t * frag,struct device * dev)907 static inline bool libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk *bq,
908 const skb_frag_t *frag,
909 struct device *dev)
910 {
911 dma_addr_t dma;
912
913 dma = skb_frag_dma_map(dev, frag);
914 if (dma_mapping_error(dev, dma))
915 return false;
916
917 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
918 .dma = dma,
919 __libeth_xdp_tx_len(skb_frag_size(frag)),
920 };
921
922 return true;
923 }
924
925 /**
926 * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
927 * @bq: XDP Tx bulk to queue the frame to
928 * @xdpf: XDP frame to queue
929 * @flush_bulk: driver callback to flush the bulk to the HW queue
930 *
931 * Return: ``LIBETH_XDP_TX`` on success,
932 * ``LIBETH_XDP_DROP`` if the frame should be dropped by the stack,
933 * ``LIBETH_XDP_ABORTED`` if the frame will be dropped by libeth_xdp.
934 */
935 static __always_inline u32
libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk * bq,struct xdp_frame * xdpf,bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags))936 libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk *bq,
937 struct xdp_frame *xdpf,
938 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
939 u32 flags))
940 {
941 u32 head, nr_frags, i, ret = LIBETH_XDP_TX;
942 struct device *dev = bq->dev->dev.parent;
943 const struct skb_shared_info *sinfo;
944
945 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
946 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
947 return LIBETH_XDP_DROP;
948
949 head = libeth_xdp_xmit_queue_head(bq, xdpf, dev);
950 if (head == LIBETH_XDP_PASS)
951 goto out;
952 else if (head == LIBETH_XDP_DROP)
953 return LIBETH_XDP_DROP;
954
955 sinfo = xdp_get_shared_info_from_frame(xdpf);
956 nr_frags = sinfo->nr_frags;
957
958 for (i = 0; i < nr_frags; i++) {
959 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
960 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
961 break;
962
963 if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev))
964 break;
965 }
966
967 if (unlikely(i < nr_frags))
968 ret = LIBETH_XDP_ABORTED;
969
970 out:
971 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST;
972
973 return ret;
974 }
975
976 /**
977 * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
978 * @frm: XDP Tx frame from the bulk
979 * @i: index on the HW queue
980 * @sq: XDPSQ abstraction for the queue
981 * @priv: private data
982 *
983 * Return: XDP Tx descriptor with the mapped DMA and other info to pass to
984 * the driver callback.
985 */
986 static inline struct libeth_xdp_tx_desc
libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm,u32 i,const struct libeth_xdpsq * sq,u64 priv)987 libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
988 const struct libeth_xdpsq *sq, u64 priv)
989 {
990 struct libeth_xdp_tx_desc desc;
991 struct libeth_sqe *sqe;
992 struct xdp_frame *xdpf;
993
994 if (frm.flags & LIBETH_XDP_TX_FIRST) {
995 xdpf = frm.xdpf;
996 desc.addr = *libeth_xdp_xmit_frame_dma(xdpf);
997 } else {
998 xdpf = NULL;
999 desc.addr = frm.dma;
1000 }
1001 desc.opts = frm.opts;
1002
1003 sqe = &sq->sqes[i];
1004 dma_unmap_addr_set(sqe, dma, desc.addr);
1005 dma_unmap_len_set(sqe, len, desc.len);
1006
1007 if (!xdpf) {
1008 sqe->type = LIBETH_SQE_XDP_XMIT_FRAG;
1009 return desc;
1010 }
1011
1012 sqe->type = LIBETH_SQE_XDP_XMIT;
1013 sqe->xdpf = xdpf;
1014 libeth_xdp_tx_fill_stats(sqe, &desc,
1015 xdp_get_shared_info_from_frame(xdpf));
1016
1017 return desc;
1018 }
1019
1020 /**
1021 * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
1022 * @bq: bulk to flush
1023 * @flags: Tx flags, see __libeth_xdp_tx_flush_bulk()
1024 * @prep: driver callback to prepare the queue
1025 * @xmit: driver callback to fill a HW descriptor
1026 *
1027 * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
1028 * callback.
1029 */
1030 #define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit) \
1031 __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep, \
1032 libeth_xdp_xmit_fill_buf, xmit)
1033
1034 u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq,
1035 u32 count, const struct net_device *dev);
1036
1037 /**
1038 * __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit()
1039 * @bq: XDP Tx bulk to queue frames to
1040 * @frames: XDP frames passed by the stack
1041 * @n: number of frames
1042 * @flags: flags passed by the stack
1043 * @flush_bulk: driver callback to flush an XDP xmit bulk
1044 * @finalize: driver callback to finalize sending XDP Tx frames on the queue
1045 *
1046 * Perform common checks, map the frags and queue them to the bulk, then flush
1047 * the bulk to the XDPSQ. If requested by the stack, finalize the queue.
1048 *
1049 * Return: number of frames send or -errno on error.
1050 */
1051 static __always_inline int
__libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk * bq,struct xdp_frame ** frames,u32 n,u32 flags,bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags),void (* finalize)(void * xdpsq,bool sent,bool flush))1052 __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
1053 struct xdp_frame **frames, u32 n, u32 flags,
1054 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1055 u32 flags),
1056 void (*finalize)(void *xdpsq, bool sent, bool flush))
1057 {
1058 u32 nxmit = 0;
1059
1060 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1061 return -EINVAL;
1062
1063 for (u32 i = 0; likely(i < n); i++) {
1064 u32 ret;
1065
1066 ret = libeth_xdp_xmit_queue_bulk(bq, frames[i], flush_bulk);
1067 if (unlikely(ret != LIBETH_XDP_TX)) {
1068 nxmit += ret == LIBETH_XDP_ABORTED;
1069 break;
1070 }
1071
1072 nxmit++;
1073 }
1074
1075 if (bq->count) {
1076 flush_bulk(bq, LIBETH_XDP_TX_NDO);
1077 if (unlikely(bq->count))
1078 nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk,
1079 bq->count,
1080 bq->dev);
1081 }
1082
1083 finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH);
1084
1085 return nxmit;
1086 }
1087
1088 /**
1089 * libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver
1090 * @dev: target &net_device
1091 * @n: number of frames to send
1092 * @fr: XDP frames to send
1093 * @f: flags passed by the stack
1094 * @xqs: array of XDPSQs driver structs
1095 * @nqs: number of active XDPSQs, the above array length
1096 * @fl: driver callback to flush an XDP xmit bulk
1097 * @fin: driver cabback to finalize the queue
1098 *
1099 * If the driver has active XDPSQs, perform common checks and send the frames.
1100 * Finalize the queue, if requested.
1101 *
1102 * Return: number of frames sent or -errno on error.
1103 */
1104 #define libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin) \
1105 _libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin, \
1106 __UNIQUE_ID(bq_), __UNIQUE_ID(ret_), \
1107 __UNIQUE_ID(nqs_))
1108
1109 #define _libeth_xdp_xmit_do_bulk(d, n, fr, f, xqs, nqs, fl, fin, ub, ur, un) \
1110 ({ \
1111 u32 un = (nqs); \
1112 int ur; \
1113 \
1114 if (likely(un)) { \
1115 LIBETH_XDP_ONSTACK_BULK(ub); \
1116 \
1117 libeth_xdp_xmit_init_bulk(&ub, d, xqs, un); \
1118 ur = __libeth_xdp_xmit_do_bulk(&ub, fr, n, f, fl, fin); \
1119 } else { \
1120 ur = -ENXIO; \
1121 } \
1122 \
1123 ur; \
1124 })
1125
1126 /* Rx polling path */
1127
1128 /**
1129 * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
1130 * @bq: bulk to initialize
1131 * @prog: RCU pointer to the XDP program (can be %NULL)
1132 * @dev: target &net_device
1133 * @xdpsqs: array of driver XDPSQ structs
1134 * @num: number of active XDPSQs, the above array length
1135 *
1136 * Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
1137 * Initializes all the needed fields to run libeth_xdp functions. If @num == 0,
1138 * assumes XDP is not enabled.
1139 * Do not use for XSk, it has its own optimized helper.
1140 */
1141 #define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \
1142 __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \
1143 __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
1144
1145 #define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \
1146 typeof(bq) ub = (bq); \
1147 u32 un = (num); \
1148 \
1149 rcu_read_lock(); \
1150 \
1151 if (un || (xsk)) { \
1152 ub->prog = rcu_dereference(pr); \
1153 ub->dev = (d); \
1154 ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \
1155 } else { \
1156 ub->prog = NULL; \
1157 } \
1158 \
1159 ub->act_mask = 0; \
1160 ub->count = 0; \
1161 } while (0)
1162
1163 void libeth_xdp_load_stash(struct libeth_xdp_buff *dst,
1164 const struct libeth_xdp_buff_stash *src);
1165 void libeth_xdp_save_stash(struct libeth_xdp_buff_stash *dst,
1166 const struct libeth_xdp_buff *src);
1167 void __libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash);
1168
1169 /**
1170 * libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll
1171 * @dst: onstack buffer to initialize
1172 * @src: XDP buffer stash placed on the queue
1173 * @rxq: registered &xdp_rxq_info corresponding to this queue
1174 *
1175 * Should be called before the main NAPI polling loop. Loads the content of
1176 * the previously saved stash or initializes the buffer from scratch.
1177 * Do not use for XSk.
1178 */
1179 static inline void
libeth_xdp_init_buff(struct libeth_xdp_buff * dst,const struct libeth_xdp_buff_stash * src,struct xdp_rxq_info * rxq)1180 libeth_xdp_init_buff(struct libeth_xdp_buff *dst,
1181 const struct libeth_xdp_buff_stash *src,
1182 struct xdp_rxq_info *rxq)
1183 {
1184 if (likely(!src->data))
1185 dst->data = NULL;
1186 else
1187 libeth_xdp_load_stash(dst, src);
1188
1189 dst->base.rxq = rxq;
1190 }
1191
1192 /**
1193 * libeth_xdp_save_buff - save a partially built buffer on a queue
1194 * @dst: XDP buffer stash placed on the queue
1195 * @src: onstack buffer to save
1196 *
1197 * Should be called after the main NAPI polling loop. If the loop exited before
1198 * the buffer was finished, saves its content on the queue, so that it can be
1199 * completed during the next poll. Otherwise, clears the stash.
1200 */
libeth_xdp_save_buff(struct libeth_xdp_buff_stash * dst,const struct libeth_xdp_buff * src)1201 static inline void libeth_xdp_save_buff(struct libeth_xdp_buff_stash *dst,
1202 const struct libeth_xdp_buff *src)
1203 {
1204 if (likely(!src->data))
1205 dst->data = NULL;
1206 else
1207 libeth_xdp_save_stash(dst, src);
1208 }
1209
1210 /**
1211 * libeth_xdp_return_stash - free an XDP buffer stash from a queue
1212 * @stash: stash to free
1213 *
1214 * If the queue is about to be destroyed, but it still has an incompleted
1215 * buffer stash, this helper should be called to free it.
1216 */
libeth_xdp_return_stash(struct libeth_xdp_buff_stash * stash)1217 static inline void libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash)
1218 {
1219 if (stash->data)
1220 __libeth_xdp_return_stash(stash);
1221 }
1222
libeth_xdp_return_va(const void * data,bool napi)1223 static inline void libeth_xdp_return_va(const void *data, bool napi)
1224 {
1225 netmem_ref netmem = virt_to_netmem(data);
1226
1227 page_pool_put_full_netmem(__netmem_get_pp(netmem), netmem, napi);
1228 }
1229
libeth_xdp_return_frags(const struct skb_shared_info * sinfo,bool napi)1230 static inline void libeth_xdp_return_frags(const struct skb_shared_info *sinfo,
1231 bool napi)
1232 {
1233 for (u32 i = 0; i < sinfo->nr_frags; i++) {
1234 netmem_ref netmem = skb_frag_netmem(&sinfo->frags[i]);
1235
1236 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, napi);
1237 }
1238 }
1239
1240 /**
1241 * libeth_xdp_return_buff - free/recycle &libeth_xdp_buff
1242 * @xdp: buffer to free
1243 *
1244 * Hotpath helper to free &libeth_xdp_buff. Comparing to xdp_return_buff(),
1245 * it's faster as it gets inlined and always assumes order-0 pages and safe
1246 * direct recycling. Zeroes @xdp->data to avoid UAFs.
1247 */
1248 #define libeth_xdp_return_buff(xdp) __libeth_xdp_return_buff(xdp, true)
1249
__libeth_xdp_return_buff(struct libeth_xdp_buff * xdp,bool napi)1250 static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp,
1251 bool napi)
1252 {
1253 if (!xdp_buff_has_frags(&xdp->base))
1254 goto out;
1255
1256 libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),
1257 napi);
1258
1259 out:
1260 libeth_xdp_return_va(xdp->data, napi);
1261 xdp->data = NULL;
1262 }
1263
1264 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1265 const struct libeth_fqe *fqe,
1266 u32 len);
1267
1268 /**
1269 * libeth_xdp_prepare_buff - fill &libeth_xdp_buff with head FQE data
1270 * @xdp: XDP buffer to attach the head to
1271 * @fqe: FQE containing the head buffer
1272 * @len: buffer len passed from HW
1273 *
1274 * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer
1275 * head with the Rx buffer data: data pointer, length, headroom, and
1276 * truesize/tailroom. Zeroes the flags.
1277 * Uses faster single u64 write instead of per-field access.
1278 */
libeth_xdp_prepare_buff(struct libeth_xdp_buff * xdp,const struct libeth_fqe * fqe,u32 len)1279 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp,
1280 const struct libeth_fqe *fqe,
1281 u32 len)
1282 {
1283 const struct page *page = __netmem_to_page(fqe->netmem);
1284
1285 #ifdef __LIBETH_WORD_ACCESS
1286 static_assert(offsetofend(typeof(xdp->base), flags) -
1287 offsetof(typeof(xdp->base), frame_sz) ==
1288 sizeof(u64));
1289
1290 *(u64 *)&xdp->base.frame_sz = fqe->truesize;
1291 #else
1292 xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);
1293 #endif
1294 xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,
1295 pp_page_to_nmdesc(page)->pp->p.offset, len, true);
1296 }
1297
1298 /**
1299 * libeth_xdp_process_buff - attach Rx buffer to &libeth_xdp_buff
1300 * @xdp: XDP buffer to attach the Rx buffer to
1301 * @fqe: Rx buffer to process
1302 * @len: received data length from the descriptor
1303 *
1304 * If the XDP buffer is empty, attaches the Rx buffer as head and initializes
1305 * the required fields. Otherwise, attaches the buffer as a frag.
1306 * Already performs DMA sync-for-CPU and frame start prefetch
1307 * (for head buffers only).
1308 *
1309 * Return: true on success, false if the descriptor must be skipped (empty or
1310 * no space for a new frag).
1311 */
libeth_xdp_process_buff(struct libeth_xdp_buff * xdp,const struct libeth_fqe * fqe,u32 len)1312 static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp,
1313 const struct libeth_fqe *fqe,
1314 u32 len)
1315 {
1316 if (!libeth_rx_sync_for_cpu(fqe, len))
1317 return false;
1318
1319 if (xdp->data)
1320 return libeth_xdp_buff_add_frag(xdp, fqe, len);
1321
1322 libeth_xdp_prepare_buff(xdp, fqe, len);
1323
1324 prefetch(xdp->data);
1325
1326 return true;
1327 }
1328
1329 /**
1330 * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
1331 * @ss: onstack stats to update
1332 * @xdp: buffer to account
1333 *
1334 * Internal helper used by __libeth_xdp_run_pass(), do not call directly.
1335 * Adds buffer's frags count and total len to the onstack stats.
1336 */
1337 static inline void
libeth_xdp_buff_stats_frags(struct libeth_rq_napi_stats * ss,const struct libeth_xdp_buff * xdp)1338 libeth_xdp_buff_stats_frags(struct libeth_rq_napi_stats *ss,
1339 const struct libeth_xdp_buff *xdp)
1340 {
1341 const struct skb_shared_info *sinfo;
1342
1343 sinfo = xdp_get_shared_info_from_buff(&xdp->base);
1344 ss->bytes += sinfo->xdp_frags_size;
1345 ss->fragments += sinfo->nr_frags + 1;
1346 }
1347
1348 u32 libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq,
1349 struct libeth_xdp_buff *xdp,
1350 enum xdp_action act, int ret);
1351
1352 /**
1353 * __libeth_xdp_run_prog - run XDP program on an XDP buffer
1354 * @xdp: XDP buffer to run the prog on
1355 * @bq: buffer bulk for ``XDP_TX`` queueing
1356 *
1357 * Internal inline abstraction to run XDP program. Handles ``XDP_DROP``
1358 * and ``XDP_REDIRECT`` only, the rest is processed levels up.
1359 * Reports an XDP prog exception on errors.
1360 *
1361 * Return: libeth_xdp prog verdict depending on the prog's verdict.
1362 */
1363 static __always_inline u32
__libeth_xdp_run_prog(struct libeth_xdp_buff * xdp,const struct libeth_xdp_tx_bulk * bq)1364 __libeth_xdp_run_prog(struct libeth_xdp_buff *xdp,
1365 const struct libeth_xdp_tx_bulk *bq)
1366 {
1367 enum xdp_action act;
1368
1369 act = bpf_prog_run_xdp(bq->prog, &xdp->base);
1370 if (unlikely(act < XDP_DROP || act > XDP_REDIRECT))
1371 goto out;
1372
1373 switch (act) {
1374 case XDP_PASS:
1375 return LIBETH_XDP_PASS;
1376 case XDP_DROP:
1377 libeth_xdp_return_buff(xdp);
1378
1379 return LIBETH_XDP_DROP;
1380 case XDP_TX:
1381 return LIBETH_XDP_TX;
1382 case XDP_REDIRECT:
1383 if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))
1384 break;
1385
1386 xdp->data = NULL;
1387
1388 return LIBETH_XDP_REDIRECT;
1389 default:
1390 break;
1391 }
1392
1393 out:
1394 return libeth_xdp_prog_exception(bq, xdp, act, 0);
1395 }
1396
1397 /**
1398 * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
1399 * @xdp: XDP buffer to run the prog on
1400 * @bq: buffer bulk for ``XDP_TX`` queueing
1401 * @run: internal callback for running XDP program
1402 * @queue: internal callback for queuing ``XDP_TX`` frame
1403 * @flush_bulk: driver callback for flushing a bulk
1404 *
1405 * Internal inline abstraction to run XDP program and additionally handle
1406 * ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue.
1407 * Do not use directly.
1408 *
1409 * Return: libeth_xdp prog verdict depending on the prog's verdict.
1410 */
1411 static __always_inline u32
__libeth_xdp_run_flush(struct libeth_xdp_buff * xdp,struct libeth_xdp_tx_bulk * bq,u32 (* run)(struct libeth_xdp_buff * xdp,const struct libeth_xdp_tx_bulk * bq),bool (* queue)(struct libeth_xdp_tx_bulk * bq,struct libeth_xdp_buff * xdp,bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags)),bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags))1412 __libeth_xdp_run_flush(struct libeth_xdp_buff *xdp,
1413 struct libeth_xdp_tx_bulk *bq,
1414 u32 (*run)(struct libeth_xdp_buff *xdp,
1415 const struct libeth_xdp_tx_bulk *bq),
1416 bool (*queue)(struct libeth_xdp_tx_bulk *bq,
1417 struct libeth_xdp_buff *xdp,
1418 bool (*flush_bulk)
1419 (struct libeth_xdp_tx_bulk *bq,
1420 u32 flags)),
1421 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1422 u32 flags))
1423 {
1424 u32 act;
1425
1426 act = run(xdp, bq);
1427 if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))
1428 act = LIBETH_XDP_DROP;
1429
1430 bq->act_mask |= act;
1431
1432 return act;
1433 }
1434
1435 /**
1436 * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
1437 * @xdp: XDP buffer to process
1438 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1439 * @fl: driver ``XDP_TX`` bulk flush callback
1440 *
1441 * Run the attached XDP program and handle all possible verdicts. XSk has its
1442 * own version.
1443 * Prefer using it via LIBETH_XDP_DEFINE_RUN{,_PASS,_PROG}().
1444 *
1445 * Return: true if the buffer should be passed up the stack, false if the poll
1446 * should go to the next buffer.
1447 */
1448 #define libeth_xdp_run_prog(xdp, bq, fl) \
1449 (__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \
1450 libeth_xdp_tx_queue_bulk, \
1451 fl) == LIBETH_XDP_PASS)
1452
1453 /**
1454 * __libeth_xdp_run_pass - helper to run XDP program and handle the result
1455 * @xdp: XDP buffer to process
1456 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1457 * @napi: NAPI to build an skb and pass it up the stack
1458 * @rs: onstack libeth RQ stats
1459 * @md: metadata that should be filled to the XDP buffer
1460 * @prep: callback for filling the metadata
1461 * @run: driver wrapper to run XDP program
1462 * @populate: driver callback to populate an skb with the HW descriptor data
1463 *
1464 * Inline abstraction that does the following (non-XSk path):
1465 * 1) adds frame size and frag number (if needed) to the onstack stats;
1466 * 2) fills the descriptor metadata to the onstack &libeth_xdp_buff
1467 * 3) runs XDP program if present;
1468 * 4) handles all possible verdicts;
1469 * 5) on ``XDP_PASS`, builds an skb from the buffer;
1470 * 6) populates it with the descriptor metadata;
1471 * 7) passes it up the stack.
1472 *
1473 * In most cases, number 2 means just writing the pointer to the HW descriptor
1474 * to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}()
1475 * wrappers to build a driver function.
1476 */
1477 static __always_inline void
__libeth_xdp_run_pass(struct libeth_xdp_buff * xdp,struct libeth_xdp_tx_bulk * bq,struct napi_struct * napi,struct libeth_rq_napi_stats * rs,const void * md,void (* prep)(struct libeth_xdp_buff * xdp,const void * md),bool (* run)(struct libeth_xdp_buff * xdp,struct libeth_xdp_tx_bulk * bq),bool (* populate)(struct sk_buff * skb,const struct libeth_xdp_buff * xdp,struct libeth_rq_napi_stats * rs))1478 __libeth_xdp_run_pass(struct libeth_xdp_buff *xdp,
1479 struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi,
1480 struct libeth_rq_napi_stats *rs, const void *md,
1481 void (*prep)(struct libeth_xdp_buff *xdp,
1482 const void *md),
1483 bool (*run)(struct libeth_xdp_buff *xdp,
1484 struct libeth_xdp_tx_bulk *bq),
1485 bool (*populate)(struct sk_buff *skb,
1486 const struct libeth_xdp_buff *xdp,
1487 struct libeth_rq_napi_stats *rs))
1488 {
1489 struct sk_buff *skb;
1490
1491 rs->bytes += xdp->base.data_end - xdp->data;
1492 rs->packets++;
1493
1494 if (xdp_buff_has_frags(&xdp->base))
1495 libeth_xdp_buff_stats_frags(rs, xdp);
1496
1497 if (prep && (!__builtin_constant_p(!!md) || md))
1498 prep(xdp, md);
1499
1500 if (!bq || !run || !bq->prog)
1501 goto build;
1502
1503 if (!run(xdp, bq))
1504 return;
1505
1506 build:
1507 skb = xdp_build_skb_from_buff(&xdp->base);
1508 if (unlikely(!skb)) {
1509 libeth_xdp_return_buff_slow(xdp);
1510 return;
1511 }
1512
1513 xdp->data = NULL;
1514
1515 if (unlikely(!populate(skb, xdp, rs))) {
1516 napi_consume_skb(skb, true);
1517 return;
1518 }
1519
1520 napi_gro_receive(napi, skb);
1521 }
1522
libeth_xdp_prep_desc(struct libeth_xdp_buff * xdp,const void * desc)1523 static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp,
1524 const void *desc)
1525 {
1526 xdp->desc = desc;
1527 }
1528
1529 /**
1530 * libeth_xdp_run_pass - helper to run XDP program and handle the result
1531 * @xdp: XDP buffer to process
1532 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1533 * @napi: NAPI to build an skb and pass it up the stack
1534 * @ss: onstack libeth RQ stats
1535 * @desc: pointer to the HW descriptor for that frame
1536 * @run: driver wrapper to run XDP program
1537 * @populate: driver callback to populate an skb with the HW descriptor data
1538 *
1539 * Wrapper around the underscored version when "fill the descriptor metadata"
1540 * means just writing the pointer to the HW descriptor as @xdp->desc.
1541 */
1542 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \
1543 __libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \
1544 run, populate)
1545
1546 /**
1547 * libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk)
1548 * @bq: ``XDP_TX`` frame bulk
1549 * @flush: driver callback to flush the bulk
1550 * @finalize: driver callback to start sending the frames and run the timer
1551 *
1552 * Flush the bulk if there are frames left to send, kick the queue and flush
1553 * the XDP maps.
1554 */
1555 #define libeth_xdp_finalize_rx(bq, flush, finalize) \
1556 __libeth_xdp_finalize_rx(bq, 0, flush, finalize)
1557
1558 static __always_inline void
__libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk * bq,u32 flags,bool (* flush_bulk)(struct libeth_xdp_tx_bulk * bq,u32 flags),void (* finalize)(void * xdpsq,bool sent,bool flush))1559 __libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk *bq, u32 flags,
1560 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1561 u32 flags),
1562 void (*finalize)(void *xdpsq, bool sent, bool flush))
1563 {
1564 if (bq->act_mask & LIBETH_XDP_TX) {
1565 if (bq->count)
1566 flush_bulk(bq, flags | LIBETH_XDP_TX_DROP);
1567 finalize(bq->xdpsq, true, true);
1568 }
1569 if (bq->act_mask & LIBETH_XDP_REDIRECT)
1570 xdp_do_flush();
1571
1572 rcu_read_unlock();
1573 }
1574
1575 /*
1576 * Helpers to reduce boilerplate code in drivers.
1577 *
1578 * Typical driver Rx flow would be (excl. bulk and buff init, frag attach):
1579 *
1580 * LIBETH_XDP_DEFINE_START();
1581 * LIBETH_XDP_DEFINE_FLUSH_TX(static driver_xdp_flush_tx, driver_xdp_tx_prep,
1582 * driver_xdp_xmit);
1583 * LIBETH_XDP_DEFINE_RUN(static driver_xdp_run, driver_xdp_run_prog,
1584 * driver_xdp_flush_tx, driver_populate_skb);
1585 * LIBETH_XDP_DEFINE_FINALIZE(static driver_xdp_finalize_rx,
1586 * driver_xdp_flush_tx, driver_xdp_finalize_sq);
1587 * LIBETH_XDP_DEFINE_END();
1588 *
1589 * This will build a set of 4 static functions. The compiler is free to decide
1590 * whether to inline them.
1591 * Then, in the NAPI polling function:
1592 *
1593 * while (packets < budget) {
1594 * // ...
1595 * driver_xdp_run(xdp, &bq, napi, &rs, desc);
1596 * }
1597 * driver_xdp_finalize_rx(&bq);
1598 */
1599
1600 #define LIBETH_XDP_DEFINE_START() \
1601 __diag_push(); \
1602 __diag_ignore(GCC, 8, "-Wold-style-declaration", \
1603 "Allow specifying \'static\' after the return type")
1604
1605 /**
1606 * LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback
1607 * @name: name of the function to define
1608 * @poll: Tx polling/completion function
1609 */
1610 #define LIBETH_XDP_DEFINE_TIMER(name, poll) \
1611 void name(struct work_struct *work) \
1612 { \
1613 libeth_xdpsq_run_timer(work, poll); \
1614 }
1615
1616 /**
1617 * LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function
1618 * @name: name of the function to define
1619 * @prep: driver callback to clean an XDPSQ
1620 * @xmit: driver callback to write a HW Tx descriptor
1621 */
1622 #define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit) \
1623 __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1624
1625 #define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx) \
1626 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
1627 { \
1628 return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit); \
1629 }
1630
1631 /**
1632 * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
1633 * @name: name of the function to define
1634 * @prep: driver callback to clean an XDPSQ
1635 * @xmit: driver callback to write a HW Tx descriptor
1636 */
1637 #define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit) \
1638 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
1639 { \
1640 return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit); \
1641 }
1642
1643 /**
1644 * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
1645 * @name: name of the function to define
1646 * @flush: driver callback to flush an ``XDP_TX`` bulk
1647 */
1648 #define LIBETH_XDP_DEFINE_RUN_PROG(name, flush) \
1649 bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1650
1651 #define __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, pfx) \
1652 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \
1653 { \
1654 return libeth_##pfx##_run_prog(xdp, bq, flush); \
1655 }
1656
1657 /**
1658 * LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function
1659 * @name: name of the function to define
1660 * @run: driver callback to run XDP program (above)
1661 * @populate: driver callback to fill an skb with HW descriptor info
1662 */
1663 #define LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate) \
1664 void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1665
1666 #define __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, pfx) \
1667 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \
1668 struct napi_struct *napi, struct libeth_rq_napi_stats *ss, \
1669 const void *desc) \
1670 { \
1671 return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \
1672 populate); \
1673 }
1674
1675 /**
1676 * LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function
1677 * @name: name of the function to define
1678 * @run: name of the XDP prog run function to define
1679 * @flush: driver callback to flush an ``XDP_TX`` bulk
1680 * @populate: driver callback to fill an skb with HW descriptor info
1681 */
1682 #define LIBETH_XDP_DEFINE_RUN(name, run, flush, populate) \
1683 __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP)
1684
1685 #define __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, pfx) \
1686 LIBETH_##pfx##_DEFINE_RUN_PROG(static run, flush); \
1687 LIBETH_##pfx##_DEFINE_RUN_PASS(name, run, populate)
1688
1689 /**
1690 * LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function
1691 * @name: name of the function to define
1692 * @flush: driver callback to flush an ``XDP_TX`` bulk
1693 * @finalize: driver callback to finalize an XDPSQ and run the timer
1694 */
1695 #define LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize) \
1696 __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1697
1698 #define __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, pfx) \
1699 void name(struct libeth_xdp_tx_bulk *bq) \
1700 { \
1701 libeth_##pfx##_finalize_rx(bq, flush, finalize); \
1702 }
1703
1704 #define LIBETH_XDP_DEFINE_END() __diag_pop()
1705
1706 /* XMO */
1707
1708 /**
1709 * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
1710 * @xdp: &libeth_xdp_buff corresponding to the queue
1711 * @type: typeof() of the driver Rx queue structure
1712 * @member: name of &xdp_rxq_info inside @type
1713 *
1714 * Often times, pointer to the RQ is needed when reading/filling metadata from
1715 * HW descriptors. The helper can be used to quickly jump from an XDP buffer
1716 * to the queue corresponding to its &xdp_rxq_info without introducing
1717 * additional fields (&libeth_xdp_buff is precisely 1 cacheline long on x64).
1718 */
1719 #define libeth_xdp_buff_to_rq(xdp, type, member) \
1720 container_of_const((xdp)->base.rxq, type, member)
1721
1722 /**
1723 * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
1724 * @hash: pointer to the variable to write the hash to
1725 * @rss_type: pointer to the variable to write the hash type to
1726 * @val: hash value from the HW descriptor
1727 * @pt: libeth parsed packet type
1728 *
1729 * Handle zeroed/non-available hash and convert libeth parsed packet type to
1730 * the corresponding XDP RSS hash type. To be called at the end of
1731 * xdp_metadata_ops idpf_xdpmo::xmo_rx_hash() implementation.
1732 * Note that if the driver doesn't use a constant packet type lookup table but
1733 * generates it at runtime, it must call libeth_rx_pt_gen_hash_type(pt) to
1734 * generate XDP RSS hash type for each packet type.
1735 *
1736 * Return: 0 on success, -ENODATA when the hash is not available.
1737 */
libeth_xdpmo_rx_hash(u32 * hash,enum xdp_rss_hash_type * rss_type,u32 val,struct libeth_rx_pt pt)1738 static inline int libeth_xdpmo_rx_hash(u32 *hash,
1739 enum xdp_rss_hash_type *rss_type,
1740 u32 val, struct libeth_rx_pt pt)
1741 {
1742 if (unlikely(!val))
1743 return -ENODATA;
1744
1745 *hash = val;
1746 *rss_type = pt.hash_type;
1747
1748 return 0;
1749 }
1750
1751 /* Tx buffer completion */
1752
1753 void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo,
1754 struct xdp_frame_bulk *bq, bool frags);
1755 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);
1756
1757 /**
1758 * __libeth_xdp_complete_tx - complete sent XDPSQE
1759 * @sqe: SQ element / Tx buffer to complete
1760 * @cp: Tx polling/completion params
1761 * @bulk: internal callback to bulk-free ``XDP_TX`` buffers
1762 * @xsk: internal callback to free XSk ``XDP_TX`` buffers
1763 *
1764 * Use the non-underscored version in drivers instead. This one is shared
1765 * internally with libeth_tx_complete_any().
1766 * Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping
1767 * when needed, buffer freeing, stats update, and SQE invalidation.
1768 */
1769 static __always_inline void
__libeth_xdp_complete_tx(struct libeth_sqe * sqe,struct libeth_cq_pp * cp,typeof(libeth_xdp_return_buff_bulk) bulk,typeof(libeth_xsk_buff_free_slow) xsk)1770 __libeth_xdp_complete_tx(struct libeth_sqe *sqe, struct libeth_cq_pp *cp,
1771 typeof(libeth_xdp_return_buff_bulk) bulk,
1772 typeof(libeth_xsk_buff_free_slow) xsk)
1773 {
1774 enum libeth_sqe_type type = sqe->type;
1775
1776 switch (type) {
1777 case LIBETH_SQE_EMPTY:
1778 return;
1779 case LIBETH_SQE_XDP_XMIT:
1780 case LIBETH_SQE_XDP_XMIT_FRAG:
1781 dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma),
1782 dma_unmap_len(sqe, len), DMA_TO_DEVICE);
1783 break;
1784 default:
1785 break;
1786 }
1787
1788 switch (type) {
1789 case LIBETH_SQE_XDP_TX:
1790 bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1);
1791 break;
1792 case LIBETH_SQE_XDP_XMIT:
1793 xdp_return_frame_bulk(sqe->xdpf, cp->bq);
1794 break;
1795 case LIBETH_SQE_XSK_TX:
1796 case LIBETH_SQE_XSK_TX_FRAG:
1797 xsk(sqe->xsk);
1798 break;
1799 default:
1800 break;
1801 }
1802
1803 switch (type) {
1804 case LIBETH_SQE_XDP_TX:
1805 case LIBETH_SQE_XDP_XMIT:
1806 case LIBETH_SQE_XSK_TX:
1807 cp->xdp_tx -= sqe->nr_frags;
1808
1809 cp->xss->packets++;
1810 cp->xss->bytes += sqe->bytes;
1811 break;
1812 default:
1813 break;
1814 }
1815
1816 sqe->type = LIBETH_SQE_EMPTY;
1817 }
1818
libeth_xdp_complete_tx(struct libeth_sqe * sqe,struct libeth_cq_pp * cp)1819 static inline void libeth_xdp_complete_tx(struct libeth_sqe *sqe,
1820 struct libeth_cq_pp *cp)
1821 {
1822 __libeth_xdp_complete_tx(sqe, cp, libeth_xdp_return_buff_bulk,
1823 libeth_xsk_buff_free_slow);
1824 }
1825
1826 /* Misc */
1827
1828 u32 libeth_xdp_queue_threshold(u32 count);
1829
1830 void __libeth_xdp_set_features(struct net_device *dev,
1831 const struct xdp_metadata_ops *xmo,
1832 u32 zc_segs,
1833 const struct xsk_tx_metadata_ops *tmo);
1834 void libeth_xdp_set_redirect(struct net_device *dev, bool enable);
1835
1836 /**
1837 * libeth_xdp_set_features - set XDP features for netdev
1838 * @dev: &net_device to configure
1839 * @...: optional params, see __libeth_xdp_set_features()
1840 *
1841 * Set all the features libeth_xdp supports, including .ndo_xdp_xmit(). That
1842 * said, it should be used only when XDPSQs are always available regardless
1843 * of whether an XDP prog is attached to @dev.
1844 */
1845 #define libeth_xdp_set_features(dev, ...) \
1846 CONCATENATE(__libeth_xdp_feat, \
1847 COUNT_ARGS(__VA_ARGS__))(dev, ##__VA_ARGS__)
1848
1849 #define __libeth_xdp_feat0(dev) \
1850 __libeth_xdp_set_features(dev, NULL, 0, NULL)
1851 #define __libeth_xdp_feat1(dev, xmo) \
1852 __libeth_xdp_set_features(dev, xmo, 0, NULL)
1853 #define __libeth_xdp_feat2(dev, xmo, zc_segs) \
1854 __libeth_xdp_set_features(dev, xmo, zc_segs, NULL)
1855 #define __libeth_xdp_feat3(dev, xmo, zc_segs, tmo) \
1856 __libeth_xdp_set_features(dev, xmo, zc_segs, tmo)
1857
1858 /**
1859 * libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir
1860 * @dev: target &net_device
1861 * @...: optional params, see __libeth_xdp_set_features()
1862 *
1863 * Enable everything except the .ndo_xdp_xmit() feature, use when XDPSQs are
1864 * not available right after netdev registration.
1865 */
1866 #define libeth_xdp_set_features_noredir(dev, ...) \
1867 __libeth_xdp_set_features_noredir(dev, __UNIQUE_ID(dev_), \
1868 ##__VA_ARGS__)
1869
1870 #define __libeth_xdp_set_features_noredir(dev, ud, ...) do { \
1871 struct net_device *ud = (dev); \
1872 \
1873 libeth_xdp_set_features(ud, ##__VA_ARGS__); \
1874 libeth_xdp_set_redirect(ud, false); \
1875 } while (0)
1876
1877 #define libeth_xsktmo ((const void *)GOLDEN_RATIO_PRIME)
1878
1879 #endif /* __LIBETH_XDP_H */
1880