Lines Matching +full:xdp +full:- +full:rx +full:- +full:metadata
1 /* SPDX-License-Identifier: GPL-2.0-only */
10 #include <net/libeth/rx.h>
15 * Defined as bits to be able to use them as a mask on Rx.
28 * pick maximum pointer-compatible alignment.
36 * struct libeth_xdp_buff - libeth extension over &xdp_buff
39 * @desc: RQ descriptor containing metadata for this buffer
40 * @priv: driver-private scratchspace
43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
45 * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk.
58 offsetof(struct xdp_buff_xsk, xdp.data));
65 * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
67 * @...: sizeof() of the driver-private data
72 * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
74 * @...: type or variable name of the driver-private data
105 * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys
106 * @rxq: current number of active Rx queues
114 * Return: number of XDP Tx queues the device needs to use.
118 return min(max(nr_cpu_ids, rxq), max - txq); in libeth_xdpsq_num()
122 * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs
133 * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
157 * libeth_xdpsq_get - initialize &libeth_xdpsq_lock
174 * libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock
184 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) in libeth_xdpsq_put()
192 * libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed
200 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) in libeth_xdpsq_lock()
205 * libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed
213 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share) in libeth_xdpsq_unlock()
217 /* XDPSQ clean-up timers */
224 * libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer
231 cancel_delayed_work_sync(&timer->dwork); in libeth_xdpsq_deinit_timer()
235 * libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer
240 * second (-> lazy cleaning won't happen).
247 &timer->dwork, HZ); in libeth_xdpsq_queue_timer()
251 * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event
253 * @poll: driver-specific completion queue poll function
266 libeth_xdpsq_lock(timer->lock); in libeth_xdpsq_run_timer()
268 if (poll(timer->xdpsq, U32_MAX)) in libeth_xdpsq_run_timer()
271 libeth_xdpsq_unlock(timer->lock); in libeth_xdpsq_run_timer()
277 * enum - libeth_xdp internal Tx flags
294 * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags
297 * @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits
317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
321 * @frag: one (non-head) frag for ``XDP_TX``
323 * @dma: DMA address of the non-head frag for .ndo_xdp_xmit()
324 * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
369 * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
371 * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure
372 * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
395 * LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack
406 * struct libeth_xdpsq - abstraction for an XDPSQ
412 * @pending: pointer to the number of sent-not-completed descs on that queue
413 * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames
416 * Abstraction for driver-independent implementation of Tx. Placed on the stack
418 * functions can access and modify driver-specific resources.
434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
437 * @flags: XDP Tx flags
440 * Filled by the generic functions and then passed to driver-specific functions
455 * libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv
466 * libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer
479 * On 64-bit systems, assigning one u64 is faster than two u32s. When ::len
494 * libeth_xdp_tx_xmit_bulk - main XDP Tx function
496 * @xdpsq: pointer to the driver-specific XDPSQ struct
499 * @priv: driver-specific private data
504 * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
536 this = sq.count - ntu; in libeth_xdp_tx_xmit_bulk()
547 u32 base = ntu + i - off; in libeth_xdp_tx_xmit_bulk()
558 xmit(fill(bulk[i], ntu + i - off, &sq, priv), in libeth_xdp_tx_xmit_bulk()
559 ntu + i - off, &sq, priv); in libeth_xdp_tx_xmit_bulk()
569 this = n - i; in libeth_xdp_tx_xmit_bulk()
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
592 * libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head
593 * @bq: XDP Tx bulk to queue the head frag to
594 * @xdp: XDP buffer with the head to queue
599 const struct libeth_xdp_buff *xdp) in libeth_xdp_tx_queue_head() argument
601 const struct xdp_buff *base = &xdp->base; in libeth_xdp_tx_queue_head()
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xdp_tx_queue_head()
604 .data = xdp->data, in libeth_xdp_tx_queue_head()
605 .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST, in libeth_xdp_tx_queue_head()
606 .soff = xdp_data_hard_end(base) - xdp->data, in libeth_xdp_tx_queue_head()
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI; in libeth_xdp_tx_queue_head()
618 * libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag
619 * @bq: XDP Tx bulk to queue the frag to
625 bq->bulk[bq->count++].frag = *frag; in libeth_xdp_tx_queue_frag()
629 * libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame
630 * @bq: XDP Tx bulk to queue the frame to
631 * @xdp: XDP buffer to queue
638 struct libeth_xdp_buff *xdp, in libeth_xdp_tx_queue_bulk() argument
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_tx_queue_bulk()
648 libeth_xdp_return_buff_slow(xdp); in libeth_xdp_tx_queue_bulk()
652 if (!libeth_xdp_tx_queue_head(bq, xdp)) in libeth_xdp_tx_queue_bulk()
655 sinfo = xdp_get_shared_info_from_buff(&xdp->base); in libeth_xdp_tx_queue_bulk()
656 nr_frags = sinfo->nr_frags; in libeth_xdp_tx_queue_bulk()
659 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_tx_queue_bulk()
665 libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]); in libeth_xdp_tx_queue_bulk()
669 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST; in libeth_xdp_tx_queue_bulk()
670 xdp->data = NULL; in libeth_xdp_tx_queue_bulk()
676 * libeth_xdp_tx_fill_stats - fill &libeth_sqe with ``XDP_TX`` frame stats
693 ue->nr_frags = 1; \
694 ue->bytes = ud->len; \
696 if (ud->flags & LIBETH_XDP_TX_MULTI) { \
698 ue->nr_frags += us->nr_frags; \
699 ue->bytes += us->xdp_frags_size; \
704 * libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe
705 * @frm: XDP Tx frame from the bulk
710 * Return: XDP Tx descriptor with the synced DMA and other info to pass to
740 dma_sync_single_for_device(__netmem_get_pp(netmem)->p.dev, desc.addr, in libeth_xdp_tx_fill_buf()
746 sqe = &sq->sqes[i]; in libeth_xdp_tx_fill_buf()
747 sqe->type = LIBETH_SQE_XDP_TX; in libeth_xdp_tx_fill_buf()
748 sqe->sinfo = sinfo; in libeth_xdp_tx_fill_buf()
758 * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
760 * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
761 * @prep: driver-specific callback to prepare the queue for sending
783 sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq, in __libeth_xdp_tx_flush_bulk()
784 min(bq->count, LIBETH_XDP_TX_BULK), in __libeth_xdp_tx_flush_bulk()
786 drops = bq->count - sent; in __libeth_xdp_tx_flush_bulk()
790 err = -ENXIO; in __libeth_xdp_tx_flush_bulk()
792 bq->count = 0; in __libeth_xdp_tx_flush_bulk()
795 trace_xdp_bulk_tx(bq->dev, sent, drops, err); in __libeth_xdp_tx_flush_bulk()
801 * libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk
817 * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
820 * @xdpsqs: array of driver-specific XDPSQ structs
830 bq->dev = dev; in __libeth_xdp_xmit_init_bulk()
831 bq->xdpsq = xdpsq; in __libeth_xdp_xmit_init_bulk()
832 bq->count = 0; in __libeth_xdp_xmit_init_bulk()
836 * libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame
837 * @xf: pointer to the XDP frame
865 * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
866 * @bq: XDP Tx bulk to queue the head frag to
867 * @xdpf: XDP frame with the head to queue
880 dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); in libeth_xdp_xmit_queue_head()
886 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xdp_xmit_queue_head()
888 __libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST), in libeth_xdp_xmit_queue_head()
894 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; in libeth_xdp_xmit_queue_head()
900 * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
901 * @bq: XDP Tx bulk to queue the frag to
917 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xdp_xmit_queue_frag()
926 * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
927 * @bq: XDP Tx bulk to queue the frame to
928 * @xdpf: XDP frame to queue
942 struct device *dev = bq->dev->dev.parent; in libeth_xdp_xmit_queue_bulk()
945 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_xmit_queue_bulk()
956 nr_frags = sinfo->nr_frags; in libeth_xdp_xmit_queue_bulk()
959 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_xmit_queue_bulk()
963 if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev)) in libeth_xdp_xmit_queue_bulk()
971 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST; in libeth_xdp_xmit_queue_bulk()
977 * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
978 * @frm: XDP Tx frame from the bulk
983 * Return: XDP Tx descriptor with the mapped DMA and other info to pass to
1003 sqe = &sq->sqes[i]; in libeth_xdp_xmit_fill_buf()
1008 sqe->type = LIBETH_SQE_XDP_XMIT_FRAG; in libeth_xdp_xmit_fill_buf()
1012 sqe->type = LIBETH_SQE_XDP_XMIT; in libeth_xdp_xmit_fill_buf()
1013 sqe->xdpf = xdpf; in libeth_xdp_xmit_fill_buf()
1021 * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
1027 * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
1038 * __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit()
1039 * @bq: XDP Tx bulk to queue frames to
1040 * @frames: XDP frames passed by the stack
1043 * @flush_bulk: driver callback to flush an XDP xmit bulk
1044 * @finalize: driver callback to finalize sending XDP Tx frames on the queue
1049 * Return: number of frames send or -errno on error.
1061 return -EINVAL; in __libeth_xdp_xmit_do_bulk()
1075 if (bq->count) { in __libeth_xdp_xmit_do_bulk()
1077 if (unlikely(bq->count)) in __libeth_xdp_xmit_do_bulk()
1078 nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk, in __libeth_xdp_xmit_do_bulk()
1079 bq->count, in __libeth_xdp_xmit_do_bulk()
1080 bq->dev); in __libeth_xdp_xmit_do_bulk()
1083 finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH); in __libeth_xdp_xmit_do_bulk()
1089 * libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver
1092 * @fr: XDP frames to send
1096 * @fl: driver callback to flush an XDP xmit bulk
1102 * Return: number of frames sent or -errno on error.
1120 ur = -ENXIO; \
1126 /* Rx polling path */
1129 * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
1131 * @prog: RCU pointer to the XDP program (can be %NULL)
1136 * Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
1138 * assumes XDP is not enabled.
1152 ub->prog = rcu_dereference(pr); \
1153 ub->dev = (d); \
1154 ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \
1156 ub->prog = NULL; \
1159 ub->act_mask = 0; \
1160 ub->count = 0; \
1170 * libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll
1172 * @src: XDP buffer stash placed on the queue
1184 if (likely(!src->data)) in libeth_xdp_init_buff()
1185 dst->data = NULL; in libeth_xdp_init_buff()
1189 dst->base.rxq = rxq; in libeth_xdp_init_buff()
1193 * libeth_xdp_save_buff - save a partially built buffer on a queue
1194 * @dst: XDP buffer stash placed on the queue
1204 if (likely(!src->data)) in libeth_xdp_save_buff()
1205 dst->data = NULL; in libeth_xdp_save_buff()
1211 * libeth_xdp_return_stash - free an XDP buffer stash from a queue
1219 if (stash->data) in libeth_xdp_return_stash()
1233 for (u32 i = 0; i < sinfo->nr_frags; i++) { in libeth_xdp_return_frags()
1234 netmem_ref netmem = skb_frag_netmem(&sinfo->frags[i]); in libeth_xdp_return_frags()
1241 * libeth_xdp_return_buff - free/recycle &libeth_xdp_buff
1242 * @xdp: buffer to free
1245 * it's faster as it gets inlined and always assumes order-0 pages and safe
1246 * direct recycling. Zeroes @xdp->data to avoid UAFs.
1248 #define libeth_xdp_return_buff(xdp) __libeth_xdp_return_buff(xdp, true) argument
1250 static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp, in __libeth_xdp_return_buff() argument
1253 if (!xdp_buff_has_frags(&xdp->base)) in __libeth_xdp_return_buff()
1256 libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base), in __libeth_xdp_return_buff()
1260 libeth_xdp_return_va(xdp->data, napi); in __libeth_xdp_return_buff()
1261 xdp->data = NULL; in __libeth_xdp_return_buff()
1264 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1269 * libeth_xdp_prepare_buff - fill &libeth_xdp_buff with head FQE data
1270 * @xdp: XDP buffer to attach the head to
1274 * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer
1275 * head with the Rx buffer data: data pointer, length, headroom, and
1277 * Uses faster single u64 write instead of per-field access.
1279 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp, in libeth_xdp_prepare_buff() argument
1283 const struct page *page = __netmem_to_page(fqe->netmem); in libeth_xdp_prepare_buff()
1286 static_assert(offsetofend(typeof(xdp->base), flags) - in libeth_xdp_prepare_buff()
1287 offsetof(typeof(xdp->base), frame_sz) == in libeth_xdp_prepare_buff()
1290 *(u64 *)&xdp->base.frame_sz = fqe->truesize; in libeth_xdp_prepare_buff()
1292 xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq); in libeth_xdp_prepare_buff()
1294 xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset, in libeth_xdp_prepare_buff()
1295 pp_page_to_nmdesc(page)->pp->p.offset, len, true); in libeth_xdp_prepare_buff()
1299 * libeth_xdp_process_buff - attach Rx buffer to &libeth_xdp_buff
1300 * @xdp: XDP buffer to attach the Rx buffer to
1301 * @fqe: Rx buffer to process
1304 * If the XDP buffer is empty, attaches the Rx buffer as head and initializes
1306 * Already performs DMA sync-for-CPU and frame start prefetch
1312 static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp, in libeth_xdp_process_buff() argument
1319 if (xdp->data) in libeth_xdp_process_buff()
1320 return libeth_xdp_buff_add_frag(xdp, fqe, len); in libeth_xdp_process_buff()
1322 libeth_xdp_prepare_buff(xdp, fqe, len); in libeth_xdp_process_buff()
1324 prefetch(xdp->data); in libeth_xdp_process_buff()
1330 * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
1332 * @xdp: buffer to account
1339 const struct libeth_xdp_buff *xdp) in libeth_xdp_buff_stats_frags() argument
1343 sinfo = xdp_get_shared_info_from_buff(&xdp->base); in libeth_xdp_buff_stats_frags()
1344 ss->bytes += sinfo->xdp_frags_size; in libeth_xdp_buff_stats_frags()
1345 ss->fragments += sinfo->nr_frags + 1; in libeth_xdp_buff_stats_frags()
1349 struct libeth_xdp_buff *xdp,
1353 * __libeth_xdp_run_prog - run XDP program on an XDP buffer
1354 * @xdp: XDP buffer to run the prog on
1357 * Internal inline abstraction to run XDP program. Handles ``XDP_DROP``
1359 * Reports an XDP prog exception on errors.
1364 __libeth_xdp_run_prog(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_prog() argument
1369 act = bpf_prog_run_xdp(bq->prog, &xdp->base); in __libeth_xdp_run_prog()
1377 libeth_xdp_return_buff(xdp); in __libeth_xdp_run_prog()
1383 if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog))) in __libeth_xdp_run_prog()
1386 xdp->data = NULL; in __libeth_xdp_run_prog()
1394 return libeth_xdp_prog_exception(bq, xdp, act, 0); in __libeth_xdp_run_prog()
1398 * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
1399 * @xdp: XDP buffer to run the prog on
1401 * @run: internal callback for running XDP program
1405 * Internal inline abstraction to run XDP program and additionally handle
1406 * ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue.
1412 __libeth_xdp_run_flush(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush() argument
1414 u32 (*run)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush() argument
1417 struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush()
1426 act = run(xdp, bq); in __libeth_xdp_run_flush()
1427 if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk))) in __libeth_xdp_run_flush()
1430 bq->act_mask |= act; in __libeth_xdp_run_flush()
1436 * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
1437 * @xdp: XDP buffer to process
1438 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1441 * Run the attached XDP program and handle all possible verdicts. XSk has its
1448 #define libeth_xdp_run_prog(xdp, bq, fl) \ argument
1449 (__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \
1454 * __libeth_xdp_run_pass - helper to run XDP program and handle the result
1455 * @xdp: XDP buffer to process
1456 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1459 * @md: metadata that should be filled to the XDP buffer
1460 * @prep: callback for filling the metadata
1461 * @run: driver wrapper to run XDP program
1464 * Inline abstraction that does the following (non-XSk path):
1466 * 2) fills the descriptor metadata to the onstack &libeth_xdp_buff
1467 * 3) runs XDP program if present;
1470 * 6) populates it with the descriptor metadata;
1474 * to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}()
1478 __libeth_xdp_run_pass(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1481 void (*prep)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1483 bool (*run)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1486 const struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1491 rs->bytes += xdp->base.data_end - xdp->data; in __libeth_xdp_run_pass()
1492 rs->packets++; in __libeth_xdp_run_pass()
1494 if (xdp_buff_has_frags(&xdp->base)) in __libeth_xdp_run_pass()
1495 libeth_xdp_buff_stats_frags(rs, xdp); in __libeth_xdp_run_pass()
1498 prep(xdp, md); in __libeth_xdp_run_pass()
1500 if (!bq || !run || !bq->prog) in __libeth_xdp_run_pass()
1503 if (!run(xdp, bq)) in __libeth_xdp_run_pass()
1507 skb = xdp_build_skb_from_buff(&xdp->base); in __libeth_xdp_run_pass()
1509 libeth_xdp_return_buff_slow(xdp); in __libeth_xdp_run_pass()
1513 xdp->data = NULL; in __libeth_xdp_run_pass()
1515 if (unlikely(!populate(skb, xdp, rs))) { in __libeth_xdp_run_pass()
1523 static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp, in libeth_xdp_prep_desc() argument
1526 xdp->desc = desc; in libeth_xdp_prep_desc()
1530 * libeth_xdp_run_pass - helper to run XDP program and handle the result
1531 * @xdp: XDP buffer to process
1532 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1536 * @run: driver wrapper to run XDP program
1539 * Wrapper around the underscored version when "fill the descriptor metadata"
1540 * means just writing the pointer to the HW descriptor as @xdp->desc.
1542 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \ argument
1543 __libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \
1547 * libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk)
1553 * the XDP maps.
1564 if (bq->act_mask & LIBETH_XDP_TX) { in __libeth_xdp_finalize_rx()
1565 if (bq->count) in __libeth_xdp_finalize_rx()
1567 finalize(bq->xdpsq, true, true); in __libeth_xdp_finalize_rx()
1569 if (bq->act_mask & LIBETH_XDP_REDIRECT) in __libeth_xdp_finalize_rx()
1578 * Typical driver Rx flow would be (excl. bulk and buff init, frag attach):
1595 * driver_xdp_run(xdp, &bq, napi, &rs, desc);
1602 __diag_ignore(GCC, 8, "-Wold-style-declaration", \
1606 * LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback
1617 * LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function
1623 __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1632 * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
1644 * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
1649 bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1652 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \
1654 return libeth_##pfx##_run_prog(xdp, bq, flush); \
1658 * LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function
1660 * @run: driver callback to run XDP program (above)
1664 void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1667 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \
1671 return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \
1676 * LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function
1678 * @run: name of the XDP prog run function to define
1683 __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP)
1690 * LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function
1696 __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1709 * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
1710 * @xdp: &libeth_xdp_buff corresponding to the queue
1711 * @type: typeof() of the driver Rx queue structure
1714 * Often times, pointer to the RQ is needed when reading/filling metadata from
1715 * HW descriptors. The helper can be used to quickly jump from an XDP buffer
1719 #define libeth_xdp_buff_to_rq(xdp, type, member) \ argument
1720 container_of_const((xdp)->base.rxq, type, member)
1723 * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
1729 * Handle zeroed/non-available hash and convert libeth parsed packet type to
1730 * the corresponding XDP RSS hash type. To be called at the end of
1734 * generate XDP RSS hash type for each packet type.
1736 * Return: 0 on success, -ENODATA when the hash is not available.
1743 return -ENODATA; in libeth_xdpmo_rx_hash()
1755 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);
1758 * __libeth_xdp_complete_tx - complete sent XDPSQE
1761 * @bulk: internal callback to bulk-free ``XDP_TX`` buffers
1764 * Use the non-underscored version in drivers instead. This one is shared
1766 * Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping
1774 enum libeth_sqe_type type = sqe->type; in __libeth_xdp_complete_tx()
1781 dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma), in __libeth_xdp_complete_tx()
1790 bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1); in __libeth_xdp_complete_tx()
1793 xdp_return_frame_bulk(sqe->xdpf, cp->bq); in __libeth_xdp_complete_tx()
1797 xsk(sqe->xsk); in __libeth_xdp_complete_tx()
1807 cp->xdp_tx -= sqe->nr_frags; in __libeth_xdp_complete_tx()
1809 cp->xss->packets++; in __libeth_xdp_complete_tx()
1810 cp->xss->bytes += sqe->bytes; in __libeth_xdp_complete_tx()
1816 sqe->type = LIBETH_SQE_EMPTY; in __libeth_xdp_complete_tx()
1837 * libeth_xdp_set_features - set XDP features for netdev
1843 * of whether an XDP prog is attached to @dev.
1859 * libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir