Lines Matching +full:num +full:- +full:tx +full:- +full:queues
1 /* SPDX-License-Identifier: GPL-2.0-only */
11 #include <net/libeth/tx.h>
16 * Also used as internal return values on Tx.
28 * pick maximum pointer-compatible alignment.
36 * struct libeth_xdp_buff - libeth extension over &xdp_buff
40 * @priv: driver-private scratchspace
43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
45 * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk.
65 * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
67 * @...: sizeof() of the driver-private data
72 * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
74 * @...: type or variable name of the driver-private data
105 * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys
106 * @rxq: current number of active Rx queues
107 * @txq: current number of active Tx queues
108 * @max: maximum number of Tx queues
114 * Return: number of XDP Tx queues the device needs to use.
118 return min(max(nr_cpu_ids, rxq), max - txq);
122 * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs
123 * @num: number of active XDPSQs
127 static inline bool libeth_xdpsq_shared(u32 num)
129 return num < nr_cpu_ids;
133 * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
134 * @num: number of active XDPSQs
140 static inline u32 libeth_xdpsq_id(u32 num)
145 libeth_xdpsq_shared(num))
146 ret %= num;
157 * libeth_xdpsq_get - initialize &libeth_xdpsq_lock
174 * libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock
184 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
192 * libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed
200 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
205 * libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed
213 if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
217 /* XDPSQ clean-up timers */
224 * libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer
231 cancel_delayed_work_sync(&timer->dwork);
235 * libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer
240 * second (-> lazy cleaning won't happen).
247 &timer->dwork, HZ);
251 * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event
253 * @poll: driver-specific completion queue poll function
266 libeth_xdpsq_lock(timer->lock);
268 if (poll(timer->xdpsq, U32_MAX))
271 libeth_xdpsq_unlock(timer->lock);
274 /* Common Tx bits */
277 * enum - libeth_xdp internal Tx flags
294 * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags
317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
321 * @frag: one (non-head) frag for ``XDP_TX``
323 * @dma: DMA address of the non-head frag for .ndo_xdp_xmit()
326 * @flags: Tx flags for the above
368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
371 * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure
374 * @bulk: array of queued frames for bulk Tx
376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
380 * For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly
395 * LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack
406 * struct libeth_xdpsq - abstraction for an XDPSQ
408 * @sqes: array of Tx buffers from the actual queue struct
412 * @pending: pointer to the number of sent-not-completed descs on that queue
413 * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames
416 * Abstraction for driver-independent implementation of Tx. Placed on the stack
418 * functions can access and modify driver-specific resources.
434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
437 * @flags: XDP Tx flags
440 * Filled by the generic functions and then passed to driver-specific functions
441 * to fill a HW Tx descriptor, always placed on the [function] stack.
455 * libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv
466 * libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer
479 * On 64-bit systems, assigning one u64 is faster than two u32s. When ::len
494 * libeth_xdp_tx_xmit_bulk - main XDP Tx function
496 * @xdpsq: pointer to the driver-specific XDPSQ struct
499 * @priv: driver-specific private data
504 * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
536 this = sq.count - ntu;
547 u32 base = ntu + i - off;
558 xmit(fill(bulk[i], ntu + i - off, &sq, priv),
559 ntu + i - off, &sq, priv);
569 this = n - i;
592 * libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head
593 * @bq: XDP Tx bulk to queue the head frag to
601 const struct xdp_buff *base = &xdp->base;
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
604 .data = xdp->data,
605 .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,
606 .soff = xdp_data_hard_end(base) - xdp->data,
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI;
618 * libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag
619 * @bq: XDP Tx bulk to queue the frag to
625 bq->bulk[bq->count++].frag = *frag;
629 * libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame
630 * @bq: XDP Tx bulk to queue the frame to
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
655 sinfo = xdp_get_shared_info_from_buff(&xdp->base);
656 nr_frags = sinfo->nr_frags;
659 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
665 libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]);
669 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST;
670 xdp->data = NULL;
676 * libeth_xdp_tx_fill_stats - fill &libeth_sqe with ``XDP_TX`` frame stats
678 * @desc: libeth_xdp Tx descriptor
693 ue->nr_frags = 1; \
694 ue->bytes = ud->len; \
696 if (ud->flags & LIBETH_XDP_TX_MULTI) { \
698 ue->nr_frags += us->nr_frags; \
699 ue->bytes += us->xdp_frags_size; \
704 * libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe
705 * @frm: XDP Tx frame from the bulk
710 * Return: XDP Tx descriptor with the synced DMA and other info to pass to
740 dma_sync_single_for_device(__netmem_get_pp(netmem)->p.dev, desc.addr,
746 sqe = &sq->sqes[i];
747 sqe->type = LIBETH_SQE_XDP_TX;
748 sqe->sinfo = sinfo;
758 * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
760 * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
761 * @prep: driver-specific callback to prepare the queue for sending
783 sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
784 min(bq->count, LIBETH_XDP_TX_BULK),
786 drops = bq->count - sent;
790 err = -ENXIO;
792 bq->count = 0;
795 trace_xdp_bulk_tx(bq->dev, sent, drops, err);
801 * libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk
803 * @flags: Tx flags, see above
817 * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
820 * @xdpsqs: array of driver-specific XDPSQ structs
821 * @num: number of active XDPSQs (the above array length)
823 #define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \
824 __libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
830 bq->dev = dev;
831 bq->xdpsq = xdpsq;
832 bq->count = 0;
836 * libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame
865 * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
866 * @bq: XDP Tx bulk to queue the head frag to
880 dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
886 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
888 __libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST),
894 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI;
900 * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
901 * @bq: XDP Tx bulk to queue the frag to
917 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
926 * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
927 * @bq: XDP Tx bulk to queue the frame to
942 struct device *dev = bq->dev->dev.parent;
945 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
956 nr_frags = sinfo->nr_frags;
959 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
963 if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev))
971 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST;
977 * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
978 * @frm: XDP Tx frame from the bulk
983 * Return: XDP Tx descriptor with the mapped DMA and other info to pass to
1003 sqe = &sq->sqes[i];
1008 sqe->type = LIBETH_SQE_XDP_XMIT_FRAG;
1012 sqe->type = LIBETH_SQE_XDP_XMIT;
1013 sqe->xdpf = xdpf;
1021 * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
1023 * @flags: Tx flags, see __libeth_xdp_tx_flush_bulk()
1038 * __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit()
1039 * @bq: XDP Tx bulk to queue frames to
1044 * @finalize: driver callback to finalize sending XDP Tx frames on the queue
1049 * Return: number of frames send or -errno on error.
1061 return -EINVAL;
1075 if (bq->count) {
1077 if (unlikely(bq->count))
1078 nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk,
1079 bq->count,
1080 bq->dev);
1083 finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH);
1089 * libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver
1102 * Return: number of frames sent or -errno on error.
1120 ur = -ENXIO; \
1129 * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
1134 * @num: number of active XDPSQs, the above array length
1136 * Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
1137 * Initializes all the needed fields to run libeth_xdp functions. If @num == 0,
1141 #define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \
1142 __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \
1145 #define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \
1147 u32 un = (num); \
1152 ub->prog = rcu_dereference(pr); \
1153 ub->dev = (d); \
1154 ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \
1156 ub->prog = NULL; \
1159 ub->act_mask = 0; \
1160 ub->count = 0; \
1170 * libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll
1184 if (likely(!src->data))
1185 dst->data = NULL;
1189 dst->base.rxq = rxq;
1193 * libeth_xdp_save_buff - save a partially built buffer on a queue
1204 if (likely(!src->data))
1205 dst->data = NULL;
1211 * libeth_xdp_return_stash - free an XDP buffer stash from a queue
1219 if (stash->data)
1233 for (u32 i = 0; i < sinfo->nr_frags; i++) {
1234 netmem_ref netmem = skb_frag_netmem(&sinfo->frags[i]);
1241 * libeth_xdp_return_buff - free/recycle &libeth_xdp_buff
1245 * it's faster as it gets inlined and always assumes order-0 pages and safe
1246 * direct recycling. Zeroes @xdp->data to avoid UAFs.
1253 if (!xdp_buff_has_frags(&xdp->base))
1256 libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),
1260 libeth_xdp_return_va(xdp->data, napi);
1261 xdp->data = NULL;
1269 * libeth_xdp_prepare_buff - fill &libeth_xdp_buff with head FQE data
1282 const struct page *page = __netmem_to_page(fqe->netmem);
1284 xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,
1285 pp_page_to_nmdesc(page)->pp->p.offset, len, true);
1286 xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);
1290 * libeth_xdp_process_buff - attach Rx buffer to &libeth_xdp_buff
1297 * Already performs DMA sync-for-CPU and frame start prefetch
1310 if (xdp->data)
1315 prefetch(xdp->data);
1321 * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
1334 sinfo = xdp_get_shared_info_from_buff(&xdp->base);
1335 ss->bytes += sinfo->xdp_frags_size;
1336 ss->fragments += sinfo->nr_frags + 1;
1344 * __libeth_xdp_run_prog - run XDP program on an XDP buffer
1360 act = bpf_prog_run_xdp(bq->prog, &xdp->base);
1374 if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))
1377 xdp->data = NULL;
1389 * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
1421 bq->act_mask |= act;
1427 * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
1429 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1445 * __libeth_xdp_run_pass - helper to run XDP program and handle the result
1447 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1455 * Inline abstraction that does the following (non-XSk path):
1482 rs->bytes += xdp->base.data_end - xdp->data;
1483 rs->packets++;
1485 if (xdp_buff_has_frags(&xdp->base))
1491 if (!bq || !run || !bq->prog)
1498 skb = xdp_build_skb_from_buff(&xdp->base);
1504 xdp->data = NULL;
1517 xdp->desc = desc;
1521 * libeth_xdp_run_pass - helper to run XDP program and handle the result
1523 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1531 * means just writing the pointer to the HW descriptor as @xdp->desc.
1538 * libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk)
1555 if (bq->act_mask & LIBETH_XDP_TX) {
1556 if (bq->count)
1558 finalize(bq->xdpsq, true, true);
1560 if (bq->act_mask & LIBETH_XDP_REDIRECT)
1593 __diag_ignore(GCC, 8, "-Wold-style-declaration", \
1597 * LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback
1599 * @poll: Tx polling/completion function
1608 * LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function
1611 * @xmit: driver callback to write a HW Tx descriptor
1623 * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
1626 * @xmit: driver callback to write a HW Tx descriptor
1635 * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
1649 * LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function
1667 * LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function
1681 * LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function
1700 * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
1711 container_of_const((xdp)->base.rxq, type, member)
1714 * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
1720 * Handle zeroed/non-available hash and convert libeth parsed packet type to
1727 * Return: 0 on success, -ENODATA when the hash is not available.
1734 return -ENODATA;
1742 /* Tx buffer completion */
1749 * __libeth_xdp_complete_tx - complete sent XDPSQE
1750 * @sqe: SQ element / Tx buffer to complete
1751 * @cp: Tx polling/completion params
1752 * @bulk: internal callback to bulk-free ``XDP_TX`` buffers
1755 * Use the non-underscored version in drivers instead. This one is shared
1765 enum libeth_sqe_type type = sqe->type;
1772 dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma),
1781 bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1);
1784 xdp_return_frame_bulk(sqe->xdpf, cp->bq);
1788 xsk(sqe->xsk);
1798 cp->xdp_tx -= sqe->nr_frags;
1800 cp->xss->packets++;
1801 cp->xss->bytes += sqe->bytes;
1807 sqe->type = LIBETH_SQE_EMPTY;
1828 * libeth_xdp_set_features - set XDP features for netdev
1850 * libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir