Lines Matching +full:xdp +full:- +full:rx +full:- +full:metadata
1 /* SPDX-License-Identifier: GPL-2.0-only */
7 #include <net/libeth/xdp.h>
18 * libeth_xsk_tx_queue_head - internal helper for queueing XSk ``XDP_TX`` head
19 * @bq: XDP Tx bulk to queue the head frag to
20 * @xdp: XSk buffer with the head to queue
25 struct libeth_xdp_buff *xdp) in libeth_xsk_tx_queue_head() argument
27 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_head()
28 .xsk = xdp, in libeth_xsk_tx_queue_head()
29 __libeth_xdp_tx_len(xdp->base.data_end - xdp->data, in libeth_xsk_tx_queue_head()
33 if (likely(!xdp_buff_has_frags(&xdp->base))) in libeth_xsk_tx_queue_head()
36 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; in libeth_xsk_tx_queue_head()
42 * libeth_xsk_tx_queue_frag - internal helper for queueing XSk ``XDP_TX`` frag
43 * @bq: XDP Tx bulk to queue the frag to
49 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_frag()
51 __libeth_xdp_tx_len(frag->base.data_end - frag->data), in libeth_xsk_tx_queue_frag()
56 * libeth_xsk_tx_queue_bulk - internal helper for queueing XSk ``XDP_TX`` frame
57 * @bq: XDP Tx bulk to queue the frame to
58 * @xdp: XSk buffer to queue
65 struct libeth_xdp_buff *xdp, in libeth_xsk_tx_queue_bulk() argument
71 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xsk_tx_queue_bulk()
73 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_tx_queue_bulk()
77 if (!libeth_xsk_tx_queue_head(bq, xdp)) in libeth_xsk_tx_queue_bulk()
80 for (const struct libeth_xdp_buff *head = xdp; ; ) { in libeth_xsk_tx_queue_bulk()
81 xdp = container_of(xsk_buff_get_frag(&head->base), in libeth_xsk_tx_queue_bulk()
82 typeof(*xdp), base); in libeth_xsk_tx_queue_bulk()
83 if (!xdp) in libeth_xsk_tx_queue_bulk()
86 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xsk_tx_queue_bulk()
92 libeth_xsk_tx_queue_frag(bq, xdp); in libeth_xsk_tx_queue_bulk()
96 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST; in libeth_xsk_tx_queue_bulk()
102 * libeth_xsk_tx_fill_buf - internal helper to fill XSk ``XDP_TX`` &libeth_sqe
103 * @frm: XDP Tx frame from the bulk
108 * Return: XDP Tx descriptor with the synced DMA and other info to pass to
115 struct libeth_xdp_buff *xdp = frm.xsk; in libeth_xsk_tx_fill_buf() local
117 .addr = xsk_buff_xdp_get_dma(&xdp->base), in libeth_xsk_tx_fill_buf()
122 xsk_buff_raw_dma_sync_for_device(sq->pool, desc.addr, desc.len); in libeth_xsk_tx_fill_buf()
124 sqe = &sq->sqes[i]; in libeth_xsk_tx_fill_buf()
125 sqe->xsk = xdp; in libeth_xsk_tx_fill_buf()
128 sqe->type = LIBETH_SQE_XSK_TX_FRAG; in libeth_xsk_tx_fill_buf()
132 sqe->type = LIBETH_SQE_XSK_TX; in libeth_xsk_tx_fill_buf()
134 xdp_get_shared_info_from_buff(&xdp->base)); in libeth_xsk_tx_fill_buf()
140 * libeth_xsk_tx_flush_bulk - wrapper to define flush of XSk ``XDP_TX`` bulk
156 * libeth_xsktmo_req_csum - XSk Tx metadata op to request checksum offload
164 * Consider using within @libeth_xsktmo unless the driver requires HW-specific
170 ((struct libeth_xdp_tx_desc *)priv)->flags |= LIBETH_XDP_TX_CSUM; in libeth_xsktmo_req_csum()
179 * __libeth_xsk_xmit_fill_buf_md - internal helper to prepare XSk xmit w/meta
182 * @priv: XSk Tx metadata ops
184 * Same as __libeth_xsk_xmit_fill_buf(), but requests metadata pointer and
185 * fills additional fields in &libeth_xdp_tx_desc to ask for metadata offload.
187 * Return: XDP Tx descriptor with the DMA, metadata request bits, and other
199 ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); in __libeth_xsk_xmit_fill_buf_md()
202 __libeth_xdp_tx_len(xdesc->len), in __libeth_xsk_xmit_fill_buf_md()
216 * __libeth_xsk_xmit_fill_buf - internal helper to prepare XSk xmit w/o meta
220 * Return: XDP Tx descriptor with the DMA and other info to pass to
228 .addr = xsk_buff_raw_get_dma(sq->pool, xdesc->addr), in __libeth_xsk_xmit_fill_buf()
229 __libeth_xdp_tx_len(xdesc->len), in __libeth_xsk_xmit_fill_buf()
234 * libeth_xsk_xmit_fill_buf - internal helper to prepare an XSk xmit
238 * @priv: XSk Tx metadata ops
240 * Depending on the metadata ops presence (determined at compile time), calls
241 * the quickest helper to build a libeth XDP Tx descriptor.
243 * Return: XDP Tx descriptor with the synced DMA, metadata request bits,
259 xsk_buff_raw_dma_sync_for_device(sq->pool, desc.addr, desc.len); in libeth_xsk_xmit_fill_buf()
265 * libeth_xsk_xmit_do_bulk - send XSk xmit frames
269 * @tmo: optional XSk Tx metadata ops
278 * Note that unlike other XDP Tx ops, the queue must be locked and cleaned
301 bulk = container_of(&pool->tx_descs[0], typeof(*bulk), desc); in libeth_xsk_xmit_do_bulk()
314 /* Rx polling path */
317 * libeth_xsk_tx_init_bulk - initialize XDP Tx bulk for an XSk Rx NAPI poll
319 * @prog: RCU pointer to the XDP program (never %NULL)
324 * Should be called on an onstack XDP Tx bulk before the XSk NAPI polling loop.
326 * Never checks if @prog is %NULL or @num == 0 as XDP must always be enabled
334 struct libeth_xdp_buff *xdp);
337 * libeth_xsk_process_buff - attach XSk Rx buffer to &libeth_xdp_buff
339 * @xdp: XSk buffer to process
344 * Already performs DMA sync-for-CPU and frame start prefetch
352 struct libeth_xdp_buff *xdp, u32 len) in libeth_xsk_process_buff() argument
355 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_process_buff()
359 xsk_buff_set_size(&xdp->base, len); in libeth_xsk_process_buff()
360 xsk_buff_dma_sync_for_cpu(&xdp->base); in libeth_xsk_process_buff()
363 return libeth_xsk_buff_add_frag(head, xdp); in libeth_xsk_process_buff()
365 prefetch(xdp->data); in libeth_xsk_process_buff()
367 return xdp; in libeth_xsk_process_buff()
371 const struct libeth_xdp_buff *xdp);
373 u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp,
378 * __libeth_xsk_run_prog - run XDP program on XSk buffer
379 * @xdp: XSk buffer to run the prog on
382 * Internal inline abstraction to run XDP program on XSk Rx path. Handles
385 * Reports an XDP prog exception on errors.
390 __libeth_xsk_run_prog(struct libeth_xdp_buff *xdp, in __libeth_xsk_run_prog() argument
396 act = bpf_prog_run_xdp(bq->prog, &xdp->base); in __libeth_xsk_run_prog()
399 return __libeth_xsk_run_prog_slow(xdp, bq, act, ret); in __libeth_xsk_run_prog()
401 ret = xdp_do_redirect(bq->dev, &xdp->base, bq->prog); in __libeth_xsk_run_prog()
409 * libeth_xsk_run_prog - run XDP program on XSk path and handle all verdicts
410 * @xdp: XSk buffer to process
411 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
414 * Run the attached XDP program and handle all possible verdicts.
419 #define libeth_xsk_run_prog(xdp, bq, fl) \ argument
420 __libeth_xdp_run_flush(xdp, bq, __libeth_xsk_run_prog, \
424 * __libeth_xsk_run_pass - helper to run XDP program and handle the result
425 * @xdp: XSk buffer to process
426 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
429 * @md: metadata that should be filled to the XSk buffer
430 * @prep: callback for filling the metadata
431 * @run: driver wrapper to run XDP program
441 __libeth_xsk_run_pass(struct libeth_xdp_buff *xdp, in __libeth_xsk_run_pass() argument
444 void (*prep)(struct libeth_xdp_buff *xdp, in __libeth_xsk_run_pass() argument
446 u32 (*run)(struct libeth_xdp_buff *xdp, in __libeth_xsk_run_pass() argument
449 const struct libeth_xdp_buff *xdp, in __libeth_xsk_run_pass() argument
455 rs->bytes += xdp->base.data_end - xdp->data; in __libeth_xsk_run_pass()
456 rs->packets++; in __libeth_xsk_run_pass()
458 if (unlikely(xdp_buff_has_frags(&xdp->base))) in __libeth_xsk_run_pass()
459 libeth_xsk_buff_stats_frags(rs, xdp); in __libeth_xsk_run_pass()
462 prep(xdp, md); in __libeth_xsk_run_pass()
464 act = run(xdp, bq); in __libeth_xsk_run_pass()
471 skb = xdp_build_skb_from_zc(&xdp->base); in __libeth_xsk_run_pass()
473 libeth_xsk_buff_free_slow(xdp); in __libeth_xsk_run_pass()
477 if (unlikely(!populate(skb, xdp, rs))) { in __libeth_xsk_run_pass()
488 * libeth_xsk_run_pass - helper to run XDP program and handle the result
489 * @xdp: XSk buffer to process
490 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
494 * @run: driver wrapper to run XDP program
497 * Wrapper around the underscored version when "fill the descriptor metadata"
498 * means just writing the pointer to the HW descriptor as @xdp->desc.
500 #define libeth_xsk_run_pass(xdp, bq, napi, rs, desc, run, populate) \ argument
501 __libeth_xsk_run_pass(xdp, bq, napi, rs, desc, libeth_xdp_prep_desc, \
505 * libeth_xsk_finalize_rx - finalize XDPSQ after an XSk NAPI polling loop
511 * the XDP maps.
519 * Typical driver XSk Rx flow would be (excl. bulk and buff init, frag attach):
536 * if (!driver_xsk_run(xdp, &bq, napi, &rs, desc))
543 * LIBETH_XSK_DEFINE_FLUSH_TX - define a driver XSk ``XDP_TX`` flush function
552 * LIBETH_XSK_DEFINE_RUN_PROG - define a driver XDP program run function
560 * LIBETH_XSK_DEFINE_RUN_PASS - define a driver buffer process + pass function
562 * @run: driver callback to run XDP program (above)
569 * LIBETH_XSK_DEFINE_RUN - define a driver buffer process, run + pass function
571 * @run: name of the XDP prog run function to define
579 * LIBETH_XSK_DEFINE_FINALIZE - define a driver XSk NAPI poll finalize function
590 * struct libeth_xskfq - structure representing an XSk buffer (fill) queue
599 * @buf_len: HW-writeable length per each buffer
624 * libeth_xsk_buff_xdp_get_dma - get DMA address of XSk &libeth_xdp_buff
625 * @xdp: buffer to get the DMA addr for
627 #define libeth_xsk_buff_xdp_get_dma(xdp) \ argument
628 xsk_buff_xdp_get_dma(&(xdp)->base)
631 * libeth_xskfqe_alloc - allocate @n XSk Rx buffers
636 * Note that @fq->ntu gets updated, but ::pending must be recalculated
648 this = fq->count - fq->ntu; in libeth_xskfqe_alloc()
653 xskb = (typeof(xskb))&fq->fqes[fq->ntu]; in libeth_xskfqe_alloc()
654 ret = xsk_buff_alloc_batch(fq->pool, xskb, this); in libeth_xskfqe_alloc()
656 for (u32 i = 0, ntu = fq->ntu; likely(i < ret); i++) in libeth_xskfqe_alloc()
660 fq->ntu += ret; in libeth_xskfqe_alloc()
662 if (likely(fq->ntu < fq->count) || unlikely(ret < this)) in libeth_xskfqe_alloc()
665 fq->ntu = 0; in libeth_xskfqe_alloc()
668 this = n - this; in libeth_xskfqe_alloc()