1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP" 5 6 #include <linux/export.h> 7 8 #include <net/libeth/xdp.h> 9 10 /* ``XDP_TX`` bulking */ 11 12 static void __cold 13 libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm) 14 { 15 if (frm->len_fl & LIBETH_XDP_TX_MULTI) 16 libeth_xdp_return_frags(frm->data + frm->soff, true); 17 18 libeth_xdp_return_va(frm->data, true); 19 } 20 21 static void __cold 22 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) 23 { 24 for (u32 i = 0; i < count; i++) { 25 const struct libeth_xdp_tx_frame *frm = &bq[i]; 26 27 if (!(frm->len_fl & LIBETH_XDP_TX_FIRST)) 28 continue; 29 30 libeth_xdp_tx_return_one(frm); 31 } 32 } 33 34 static void __cold libeth_trace_xdp_exception(const struct net_device *dev, 35 const struct bpf_prog *prog, 36 u32 act) 37 { 38 trace_xdp_exception(dev, prog, act); 39 } 40 41 /** 42 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames 43 * @bq: XDP Tx frame bulk 44 * @sent: number of frames sent successfully (from this bulk) 45 * @flags: internal libeth_xdp flags (.ndo_xdp_xmit etc.) 46 * 47 * Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly. 48 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust 49 * the Tx bulk to try again later. 50 */ 51 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, 52 u32 flags) 53 { 54 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; 55 u32 left = bq->count - sent; 56 57 if (!(flags & LIBETH_XDP_TX_NDO)) 58 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); 59 60 if (!(flags & LIBETH_XDP_TX_DROP)) { 61 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); 62 bq->count = left; 63 64 return; 65 } 66 67 if (!(flags & LIBETH_XDP_TX_NDO)) 68 libeth_xdp_tx_return_bulk(pos, left); 69 else 70 libeth_xdp_xmit_return_bulk(pos, left, bq->dev); 71 72 bq->count = 0; 73 } 74 EXPORT_SYMBOL_GPL(libeth_xdp_tx_exception); 75 76 /* .ndo_xdp_xmit() implementation */ 77 78 u32 __cold libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, 79 u32 count, const struct net_device *dev) 80 { 81 u32 n = 0; 82 83 for (u32 i = 0; i < count; i++) { 84 const struct libeth_xdp_tx_frame *frm = &bq[i]; 85 dma_addr_t dma; 86 87 if (frm->flags & LIBETH_XDP_TX_FIRST) 88 dma = *libeth_xdp_xmit_frame_dma(frm->xdpf); 89 else 90 dma = dma_unmap_addr(frm, dma); 91 92 dma_unmap_page(dev->dev.parent, dma, dma_unmap_len(frm, len), 93 DMA_TO_DEVICE); 94 95 /* Actual xdp_frames are freed by the core */ 96 n += !!(frm->flags & LIBETH_XDP_TX_FIRST); 97 } 98 99 return n; 100 } 101 EXPORT_SYMBOL_GPL(libeth_xdp_xmit_return_bulk); 102 103 /* Rx polling path */ 104 105 /** 106 * libeth_xdp_return_buff_slow - free &libeth_xdp_buff 107 * @xdp: buffer to free/return 108 * 109 * Slowpath version of libeth_xdp_return_buff() to be called on exceptions, 110 * queue clean-ups etc., without unwanted inlining. 111 */ 112 void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp) 113 { 114 __libeth_xdp_return_buff(xdp, false); 115 } 116 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_slow); 117 118 MODULE_DESCRIPTION("Common Ethernet library - XDP infra"); 119 MODULE_IMPORT_NS("LIBETH"); 120 MODULE_LICENSE("GPL"); 121