1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP" 5 6 #include <linux/export.h> 7 8 #include <net/libeth/xdp.h> 9 10 /* ``XDP_TX`` bulking */ 11 12 static void __cold 13 libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm) 14 { 15 if (frm->len_fl & LIBETH_XDP_TX_MULTI) 16 libeth_xdp_return_frags(frm->data + frm->soff, true); 17 18 libeth_xdp_return_va(frm->data, true); 19 } 20 21 static void __cold 22 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) 23 { 24 for (u32 i = 0; i < count; i++) { 25 const struct libeth_xdp_tx_frame *frm = &bq[i]; 26 27 if (!(frm->len_fl & LIBETH_XDP_TX_FIRST)) 28 continue; 29 30 libeth_xdp_tx_return_one(frm); 31 } 32 } 33 34 static void __cold libeth_trace_xdp_exception(const struct net_device *dev, 35 const struct bpf_prog *prog, 36 u32 act) 37 { 38 trace_xdp_exception(dev, prog, act); 39 } 40 41 /** 42 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames 43 * @bq: XDP Tx frame bulk 44 * @sent: number of frames sent successfully (from this bulk) 45 * @flags: internal libeth_xdp flags 46 * 47 * Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly. 48 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust 49 * the Tx bulk to try again later. 50 */ 51 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, 52 u32 flags) 53 { 54 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; 55 u32 left = bq->count - sent; 56 57 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); 58 59 if (!(flags & LIBETH_XDP_TX_DROP)) { 60 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); 61 bq->count = left; 62 63 return; 64 } 65 66 libeth_xdp_tx_return_bulk(pos, left); 67 68 bq->count = 0; 69 } 70 EXPORT_SYMBOL_GPL(libeth_xdp_tx_exception); 71 72 /* Rx polling path */ 73 74 /** 75 * libeth_xdp_return_buff_slow - free &libeth_xdp_buff 76 * @xdp: buffer to free/return 77 * 78 * Slowpath version of libeth_xdp_return_buff() to be called on exceptions, 79 * queue clean-ups etc., without unwanted inlining. 80 */ 81 void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp) 82 { 83 __libeth_xdp_return_buff(xdp, false); 84 } 85 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_slow); 86 87 MODULE_DESCRIPTION("Common Ethernet library - XDP infra"); 88 MODULE_IMPORT_NS("LIBETH"); 89 MODULE_LICENSE("GPL"); 90