1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP" 5 6 #include <linux/export.h> 7 8 #include <net/libeth/xdp.h> 9 10 #include "priv.h" 11 12 /* ``XDP_TX`` bulking */ 13 14 static void __cold 15 libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm) 16 { 17 if (frm->len_fl & LIBETH_XDP_TX_MULTI) 18 libeth_xdp_return_frags(frm->data + frm->soff, true); 19 20 libeth_xdp_return_va(frm->data, true); 21 } 22 23 static void __cold 24 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) 25 { 26 for (u32 i = 0; i < count; i++) { 27 const struct libeth_xdp_tx_frame *frm = &bq[i]; 28 29 if (!(frm->len_fl & LIBETH_XDP_TX_FIRST)) 30 continue; 31 32 libeth_xdp_tx_return_one(frm); 33 } 34 } 35 36 static void __cold libeth_trace_xdp_exception(const struct net_device *dev, 37 const struct bpf_prog *prog, 38 u32 act) 39 { 40 trace_xdp_exception(dev, prog, act); 41 } 42 43 /** 44 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames 45 * @bq: XDP Tx frame bulk 46 * @sent: number of frames sent successfully (from this bulk) 47 * @flags: internal libeth_xdp flags (.ndo_xdp_xmit etc.) 48 * 49 * Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly. 50 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust 51 * the Tx bulk to try again later. 52 */ 53 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, 54 u32 flags) 55 { 56 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; 57 u32 left = bq->count - sent; 58 59 if (!(flags & LIBETH_XDP_TX_NDO)) 60 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); 61 62 if (!(flags & LIBETH_XDP_TX_DROP)) { 63 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); 64 bq->count = left; 65 66 return; 67 } 68 69 if (!(flags & LIBETH_XDP_TX_NDO)) 70 libeth_xdp_tx_return_bulk(pos, left); 71 else 72 libeth_xdp_xmit_return_bulk(pos, left, bq->dev); 73 74 bq->count = 0; 75 } 76 EXPORT_SYMBOL_GPL(libeth_xdp_tx_exception); 77 78 /* .ndo_xdp_xmit() implementation */ 79 80 u32 __cold libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, 81 u32 count, const struct net_device *dev) 82 { 83 u32 n = 0; 84 85 for (u32 i = 0; i < count; i++) { 86 const struct libeth_xdp_tx_frame *frm = &bq[i]; 87 dma_addr_t dma; 88 89 if (frm->flags & LIBETH_XDP_TX_FIRST) 90 dma = *libeth_xdp_xmit_frame_dma(frm->xdpf); 91 else 92 dma = dma_unmap_addr(frm, dma); 93 94 dma_unmap_page(dev->dev.parent, dma, dma_unmap_len(frm, len), 95 DMA_TO_DEVICE); 96 97 /* Actual xdp_frames are freed by the core */ 98 n += !!(frm->flags & LIBETH_XDP_TX_FIRST); 99 } 100 101 return n; 102 } 103 EXPORT_SYMBOL_GPL(libeth_xdp_xmit_return_bulk); 104 105 /* Rx polling path */ 106 107 /** 108 * libeth_xdp_return_buff_slow - free &libeth_xdp_buff 109 * @xdp: buffer to free/return 110 * 111 * Slowpath version of libeth_xdp_return_buff() to be called on exceptions, 112 * queue clean-ups etc., without unwanted inlining. 113 */ 114 void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp) 115 { 116 __libeth_xdp_return_buff(xdp, false); 117 } 118 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_slow); 119 120 /* Tx buffer completion */ 121 122 static void libeth_xdp_put_netmem_bulk(netmem_ref netmem, 123 struct xdp_frame_bulk *bq) 124 { 125 if (unlikely(bq->count == XDP_BULK_QUEUE_SIZE)) 126 xdp_flush_frame_bulk(bq); 127 128 bq->q[bq->count++] = netmem; 129 } 130 131 /** 132 * libeth_xdp_return_buff_bulk - free &xdp_buff as part of a bulk 133 * @sinfo: shared info corresponding to the buffer 134 * @bq: XDP frame bulk to store the buffer 135 * @frags: whether the buffer has frags 136 * 137 * Same as xdp_return_frame_bulk(), but for &libeth_xdp_buff, speeds up Tx 138 * completion of ``XDP_TX`` buffers and allows to free them in same bulks 139 * with &xdp_frame buffers. 140 */ 141 void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo, 142 struct xdp_frame_bulk *bq, bool frags) 143 { 144 if (!frags) 145 goto head; 146 147 for (u32 i = 0; i < sinfo->nr_frags; i++) 148 libeth_xdp_put_netmem_bulk(skb_frag_netmem(&sinfo->frags[i]), 149 bq); 150 151 head: 152 libeth_xdp_put_netmem_bulk(virt_to_netmem(sinfo), bq); 153 } 154 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_bulk); 155 156 /* Module */ 157 158 static const struct libeth_xdp_ops xdp_ops __initconst = { 159 .bulk = libeth_xdp_return_buff_bulk, 160 }; 161 162 static int __init libeth_xdp_module_init(void) 163 { 164 libeth_attach_xdp(&xdp_ops); 165 166 return 0; 167 } 168 module_init(libeth_xdp_module_init); 169 170 static void __exit libeth_xdp_module_exit(void) 171 { 172 libeth_detach_xdp(); 173 } 174 module_exit(libeth_xdp_module_exit); 175 176 MODULE_DESCRIPTION("Common Ethernet library - XDP infra"); 177 MODULE_IMPORT_NS("LIBETH"); 178 MODULE_LICENSE("GPL"); 179