1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP" 5 6 #include <linux/export.h> 7 8 #include <net/libeth/xdp.h> 9 10 #include "priv.h" 11 12 /* XDPSQ sharing */ 13 14 DEFINE_STATIC_KEY_FALSE(libeth_xdpsq_share); 15 EXPORT_SYMBOL_GPL(libeth_xdpsq_share); 16 17 void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock, 18 const struct net_device *dev) 19 { 20 bool warn; 21 22 spin_lock_init(&lock->lock); 23 lock->share = true; 24 25 warn = !static_key_enabled(&libeth_xdpsq_share); 26 static_branch_inc(&libeth_xdpsq_share); 27 28 if (warn && net_ratelimit()) 29 netdev_warn(dev, "XDPSQ sharing enabled, possible XDP Tx slowdown\n"); 30 } 31 EXPORT_SYMBOL_GPL(__libeth_xdpsq_get); 32 33 void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock, 34 const struct net_device *dev) 35 { 36 static_branch_dec(&libeth_xdpsq_share); 37 38 if (!static_key_enabled(&libeth_xdpsq_share) && net_ratelimit()) 39 netdev_notice(dev, "XDPSQ sharing disabled\n"); 40 41 lock->share = false; 42 } 43 EXPORT_SYMBOL_GPL(__libeth_xdpsq_put); 44 45 void __acquires(&lock->lock) 46 __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock) 47 { 48 spin_lock(&lock->lock); 49 } 50 EXPORT_SYMBOL_GPL(__libeth_xdpsq_lock); 51 52 void __releases(&lock->lock) 53 __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock) 54 { 55 spin_unlock(&lock->lock); 56 } 57 EXPORT_SYMBOL_GPL(__libeth_xdpsq_unlock); 58 59 /* ``XDP_TX`` bulking */ 60 61 static void __cold 62 libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm) 63 { 64 if (frm->len_fl & LIBETH_XDP_TX_MULTI) 65 libeth_xdp_return_frags(frm->data + frm->soff, true); 66 67 libeth_xdp_return_va(frm->data, true); 68 } 69 70 static void __cold 71 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) 72 { 73 for (u32 i = 0; i < count; i++) { 74 const struct libeth_xdp_tx_frame *frm = &bq[i]; 75 76 if (!(frm->len_fl & LIBETH_XDP_TX_FIRST)) 77 continue; 78 79 libeth_xdp_tx_return_one(frm); 80 } 81 } 82 83 static void __cold libeth_trace_xdp_exception(const struct net_device *dev, 84 const struct bpf_prog *prog, 85 u32 act) 86 { 87 trace_xdp_exception(dev, prog, act); 88 } 89 90 /** 91 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames 92 * @bq: XDP Tx frame bulk 93 * @sent: number of frames sent successfully (from this bulk) 94 * @flags: internal libeth_xdp flags (.ndo_xdp_xmit etc.) 95 * 96 * Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly. 97 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust 98 * the Tx bulk to try again later. 99 */ 100 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, 101 u32 flags) 102 { 103 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; 104 u32 left = bq->count - sent; 105 106 if (!(flags & LIBETH_XDP_TX_NDO)) 107 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); 108 109 if (!(flags & LIBETH_XDP_TX_DROP)) { 110 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); 111 bq->count = left; 112 113 return; 114 } 115 116 if (!(flags & LIBETH_XDP_TX_NDO)) 117 libeth_xdp_tx_return_bulk(pos, left); 118 else 119 libeth_xdp_xmit_return_bulk(pos, left, bq->dev); 120 121 bq->count = 0; 122 } 123 EXPORT_SYMBOL_GPL(libeth_xdp_tx_exception); 124 125 /* .ndo_xdp_xmit() implementation */ 126 127 u32 __cold libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq, 128 u32 count, const struct net_device *dev) 129 { 130 u32 n = 0; 131 132 for (u32 i = 0; i < count; i++) { 133 const struct libeth_xdp_tx_frame *frm = &bq[i]; 134 dma_addr_t dma; 135 136 if (frm->flags & LIBETH_XDP_TX_FIRST) 137 dma = *libeth_xdp_xmit_frame_dma(frm->xdpf); 138 else 139 dma = dma_unmap_addr(frm, dma); 140 141 dma_unmap_page(dev->dev.parent, dma, dma_unmap_len(frm, len), 142 DMA_TO_DEVICE); 143 144 /* Actual xdp_frames are freed by the core */ 145 n += !!(frm->flags & LIBETH_XDP_TX_FIRST); 146 } 147 148 return n; 149 } 150 EXPORT_SYMBOL_GPL(libeth_xdp_xmit_return_bulk); 151 152 /* Rx polling path */ 153 154 /** 155 * libeth_xdp_return_buff_slow - free &libeth_xdp_buff 156 * @xdp: buffer to free/return 157 * 158 * Slowpath version of libeth_xdp_return_buff() to be called on exceptions, 159 * queue clean-ups etc., without unwanted inlining. 160 */ 161 void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp) 162 { 163 __libeth_xdp_return_buff(xdp, false); 164 } 165 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_slow); 166 167 /* Tx buffer completion */ 168 169 static void libeth_xdp_put_netmem_bulk(netmem_ref netmem, 170 struct xdp_frame_bulk *bq) 171 { 172 if (unlikely(bq->count == XDP_BULK_QUEUE_SIZE)) 173 xdp_flush_frame_bulk(bq); 174 175 bq->q[bq->count++] = netmem; 176 } 177 178 /** 179 * libeth_xdp_return_buff_bulk - free &xdp_buff as part of a bulk 180 * @sinfo: shared info corresponding to the buffer 181 * @bq: XDP frame bulk to store the buffer 182 * @frags: whether the buffer has frags 183 * 184 * Same as xdp_return_frame_bulk(), but for &libeth_xdp_buff, speeds up Tx 185 * completion of ``XDP_TX`` buffers and allows to free them in same bulks 186 * with &xdp_frame buffers. 187 */ 188 void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo, 189 struct xdp_frame_bulk *bq, bool frags) 190 { 191 if (!frags) 192 goto head; 193 194 for (u32 i = 0; i < sinfo->nr_frags; i++) 195 libeth_xdp_put_netmem_bulk(skb_frag_netmem(&sinfo->frags[i]), 196 bq); 197 198 head: 199 libeth_xdp_put_netmem_bulk(virt_to_netmem(sinfo), bq); 200 } 201 EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_bulk); 202 203 /* Module */ 204 205 static const struct libeth_xdp_ops xdp_ops __initconst = { 206 .bulk = libeth_xdp_return_buff_bulk, 207 }; 208 209 static int __init libeth_xdp_module_init(void) 210 { 211 libeth_attach_xdp(&xdp_ops); 212 213 return 0; 214 } 215 module_init(libeth_xdp_module_init); 216 217 static void __exit libeth_xdp_module_exit(void) 218 { 219 libeth_detach_xdp(); 220 } 221 module_exit(libeth_xdp_module_exit); 222 223 MODULE_DESCRIPTION("Common Ethernet library - XDP infra"); 224 MODULE_IMPORT_NS("LIBETH"); 225 MODULE_LICENSE("GPL"); 226