1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP" 5 6 #include <linux/export.h> 7 8 #include <net/libeth/xsk.h> 9 10 #include "priv.h" 11 12 /* ``XDP_TX`` bulking */ 13 14 void __cold libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, 15 u32 count) 16 { 17 for (u32 i = 0; i < count; i++) 18 libeth_xsk_buff_free_slow(bq[i].xsk); 19 } 20 21 /* XSk TMO */ 22 23 const struct xsk_tx_metadata_ops libeth_xsktmo_slow = { 24 .tmo_request_checksum = libeth_xsktmo_req_csum, 25 }; 26 27 /* Rx polling path */ 28 29 /** 30 * libeth_xsk_buff_free_slow - free an XSk Rx buffer 31 * @xdp: buffer to free 32 * 33 * Slowpath version of xsk_buff_free() to be used on exceptions, cleanups etc. 34 * to avoid unwanted inlining. 35 */ 36 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp) 37 { 38 xsk_buff_free(&xdp->base); 39 } 40 EXPORT_SYMBOL_GPL(libeth_xsk_buff_free_slow); 41 42 /** 43 * libeth_xsk_buff_add_frag - add frag to XSk Rx buffer 44 * @head: head buffer 45 * @xdp: frag buffer 46 * 47 * External helper used by libeth_xsk_process_buff(), do not call directly. 48 * Frees both main and frag buffers on error. 49 * 50 * Return: main buffer with attached frag on success, %NULL on error (no space 51 * for a new frag). 52 */ 53 struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head, 54 struct libeth_xdp_buff *xdp) 55 { 56 if (!xsk_buff_add_frag(&head->base, &xdp->base)) 57 goto free; 58 59 return head; 60 61 free: 62 libeth_xsk_buff_free_slow(xdp); 63 libeth_xsk_buff_free_slow(head); 64 65 return NULL; 66 } 67 EXPORT_SYMBOL_GPL(libeth_xsk_buff_add_frag); 68 69 /** 70 * libeth_xsk_buff_stats_frags - update onstack RQ stats with XSk frags info 71 * @rs: onstack stats to update 72 * @xdp: buffer to account 73 * 74 * External helper used by __libeth_xsk_run_pass(), do not call directly. 75 * Adds buffer's frags count and total len to the onstack stats. 76 */ 77 void libeth_xsk_buff_stats_frags(struct libeth_rq_napi_stats *rs, 78 const struct libeth_xdp_buff *xdp) 79 { 80 libeth_xdp_buff_stats_frags(rs, xdp); 81 } 82 EXPORT_SYMBOL_GPL(libeth_xsk_buff_stats_frags); 83 84 /** 85 * __libeth_xsk_run_prog_slow - process the non-``XDP_REDIRECT`` verdicts 86 * @xdp: buffer to process 87 * @bq: Tx bulk for queueing on ``XDP_TX`` 88 * @act: verdict to process 89 * @ret: error code if ``XDP_REDIRECT`` failed 90 * 91 * External helper used by __libeth_xsk_run_prog(), do not call directly. 92 * ``XDP_REDIRECT`` is the most common and hottest verdict on XSk, thus 93 * it is processed inline. The rest goes here for out-of-line processing, 94 * together with redirect errors. 95 * 96 * Return: libeth_xdp XDP prog verdict. 97 */ 98 u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp, 99 const struct libeth_xdp_tx_bulk *bq, 100 enum xdp_action act, int ret) 101 { 102 switch (act) { 103 case XDP_DROP: 104 xsk_buff_free(&xdp->base); 105 106 return LIBETH_XDP_DROP; 107 case XDP_TX: 108 return LIBETH_XDP_TX; 109 case XDP_PASS: 110 return LIBETH_XDP_PASS; 111 default: 112 break; 113 } 114 115 return libeth_xdp_prog_exception(bq, xdp, act, ret); 116 } 117 EXPORT_SYMBOL_GPL(__libeth_xsk_run_prog_slow); 118 119 /** 120 * libeth_xsk_prog_exception - handle XDP prog exceptions on XSk 121 * @xdp: buffer to process 122 * @act: verdict returned by the prog 123 * @ret: error code if ``XDP_REDIRECT`` failed 124 * 125 * Internal. Frees the buffer and, if the queue uses XSk wakeups, stop the 126 * current NAPI poll when there are no free buffers left. 127 * 128 * Return: libeth_xdp's XDP prog verdict. 129 */ 130 u32 __cold libeth_xsk_prog_exception(struct libeth_xdp_buff *xdp, 131 enum xdp_action act, int ret) 132 { 133 const struct xdp_buff_xsk *xsk; 134 u32 __ret = LIBETH_XDP_DROP; 135 136 if (act != XDP_REDIRECT) 137 goto drop; 138 139 xsk = container_of(&xdp->base, typeof(*xsk), xdp); 140 if (xsk_uses_need_wakeup(xsk->pool) && ret == -ENOBUFS) 141 __ret = LIBETH_XDP_ABORTED; 142 143 drop: 144 libeth_xsk_buff_free_slow(xdp); 145 146 return __ret; 147 } 148