1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (C) 2024-2025 Intel Corporation */ 3 4 #ifndef __LIBETH_TX_H 5 #define __LIBETH_TX_H 6 7 #include <linux/skbuff.h> 8 9 #include <net/libeth/types.h> 10 11 /* Tx buffer completion */ 12 13 /** 14 * enum libeth_sqe_type - type of &libeth_sqe to act on Tx completion 15 * @LIBETH_SQE_EMPTY: unused/empty OR XDP_TX/XSk frame, no action required 16 * @LIBETH_SQE_CTX: context descriptor with empty SQE, no action required 17 * @LIBETH_SQE_SLAB: kmalloc-allocated buffer, unmap and kfree() 18 * @LIBETH_SQE_FRAG: mapped skb frag, only unmap DMA 19 * @LIBETH_SQE_SKB: &sk_buff, unmap and napi_consume_skb(), update stats 20 * @__LIBETH_SQE_XDP_START: separator between skb and XDP types 21 * @LIBETH_SQE_XDP_TX: &skb_shared_info, libeth_xdp_return_buff_bulk(), stats 22 * @LIBETH_SQE_XDP_XMIT: &xdp_frame, unmap and xdp_return_frame_bulk(), stats 23 * @LIBETH_SQE_XDP_XMIT_FRAG: &xdp_frame frag, only unmap DMA 24 * @LIBETH_SQE_XSK_TX: &libeth_xdp_buff on XSk queue, xsk_buff_free(), stats 25 * @LIBETH_SQE_XSK_TX_FRAG: &libeth_xdp_buff frag on XSk queue, xsk_buff_free() 26 */ 27 enum libeth_sqe_type { 28 LIBETH_SQE_EMPTY = 0U, 29 LIBETH_SQE_CTX, 30 LIBETH_SQE_SLAB, 31 LIBETH_SQE_FRAG, 32 LIBETH_SQE_SKB, 33 34 __LIBETH_SQE_XDP_START, 35 LIBETH_SQE_XDP_TX = __LIBETH_SQE_XDP_START, 36 LIBETH_SQE_XDP_XMIT, 37 LIBETH_SQE_XDP_XMIT_FRAG, 38 LIBETH_SQE_XSK_TX, 39 LIBETH_SQE_XSK_TX_FRAG, 40 }; 41 42 /** 43 * struct libeth_sqe - represents a Send Queue Element / Tx buffer 44 * @type: type of the buffer, see the enum above 45 * @rs_idx: index of the last buffer from the batch this one was sent in 46 * @raw: slab buffer to free via kfree() 47 * @skb: &sk_buff to consume 48 * @sinfo: skb shared info of an XDP_TX frame 49 * @xdpf: XDP frame from ::ndo_xdp_xmit() 50 * @xsk: XSk Rx frame from XDP_TX action 51 * @dma: DMA address to unmap 52 * @len: length of the mapped region to unmap 53 * @nr_frags: number of frags in the frame this buffer belongs to 54 * @packets: number of physical packets sent for this frame 55 * @bytes: number of physical bytes sent for this frame 56 * @priv: driver-private scratchpad 57 */ 58 struct libeth_sqe { 59 enum libeth_sqe_type type:32; 60 u32 rs_idx; 61 62 union { 63 void *raw; 64 struct sk_buff *skb; 65 struct skb_shared_info *sinfo; 66 struct xdp_frame *xdpf; 67 struct libeth_xdp_buff *xsk; 68 }; 69 70 DEFINE_DMA_UNMAP_ADDR(dma); 71 DEFINE_DMA_UNMAP_LEN(len); 72 73 u32 nr_frags; 74 u32 packets; 75 u32 bytes; 76 77 unsigned long priv; 78 } __aligned_largest; 79 80 /** 81 * LIBETH_SQE_CHECK_PRIV - check the driver's private SQE data 82 * @p: type or name of the object the driver wants to fit into &libeth_sqe 83 * 84 * Make sure the driver's private data fits into libeth_sqe::priv. To be used 85 * right after its declaration. 86 */ 87 #define LIBETH_SQE_CHECK_PRIV(p) \ 88 static_assert(sizeof(p) <= sizeof_field(struct libeth_sqe, priv)) 89 90 /** 91 * struct libeth_cq_pp - completion queue poll params 92 * @dev: &device to perform DMA unmapping 93 * @bq: XDP frame bulk to combine return operations 94 * @ss: onstack NAPI stats to fill 95 * @xss: onstack XDPSQ NAPI stats to fill 96 * @xdp_tx: number of XDP-not-XSk frames processed 97 * @napi: whether it's called from the NAPI context 98 * 99 * libeth uses this structure to access objects needed for performing full 100 * Tx complete operation without passing lots of arguments and change the 101 * prototypes each time a new one is added. 102 */ 103 struct libeth_cq_pp { 104 struct device *dev; 105 struct xdp_frame_bulk *bq; 106 107 union { 108 struct libeth_sq_napi_stats *ss; 109 struct libeth_xdpsq_napi_stats *xss; 110 }; 111 u32 xdp_tx; 112 113 bool napi; 114 }; 115 116 /** 117 * libeth_tx_complete - perform Tx completion for one SQE 118 * @sqe: SQE to complete 119 * @cp: poll params 120 * 121 * Do Tx complete for all the types of buffers, incl. freeing, unmapping, 122 * updating the stats etc. 123 */ 124 static inline void libeth_tx_complete(struct libeth_sqe *sqe, 125 const struct libeth_cq_pp *cp) 126 { 127 switch (sqe->type) { 128 case LIBETH_SQE_EMPTY: 129 return; 130 case LIBETH_SQE_SKB: 131 case LIBETH_SQE_FRAG: 132 case LIBETH_SQE_SLAB: 133 dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma), 134 dma_unmap_len(sqe, len), DMA_TO_DEVICE); 135 break; 136 default: 137 break; 138 } 139 140 switch (sqe->type) { 141 case LIBETH_SQE_SKB: 142 cp->ss->packets += sqe->packets; 143 cp->ss->bytes += sqe->bytes; 144 145 napi_consume_skb(sqe->skb, cp->napi); 146 break; 147 case LIBETH_SQE_SLAB: 148 kfree(sqe->raw); 149 break; 150 default: 151 break; 152 } 153 154 sqe->type = LIBETH_SQE_EMPTY; 155 } 156 157 void libeth_tx_complete_any(struct libeth_sqe *sqe, struct libeth_cq_pp *cp); 158 159 #endif /* __LIBETH_TX_H */ 160