1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (C) 2024-2025 Intel Corporation */ 3 4 #ifndef __LIBETH_RX_H 5 #define __LIBETH_RX_H 6 7 #include <linux/if_vlan.h> 8 9 #include <net/page_pool/helpers.h> 10 #include <net/xdp.h> 11 12 /* Rx buffer management */ 13 14 /* Space reserved in front of each frame */ 15 #define LIBETH_SKB_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN) 16 #define LIBETH_XDP_HEADROOM (ALIGN(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \ 17 NET_IP_ALIGN) 18 /* Maximum headroom for worst-case calculations */ 19 #define LIBETH_MAX_HEADROOM LIBETH_XDP_HEADROOM 20 /* Link layer / L2 overhead: Ethernet, 2 VLAN tags (C + S), FCS */ 21 #define LIBETH_RX_LL_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN) 22 /* Maximum supported L2-L4 header length */ 23 #define LIBETH_MAX_HEAD roundup_pow_of_two(max(MAX_HEADER, 256)) 24 25 /* Always use order-0 pages */ 26 #define LIBETH_RX_PAGE_ORDER 0 27 /* Pick a sane buffer stride and align to a cacheline boundary */ 28 #define LIBETH_RX_BUF_STRIDE SKB_DATA_ALIGN(128) 29 /* HW-writeable space in one buffer: truesize - headroom/tailroom, aligned */ 30 #define LIBETH_RX_PAGE_LEN(hr) \ 31 ALIGN_DOWN(SKB_MAX_ORDER(hr, LIBETH_RX_PAGE_ORDER), \ 32 LIBETH_RX_BUF_STRIDE) 33 34 /** 35 * struct libeth_fqe - structure representing an Rx buffer (fill queue element) 36 * @netmem: network memory reference holding the buffer 37 * @offset: offset from the page start (to the headroom) 38 * @truesize: total space occupied by the buffer (w/ headroom and tailroom) 39 * 40 * Depending on the MTU, API switches between one-page-per-frame and shared 41 * page model (to conserve memory on bigger-page platforms). In case of the 42 * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```. 43 */ 44 struct libeth_fqe { 45 netmem_ref netmem; 46 u32 offset; 47 u32 truesize; 48 } __aligned_largest; 49 50 /** 51 * enum libeth_fqe_type - enum representing types of Rx buffers 52 * @LIBETH_FQE_MTU: buffer size is determined by MTU 53 * @LIBETH_FQE_SHORT: buffer size is smaller than MTU, for short frames 54 * @LIBETH_FQE_HDR: buffer size is ```LIBETH_MAX_HEAD```-sized, for headers 55 */ 56 enum libeth_fqe_type { 57 LIBETH_FQE_MTU = 0U, 58 LIBETH_FQE_SHORT, 59 LIBETH_FQE_HDR, 60 }; 61 62 /** 63 * struct libeth_fq - structure representing a buffer (fill) queue 64 * @fp: hotpath part of the structure 65 * @pp: &page_pool for buffer management 66 * @fqes: array of Rx buffers 67 * @truesize: size to allocate per buffer, w/overhead 68 * @count: number of descriptors/buffers the queue has 69 * @type: type of the buffers this queue has 70 * @hsplit: flag whether header split is enabled 71 * @xdp: flag indicating whether XDP is enabled 72 * @no_napi: the queue is not a data queue and does not have NAPI 73 * @buf_len: HW-writeable length per each buffer 74 * @nid: ID of the closest NUMA node with memory 75 */ 76 struct libeth_fq { 77 struct_group_tagged(libeth_fq_fp, fp, 78 struct page_pool *pp; 79 struct libeth_fqe *fqes; 80 81 u32 truesize; 82 u32 count; 83 ); 84 85 /* Cold fields */ 86 enum libeth_fqe_type type:2; 87 bool hsplit:1; 88 bool xdp:1; 89 bool no_napi:1; 90 91 u32 buf_len; 92 int nid; 93 }; 94 95 int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev); 96 void libeth_rx_fq_destroy(struct libeth_fq *fq); 97 98 /** 99 * libeth_rx_alloc - allocate a new Rx buffer 100 * @fq: fill queue to allocate for 101 * @i: index of the buffer within the queue 102 * 103 * Return: DMA address to be passed to HW for Rx on successful allocation, 104 * ```DMA_MAPPING_ERROR``` otherwise. 105 */ 106 static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i) 107 { 108 struct libeth_fqe *buf = &fq->fqes[i]; 109 110 buf->truesize = fq->truesize; 111 buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset, 112 &buf->truesize); 113 if (unlikely(!buf->netmem)) 114 return DMA_MAPPING_ERROR; 115 116 return page_pool_get_dma_addr_netmem(buf->netmem) + buf->offset + 117 fq->pp->p.offset; 118 } 119 120 void libeth_rx_recycle_slow(netmem_ref netmem); 121 122 /** 123 * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA 124 * @fqe: buffer to process 125 * @len: frame length from the descriptor 126 * 127 * Process the buffer after it's written by HW. The regular path is to 128 * synchronize DMA for CPU, but in case of no data it will be immediately 129 * recycled back to its PP. 130 * 131 * Return: true when there's data to process, false otherwise. 132 */ 133 static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe, 134 u32 len) 135 { 136 netmem_ref netmem = fqe->netmem; 137 138 /* Very rare, but possible case. The most common reason: 139 * the last fragment contained FCS only, which was then 140 * stripped by the HW. 141 */ 142 if (unlikely(!len)) { 143 libeth_rx_recycle_slow(netmem); 144 return false; 145 } 146 147 page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem), netmem, 148 fqe->offset, len); 149 150 return true; 151 } 152 153 /* Converting abstract packet type numbers into a software structure with 154 * the packet parameters to do O(1) lookup on Rx. 155 */ 156 157 enum { 158 LIBETH_RX_PT_OUTER_L2 = 0U, 159 LIBETH_RX_PT_OUTER_IPV4, 160 LIBETH_RX_PT_OUTER_IPV6, 161 }; 162 163 enum { 164 LIBETH_RX_PT_NOT_FRAG = 0U, 165 LIBETH_RX_PT_FRAG, 166 }; 167 168 enum { 169 LIBETH_RX_PT_TUNNEL_IP_NONE = 0U, 170 LIBETH_RX_PT_TUNNEL_IP_IP, 171 LIBETH_RX_PT_TUNNEL_IP_GRENAT, 172 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC, 173 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC_VLAN, 174 }; 175 176 enum { 177 LIBETH_RX_PT_TUNNEL_END_NONE = 0U, 178 LIBETH_RX_PT_TUNNEL_END_IPV4, 179 LIBETH_RX_PT_TUNNEL_END_IPV6, 180 }; 181 182 enum { 183 LIBETH_RX_PT_INNER_NONE = 0U, 184 LIBETH_RX_PT_INNER_UDP, 185 LIBETH_RX_PT_INNER_TCP, 186 LIBETH_RX_PT_INNER_SCTP, 187 LIBETH_RX_PT_INNER_ICMP, 188 LIBETH_RX_PT_INNER_TIMESYNC, 189 }; 190 191 #define LIBETH_RX_PT_PAYLOAD_NONE PKT_HASH_TYPE_NONE 192 #define LIBETH_RX_PT_PAYLOAD_L2 PKT_HASH_TYPE_L2 193 #define LIBETH_RX_PT_PAYLOAD_L3 PKT_HASH_TYPE_L3 194 #define LIBETH_RX_PT_PAYLOAD_L4 PKT_HASH_TYPE_L4 195 196 struct libeth_rx_pt { 197 u32 outer_ip:2; 198 u32 outer_frag:1; 199 u32 tunnel_type:3; 200 u32 tunnel_end_prot:2; 201 u32 tunnel_end_frag:1; 202 u32 inner_prot:3; 203 enum pkt_hash_types payload_layer:2; 204 205 u32 pad:2; 206 enum xdp_rss_hash_type hash_type:16; 207 }; 208 209 /** 210 * struct libeth_rx_csum - checksum offload bits decoded from the Rx descriptor 211 * @l3l4p: detectable L3 and L4 integrity check is processed by the hardware 212 * @ipe: IP checksum error 213 * @eipe: external (outermost) IP header (only for tunels) 214 * @eudpe: external (outermost) UDP checksum error (only for tunels) 215 * @ipv6exadd: IPv6 header with extension headers 216 * @l4e: L4 integrity error 217 * @pprs: set for packets that skip checksum calculation in the HW pre parser 218 * @nat: the packet is a UDP tunneled packet 219 * @raw_csum_valid: set if raw checksum is valid 220 * @pad: padding to naturally align raw_csum field 221 * @raw_csum: raw checksum 222 */ 223 struct libeth_rx_csum { 224 u32 l3l4p:1; 225 u32 ipe:1; 226 u32 eipe:1; 227 u32 eudpe:1; 228 u32 ipv6exadd:1; 229 u32 l4e:1; 230 u32 pprs:1; 231 u32 nat:1; 232 233 u32 raw_csum_valid:1; 234 u32 pad:7; 235 u32 raw_csum:16; 236 }; 237 238 /** 239 * struct libeth_rqe_info - receive queue element info 240 * @len: packet length 241 * @ptype: packet type based on types programmed into the device 242 * @eop: whether it's the last fragment of the packet 243 * @rxe: MAC errors: CRC, Alignment, Oversize, Undersizes, Length error 244 * @vlan: C-VLAN or S-VLAN tag depending on the VLAN offload configuration 245 */ 246 struct libeth_rqe_info { 247 u32 len; 248 249 u32 ptype:14; 250 u32 eop:1; 251 u32 rxe:1; 252 253 u32 vlan:16; 254 }; 255 256 void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt); 257 258 /** 259 * libeth_rx_pt_get_ip_ver - get IP version from a packet type structure 260 * @pt: packet type params 261 * 262 * Wrapper to compile out the IPv6 code from the drivers when not supported 263 * by the kernel. 264 * 265 * Return: @pt.outer_ip or stub for IPv6 when not compiled-in. 266 */ 267 static inline u32 libeth_rx_pt_get_ip_ver(struct libeth_rx_pt pt) 268 { 269 #if !IS_ENABLED(CONFIG_IPV6) 270 switch (pt.outer_ip) { 271 case LIBETH_RX_PT_OUTER_IPV4: 272 return LIBETH_RX_PT_OUTER_IPV4; 273 default: 274 return LIBETH_RX_PT_OUTER_L2; 275 } 276 #else 277 return pt.outer_ip; 278 #endif 279 } 280 281 /* libeth_has_*() can be used to quickly check whether the HW metadata is 282 * available to avoid further expensive processing such as descriptor reads. 283 * They already check for the corresponding netdev feature to be enabled, 284 * thus can be used as drop-in replacements. 285 */ 286 287 static inline bool libeth_rx_pt_has_checksum(const struct net_device *dev, 288 struct libeth_rx_pt pt) 289 { 290 /* Non-zero _INNER* is only possible when _OUTER_IPV* is set, 291 * it is enough to check only for the L4 type. 292 */ 293 return likely(pt.inner_prot > LIBETH_RX_PT_INNER_NONE && 294 (dev->features & NETIF_F_RXCSUM)); 295 } 296 297 static inline bool libeth_rx_pt_has_hash(const struct net_device *dev, 298 struct libeth_rx_pt pt) 299 { 300 return likely(pt.payload_layer > LIBETH_RX_PT_PAYLOAD_NONE && 301 (dev->features & NETIF_F_RXHASH)); 302 } 303 304 /** 305 * libeth_rx_pt_set_hash - fill in skb hash value basing on the PT 306 * @skb: skb to fill the hash in 307 * @hash: 32-bit hash value from the descriptor 308 * @pt: packet type 309 */ 310 static inline void libeth_rx_pt_set_hash(struct sk_buff *skb, u32 hash, 311 struct libeth_rx_pt pt) 312 { 313 skb_set_hash(skb, hash, pt.payload_layer); 314 } 315 316 #endif /* __LIBETH_RX_H */ 317