1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (C) 2024 Intel Corporation */
3
4 #ifndef __LIBETH_RX_H
5 #define __LIBETH_RX_H
6
7 #include <linux/if_vlan.h>
8
9 #include <net/page_pool/helpers.h>
10 #include <net/xdp.h>
11
12 /* Rx buffer management */
13
14 /* Space reserved in front of each frame */
15 #define LIBETH_SKB_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
16 /* Maximum headroom for worst-case calculations */
17 #define LIBETH_MAX_HEADROOM LIBETH_SKB_HEADROOM
18 /* Link layer / L2 overhead: Ethernet, 2 VLAN tags (C + S), FCS */
19 #define LIBETH_RX_LL_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN)
20 /* Maximum supported L2-L4 header length */
21 #define LIBETH_MAX_HEAD roundup_pow_of_two(max(MAX_HEADER, 256))
22
23 /* Always use order-0 pages */
24 #define LIBETH_RX_PAGE_ORDER 0
25 /* Pick a sane buffer stride and align to a cacheline boundary */
26 #define LIBETH_RX_BUF_STRIDE SKB_DATA_ALIGN(128)
27 /* HW-writeable space in one buffer: truesize - headroom/tailroom, aligned */
28 #define LIBETH_RX_PAGE_LEN(hr) \
29 ALIGN_DOWN(SKB_MAX_ORDER(hr, LIBETH_RX_PAGE_ORDER), \
30 LIBETH_RX_BUF_STRIDE)
31
32 /**
33 * struct libeth_fqe - structure representing an Rx buffer (fill queue element)
34 * @page: page holding the buffer
35 * @offset: offset from the page start (to the headroom)
36 * @truesize: total space occupied by the buffer (w/ headroom and tailroom)
37 *
38 * Depending on the MTU, API switches between one-page-per-frame and shared
39 * page model (to conserve memory on bigger-page platforms). In case of the
40 * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
41 */
42 struct libeth_fqe {
43 struct page *page;
44 u32 offset;
45 u32 truesize;
46 } __aligned_largest;
47
48 /**
49 * enum libeth_fqe_type - enum representing types of Rx buffers
50 * @LIBETH_FQE_MTU: buffer size is determined by MTU
51 * @LIBETH_FQE_SHORT: buffer size is smaller than MTU, for short frames
52 * @LIBETH_FQE_HDR: buffer size is ```LIBETH_MAX_HEAD```-sized, for headers
53 */
54 enum libeth_fqe_type {
55 LIBETH_FQE_MTU = 0U,
56 LIBETH_FQE_SHORT,
57 LIBETH_FQE_HDR,
58 };
59
60 /**
61 * struct libeth_fq - structure representing a buffer (fill) queue
62 * @fp: hotpath part of the structure
63 * @pp: &page_pool for buffer management
64 * @fqes: array of Rx buffers
65 * @truesize: size to allocate per buffer, w/overhead
66 * @count: number of descriptors/buffers the queue has
67 * @type: type of the buffers this queue has
68 * @hsplit: flag whether header split is enabled
69 * @buf_len: HW-writeable length per each buffer
70 * @nid: ID of the closest NUMA node with memory
71 */
72 struct libeth_fq {
73 struct_group_tagged(libeth_fq_fp, fp,
74 struct page_pool *pp;
75 struct libeth_fqe *fqes;
76
77 u32 truesize;
78 u32 count;
79 );
80
81 /* Cold fields */
82 enum libeth_fqe_type type:2;
83 bool hsplit:1;
84
85 u32 buf_len;
86 int nid;
87 };
88
89 int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi);
90 void libeth_rx_fq_destroy(struct libeth_fq *fq);
91
92 /**
93 * libeth_rx_alloc - allocate a new Rx buffer
94 * @fq: fill queue to allocate for
95 * @i: index of the buffer within the queue
96 *
97 * Return: DMA address to be passed to HW for Rx on successful allocation,
98 * ```DMA_MAPPING_ERROR``` otherwise.
99 */
libeth_rx_alloc(const struct libeth_fq_fp * fq,u32 i)100 static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i)
101 {
102 struct libeth_fqe *buf = &fq->fqes[i];
103
104 buf->truesize = fq->truesize;
105 buf->page = page_pool_dev_alloc(fq->pp, &buf->offset, &buf->truesize);
106 if (unlikely(!buf->page))
107 return DMA_MAPPING_ERROR;
108
109 return page_pool_get_dma_addr(buf->page) + buf->offset +
110 fq->pp->p.offset;
111 }
112
113 void libeth_rx_recycle_slow(struct page *page);
114
115 /**
116 * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA
117 * @fqe: buffer to process
118 * @len: frame length from the descriptor
119 *
120 * Process the buffer after it's written by HW. The regular path is to
121 * synchronize DMA for CPU, but in case of no data it will be immediately
122 * recycled back to its PP.
123 *
124 * Return: true when there's data to process, false otherwise.
125 */
libeth_rx_sync_for_cpu(const struct libeth_fqe * fqe,u32 len)126 static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
127 u32 len)
128 {
129 struct page *page = fqe->page;
130
131 /* Very rare, but possible case. The most common reason:
132 * the last fragment contained FCS only, which was then
133 * stripped by the HW.
134 */
135 if (unlikely(!len)) {
136 libeth_rx_recycle_slow(page);
137 return false;
138 }
139
140 page_pool_dma_sync_for_cpu(page->pp, page, fqe->offset, len);
141
142 return true;
143 }
144
145 /* Converting abstract packet type numbers into a software structure with
146 * the packet parameters to do O(1) lookup on Rx.
147 */
148
149 enum {
150 LIBETH_RX_PT_OUTER_L2 = 0U,
151 LIBETH_RX_PT_OUTER_IPV4,
152 LIBETH_RX_PT_OUTER_IPV6,
153 };
154
155 enum {
156 LIBETH_RX_PT_NOT_FRAG = 0U,
157 LIBETH_RX_PT_FRAG,
158 };
159
160 enum {
161 LIBETH_RX_PT_TUNNEL_IP_NONE = 0U,
162 LIBETH_RX_PT_TUNNEL_IP_IP,
163 LIBETH_RX_PT_TUNNEL_IP_GRENAT,
164 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC,
165 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC_VLAN,
166 };
167
168 enum {
169 LIBETH_RX_PT_TUNNEL_END_NONE = 0U,
170 LIBETH_RX_PT_TUNNEL_END_IPV4,
171 LIBETH_RX_PT_TUNNEL_END_IPV6,
172 };
173
174 enum {
175 LIBETH_RX_PT_INNER_NONE = 0U,
176 LIBETH_RX_PT_INNER_UDP,
177 LIBETH_RX_PT_INNER_TCP,
178 LIBETH_RX_PT_INNER_SCTP,
179 LIBETH_RX_PT_INNER_ICMP,
180 LIBETH_RX_PT_INNER_TIMESYNC,
181 };
182
183 #define LIBETH_RX_PT_PAYLOAD_NONE PKT_HASH_TYPE_NONE
184 #define LIBETH_RX_PT_PAYLOAD_L2 PKT_HASH_TYPE_L2
185 #define LIBETH_RX_PT_PAYLOAD_L3 PKT_HASH_TYPE_L3
186 #define LIBETH_RX_PT_PAYLOAD_L4 PKT_HASH_TYPE_L4
187
188 struct libeth_rx_pt {
189 u32 outer_ip:2;
190 u32 outer_frag:1;
191 u32 tunnel_type:3;
192 u32 tunnel_end_prot:2;
193 u32 tunnel_end_frag:1;
194 u32 inner_prot:3;
195 enum pkt_hash_types payload_layer:2;
196
197 u32 pad:2;
198 enum xdp_rss_hash_type hash_type:16;
199 };
200
201 void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt);
202
203 /**
204 * libeth_rx_pt_get_ip_ver - get IP version from a packet type structure
205 * @pt: packet type params
206 *
207 * Wrapper to compile out the IPv6 code from the drivers when not supported
208 * by the kernel.
209 *
210 * Return: @pt.outer_ip or stub for IPv6 when not compiled-in.
211 */
libeth_rx_pt_get_ip_ver(struct libeth_rx_pt pt)212 static inline u32 libeth_rx_pt_get_ip_ver(struct libeth_rx_pt pt)
213 {
214 #if !IS_ENABLED(CONFIG_IPV6)
215 switch (pt.outer_ip) {
216 case LIBETH_RX_PT_OUTER_IPV4:
217 return LIBETH_RX_PT_OUTER_IPV4;
218 default:
219 return LIBETH_RX_PT_OUTER_L2;
220 }
221 #else
222 return pt.outer_ip;
223 #endif
224 }
225
226 /* libeth_has_*() can be used to quickly check whether the HW metadata is
227 * available to avoid further expensive processing such as descriptor reads.
228 * They already check for the corresponding netdev feature to be enabled,
229 * thus can be used as drop-in replacements.
230 */
231
libeth_rx_pt_has_checksum(const struct net_device * dev,struct libeth_rx_pt pt)232 static inline bool libeth_rx_pt_has_checksum(const struct net_device *dev,
233 struct libeth_rx_pt pt)
234 {
235 /* Non-zero _INNER* is only possible when _OUTER_IPV* is set,
236 * it is enough to check only for the L4 type.
237 */
238 return likely(pt.inner_prot > LIBETH_RX_PT_INNER_NONE &&
239 (dev->features & NETIF_F_RXCSUM));
240 }
241
libeth_rx_pt_has_hash(const struct net_device * dev,struct libeth_rx_pt pt)242 static inline bool libeth_rx_pt_has_hash(const struct net_device *dev,
243 struct libeth_rx_pt pt)
244 {
245 return likely(pt.payload_layer > LIBETH_RX_PT_PAYLOAD_NONE &&
246 (dev->features & NETIF_F_RXHASH));
247 }
248
249 /**
250 * libeth_rx_pt_set_hash - fill in skb hash value basing on the PT
251 * @skb: skb to fill the hash in
252 * @hash: 32-bit hash value from the descriptor
253 * @pt: packet type
254 */
libeth_rx_pt_set_hash(struct sk_buff * skb,u32 hash,struct libeth_rx_pt pt)255 static inline void libeth_rx_pt_set_hash(struct sk_buff *skb, u32 hash,
256 struct libeth_rx_pt pt)
257 {
258 skb_set_hash(skb, hash, pt.payload_layer);
259 }
260
261 #endif /* __LIBETH_RX_H */
262