1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2024-2025 Intel Corporation */
3
4 #define DEFAULT_SYMBOL_NAMESPACE "LIBETH"
5
6 #include <linux/export.h>
7
8 #include <net/libeth/rx.h>
9
10 /* Rx buffer management */
11
12 /**
13 * libeth_rx_hw_len_mtu - get the actual buffer size to be passed to HW
14 * @pp: &page_pool_params of the netdev to calculate the size for
15 * @max_len: maximum buffer size for a single descriptor
16 *
17 * Return: HW-writeable length per one buffer to pass it to the HW accounting:
18 * MTU the @dev has, HW required alignment, minimum and maximum allowed values,
19 * and system's page size.
20 */
libeth_rx_hw_len_mtu(const struct page_pool_params * pp,u32 max_len)21 static u32 libeth_rx_hw_len_mtu(const struct page_pool_params *pp, u32 max_len)
22 {
23 u32 len;
24
25 len = READ_ONCE(pp->netdev->mtu) + LIBETH_RX_LL_LEN;
26 len = ALIGN(len, LIBETH_RX_BUF_STRIDE);
27 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE),
28 pp->max_len);
29
30 return len;
31 }
32
33 /**
34 * libeth_rx_hw_len_truesize - get the short buffer size to be passed to HW
35 * @pp: &page_pool_params of the netdev to calculate the size for
36 * @max_len: maximum buffer size for a single descriptor
37 * @truesize: desired truesize for the buffers
38 *
39 * Return: HW-writeable length per one buffer to pass it to the HW ignoring the
40 * MTU and closest to the passed truesize. Can be used for "short" buffer
41 * queues to fragment pages more efficiently.
42 */
libeth_rx_hw_len_truesize(const struct page_pool_params * pp,u32 max_len,u32 truesize)43 static u32 libeth_rx_hw_len_truesize(const struct page_pool_params *pp,
44 u32 max_len, u32 truesize)
45 {
46 u32 min, len;
47
48 min = SKB_HEAD_ALIGN(pp->offset + LIBETH_RX_BUF_STRIDE);
49 truesize = clamp(roundup_pow_of_two(truesize), roundup_pow_of_two(min),
50 PAGE_SIZE << LIBETH_RX_PAGE_ORDER);
51
52 len = SKB_WITH_OVERHEAD(truesize - pp->offset);
53 len = ALIGN_DOWN(len, LIBETH_RX_BUF_STRIDE) ? : LIBETH_RX_BUF_STRIDE;
54 len = min3(len, ALIGN_DOWN(max_len ? : U32_MAX, LIBETH_RX_BUF_STRIDE),
55 pp->max_len);
56
57 return len;
58 }
59
60 /**
61 * libeth_rx_page_pool_params - calculate params with the stack overhead
62 * @fq: buffer queue to calculate the size for
63 * @pp: &page_pool_params of the netdev
64 *
65 * Set the PP params to will all needed stack overhead (headroom, tailroom) and
66 * both the HW buffer length and the truesize for all types of buffers. For
67 * "short" buffers, truesize never exceeds the "wanted" one; for the rest,
68 * it can be up to the page size.
69 *
70 * Return: true on success, false on invalid input params.
71 */
libeth_rx_page_pool_params(struct libeth_fq * fq,struct page_pool_params * pp)72 static bool libeth_rx_page_pool_params(struct libeth_fq *fq,
73 struct page_pool_params *pp)
74 {
75 pp->offset = fq->xdp ? LIBETH_XDP_HEADROOM : LIBETH_SKB_HEADROOM;
76 /* HW-writeable / syncable length per one page */
77 pp->max_len = LIBETH_RX_PAGE_LEN(pp->offset);
78
79 /* HW-writeable length per buffer */
80 switch (fq->type) {
81 case LIBETH_FQE_MTU:
82 fq->buf_len = libeth_rx_hw_len_mtu(pp, fq->buf_len);
83 break;
84 case LIBETH_FQE_SHORT:
85 fq->buf_len = libeth_rx_hw_len_truesize(pp, fq->buf_len,
86 fq->truesize);
87 break;
88 case LIBETH_FQE_HDR:
89 fq->buf_len = ALIGN(LIBETH_MAX_HEAD, LIBETH_RX_BUF_STRIDE);
90 break;
91 default:
92 return false;
93 }
94
95 /* Buffer size to allocate */
96 fq->truesize = roundup_pow_of_two(SKB_HEAD_ALIGN(pp->offset +
97 fq->buf_len));
98
99 return true;
100 }
101
102 /**
103 * libeth_rx_page_pool_params_zc - calculate params without the stack overhead
104 * @fq: buffer queue to calculate the size for
105 * @pp: &page_pool_params of the netdev
106 *
107 * Set the PP params to exclude the stack overhead and both the buffer length
108 * and the truesize, which are equal for the data buffers. Note that this
109 * requires separate header buffers to be always active and account the
110 * overhead.
111 * With the MTU == ``PAGE_SIZE``, this allows the kernel to enable the zerocopy
112 * mode.
113 *
114 * Return: true on success, false on invalid input params.
115 */
libeth_rx_page_pool_params_zc(struct libeth_fq * fq,struct page_pool_params * pp)116 static bool libeth_rx_page_pool_params_zc(struct libeth_fq *fq,
117 struct page_pool_params *pp)
118 {
119 u32 mtu, max;
120
121 pp->offset = 0;
122 pp->max_len = PAGE_SIZE << LIBETH_RX_PAGE_ORDER;
123
124 switch (fq->type) {
125 case LIBETH_FQE_MTU:
126 mtu = READ_ONCE(pp->netdev->mtu);
127 break;
128 case LIBETH_FQE_SHORT:
129 mtu = fq->truesize;
130 break;
131 default:
132 return false;
133 }
134
135 mtu = roundup_pow_of_two(mtu);
136 max = min(rounddown_pow_of_two(fq->buf_len ? : U32_MAX),
137 pp->max_len);
138
139 fq->buf_len = clamp(mtu, LIBETH_RX_BUF_STRIDE, max);
140 fq->truesize = fq->buf_len;
141
142 return true;
143 }
144
145 /**
146 * libeth_rx_fq_create - create a PP with the default libeth settings
147 * @fq: buffer queue struct to fill
148 * @napi_dev: &napi_struct for NAPI (data) queues, &device for others
149 *
150 * Return: %0 on success, -%errno on failure.
151 */
libeth_rx_fq_create(struct libeth_fq * fq,void * napi_dev)152 int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev)
153 {
154 struct napi_struct *napi = fq->no_napi ? NULL : napi_dev;
155 struct page_pool_params pp = {
156 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
157 .order = LIBETH_RX_PAGE_ORDER,
158 .pool_size = fq->count,
159 .nid = fq->nid,
160 .dev = napi ? napi->dev->dev.parent : napi_dev,
161 .netdev = napi ? napi->dev : NULL,
162 .napi = napi,
163 };
164 struct libeth_fqe *fqes;
165 struct page_pool *pool;
166 int ret;
167
168 if (!pp.netdev && fq->type == LIBETH_FQE_MTU)
169 return -EINVAL;
170
171 pp.dma_dir = fq->xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
172
173 if (!fq->hsplit)
174 ret = libeth_rx_page_pool_params(fq, &pp);
175 else
176 ret = libeth_rx_page_pool_params_zc(fq, &pp);
177 if (!ret)
178 return -EINVAL;
179
180 pool = page_pool_create(&pp);
181 if (IS_ERR(pool))
182 return PTR_ERR(pool);
183
184 fqes = kvcalloc_node(fq->count, sizeof(*fqes), GFP_KERNEL, fq->nid);
185 if (!fqes) {
186 ret = -ENOMEM;
187 goto err_buf;
188 }
189
190 ret = xdp_reg_page_pool(pool);
191 if (ret)
192 goto err_mem;
193
194 fq->fqes = fqes;
195 fq->pp = pool;
196
197 return 0;
198
199 err_mem:
200 kvfree(fqes);
201 err_buf:
202 page_pool_destroy(pool);
203
204 return ret;
205 }
206 EXPORT_SYMBOL_GPL(libeth_rx_fq_create);
207
208 /**
209 * libeth_rx_fq_destroy - destroy a &page_pool created by libeth
210 * @fq: buffer queue to process
211 */
libeth_rx_fq_destroy(struct libeth_fq * fq)212 void libeth_rx_fq_destroy(struct libeth_fq *fq)
213 {
214 xdp_unreg_page_pool(fq->pp);
215 kvfree(fq->fqes);
216 page_pool_destroy(fq->pp);
217 }
218 EXPORT_SYMBOL_GPL(libeth_rx_fq_destroy);
219
220 /**
221 * libeth_rx_recycle_slow - recycle libeth netmem
222 * @netmem: network memory to recycle
223 *
224 * To be used on exceptions or rare cases not requiring fast inline recycling.
225 */
libeth_rx_recycle_slow(netmem_ref netmem)226 void __cold libeth_rx_recycle_slow(netmem_ref netmem)
227 {
228 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
229 }
230 EXPORT_SYMBOL_GPL(libeth_rx_recycle_slow);
231
232 /* Converting abstract packet type numbers into a software structure with
233 * the packet parameters to do O(1) lookup on Rx.
234 */
235
236 static const u16 libeth_rx_pt_xdp_oip[] = {
237 [LIBETH_RX_PT_OUTER_L2] = XDP_RSS_TYPE_NONE,
238 [LIBETH_RX_PT_OUTER_IPV4] = XDP_RSS_L3_IPV4,
239 [LIBETH_RX_PT_OUTER_IPV6] = XDP_RSS_L3_IPV6,
240 };
241
242 static const u16 libeth_rx_pt_xdp_iprot[] = {
243 [LIBETH_RX_PT_INNER_NONE] = XDP_RSS_TYPE_NONE,
244 [LIBETH_RX_PT_INNER_UDP] = XDP_RSS_L4_UDP,
245 [LIBETH_RX_PT_INNER_TCP] = XDP_RSS_L4_TCP,
246 [LIBETH_RX_PT_INNER_SCTP] = XDP_RSS_L4_SCTP,
247 [LIBETH_RX_PT_INNER_ICMP] = XDP_RSS_L4_ICMP,
248 [LIBETH_RX_PT_INNER_TIMESYNC] = XDP_RSS_TYPE_NONE,
249 };
250
251 static const u16 libeth_rx_pt_xdp_pl[] = {
252 [LIBETH_RX_PT_PAYLOAD_NONE] = XDP_RSS_TYPE_NONE,
253 [LIBETH_RX_PT_PAYLOAD_L2] = XDP_RSS_TYPE_NONE,
254 [LIBETH_RX_PT_PAYLOAD_L3] = XDP_RSS_TYPE_NONE,
255 [LIBETH_RX_PT_PAYLOAD_L4] = XDP_RSS_L4,
256 };
257
258 /**
259 * libeth_rx_pt_gen_hash_type - generate an XDP RSS hash type for a PT
260 * @pt: PT structure to evaluate
261 *
262 * Generates ```hash_type``` field with XDP RSS type values from the parsed
263 * packet parameters if they're obtained dynamically at runtime.
264 */
libeth_rx_pt_gen_hash_type(struct libeth_rx_pt * pt)265 void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt)
266 {
267 pt->hash_type = 0;
268 pt->hash_type |= libeth_rx_pt_xdp_oip[pt->outer_ip];
269 pt->hash_type |= libeth_rx_pt_xdp_iprot[pt->inner_prot];
270 pt->hash_type |= libeth_rx_pt_xdp_pl[pt->payload_layer];
271 }
272 EXPORT_SYMBOL_GPL(libeth_rx_pt_gen_hash_type);
273
274 /* Module */
275
276 MODULE_DESCRIPTION("Common Ethernet library");
277 MODULE_LICENSE("GPL");
278