1 /*-
2 * Copyright (c) 2020-2025 The FreeBSD Foundation
3 * Copyright (c) 2021-2022 Bjoern A. Zeeb
4 *
5 * This software was developed by Björn Zeeb under sponsorship from
6 * the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 /*
31 * NOTE: this socket buffer compatibility code is highly EXPERIMENTAL.
32 * Do not rely on the internals of this implementation. They are highly
33 * likely to change as we will improve the integration to FreeBSD mbufs.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_ddb.h"
38
39 #include <sys/param.h>
40 #include <sys/types.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
43 #include <sys/sysctl.h>
44
45 #include <vm/uma.h>
46
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50
51 #include <linux/skbuff.h>
52 #include <linux/slab.h>
53 #include <linux/gfp.h>
54 #ifdef __LP64__
55 #include <linux/log2.h>
56 #endif
57
58 #include <net/page_pool/helpers.h>
59
60 SYSCTL_DECL(_compat_linuxkpi);
61 SYSCTL_NODE(_compat_linuxkpi, OID_AUTO, skb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
62 "LinuxKPI skbuff");
63
64 #ifdef SKB_DEBUG
65 int linuxkpi_debug_skb;
66 SYSCTL_INT(_compat_linuxkpi_skb, OID_AUTO, debug, CTLFLAG_RWTUN,
67 &linuxkpi_debug_skb, 0, "SKB debug level");
68 #endif
69
70 static uma_zone_t skbzone;
71
72 #define SKB_DMA32_MALLOC
73 #ifdef SKB_DMA32_MALLOC
74 /*
75 * Realtek wireless drivers (e.g., rtw88) require 32bit DMA in a single segment.
76 * busdma(9) has a hard time providing this currently for 3-ish pages at large
77 * quantities (see lkpi_pci_nseg1_fail in linux_pci.c).
78 * Work around this for now by allowing a tunable to enforce physical addresses
79 * allocation limits using "old-school" contigmalloc(9) to avoid bouncing.
80 * Note: with the malloc/contigmalloc + kmalloc changes also providing physical
81 * contiguous memory, and the nseg=1 limit for bouncing we should in theory be
82 * fine now and not need any of this anymore, however busdma still has troubles
83 * boncing three contiguous pages so for now this stays.
84 */
85 static int linuxkpi_skb_memlimit;
86 SYSCTL_INT(_compat_linuxkpi_skb, OID_AUTO, mem_limit, CTLFLAG_RDTUN,
87 &linuxkpi_skb_memlimit, 0, "SKB memory limit: 0=no limit, "
88 "1=32bit, 2=36bit, other=undef (currently 32bit)");
89
90 static MALLOC_DEFINE(M_LKPISKB, "lkpiskb", "Linux KPI skbuff compat");
91 #endif
92
93 struct sk_buff *
linuxkpi_alloc_skb(size_t size,gfp_t gfp)94 linuxkpi_alloc_skb(size_t size, gfp_t gfp)
95 {
96 struct sk_buff *skb;
97 void *p;
98 size_t len;
99
100 skb = uma_zalloc(skbzone, linux_check_m_flags(gfp) | M_ZERO);
101 if (skb == NULL)
102 return (NULL);
103
104 skb->prev = skb->next = skb;
105 skb->truesize = size;
106 skb->shinfo = (struct skb_shared_info *)(skb + 1);
107
108 if (size == 0)
109 return (skb);
110
111 len = size;
112 #ifdef SKB_DMA32_MALLOC
113 /*
114 * Using our own type here not backing my kmalloc.
115 * We assume no one calls kfree directly on the skb.
116 */
117 if (__predict_false(linuxkpi_skb_memlimit != 0)) {
118 vm_paddr_t high;
119
120 switch (linuxkpi_skb_memlimit) {
121 #ifdef __LP64__
122 case 2:
123 high = (0xfffffffff); /* 1<<36 really. */
124 break;
125 #endif
126 case 1:
127 default:
128 high = (0xffffffff); /* 1<<32 really. */
129 break;
130 }
131 len = roundup_pow_of_two(len);
132 p = contigmalloc(len, M_LKPISKB,
133 linux_check_m_flags(gfp) | M_ZERO, 0, high, PAGE_SIZE, 0);
134 } else
135 #endif
136 p = __kmalloc(len, linux_check_m_flags(gfp) | M_ZERO);
137 if (p == NULL) {
138 uma_zfree(skbzone, skb);
139 return (NULL);
140 }
141
142 skb->head = skb->data = (uint8_t *)p;
143 skb_reset_tail_pointer(skb);
144 skb->end = skb->head + size;
145 refcount_set(&skb->refcnt, 1);
146
147 SKB_TRACE_FMT(skb, "data %p size %zu", (skb) ? skb->data : NULL, size);
148 return (skb);
149 }
150
151 struct sk_buff *
linuxkpi_dev_alloc_skb(size_t size,gfp_t gfp)152 linuxkpi_dev_alloc_skb(size_t size, gfp_t gfp)
153 {
154 struct sk_buff *skb;
155 size_t len;
156
157 len = size + NET_SKB_PAD;
158 skb = linuxkpi_alloc_skb(len, gfp);
159
160 if (skb != NULL)
161 skb_reserve(skb, NET_SKB_PAD);
162
163 SKB_TRACE_FMT(skb, "data %p size %zu len %zu",
164 (skb) ? skb->data : NULL, size, len);
165 return (skb);
166 }
167
168 struct sk_buff *
linuxkpi_build_skb(void * data,size_t fragsz)169 linuxkpi_build_skb(void *data, size_t fragsz)
170 {
171 struct sk_buff *skb;
172
173 if (data == NULL || fragsz == 0)
174 return (NULL);
175
176 /* Just allocate a skb without data area. */
177 skb = linuxkpi_alloc_skb(0, GFP_KERNEL);
178 if (skb == NULL)
179 return (NULL);
180
181 skb->_flags |= _SKB_FLAGS_SKBEXTFRAG;
182 skb->truesize = fragsz;
183 skb->head = skb->data = data;
184 skb_reset_tail_pointer(skb);
185 skb->end = skb->head + fragsz;
186 refcount_set(&skb->refcnt, 1);
187
188 return (skb);
189 }
190
191 struct sk_buff *
linuxkpi_skb_copy(const struct sk_buff * skb,gfp_t gfp)192 linuxkpi_skb_copy(const struct sk_buff *skb, gfp_t gfp)
193 {
194 struct sk_buff *new;
195 struct skb_shared_info *shinfo;
196 size_t len;
197 unsigned int headroom;
198 uint16_t fragno, count;
199
200 /* Full buffer size + any fragments. */
201 len = skb->end - skb->head + skb->data_len;
202
203 new = linuxkpi_alloc_skb(len, gfp);
204 if (new == NULL)
205 return (NULL);
206
207 headroom = skb_headroom(skb);
208 /* Fixup head and end. */
209 skb_reserve(new, headroom); /* data and tail move headroom forward. */
210 skb_put(new, skb->len); /* tail and len get adjusted */
211
212 /* Copy data. */
213 memcpy(new->head, skb->data - headroom, headroom + skb->len);
214
215 /* Deal with fragments. */
216 shinfo = skb->shinfo;
217 for (count = fragno = 0;
218 count < shinfo->nr_frags && fragno < nitems(shinfo->frags);
219 fragno++) {
220 if (shinfo->frags[fragno].page != NULL) {
221 skb_put_data(new,
222 skb_frag_address(&shinfo->frags[fragno]),
223 shinfo->frags[fragno].size);
224 count++;
225 }
226 }
227
228 /* Deal with header fields. */
229 memcpy(new->cb, skb->cb, sizeof(skb->cb));
230 SKB_IMPROVE("more header fields to copy?");
231
232 return (new);
233 }
234
235 int
lkpi___skb_linearize(struct sk_buff * skb)236 lkpi___skb_linearize(struct sk_buff *skb)
237 {
238 struct sk_buff *new;
239 struct skb_shared_info *shinfo;
240 uint16_t fragno, count;
241
242 SKB_TRACE(skb);
243 SKB_IMPROVE("Hack completely re-allocating and freeing; FIXME");
244 new = skb_copy(skb, GFP_NOWAIT);
245 if (new == NULL)
246 return (-ENOMEM);
247
248 /* Now need to swap head, data, tail, ... and free from old (and then new). */
249 shinfo = skb->shinfo;
250 for (count = fragno = 0;
251 count < shinfo->nr_frags && fragno < nitems(shinfo->frags);
252 fragno++) {
253
254 if (shinfo->frags[fragno].page != NULL) {
255 struct page *p;
256
257 p = shinfo->frags[fragno].page;
258 shinfo->frags[fragno].size = 0;
259 shinfo->frags[fragno].offset = 0;
260 shinfo->frags[fragno].page = NULL;
261 __free_page(p);
262 count++;
263 }
264 }
265
266 if ((skb->_flags & _SKB_FLAGS_SKBEXTFRAG) != 0) {
267 void *p;
268
269 p = skb->head;
270 skb_free_frag(p);
271 skb->head = NULL;
272 skb->_flags &= ~_SKB_FLAGS_SKBEXTFRAG;
273 }
274
275 #ifdef SKB_DMA32_MALLOC
276 if (__predict_false(linuxkpi_skb_memlimit != 0))
277 free(skb->head, M_LKPISKB);
278 else
279 #endif
280 kfree(skb->head);
281
282 skb->head = new->head;
283 skb->data = new->data;
284 skb->tail = new->tail;
285 skb->end = new->end;
286 skb->len = new->len;
287 skb->data_len = new->data_len;
288 skb->truesize = new->truesize;
289
290 uma_zfree(skbzone, new);
291
292 return (0);
293 }
294
295 static bool
lkpi_skb_refcount_release(struct sk_buff * skb)296 lkpi_skb_refcount_release(struct sk_buff *skb)
297 {
298 if (skb == NULL)
299 return (false);
300
301 /* Do we need further tests to avoid freeing this one? */
302
303 if (!refcount_dec_and_test(&skb->refcnt))
304 return (false);
305
306 return (true);
307 }
308
309 void
linuxkpi_kfree_skb(struct sk_buff * skb)310 linuxkpi_kfree_skb(struct sk_buff *skb)
311 {
312 struct skb_shared_info *shinfo;
313 uint16_t fragno, count;
314
315 SKB_TRACE(skb);
316 if (skb == NULL)
317 return;
318
319 if (!lkpi_skb_refcount_release(skb)) {
320 SKB_TRACE_FMT(skb, "not freed due to refcnt");
321 return;
322 }
323
324 /*
325 * XXX TODO this will go away once we have skb backed by mbuf.
326 * currently we allow the mbuf to stay around and use a private
327 * free function to allow secondary resources to be freed along.
328 */
329 if (skb->m != NULL) {
330 void *m;
331
332 m = skb->m;
333 skb->m = NULL;
334
335 KASSERT(skb->m_free_func != NULL, ("%s: skb %p has m %p but no "
336 "m_free_func %p\n", __func__, skb, m, skb->m_free_func));
337 skb->m_free_func(m);
338 }
339 KASSERT(skb->m == NULL,
340 ("%s: skb %p m %p != NULL\n", __func__, skb, skb->m));
341
342 shinfo = skb->shinfo;
343 for (count = fragno = 0;
344 count < shinfo->nr_frags && fragno < nitems(shinfo->frags);
345 fragno++) {
346
347 if (shinfo->frags[fragno].page != NULL) {
348 struct page *p;
349
350 p = shinfo->frags[fragno].page;
351 shinfo->frags[fragno].size = 0;
352 shinfo->frags[fragno].offset = 0;
353 shinfo->frags[fragno].page = NULL;
354 #ifdef PAGE_IS_LKPI_PAGE
355 if ((skb->_flags & _SKB_PP_RECYCLE) != 0)
356 page_pool_put_full_page(p->pp, p, false);
357 else
358 #endif
359 __free_page(p);
360 count++;
361 }
362 }
363
364 if ((skb->_flags & _SKB_FLAGS_SKBEXTFRAG) != 0) {
365 void *p;
366
367 p = skb->head;
368 skb_free_frag(p);
369 skb->head = NULL;
370 }
371
372 #ifdef SKB_DMA32_MALLOC
373 if (__predict_false(linuxkpi_skb_memlimit != 0))
374 free(skb->head, M_LKPISKB);
375 else
376 #endif
377 kfree(skb->head);
378 uma_zfree(skbzone, skb);
379 }
380
381 static void
lkpi_skbuff_init(void * arg __unused)382 lkpi_skbuff_init(void *arg __unused)
383 {
384 skbzone = uma_zcreate("skbuff",
385 sizeof(struct sk_buff) + sizeof(struct skb_shared_info),
386 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
387 /* Do we need to apply limits? */
388 }
389 SYSINIT(linuxkpi_skbuff, SI_SUB_DRIVERS, SI_ORDER_FIRST, lkpi_skbuff_init, NULL);
390
391 static void
lkpi_skbuff_destroy(void * arg __unused)392 lkpi_skbuff_destroy(void *arg __unused)
393 {
394 uma_zdestroy(skbzone);
395 }
396 SYSUNINIT(linuxkpi_skbuff, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_skbuff_destroy, NULL);
397
398 #ifdef DDB
DB_SHOW_COMMAND(skb,db_show_skb)399 DB_SHOW_COMMAND(skb, db_show_skb)
400 {
401 struct sk_buff *skb;
402 int i;
403
404 if (!have_addr) {
405 db_printf("usage: show skb <addr>\n");
406 return;
407 }
408
409 skb = (struct sk_buff *)addr;
410
411 db_printf("skb %p\n", skb);
412 db_printf("\tnext %p prev %p\n", skb->next, skb->prev);
413 db_printf("\tlist %p\n", &skb->list);
414 db_printf("\tlen %u data_len %u truesize %u mac_len %u\n",
415 skb->len, skb->data_len, skb->truesize, skb->mac_len);
416 db_printf("\tcsum %#06x l3hdroff %u l4hdroff %u priority %u qmap %u\n",
417 skb->csum, skb->l3hdroff, skb->l4hdroff, skb->priority, skb->qmap);
418 db_printf("\tpkt_type %d dev %p sk %p\n",
419 skb->pkt_type, skb->dev, skb->sk);
420 db_printf("\tcsum_offset %d csum_start %d ip_summed %d protocol %d\n",
421 skb->csum_offset, skb->csum_start, skb->ip_summed, skb->protocol);
422 db_printf("\t_flags %#06x\n", skb->_flags); /* XXX-BZ print names? */
423 db_printf("\thead %p data %p tail %p end %p\n",
424 skb->head, skb->data, skb->tail, skb->end);
425 db_printf("\tshinfo %p m %p m_free_func %p\n",
426 skb->shinfo, skb->m, skb->m_free_func);
427
428 if (skb->shinfo != NULL) {
429 struct skb_shared_info *shinfo;
430
431 shinfo = skb->shinfo;
432 db_printf("\t\tgso_type %d gso_size %u nr_frags %u\n",
433 shinfo->gso_type, shinfo->gso_size, shinfo->nr_frags);
434 for (i = 0; i < nitems(shinfo->frags); i++) {
435 struct skb_frag *frag;
436
437 frag = &shinfo->frags[i];
438 if (frag == NULL || frag->page == NULL)
439 continue;
440 db_printf("\t\t\tfrag %p fragno %d page %p %p "
441 "offset %ju size %zu\n",
442 frag, i, frag->page, linux_page_address(frag->page),
443 (uintmax_t)frag->offset, frag->size);
444 }
445 }
446 db_printf("\tcb[] %p {", skb->cb);
447 for (i = 0; i < nitems(skb->cb); i++) {
448 db_printf("%#04x%s",
449 skb->cb[i], (i < (nitems(skb->cb)-1)) ? ", " : "");
450 }
451 db_printf("}\n");
452
453 db_printf("\t__scratch[0] %p\n", skb->__scratch);
454 };
455 #endif
456