1 /*- 2 * Copyright (c) 2020-2025 The FreeBSD Foundation 3 * Copyright (c) 2021-2022 Bjoern A. Zeeb 4 * 5 * This software was developed by Björn Zeeb under sponsorship from 6 * the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * NOTE: this socket buffer compatibility code is highly EXPERIMENTAL. 32 * Do not rely on the internals of this implementation. They are highly 33 * likely to change as we will improve the integration to FreeBSD mbufs. 34 */ 35 36 #include <sys/cdefs.h> 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/types.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/sysctl.h> 44 45 #include <vm/uma.h> 46 47 #ifdef DDB 48 #include <ddb/ddb.h> 49 #endif 50 51 #include <linux/skbuff.h> 52 #include <linux/slab.h> 53 #include <linux/gfp.h> 54 #ifdef __LP64__ 55 #include <linux/log2.h> 56 #endif 57 58 #include <net/page_pool/helpers.h> 59 60 SYSCTL_DECL(_compat_linuxkpi); 61 SYSCTL_NODE(_compat_linuxkpi, OID_AUTO, skb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 62 "LinuxKPI skbuff"); 63 64 #ifdef SKB_DEBUG 65 int linuxkpi_debug_skb; 66 SYSCTL_INT(_compat_linuxkpi_skb, OID_AUTO, debug, CTLFLAG_RWTUN, 67 &linuxkpi_debug_skb, 0, "SKB debug level"); 68 #endif 69 70 static uma_zone_t skbzone; 71 72 #define SKB_DMA32_MALLOC 73 #ifdef SKB_DMA32_MALLOC 74 /* 75 * Realtek wireless drivers (e.g., rtw88) require 32bit DMA in a single segment. 76 * busdma(9) has a hard time providing this currently for 3-ish pages at large 77 * quantities (see lkpi_pci_nseg1_fail in linux_pci.c). 78 * Work around this for now by allowing a tunable to enforce physical addresses 79 * allocation limits using "old-school" contigmalloc(9) to avoid bouncing. 80 * Note: with the malloc/contigmalloc + kmalloc changes also providing physical 81 * contiguous memory, and the nseg=1 limit for bouncing we should in theory be 82 * fine now and not need any of this anymore, however busdma still has troubles 83 * boncing three contiguous pages so for now this stays. 84 */ 85 static int linuxkpi_skb_memlimit; 86 SYSCTL_INT(_compat_linuxkpi_skb, OID_AUTO, mem_limit, CTLFLAG_RDTUN, 87 &linuxkpi_skb_memlimit, 0, "SKB memory limit: 0=no limit, " 88 "1=32bit, 2=36bit, other=undef (currently 32bit)"); 89 90 static MALLOC_DEFINE(M_LKPISKB, "lkpiskb", "Linux KPI skbuff compat"); 91 #endif 92 93 struct sk_buff * 94 linuxkpi_alloc_skb(size_t size, gfp_t gfp) 95 { 96 struct sk_buff *skb; 97 void *p; 98 size_t len; 99 100 skb = uma_zalloc(skbzone, linux_check_m_flags(gfp) | M_ZERO); 101 if (skb == NULL) 102 return (NULL); 103 104 skb->prev = skb->next = skb; 105 skb->truesize = size; 106 skb->shinfo = (struct skb_shared_info *)(skb + 1); 107 108 if (size == 0) 109 return (skb); 110 111 len = size; 112 #ifdef SKB_DMA32_MALLOC 113 /* 114 * Using our own type here not backing my kmalloc. 115 * We assume no one calls kfree directly on the skb. 116 */ 117 if (__predict_false(linuxkpi_skb_memlimit != 0)) { 118 vm_paddr_t high; 119 120 switch (linuxkpi_skb_memlimit) { 121 #ifdef __LP64__ 122 case 2: 123 high = (0xfffffffff); /* 1<<36 really. */ 124 break; 125 #endif 126 case 1: 127 default: 128 high = (0xffffffff); /* 1<<32 really. */ 129 break; 130 } 131 len = roundup_pow_of_two(len); 132 p = contigmalloc(len, M_LKPISKB, 133 linux_check_m_flags(gfp) | M_ZERO, 0, high, PAGE_SIZE, 0); 134 } else 135 #endif 136 p = __kmalloc(len, linux_check_m_flags(gfp) | M_ZERO); 137 if (p == NULL) { 138 uma_zfree(skbzone, skb); 139 return (NULL); 140 } 141 142 skb->head = skb->data = (uint8_t *)p; 143 skb_reset_tail_pointer(skb); 144 skb->end = skb->head + size; 145 refcount_set(&skb->refcnt, 1); 146 147 SKB_TRACE_FMT(skb, "data %p size %zu", (skb) ? skb->data : NULL, size); 148 return (skb); 149 } 150 151 struct sk_buff * 152 linuxkpi_dev_alloc_skb(size_t size, gfp_t gfp) 153 { 154 struct sk_buff *skb; 155 size_t len; 156 157 len = size + NET_SKB_PAD; 158 skb = linuxkpi_alloc_skb(len, gfp); 159 160 if (skb != NULL) 161 skb_reserve(skb, NET_SKB_PAD); 162 163 SKB_TRACE_FMT(skb, "data %p size %zu len %zu", 164 (skb) ? skb->data : NULL, size, len); 165 return (skb); 166 } 167 168 struct sk_buff * 169 linuxkpi_build_skb(void *data, size_t fragsz) 170 { 171 struct sk_buff *skb; 172 173 if (data == NULL || fragsz == 0) 174 return (NULL); 175 176 /* Just allocate a skb without data area. */ 177 skb = linuxkpi_alloc_skb(0, GFP_KERNEL); 178 if (skb == NULL) 179 return (NULL); 180 181 skb->_flags |= _SKB_FLAGS_SKBEXTFRAG; 182 skb->truesize = fragsz; 183 skb->head = skb->data = data; 184 skb_reset_tail_pointer(skb); 185 skb->end = skb->head + fragsz; 186 refcount_set(&skb->refcnt, 1); 187 188 return (skb); 189 } 190 191 struct sk_buff * 192 linuxkpi_skb_copy(const struct sk_buff *skb, gfp_t gfp) 193 { 194 struct sk_buff *new; 195 struct skb_shared_info *shinfo; 196 size_t len; 197 unsigned int headroom; 198 uint16_t fragno, count; 199 200 /* Full buffer size + any fragments. */ 201 len = skb->end - skb->head + skb->data_len; 202 203 new = linuxkpi_alloc_skb(len, gfp); 204 if (new == NULL) 205 return (NULL); 206 207 headroom = skb_headroom(skb); 208 /* Fixup head and end. */ 209 skb_reserve(new, headroom); /* data and tail move headroom forward. */ 210 skb_put(new, skb->len); /* tail and len get adjusted */ 211 212 /* Copy data. */ 213 memcpy(new->head, skb->data - headroom, headroom + skb->len); 214 215 /* Deal with fragments. */ 216 shinfo = skb->shinfo; 217 for (count = fragno = 0; 218 count < shinfo->nr_frags && fragno < nitems(shinfo->frags); 219 fragno++) { 220 if (shinfo->frags[fragno].page != NULL) { 221 skb_put_data(new, 222 skb_frag_address(&shinfo->frags[fragno]), 223 shinfo->frags[fragno].size); 224 count++; 225 } 226 } 227 228 /* Deal with header fields. */ 229 memcpy(new->cb, skb->cb, sizeof(skb->cb)); 230 SKB_IMPROVE("more header fields to copy?"); 231 232 return (new); 233 } 234 235 int 236 lkpi___skb_linearize(struct sk_buff *skb) 237 { 238 struct sk_buff *new; 239 struct skb_shared_info *shinfo; 240 uint16_t fragno, count; 241 242 SKB_TRACE(skb); 243 SKB_IMPROVE("Hack completely re-allocating and freeing; FIXME"); 244 new = skb_copy(skb, GFP_NOWAIT); 245 if (new == NULL) 246 return (-ENOMEM); 247 248 /* Now need to swap head, data, tail, ... and free from old (and then new). */ 249 shinfo = skb->shinfo; 250 for (count = fragno = 0; 251 count < shinfo->nr_frags && fragno < nitems(shinfo->frags); 252 fragno++) { 253 254 if (shinfo->frags[fragno].page != NULL) { 255 struct page *p; 256 257 p = shinfo->frags[fragno].page; 258 shinfo->frags[fragno].size = 0; 259 shinfo->frags[fragno].offset = 0; 260 shinfo->frags[fragno].page = NULL; 261 __free_page(p); 262 count++; 263 } 264 } 265 266 if ((skb->_flags & _SKB_FLAGS_SKBEXTFRAG) != 0) { 267 void *p; 268 269 p = skb->head; 270 skb_free_frag(p); 271 skb->head = NULL; 272 skb->_flags &= ~_SKB_FLAGS_SKBEXTFRAG; 273 } 274 275 #ifdef SKB_DMA32_MALLOC 276 if (__predict_false(linuxkpi_skb_memlimit != 0)) 277 free(skb->head, M_LKPISKB); 278 else 279 #endif 280 kfree(skb->head); 281 282 skb->head = new->head; 283 skb->data = new->data; 284 skb->tail = new->tail; 285 skb->end = new->end; 286 skb->len = new->len; 287 skb->data_len = new->data_len; 288 skb->truesize = new->truesize; 289 290 uma_zfree(skbzone, new); 291 292 return (0); 293 } 294 295 static bool 296 lkpi_skb_refcount_release(struct sk_buff *skb) 297 { 298 if (skb == NULL) 299 return (false); 300 301 /* Do we need further tests to avoid freeing this one? */ 302 303 if (!refcount_dec_and_test(&skb->refcnt)) 304 return (false); 305 306 return (true); 307 } 308 309 void 310 linuxkpi_kfree_skb(struct sk_buff *skb) 311 { 312 struct skb_shared_info *shinfo; 313 uint16_t fragno, count; 314 315 SKB_TRACE(skb); 316 if (skb == NULL) 317 return; 318 319 if (!lkpi_skb_refcount_release(skb)) { 320 SKB_TRACE_FMT(skb, "not freed due to refcnt"); 321 return; 322 } 323 324 /* 325 * XXX TODO this will go away once we have skb backed by mbuf. 326 * currently we allow the mbuf to stay around and use a private 327 * free function to allow secondary resources to be freed along. 328 */ 329 if (skb->m != NULL) { 330 void *m; 331 332 m = skb->m; 333 skb->m = NULL; 334 335 KASSERT(skb->m_free_func != NULL, ("%s: skb %p has m %p but no " 336 "m_free_func %p\n", __func__, skb, m, skb->m_free_func)); 337 skb->m_free_func(m); 338 } 339 KASSERT(skb->m == NULL, 340 ("%s: skb %p m %p != NULL\n", __func__, skb, skb->m)); 341 342 shinfo = skb->shinfo; 343 for (count = fragno = 0; 344 count < shinfo->nr_frags && fragno < nitems(shinfo->frags); 345 fragno++) { 346 347 if (shinfo->frags[fragno].page != NULL) { 348 struct page *p; 349 350 p = shinfo->frags[fragno].page; 351 shinfo->frags[fragno].size = 0; 352 shinfo->frags[fragno].offset = 0; 353 shinfo->frags[fragno].page = NULL; 354 #ifdef PAGE_IS_LKPI_PAGE 355 if ((skb->_flags & _SKB_PP_RECYCLE) != 0) 356 page_pool_put_full_page(p->pp, p, false); 357 else 358 #endif 359 __free_page(p); 360 count++; 361 } 362 } 363 364 if ((skb->_flags & _SKB_FLAGS_SKBEXTFRAG) != 0) { 365 void *p; 366 367 p = skb->head; 368 skb_free_frag(p); 369 skb->head = NULL; 370 } 371 372 #ifdef SKB_DMA32_MALLOC 373 if (__predict_false(linuxkpi_skb_memlimit != 0)) 374 free(skb->head, M_LKPISKB); 375 else 376 #endif 377 kfree(skb->head); 378 uma_zfree(skbzone, skb); 379 } 380 381 static void 382 lkpi_skbuff_init(void *arg __unused) 383 { 384 skbzone = uma_zcreate("skbuff", 385 sizeof(struct sk_buff) + sizeof(struct skb_shared_info), 386 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 387 /* Do we need to apply limits? */ 388 } 389 SYSINIT(linuxkpi_skbuff, SI_SUB_DRIVERS, SI_ORDER_FIRST, lkpi_skbuff_init, NULL); 390 391 static void 392 lkpi_skbuff_destroy(void *arg __unused) 393 { 394 uma_zdestroy(skbzone); 395 } 396 SYSUNINIT(linuxkpi_skbuff, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_skbuff_destroy, NULL); 397 398 #ifdef DDB 399 DB_SHOW_COMMAND(skb, db_show_skb) 400 { 401 struct sk_buff *skb; 402 int i; 403 404 if (!have_addr) { 405 db_printf("usage: show skb <addr>\n"); 406 return; 407 } 408 409 skb = (struct sk_buff *)addr; 410 411 db_printf("skb %p\n", skb); 412 db_printf("\tnext %p prev %p\n", skb->next, skb->prev); 413 db_printf("\tlist %p\n", &skb->list); 414 db_printf("\tlen %u data_len %u truesize %u mac_len %u\n", 415 skb->len, skb->data_len, skb->truesize, skb->mac_len); 416 db_printf("\tcsum %#06x l3hdroff %u l4hdroff %u priority %u qmap %u\n", 417 skb->csum, skb->l3hdroff, skb->l4hdroff, skb->priority, skb->qmap); 418 db_printf("\tpkt_type %d dev %p sk %p\n", 419 skb->pkt_type, skb->dev, skb->sk); 420 db_printf("\tcsum_offset %d csum_start %d ip_summed %d protocol %d\n", 421 skb->csum_offset, skb->csum_start, skb->ip_summed, skb->protocol); 422 db_printf("\t_flags %#06x\n", skb->_flags); /* XXX-BZ print names? */ 423 db_printf("\thead %p data %p tail %p end %p\n", 424 skb->head, skb->data, skb->tail, skb->end); 425 db_printf("\tshinfo %p m %p m_free_func %p\n", 426 skb->shinfo, skb->m, skb->m_free_func); 427 428 if (skb->shinfo != NULL) { 429 struct skb_shared_info *shinfo; 430 431 shinfo = skb->shinfo; 432 db_printf("\t\tgso_type %d gso_size %u nr_frags %u\n", 433 shinfo->gso_type, shinfo->gso_size, shinfo->nr_frags); 434 for (i = 0; i < nitems(shinfo->frags); i++) { 435 struct skb_frag *frag; 436 437 frag = &shinfo->frags[i]; 438 if (frag == NULL || frag->page == NULL) 439 continue; 440 db_printf("\t\t\tfrag %p fragno %d page %p %p " 441 "offset %ju size %zu\n", 442 frag, i, frag->page, linux_page_address(frag->page), 443 (uintmax_t)frag->offset, frag->size); 444 } 445 } 446 db_printf("\tcb[] %p {", skb->cb); 447 for (i = 0; i < nitems(skb->cb); i++) { 448 db_printf("%#04x%s", 449 skb->cb[i], (i < (nitems(skb->cb)-1)) ? ", " : ""); 450 } 451 db_printf("}\n"); 452 453 db_printf("\t__scratch[0] %p\n", skb->__scratch); 454 }; 455 #endif 456