1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/skbuff_ref.h> 55 #include <linux/splice.h> 56 #include <linux/cache.h> 57 #include <linux/rtnetlink.h> 58 #include <linux/init.h> 59 #include <linux/scatterlist.h> 60 #include <linux/errqueue.h> 61 #include <linux/prefetch.h> 62 #include <linux/bitfield.h> 63 #include <linux/if_vlan.h> 64 #include <linux/mpls.h> 65 #include <linux/kcov.h> 66 #include <linux/iov_iter.h> 67 #include <linux/crc32.h> 68 #include <kunit/visibility.h> 69 70 #include <net/protocol.h> 71 #include <net/dst.h> 72 #include <net/sock.h> 73 #include <net/checksum.h> 74 #include <net/gro.h> 75 #include <net/gso.h> 76 #include <net/hotdata.h> 77 #include <net/ip6_checksum.h> 78 #include <net/xfrm.h> 79 #include <net/mpls.h> 80 #include <net/mptcp.h> 81 #include <net/mctp.h> 82 #include <net/tcp.h> 83 #include <net/can.h> 84 #include <net/page_pool/helpers.h> 85 #include <net/psp/types.h> 86 #include <net/dropreason.h> 87 #include <net/xdp_sock.h> 88 89 #include <linux/uaccess.h> 90 #include <trace/events/skb.h> 91 #include <linux/highmem.h> 92 #include <linux/capability.h> 93 #include <linux/user_namespace.h> 94 #include <linux/indirect_call_wrapper.h> 95 #include <linux/textsearch.h> 96 97 #include "dev.h" 98 #include "devmem.h" 99 #include "net-sysfs.h" 100 #include "netmem_priv.h" 101 102 #ifdef CONFIG_SKB_EXTENSIONS 103 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 104 #endif 105 106 #define GRO_MAX_HEAD_PAD (GRO_MAX_HEAD + NET_SKB_PAD + NET_IP_ALIGN) 107 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(max(MAX_TCP_HEADER, \ 108 GRO_MAX_HEAD_PAD)) 109 110 /* SKB_SMALL_HEAD_CACHE_SIZE is the size used for the skbuff_small_head 111 * kmem_cache. The non-power-of-2 padding is kept for historical reasons and 112 * to avoid potential collisions with generic kmalloc bucket sizes. 113 */ 114 #define SKB_SMALL_HEAD_CACHE_SIZE \ 115 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \ 116 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \ 117 SKB_SMALL_HEAD_SIZE) 118 119 #define SKB_SMALL_HEAD_HEADROOM \ 120 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) 121 122 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use 123 * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the 124 * netmem is a page. 125 */ 126 static_assert(offsetof(struct bio_vec, bv_page) == 127 offsetof(skb_frag_t, netmem)); 128 static_assert(sizeof_field(struct bio_vec, bv_page) == 129 sizeof_field(skb_frag_t, netmem)); 130 131 static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len)); 132 static_assert(sizeof_field(struct bio_vec, bv_len) == 133 sizeof_field(skb_frag_t, len)); 134 135 static_assert(offsetof(struct bio_vec, bv_offset) == 136 offsetof(skb_frag_t, offset)); 137 static_assert(sizeof_field(struct bio_vec, bv_offset) == 138 sizeof_field(skb_frag_t, offset)); 139 140 #undef FN 141 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 142 static const char * const drop_reasons[] = { 143 [SKB_CONSUMED] = "CONSUMED", 144 DEFINE_DROP_REASON(FN, FN) 145 }; 146 147 static const struct drop_reason_list drop_reasons_core = { 148 .reasons = drop_reasons, 149 .n_reasons = ARRAY_SIZE(drop_reasons), 150 }; 151 152 const struct drop_reason_list __rcu * 153 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = { 154 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core), 155 }; 156 EXPORT_SYMBOL(drop_reasons_by_subsys); 157 158 /** 159 * drop_reasons_register_subsys - register another drop reason subsystem 160 * @subsys: the subsystem to register, must not be the core 161 * @list: the list of drop reasons within the subsystem, must point to 162 * a statically initialized list 163 */ 164 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys, 165 const struct drop_reason_list *list) 166 { 167 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 168 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 169 "invalid subsystem %d\n", subsys)) 170 return; 171 172 /* must point to statically allocated memory, so INIT is OK */ 173 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list); 174 } 175 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys); 176 177 /** 178 * drop_reasons_unregister_subsys - unregister a drop reason subsystem 179 * @subsys: the subsystem to remove, must not be the core 180 * 181 * Note: This will synchronize_rcu() to ensure no users when it returns. 182 */ 183 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys) 184 { 185 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 186 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 187 "invalid subsystem %d\n", subsys)) 188 return; 189 190 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL); 191 192 synchronize_rcu(); 193 } 194 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys); 195 196 /** 197 * skb_panic - private function for out-of-line support 198 * @skb: buffer 199 * @sz: size 200 * @addr: address 201 * @msg: skb_over_panic or skb_under_panic 202 * 203 * Out-of-line support for skb_put() and skb_push(). 204 * Called via the wrapper skb_over_panic() or skb_under_panic(). 205 * Keep out of line to prevent kernel bloat. 206 * __builtin_return_address is not used because it is not always reliable. 207 */ 208 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 209 const char msg[]) 210 { 211 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 212 msg, addr, skb->len, sz, skb->head, skb->data, 213 (unsigned long)skb->tail, (unsigned long)skb->end, 214 skb->dev ? skb->dev->name : "<NULL>"); 215 BUG(); 216 } 217 218 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 219 { 220 skb_panic(skb, sz, addr, __func__); 221 } 222 223 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 224 { 225 skb_panic(skb, sz, addr, __func__); 226 } 227 228 #define NAPI_SKB_CACHE_SIZE 128 229 #define NAPI_SKB_CACHE_BULK 32 230 #define NAPI_SKB_CACHE_FREE 32 231 232 struct napi_alloc_cache { 233 local_lock_t bh_lock; 234 struct page_frag_cache page; 235 unsigned int skb_count; 236 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 237 }; 238 239 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 240 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache) = { 241 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 242 }; 243 244 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 245 { 246 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 247 void *data; 248 249 fragsz = SKB_DATA_ALIGN(fragsz); 250 251 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 252 data = __page_frag_alloc_align(&nc->page, fragsz, 253 GFP_ATOMIC | __GFP_NOWARN, align_mask); 254 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 255 return data; 256 257 } 258 EXPORT_SYMBOL(__napi_alloc_frag_align); 259 260 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 261 { 262 void *data; 263 264 if (in_hardirq() || irqs_disabled()) { 265 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 266 267 fragsz = SKB_DATA_ALIGN(fragsz); 268 data = __page_frag_alloc_align(nc, fragsz, 269 GFP_ATOMIC | __GFP_NOWARN, 270 align_mask); 271 } else { 272 local_bh_disable(); 273 data = __napi_alloc_frag_align(fragsz, align_mask); 274 local_bh_enable(); 275 } 276 return data; 277 } 278 EXPORT_SYMBOL(__netdev_alloc_frag_align); 279 280 /* Cache kmem_cache_size(net_hotdata.skbuff_cache) to help the compiler 281 * remove dead code (and skbuff_cache_size) when CONFIG_KASAN is unset. 282 */ 283 static u32 skbuff_cache_size __read_mostly; 284 285 static inline struct sk_buff *napi_skb_cache_get(bool alloc) 286 { 287 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 288 struct sk_buff *skb; 289 290 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 291 if (unlikely(!nc->skb_count)) { 292 if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 293 GFP_ATOMIC | __GFP_NOWARN, 294 NAPI_SKB_CACHE_BULK, 295 nc->skb_cache)) 296 nc->skb_count = NAPI_SKB_CACHE_BULK; 297 if (unlikely(!nc->skb_count)) { 298 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 299 return NULL; 300 } 301 } 302 303 skb = nc->skb_cache[--nc->skb_count]; 304 if (nc->skb_count) 305 prefetch(nc->skb_cache[nc->skb_count - 1]); 306 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 307 kasan_mempool_unpoison_object(skb, skbuff_cache_size); 308 309 return skb; 310 } 311 312 /* 313 * Only clear those fields we need to clear, not those that we will 314 * actually initialise later. Hence, don't put any more fields after 315 * the tail pointer in struct sk_buff! 316 */ 317 static inline void skbuff_clear(struct sk_buff *skb) 318 { 319 /* Replace memset(skb, 0, offsetof(struct sk_buff, tail)) 320 * with two smaller memset(), with a barrier() between them. 321 * This forces the compiler to inline both calls. 322 */ 323 BUILD_BUG_ON(offsetof(struct sk_buff, tail) <= 128); 324 memset(skb, 0, 128); 325 barrier(); 326 memset((void *)skb + 128, 0, offsetof(struct sk_buff, tail) - 128); 327 } 328 329 /** 330 * napi_skb_cache_get_bulk - obtain a number of zeroed skb heads from the cache 331 * @skbs: pointer to an at least @n-sized array to fill with skb pointers 332 * @n: number of entries to provide 333 * 334 * Tries to obtain @n &sk_buff entries from the NAPI percpu cache and writes 335 * the pointers into the provided array @skbs. If there are less entries 336 * available, tries to replenish the cache and bulk-allocates the diff from 337 * the MM layer if needed. 338 * The heads are being zeroed with either memset() or %__GFP_ZERO, so they are 339 * ready for {,__}build_skb_around() and don't have any data buffers attached. 340 * Must be called *only* from the BH context. 341 * 342 * Return: number of successfully allocated skbs (@n if no actual allocation 343 * needed or kmem_cache_alloc_bulk() didn't fail). 344 */ 345 u32 napi_skb_cache_get_bulk(void **skbs, u32 n) 346 { 347 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 348 u32 bulk, total = n; 349 350 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 351 352 if (nc->skb_count >= n) 353 goto get; 354 355 /* No enough cached skbs. Try refilling the cache first */ 356 bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK); 357 if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 358 GFP_ATOMIC | __GFP_NOWARN, bulk, 359 &nc->skb_cache[nc->skb_count])) 360 nc->skb_count += bulk; 361 if (likely(nc->skb_count >= n)) 362 goto get; 363 364 /* Still not enough. Bulk-allocate the missing part directly, zeroed */ 365 if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 366 GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, 367 n - nc->skb_count, &skbs[nc->skb_count])) 368 n = nc->skb_count; 369 if (likely(nc->skb_count >= n)) 370 goto get; 371 372 /* kmem_cache didn't allocate the number we need, limit the output */ 373 total -= n - nc->skb_count; 374 n = nc->skb_count; 375 376 get: 377 for (u32 base = nc->skb_count - n, i = 0; i < n; i++) { 378 skbs[i] = nc->skb_cache[base + i]; 379 380 kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size); 381 skbuff_clear(skbs[i]); 382 } 383 384 nc->skb_count -= n; 385 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 386 387 return total; 388 } 389 EXPORT_SYMBOL_GPL(napi_skb_cache_get_bulk); 390 391 static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 392 unsigned int size) 393 { 394 struct skb_shared_info *shinfo; 395 396 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 397 398 /* Assumes caller memset cleared SKB */ 399 skb->truesize = SKB_TRUESIZE(size); 400 refcount_set(&skb->users, 1); 401 skb->head = data; 402 skb->data = data; 403 skb_reset_tail_pointer(skb); 404 skb_set_end_offset(skb, size); 405 skb->mac_header = (typeof(skb->mac_header))~0U; 406 skb->transport_header = (typeof(skb->transport_header))~0U; 407 skb->alloc_cpu = raw_smp_processor_id(); 408 /* make sure we initialize shinfo sequentially */ 409 shinfo = skb_shinfo(skb); 410 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 411 atomic_set(&shinfo->dataref, 1); 412 413 skb_set_kcov_handle(skb, kcov_common_handle()); 414 } 415 416 static inline void *__slab_build_skb(void *data, unsigned int *size) 417 { 418 void *resized; 419 420 /* Must find the allocation size (and grow it to match). */ 421 *size = ksize(data); 422 /* krealloc() will immediately return "data" when 423 * "ksize(data)" is requested: it is the existing upper 424 * bounds. As a result, GFP_ATOMIC will be ignored. Note 425 * that this "new" pointer needs to be passed back to the 426 * caller for use so the __alloc_size hinting will be 427 * tracked correctly. 428 */ 429 resized = krealloc(data, *size, GFP_ATOMIC); 430 WARN_ON_ONCE(resized != data); 431 return resized; 432 } 433 434 /* build_skb() variant which can operate on slab buffers. 435 * Note that this should be used sparingly as slab buffers 436 * cannot be combined efficiently by GRO! 437 */ 438 struct sk_buff *slab_build_skb(void *data) 439 { 440 struct sk_buff *skb; 441 unsigned int size; 442 443 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, 444 GFP_ATOMIC | __GFP_NOWARN); 445 if (unlikely(!skb)) 446 return NULL; 447 448 skbuff_clear(skb); 449 data = __slab_build_skb(data, &size); 450 __finalize_skb_around(skb, data, size); 451 452 return skb; 453 } 454 EXPORT_SYMBOL(slab_build_skb); 455 456 /* Caller must provide SKB that is memset cleared */ 457 static void __build_skb_around(struct sk_buff *skb, void *data, 458 unsigned int frag_size) 459 { 460 unsigned int size = frag_size; 461 462 /* frag_size == 0 is considered deprecated now. Callers 463 * using slab buffer should use slab_build_skb() instead. 464 */ 465 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 466 data = __slab_build_skb(data, &size); 467 468 __finalize_skb_around(skb, data, size); 469 } 470 471 /** 472 * __build_skb - build a network buffer 473 * @data: data buffer provided by caller 474 * @frag_size: size of data (must not be 0) 475 * 476 * Allocate a new &sk_buff. Caller provides space holding head and 477 * skb_shared_info. @data must have been allocated from the page 478 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 479 * allocation is deprecated, and callers should use slab_build_skb() 480 * instead.) 481 * The return is the new skb buffer. 482 * On a failure the return is %NULL, and @data is not freed. 483 * Notes : 484 * Before IO, driver allocates only data buffer where NIC put incoming frame 485 * Driver should add room at head (NET_SKB_PAD) and 486 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 487 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 488 * before giving packet to stack. 489 * RX rings only contains data buffers, not full skbs. 490 */ 491 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 492 { 493 struct sk_buff *skb; 494 495 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, 496 GFP_ATOMIC | __GFP_NOWARN); 497 if (unlikely(!skb)) 498 return NULL; 499 500 skbuff_clear(skb); 501 __build_skb_around(skb, data, frag_size); 502 503 return skb; 504 } 505 506 /* build_skb() is wrapper over __build_skb(), that specifically 507 * takes care of skb->head and skb->pfmemalloc 508 */ 509 struct sk_buff *build_skb(void *data, unsigned int frag_size) 510 { 511 struct sk_buff *skb = __build_skb(data, frag_size); 512 513 if (likely(skb && frag_size)) { 514 skb->head_frag = 1; 515 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 516 } 517 return skb; 518 } 519 EXPORT_SYMBOL(build_skb); 520 521 /** 522 * build_skb_around - build a network buffer around provided skb 523 * @skb: sk_buff provide by caller, must be memset cleared 524 * @data: data buffer provided by caller 525 * @frag_size: size of data 526 */ 527 struct sk_buff *build_skb_around(struct sk_buff *skb, 528 void *data, unsigned int frag_size) 529 { 530 if (unlikely(!skb)) 531 return NULL; 532 533 __build_skb_around(skb, data, frag_size); 534 535 if (frag_size) { 536 skb->head_frag = 1; 537 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 538 } 539 return skb; 540 } 541 EXPORT_SYMBOL(build_skb_around); 542 543 /** 544 * __napi_build_skb - build a network buffer 545 * @data: data buffer provided by caller 546 * @frag_size: size of data 547 * 548 * Version of __build_skb() that uses NAPI percpu caches to obtain 549 * skbuff_head instead of inplace allocation. 550 * 551 * Returns a new &sk_buff on success, %NULL on allocation failure. 552 */ 553 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 554 { 555 struct sk_buff *skb; 556 557 skb = napi_skb_cache_get(true); 558 if (unlikely(!skb)) 559 return NULL; 560 561 skbuff_clear(skb); 562 __build_skb_around(skb, data, frag_size); 563 564 return skb; 565 } 566 567 /** 568 * napi_build_skb - build a network buffer 569 * @data: data buffer provided by caller 570 * @frag_size: size of data 571 * 572 * Version of __napi_build_skb() that takes care of skb->head_frag 573 * and skb->pfmemalloc when the data is a page or page fragment. 574 * 575 * Returns a new &sk_buff on success, %NULL on allocation failure. 576 */ 577 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 578 { 579 struct sk_buff *skb = __napi_build_skb(data, frag_size); 580 581 if (likely(skb) && frag_size) { 582 skb->head_frag = 1; 583 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 584 } 585 586 return skb; 587 } 588 EXPORT_SYMBOL(napi_build_skb); 589 590 static void *kmalloc_pfmemalloc(size_t obj_size, gfp_t flags, int node) 591 { 592 if (!gfp_pfmemalloc_allowed(flags)) 593 return NULL; 594 if (!obj_size) 595 return kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, 596 flags, node); 597 return kmalloc_node_track_caller(obj_size, flags, node); 598 } 599 600 /* 601 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 602 * the caller if emergency pfmemalloc reserves are being used. If it is and 603 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 604 * may be used. Otherwise, the packet data may be discarded until enough 605 * memory is free 606 */ 607 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 608 struct sk_buff *skb) 609 { 610 size_t obj_size; 611 void *obj; 612 613 obj_size = SKB_HEAD_ALIGN(*size); 614 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && 615 !(flags & KMALLOC_NOT_NORMAL_BITS)) { 616 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, 617 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 618 node); 619 *size = SKB_SMALL_HEAD_CACHE_SIZE; 620 if (likely(obj)) 621 goto out; 622 /* Try again but now we are using pfmemalloc reserves */ 623 if (skb) 624 skb->pfmemalloc = true; 625 return kmalloc_pfmemalloc(0, flags, node); 626 } 627 628 obj_size = kmalloc_size_roundup(obj_size); 629 /* The following cast might truncate high-order bits of obj_size, this 630 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway. 631 */ 632 *size = (unsigned int)obj_size; 633 634 /* 635 * Try a regular allocation, when that fails and we're not entitled 636 * to the reserves, fail. 637 */ 638 obj = kmalloc_node_track_caller(obj_size, 639 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 640 node); 641 if (likely(obj)) 642 goto out; 643 644 /* Try again but now we are using pfmemalloc reserves */ 645 if (skb) 646 skb->pfmemalloc = true; 647 obj = kmalloc_pfmemalloc(obj_size, flags, node); 648 out: 649 return obj; 650 } 651 652 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 653 * 'private' fields and also do memory statistics to find all the 654 * [BEEP] leaks. 655 * 656 */ 657 658 /** 659 * __alloc_skb - allocate a network buffer 660 * @size: size to allocate 661 * @gfp_mask: allocation mask 662 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 663 * instead of head cache and allocate a cloned (child) skb. 664 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 665 * allocations in case the data is required for writeback 666 * @node: numa node to allocate memory on 667 * 668 * Allocate a new &sk_buff. The returned buffer has no headroom and a 669 * tail room of at least size bytes. The object has a reference count 670 * of one. The return is the buffer. On a failure the return is %NULL. 671 * 672 * Buffers may only be allocated from interrupts using a @gfp_mask of 673 * %GFP_ATOMIC. 674 */ 675 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 676 int flags, int node) 677 { 678 struct sk_buff *skb = NULL; 679 struct kmem_cache *cache; 680 u8 *data; 681 682 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 683 gfp_mask |= __GFP_MEMALLOC; 684 685 if (flags & SKB_ALLOC_FCLONE) { 686 cache = net_hotdata.skbuff_fclone_cache; 687 goto fallback; 688 } 689 cache = net_hotdata.skbuff_cache; 690 if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id())) 691 goto fallback; 692 693 if (flags & SKB_ALLOC_NAPI) { 694 skb = napi_skb_cache_get(true); 695 if (unlikely(!skb)) 696 return NULL; 697 } else if (!in_hardirq() && !irqs_disabled()) { 698 local_bh_disable(); 699 skb = napi_skb_cache_get(false); 700 local_bh_enable(); 701 } 702 703 if (!skb) { 704 fallback: 705 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 706 if (unlikely(!skb)) 707 return NULL; 708 } 709 skbuff_clear(skb); 710 711 /* We do our best to align skb_shared_info on a separate cache 712 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 713 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 714 * Both skb->head and skb_shared_info are cache line aligned. 715 */ 716 data = kmalloc_reserve(&size, gfp_mask, node, skb); 717 if (unlikely(!data)) 718 goto nodata; 719 /* kmalloc_size_roundup() might give us more room than requested. 720 * Put skb_shared_info exactly at the end of allocated zone, 721 * to allow max possible filling before reallocation. 722 */ 723 __finalize_skb_around(skb, data, size); 724 725 if (flags & SKB_ALLOC_FCLONE) { 726 struct sk_buff_fclones *fclones; 727 728 fclones = container_of(skb, struct sk_buff_fclones, skb1); 729 730 /* skb->fclone is a 2bits field. 731 * Replace expensive RMW (skb->fclone = SKB_FCLONE_ORIG) 732 * with a single OR. 733 */ 734 BUILD_BUG_ON(SKB_FCLONE_UNAVAILABLE != 0); 735 DEBUG_NET_WARN_ON_ONCE(skb->fclone != SKB_FCLONE_UNAVAILABLE); 736 skb->fclone |= SKB_FCLONE_ORIG; 737 738 refcount_set(&fclones->fclone_ref, 1); 739 } 740 741 return skb; 742 743 nodata: 744 kmem_cache_free(cache, skb); 745 return NULL; 746 } 747 EXPORT_SYMBOL(__alloc_skb); 748 749 /** 750 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 751 * @dev: network device to receive on 752 * @len: length to allocate 753 * @gfp_mask: get_free_pages mask, passed to alloc_skb 754 * 755 * Allocate a new &sk_buff and assign it a usage count of one. The 756 * buffer has NET_SKB_PAD headroom built in. Users should allocate 757 * the headroom they think they need without accounting for the 758 * built in space. The built in space is used for optimisations. 759 * 760 * %NULL is returned if there is no free memory. 761 */ 762 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 763 gfp_t gfp_mask) 764 { 765 struct page_frag_cache *nc; 766 struct sk_buff *skb; 767 bool pfmemalloc; 768 void *data; 769 770 len += NET_SKB_PAD; 771 772 /* If requested length is either too small or too big, 773 * we use kmalloc() for skb->head allocation. 774 */ 775 if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) || 776 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 777 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 778 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 779 if (!skb) 780 goto skb_fail; 781 goto skb_success; 782 } 783 784 len = SKB_HEAD_ALIGN(len); 785 786 if (sk_memalloc_socks()) 787 gfp_mask |= __GFP_MEMALLOC; 788 789 if (in_hardirq() || irqs_disabled()) { 790 nc = this_cpu_ptr(&netdev_alloc_cache); 791 data = page_frag_alloc(nc, len, gfp_mask); 792 pfmemalloc = page_frag_cache_is_pfmemalloc(nc); 793 } else { 794 local_bh_disable(); 795 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 796 797 nc = this_cpu_ptr(&napi_alloc_cache.page); 798 data = page_frag_alloc(nc, len, gfp_mask); 799 pfmemalloc = page_frag_cache_is_pfmemalloc(nc); 800 801 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 802 local_bh_enable(); 803 } 804 805 if (unlikely(!data)) 806 return NULL; 807 808 skb = __build_skb(data, len); 809 if (unlikely(!skb)) { 810 skb_free_frag(data); 811 return NULL; 812 } 813 814 if (pfmemalloc) 815 skb->pfmemalloc = 1; 816 skb->head_frag = 1; 817 818 skb_success: 819 skb_reserve(skb, NET_SKB_PAD); 820 skb->dev = dev; 821 822 skb_fail: 823 return skb; 824 } 825 EXPORT_SYMBOL(__netdev_alloc_skb); 826 827 /** 828 * napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 829 * @napi: napi instance this buffer was allocated for 830 * @len: length to allocate 831 * 832 * Allocate a new sk_buff for use in NAPI receive. This buffer will 833 * attempt to allocate the head from a special reserved region used 834 * only for NAPI Rx allocation. By doing this we can save several 835 * CPU cycles by avoiding having to disable and re-enable IRQs. 836 * 837 * %NULL is returned if there is no free memory. 838 */ 839 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int len) 840 { 841 gfp_t gfp_mask = GFP_ATOMIC | __GFP_NOWARN; 842 struct napi_alloc_cache *nc; 843 struct sk_buff *skb; 844 bool pfmemalloc; 845 void *data; 846 847 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 848 len += NET_SKB_PAD + NET_IP_ALIGN; 849 850 /* If requested length is either too small or too big, 851 * we use kmalloc() for skb->head allocation. 852 */ 853 if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) || 854 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 855 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 856 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 857 NUMA_NO_NODE); 858 if (!skb) 859 goto skb_fail; 860 goto skb_success; 861 } 862 863 len = SKB_HEAD_ALIGN(len); 864 865 if (sk_memalloc_socks()) 866 gfp_mask |= __GFP_MEMALLOC; 867 868 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 869 nc = this_cpu_ptr(&napi_alloc_cache); 870 871 data = page_frag_alloc(&nc->page, len, gfp_mask); 872 pfmemalloc = page_frag_cache_is_pfmemalloc(&nc->page); 873 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 874 875 if (unlikely(!data)) 876 return NULL; 877 878 skb = __napi_build_skb(data, len); 879 if (unlikely(!skb)) { 880 skb_free_frag(data); 881 return NULL; 882 } 883 884 if (pfmemalloc) 885 skb->pfmemalloc = 1; 886 skb->head_frag = 1; 887 888 skb_success: 889 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 890 skb->dev = napi->dev; 891 892 skb_fail: 893 return skb; 894 } 895 EXPORT_SYMBOL(napi_alloc_skb); 896 897 898 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 899 unsigned int truesize) 900 { 901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 902 903 DEBUG_NET_WARN_ON_ONCE(size > truesize); 904 905 skb_frag_size_add(frag, size); 906 skb->len += size; 907 skb->data_len += size; 908 skb->truesize += truesize; 909 } 910 EXPORT_SYMBOL(skb_coalesce_rx_frag); 911 912 static void skb_drop_list(struct sk_buff **listp) 913 { 914 kfree_skb_list(*listp); 915 *listp = NULL; 916 } 917 918 static inline void skb_drop_fraglist(struct sk_buff *skb) 919 { 920 skb_drop_list(&skb_shinfo(skb)->frag_list); 921 } 922 923 static void skb_clone_fraglist(struct sk_buff *skb) 924 { 925 struct sk_buff *list; 926 927 skb_walk_frags(skb, list) 928 skb_get(list); 929 } 930 931 /** 932 * skb_pp_cow_data() - copy skb data into page-pool backed storage 933 * @pool: page pool to allocate from 934 * @pskb: pointer to skb pointer, replaced with the copied skb on success 935 * @headroom: headroom to reserve in the copied skb 936 * 937 * skb_copy_bits() handles both frags[] and frag_list input. If the copied 938 * skb remains non-linear, it uses frags[], which is the representation used 939 * by XDP multi-buffer. 940 * 941 * Return: 0 on success or a negative errno on failure. 942 */ 943 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, 944 unsigned int headroom) 945 { 946 #if IS_ENABLED(CONFIG_PAGE_POOL) 947 u32 size, truesize, len, max_head_size, off; 948 struct sk_buff *skb = *pskb, *nskb; 949 int err, i, head_off; 950 void *data; 951 952 max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); 953 if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) 954 return -ENOMEM; 955 956 size = min_t(u32, skb->len, max_head_size); 957 truesize = SKB_HEAD_ALIGN(size) + headroom; 958 data = page_pool_dev_alloc_va(pool, &truesize); 959 if (!data) 960 return -ENOMEM; 961 962 nskb = napi_build_skb(data, truesize); 963 if (!nskb) { 964 page_pool_free_va(pool, data, true); 965 return -ENOMEM; 966 } 967 968 skb_reserve(nskb, headroom); 969 skb_copy_header(nskb, skb); 970 nskb->mac_len = skb->mac_len; 971 skb_mark_for_recycle(nskb); 972 973 err = skb_copy_bits(skb, 0, nskb->data, size); 974 if (err) { 975 consume_skb(nskb); 976 return err; 977 } 978 skb_put(nskb, size); 979 980 head_off = skb_headroom(nskb) - skb_headroom(skb); 981 skb_headers_offset_update(nskb, head_off); 982 983 off = size; 984 len = skb->len - off; 985 for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) { 986 struct page *page; 987 u32 page_off; 988 989 size = min_t(u32, len, PAGE_SIZE); 990 truesize = size; 991 992 page = page_pool_dev_alloc(pool, &page_off, &truesize); 993 if (!page) { 994 consume_skb(nskb); 995 return -ENOMEM; 996 } 997 998 skb_add_rx_frag(nskb, i, page, page_off, size, truesize); 999 err = skb_copy_bits(skb, off, page_address(page) + page_off, 1000 size); 1001 if (err) { 1002 consume_skb(nskb); 1003 return err; 1004 } 1005 1006 len -= size; 1007 off += size; 1008 } 1009 1010 consume_skb(skb); 1011 *pskb = nskb; 1012 1013 return 0; 1014 #else 1015 return -EOPNOTSUPP; 1016 #endif 1017 } 1018 EXPORT_SYMBOL(skb_pp_cow_data); 1019 1020 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, 1021 const struct bpf_prog *prog) 1022 { 1023 if (!prog->aux->xdp_has_frags) 1024 return -EINVAL; 1025 1026 return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM); 1027 } 1028 EXPORT_SYMBOL(skb_cow_data_for_xdp); 1029 1030 #if IS_ENABLED(CONFIG_PAGE_POOL) 1031 /** 1032 * napi_pp_put_page - release a page_pool reference on a netmem 1033 * @netmem: netmem to unreference 1034 * 1035 * Drops a page_pool reference (pp_ref_count) on @netmem if it belongs to a 1036 * page_pool. Counterpart to napi_pp_get_page(). 1037 * 1038 * Return: true if @netmem belongs to a page_pool, false otherwise. 1039 */ 1040 bool napi_pp_put_page(netmem_ref netmem) 1041 { 1042 netmem = netmem_compound_head(netmem); 1043 1044 if (unlikely(!netmem_is_pp(netmem))) 1045 return false; 1046 1047 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false); 1048 1049 return true; 1050 } 1051 EXPORT_SYMBOL(napi_pp_put_page); 1052 1053 /** 1054 * napi_pp_get_page - acquire a page_pool reference on a netmem 1055 * @netmem: netmem to reference 1056 * 1057 * Acquires a page_pool reference (pp_ref_count) on @netmem if it belongs to a 1058 * page_pool. Counterpart to napi_pp_put_page(). 1059 * 1060 * Return: true if @netmem belongs to a page_pool, false otherwise. 1061 */ 1062 bool napi_pp_get_page(netmem_ref netmem) 1063 { 1064 netmem = netmem_compound_head(netmem); 1065 1066 if (unlikely(!netmem_is_pp(netmem))) 1067 return false; 1068 1069 page_pool_ref_netmem(netmem); 1070 1071 return true; 1072 } 1073 EXPORT_SYMBOL(napi_pp_get_page); 1074 #endif 1075 1076 static bool skb_pp_recycle(struct sk_buff *skb, void *data) 1077 { 1078 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 1079 return false; 1080 return napi_pp_put_page(page_to_netmem(virt_to_page(data))); 1081 } 1082 1083 static void skb_kfree_head(void *head) 1084 { 1085 kfree(head); 1086 } 1087 1088 static void skb_free_head(struct sk_buff *skb) 1089 { 1090 unsigned char *head = skb->head; 1091 1092 if (skb->head_frag) { 1093 if (skb_pp_recycle(skb, head)) 1094 return; 1095 skb_free_frag(head); 1096 } else { 1097 skb_kfree_head(head); 1098 } 1099 } 1100 1101 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason) 1102 { 1103 struct skb_shared_info *shinfo = skb_shinfo(skb); 1104 int i; 1105 1106 if (!skb_data_unref(skb, shinfo)) 1107 return; 1108 1109 if (skb_zcopy(skb)) { 1110 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 1111 1112 skb_zcopy_clear(skb, true); 1113 if (skip_unref) 1114 goto free_head; 1115 } 1116 1117 for (i = 0; i < shinfo->nr_frags; i++) 1118 skb_netmem_unref(skb_frag_netmem(&shinfo->frags[i]), 1119 skb->pp_recycle); 1120 1121 free_head: 1122 if (shinfo->frag_list) 1123 kfree_skb_list_reason(shinfo->frag_list, reason); 1124 1125 skb_free_head(skb); 1126 } 1127 1128 /* 1129 * Free an skbuff by memory without cleaning the state. 1130 */ 1131 static void kfree_skbmem(struct sk_buff *skb) 1132 { 1133 struct sk_buff_fclones *fclones; 1134 1135 switch (skb->fclone) { 1136 case SKB_FCLONE_UNAVAILABLE: 1137 kmem_cache_free(net_hotdata.skbuff_cache, skb); 1138 return; 1139 1140 case SKB_FCLONE_ORIG: 1141 fclones = container_of(skb, struct sk_buff_fclones, skb1); 1142 1143 /* We usually free the clone (TX completion) before original skb 1144 * This test would have no chance to be true for the clone, 1145 * while here, branch prediction will be good. 1146 */ 1147 if (refcount_read(&fclones->fclone_ref) == 1) 1148 goto fastpath; 1149 break; 1150 1151 default: /* SKB_FCLONE_CLONE */ 1152 fclones = container_of(skb, struct sk_buff_fclones, skb2); 1153 break; 1154 } 1155 if (!refcount_dec_and_test(&fclones->fclone_ref)) 1156 return; 1157 fastpath: 1158 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones); 1159 } 1160 1161 void skb_release_head_state(struct sk_buff *skb) 1162 { 1163 skb_dst_drop(skb); 1164 if (skb->destructor) { 1165 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 1166 #ifdef CONFIG_INET 1167 INDIRECT_CALL_4(skb->destructor, 1168 tcp_wfree, __sock_wfree, sock_wfree, 1169 xsk_destruct_skb, 1170 skb); 1171 #else 1172 INDIRECT_CALL_2(skb->destructor, 1173 sock_wfree, xsk_destruct_skb, 1174 skb); 1175 1176 #endif 1177 skb->destructor = NULL; 1178 skb->sk = NULL; 1179 } 1180 nf_reset_ct(skb); 1181 skb_ext_reset(skb); 1182 } 1183 1184 /* Free everything but the sk_buff shell. */ 1185 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason) 1186 { 1187 skb_release_head_state(skb); 1188 if (likely(skb->head)) 1189 skb_release_data(skb, reason); 1190 } 1191 1192 /** 1193 * __kfree_skb - private function 1194 * @skb: buffer 1195 * 1196 * Free an sk_buff. Release anything attached to the buffer. 1197 * Clean the state. This is an internal helper function. Users should 1198 * always call kfree_skb 1199 */ 1200 1201 void __kfree_skb(struct sk_buff *skb) 1202 { 1203 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED); 1204 kfree_skbmem(skb); 1205 } 1206 EXPORT_SYMBOL(__kfree_skb); 1207 1208 static __always_inline 1209 bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, 1210 enum skb_drop_reason reason) 1211 { 1212 if (unlikely(!skb_unref(skb))) 1213 return false; 1214 1215 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET || 1216 u32_get_bits(reason, 1217 SKB_DROP_REASON_SUBSYS_MASK) >= 1218 SKB_DROP_REASON_SUBSYS_NUM); 1219 1220 if (reason == SKB_CONSUMED) 1221 trace_consume_skb(skb, __builtin_return_address(0)); 1222 else 1223 trace_kfree_skb(skb, __builtin_return_address(0), reason, sk); 1224 return true; 1225 } 1226 1227 /** 1228 * sk_skb_reason_drop - free an sk_buff with special reason 1229 * @sk: the socket to receive @skb, or NULL if not applicable 1230 * @skb: buffer to free 1231 * @reason: reason why this skb is dropped 1232 * 1233 * Drop a reference to the buffer and free it if the usage count has hit 1234 * zero. Meanwhile, pass the receiving socket and drop reason to 1235 * 'kfree_skb' tracepoint. 1236 */ 1237 void __fix_address 1238 sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, 1239 enum skb_drop_reason reason) 1240 { 1241 if (__sk_skb_reason_drop(sk, skb, reason)) 1242 __kfree_skb(skb); 1243 } 1244 EXPORT_SYMBOL(sk_skb_reason_drop); 1245 1246 #define KFREE_SKB_BULK_SIZE 16 1247 1248 struct skb_free_array { 1249 unsigned int skb_count; 1250 void *skb_array[KFREE_SKB_BULK_SIZE]; 1251 }; 1252 1253 static void kfree_skb_add_bulk(struct sk_buff *skb, 1254 struct skb_free_array *sa, 1255 enum skb_drop_reason reason) 1256 { 1257 /* if SKB is a clone, don't handle this case */ 1258 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) { 1259 __kfree_skb(skb); 1260 return; 1261 } 1262 1263 skb_release_all(skb, reason); 1264 sa->skb_array[sa->skb_count++] = skb; 1265 1266 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { 1267 kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE, 1268 sa->skb_array); 1269 sa->skb_count = 0; 1270 } 1271 } 1272 1273 void __fix_address 1274 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 1275 { 1276 struct skb_free_array sa; 1277 1278 sa.skb_count = 0; 1279 1280 while (segs) { 1281 struct sk_buff *next = segs->next; 1282 1283 if (__sk_skb_reason_drop(NULL, segs, reason)) { 1284 skb_poison_list(segs); 1285 kfree_skb_add_bulk(segs, &sa, reason); 1286 } 1287 1288 segs = next; 1289 } 1290 1291 if (sa.skb_count) 1292 kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array); 1293 } 1294 EXPORT_SYMBOL(kfree_skb_list_reason); 1295 1296 /* Dump skb information and contents. 1297 * 1298 * Must only be called from net_ratelimit()-ed paths. 1299 * 1300 * Dumps whole packets if full_pkt, only headers otherwise. 1301 */ 1302 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 1303 { 1304 struct skb_shared_info *sh = skb_shinfo(skb); 1305 struct net_device *dev = skb->dev; 1306 struct sock *sk = skb->sk; 1307 struct sk_buff *list_skb; 1308 bool has_mac, has_trans; 1309 int headroom, tailroom; 1310 int i, len, seg_len; 1311 1312 if (full_pkt) 1313 len = skb->len; 1314 else 1315 len = min_t(int, skb->len, MAX_HEADER + 128); 1316 1317 headroom = skb_headroom(skb); 1318 tailroom = skb_tailroom(skb); 1319 1320 has_mac = skb_mac_header_was_set(skb); 1321 has_trans = skb_transport_header_was_set(skb); 1322 1323 printk("%sskb len=%u data_len=%u headroom=%u headlen=%u tailroom=%u\n" 1324 "end-tail=%u mac=(%d,%d) mac_len=%u net=(%d,%d) trans=%d\n" 1325 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1326 "csum(0x%x start=%u offset=%u ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1327 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n" 1328 "priority=0x%x mark=0x%x alloc_cpu=%u vlan_all=0x%x\n" 1329 "encapsulation=%d inner(proto=0x%04x, mac=%u, net=%u, trans=%u)\n", 1330 level, skb->len, skb->data_len, headroom, skb_headlen(skb), 1331 tailroom, skb->end - skb->tail, 1332 has_mac ? skb->mac_header : -1, 1333 has_mac ? skb_mac_header_len(skb) : -1, 1334 skb->mac_len, 1335 skb->network_header, 1336 has_trans ? skb_network_header_len(skb) : -1, 1337 has_trans ? skb->transport_header : -1, 1338 sh->tx_flags, sh->nr_frags, 1339 sh->gso_size, sh->gso_type, sh->gso_segs, 1340 skb->csum, skb->csum_start, skb->csum_offset, skb->ip_summed, 1341 skb->csum_complete_sw, skb->csum_valid, skb->csum_level, 1342 skb->hash, skb->sw_hash, skb->l4_hash, 1343 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif, 1344 skb->priority, skb->mark, skb->alloc_cpu, skb->vlan_all, 1345 skb->encapsulation, skb->inner_protocol, skb->inner_mac_header, 1346 skb->inner_network_header, skb->inner_transport_header); 1347 1348 if (dev) 1349 printk("%sdev name=%s feat=%pNF\n", 1350 level, dev->name, &dev->features); 1351 if (sk) 1352 printk("%ssk family=%hu type=%u proto=%u\n", 1353 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1354 1355 if (full_pkt && headroom) 1356 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1357 16, 1, skb->head, headroom, false); 1358 1359 seg_len = min_t(int, skb_headlen(skb), len); 1360 if (seg_len) 1361 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1362 16, 1, skb->data, seg_len, false); 1363 len -= seg_len; 1364 1365 if (full_pkt && tailroom) 1366 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1367 16, 1, skb_tail_pointer(skb), tailroom, false); 1368 1369 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1370 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1371 u32 p_off, p_len, copied; 1372 struct page *p; 1373 u8 *vaddr; 1374 1375 if (skb_frag_is_net_iov(frag)) { 1376 printk("%sskb frag %d: not readable\n", level, i); 1377 len -= skb_frag_size(frag); 1378 if (!len) 1379 break; 1380 continue; 1381 } 1382 1383 skb_frag_foreach_page(frag, skb_frag_off(frag), 1384 skb_frag_size(frag), p, p_off, p_len, 1385 copied) { 1386 seg_len = min_t(int, p_len, len); 1387 vaddr = kmap_atomic(p); 1388 print_hex_dump(level, "skb frag: ", 1389 DUMP_PREFIX_OFFSET, 1390 16, 1, vaddr + p_off, seg_len, false); 1391 kunmap_atomic(vaddr); 1392 len -= seg_len; 1393 if (!len) 1394 break; 1395 } 1396 } 1397 1398 if (full_pkt && skb_has_frag_list(skb)) { 1399 printk("skb fraglist:\n"); 1400 skb_walk_frags(skb, list_skb) 1401 skb_dump(level, list_skb, true); 1402 } 1403 } 1404 EXPORT_SYMBOL(skb_dump); 1405 1406 /** 1407 * skb_tx_error - report an sk_buff xmit error 1408 * @skb: buffer that triggered an error 1409 * 1410 * Report xmit error if a device callback is tracking this skb. 1411 * skb must be freed afterwards. 1412 * 1413 * Does nothing for a cloned skb: the zerocopy state lives in 1414 * skb_shinfo(), which the clones share. 1415 */ 1416 void skb_tx_error(struct sk_buff *skb) 1417 { 1418 if (skb && !skb_cloned(skb)) { 1419 skb_zcopy_downgrade_managed(skb); 1420 skb_zcopy_clear(skb, true); 1421 } 1422 } 1423 EXPORT_SYMBOL(skb_tx_error); 1424 1425 #ifdef CONFIG_TRACEPOINTS 1426 /** 1427 * consume_skb - free an skbuff 1428 * @skb: buffer to free 1429 * 1430 * Drop a ref to the buffer and free it if the usage count has hit zero 1431 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1432 * is being dropped after a failure and notes that 1433 */ 1434 void consume_skb(struct sk_buff *skb) 1435 { 1436 if (!skb_unref(skb)) 1437 return; 1438 1439 trace_consume_skb(skb, __builtin_return_address(0)); 1440 __kfree_skb(skb); 1441 } 1442 EXPORT_SYMBOL(consume_skb); 1443 #endif 1444 1445 /** 1446 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1447 * @skb: buffer to free 1448 * 1449 * Alike consume_skb(), but this variant assumes that this is the last 1450 * skb reference and all the head states have been already dropped 1451 */ 1452 void __consume_stateless_skb(struct sk_buff *skb) 1453 { 1454 trace_consume_skb(skb, __builtin_return_address(0)); 1455 skb_release_data(skb, SKB_CONSUMED); 1456 kfree_skbmem(skb); 1457 } 1458 1459 static void napi_skb_cache_put(struct sk_buff *skb) 1460 { 1461 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1462 1463 if (!kasan_mempool_poison_object(skb)) 1464 return; 1465 1466 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 1467 nc->skb_cache[nc->skb_count++] = skb; 1468 1469 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1470 u32 i, remaining = NAPI_SKB_CACHE_SIZE - NAPI_SKB_CACHE_FREE; 1471 1472 for (i = remaining; i < NAPI_SKB_CACHE_SIZE; i++) 1473 kasan_mempool_unpoison_object(nc->skb_cache[i], 1474 skbuff_cache_size); 1475 1476 kmem_cache_free_bulk(net_hotdata.skbuff_cache, 1477 NAPI_SKB_CACHE_FREE, 1478 nc->skb_cache + remaining); 1479 nc->skb_count = remaining; 1480 } 1481 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 1482 } 1483 1484 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason) 1485 { 1486 skb_release_all(skb, reason); 1487 napi_skb_cache_put(skb); 1488 } 1489 1490 void napi_skb_free_stolen_head(struct sk_buff *skb) 1491 { 1492 if (unlikely(skb->slow_gro)) { 1493 nf_reset_ct(skb); 1494 skb_dst_drop(skb); 1495 skb_ext_put(skb); 1496 skb_orphan(skb); 1497 skb->slow_gro = 0; 1498 } 1499 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1500 kfree_skbmem(skb); 1501 return; 1502 } 1503 napi_skb_cache_put(skb); 1504 } 1505 1506 /** 1507 * napi_consume_skb() - consume skb in NAPI context, try to feed skb cache 1508 * @skb: buffer to free 1509 * @budget: NAPI budget 1510 * 1511 * Non-zero @budget must come from the @budget argument passed by the core 1512 * to a NAPI poll function. Note that core may pass budget of 0 to NAPI poll 1513 * for example when polling for netpoll / netconsole. 1514 * 1515 * Passing @budget of 0 is safe from any context, it turns this function 1516 * into dev_consume_skb_any(). 1517 */ 1518 void napi_consume_skb(struct sk_buff *skb, int budget) 1519 { 1520 if (unlikely(!budget || !skb)) { 1521 dev_consume_skb_any(skb); 1522 return; 1523 } 1524 1525 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1526 1527 if (!static_branch_unlikely(&skb_defer_disable_key) && 1528 skb->alloc_cpu != smp_processor_id() && !skb_shared(skb)) { 1529 skb_release_head_state(skb); 1530 return skb_attempt_defer_free(skb); 1531 } 1532 1533 if (!skb_unref(skb)) 1534 return; 1535 1536 /* if reaching here SKB is ready to free */ 1537 trace_consume_skb(skb, __builtin_return_address(0)); 1538 1539 /* if SKB is a clone, don't handle this case */ 1540 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1541 __kfree_skb(skb); 1542 return; 1543 } 1544 1545 skb_release_all(skb, SKB_CONSUMED); 1546 napi_skb_cache_put(skb); 1547 } 1548 EXPORT_SYMBOL(napi_consume_skb); 1549 1550 /* Make sure a field is contained by headers group */ 1551 #define CHECK_SKB_FIELD(field) \ 1552 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1553 offsetof(struct sk_buff, headers.field)); \ 1554 1555 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1556 { 1557 new->tstamp = old->tstamp; 1558 /* We do not copy old->sk */ 1559 new->dev = old->dev; 1560 memcpy(new->cb, old->cb, sizeof(old->cb)); 1561 skb_dst_copy(new, old); 1562 __skb_ext_copy(new, old); 1563 __nf_copy(new, old, false); 1564 1565 /* Note : this field could be in the headers group. 1566 * It is not yet because we do not want to have a 16 bit hole 1567 */ 1568 new->queue_mapping = old->queue_mapping; 1569 1570 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1571 CHECK_SKB_FIELD(protocol); 1572 CHECK_SKB_FIELD(csum); 1573 CHECK_SKB_FIELD(hash); 1574 CHECK_SKB_FIELD(priority); 1575 CHECK_SKB_FIELD(skb_iif); 1576 CHECK_SKB_FIELD(vlan_proto); 1577 CHECK_SKB_FIELD(vlan_tci); 1578 CHECK_SKB_FIELD(transport_header); 1579 CHECK_SKB_FIELD(network_header); 1580 CHECK_SKB_FIELD(mac_header); 1581 CHECK_SKB_FIELD(inner_protocol); 1582 CHECK_SKB_FIELD(inner_transport_header); 1583 CHECK_SKB_FIELD(inner_network_header); 1584 CHECK_SKB_FIELD(inner_mac_header); 1585 CHECK_SKB_FIELD(mark); 1586 #ifdef CONFIG_NETWORK_SECMARK 1587 CHECK_SKB_FIELD(secmark); 1588 #endif 1589 #ifdef CONFIG_NET_RX_BUSY_POLL 1590 CHECK_SKB_FIELD(napi_id); 1591 #endif 1592 #ifdef CONFIG_XPS 1593 CHECK_SKB_FIELD(sender_cpu); 1594 #endif 1595 #ifdef CONFIG_NET_SCHED 1596 CHECK_SKB_FIELD(tc_index); 1597 #endif 1598 1599 } 1600 1601 /* 1602 * You should not add any new code to this function. Add it to 1603 * __copy_skb_header above instead. 1604 */ 1605 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1606 { 1607 #define C(x) n->x = skb->x 1608 1609 n->next = n->prev = NULL; 1610 n->sk = NULL; 1611 __copy_skb_header(n, skb); 1612 1613 C(len); 1614 C(data_len); 1615 C(mac_len); 1616 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1617 n->cloned = 1; 1618 n->nohdr = 0; 1619 n->peeked = 0; 1620 C(pfmemalloc); 1621 C(pp_recycle); 1622 n->destructor = NULL; 1623 C(alloc_cpu); 1624 C(tail); 1625 C(end); 1626 C(head); 1627 C(head_frag); 1628 C(data); 1629 C(truesize); 1630 refcount_set(&n->users, 1); 1631 1632 atomic_inc(&(skb_shinfo(skb)->dataref)); 1633 skb->cloned = 1; 1634 1635 return n; 1636 #undef C 1637 } 1638 1639 /** 1640 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1641 * @first: first sk_buff of the msg 1642 */ 1643 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1644 { 1645 struct sk_buff *n; 1646 1647 n = alloc_skb(0, GFP_ATOMIC); 1648 if (!n) 1649 return NULL; 1650 1651 n->len = first->len; 1652 n->data_len = first->len; 1653 n->truesize = first->truesize; 1654 1655 skb_shinfo(n)->frag_list = first; 1656 1657 __copy_skb_header(n, first); 1658 n->destructor = NULL; 1659 1660 return n; 1661 } 1662 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1663 1664 /** 1665 * skb_morph - morph one skb into another 1666 * @dst: the skb to receive the contents 1667 * @src: the skb to supply the contents 1668 * 1669 * This is identical to skb_clone except that the target skb is 1670 * supplied by the user. 1671 * 1672 * The target skb is returned upon exit. 1673 */ 1674 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1675 { 1676 skb_release_all(dst, SKB_CONSUMED); 1677 return __skb_clone(dst, src); 1678 } 1679 EXPORT_SYMBOL_GPL(skb_morph); 1680 1681 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1682 { 1683 unsigned long max_pg, num_pg, new_pg, old_pg, rlim; 1684 struct user_struct *user; 1685 1686 if (capable(CAP_IPC_LOCK) || !size) 1687 return 0; 1688 1689 rlim = rlimit(RLIMIT_MEMLOCK); 1690 if (rlim == RLIM_INFINITY) 1691 return 0; 1692 1693 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1694 max_pg = rlim >> PAGE_SHIFT; 1695 user = mmp->user ? : current_user(); 1696 1697 old_pg = atomic_long_read(&user->locked_vm); 1698 do { 1699 new_pg = old_pg + num_pg; 1700 if (new_pg > max_pg) 1701 return -ENOBUFS; 1702 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1703 1704 if (!mmp->user) { 1705 mmp->user = get_uid(user); 1706 mmp->num_pg = num_pg; 1707 } else { 1708 mmp->num_pg += num_pg; 1709 } 1710 1711 return 0; 1712 } 1713 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1714 1715 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1716 { 1717 if (mmp->user) { 1718 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1719 free_uid(mmp->user); 1720 } 1721 } 1722 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1723 1724 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size, 1725 bool devmem) 1726 { 1727 struct ubuf_info_msgzc *uarg; 1728 struct sk_buff *skb; 1729 1730 WARN_ON_ONCE(!in_task()); 1731 1732 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1733 if (!skb) 1734 return NULL; 1735 1736 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1737 uarg = (void *)skb->cb; 1738 uarg->mmp.user = NULL; 1739 1740 if (likely(!devmem) && mm_account_pinned_pages(&uarg->mmp, size)) { 1741 kfree_skb(skb); 1742 return NULL; 1743 } 1744 1745 uarg->ubuf.ops = &msg_zerocopy_ubuf_ops; 1746 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1747 uarg->len = 1; 1748 uarg->bytelen = size; 1749 uarg->zerocopy = 1; 1750 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1751 refcount_set(&uarg->ubuf.refcnt, 1); 1752 sock_hold(sk); 1753 1754 return &uarg->ubuf; 1755 } 1756 1757 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1758 { 1759 return container_of((void *)uarg, struct sk_buff, cb); 1760 } 1761 1762 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1763 struct ubuf_info *uarg, bool devmem) 1764 { 1765 if (uarg) { 1766 struct ubuf_info_msgzc *uarg_zc; 1767 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1768 u32 bytelen, next; 1769 1770 /* there might be non MSG_ZEROCOPY users */ 1771 if (uarg->ops != &msg_zerocopy_ubuf_ops) 1772 return NULL; 1773 1774 /* realloc only when socket is locked (TCP, UDP cork), 1775 * so uarg->len and sk_zckey access is serialized 1776 */ 1777 if (!sock_owned_by_user(sk)) { 1778 WARN_ON_ONCE(1); 1779 return NULL; 1780 } 1781 1782 uarg_zc = uarg_to_msgzc(uarg); 1783 bytelen = uarg_zc->bytelen + size; 1784 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1785 /* TCP can create new skb to attach new uarg */ 1786 if (sk->sk_type == SOCK_STREAM) 1787 goto new_alloc; 1788 return NULL; 1789 } 1790 1791 next = (u32)atomic_read(&sk->sk_zckey); 1792 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1793 if (likely(!devmem) && 1794 mm_account_pinned_pages(&uarg_zc->mmp, size)) 1795 return NULL; 1796 uarg_zc->len++; 1797 uarg_zc->bytelen = bytelen; 1798 atomic_set(&sk->sk_zckey, ++next); 1799 1800 /* no extra ref when appending to datagram (MSG_MORE) */ 1801 if (sk->sk_type == SOCK_STREAM) 1802 net_zcopy_get(uarg); 1803 1804 return uarg; 1805 } 1806 } 1807 1808 new_alloc: 1809 return msg_zerocopy_alloc(sk, size, devmem); 1810 } 1811 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1812 1813 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1814 { 1815 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1816 u32 old_lo, old_hi; 1817 u64 sum_len; 1818 1819 old_lo = serr->ee.ee_info; 1820 old_hi = serr->ee.ee_data; 1821 sum_len = old_hi - old_lo + 1ULL + len; 1822 1823 if (sum_len >= (1ULL << 32)) 1824 return false; 1825 1826 if (lo != old_hi + 1) 1827 return false; 1828 1829 serr->ee.ee_data += len; 1830 return true; 1831 } 1832 1833 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1834 { 1835 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1836 struct sock_exterr_skb *serr; 1837 struct sock *sk = skb->sk; 1838 struct sk_buff_head *q; 1839 unsigned long flags; 1840 bool is_zerocopy; 1841 u32 lo, hi; 1842 u16 len; 1843 1844 mm_unaccount_pinned_pages(&uarg->mmp); 1845 1846 /* if !len, there was only 1 call, and it was aborted 1847 * so do not queue a completion notification 1848 */ 1849 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1850 goto release; 1851 1852 len = uarg->len; 1853 lo = uarg->id; 1854 hi = uarg->id + len - 1; 1855 is_zerocopy = uarg->zerocopy; 1856 1857 serr = SKB_EXT_ERR(skb); 1858 memset(serr, 0, sizeof(*serr)); 1859 serr->ee.ee_errno = 0; 1860 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1861 serr->ee.ee_data = hi; 1862 serr->ee.ee_info = lo; 1863 if (!is_zerocopy) 1864 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1865 1866 q = &sk->sk_error_queue; 1867 spin_lock_irqsave(&q->lock, flags); 1868 tail = skb_peek_tail(q); 1869 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1870 !skb_zerocopy_notify_extend(tail, lo, len)) { 1871 __skb_queue_tail(q, skb); 1872 skb = NULL; 1873 } 1874 spin_unlock_irqrestore(&q->lock, flags); 1875 1876 sk_error_report(sk); 1877 1878 release: 1879 consume_skb(skb); 1880 sock_put(sk); 1881 } 1882 1883 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg, 1884 bool success) 1885 { 1886 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1887 1888 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1889 1890 if (refcount_dec_and_test(&uarg->refcnt)) 1891 __msg_zerocopy_callback(uarg_zc); 1892 } 1893 1894 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1895 { 1896 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1897 1898 atomic_dec(&sk->sk_zckey); 1899 uarg_to_msgzc(uarg)->len--; 1900 1901 if (have_uref) 1902 msg_zerocopy_complete(NULL, uarg, true); 1903 } 1904 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1905 1906 const struct ubuf_info_ops msg_zerocopy_ubuf_ops = { 1907 .complete = msg_zerocopy_complete, 1908 }; 1909 EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops); 1910 1911 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1912 struct msghdr *msg, int len, 1913 struct ubuf_info *uarg, 1914 struct net_devmem_dmabuf_binding *binding) 1915 { 1916 unsigned int orig_truesize = skb->truesize; 1917 int err, orig_len = skb->len; 1918 1919 if (uarg->ops->link_skb) { 1920 err = uarg->ops->link_skb(skb, uarg); 1921 if (err) 1922 return err; 1923 } else { 1924 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1925 1926 /* An skb can only point to one uarg. This edge case happens 1927 * when TCP appends to an skb, but zerocopy_realloc triggered 1928 * a new alloc. 1929 */ 1930 if (orig_uarg && uarg != orig_uarg) 1931 return -EEXIST; 1932 } 1933 1934 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len, 1935 binding); 1936 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1937 u32 delta_truesize = skb->truesize - orig_truesize; 1938 struct sock *save_sk = skb->sk; 1939 1940 /* Streams do not free skb on error. Reset to prev state. */ 1941 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1942 skb->sk = sk; 1943 if (!___pskb_trim(skb, orig_len)) { 1944 skb->truesize -= delta_truesize; 1945 sk_wmem_queued_add(sk, -delta_truesize); 1946 if (!skb_zcopy_pure(skb)) 1947 sk_mem_uncharge(sk, delta_truesize); 1948 } 1949 skb->sk = save_sk; 1950 return err; 1951 } 1952 1953 skb_zcopy_set(skb, uarg, NULL); 1954 return skb->len - orig_len; 1955 } 1956 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1957 1958 void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1959 { 1960 int i; 1961 1962 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1963 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1964 skb_frag_ref(skb, i); 1965 } 1966 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1967 1968 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1969 gfp_t gfp_mask) 1970 { 1971 if (skb_zcopy(orig)) { 1972 if (skb_zcopy(nskb)) { 1973 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1974 if (!gfp_mask) { 1975 WARN_ON_ONCE(1); 1976 return -ENOMEM; 1977 } 1978 if (skb_uarg(nskb) == skb_uarg(orig)) 1979 return 0; 1980 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1981 return -EIO; 1982 } 1983 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1984 } 1985 return 0; 1986 } 1987 1988 /** 1989 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1990 * @skb: the skb to modify 1991 * @gfp_mask: allocation priority 1992 * 1993 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1994 * It will copy all frags into kernel and drop the reference 1995 * to userspace pages. 1996 * 1997 * If this function is called from an interrupt gfp_mask() must be 1998 * %GFP_ATOMIC. 1999 * 2000 * Returns 0 on success or a negative error code on failure 2001 * to allocate kernel memory to copy to. 2002 */ 2003 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 2004 { 2005 int num_frags = skb_shinfo(skb)->nr_frags; 2006 struct page *page, *head = NULL; 2007 int i, order, psize, new_frags; 2008 u32 d_off; 2009 2010 if (!skb_frags_readable(skb)) 2011 return -EFAULT; 2012 2013 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 2014 return -EINVAL; 2015 2016 if (!num_frags) 2017 goto release; 2018 2019 /* We might have to allocate high order pages, so compute what minimum 2020 * page order is needed. 2021 */ 2022 order = 0; 2023 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb)) 2024 order++; 2025 psize = (PAGE_SIZE << order); 2026 2027 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order); 2028 for (i = 0; i < new_frags; i++) { 2029 page = alloc_pages(gfp_mask | __GFP_COMP, order); 2030 if (!page) { 2031 while (head) { 2032 struct page *next = (struct page *)page_private(head); 2033 put_page(head); 2034 head = next; 2035 } 2036 return -ENOMEM; 2037 } 2038 set_page_private(page, (unsigned long)head); 2039 head = page; 2040 } 2041 2042 page = head; 2043 d_off = 0; 2044 for (i = 0; i < num_frags; i++) { 2045 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2046 u32 p_off, p_len, copied; 2047 struct page *p; 2048 u8 *vaddr; 2049 2050 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 2051 p, p_off, p_len, copied) { 2052 u32 copy, done = 0; 2053 vaddr = kmap_atomic(p); 2054 2055 while (done < p_len) { 2056 if (d_off == psize) { 2057 d_off = 0; 2058 page = (struct page *)page_private(page); 2059 } 2060 copy = min_t(u32, psize - d_off, p_len - done); 2061 memcpy(page_address(page) + d_off, 2062 vaddr + p_off + done, copy); 2063 done += copy; 2064 d_off += copy; 2065 } 2066 kunmap_atomic(vaddr); 2067 } 2068 } 2069 2070 /* skb frags release userspace buffers */ 2071 for (i = 0; i < num_frags; i++) 2072 skb_frag_unref(skb, i); 2073 2074 /* skb frags point to kernel buffers */ 2075 for (i = 0; i < new_frags - 1; i++) { 2076 __skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize); 2077 head = (struct page *)page_private(head); 2078 } 2079 __skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0, 2080 d_off); 2081 skb_shinfo(skb)->nr_frags = new_frags; 2082 2083 release: 2084 skb_zcopy_clear(skb, false); 2085 return 0; 2086 } 2087 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 2088 2089 /** 2090 * skb_clone - duplicate an sk_buff 2091 * @skb: buffer to clone 2092 * @gfp_mask: allocation priority 2093 * 2094 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 2095 * copies share the same packet data but not structure. The new 2096 * buffer has a reference count of 1. If the allocation fails the 2097 * function returns %NULL otherwise the new buffer is returned. 2098 * 2099 * If this function is called from an interrupt gfp_mask() must be 2100 * %GFP_ATOMIC. 2101 */ 2102 2103 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 2104 { 2105 struct sk_buff_fclones *fclones = container_of(skb, 2106 struct sk_buff_fclones, 2107 skb1); 2108 struct sk_buff *n; 2109 2110 if (skb_orphan_frags(skb, gfp_mask)) 2111 return NULL; 2112 2113 if (skb->fclone == SKB_FCLONE_ORIG && 2114 refcount_read(&fclones->fclone_ref) == 1) { 2115 n = &fclones->skb2; 2116 refcount_set(&fclones->fclone_ref, 2); 2117 n->fclone = SKB_FCLONE_CLONE; 2118 } else { 2119 if (skb_pfmemalloc(skb)) 2120 gfp_mask |= __GFP_MEMALLOC; 2121 2122 n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask); 2123 if (!n) 2124 return NULL; 2125 2126 n->fclone = SKB_FCLONE_UNAVAILABLE; 2127 } 2128 2129 return __skb_clone(n, skb); 2130 } 2131 EXPORT_SYMBOL(skb_clone); 2132 2133 void skb_headers_offset_update(struct sk_buff *skb, int off) 2134 { 2135 /* Only adjust this if it actually is csum_start rather than csum */ 2136 if (skb->ip_summed == CHECKSUM_PARTIAL) 2137 skb->csum_start += off; 2138 /* {transport,network,mac}_header and tail are relative to skb->head */ 2139 skb->transport_header += off; 2140 skb->network_header += off; 2141 if (skb_mac_header_was_set(skb)) 2142 skb->mac_header += off; 2143 skb->inner_transport_header += off; 2144 skb->inner_network_header += off; 2145 skb->inner_mac_header += off; 2146 } 2147 EXPORT_SYMBOL(skb_headers_offset_update); 2148 2149 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 2150 { 2151 __copy_skb_header(new, old); 2152 2153 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 2154 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 2155 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 2156 } 2157 EXPORT_SYMBOL(skb_copy_header); 2158 2159 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 2160 { 2161 if (skb_pfmemalloc(skb)) 2162 return SKB_ALLOC_RX; 2163 return 0; 2164 } 2165 2166 /** 2167 * skb_copy - create private copy of an sk_buff 2168 * @skb: buffer to copy 2169 * @gfp_mask: allocation priority 2170 * 2171 * Make a copy of both an &sk_buff and its data. This is used when the 2172 * caller wishes to modify the data and needs a private copy of the 2173 * data to alter. Returns %NULL on failure or the pointer to the buffer 2174 * on success. The returned buffer has a reference count of 1. 2175 * 2176 * As by-product this function converts non-linear &sk_buff to linear 2177 * one, so that &sk_buff becomes completely private and caller is allowed 2178 * to modify all the data of returned buffer. This means that this 2179 * function is not recommended for use in circumstances when only 2180 * header is going to be modified. Use pskb_copy() instead. 2181 */ 2182 2183 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 2184 { 2185 struct sk_buff *n; 2186 unsigned int size; 2187 int headerlen; 2188 2189 if (!skb_frags_readable(skb)) 2190 return NULL; 2191 2192 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)) 2193 return NULL; 2194 2195 headerlen = skb_headroom(skb); 2196 size = skb_end_offset(skb) + skb->data_len; 2197 n = __alloc_skb(size, gfp_mask, 2198 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 2199 if (!n) 2200 return NULL; 2201 2202 /* Set the data pointer */ 2203 skb_reserve(n, headerlen); 2204 /* Set the tail pointer and length */ 2205 skb_put(n, skb->len); 2206 2207 if (unlikely(skb_copy_bits(skb, -headerlen, n->head, 2208 headerlen + skb->len))) { 2209 DEBUG_NET_WARN_ON_ONCE(1); 2210 kfree_skb(n); 2211 return NULL; 2212 } 2213 2214 skb_copy_header(n, skb); 2215 return n; 2216 } 2217 EXPORT_SYMBOL(skb_copy); 2218 2219 /** 2220 * __pskb_copy_fclone - create copy of an sk_buff with private head. 2221 * @skb: buffer to copy 2222 * @headroom: headroom of new skb 2223 * @gfp_mask: allocation priority 2224 * @fclone: if true allocate the copy of the skb from the fclone 2225 * cache instead of the head cache; it is recommended to set this 2226 * to true for the cases where the copy will likely be cloned 2227 * 2228 * Make a copy of both an &sk_buff and part of its data, located 2229 * in header. Fragmented data remain shared. This is used when 2230 * the caller wishes to modify only header of &sk_buff and needs 2231 * private copy of the header to alter. Returns %NULL on failure 2232 * or the pointer to the buffer on success. 2233 * The returned buffer has a reference count of 1. 2234 */ 2235 2236 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 2237 gfp_t gfp_mask, bool fclone) 2238 { 2239 unsigned int size = skb_headlen(skb) + headroom; 2240 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 2241 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 2242 2243 if (!n) 2244 goto out; 2245 2246 /* Set the data pointer */ 2247 skb_reserve(n, headroom); 2248 /* Set the tail pointer and length */ 2249 skb_put(n, skb_headlen(skb)); 2250 /* Copy the bytes */ 2251 skb_copy_from_linear_data(skb, n->data, n->len); 2252 2253 n->truesize += skb->data_len; 2254 n->data_len = skb->data_len; 2255 n->len = skb->len; 2256 2257 if (skb_shinfo(skb)->nr_frags) { 2258 int i; 2259 2260 if (skb_orphan_frags(skb, gfp_mask) || 2261 skb_zerocopy_clone(n, skb, gfp_mask)) { 2262 kfree_skb(n); 2263 n = NULL; 2264 goto out; 2265 } 2266 n->pp_recycle = skb->pp_recycle; 2267 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2268 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 2269 skb_frag_ref(n, i); 2270 } 2271 skb_shinfo(n)->nr_frags = i; 2272 skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 2273 } 2274 2275 if (skb_has_frag_list(skb)) { 2276 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 2277 skb_clone_fraglist(n); 2278 } 2279 2280 skb_copy_header(n, skb); 2281 out: 2282 return n; 2283 } 2284 EXPORT_SYMBOL(__pskb_copy_fclone); 2285 2286 /** 2287 * pskb_expand_head - reallocate header of &sk_buff 2288 * @skb: buffer to reallocate 2289 * @nhead: room to add at head 2290 * @ntail: room to add at tail 2291 * @gfp_mask: allocation priority 2292 * 2293 * Expands (or creates identical copy, if @nhead and @ntail are zero) 2294 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 2295 * reference count of 1. Returns zero in the case of success or error, 2296 * if expansion failed. In the last case, &sk_buff is not changed. 2297 * 2298 * All the pointers pointing into skb header may change and must be 2299 * reloaded after call to this function. 2300 * 2301 * Note: If you skb_push() the start of the buffer after reallocating the 2302 * header, call skb_postpush_data_move() first to move the metadata out of 2303 * the way before writing to &sk_buff->data. 2304 */ 2305 2306 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 2307 gfp_t gfp_mask) 2308 { 2309 unsigned int osize = skb_end_offset(skb); 2310 unsigned int size = osize + nhead + ntail; 2311 long off; 2312 u8 *data; 2313 int i; 2314 2315 if (unlikely(nhead < 0)) { 2316 DEBUG_NET_WARN_ON_ONCE(1); 2317 return -EINVAL; 2318 } 2319 2320 if (unlikely(skb_shared(skb))) { 2321 DEBUG_NET_WARN_ON_ONCE(1); 2322 return -EINVAL; 2323 } 2324 2325 skb_zcopy_downgrade_managed(skb); 2326 2327 if (skb_pfmemalloc(skb)) 2328 gfp_mask |= __GFP_MEMALLOC; 2329 2330 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 2331 if (!data) 2332 goto nodata; 2333 size = SKB_WITH_OVERHEAD(size); 2334 2335 /* Copy only real data... and, alas, header. This should be 2336 * optimized for the cases when header is void. 2337 */ 2338 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 2339 2340 memcpy((struct skb_shared_info *)(data + size), 2341 skb_shinfo(skb), 2342 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 2343 2344 /* 2345 * if shinfo is shared we must drop the old head gracefully, but if it 2346 * is not we can just drop the old head and let the existing refcount 2347 * be since all we did is relocate the values 2348 */ 2349 if (skb_cloned(skb)) { 2350 if (skb_orphan_frags(skb, gfp_mask)) 2351 goto nofrags; 2352 if (skb_zcopy(skb)) 2353 net_zcopy_get(skb_uarg(skb)); 2354 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2355 skb_frag_ref(skb, i); 2356 2357 if (skb_has_frag_list(skb)) 2358 skb_clone_fraglist(skb); 2359 2360 skb_release_data(skb, SKB_CONSUMED); 2361 } else { 2362 skb_free_head(skb); 2363 } 2364 off = (data + nhead) - skb->head; 2365 2366 skb->head = data; 2367 skb->head_frag = 0; 2368 skb->data += off; 2369 2370 skb_set_end_offset(skb, size); 2371 #ifdef NET_SKBUFF_DATA_USES_OFFSET 2372 off = nhead; 2373 #endif 2374 skb->tail += off; 2375 skb_headers_offset_update(skb, nhead); 2376 skb->cloned = 0; 2377 skb->hdr_len = 0; 2378 skb->nohdr = 0; 2379 atomic_set(&skb_shinfo(skb)->dataref, 1); 2380 2381 /* It is not generally safe to change skb->truesize. 2382 * For the moment, we really care of rx path, or 2383 * when skb is orphaned (not attached to a socket). 2384 */ 2385 if (!skb->sk || skb->destructor == sock_edemux) 2386 skb->truesize += size - osize; 2387 2388 return 0; 2389 2390 nofrags: 2391 skb_kfree_head(data); 2392 nodata: 2393 return -ENOMEM; 2394 } 2395 EXPORT_SYMBOL(pskb_expand_head); 2396 2397 /* Make private copy of skb with writable head and some headroom */ 2398 2399 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 2400 { 2401 struct sk_buff *skb2; 2402 int delta = headroom - skb_headroom(skb); 2403 2404 if (delta <= 0) 2405 skb2 = pskb_copy(skb, GFP_ATOMIC); 2406 else { 2407 skb2 = skb_clone(skb, GFP_ATOMIC); 2408 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 2409 GFP_ATOMIC)) { 2410 kfree_skb(skb2); 2411 skb2 = NULL; 2412 } 2413 } 2414 return skb2; 2415 } 2416 EXPORT_SYMBOL(skb_realloc_headroom); 2417 2418 /* Note: We plan to rework this in linux-6.4 */ 2419 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 2420 { 2421 unsigned int saved_end_offset, saved_truesize; 2422 struct skb_shared_info *shinfo; 2423 int res; 2424 2425 saved_end_offset = skb_end_offset(skb); 2426 saved_truesize = skb->truesize; 2427 2428 res = pskb_expand_head(skb, 0, 0, pri); 2429 if (res) 2430 return res; 2431 2432 skb->truesize = saved_truesize; 2433 2434 if (likely(skb_end_offset(skb) == saved_end_offset)) 2435 return 0; 2436 2437 shinfo = skb_shinfo(skb); 2438 2439 /* We are about to change back skb->end, 2440 * we need to move skb_shinfo() to its new location. 2441 */ 2442 memmove(skb->head + saved_end_offset, 2443 shinfo, 2444 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2445 2446 skb_set_end_offset(skb, saved_end_offset); 2447 2448 return 0; 2449 } 2450 2451 /** 2452 * skb_expand_head - reallocate header of &sk_buff 2453 * @skb: buffer to reallocate 2454 * @headroom: needed headroom 2455 * 2456 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2457 * if possible; copies skb->sk to new skb as needed 2458 * and frees original skb in case of failures. 2459 * 2460 * It expect increased headroom and generates warning otherwise. 2461 */ 2462 2463 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2464 { 2465 int delta = headroom - skb_headroom(skb); 2466 int osize = skb_end_offset(skb); 2467 struct sock *sk = skb->sk; 2468 2469 if (WARN_ONCE(delta <= 0, 2470 "%s is expecting an increase in the headroom", __func__)) 2471 return skb; 2472 2473 delta = SKB_DATA_ALIGN(delta); 2474 /* pskb_expand_head() might crash, if skb is shared. */ 2475 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2476 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2477 2478 if (unlikely(!nskb)) 2479 goto fail; 2480 2481 if (sk) 2482 skb_set_owner_w(nskb, sk); 2483 consume_skb(skb); 2484 skb = nskb; 2485 } 2486 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2487 goto fail; 2488 2489 if (sk && is_skb_wmem(skb)) { 2490 delta = skb_end_offset(skb) - osize; 2491 refcount_add(delta, &sk->sk_wmem_alloc); 2492 skb->truesize += delta; 2493 } 2494 return skb; 2495 2496 fail: 2497 kfree_skb(skb); 2498 return NULL; 2499 } 2500 EXPORT_SYMBOL(skb_expand_head); 2501 2502 /** 2503 * skb_copy_expand - copy and expand sk_buff 2504 * @skb: buffer to copy 2505 * @newheadroom: new free bytes at head 2506 * @newtailroom: new free bytes at tail 2507 * @gfp_mask: allocation priority 2508 * 2509 * Make a copy of both an &sk_buff and its data and while doing so 2510 * allocate additional space. 2511 * 2512 * This is used when the caller wishes to modify the data and needs a 2513 * private copy of the data to alter as well as more space for new fields. 2514 * Returns %NULL on failure or the pointer to the buffer 2515 * on success. The returned buffer has a reference count of 1. 2516 * 2517 * You must pass %GFP_ATOMIC as the allocation priority if this function 2518 * is called from an interrupt. 2519 */ 2520 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2521 int newheadroom, int newtailroom, 2522 gfp_t gfp_mask) 2523 { 2524 /* 2525 * Allocate the copy buffer 2526 */ 2527 int head_copy_len, head_copy_off; 2528 struct sk_buff *n; 2529 int oldheadroom; 2530 2531 if (!skb_frags_readable(skb)) 2532 return NULL; 2533 2534 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)) 2535 return NULL; 2536 2537 oldheadroom = skb_headroom(skb); 2538 n = __alloc_skb(newheadroom + skb->len + newtailroom, 2539 gfp_mask, skb_alloc_rx_flag(skb), 2540 NUMA_NO_NODE); 2541 if (!n) 2542 return NULL; 2543 2544 skb_reserve(n, newheadroom); 2545 2546 /* Set the tail pointer and length */ 2547 skb_put(n, skb->len); 2548 2549 head_copy_len = oldheadroom; 2550 head_copy_off = 0; 2551 if (newheadroom <= head_copy_len) 2552 head_copy_len = newheadroom; 2553 else 2554 head_copy_off = newheadroom - head_copy_len; 2555 2556 /* Copy the linear header and data. */ 2557 if (unlikely(skb_copy_bits(skb, -head_copy_len, 2558 n->head + head_copy_off, 2559 skb->len + head_copy_len))) { 2560 DEBUG_NET_WARN_ON_ONCE(1); 2561 kfree_skb(n); 2562 return NULL; 2563 } 2564 2565 skb_copy_header(n, skb); 2566 2567 skb_headers_offset_update(n, newheadroom - oldheadroom); 2568 2569 return n; 2570 } 2571 EXPORT_SYMBOL(skb_copy_expand); 2572 2573 /** 2574 * __skb_pad - zero pad the tail of an skb 2575 * @skb: buffer to pad 2576 * @pad: space to pad 2577 * @free_on_error: free buffer on error 2578 * 2579 * Ensure that a buffer is followed by a padding area that is zero 2580 * filled. Used by network drivers which may DMA or transfer data 2581 * beyond the buffer end onto the wire. 2582 * 2583 * May return error in out of memory cases. The skb is freed on error 2584 * if @free_on_error is true. 2585 */ 2586 2587 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2588 { 2589 int err; 2590 int ntail; 2591 2592 /* If the skbuff is non linear tailroom is always zero.. */ 2593 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2594 memset(skb->data+skb->len, 0, pad); 2595 return 0; 2596 } 2597 2598 ntail = skb->data_len + pad - (skb->end - skb->tail); 2599 if (likely(skb_cloned(skb) || ntail > 0)) { 2600 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2601 if (unlikely(err)) 2602 goto free_skb; 2603 } 2604 2605 /* FIXME: The use of this function with non-linear skb's really needs 2606 * to be audited. 2607 */ 2608 err = skb_linearize(skb); 2609 if (unlikely(err)) 2610 goto free_skb; 2611 2612 memset(skb->data + skb->len, 0, pad); 2613 return 0; 2614 2615 free_skb: 2616 if (free_on_error) 2617 kfree_skb(skb); 2618 return err; 2619 } 2620 EXPORT_SYMBOL(__skb_pad); 2621 2622 /** 2623 * pskb_put - add data to the tail of a potentially fragmented buffer 2624 * @skb: start of the buffer to use 2625 * @tail: tail fragment of the buffer to use 2626 * @len: amount of data to add 2627 * 2628 * This function extends the used data area of the potentially 2629 * fragmented buffer. @tail must be the last fragment of @skb -- or 2630 * @skb itself. If this would exceed the total buffer size the kernel 2631 * will panic. A pointer to the first byte of the extra data is 2632 * returned. 2633 */ 2634 2635 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2636 { 2637 if (tail != skb) { 2638 skb->data_len += len; 2639 skb->len += len; 2640 } 2641 return skb_put(tail, len); 2642 } 2643 EXPORT_SYMBOL_GPL(pskb_put); 2644 2645 /** 2646 * skb_put - add data to a buffer 2647 * @skb: buffer to use 2648 * @len: amount of data to add 2649 * 2650 * This function extends the used data area of the buffer. If this would 2651 * exceed the total buffer size the kernel will panic. A pointer to the 2652 * first byte of the extra data is returned. 2653 */ 2654 void *skb_put(struct sk_buff *skb, unsigned int len) 2655 { 2656 void *tmp = skb_tail_pointer(skb); 2657 SKB_LINEAR_ASSERT(skb); 2658 skb->tail += len; 2659 skb->len += len; 2660 if (unlikely(skb->tail > skb->end)) 2661 skb_over_panic(skb, len, __builtin_return_address(0)); 2662 return tmp; 2663 } 2664 EXPORT_SYMBOL(skb_put); 2665 2666 /** 2667 * skb_push - add data to the start of a buffer 2668 * @skb: buffer to use 2669 * @len: amount of data to add 2670 * 2671 * This function extends the used data area of the buffer at the buffer 2672 * start. If this would exceed the total buffer headroom the kernel will 2673 * panic. A pointer to the first byte of the extra data is returned. 2674 */ 2675 void *skb_push(struct sk_buff *skb, unsigned int len) 2676 { 2677 skb->data -= len; 2678 skb->len += len; 2679 if (unlikely(skb->data < skb->head)) 2680 skb_under_panic(skb, len, __builtin_return_address(0)); 2681 return skb->data; 2682 } 2683 EXPORT_SYMBOL(skb_push); 2684 2685 /** 2686 * skb_pull - remove data from the start of a buffer 2687 * @skb: buffer to use 2688 * @len: amount of data to remove 2689 * 2690 * This function removes data from the start of a buffer, returning 2691 * the memory to the headroom. A pointer to the next data in the buffer 2692 * is returned. Once the data has been pulled future pushes will overwrite 2693 * the old data. 2694 */ 2695 void *skb_pull(struct sk_buff *skb, unsigned int len) 2696 { 2697 return skb_pull_inline(skb, len); 2698 } 2699 EXPORT_SYMBOL(skb_pull); 2700 2701 /** 2702 * skb_pull_data - remove data from the start of a buffer returning its 2703 * original position. 2704 * @skb: buffer to use 2705 * @len: amount of data to remove 2706 * 2707 * This function removes data from the start of a buffer, returning 2708 * the memory to the headroom. A pointer to the original data in the buffer 2709 * is returned after checking if there is enough data to pull. Once the 2710 * data has been pulled future pushes will overwrite the old data. 2711 */ 2712 void *skb_pull_data(struct sk_buff *skb, size_t len) 2713 { 2714 void *data = skb->data; 2715 2716 if (skb->len < len) 2717 return NULL; 2718 2719 skb_pull(skb, len); 2720 2721 return data; 2722 } 2723 EXPORT_SYMBOL(skb_pull_data); 2724 2725 /** 2726 * skb_trim - remove end from a buffer 2727 * @skb: buffer to alter 2728 * @len: new length 2729 * 2730 * Cut the length of a buffer down by removing data from the tail. If 2731 * the buffer is already under the length specified it is not modified. 2732 * The skb must be linear. 2733 */ 2734 void skb_trim(struct sk_buff *skb, unsigned int len) 2735 { 2736 if (skb->len > len) 2737 __skb_trim(skb, len); 2738 } 2739 EXPORT_SYMBOL(skb_trim); 2740 2741 /* Trims skb to length len. It can change skb pointers. 2742 */ 2743 2744 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2745 { 2746 struct sk_buff **fragp; 2747 struct sk_buff *frag; 2748 int offset = skb_headlen(skb); 2749 int nfrags = skb_shinfo(skb)->nr_frags; 2750 int i; 2751 int err; 2752 2753 if (skb_cloned(skb) && 2754 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2755 return err; 2756 2757 i = 0; 2758 if (offset >= len) 2759 goto drop_pages; 2760 2761 for (; i < nfrags; i++) { 2762 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2763 2764 if (end < len) { 2765 offset = end; 2766 continue; 2767 } 2768 2769 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2770 2771 drop_pages: 2772 skb_shinfo(skb)->nr_frags = i; 2773 2774 for (; i < nfrags; i++) 2775 skb_frag_unref(skb, i); 2776 2777 if (skb_has_frag_list(skb)) 2778 skb_drop_fraglist(skb); 2779 goto done; 2780 } 2781 2782 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2783 fragp = &frag->next) { 2784 int end = offset + frag->len; 2785 2786 if (skb_shared(frag)) { 2787 struct sk_buff *nfrag; 2788 2789 nfrag = skb_clone(frag, GFP_ATOMIC); 2790 if (unlikely(!nfrag)) 2791 return -ENOMEM; 2792 2793 nfrag->next = frag->next; 2794 consume_skb(frag); 2795 frag = nfrag; 2796 *fragp = frag; 2797 } 2798 2799 if (end < len) { 2800 offset = end; 2801 continue; 2802 } 2803 2804 if (end > len && 2805 unlikely((err = pskb_trim(frag, len - offset)))) 2806 return err; 2807 2808 if (frag->next) 2809 skb_drop_list(&frag->next); 2810 break; 2811 } 2812 2813 done: 2814 if (len > skb_headlen(skb)) { 2815 skb->data_len -= skb->len - len; 2816 skb->len = len; 2817 } else { 2818 skb->len = len; 2819 skb->data_len = 0; 2820 skb_set_tail_pointer(skb, len); 2821 } 2822 if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb)) 2823 skb->unreadable = 0; 2824 2825 if (!skb->sk || skb->destructor == sock_edemux) 2826 skb_condense(skb); 2827 return 0; 2828 } 2829 EXPORT_SYMBOL(___pskb_trim); 2830 2831 static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len) 2832 { 2833 int delta = skb->len - len; 2834 2835 if (skb_frags_readable(skb)) { 2836 skb->csum = csum_block_sub(skb->csum, 2837 skb_checksum(skb, len, delta, 0), 2838 len); 2839 return 0; 2840 } 2841 2842 if (len > skb_headlen(skb)) 2843 return -EFAULT; 2844 2845 /* The trimmed bytes are unreadable, but the remaining packet can be 2846 * checksummed by software after trimming. 2847 */ 2848 skb->ip_summed = CHECKSUM_NONE; 2849 return 0; 2850 } 2851 2852 /* Note : use pskb_trim_rcsum() instead of calling this directly 2853 */ 2854 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2855 { 2856 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2857 int err; 2858 2859 err = pskb_trim_rcsum_complete(skb, len); 2860 if (err) 2861 return err; 2862 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2863 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2864 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2865 2866 if (offset + sizeof(__sum16) > hdlen) 2867 return -EINVAL; 2868 } 2869 return __pskb_trim(skb, len); 2870 } 2871 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2872 2873 /** 2874 * __pskb_pull_tail - advance tail of skb header 2875 * @skb: buffer to reallocate 2876 * @delta: number of bytes to advance tail 2877 * 2878 * The function makes a sense only on a fragmented &sk_buff, 2879 * it expands header moving its tail forward and copying necessary 2880 * data from fragmented part. 2881 * 2882 * &sk_buff MUST have reference count of 1. 2883 * 2884 * Returns %NULL (and &sk_buff does not change) if pull failed 2885 * or value of new tail of skb in the case of success. 2886 * 2887 * All the pointers pointing into skb header may change and must be 2888 * reloaded after call to this function. 2889 */ 2890 2891 /* Moves tail of skb head forward, copying data from fragmented part, 2892 * when it is necessary. 2893 * 1. It may fail due to malloc failure. 2894 * 2. It may change skb pointers. 2895 * 2896 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2897 */ 2898 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2899 { 2900 /* If skb has not enough free space at tail, get new one 2901 * plus 128 bytes for future expansions. If we have enough 2902 * room at tail, reallocate without expansion only if skb is cloned. 2903 */ 2904 int i, k, eat = (skb->tail + delta) - skb->end; 2905 2906 if (!skb_frags_readable(skb)) 2907 return NULL; 2908 2909 if (eat > 0 || skb_cloned(skb)) { 2910 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2911 GFP_ATOMIC)) 2912 return NULL; 2913 } 2914 2915 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2916 skb_tail_pointer(skb), delta)); 2917 2918 /* Optimization: no fragments, no reasons to preestimate 2919 * size of pulled pages. Superb. 2920 */ 2921 if (!skb_has_frag_list(skb)) 2922 goto pull_pages; 2923 2924 /* Estimate size of pulled pages. */ 2925 eat = delta; 2926 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2927 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2928 2929 if (size >= eat) 2930 goto pull_pages; 2931 eat -= size; 2932 } 2933 2934 /* If we need update frag list, we are in troubles. 2935 * Certainly, it is possible to add an offset to skb data, 2936 * but taking into account that pulling is expected to 2937 * be very rare operation, it is worth to fight against 2938 * further bloating skb head and crucify ourselves here instead. 2939 * Pure masohism, indeed. 8)8) 2940 */ 2941 if (eat) { 2942 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2943 struct sk_buff *clone = NULL; 2944 struct sk_buff *insp = NULL; 2945 2946 do { 2947 if (list->len <= eat) { 2948 /* Eaten as whole. */ 2949 eat -= list->len; 2950 list = list->next; 2951 insp = list; 2952 } else { 2953 /* Eaten partially. */ 2954 if (skb_is_gso(skb) && !list->head_frag && 2955 skb_headlen(list)) 2956 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2957 2958 if (skb_shared(list)) { 2959 /* Sucks! We need to fork list. :-( */ 2960 clone = skb_clone(list, GFP_ATOMIC); 2961 if (!clone) 2962 return NULL; 2963 insp = list->next; 2964 list = clone; 2965 } else { 2966 /* This may be pulled without 2967 * problems. */ 2968 insp = list; 2969 } 2970 if (!pskb_pull(list, eat)) { 2971 kfree_skb(clone); 2972 return NULL; 2973 } 2974 break; 2975 } 2976 } while (eat); 2977 2978 /* Free pulled out fragments. */ 2979 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2980 skb_shinfo(skb)->frag_list = list->next; 2981 consume_skb(list); 2982 } 2983 /* And insert new clone at head. */ 2984 if (clone) { 2985 clone->next = list; 2986 skb_shinfo(skb)->frag_list = clone; 2987 } 2988 } 2989 /* Success! Now we may commit changes to skb data. */ 2990 2991 pull_pages: 2992 eat = delta; 2993 k = 0; 2994 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2995 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2996 2997 if (size <= eat) { 2998 skb_frag_unref(skb, i); 2999 eat -= size; 3000 } else { 3001 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 3002 3003 *frag = skb_shinfo(skb)->frags[i]; 3004 if (eat) { 3005 skb_frag_off_add(frag, eat); 3006 skb_frag_size_sub(frag, eat); 3007 if (!i) 3008 goto end; 3009 eat = 0; 3010 } 3011 k++; 3012 } 3013 } 3014 skb_shinfo(skb)->nr_frags = k; 3015 3016 end: 3017 skb->tail += delta; 3018 skb->data_len -= delta; 3019 3020 if (!skb->data_len) 3021 skb_zcopy_clear(skb, false); 3022 3023 return skb_tail_pointer(skb); 3024 } 3025 EXPORT_SYMBOL(__pskb_pull_tail); 3026 3027 /** 3028 * skb_copy_bits - copy bits from skb to kernel buffer 3029 * @skb: source skb 3030 * @offset: offset in source 3031 * @to: destination buffer 3032 * @len: number of bytes to copy 3033 * 3034 * Copy the specified number of bytes from the source skb to the 3035 * destination buffer. 3036 * 3037 * CAUTION ! : 3038 * If its prototype is ever changed, 3039 * check arch/{*}/net/{*}.S files, 3040 * since it is called from BPF assembly code. 3041 */ 3042 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 3043 { 3044 int start = skb_headlen(skb); 3045 struct sk_buff *frag_iter; 3046 int i, copy; 3047 3048 if (offset > (int)skb->len - len) 3049 goto fault; 3050 3051 /* Copy header. */ 3052 if ((copy = start - offset) > 0) { 3053 if (copy > len) 3054 copy = len; 3055 skb_copy_from_linear_data_offset(skb, offset, to, copy); 3056 if ((len -= copy) == 0) 3057 return 0; 3058 offset += copy; 3059 to += copy; 3060 } 3061 3062 if (!skb_frags_readable(skb)) 3063 goto fault; 3064 3065 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3066 int end; 3067 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 3068 3069 WARN_ON(start > offset + len); 3070 3071 end = start + skb_frag_size(f); 3072 if ((copy = end - offset) > 0) { 3073 u32 p_off, p_len, copied; 3074 struct page *p; 3075 u8 *vaddr; 3076 3077 if (copy > len) 3078 copy = len; 3079 3080 skb_frag_foreach_page(f, 3081 skb_frag_off(f) + offset - start, 3082 copy, p, p_off, p_len, copied) { 3083 vaddr = kmap_atomic(p); 3084 memcpy(to + copied, vaddr + p_off, p_len); 3085 kunmap_atomic(vaddr); 3086 } 3087 3088 if ((len -= copy) == 0) 3089 return 0; 3090 offset += copy; 3091 to += copy; 3092 } 3093 start = end; 3094 } 3095 3096 skb_walk_frags(skb, frag_iter) { 3097 int end; 3098 3099 WARN_ON(start > offset + len); 3100 3101 end = start + frag_iter->len; 3102 if ((copy = end - offset) > 0) { 3103 if (copy > len) 3104 copy = len; 3105 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 3106 goto fault; 3107 if ((len -= copy) == 0) 3108 return 0; 3109 offset += copy; 3110 to += copy; 3111 } 3112 start = end; 3113 } 3114 3115 if (!len) 3116 return 0; 3117 3118 fault: 3119 return -EFAULT; 3120 } 3121 EXPORT_SYMBOL(skb_copy_bits); 3122 3123 /* 3124 * Callback from splice_to_pipe(), if we need to release some pages 3125 * at the end of the spd in case we error'ed out in filling the pipe. 3126 */ 3127 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 3128 { 3129 put_page(spd->pages[i]); 3130 } 3131 3132 static struct page *linear_to_page(struct page *page, unsigned int *len, 3133 unsigned int *offset, 3134 struct sock *sk) 3135 { 3136 struct page_frag *pfrag = sk_page_frag(sk); 3137 3138 if (!sk_page_frag_refill(sk, pfrag)) 3139 return NULL; 3140 3141 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 3142 3143 memcpy(page_address(pfrag->page) + pfrag->offset, 3144 page_address(page) + *offset, *len); 3145 *offset = pfrag->offset; 3146 pfrag->offset += *len; 3147 3148 return pfrag->page; 3149 } 3150 3151 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 3152 struct page *page, 3153 unsigned int offset) 3154 { 3155 return spd->nr_pages && 3156 spd->pages[spd->nr_pages - 1] == page && 3157 (spd->partial[spd->nr_pages - 1].offset + 3158 spd->partial[spd->nr_pages - 1].len == offset); 3159 } 3160 3161 /* 3162 * Fill page/offset/length into spd, if it can hold more pages. 3163 */ 3164 static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, 3165 unsigned int *len, unsigned int offset, bool linear, 3166 struct sock *sk) 3167 { 3168 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 3169 return true; 3170 3171 if (linear) { 3172 page = linear_to_page(page, len, &offset, sk); 3173 if (!page) 3174 return true; 3175 } 3176 if (spd_can_coalesce(spd, page, offset)) { 3177 spd->partial[spd->nr_pages - 1].len += *len; 3178 return false; 3179 } 3180 get_page(page); 3181 spd->pages[spd->nr_pages] = page; 3182 spd->partial[spd->nr_pages].len = *len; 3183 spd->partial[spd->nr_pages].offset = offset; 3184 spd->nr_pages++; 3185 3186 return false; 3187 } 3188 3189 static bool __splice_segment(struct page *page, unsigned int poff, 3190 unsigned int plen, unsigned int *off, 3191 unsigned int *len, 3192 struct splice_pipe_desc *spd, bool linear, 3193 struct sock *sk) 3194 { 3195 if (!*len) 3196 return true; 3197 3198 /* skip this segment if already processed */ 3199 if (*off >= plen) { 3200 *off -= plen; 3201 return false; 3202 } 3203 3204 /* ignore any bits we already processed */ 3205 poff += *off; 3206 plen -= *off; 3207 *off = 0; 3208 3209 do { 3210 unsigned int flen = min(*len, plen); 3211 3212 if (spd_fill_page(spd, page, &flen, poff, linear, sk)) 3213 return true; 3214 poff += flen; 3215 plen -= flen; 3216 *len -= flen; 3217 if (!*len) 3218 return true; 3219 } while (plen); 3220 3221 return false; 3222 } 3223 3224 /* 3225 * Map linear and fragment data from the skb to spd. It reports true if the 3226 * pipe is full or if we already spliced the requested length. 3227 */ 3228 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 3229 unsigned int *offset, unsigned int *len, 3230 struct splice_pipe_desc *spd, struct sock *sk) 3231 { 3232 struct sk_buff *iter; 3233 int seg; 3234 3235 /* map the linear part : 3236 * If skb->head_frag is set, this 'linear' part is backed by a 3237 * fragment, and if the head is not shared with any clones then 3238 * we can avoid a copy since we own the head portion of this page. 3239 */ 3240 if (__splice_segment(virt_to_page(skb->data), 3241 (unsigned long) skb->data & (PAGE_SIZE - 1), 3242 skb_headlen(skb), 3243 offset, len, spd, 3244 skb_head_is_locked(skb), 3245 sk)) 3246 return true; 3247 3248 /* 3249 * then map the fragments 3250 */ 3251 if (!skb_frags_readable(skb)) 3252 return false; 3253 3254 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 3255 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 3256 3257 if (WARN_ON_ONCE(!skb_frag_page(f))) 3258 return false; 3259 3260 if (__splice_segment(skb_frag_page(f), 3261 skb_frag_off(f), skb_frag_size(f), 3262 offset, len, spd, false, sk)) 3263 return true; 3264 } 3265 3266 skb_walk_frags(skb, iter) { 3267 if (*offset >= iter->len) { 3268 *offset -= iter->len; 3269 continue; 3270 } 3271 /* __skb_splice_bits() only fails if the output has no room 3272 * left, so no point in going over the frag_list for the error 3273 * case. 3274 */ 3275 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 3276 return true; 3277 } 3278 3279 return false; 3280 } 3281 3282 /* 3283 * Map data from the skb to a pipe. Should handle both the linear part, 3284 * the fragments, and the frag list. 3285 */ 3286 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 3287 struct pipe_inode_info *pipe, unsigned int tlen, 3288 unsigned int flags) 3289 { 3290 struct partial_page partial[MAX_SKB_FRAGS]; 3291 struct page *pages[MAX_SKB_FRAGS]; 3292 struct splice_pipe_desc spd = { 3293 .pages = pages, 3294 .partial = partial, 3295 .nr_pages_max = MAX_SKB_FRAGS, 3296 .ops = &default_pipe_buf_ops, 3297 .spd_release = sock_spd_release, 3298 }; 3299 int ret = 0; 3300 3301 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 3302 3303 if (spd.nr_pages) 3304 ret = splice_to_pipe(pipe, &spd); 3305 3306 return ret; 3307 } 3308 EXPORT_SYMBOL_GPL(skb_splice_bits); 3309 3310 static int sendmsg_locked(struct sock *sk, struct msghdr *msg) 3311 { 3312 struct socket *sock = sk->sk_socket; 3313 size_t size = msg_data_left(msg); 3314 3315 if (!sock) 3316 return -EINVAL; 3317 3318 if (!sock->ops->sendmsg_locked) 3319 return sock_no_sendmsg_locked(sk, msg, size); 3320 3321 return sock->ops->sendmsg_locked(sk, msg, size); 3322 } 3323 3324 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg) 3325 { 3326 struct socket *sock = sk->sk_socket; 3327 3328 if (!sock) 3329 return -EINVAL; 3330 return sock_sendmsg(sock, msg); 3331 } 3332 3333 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg); 3334 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 3335 int len, sendmsg_func sendmsg, int flags) 3336 { 3337 int more_hint = sk_is_tcp(sk) ? MSG_MORE : 0; 3338 unsigned int orig_len = len; 3339 struct sk_buff *head = skb; 3340 unsigned short fragidx; 3341 int slen, ret; 3342 3343 do_frag_list: 3344 3345 /* Deal with head data */ 3346 while (offset < skb_headlen(skb) && len) { 3347 struct kvec kv; 3348 struct msghdr msg; 3349 3350 slen = min_t(int, len, skb_headlen(skb) - offset); 3351 kv.iov_base = skb->data + offset; 3352 kv.iov_len = slen; 3353 memset(&msg, 0, sizeof(msg)); 3354 msg.msg_flags = MSG_DONTWAIT | flags; 3355 if (slen < len) 3356 msg.msg_flags |= more_hint; 3357 3358 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen); 3359 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3360 sendmsg_unlocked, sk, &msg); 3361 if (ret <= 0) 3362 goto error; 3363 3364 offset += ret; 3365 len -= ret; 3366 } 3367 3368 /* All the data was skb head? */ 3369 if (!len) 3370 goto out; 3371 3372 /* Make offset relative to start of frags */ 3373 offset -= skb_headlen(skb); 3374 3375 /* Find where we are in frag list */ 3376 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3377 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3378 3379 if (offset < skb_frag_size(frag)) 3380 break; 3381 3382 offset -= skb_frag_size(frag); 3383 } 3384 3385 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3386 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3387 3388 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 3389 3390 while (slen) { 3391 struct bio_vec bvec; 3392 struct msghdr msg = { 3393 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | 3394 flags, 3395 }; 3396 3397 if (slen < len) 3398 msg.msg_flags |= more_hint; 3399 bvec_set_page(&bvec, skb_frag_page(frag), slen, 3400 skb_frag_off(frag) + offset); 3401 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, 3402 slen); 3403 3404 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3405 sendmsg_unlocked, sk, &msg); 3406 if (ret <= 0) 3407 goto error; 3408 3409 len -= ret; 3410 offset += ret; 3411 slen -= ret; 3412 } 3413 3414 offset = 0; 3415 } 3416 3417 if (len) { 3418 /* Process any frag lists */ 3419 3420 if (skb == head) { 3421 if (skb_has_frag_list(skb)) { 3422 skb = skb_shinfo(skb)->frag_list; 3423 goto do_frag_list; 3424 } 3425 } else if (skb->next) { 3426 skb = skb->next; 3427 goto do_frag_list; 3428 } 3429 } 3430 3431 out: 3432 return orig_len - len; 3433 3434 error: 3435 return orig_len == len ? ret : orig_len - len; 3436 } 3437 3438 /* Send skb data on a socket. Socket must be locked. */ 3439 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3440 int len) 3441 { 3442 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, 0); 3443 } 3444 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 3445 3446 int skb_send_sock_locked_with_flags(struct sock *sk, struct sk_buff *skb, 3447 int offset, int len, int flags) 3448 { 3449 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, flags); 3450 } 3451 EXPORT_SYMBOL_GPL(skb_send_sock_locked_with_flags); 3452 3453 /* Send skb data on a socket. Socket must be unlocked. */ 3454 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 3455 { 3456 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 0); 3457 } 3458 3459 /** 3460 * skb_store_bits - store bits from kernel buffer to skb 3461 * @skb: destination buffer 3462 * @offset: offset in destination 3463 * @from: source buffer 3464 * @len: number of bytes to copy 3465 * 3466 * Copy the specified number of bytes from the source buffer to the 3467 * destination skb. This function handles all the messy bits of 3468 * traversing fragment lists and such. 3469 */ 3470 3471 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 3472 { 3473 int start = skb_headlen(skb); 3474 struct sk_buff *frag_iter; 3475 int i, copy; 3476 3477 if (offset > (int)skb->len - len) 3478 goto fault; 3479 3480 if ((copy = start - offset) > 0) { 3481 if (copy > len) 3482 copy = len; 3483 skb_copy_to_linear_data_offset(skb, offset, from, copy); 3484 if ((len -= copy) == 0) 3485 return 0; 3486 offset += copy; 3487 from += copy; 3488 } 3489 3490 if (!skb_frags_readable(skb)) 3491 goto fault; 3492 3493 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3494 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3495 int end; 3496 3497 WARN_ON(start > offset + len); 3498 3499 end = start + skb_frag_size(frag); 3500 if ((copy = end - offset) > 0) { 3501 u32 p_off, p_len, copied; 3502 struct page *p; 3503 u8 *vaddr; 3504 3505 if (copy > len) 3506 copy = len; 3507 3508 skb_frag_foreach_page(frag, 3509 skb_frag_off(frag) + offset - start, 3510 copy, p, p_off, p_len, copied) { 3511 vaddr = kmap_atomic(p); 3512 memcpy(vaddr + p_off, from + copied, p_len); 3513 kunmap_atomic(vaddr); 3514 } 3515 3516 if ((len -= copy) == 0) 3517 return 0; 3518 offset += copy; 3519 from += copy; 3520 } 3521 start = end; 3522 } 3523 3524 skb_walk_frags(skb, frag_iter) { 3525 int end; 3526 3527 WARN_ON(start > offset + len); 3528 3529 end = start + frag_iter->len; 3530 if ((copy = end - offset) > 0) { 3531 if (copy > len) 3532 copy = len; 3533 if (skb_store_bits(frag_iter, offset - start, 3534 from, copy)) 3535 goto fault; 3536 if ((len -= copy) == 0) 3537 return 0; 3538 offset += copy; 3539 from += copy; 3540 } 3541 start = end; 3542 } 3543 if (!len) 3544 return 0; 3545 3546 fault: 3547 return -EFAULT; 3548 } 3549 EXPORT_SYMBOL(skb_store_bits); 3550 3551 /* Checksum skb data. */ 3552 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, __wsum csum) 3553 { 3554 int start = skb_headlen(skb); 3555 int i, copy = start - offset; 3556 struct sk_buff *frag_iter; 3557 int pos = 0; 3558 3559 /* Checksum header. */ 3560 if (copy > 0) { 3561 if (copy > len) 3562 copy = len; 3563 csum = csum_partial(skb->data + offset, copy, csum); 3564 if ((len -= copy) == 0) 3565 return csum; 3566 offset += copy; 3567 pos = copy; 3568 } 3569 3570 if (WARN_ON_ONCE(!skb_frags_readable(skb))) 3571 return 0; 3572 3573 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3574 int end; 3575 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3576 3577 WARN_ON(start > offset + len); 3578 3579 end = start + skb_frag_size(frag); 3580 if ((copy = end - offset) > 0) { 3581 u32 p_off, p_len, copied; 3582 struct page *p; 3583 __wsum csum2; 3584 u8 *vaddr; 3585 3586 if (copy > len) 3587 copy = len; 3588 3589 skb_frag_foreach_page(frag, 3590 skb_frag_off(frag) + offset - start, 3591 copy, p, p_off, p_len, copied) { 3592 vaddr = kmap_atomic(p); 3593 csum2 = csum_partial(vaddr + p_off, p_len, 0); 3594 kunmap_atomic(vaddr); 3595 csum = csum_block_add(csum, csum2, pos); 3596 pos += p_len; 3597 } 3598 3599 if (!(len -= copy)) 3600 return csum; 3601 offset += copy; 3602 } 3603 start = end; 3604 } 3605 3606 skb_walk_frags(skb, frag_iter) { 3607 int end; 3608 3609 WARN_ON(start > offset + len); 3610 3611 end = start + frag_iter->len; 3612 if ((copy = end - offset) > 0) { 3613 __wsum csum2; 3614 if (copy > len) 3615 copy = len; 3616 csum2 = skb_checksum(frag_iter, offset - start, copy, 3617 0); 3618 csum = csum_block_add(csum, csum2, pos); 3619 if ((len -= copy) == 0) 3620 return csum; 3621 offset += copy; 3622 pos += copy; 3623 } 3624 start = end; 3625 } 3626 if (unlikely(len)) { 3627 DEBUG_NET_WARN_ON_ONCE(1); 3628 return 0; 3629 } 3630 3631 return csum; 3632 } 3633 EXPORT_SYMBOL(skb_checksum); 3634 3635 /* Both of above in one bottle. */ 3636 3637 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3638 u8 *to, int len) 3639 { 3640 int start = skb_headlen(skb); 3641 int i, copy = start - offset; 3642 struct sk_buff *frag_iter; 3643 int pos = 0; 3644 __wsum csum = 0; 3645 3646 /* Copy header. */ 3647 if (copy > 0) { 3648 if (copy > len) 3649 copy = len; 3650 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3651 copy); 3652 if ((len -= copy) == 0) 3653 return csum; 3654 offset += copy; 3655 to += copy; 3656 pos = copy; 3657 } 3658 3659 if (!skb_frags_readable(skb)) { 3660 /* Don't hand the caller a buffer with stale bytes in it. */ 3661 if (len > 0) 3662 memset(to, 0, len); 3663 return 0; 3664 } 3665 3666 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3667 int end; 3668 3669 WARN_ON(start > offset + len); 3670 3671 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3672 if ((copy = end - offset) > 0) { 3673 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3674 u32 p_off, p_len, copied; 3675 struct page *p; 3676 __wsum csum2; 3677 u8 *vaddr; 3678 3679 if (copy > len) 3680 copy = len; 3681 3682 skb_frag_foreach_page(frag, 3683 skb_frag_off(frag) + offset - start, 3684 copy, p, p_off, p_len, copied) { 3685 vaddr = kmap_atomic(p); 3686 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3687 to + copied, 3688 p_len); 3689 kunmap_atomic(vaddr); 3690 csum = csum_block_add(csum, csum2, pos); 3691 pos += p_len; 3692 } 3693 3694 if (!(len -= copy)) 3695 return csum; 3696 offset += copy; 3697 to += copy; 3698 } 3699 start = end; 3700 } 3701 3702 skb_walk_frags(skb, frag_iter) { 3703 __wsum csum2; 3704 int end; 3705 3706 WARN_ON(start > offset + len); 3707 3708 end = start + frag_iter->len; 3709 if ((copy = end - offset) > 0) { 3710 if (copy > len) 3711 copy = len; 3712 csum2 = skb_copy_and_csum_bits(frag_iter, 3713 offset - start, 3714 to, copy); 3715 csum = csum_block_add(csum, csum2, pos); 3716 if ((len -= copy) == 0) 3717 return csum; 3718 offset += copy; 3719 to += copy; 3720 pos += copy; 3721 } 3722 start = end; 3723 } 3724 if (unlikely(len)) { 3725 DEBUG_NET_WARN_ON_ONCE(1); 3726 /* Don't hand the caller a buffer with stale bytes in it. */ 3727 if (len > 0) 3728 memset(to, 0, len); 3729 return 0; 3730 } 3731 return csum; 3732 } 3733 EXPORT_SYMBOL(skb_copy_and_csum_bits); 3734 3735 #ifdef CONFIG_NET_CRC32C 3736 u32 skb_crc32c(const struct sk_buff *skb, int offset, int len, u32 crc) 3737 { 3738 int start = skb_headlen(skb); 3739 int i, copy = start - offset; 3740 struct sk_buff *frag_iter; 3741 3742 if (copy > 0) { 3743 copy = min(copy, len); 3744 crc = crc32c(crc, skb->data + offset, copy); 3745 len -= copy; 3746 if (len == 0) 3747 return crc; 3748 offset += copy; 3749 } 3750 3751 if (WARN_ON_ONCE(!skb_frags_readable(skb))) 3752 return 0; 3753 3754 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3755 int end; 3756 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3757 3758 WARN_ON(start > offset + len); 3759 3760 end = start + skb_frag_size(frag); 3761 copy = end - offset; 3762 if (copy > 0) { 3763 u32 p_off, p_len, copied; 3764 struct page *p; 3765 u8 *vaddr; 3766 3767 copy = min(copy, len); 3768 skb_frag_foreach_page(frag, 3769 skb_frag_off(frag) + offset - start, 3770 copy, p, p_off, p_len, copied) { 3771 vaddr = kmap_atomic(p); 3772 crc = crc32c(crc, vaddr + p_off, p_len); 3773 kunmap_atomic(vaddr); 3774 } 3775 len -= copy; 3776 if (len == 0) 3777 return crc; 3778 offset += copy; 3779 } 3780 start = end; 3781 } 3782 3783 skb_walk_frags(skb, frag_iter) { 3784 int end; 3785 3786 WARN_ON(start > offset + len); 3787 3788 end = start + frag_iter->len; 3789 copy = end - offset; 3790 if (copy > 0) { 3791 copy = min(copy, len); 3792 crc = skb_crc32c(frag_iter, offset - start, copy, crc); 3793 len -= copy; 3794 if (len == 0) 3795 return crc; 3796 offset += copy; 3797 } 3798 start = end; 3799 } 3800 if (unlikely(len)) { 3801 DEBUG_NET_WARN_ON_ONCE(1); 3802 return 0; 3803 } 3804 3805 return crc; 3806 } 3807 EXPORT_SYMBOL(skb_crc32c); 3808 #endif /* CONFIG_NET_CRC32C */ 3809 3810 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3811 { 3812 __sum16 sum; 3813 3814 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3815 /* See comments in __skb_checksum_complete(). */ 3816 if (likely(!sum)) { 3817 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3818 !skb->csum_complete_sw) 3819 netdev_rx_csum_fault(skb->dev, skb); 3820 } 3821 if (!skb_shared(skb)) 3822 skb->csum_valid = !sum; 3823 return sum; 3824 } 3825 EXPORT_SYMBOL(__skb_checksum_complete_head); 3826 3827 /* This function assumes skb->csum already holds pseudo header's checksum, 3828 * which has been changed from the hardware checksum, for example, by 3829 * __skb_checksum_validate_complete(). And, the original skb->csum must 3830 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3831 * 3832 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3833 * zero. The new checksum is stored back into skb->csum unless the skb is 3834 * shared. 3835 */ 3836 __sum16 __skb_checksum_complete(struct sk_buff *skb) 3837 { 3838 __wsum csum; 3839 __sum16 sum; 3840 3841 csum = skb_checksum(skb, 0, skb->len, 0); 3842 3843 sum = csum_fold(csum_add(skb->csum, csum)); 3844 /* This check is inverted, because we already knew the hardware 3845 * checksum is invalid before calling this function. So, if the 3846 * re-computed checksum is valid instead, then we have a mismatch 3847 * between the original skb->csum and skb_checksum(). This means either 3848 * the original hardware checksum is incorrect or we screw up skb->csum 3849 * when moving skb->data around. 3850 */ 3851 if (likely(!sum)) { 3852 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3853 !skb->csum_complete_sw) 3854 netdev_rx_csum_fault(skb->dev, skb); 3855 } 3856 3857 if (!skb_shared(skb)) { 3858 /* Save full packet checksum */ 3859 skb->csum = csum; 3860 skb->ip_summed = CHECKSUM_COMPLETE; 3861 skb->csum_complete_sw = 1; 3862 skb->csum_valid = !sum; 3863 } 3864 3865 return sum; 3866 } 3867 EXPORT_SYMBOL(__skb_checksum_complete); 3868 3869 /** 3870 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3871 * @from: source buffer 3872 * 3873 * Calculates the amount of linear headroom needed in the 'to' skb passed 3874 * into skb_zerocopy(). 3875 */ 3876 unsigned int 3877 skb_zerocopy_headlen(const struct sk_buff *from) 3878 { 3879 unsigned int hlen = 0; 3880 3881 if (!from->head_frag || 3882 skb_headlen(from) < L1_CACHE_BYTES || 3883 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3884 hlen = skb_headlen(from); 3885 if (!hlen) 3886 hlen = from->len; 3887 } 3888 3889 if (skb_has_frag_list(from)) 3890 hlen = from->len; 3891 3892 return hlen; 3893 } 3894 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3895 3896 /** 3897 * skb_zerocopy - Zero copy skb to skb 3898 * @to: destination buffer 3899 * @from: source buffer 3900 * @len: number of bytes to copy from source buffer 3901 * @hlen: size of linear headroom in destination buffer 3902 * 3903 * Copies up to `len` bytes from `from` to `to` by creating references 3904 * to the frags in the source buffer. 3905 * 3906 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3907 * headroom in the `to` buffer. 3908 * 3909 * Return value: 3910 * 0: everything is OK 3911 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3912 * -EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head 3913 * payload would be mixed with unreadable frags. 3914 */ 3915 int 3916 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3917 { 3918 int i, j = 0; 3919 int plen = 0; /* length of skb->head fragment */ 3920 int ret; 3921 struct page *page; 3922 unsigned int offset; 3923 3924 if (unlikely(!from->head_frag && !hlen)) { 3925 DEBUG_NET_WARN_ON_ONCE(1); 3926 return -EFAULT; 3927 } 3928 3929 /* dont bother with small payloads */ 3930 if (len <= skb_tailroom(to)) 3931 return skb_copy_bits(from, 0, skb_put(to, len), len); 3932 3933 to->pp_recycle = from->pp_recycle; 3934 if (hlen) { 3935 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3936 if (unlikely(ret)) 3937 return ret; 3938 len -= hlen; 3939 } else { 3940 plen = min_t(int, skb_headlen(from), len); 3941 if (plen) { 3942 page = virt_to_head_page(from->head); 3943 offset = from->data - (unsigned char *)page_address(page); 3944 __skb_fill_netmem_desc(to, 0, page_to_netmem(page), 3945 offset, plen); 3946 skb_frag_ref(to, 0); 3947 j = 1; 3948 len -= plen; 3949 } 3950 } 3951 3952 if (!skb_frags_readable(from) && j > 0 && len) { 3953 skb_frag_unref(to, 0); 3954 return -EFAULT; 3955 } 3956 3957 skb_len_add(to, len + plen); 3958 3959 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3960 if (j > 0) 3961 skb_frag_unref(to, 0); 3962 return -ENOMEM; 3963 } 3964 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3965 3966 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3967 int size; 3968 3969 if (!len) 3970 break; 3971 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3972 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3973 len); 3974 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3975 len -= size; 3976 skb_frag_ref(to, j); 3977 j++; 3978 } 3979 skb_shinfo(to)->nr_frags = j; 3980 3981 skb_shinfo(to)->flags |= skb_shinfo(from)->flags & SKBFL_SHARED_FRAG; 3982 3983 if (i > 0 && from->unreadable) 3984 to->unreadable = 1; 3985 3986 return 0; 3987 } 3988 EXPORT_SYMBOL_GPL(skb_zerocopy); 3989 3990 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3991 { 3992 __wsum csum; 3993 long csstart; 3994 3995 if (skb->ip_summed == CHECKSUM_PARTIAL) 3996 csstart = skb_checksum_start_offset(skb); 3997 else 3998 csstart = skb_headlen(skb); 3999 4000 if (unlikely(csstart < 0 || csstart > skb_headlen(skb) || 4001 (skb->ip_summed == CHECKSUM_PARTIAL && 4002 csstart + skb->csum_offset + sizeof(__sum16) > 4003 skb->len))) { 4004 DEBUG_NET_WARN_ON_ONCE(1); 4005 /* Send the frame without the checksum filled in, or send 4006 * zeroes if we can't even copy it. 4007 */ 4008 if (skb_copy_bits(skb, 0, to, skb->len)) 4009 memset(to, 0, skb->len); 4010 return; 4011 } 4012 4013 skb_copy_from_linear_data(skb, to, csstart); 4014 4015 csum = 0; 4016 if (csstart != skb->len) 4017 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 4018 skb->len - csstart); 4019 4020 if (skb->ip_summed == CHECKSUM_PARTIAL) { 4021 long csstuff = csstart + skb->csum_offset; 4022 4023 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 4024 } 4025 } 4026 EXPORT_SYMBOL(skb_copy_and_csum_dev); 4027 4028 /** 4029 * skb_dequeue - remove from the head of the queue 4030 * @list: list to dequeue from 4031 * 4032 * Remove the head of the list. The list lock is taken so the function 4033 * may be used safely with other locking list functions. The head item is 4034 * returned or %NULL if the list is empty. 4035 */ 4036 4037 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 4038 { 4039 unsigned long flags; 4040 struct sk_buff *result; 4041 4042 spin_lock_irqsave(&list->lock, flags); 4043 result = __skb_dequeue(list); 4044 spin_unlock_irqrestore(&list->lock, flags); 4045 return result; 4046 } 4047 EXPORT_SYMBOL(skb_dequeue); 4048 4049 /** 4050 * skb_dequeue_tail - remove from the tail of the queue 4051 * @list: list to dequeue from 4052 * 4053 * Remove the tail of the list. The list lock is taken so the function 4054 * may be used safely with other locking list functions. The tail item is 4055 * returned or %NULL if the list is empty. 4056 */ 4057 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 4058 { 4059 unsigned long flags; 4060 struct sk_buff *result; 4061 4062 spin_lock_irqsave(&list->lock, flags); 4063 result = __skb_dequeue_tail(list); 4064 spin_unlock_irqrestore(&list->lock, flags); 4065 return result; 4066 } 4067 EXPORT_SYMBOL(skb_dequeue_tail); 4068 4069 /** 4070 * skb_queue_purge_reason - empty a list 4071 * @list: list to empty 4072 * @reason: drop reason 4073 * 4074 * Delete all buffers on an &sk_buff list. Each buffer is removed from 4075 * the list and one reference dropped. This function takes the list 4076 * lock and is atomic with respect to other list locking functions. 4077 */ 4078 void skb_queue_purge_reason(struct sk_buff_head *list, 4079 enum skb_drop_reason reason) 4080 { 4081 struct sk_buff_head tmp; 4082 unsigned long flags; 4083 4084 if (skb_queue_empty_lockless(list)) 4085 return; 4086 4087 __skb_queue_head_init(&tmp); 4088 4089 spin_lock_irqsave(&list->lock, flags); 4090 skb_queue_splice_init(list, &tmp); 4091 spin_unlock_irqrestore(&list->lock, flags); 4092 4093 __skb_queue_purge_reason(&tmp, reason); 4094 } 4095 EXPORT_SYMBOL(skb_queue_purge_reason); 4096 4097 /** 4098 * skb_rbtree_purge - empty a skb rbtree 4099 * @root: root of the rbtree to empty 4100 * Return value: the sum of truesizes of all purged skbs. 4101 * 4102 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 4103 * the list and one reference dropped. This function does not take 4104 * any lock. Synchronization should be handled by the caller (e.g., TCP 4105 * out-of-order queue is protected by the socket lock). 4106 */ 4107 unsigned int skb_rbtree_purge(struct rb_root *root) 4108 { 4109 struct rb_node *p = rb_first(root); 4110 unsigned int sum = 0; 4111 4112 while (p) { 4113 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 4114 4115 p = rb_next(p); 4116 rb_erase(&skb->rbnode, root); 4117 sum += skb->truesize; 4118 kfree_skb(skb); 4119 } 4120 return sum; 4121 } 4122 4123 void skb_errqueue_purge(struct sk_buff_head *list) 4124 { 4125 struct sk_buff *skb, *next; 4126 struct sk_buff_head kill; 4127 unsigned long flags; 4128 4129 __skb_queue_head_init(&kill); 4130 4131 spin_lock_irqsave(&list->lock, flags); 4132 skb_queue_walk_safe(list, skb, next) { 4133 if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY || 4134 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) 4135 continue; 4136 __skb_unlink(skb, list); 4137 __skb_queue_tail(&kill, skb); 4138 } 4139 spin_unlock_irqrestore(&list->lock, flags); 4140 __skb_queue_purge(&kill); 4141 } 4142 EXPORT_SYMBOL(skb_errqueue_purge); 4143 4144 /** 4145 * skb_queue_head - queue a buffer at the list head 4146 * @list: list to use 4147 * @newsk: buffer to queue 4148 * 4149 * Queue a buffer at the start of the list. This function takes the 4150 * list lock and can be used safely with other locking &sk_buff functions 4151 * safely. 4152 * 4153 * A buffer cannot be placed on two lists at the same time. 4154 */ 4155 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 4156 { 4157 unsigned long flags; 4158 4159 spin_lock_irqsave(&list->lock, flags); 4160 __skb_queue_head(list, newsk); 4161 spin_unlock_irqrestore(&list->lock, flags); 4162 } 4163 EXPORT_SYMBOL(skb_queue_head); 4164 4165 /** 4166 * skb_queue_tail - queue a buffer at the list tail 4167 * @list: list to use 4168 * @newsk: buffer to queue 4169 * 4170 * Queue a buffer at the tail of the list. This function takes the 4171 * list lock and can be used safely with other locking &sk_buff functions 4172 * safely. 4173 * 4174 * A buffer cannot be placed on two lists at the same time. 4175 */ 4176 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 4177 { 4178 unsigned long flags; 4179 4180 spin_lock_irqsave(&list->lock, flags); 4181 __skb_queue_tail(list, newsk); 4182 spin_unlock_irqrestore(&list->lock, flags); 4183 } 4184 EXPORT_SYMBOL(skb_queue_tail); 4185 4186 /** 4187 * skb_unlink - remove a buffer from a list 4188 * @skb: buffer to remove 4189 * @list: list to use 4190 * 4191 * Remove a packet from a list. The list locks are taken and this 4192 * function is atomic with respect to other list locked calls 4193 * 4194 * You must know what list the SKB is on. 4195 */ 4196 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 4197 { 4198 unsigned long flags; 4199 4200 spin_lock_irqsave(&list->lock, flags); 4201 __skb_unlink(skb, list); 4202 spin_unlock_irqrestore(&list->lock, flags); 4203 } 4204 EXPORT_SYMBOL(skb_unlink); 4205 4206 /** 4207 * skb_append - append a buffer 4208 * @old: buffer to insert after 4209 * @newsk: buffer to insert 4210 * @list: list to use 4211 * 4212 * Place a packet after a given packet in a list. The list locks are taken 4213 * and this function is atomic with respect to other list locked calls. 4214 * A buffer cannot be placed on two lists at the same time. 4215 */ 4216 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 4217 { 4218 unsigned long flags; 4219 4220 spin_lock_irqsave(&list->lock, flags); 4221 __skb_queue_after(list, old, newsk); 4222 spin_unlock_irqrestore(&list->lock, flags); 4223 } 4224 EXPORT_SYMBOL(skb_append); 4225 4226 static inline void skb_split_inside_header(struct sk_buff *skb, 4227 struct sk_buff* skb1, 4228 const u32 len, const int pos) 4229 { 4230 int i; 4231 4232 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 4233 pos - len); 4234 /* And move data appendix as is. */ 4235 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 4236 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 4237 4238 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 4239 skb1->unreadable = skb->unreadable; 4240 skb_shinfo(skb)->nr_frags = 0; 4241 skb1->data_len = skb->data_len; 4242 skb1->len += skb1->data_len; 4243 skb->data_len = 0; 4244 skb->len = len; 4245 skb_set_tail_pointer(skb, len); 4246 } 4247 4248 static inline void skb_split_no_header(struct sk_buff *skb, 4249 struct sk_buff* skb1, 4250 const u32 len, int pos) 4251 { 4252 int i, k = 0; 4253 const int nfrags = skb_shinfo(skb)->nr_frags; 4254 4255 skb_shinfo(skb)->nr_frags = 0; 4256 skb1->len = skb1->data_len = skb->len - len; 4257 skb->len = len; 4258 skb->data_len = len - pos; 4259 4260 for (i = 0; i < nfrags; i++) { 4261 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 4262 4263 if (pos + size > len) { 4264 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 4265 4266 if (pos < len) { 4267 /* Split frag. 4268 * We have two variants in this case: 4269 * 1. Move all the frag to the second 4270 * part, if it is possible. F.e. 4271 * this approach is mandatory for TUX, 4272 * where splitting is expensive. 4273 * 2. Split is accurately. We make this. 4274 */ 4275 skb_frag_ref(skb1, k); 4276 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 4277 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 4278 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 4279 skb_shinfo(skb)->nr_frags++; 4280 } 4281 k++; 4282 } else 4283 skb_shinfo(skb)->nr_frags++; 4284 pos += size; 4285 } 4286 skb_shinfo(skb1)->nr_frags = k; 4287 4288 skb1->unreadable = skb->unreadable; 4289 } 4290 4291 /** 4292 * skb_split - Split fragmented skb to two parts at length len. 4293 * @skb: the buffer to split 4294 * @skb1: the buffer to receive the second part 4295 * @len: new length for skb 4296 */ 4297 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 4298 { 4299 int pos = skb_headlen(skb); 4300 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 4301 4302 skb_zcopy_downgrade_managed(skb); 4303 4304 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 4305 skb_zerocopy_clone(skb1, skb, 0); 4306 skb1->pp_recycle = skb->pp_recycle; 4307 if (len < pos) /* Split line is inside header. */ 4308 skb_split_inside_header(skb, skb1, len, pos); 4309 else /* Second chunk has no header, nothing to copy. */ 4310 skb_split_no_header(skb, skb1, len, pos); 4311 } 4312 EXPORT_SYMBOL(skb_split); 4313 4314 /* Shifting from/to a cloned skb is a no-go. 4315 * 4316 * Caller cannot keep skb_shinfo related pointers past calling here! 4317 */ 4318 static int skb_prepare_for_shift(struct sk_buff *skb) 4319 { 4320 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 4321 } 4322 4323 /** 4324 * skb_shift - Shifts paged data partially from skb to another 4325 * @tgt: buffer into which tail data gets added 4326 * @skb: buffer from which the paged data comes from 4327 * @shiftlen: shift up to this many bytes 4328 * 4329 * Attempts to shift up to shiftlen worth of bytes, which may be less than 4330 * the length of the skb, from skb to tgt. Returns number bytes shifted. 4331 * It's up to caller to free skb if everything was shifted. 4332 * 4333 * If @tgt runs out of frags, the whole operation is aborted. 4334 * 4335 * Skb cannot include anything else but paged data while tgt is allowed 4336 * to have non-paged data as well. 4337 * 4338 * TODO: full sized shift could be optimized but that would need 4339 * specialized skb free'er to handle frags without up-to-date nr_frags. 4340 */ 4341 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 4342 { 4343 int from, to, merge, todo; 4344 skb_frag_t *fragfrom, *fragto; 4345 4346 if (unlikely(shiftlen > skb->len)) { 4347 DEBUG_NET_WARN_ON_ONCE(1); 4348 return 0; 4349 } 4350 4351 if (skb_headlen(skb)) 4352 return 0; 4353 if (skb_zcopy(tgt) || skb_zcopy(skb)) 4354 return 0; 4355 4356 DEBUG_NET_WARN_ON_ONCE(tgt->pp_recycle != skb->pp_recycle); 4357 DEBUG_NET_WARN_ON_ONCE(skb_cmp_decrypted(tgt, skb)); 4358 4359 todo = shiftlen; 4360 from = 0; 4361 to = skb_shinfo(tgt)->nr_frags; 4362 fragfrom = &skb_shinfo(skb)->frags[from]; 4363 4364 /* Actual merge is delayed until the point when we know we can 4365 * commit all, so that we don't have to undo partial changes 4366 */ 4367 if (!skb_can_coalesce_netmem(tgt, to, skb_frag_netmem(fragfrom), 4368 skb_frag_off(fragfrom))) { 4369 merge = -1; 4370 } else { 4371 merge = to - 1; 4372 4373 todo -= skb_frag_size(fragfrom); 4374 if (todo < 0) { 4375 if (skb_prepare_for_shift(skb) || 4376 skb_prepare_for_shift(tgt)) 4377 return 0; 4378 4379 /* All previous frag pointers might be stale! */ 4380 fragfrom = &skb_shinfo(skb)->frags[from]; 4381 fragto = &skb_shinfo(tgt)->frags[merge]; 4382 4383 skb_frag_size_add(fragto, shiftlen); 4384 skb_frag_size_sub(fragfrom, shiftlen); 4385 skb_frag_off_add(fragfrom, shiftlen); 4386 4387 goto onlymerged; 4388 } 4389 4390 from++; 4391 } 4392 4393 /* Skip full, not-fitting skb to avoid expensive operations */ 4394 if ((shiftlen == skb->len) && 4395 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 4396 return 0; 4397 4398 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 4399 return 0; 4400 4401 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 4402 if (to == MAX_SKB_FRAGS) 4403 return 0; 4404 4405 fragfrom = &skb_shinfo(skb)->frags[from]; 4406 fragto = &skb_shinfo(tgt)->frags[to]; 4407 4408 if (todo >= skb_frag_size(fragfrom)) { 4409 *fragto = *fragfrom; 4410 todo -= skb_frag_size(fragfrom); 4411 from++; 4412 to++; 4413 4414 } else { 4415 skb_frag_page_copy(fragto, fragfrom); 4416 skb_frag_ref(tgt, to); 4417 skb_frag_off_copy(fragto, fragfrom); 4418 skb_frag_size_set(fragto, todo); 4419 4420 skb_frag_off_add(fragfrom, todo); 4421 skb_frag_size_sub(fragfrom, todo); 4422 todo = 0; 4423 4424 to++; 4425 break; 4426 } 4427 } 4428 4429 /* The frags ran out before shiftlen did. Nothing has been committed 4430 * yet, so back out. 4431 */ 4432 if (unlikely(todo > 0)) { 4433 DEBUG_NET_WARN_ON_ONCE(1); 4434 return 0; 4435 } 4436 4437 /* Ready to "commit" this state change to tgt */ 4438 skb_shinfo(tgt)->nr_frags = to; 4439 4440 if (merge >= 0) { 4441 fragfrom = &skb_shinfo(skb)->frags[0]; 4442 fragto = &skb_shinfo(tgt)->frags[merge]; 4443 4444 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 4445 skb_frag_unref(skb, 0); 4446 } 4447 4448 /* Reposition in the original skb */ 4449 to = 0; 4450 while (from < skb_shinfo(skb)->nr_frags) 4451 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 4452 skb_shinfo(skb)->nr_frags = to; 4453 4454 onlymerged: 4455 /* Most likely the tgt won't ever need its checksum anymore, skb on 4456 * the other hand might need it if it needs to be resent 4457 */ 4458 tgt->ip_summed = CHECKSUM_PARTIAL; 4459 skb->ip_summed = CHECKSUM_PARTIAL; 4460 4461 skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 4462 4463 skb_len_add(skb, -shiftlen); 4464 skb_len_add(tgt, shiftlen); 4465 4466 return shiftlen; 4467 } 4468 EXPORT_SYMBOL_IF_KUNIT(skb_shift); 4469 4470 /** 4471 * skb_prepare_seq_read - Prepare a sequential read of skb data 4472 * @skb: the buffer to read 4473 * @from: lower offset of data to be read 4474 * @to: upper offset of data to be read 4475 * @st: state variable 4476 * 4477 * Initializes the specified state variable. Must be called before 4478 * invoking skb_seq_read() for the first time. 4479 */ 4480 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 4481 unsigned int to, struct skb_seq_state *st) 4482 { 4483 st->lower_offset = from; 4484 st->upper_offset = to; 4485 st->root_skb = st->cur_skb = skb; 4486 st->frag_idx = st->stepped_offset = 0; 4487 st->frag_data = NULL; 4488 st->frag_off = 0; 4489 } 4490 EXPORT_SYMBOL(skb_prepare_seq_read); 4491 4492 /** 4493 * skb_seq_read - Sequentially read skb data 4494 * @consumed: number of bytes consumed by the caller so far 4495 * @data: destination pointer for data to be returned 4496 * @st: state variable 4497 * 4498 * Reads a block of skb data at @consumed relative to the 4499 * lower offset specified to skb_prepare_seq_read(). Assigns 4500 * the head of the data block to @data and returns the length 4501 * of the block or 0 if the end of the skb data or the upper 4502 * offset has been reached. 4503 * 4504 * The caller is not required to consume all of the data 4505 * returned, i.e. @consumed is typically set to the number 4506 * of bytes already consumed and the next call to 4507 * skb_seq_read() will return the remaining part of the block. 4508 * 4509 * Note 1: The size of each block of data returned can be arbitrary, 4510 * this limitation is the cost for zerocopy sequential 4511 * reads of potentially non linear data. 4512 * 4513 * Note 2: Fragment lists within fragments are not implemented 4514 * at the moment, state->root_skb could be replaced with 4515 * a stack for this purpose. 4516 */ 4517 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 4518 struct skb_seq_state *st) 4519 { 4520 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 4521 skb_frag_t *frag; 4522 4523 if (unlikely(abs_offset >= st->upper_offset)) { 4524 if (st->frag_data) { 4525 kunmap_atomic(st->frag_data); 4526 st->frag_data = NULL; 4527 } 4528 return 0; 4529 } 4530 4531 next_skb: 4532 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 4533 4534 if (abs_offset < block_limit && !st->frag_data) { 4535 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 4536 return block_limit - abs_offset; 4537 } 4538 4539 if (!skb_frags_readable(st->cur_skb)) 4540 return 0; 4541 4542 if (st->frag_idx == 0 && !st->frag_data) 4543 st->stepped_offset += skb_headlen(st->cur_skb); 4544 4545 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 4546 unsigned int pg_idx, pg_off, pg_sz; 4547 4548 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 4549 4550 pg_idx = 0; 4551 pg_off = skb_frag_off(frag); 4552 pg_sz = skb_frag_size(frag); 4553 4554 if (skb_frag_must_loop(skb_frag_page(frag))) { 4555 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 4556 pg_off = offset_in_page(pg_off + st->frag_off); 4557 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 4558 PAGE_SIZE - pg_off); 4559 } 4560 4561 block_limit = pg_sz + st->stepped_offset; 4562 if (abs_offset < block_limit) { 4563 if (!st->frag_data) 4564 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 4565 4566 *data = (u8 *)st->frag_data + pg_off + 4567 (abs_offset - st->stepped_offset); 4568 4569 return block_limit - abs_offset; 4570 } 4571 4572 if (st->frag_data) { 4573 kunmap_atomic(st->frag_data); 4574 st->frag_data = NULL; 4575 } 4576 4577 st->stepped_offset += pg_sz; 4578 st->frag_off += pg_sz; 4579 if (st->frag_off == skb_frag_size(frag)) { 4580 st->frag_off = 0; 4581 st->frag_idx++; 4582 } 4583 } 4584 4585 if (st->frag_data) { 4586 kunmap_atomic(st->frag_data); 4587 st->frag_data = NULL; 4588 } 4589 4590 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 4591 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 4592 st->frag_idx = 0; 4593 goto next_skb; 4594 } else if (st->cur_skb->next) { 4595 st->cur_skb = st->cur_skb->next; 4596 st->frag_idx = 0; 4597 goto next_skb; 4598 } 4599 4600 return 0; 4601 } 4602 EXPORT_SYMBOL(skb_seq_read); 4603 4604 /** 4605 * skb_abort_seq_read - Abort a sequential read of skb data 4606 * @st: state variable 4607 * 4608 * Must be called if skb_seq_read() was not called until it 4609 * returned 0. 4610 */ 4611 void skb_abort_seq_read(struct skb_seq_state *st) 4612 { 4613 if (st->frag_data) 4614 kunmap_atomic(st->frag_data); 4615 } 4616 EXPORT_SYMBOL(skb_abort_seq_read); 4617 4618 /** 4619 * skb_copy_seq_read() - copy from a skb_seq_state to a buffer 4620 * @st: source skb_seq_state 4621 * @offset: offset in source 4622 * @to: destination buffer 4623 * @len: number of bytes to copy 4624 * 4625 * Copy @len bytes from @offset bytes into the source @st to the destination 4626 * buffer @to. `offset` should increase (or be unchanged) with each subsequent 4627 * call to this function. If offset needs to decrease from the previous use `st` 4628 * should be reset first. 4629 * 4630 * Return: 0 on success or -EINVAL if the copy ended early 4631 */ 4632 int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len) 4633 { 4634 const u8 *data; 4635 u32 sqlen; 4636 4637 for (;;) { 4638 sqlen = skb_seq_read(offset, &data, st); 4639 if (sqlen == 0) 4640 return -EINVAL; 4641 if (sqlen >= len) { 4642 memcpy(to, data, len); 4643 return 0; 4644 } 4645 memcpy(to, data, sqlen); 4646 to += sqlen; 4647 offset += sqlen; 4648 len -= sqlen; 4649 } 4650 } 4651 EXPORT_SYMBOL(skb_copy_seq_read); 4652 4653 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 4654 4655 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 4656 struct ts_config *conf, 4657 struct ts_state *state) 4658 { 4659 return skb_seq_read(offset, text, TS_SKB_CB(state)); 4660 } 4661 4662 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 4663 { 4664 skb_abort_seq_read(TS_SKB_CB(state)); 4665 } 4666 4667 /** 4668 * skb_find_text - Find a text pattern in skb data 4669 * @skb: the buffer to look in 4670 * @from: search offset 4671 * @to: search limit 4672 * @config: textsearch configuration 4673 * 4674 * Finds a pattern in the skb data according to the specified 4675 * textsearch configuration. Use textsearch_next() to retrieve 4676 * subsequent occurrences of the pattern. Returns the offset 4677 * to the first occurrence or UINT_MAX if no match was found. 4678 */ 4679 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4680 unsigned int to, struct ts_config *config) 4681 { 4682 unsigned int patlen = config->ops->get_pattern_len(config); 4683 struct ts_state state; 4684 unsigned int ret; 4685 4686 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4687 4688 config->get_next_block = skb_ts_get_next_block; 4689 config->finish = skb_ts_finish; 4690 4691 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4692 4693 ret = textsearch_find(config, &state); 4694 return (ret + patlen <= to - from ? ret : UINT_MAX); 4695 } 4696 EXPORT_SYMBOL(skb_find_text); 4697 4698 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4699 int offset, size_t size, size_t max_frags) 4700 { 4701 int i = skb_shinfo(skb)->nr_frags; 4702 4703 if (skb_can_coalesce(skb, i, page, offset)) { 4704 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4705 } else if (i < max_frags) { 4706 skb_zcopy_downgrade_managed(skb); 4707 get_page(page); 4708 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4709 } else { 4710 return -EMSGSIZE; 4711 } 4712 4713 return 0; 4714 } 4715 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4716 4717 /** 4718 * skb_pull_rcsum - pull skb and update receive checksum 4719 * @skb: buffer to update 4720 * @len: length of data pulled 4721 * 4722 * This function performs an skb_pull on the packet and updates 4723 * the CHECKSUM_COMPLETE checksum. It should be used on 4724 * receive path processing instead of skb_pull unless you know 4725 * that the checksum difference is zero (e.g., a valid IP header) 4726 * or you are setting ip_summed to CHECKSUM_NONE. 4727 */ 4728 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4729 { 4730 unsigned char *data = skb->data; 4731 4732 BUG_ON(len > skb->len); 4733 __skb_pull(skb, len); 4734 skb_postpull_rcsum(skb, data, len); 4735 return skb->data; 4736 } 4737 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4738 4739 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4740 { 4741 skb_frag_t head_frag; 4742 struct page *page; 4743 4744 page = virt_to_head_page(frag_skb->head); 4745 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data - 4746 (unsigned char *)page_address(page), 4747 skb_headlen(frag_skb)); 4748 return head_frag; 4749 } 4750 4751 struct sk_buff *skb_segment_list(struct sk_buff *skb, 4752 netdev_features_t features, 4753 unsigned int offset) 4754 { 4755 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4756 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4757 unsigned int delta_len = 0; 4758 struct sk_buff *tail = NULL; 4759 struct sk_buff *nskb, *tmp; 4760 int len_diff, err; 4761 4762 /* Only skb_gro_receive_list generated skbs arrive here */ 4763 DEBUG_NET_WARN_ON_ONCE(!(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)); 4764 4765 skb_push(skb, -skb_network_offset(skb) + offset); 4766 4767 /* Ensure the head is writeable before touching the shared info */ 4768 err = skb_unclone(skb, GFP_ATOMIC); 4769 if (err) 4770 goto err_linearize; 4771 4772 skb_shinfo(skb)->frag_list = NULL; 4773 4774 while (list_skb) { 4775 nskb = list_skb; 4776 list_skb = list_skb->next; 4777 4778 DEBUG_NET_WARN_ON_ONCE(nskb->sk); 4779 4780 err = 0; 4781 if (skb_shared(nskb)) { 4782 tmp = skb_clone(nskb, GFP_ATOMIC); 4783 if (tmp) { 4784 consume_skb(nskb); 4785 nskb = tmp; 4786 err = skb_unclone(nskb, GFP_ATOMIC); 4787 } else { 4788 err = -ENOMEM; 4789 } 4790 } 4791 4792 if (!err) 4793 err = skb_cow_head(nskb, offset + tnl_hlen - 4794 skb_network_offset(nskb)); 4795 4796 if (!tail) 4797 skb->next = nskb; 4798 else 4799 tail->next = nskb; 4800 4801 /* nskb may be a clone with a NULL ->next: keep the remaining 4802 * members linked so that err_linearize frees them. 4803 */ 4804 nskb->next = list_skb; 4805 4806 if (unlikely(err)) 4807 goto err_linearize; 4808 4809 tail = nskb; 4810 4811 delta_len += nskb->len; 4812 4813 skb_push(nskb, -skb_network_offset(nskb) + offset); 4814 4815 skb_release_head_state(nskb); 4816 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4817 __copy_skb_header(nskb, skb); 4818 4819 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4820 nskb->transport_header += len_diff; 4821 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4822 nskb->data - tnl_hlen, 4823 offset + tnl_hlen); 4824 4825 if (skb_needs_linearize(nskb, features) && 4826 __skb_linearize(nskb)) 4827 goto err_linearize; 4828 } 4829 4830 skb->data_len = skb->data_len - delta_len; 4831 skb->len = skb->len - delta_len; 4832 4833 skb_gso_reset(skb); 4834 4835 skb->prev = tail; 4836 4837 if (skb_needs_linearize(skb, features) && 4838 __skb_linearize(skb)) 4839 goto err_linearize; 4840 4841 skb_get(skb); 4842 4843 return skb; 4844 4845 err_linearize: 4846 kfree_skb_list(skb->next); 4847 skb->next = NULL; 4848 return ERR_PTR(-ENOMEM); 4849 } 4850 EXPORT_SYMBOL_GPL(skb_segment_list); 4851 4852 /** 4853 * skb_segment - Perform protocol segmentation on skb. 4854 * @head_skb: buffer to segment 4855 * @features: features for the output path (see dev->features) 4856 * 4857 * This function performs segmentation on the given skb. It returns 4858 * a pointer to the first in a list of new skbs for the segments. 4859 * In case of error it returns ERR_PTR(err). 4860 */ 4861 struct sk_buff *skb_segment(struct sk_buff *head_skb, 4862 netdev_features_t features) 4863 { 4864 struct sk_buff *segs = NULL; 4865 struct sk_buff *tail = NULL; 4866 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4867 unsigned int max_segs = SKB_GSO_CB(head_skb)->max_segs; 4868 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4869 bool gso_by_frags = mss == GSO_BY_FRAGS; 4870 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4871 unsigned int offset = doffset; 4872 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4873 unsigned int partial_segs = 0; 4874 unsigned int headroom; 4875 unsigned int len = head_skb->len; 4876 struct sk_buff *frag_skb; 4877 skb_frag_t *frag; 4878 __be16 proto; 4879 bool csum, sg; 4880 int err = -ENOMEM; 4881 int i = 0; 4882 int nfrags, pos; 4883 4884 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4885 !gso_by_frags && mss != skb_headlen(head_skb)) { 4886 struct sk_buff *check_skb; 4887 4888 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4889 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4890 /* gso_size is untrusted, and we have a frag_list with 4891 * a linear non head_frag item. 4892 * 4893 * If head_skb's headlen does not fit requested gso_size, 4894 * it means that the frag_list members do NOT terminate 4895 * on exact gso_size boundaries. Hence we cannot perform 4896 * skb_frag_t page sharing. Therefore we must fallback to 4897 * copying the frag_list skbs; we do so by disabling SG. 4898 */ 4899 features &= ~NETIF_F_SG; 4900 break; 4901 } 4902 } 4903 } 4904 4905 __skb_push(head_skb, doffset); 4906 proto = skb_network_protocol(head_skb, NULL); 4907 if (unlikely(!proto)) 4908 return ERR_PTR(-EINVAL); 4909 4910 sg = !!(features & NETIF_F_SG); 4911 csum = !!can_checksum_protocol(features, proto); 4912 4913 if (sg && csum && !gso_by_frags) { 4914 if (!max_segs && !(features & NETIF_F_GSO_PARTIAL)) { 4915 struct sk_buff *iter; 4916 unsigned int frag_len; 4917 4918 if (!list_skb || 4919 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4920 goto normal; 4921 4922 /* If we get here then all the required 4923 * GSO features except frag_list are supported. 4924 * Try to split the SKB to multiple GSO SKBs 4925 * with no frag_list. 4926 * Currently we can do that only when the buffers don't 4927 * have a linear part and all the buffers except 4928 * the last are of the same length. 4929 */ 4930 frag_len = list_skb->len; 4931 skb_walk_frags(head_skb, iter) { 4932 if (frag_len != iter->len && iter->next) 4933 goto normal; 4934 if (skb_headlen(iter) && !iter->head_frag) 4935 goto normal; 4936 4937 len -= iter->len; 4938 } 4939 4940 if (len != frag_len) 4941 goto normal; 4942 } 4943 4944 /* GSO partial only requires that we trim off any excess that 4945 * doesn't fit into an MSS sized block, so take care of that 4946 * now. 4947 */ 4948 DEBUG_NET_WARN_ON_ONCE(!max_segs && len / mss > GSO_MAX_SEGS); 4949 if (max_segs) 4950 partial_segs = min(len / mss, max_segs); 4951 else 4952 partial_segs = min(len / mss, GSO_MAX_SEGS); 4953 if (partial_segs > 1) 4954 mss *= partial_segs; 4955 else 4956 partial_segs = 0; 4957 } 4958 4959 normal: 4960 headroom = skb_headroom(head_skb); 4961 pos = skb_headlen(head_skb); 4962 4963 if (skb_orphan_frags(head_skb, GFP_ATOMIC)) 4964 return ERR_PTR(-ENOMEM); 4965 4966 nfrags = skb_shinfo(head_skb)->nr_frags; 4967 frag = skb_shinfo(head_skb)->frags; 4968 frag_skb = head_skb; 4969 4970 do { 4971 struct sk_buff *nskb; 4972 skb_frag_t *nskb_frag; 4973 int hsize; 4974 int size; 4975 4976 if (unlikely(gso_by_frags)) { 4977 len = list_skb->len; 4978 } else { 4979 len = head_skb->len - offset; 4980 if (len > mss) 4981 len = mss; 4982 } 4983 4984 hsize = skb_headlen(head_skb) - offset; 4985 4986 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4987 (skb_headlen(list_skb) == len || sg)) { 4988 if (unlikely(skb_headlen(list_skb) > len)) { 4989 DEBUG_NET_WARN_ON_ONCE(1); 4990 err = -EINVAL; 4991 goto err; 4992 } 4993 4994 nskb = skb_clone(list_skb, GFP_ATOMIC); 4995 if (unlikely(!nskb)) 4996 goto err; 4997 4998 i = 0; 4999 nfrags = skb_shinfo(list_skb)->nr_frags; 5000 frag = skb_shinfo(list_skb)->frags; 5001 frag_skb = list_skb; 5002 pos += skb_headlen(list_skb); 5003 5004 while (pos < offset + len) { 5005 if (unlikely(i >= nfrags)) { 5006 DEBUG_NET_WARN_ON_ONCE(1); 5007 kfree_skb(nskb); 5008 err = -EINVAL; 5009 goto err; 5010 } 5011 5012 size = skb_frag_size(frag); 5013 if (pos + size > offset + len) 5014 break; 5015 5016 i++; 5017 pos += size; 5018 frag++; 5019 } 5020 5021 list_skb = list_skb->next; 5022 5023 if (unlikely(pskb_trim(nskb, len))) { 5024 kfree_skb(nskb); 5025 goto err; 5026 } 5027 5028 hsize = skb_end_offset(nskb); 5029 if (skb_cow_head(nskb, doffset + headroom)) { 5030 kfree_skb(nskb); 5031 goto err; 5032 } 5033 5034 nskb->truesize += skb_end_offset(nskb) - hsize; 5035 skb_release_head_state(nskb); 5036 __skb_push(nskb, doffset); 5037 } else { 5038 if (hsize < 0) 5039 hsize = 0; 5040 if (hsize > len || !sg) 5041 hsize = len; 5042 5043 nskb = __alloc_skb(hsize + doffset + headroom, 5044 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 5045 NUMA_NO_NODE); 5046 5047 if (unlikely(!nskb)) 5048 goto err; 5049 5050 skb_reserve(nskb, headroom); 5051 __skb_put(nskb, doffset); 5052 } 5053 5054 if (segs) 5055 tail->next = nskb; 5056 else 5057 segs = nskb; 5058 tail = nskb; 5059 5060 __copy_skb_header(nskb, head_skb); 5061 5062 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 5063 skb_reset_mac_len(nskb); 5064 5065 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 5066 nskb->data - tnl_hlen, 5067 doffset + tnl_hlen); 5068 5069 if (nskb->len == len + doffset) 5070 goto perform_csum_check; 5071 5072 if (!sg) { 5073 if (!csum) { 5074 if (!nskb->remcsum_offload) 5075 nskb->ip_summed = CHECKSUM_NONE; 5076 SKB_GSO_CB(nskb)->csum = 5077 skb_copy_and_csum_bits(head_skb, offset, 5078 skb_put(nskb, 5079 len), 5080 len); 5081 SKB_GSO_CB(nskb)->csum_start = 5082 skb_headroom(nskb) + doffset; 5083 } else { 5084 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 5085 goto err; 5086 } 5087 continue; 5088 } 5089 5090 nskb_frag = skb_shinfo(nskb)->frags; 5091 5092 skb_copy_from_linear_data_offset(head_skb, offset, 5093 skb_put(nskb, hsize), hsize); 5094 5095 skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags | 5096 skb_shinfo(frag_skb)->flags) & 5097 SKBFL_SHARED_FRAG; 5098 5099 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 5100 goto err; 5101 5102 while (pos < offset + len) { 5103 if (i >= nfrags) { 5104 if (skb_shinfo(nskb)->nr_frags && 5105 nskb->pp_recycle != list_skb->pp_recycle) { 5106 nskb->data_len = pos - offset - hsize; 5107 nskb->len += nskb->data_len; 5108 nskb->truesize += nskb->data_len; 5109 if (__skb_linearize(nskb)) 5110 goto err; 5111 hsize = pos - offset; 5112 nskb_frag = skb_shinfo(nskb)->frags; 5113 } 5114 if (skb_orphan_frags(list_skb, GFP_ATOMIC) || 5115 skb_zerocopy_clone(nskb, list_skb, 5116 GFP_ATOMIC)) 5117 goto err; 5118 5119 i = 0; 5120 nfrags = skb_shinfo(list_skb)->nr_frags; 5121 frag = skb_shinfo(list_skb)->frags; 5122 frag_skb = list_skb; 5123 5124 skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG; 5125 5126 if (!skb_headlen(list_skb)) { 5127 if (unlikely(!nfrags)) { 5128 DEBUG_NET_WARN_ON_ONCE(1); 5129 err = -EINVAL; 5130 goto err; 5131 } 5132 } else { 5133 if (unlikely(!list_skb->head_frag)) { 5134 DEBUG_NET_WARN_ON_ONCE(1); 5135 err = -EINVAL; 5136 goto err; 5137 } 5138 5139 /* to make room for head_frag. */ 5140 i--; 5141 frag--; 5142 } 5143 5144 list_skb = list_skb->next; 5145 } 5146 5147 if (unlikely(skb_shinfo(nskb)->nr_frags >= 5148 MAX_SKB_FRAGS)) { 5149 net_warn_ratelimited( 5150 "skb_segment: too many frags: %u %u\n", 5151 pos, mss); 5152 err = -EINVAL; 5153 goto err; 5154 } 5155 5156 if (!skb_shinfo(nskb)->nr_frags) 5157 nskb->pp_recycle = frag_skb->pp_recycle; 5158 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 5159 skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags); 5160 size = skb_frag_size(nskb_frag); 5161 5162 if (pos < offset) { 5163 skb_frag_off_add(nskb_frag, offset - pos); 5164 skb_frag_size_sub(nskb_frag, offset - pos); 5165 } 5166 5167 skb_shinfo(nskb)->nr_frags++; 5168 5169 if (pos + size <= offset + len) { 5170 i++; 5171 frag++; 5172 pos += size; 5173 } else { 5174 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 5175 goto skip_fraglist; 5176 } 5177 5178 nskb_frag++; 5179 } 5180 5181 skip_fraglist: 5182 nskb->data_len = len - hsize; 5183 nskb->len += nskb->data_len; 5184 nskb->truesize += nskb->data_len; 5185 5186 perform_csum_check: 5187 if (!csum) { 5188 if (skb_has_shared_frag(nskb) && 5189 __skb_linearize(nskb)) 5190 goto err; 5191 5192 if (!nskb->remcsum_offload) 5193 nskb->ip_summed = CHECKSUM_NONE; 5194 SKB_GSO_CB(nskb)->csum = 5195 skb_checksum(nskb, doffset, 5196 nskb->len - doffset, 0); 5197 SKB_GSO_CB(nskb)->csum_start = 5198 skb_headroom(nskb) + doffset; 5199 } 5200 } while ((offset += len) < head_skb->len); 5201 5202 /* Some callers want to get the end of the list. 5203 * Put it in segs->prev to avoid walking the list. 5204 * (see validate_xmit_skb_list() for example) 5205 */ 5206 segs->prev = tail; 5207 5208 if (partial_segs) { 5209 struct sk_buff *iter; 5210 int type = skb_shinfo(head_skb)->gso_type; 5211 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 5212 5213 /* Update type to add partial and then remove dodgy if set */ 5214 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 5215 type &= ~SKB_GSO_DODGY; 5216 5217 /* Update GSO info and prepare to start updating headers on 5218 * our way back down the stack of protocols. 5219 */ 5220 for (iter = segs; iter; iter = iter->next) { 5221 skb_shinfo(iter)->gso_size = gso_size; 5222 skb_shinfo(iter)->gso_segs = partial_segs; 5223 skb_shinfo(iter)->gso_type = type; 5224 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 5225 } 5226 5227 if (tail->len - doffset <= gso_size) 5228 skb_gso_reset(tail); 5229 else if (tail != segs) 5230 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 5231 } 5232 5233 /* Following permits correct backpressure, for protocols 5234 * using skb_set_owner_w(). 5235 * Idea is to tranfert ownership from head_skb to last segment. 5236 */ 5237 if (head_skb->destructor == sock_wfree) { 5238 swap(tail->truesize, head_skb->truesize); 5239 swap(tail->destructor, head_skb->destructor); 5240 swap(tail->sk, head_skb->sk); 5241 } 5242 return segs; 5243 5244 err: 5245 kfree_skb_list(segs); 5246 return ERR_PTR(err); 5247 } 5248 EXPORT_SYMBOL_GPL(skb_segment); 5249 5250 #ifdef CONFIG_SKB_EXTENSIONS 5251 #define SKB_EXT_ALIGN_VALUE 8 5252 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 5253 5254 static const u8 skb_ext_type_len[] = { 5255 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 5256 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 5257 #endif 5258 #ifdef CONFIG_XFRM 5259 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 5260 #endif 5261 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 5262 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 5263 #endif 5264 #if IS_ENABLED(CONFIG_MPTCP) 5265 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 5266 #endif 5267 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 5268 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 5269 #endif 5270 #if IS_ENABLED(CONFIG_INET_PSP) 5271 [SKB_EXT_PSP] = SKB_EXT_CHUNKSIZEOF(struct psp_skb_ext), 5272 #endif 5273 #if IS_ENABLED(CONFIG_CAN) 5274 [SKB_EXT_CAN] = SKB_EXT_CHUNKSIZEOF(struct can_skb_ext), 5275 #endif 5276 }; 5277 5278 static __always_inline __no_profile unsigned int skb_ext_total_length(void) 5279 { 5280 unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext); 5281 int i; 5282 5283 for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++) 5284 l += skb_ext_type_len[i]; 5285 5286 return l; 5287 } 5288 5289 static noinline void __init __no_profile skb_extensions_init(void) 5290 { 5291 BUILD_BUG_ON(SKB_EXT_NUM > 8); 5292 BUILD_BUG_ON(skb_ext_total_length() > 255); 5293 5294 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 5295 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 5296 0, 5297 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 5298 NULL); 5299 } 5300 #else 5301 static void skb_extensions_init(void) {} 5302 #endif 5303 5304 /* The SKB kmem_cache slab is critical for network performance. Never 5305 * merge/alias the slab with similar sized objects. This avoids fragmentation 5306 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs. 5307 */ 5308 #ifndef CONFIG_SLUB_TINY 5309 #define FLAG_SKB_NO_MERGE SLAB_NO_MERGE 5310 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */ 5311 #define FLAG_SKB_NO_MERGE 0 5312 #endif 5313 5314 void __init skb_init(void) 5315 { 5316 net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", 5317 sizeof(struct sk_buff), 5318 0, 5319 SLAB_HWCACHE_ALIGN|SLAB_PANIC| 5320 FLAG_SKB_NO_MERGE, 5321 offsetof(struct sk_buff, cb), 5322 sizeof_field(struct sk_buff, cb), 5323 NULL); 5324 skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache); 5325 5326 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 5327 sizeof(struct sk_buff_fclones), 5328 0, 5329 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 5330 NULL); 5331 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes. 5332 * struct skb_shared_info is located at the end of skb->head, 5333 * and should not be copied to/from user. 5334 */ 5335 net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", 5336 SKB_SMALL_HEAD_CACHE_SIZE, 5337 0, 5338 SLAB_HWCACHE_ALIGN | SLAB_PANIC, 5339 0, 5340 SKB_SMALL_HEAD_HEADROOM, 5341 NULL); 5342 skb_extensions_init(); 5343 } 5344 5345 static int 5346 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 5347 unsigned int recursion_level) 5348 { 5349 int start = skb_headlen(skb); 5350 int i, copy = start - offset; 5351 struct sk_buff *frag_iter; 5352 int elt = 0; 5353 5354 if (unlikely(recursion_level >= 24)) 5355 return -EMSGSIZE; 5356 5357 if (copy > 0) { 5358 if (copy > len) 5359 copy = len; 5360 sg_set_buf(sg, skb->data + offset, copy); 5361 elt++; 5362 if ((len -= copy) == 0) 5363 return elt; 5364 offset += copy; 5365 } 5366 5367 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5368 int end; 5369 5370 WARN_ON(start > offset + len); 5371 5372 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 5373 if ((copy = end - offset) > 0) { 5374 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5375 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5376 return -EMSGSIZE; 5377 5378 if (copy > len) 5379 copy = len; 5380 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 5381 skb_frag_off(frag) + offset - start); 5382 elt++; 5383 if (!(len -= copy)) 5384 return elt; 5385 offset += copy; 5386 } 5387 start = end; 5388 } 5389 5390 skb_walk_frags(skb, frag_iter) { 5391 int end, ret; 5392 5393 WARN_ON(start > offset + len); 5394 5395 end = start + frag_iter->len; 5396 if ((copy = end - offset) > 0) { 5397 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5398 return -EMSGSIZE; 5399 5400 if (copy > len) 5401 copy = len; 5402 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 5403 copy, recursion_level + 1); 5404 if (unlikely(ret < 0)) 5405 return ret; 5406 elt += ret; 5407 if ((len -= copy) == 0) 5408 return elt; 5409 offset += copy; 5410 } 5411 start = end; 5412 } 5413 if (unlikely(len)) { 5414 DEBUG_NET_WARN_ON_ONCE(1); 5415 return -EINVAL; 5416 } 5417 return elt; 5418 } 5419 5420 /** 5421 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 5422 * @skb: Socket buffer containing the buffers to be mapped 5423 * @sg: The scatter-gather list to map into 5424 * @offset: The offset into the buffer's contents to start mapping 5425 * @len: Length of buffer space to be mapped 5426 * 5427 * Fill the specified scatter-gather list with mappings/pointers into a 5428 * region of the buffer space attached to a socket buffer. Returns either 5429 * the number of scatterlist items used, or -EMSGSIZE if the contents 5430 * could not fit. 5431 */ 5432 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 5433 { 5434 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 5435 5436 if (nsg <= 0) 5437 return nsg; 5438 5439 sg_mark_end(&sg[nsg - 1]); 5440 5441 return nsg; 5442 } 5443 EXPORT_SYMBOL_GPL(skb_to_sgvec); 5444 5445 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 5446 * sglist without mark the sg which contain last skb data as the end. 5447 * So the caller can mannipulate sg list as will when padding new data after 5448 * the first call without calling sg_unmark_end to expend sg list. 5449 * 5450 * Scenario to use skb_to_sgvec_nomark: 5451 * 1. sg_init_table 5452 * 2. skb_to_sgvec_nomark(payload1) 5453 * 3. skb_to_sgvec_nomark(payload2) 5454 * 5455 * This is equivalent to: 5456 * 1. sg_init_table 5457 * 2. skb_to_sgvec(payload1) 5458 * 3. sg_unmark_end 5459 * 4. skb_to_sgvec(payload2) 5460 * 5461 * When mapping multiple payload conditionally, skb_to_sgvec_nomark 5462 * is more preferable. 5463 */ 5464 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 5465 int offset, int len) 5466 { 5467 return __skb_to_sgvec(skb, sg, offset, len, 0); 5468 } 5469 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 5470 5471 5472 5473 /** 5474 * skb_cow_data - Check that a socket buffer's data buffers are writable 5475 * @skb: The socket buffer to check. 5476 * @tailbits: Amount of trailing space to be added 5477 * @trailer: Returned pointer to the skb where the @tailbits space begins 5478 * 5479 * Make sure that the data buffers attached to a socket buffer are 5480 * writable. If they are not, private copies are made of the data buffers 5481 * and the socket buffer is set to use these instead. 5482 * 5483 * If @tailbits is given, make sure that there is space to write @tailbits 5484 * bytes of data beyond current end of socket buffer. @trailer will be 5485 * set to point to the skb in which this space begins. 5486 * 5487 * The number of scatterlist elements required to completely map the 5488 * COW'd and extended socket buffer will be returned. 5489 */ 5490 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 5491 { 5492 int copyflag; 5493 int elt; 5494 struct sk_buff *skb1, **skb_p; 5495 5496 /* If skb is cloned or its head is paged, reallocate 5497 * head pulling out all the pages (pages are considered not writable 5498 * at the moment even if they are anonymous). 5499 */ 5500 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 5501 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 5502 return -ENOMEM; 5503 5504 /* Easy case. Most of packets will go this way. */ 5505 if (!skb_has_frag_list(skb)) { 5506 /* A little of trouble, not enough of space for trailer. 5507 * This should not happen, when stack is tuned to generate 5508 * good frames. OK, on miss we reallocate and reserve even more 5509 * space, 128 bytes is fair. */ 5510 5511 if (skb_tailroom(skb) < tailbits && 5512 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 5513 return -ENOMEM; 5514 5515 /* Voila! */ 5516 *trailer = skb; 5517 return 1; 5518 } 5519 5520 /* Misery. We are in troubles, going to mincer fragments... */ 5521 5522 elt = 1; 5523 skb_p = &skb_shinfo(skb)->frag_list; 5524 copyflag = 0; 5525 5526 while ((skb1 = *skb_p) != NULL) { 5527 int ntail = 0; 5528 5529 /* The fragment is partially pulled by someone, 5530 * this can happen on input. Copy it and everything 5531 * after it. */ 5532 5533 if (skb_shared(skb1)) 5534 copyflag = 1; 5535 5536 /* If the skb is the last, worry about trailer. */ 5537 5538 if (skb1->next == NULL && tailbits) { 5539 if (skb_shinfo(skb1)->nr_frags || 5540 skb_has_frag_list(skb1) || 5541 skb_tailroom(skb1) < tailbits) 5542 ntail = tailbits + 128; 5543 } 5544 5545 if (copyflag || 5546 skb_cloned(skb1) || 5547 ntail || 5548 skb_shinfo(skb1)->nr_frags || 5549 skb_has_frag_list(skb1)) { 5550 struct sk_buff *skb2; 5551 5552 /* Fuck, we are miserable poor guys... */ 5553 if (ntail == 0) 5554 skb2 = skb_copy(skb1, GFP_ATOMIC); 5555 else 5556 skb2 = skb_copy_expand(skb1, 5557 skb_headroom(skb1), 5558 ntail, 5559 GFP_ATOMIC); 5560 if (unlikely(skb2 == NULL)) 5561 return -ENOMEM; 5562 5563 if (skb1->sk) 5564 skb_set_owner_w(skb2, skb1->sk); 5565 5566 /* Looking around. Are we still alive? 5567 * OK, link new skb, drop old one */ 5568 5569 skb2->next = skb1->next; 5570 *skb_p = skb2; 5571 kfree_skb(skb1); 5572 skb1 = skb2; 5573 } 5574 elt++; 5575 *trailer = skb1; 5576 skb_p = &skb1->next; 5577 } 5578 5579 return elt; 5580 } 5581 EXPORT_SYMBOL_GPL(skb_cow_data); 5582 5583 void sock_rmem_free(struct sk_buff *skb) 5584 { 5585 struct sock *sk = skb->sk; 5586 5587 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 5588 } 5589 5590 static void skb_set_err_queue(struct sk_buff *skb) 5591 { 5592 /* The error-queue test in skb_is_err_queue() matches this marker 5593 * with the sock_rmem_free destructor installed by sock_queue_err_skb(). 5594 */ 5595 skb->pkt_type = PACKET_OUTGOING; 5596 BUILD_BUG_ON(PACKET_OUTGOING == 0); 5597 } 5598 5599 /* 5600 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 5601 */ 5602 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 5603 { 5604 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 5605 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 5606 return -ENOMEM; 5607 5608 skb_orphan(skb); 5609 skb->sk = sk; 5610 skb->destructor = sock_rmem_free; 5611 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 5612 skb_set_err_queue(skb); 5613 5614 /* before exiting rcu section, make sure dst is refcounted */ 5615 skb_dst_force(skb); 5616 5617 skb_queue_tail(&sk->sk_error_queue, skb); 5618 if (!sock_flag(sk, SOCK_DEAD)) 5619 sk_error_report(sk); 5620 return 0; 5621 } 5622 EXPORT_SYMBOL(sock_queue_err_skb); 5623 5624 static bool is_icmp_err_skb(const struct sk_buff *skb) 5625 { 5626 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 5627 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 5628 } 5629 5630 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 5631 { 5632 struct sk_buff_head *q = &sk->sk_error_queue; 5633 struct sk_buff *skb, *skb_next = NULL; 5634 bool icmp_next = false; 5635 unsigned long flags; 5636 5637 if (skb_queue_empty_lockless(q)) 5638 return NULL; 5639 5640 spin_lock_irqsave(&q->lock, flags); 5641 skb = __skb_dequeue(q); 5642 if (skb && (skb_next = skb_peek(q))) { 5643 icmp_next = is_icmp_err_skb(skb_next); 5644 if (icmp_next) 5645 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 5646 } 5647 spin_unlock_irqrestore(&q->lock, flags); 5648 5649 if (is_icmp_err_skb(skb) && !icmp_next) 5650 sk->sk_err = 0; 5651 5652 if (skb_next) 5653 sk_error_report(sk); 5654 5655 return skb; 5656 } 5657 EXPORT_SYMBOL(sock_dequeue_err_skb); 5658 5659 /** 5660 * skb_clone_sk - create clone of skb, and take reference to socket 5661 * @skb: the skb to clone 5662 * 5663 * This function creates a clone of a buffer that holds a reference on 5664 * sk_refcnt. Buffers created via this function are meant to be 5665 * returned using sock_queue_err_skb, or free via kfree_skb. 5666 * 5667 * When passing buffers allocated with this function to sock_queue_err_skb 5668 * it is necessary to wrap the call with sock_hold/sock_put in order to 5669 * prevent the socket from being released prior to being enqueued on 5670 * the sk_error_queue. 5671 */ 5672 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 5673 { 5674 struct sock *sk = skb->sk; 5675 struct sk_buff *clone; 5676 5677 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 5678 return NULL; 5679 5680 clone = skb_clone(skb, GFP_ATOMIC); 5681 if (!clone) { 5682 sock_put(sk); 5683 return NULL; 5684 } 5685 5686 clone->sk = sk; 5687 clone->destructor = sock_efree; 5688 5689 return clone; 5690 } 5691 EXPORT_SYMBOL(skb_clone_sk); 5692 5693 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 5694 struct sock *sk, 5695 int tstype, 5696 bool opt_stats) 5697 { 5698 struct sock_exterr_skb *serr; 5699 int err; 5700 5701 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 5702 5703 serr = SKB_EXT_ERR(skb); 5704 memset(serr, 0, sizeof(*serr)); 5705 serr->ee.ee_errno = ENOMSG; 5706 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 5707 serr->ee.ee_info = tstype; 5708 serr->opt_stats = opt_stats; 5709 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 5710 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) { 5711 serr->ee.ee_data = skb_shinfo(skb)->tskey; 5712 if (sk_is_tcp(sk)) 5713 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 5714 } 5715 5716 err = sock_queue_err_skb(sk, skb); 5717 5718 if (err) 5719 kfree_skb(skb); 5720 } 5721 5722 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 5723 { 5724 struct socket *sock; 5725 struct file *file; 5726 bool ret = false; 5727 5728 if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data))) 5729 return true; 5730 5731 /* The sk pointer remains valid as long as the skb is. The sk_socket and 5732 * file pointer may become NULL if the socket is closed. Both structures 5733 * (including file->cred) are RCU freed which means they can be accessed 5734 * within a RCU read section. 5735 */ 5736 rcu_read_lock(); 5737 sock = READ_ONCE(sk->sk_socket); 5738 if (!sock) 5739 goto out; 5740 file = READ_ONCE(sock->file); 5741 if (!file) 5742 goto out; 5743 ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW); 5744 out: 5745 rcu_read_unlock(); 5746 return ret; 5747 } 5748 5749 void skb_complete_tx_timestamp(struct sk_buff *skb, 5750 struct skb_shared_hwtstamps *hwtstamps) 5751 { 5752 struct sock *sk = skb->sk; 5753 5754 if (!skb_may_tx_timestamp(sk, false)) 5755 goto err; 5756 5757 /* Take a reference to prevent skb_orphan() from freeing the socket, 5758 * but only if the socket refcount is not zero. 5759 */ 5760 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5761 *skb_hwtstamps(skb) = *hwtstamps; 5762 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5763 sock_put(sk); 5764 return; 5765 } 5766 5767 err: 5768 kfree_skb(skb); 5769 } 5770 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5771 5772 static bool skb_tstamp_tx_report_so_timestamping(struct sk_buff *skb, 5773 struct skb_shared_hwtstamps *hwtstamps, 5774 int tstype) 5775 { 5776 switch (tstype) { 5777 case SCM_TSTAMP_SCHED: 5778 return skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP; 5779 case SCM_TSTAMP_SND: 5780 return skb_shinfo(skb)->tx_flags & (hwtstamps ? SKBTX_HW_TSTAMP_NOBPF : 5781 SKBTX_SW_TSTAMP); 5782 case SCM_TSTAMP_ACK: 5783 return TCP_SKB_CB(skb)->txstamp_ack & TSTAMP_ACK_SK; 5784 case SCM_TSTAMP_COMPLETION: 5785 return skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP; 5786 } 5787 5788 return false; 5789 } 5790 5791 static void skb_tstamp_tx_report_bpf_timestamping(struct sk_buff *skb, 5792 struct skb_shared_hwtstamps *hwtstamps, 5793 struct sock *sk, 5794 int tstype) 5795 { 5796 int op; 5797 5798 switch (tstype) { 5799 case SCM_TSTAMP_SCHED: 5800 op = BPF_SOCK_OPS_TSTAMP_SCHED_CB; 5801 break; 5802 case SCM_TSTAMP_SND: 5803 if (hwtstamps) { 5804 op = BPF_SOCK_OPS_TSTAMP_SND_HW_CB; 5805 *skb_hwtstamps(skb) = *hwtstamps; 5806 } else { 5807 op = BPF_SOCK_OPS_TSTAMP_SND_SW_CB; 5808 } 5809 break; 5810 case SCM_TSTAMP_ACK: 5811 op = BPF_SOCK_OPS_TSTAMP_ACK_CB; 5812 break; 5813 default: 5814 return; 5815 } 5816 5817 bpf_skops_tx_timestamping(sk, skb, op); 5818 } 5819 5820 void __skb_tstamp_tx(struct sk_buff *orig_skb, 5821 const struct sk_buff *ack_skb, 5822 struct skb_shared_hwtstamps *hwtstamps, 5823 struct sock *sk, int tstype) 5824 { 5825 struct sk_buff *skb; 5826 bool tsonly, opt_stats = false; 5827 u32 tsflags; 5828 5829 if (!sk) 5830 return; 5831 5832 if (skb_shinfo(orig_skb)->tx_flags & SKBTX_BPF) 5833 skb_tstamp_tx_report_bpf_timestamping(orig_skb, hwtstamps, 5834 sk, tstype); 5835 5836 if (!skb_tstamp_tx_report_so_timestamping(orig_skb, hwtstamps, tstype)) 5837 return; 5838 5839 tsflags = READ_ONCE(sk->sk_tsflags); 5840 if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5841 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5842 return; 5843 5844 tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5845 if (!skb_may_tx_timestamp(sk, tsonly)) 5846 return; 5847 5848 if (tsonly) { 5849 #ifdef CONFIG_INET 5850 if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5851 sk_is_tcp(sk)) { 5852 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5853 ack_skb); 5854 opt_stats = true; 5855 } else 5856 #endif 5857 skb = alloc_skb(0, GFP_ATOMIC); 5858 } else { 5859 skb = skb_clone(orig_skb, GFP_ATOMIC); 5860 5861 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) { 5862 kfree_skb(skb); 5863 return; 5864 } 5865 } 5866 if (!skb) 5867 return; 5868 5869 if (tsonly) { 5870 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5871 SKBTX_ANY_TSTAMP; 5872 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5873 } 5874 5875 if (hwtstamps) 5876 *skb_hwtstamps(skb) = *hwtstamps; 5877 else 5878 __net_timestamp(skb); 5879 5880 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5881 } 5882 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5883 5884 void skb_tstamp_tx(struct sk_buff *orig_skb, 5885 struct skb_shared_hwtstamps *hwtstamps) 5886 { 5887 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5888 SCM_TSTAMP_SND); 5889 } 5890 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5891 5892 #ifdef CONFIG_WIRELESS 5893 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5894 { 5895 struct sock *sk = skb->sk; 5896 struct sock_exterr_skb *serr; 5897 int err = 1; 5898 5899 skb->wifi_acked_valid = 1; 5900 skb->wifi_acked = acked; 5901 5902 serr = SKB_EXT_ERR(skb); 5903 memset(serr, 0, sizeof(*serr)); 5904 serr->ee.ee_errno = ENOMSG; 5905 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5906 5907 /* Take a reference to prevent skb_orphan() from freeing the socket, 5908 * but only if the socket refcount is not zero. 5909 */ 5910 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5911 err = sock_queue_err_skb(sk, skb); 5912 sock_put(sk); 5913 } 5914 if (err) 5915 kfree_skb(skb); 5916 } 5917 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5918 #endif /* CONFIG_WIRELESS */ 5919 5920 /** 5921 * skb_partial_csum_set - set up and verify partial csum values for packet 5922 * @skb: the skb to set 5923 * @start: the number of bytes after skb->data to start checksumming. 5924 * @off: the offset from start to place the checksum. 5925 * 5926 * For untrusted partially-checksummed packets, we need to make sure the values 5927 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5928 * 5929 * This function checks and sets those values and skb->ip_summed: if this 5930 * returns false you should drop the packet. 5931 */ 5932 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5933 { 5934 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5935 u32 csum_start = skb_headroom(skb) + (u32)start; 5936 5937 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) { 5938 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5939 start, off, skb_headroom(skb), skb_headlen(skb)); 5940 return false; 5941 } 5942 skb->ip_summed = CHECKSUM_PARTIAL; 5943 skb->csum_start = csum_start; 5944 skb->csum_offset = off; 5945 skb->transport_header = csum_start; 5946 return true; 5947 } 5948 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5949 5950 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5951 unsigned int max) 5952 { 5953 if (skb_headlen(skb) >= len) 5954 return 0; 5955 5956 /* If we need to pullup then pullup to the max, so we 5957 * won't need to do it again. 5958 */ 5959 if (max > skb->len) 5960 max = skb->len; 5961 5962 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5963 return -ENOMEM; 5964 5965 if (skb_headlen(skb) < len) 5966 return -EPROTO; 5967 5968 return 0; 5969 } 5970 5971 #define MAX_TCP_HDR_LEN (15 * 4) 5972 5973 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5974 typeof(IPPROTO_IP) proto, 5975 unsigned int off) 5976 { 5977 int err; 5978 5979 switch (proto) { 5980 case IPPROTO_TCP: 5981 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5982 off + MAX_TCP_HDR_LEN); 5983 if (!err && !skb_partial_csum_set(skb, off, 5984 offsetof(struct tcphdr, 5985 check))) 5986 err = -EPROTO; 5987 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5988 5989 case IPPROTO_UDP: 5990 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5991 off + sizeof(struct udphdr)); 5992 if (!err && !skb_partial_csum_set(skb, off, 5993 offsetof(struct udphdr, 5994 check))) 5995 err = -EPROTO; 5996 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5997 } 5998 5999 return ERR_PTR(-EPROTO); 6000 } 6001 6002 /* This value should be large enough to cover a tagged ethernet header plus 6003 * maximally sized IP and TCP or UDP headers. 6004 */ 6005 #define MAX_IP_HDR_LEN 128 6006 6007 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 6008 { 6009 unsigned int off; 6010 bool fragment; 6011 __sum16 *csum; 6012 int err; 6013 6014 fragment = false; 6015 6016 err = skb_maybe_pull_tail(skb, 6017 sizeof(struct iphdr), 6018 MAX_IP_HDR_LEN); 6019 if (err < 0) 6020 goto out; 6021 6022 if (ip_is_fragment(ip_hdr(skb))) 6023 fragment = true; 6024 6025 off = ip_hdrlen(skb); 6026 6027 err = -EPROTO; 6028 6029 if (fragment) 6030 goto out; 6031 6032 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 6033 if (IS_ERR(csum)) 6034 return PTR_ERR(csum); 6035 6036 if (recalculate) 6037 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 6038 ip_hdr(skb)->daddr, 6039 skb->len - off, 6040 ip_hdr(skb)->protocol, 0); 6041 err = 0; 6042 6043 out: 6044 return err; 6045 } 6046 6047 /* This value should be large enough to cover a tagged ethernet header plus 6048 * an IPv6 header, all options, and a maximal TCP or UDP header. 6049 */ 6050 #define MAX_IPV6_HDR_LEN 256 6051 6052 #define OPT_HDR(type, skb, off) \ 6053 (type *)(skb_network_header(skb) + (off)) 6054 6055 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 6056 { 6057 int err; 6058 u8 nexthdr; 6059 unsigned int off; 6060 unsigned int len; 6061 bool fragment; 6062 bool done; 6063 __sum16 *csum; 6064 6065 fragment = false; 6066 done = false; 6067 6068 off = sizeof(struct ipv6hdr); 6069 6070 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 6071 if (err < 0) 6072 goto out; 6073 6074 nexthdr = ipv6_hdr(skb)->nexthdr; 6075 6076 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 6077 while (off <= len && !done) { 6078 switch (nexthdr) { 6079 case IPPROTO_DSTOPTS: 6080 case IPPROTO_HOPOPTS: 6081 case IPPROTO_ROUTING: { 6082 struct ipv6_opt_hdr *hp; 6083 6084 err = skb_maybe_pull_tail(skb, 6085 off + 6086 sizeof(struct ipv6_opt_hdr), 6087 off + 6088 sizeof(struct ipv6_opt_hdr)); 6089 if (err < 0) 6090 goto out; 6091 6092 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 6093 nexthdr = hp->nexthdr; 6094 off += ipv6_optlen(hp); 6095 break; 6096 } 6097 case IPPROTO_AH: { 6098 struct ip_auth_hdr *hp; 6099 6100 err = skb_maybe_pull_tail(skb, 6101 off + 6102 sizeof(struct ip_auth_hdr), 6103 off + 6104 sizeof(struct ip_auth_hdr)); 6105 if (err < 0) 6106 goto out; 6107 6108 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 6109 nexthdr = hp->nexthdr; 6110 off += ipv6_authlen(hp); 6111 break; 6112 } 6113 case IPPROTO_FRAGMENT: { 6114 struct frag_hdr *hp; 6115 6116 err = skb_maybe_pull_tail(skb, 6117 off + 6118 sizeof(struct frag_hdr), 6119 off + 6120 sizeof(struct frag_hdr)); 6121 if (err < 0) 6122 goto out; 6123 6124 hp = OPT_HDR(struct frag_hdr, skb, off); 6125 6126 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 6127 fragment = true; 6128 6129 nexthdr = hp->nexthdr; 6130 off += sizeof(struct frag_hdr); 6131 break; 6132 } 6133 default: 6134 done = true; 6135 break; 6136 } 6137 } 6138 6139 err = -EPROTO; 6140 6141 if (!done || fragment) 6142 goto out; 6143 6144 csum = skb_checksum_setup_ip(skb, nexthdr, off); 6145 if (IS_ERR(csum)) 6146 return PTR_ERR(csum); 6147 6148 if (recalculate) 6149 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 6150 &ipv6_hdr(skb)->daddr, 6151 skb->len - off, nexthdr, 0); 6152 err = 0; 6153 6154 out: 6155 return err; 6156 } 6157 6158 /** 6159 * skb_checksum_setup - set up partial checksum offset 6160 * @skb: the skb to set up 6161 * @recalculate: if true the pseudo-header checksum will be recalculated 6162 */ 6163 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 6164 { 6165 int err; 6166 6167 switch (skb->protocol) { 6168 case htons(ETH_P_IP): 6169 err = skb_checksum_setup_ipv4(skb, recalculate); 6170 break; 6171 6172 case htons(ETH_P_IPV6): 6173 err = skb_checksum_setup_ipv6(skb, recalculate); 6174 break; 6175 6176 default: 6177 err = -EPROTO; 6178 break; 6179 } 6180 6181 return err; 6182 } 6183 EXPORT_SYMBOL(skb_checksum_setup); 6184 6185 /** 6186 * skb_checksum_maybe_trim - maybe trims the given skb 6187 * @skb: the skb to check 6188 * @transport_len: the data length beyond the network header 6189 * 6190 * Checks whether the given skb has data beyond the given transport length. 6191 * If so, returns a cloned skb trimmed to this transport length. 6192 * Otherwise returns the provided skb. Returns NULL in error cases 6193 * (e.g. transport_len exceeds skb length or out-of-memory). 6194 * 6195 * Caller needs to set the skb transport header and free any returned skb if it 6196 * differs from the provided skb. 6197 */ 6198 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 6199 unsigned int transport_len) 6200 { 6201 struct sk_buff *skb_chk; 6202 unsigned int len = skb_transport_offset(skb) + transport_len; 6203 int ret; 6204 6205 if (skb->len < len) 6206 return NULL; 6207 else if (skb->len == len) 6208 return skb; 6209 6210 skb_chk = skb_clone(skb, GFP_ATOMIC); 6211 if (!skb_chk) 6212 return NULL; 6213 6214 ret = pskb_trim_rcsum(skb_chk, len); 6215 if (ret) { 6216 kfree_skb(skb_chk); 6217 return NULL; 6218 } 6219 6220 return skb_chk; 6221 } 6222 6223 /** 6224 * skb_checksum_trimmed - validate checksum of an skb 6225 * @skb: the skb to check 6226 * @transport_len: the data length beyond the network header 6227 * @skb_chkf: checksum function to use 6228 * 6229 * Applies the given checksum function skb_chkf to the provided skb. 6230 * Returns a checked and maybe trimmed skb. Returns NULL on error. 6231 * 6232 * If the skb has data beyond the given transport length, then a 6233 * trimmed & cloned skb is checked and returned. 6234 * 6235 * Caller needs to set the skb transport header and free any returned skb if it 6236 * differs from the provided skb. 6237 */ 6238 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 6239 unsigned int transport_len, 6240 __sum16(*skb_chkf)(struct sk_buff *skb)) 6241 { 6242 struct sk_buff *skb_chk; 6243 unsigned int offset = skb_transport_offset(skb); 6244 __sum16 ret; 6245 6246 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 6247 if (!skb_chk) 6248 goto err; 6249 6250 if (!pskb_may_pull(skb_chk, offset)) 6251 goto err; 6252 6253 skb_pull_rcsum(skb_chk, offset); 6254 ret = skb_chkf(skb_chk); 6255 skb_push_rcsum(skb_chk, offset); 6256 6257 if (ret) 6258 goto err; 6259 6260 return skb_chk; 6261 6262 err: 6263 if (skb_chk && skb_chk != skb) 6264 kfree_skb(skb_chk); 6265 6266 return NULL; 6267 6268 } 6269 EXPORT_SYMBOL(skb_checksum_trimmed); 6270 6271 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 6272 { 6273 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 6274 skb->dev->name); 6275 } 6276 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 6277 6278 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 6279 { 6280 if (head_stolen) { 6281 skb_release_head_state(skb); 6282 kfree_skbmem(skb); 6283 } else { 6284 __kfree_skb(skb); 6285 } 6286 } 6287 EXPORT_SYMBOL(kfree_skb_partial); 6288 6289 /** 6290 * skb_try_coalesce - try to merge skb to prior one 6291 * @to: prior buffer 6292 * @from: buffer to add 6293 * @fragstolen: pointer to boolean 6294 * @delta_truesize: how much more was allocated than was requested 6295 */ 6296 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 6297 bool *fragstolen, int *delta_truesize) 6298 { 6299 struct skb_shared_info *to_shinfo, *from_shinfo; 6300 int i, delta, len = from->len; 6301 6302 *fragstolen = false; 6303 6304 if (skb_cloned(to)) 6305 return false; 6306 6307 /* In general, avoid mixing page_pool and non-page_pool allocated 6308 * pages within the same SKB. In theory we could take full 6309 * references if @from is cloned and !@to->pp_recycle but its 6310 * tricky (due to potential race with the clone disappearing) and 6311 * rare, so not worth dealing with. 6312 */ 6313 if (to->pp_recycle != from->pp_recycle) 6314 return false; 6315 6316 /* All frags in an skb must either all be struct page-backed or all 6317 * belong to the same memory provider instance; do not coalesce skbs 6318 * that would mix them. 6319 */ 6320 if (skb_frags_readable(from) != skb_frags_readable(to)) 6321 return false; 6322 6323 if (len <= skb_tailroom(to) && skb_frags_readable(from)) { 6324 if (len) { 6325 if (unlikely(skb_copy_bits(from, 0, 6326 skb_tail_pointer(to), 6327 len))) { 6328 DEBUG_NET_WARN_ON_ONCE(1); 6329 return false; 6330 } 6331 skb_put(to, len); 6332 } 6333 *delta_truesize = 0; 6334 return true; 6335 } 6336 6337 to_shinfo = skb_shinfo(to); 6338 from_shinfo = skb_shinfo(from); 6339 if (to_shinfo->frag_list || from_shinfo->frag_list) 6340 return false; 6341 if (skb_zcopy(to) || skb_zcopy(from)) 6342 return false; 6343 6344 if (skb_headlen(from) != 0) { 6345 struct page *page; 6346 unsigned int offset; 6347 6348 if (to_shinfo->nr_frags + 6349 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 6350 return false; 6351 6352 if (skb_head_is_locked(from)) 6353 return false; 6354 6355 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 6356 6357 page = virt_to_head_page(from->head); 6358 offset = from->data - (unsigned char *)page_address(page); 6359 6360 skb_fill_page_desc(to, to_shinfo->nr_frags, 6361 page, offset, skb_headlen(from)); 6362 *fragstolen = true; 6363 } else { 6364 if (to_shinfo->nr_frags + 6365 from_shinfo->nr_frags > MAX_SKB_FRAGS) 6366 return false; 6367 6368 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 6369 } 6370 6371 WARN_ON_ONCE(delta < len); 6372 6373 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 6374 from_shinfo->frags, 6375 from_shinfo->nr_frags * sizeof(skb_frag_t)); 6376 to_shinfo->nr_frags += from_shinfo->nr_frags; 6377 if (from_shinfo->nr_frags) 6378 to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG; 6379 6380 if (!skb_cloned(from)) 6381 from_shinfo->nr_frags = 0; 6382 6383 /* if the skb is not cloned this does nothing 6384 * since we set nr_frags to 0. 6385 */ 6386 for (i = 0; i < from_shinfo->nr_frags; i++) 6387 skb_frag_ref(from, i); 6388 6389 to->truesize += delta; 6390 to->len += len; 6391 to->data_len += len; 6392 6393 *delta_truesize = delta; 6394 return true; 6395 } 6396 EXPORT_SYMBOL(skb_try_coalesce); 6397 6398 static void skb_ext_scrub(struct sk_buff *skb) 6399 { 6400 #if IS_ENABLED(CONFIG_INET_PSP) 6401 struct psp_skb_ext *pse = skb_ext_find(skb, SKB_EXT_PSP); 6402 6403 if (pse) { 6404 struct skb_ext *ext = skb->extensions; 6405 struct psp_skb_ext tmp; 6406 6407 /* Nothing else to scrub if PSP is the only extension in ext. */ 6408 if (ext->chunks == SKB_EXT_CHUNKSIZEOF(*ext) + 6409 skb_ext_type_len[SKB_EXT_PSP]) 6410 return; 6411 6412 tmp = *pse; 6413 skb_ext_reset(skb); 6414 pse = skb_ext_add(skb, SKB_EXT_PSP); 6415 if (pse) 6416 memcpy(pse, &tmp, sizeof(*pse)); 6417 return; 6418 } 6419 #endif 6420 skb_ext_reset(skb); 6421 } 6422 6423 /** 6424 * skb_scrub_packet - scrub an skb 6425 * 6426 * @skb: buffer to clean 6427 * @xnet: packet is crossing netns 6428 * 6429 * skb_scrub_packet can be used after encapsulating or decapsulating a packet 6430 * into/from a tunnel. Some information have to be cleared during these 6431 * operations. 6432 * skb_scrub_packet can also be used to clean a skb before injecting it in 6433 * another namespace (@xnet == true). We have to clear all information in the 6434 * skb that could impact namespace isolation. Note that PSP skb extension is 6435 * preserved across packet scrubbing by design. 6436 */ 6437 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 6438 { 6439 skb->pkt_type = PACKET_HOST; 6440 skb->skb_iif = 0; 6441 skb->ignore_df = 0; 6442 skb_dst_drop(skb); 6443 skb_ext_scrub(skb); 6444 nf_reset_ct(skb); 6445 nf_reset_trace(skb); 6446 6447 #ifdef CONFIG_NET_SWITCHDEV 6448 skb->offload_fwd_mark = 0; 6449 skb->offload_l3_fwd_mark = 0; 6450 #endif 6451 ipvs_reset(skb); 6452 6453 if (!xnet) 6454 return; 6455 6456 skb->mark = 0; 6457 skb_clear_tstamp(skb); 6458 } 6459 EXPORT_SYMBOL_GPL(skb_scrub_packet); 6460 6461 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 6462 { 6463 int mac_len, meta_len; 6464 void *meta; 6465 6466 if (skb_cow(skb, skb_headroom(skb)) < 0) { 6467 kfree_skb(skb); 6468 return NULL; 6469 } 6470 6471 mac_len = skb->data - skb_mac_header(skb); 6472 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 6473 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 6474 mac_len - VLAN_HLEN - ETH_TLEN); 6475 } 6476 6477 meta_len = skb_metadata_len(skb); 6478 if (meta_len) { 6479 meta = skb_metadata_end(skb) - meta_len; 6480 memmove(meta + VLAN_HLEN, meta, meta_len); 6481 } 6482 6483 skb->mac_header += VLAN_HLEN; 6484 return skb; 6485 } 6486 6487 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 6488 { 6489 struct vlan_hdr *vhdr; 6490 u16 vlan_tci; 6491 6492 if (unlikely(skb_vlan_tag_present(skb))) { 6493 /* vlan_tci is already set-up so leave this for another time */ 6494 return skb; 6495 } 6496 6497 skb = skb_share_check(skb, GFP_ATOMIC); 6498 if (unlikely(!skb)) 6499 goto err_free; 6500 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 6501 goto err_free; 6502 6503 vhdr = (struct vlan_hdr *)skb->data; 6504 vlan_tci = ntohs(vhdr->h_vlan_TCI); 6505 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 6506 6507 skb_pull_rcsum(skb, VLAN_HLEN); 6508 vlan_set_encap_proto(skb, vhdr); 6509 6510 skb = skb_reorder_vlan_header(skb); 6511 if (unlikely(!skb)) 6512 goto err_free; 6513 6514 skb_reset_network_header(skb); 6515 if (!skb_transport_header_was_set(skb)) 6516 skb_reset_transport_header(skb); 6517 skb_reset_mac_len(skb); 6518 6519 return skb; 6520 6521 err_free: 6522 kfree_skb(skb); 6523 return NULL; 6524 } 6525 EXPORT_SYMBOL(skb_vlan_untag); 6526 6527 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 6528 { 6529 if (!pskb_may_pull(skb, write_len)) 6530 return -ENOMEM; 6531 6532 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 6533 return 0; 6534 6535 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 6536 } 6537 EXPORT_SYMBOL(skb_ensure_writable); 6538 6539 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev) 6540 { 6541 int needed_headroom = dev->needed_headroom; 6542 int needed_tailroom = dev->needed_tailroom; 6543 6544 /* For tail taggers, we need to pad short frames ourselves, to ensure 6545 * that the tail tag does not fail at its role of being at the end of 6546 * the packet, once the conduit interface pads the frame. Account for 6547 * that pad length here, and pad later. 6548 */ 6549 if (unlikely(needed_tailroom && skb->len < ETH_ZLEN)) 6550 needed_tailroom += ETH_ZLEN - skb->len; 6551 /* skb_headroom() returns unsigned int... */ 6552 needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0); 6553 needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0); 6554 6555 if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb))) 6556 /* No reallocation needed, yay! */ 6557 return 0; 6558 6559 return pskb_expand_head(skb, needed_headroom, needed_tailroom, 6560 GFP_ATOMIC); 6561 } 6562 EXPORT_SYMBOL(skb_ensure_writable_head_tail); 6563 6564 /* remove VLAN header from packet and update csum accordingly. 6565 * expects a non skb_vlan_tag_present skb with a vlan tag payload 6566 */ 6567 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 6568 { 6569 int offset = skb->data - skb_mac_header(skb); 6570 int err; 6571 6572 if (WARN_ONCE(offset, 6573 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 6574 offset)) { 6575 return -EINVAL; 6576 } 6577 6578 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 6579 if (unlikely(err)) 6580 return err; 6581 6582 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6583 6584 vlan_remove_tag(skb, vlan_tci); 6585 6586 skb->mac_header += VLAN_HLEN; 6587 6588 if (skb_network_offset(skb) < ETH_HLEN) 6589 skb_set_network_header(skb, ETH_HLEN); 6590 6591 skb_reset_mac_len(skb); 6592 6593 return err; 6594 } 6595 EXPORT_SYMBOL(__skb_vlan_pop); 6596 6597 /* Pop a vlan tag either from hwaccel or from payload. 6598 * Expects skb->data at mac header. 6599 */ 6600 int skb_vlan_pop(struct sk_buff *skb) 6601 { 6602 u16 vlan_tci; 6603 __be16 vlan_proto; 6604 int err; 6605 6606 if (likely(skb_vlan_tag_present(skb))) { 6607 __vlan_hwaccel_clear_tag(skb); 6608 } else { 6609 if (unlikely(!eth_type_vlan(skb->protocol))) 6610 return 0; 6611 6612 err = __skb_vlan_pop(skb, &vlan_tci); 6613 if (err) 6614 return err; 6615 } 6616 /* move next vlan tag to hw accel tag */ 6617 if (likely(!eth_type_vlan(skb->protocol))) 6618 return 0; 6619 6620 vlan_proto = skb->protocol; 6621 err = __skb_vlan_pop(skb, &vlan_tci); 6622 if (unlikely(err)) 6623 return err; 6624 6625 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6626 return 0; 6627 } 6628 EXPORT_SYMBOL(skb_vlan_pop); 6629 6630 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 6631 * Expects skb->data at mac header. 6632 */ 6633 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 6634 { 6635 if (skb_vlan_tag_present(skb)) { 6636 int offset = skb->data - skb_mac_header(skb); 6637 int err; 6638 6639 if (WARN_ONCE(offset, 6640 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 6641 offset)) { 6642 return -EINVAL; 6643 } 6644 6645 err = __vlan_insert_tag(skb, skb->vlan_proto, 6646 skb_vlan_tag_get(skb)); 6647 if (err) 6648 return err; 6649 6650 skb->protocol = skb->vlan_proto; 6651 skb->network_header -= VLAN_HLEN; 6652 6653 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6654 } 6655 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6656 return 0; 6657 } 6658 EXPORT_SYMBOL(skb_vlan_push); 6659 6660 /** 6661 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 6662 * 6663 * @skb: Socket buffer to modify 6664 * 6665 * Drop the Ethernet header of @skb. 6666 * 6667 * Expects that skb->data points to the mac header and that no VLAN tags are 6668 * present. 6669 * 6670 * Returns 0 on success, -errno otherwise. 6671 */ 6672 int skb_eth_pop(struct sk_buff *skb) 6673 { 6674 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 6675 skb_network_offset(skb) < ETH_HLEN) 6676 return -EPROTO; 6677 6678 skb_pull_rcsum(skb, ETH_HLEN); 6679 skb_reset_mac_header(skb); 6680 skb_reset_mac_len(skb); 6681 6682 return 0; 6683 } 6684 EXPORT_SYMBOL(skb_eth_pop); 6685 6686 /** 6687 * skb_eth_push() - Add a new Ethernet header at the head of a packet 6688 * 6689 * @skb: Socket buffer to modify 6690 * @dst: Destination MAC address of the new header 6691 * @src: Source MAC address of the new header 6692 * 6693 * Prepend @skb with a new Ethernet header. 6694 * 6695 * Expects that skb->data points to the mac header, which must be empty. 6696 * 6697 * Returns 0 on success, -errno otherwise. 6698 */ 6699 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 6700 const unsigned char *src) 6701 { 6702 struct ethhdr *eth; 6703 int err; 6704 6705 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 6706 return -EPROTO; 6707 6708 err = skb_cow_head(skb, sizeof(*eth)); 6709 if (err < 0) 6710 return err; 6711 6712 skb_push(skb, sizeof(*eth)); 6713 skb_reset_mac_header(skb); 6714 skb_reset_mac_len(skb); 6715 6716 eth = eth_hdr(skb); 6717 ether_addr_copy(eth->h_dest, dst); 6718 ether_addr_copy(eth->h_source, src); 6719 eth->h_proto = skb->protocol; 6720 6721 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 6722 6723 return 0; 6724 } 6725 EXPORT_SYMBOL(skb_eth_push); 6726 6727 /* Update the ethertype of hdr and the skb csum value if required. */ 6728 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 6729 __be16 ethertype) 6730 { 6731 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6732 __be16 diff[] = { ~hdr->h_proto, ethertype }; 6733 6734 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6735 } 6736 6737 hdr->h_proto = ethertype; 6738 } 6739 6740 /** 6741 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 6742 * the packet 6743 * 6744 * @skb: buffer 6745 * @mpls_lse: MPLS label stack entry to push 6746 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6747 * @mac_len: length of the MAC header 6748 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6749 * ethernet 6750 * 6751 * Expects skb->data at mac header. 6752 * 6753 * Returns 0 on success, -errno otherwise. 6754 */ 6755 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6756 int mac_len, bool ethernet) 6757 { 6758 struct mpls_shim_hdr *lse; 6759 int err; 6760 6761 if (unlikely(!eth_p_mpls(mpls_proto))) 6762 return -EINVAL; 6763 6764 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6765 if (skb->encapsulation) 6766 return -EINVAL; 6767 6768 err = skb_cow_head(skb, MPLS_HLEN); 6769 if (unlikely(err)) 6770 return err; 6771 6772 if (!skb->inner_protocol) { 6773 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6774 skb_set_inner_protocol(skb, skb->protocol); 6775 } 6776 6777 skb_push(skb, MPLS_HLEN); 6778 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6779 mac_len); 6780 skb_reset_mac_header(skb); 6781 skb_set_network_header(skb, mac_len); 6782 skb_reset_mac_len(skb); 6783 6784 lse = mpls_hdr(skb); 6785 lse->label_stack_entry = mpls_lse; 6786 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6787 6788 if (ethernet && mac_len >= ETH_HLEN) 6789 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6790 skb->protocol = mpls_proto; 6791 6792 return 0; 6793 } 6794 EXPORT_SYMBOL_GPL(skb_mpls_push); 6795 6796 /** 6797 * skb_mpls_pop() - pop the outermost MPLS header 6798 * 6799 * @skb: buffer 6800 * @next_proto: ethertype of header after popped MPLS header 6801 * @mac_len: length of the MAC header 6802 * @ethernet: flag to indicate if the packet is ethernet 6803 * 6804 * Expects skb->data at mac header. 6805 * 6806 * Returns 0 on success, -errno otherwise. 6807 */ 6808 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6809 bool ethernet) 6810 { 6811 int err; 6812 6813 if (unlikely(!eth_p_mpls(skb->protocol))) 6814 return 0; 6815 6816 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6817 if (unlikely(err)) 6818 return err; 6819 6820 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6821 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6822 mac_len); 6823 6824 __skb_pull(skb, MPLS_HLEN); 6825 skb_reset_mac_header(skb); 6826 skb_set_network_header(skb, mac_len); 6827 6828 if (ethernet && mac_len >= ETH_HLEN) { 6829 struct ethhdr *hdr; 6830 6831 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6832 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6833 skb_mod_eth_type(skb, hdr, next_proto); 6834 } 6835 skb->protocol = next_proto; 6836 6837 /* The last label is gone, so the inner header recorded by 6838 * skb_mpls_push() no longer describes this packet. Drop it, or a 6839 * later push keeps the stale offset. 6840 */ 6841 if (!eth_p_mpls(next_proto)) 6842 skb->inner_protocol = 0; 6843 6844 return 0; 6845 } 6846 EXPORT_SYMBOL_GPL(skb_mpls_pop); 6847 6848 /** 6849 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6850 * 6851 * @skb: buffer 6852 * @mpls_lse: new MPLS label stack entry to update to 6853 * 6854 * Expects skb->data at mac header. 6855 * 6856 * Returns 0 on success, -errno otherwise. 6857 */ 6858 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6859 { 6860 int err; 6861 6862 if (unlikely(!eth_p_mpls(skb->protocol))) 6863 return -EINVAL; 6864 6865 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6866 if (unlikely(err)) 6867 return err; 6868 6869 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6870 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6871 6872 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6873 } 6874 6875 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6876 6877 return 0; 6878 } 6879 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6880 6881 /** 6882 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6883 * 6884 * @skb: buffer 6885 * 6886 * Expects skb->data at mac header. 6887 * 6888 * Returns 0 on success, -errno otherwise. 6889 */ 6890 int skb_mpls_dec_ttl(struct sk_buff *skb) 6891 { 6892 u32 lse; 6893 u8 ttl; 6894 6895 if (unlikely(!eth_p_mpls(skb->protocol))) 6896 return -EINVAL; 6897 6898 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6899 return -ENOMEM; 6900 6901 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6902 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6903 if (!ttl || !--ttl) 6904 return -EINVAL; 6905 6906 lse &= ~MPLS_LS_TTL_MASK; 6907 lse |= ttl << MPLS_LS_TTL_SHIFT; 6908 6909 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6910 } 6911 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6912 6913 /** 6914 * alloc_skb_with_frags - allocate skb with page frags 6915 * 6916 * @header_len: size of linear part 6917 * @data_len: needed length in frags 6918 * @order: max page order desired. 6919 * @errcode: pointer to error code if any 6920 * @gfp_mask: allocation mask 6921 * 6922 * This can be used to allocate a paged skb, given a maximal order for frags. 6923 */ 6924 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6925 unsigned long data_len, 6926 int order, 6927 int *errcode, 6928 gfp_t gfp_mask) 6929 { 6930 unsigned long chunk; 6931 struct sk_buff *skb; 6932 struct page *page; 6933 int nr_frags = 0; 6934 6935 *errcode = -EMSGSIZE; 6936 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) 6937 return NULL; 6938 6939 *errcode = -ENOBUFS; 6940 skb = alloc_skb(header_len, gfp_mask); 6941 if (!skb) 6942 return NULL; 6943 6944 while (data_len) { 6945 if (nr_frags == MAX_SKB_FRAGS) 6946 goto failure; 6947 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) 6948 order--; 6949 6950 if (order) { 6951 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6952 __GFP_COMP | 6953 __GFP_NOWARN, 6954 order); 6955 if (!page) { 6956 order--; 6957 continue; 6958 } 6959 } else { 6960 page = alloc_page(gfp_mask); 6961 if (!page) 6962 goto failure; 6963 } 6964 chunk = min_t(unsigned long, data_len, 6965 PAGE_SIZE << order); 6966 skb_fill_page_desc(skb, nr_frags, page, 0, chunk); 6967 nr_frags++; 6968 skb->truesize += (PAGE_SIZE << order); 6969 data_len -= chunk; 6970 } 6971 return skb; 6972 6973 failure: 6974 kfree_skb(skb); 6975 return NULL; 6976 } 6977 EXPORT_SYMBOL(alloc_skb_with_frags); 6978 6979 /* pskb_carve_inside_header() and pskb_carve_inside_nonlinear() 6980 * remove the first bytes of a packet and reallocate skb->head. 6981 * 6982 * Whatever headers were present before the operation are gone, 6983 * we must not leave stale offsets, otherwise users of this skb 6984 * (skb_dump(), drop_monitor, taps, ...) would read or pull garbage. 6985 */ 6986 static void skb_carve_reset_headers(struct sk_buff *skb) 6987 { 6988 skb_unset_mac_header(skb); 6989 skb_unset_transport_header(skb); 6990 skb_reset_network_header(skb); 6991 skb->mac_len = 0; 6992 6993 /* Inner offsets have no "unset" marker, zero them so that 6994 * skb_inner_network_header_was_set() becomes false and no 6995 * consumer mistakes them for a real (and long gone) header. 6996 */ 6997 skb->inner_mac_header = 0; 6998 skb->inner_network_header = 0; 6999 skb->inner_transport_header = 0; 7000 skb->inner_protocol = 0; 7001 skb->encapsulation = 0; 7002 7003 if (skb->ip_summed == CHECKSUM_PARTIAL) 7004 skb->ip_summed = CHECKSUM_NONE; 7005 } 7006 7007 /* carve out the first off bytes from skb when off < headlen */ 7008 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 7009 const int headlen, gfp_t gfp_mask) 7010 { 7011 int i; 7012 unsigned int size = skb_end_offset(skb); 7013 int new_hlen = headlen - off; 7014 u8 *data; 7015 7016 if (skb_pfmemalloc(skb)) 7017 gfp_mask |= __GFP_MEMALLOC; 7018 7019 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 7020 if (!data) 7021 return -ENOMEM; 7022 size = SKB_WITH_OVERHEAD(size); 7023 7024 /* Copy real data, and all frags */ 7025 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 7026 skb->len -= off; 7027 7028 /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it 7029 * while refcounting frags below. 7030 */ 7031 skb_zcopy_downgrade_managed(skb); 7032 7033 memcpy((struct skb_shared_info *)(data + size), 7034 skb_shinfo(skb), 7035 offsetof(struct skb_shared_info, 7036 frags[skb_shinfo(skb)->nr_frags])); 7037 if (skb_cloned(skb)) { 7038 /* drop the old head gracefully */ 7039 if (skb_orphan_frags(skb, gfp_mask)) { 7040 skb_kfree_head(data); 7041 return -ENOMEM; 7042 } 7043 if (skb_zcopy(skb)) 7044 net_zcopy_get(skb_uarg(skb)); 7045 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 7046 skb_frag_ref(skb, i); 7047 if (skb_has_frag_list(skb)) 7048 skb_clone_fraglist(skb); 7049 skb_release_data(skb, SKB_CONSUMED); 7050 } else { 7051 /* we can reuse existing recount- all we did was 7052 * relocate values 7053 */ 7054 skb_free_head(skb); 7055 } 7056 7057 skb->head = data; 7058 skb->data = data; 7059 skb->head_frag = 0; 7060 skb_set_end_offset(skb, size); 7061 skb_set_tail_pointer(skb, skb_headlen(skb)); 7062 skb_carve_reset_headers(skb); 7063 skb->cloned = 0; 7064 skb->hdr_len = 0; 7065 skb->nohdr = 0; 7066 atomic_set(&skb_shinfo(skb)->dataref, 1); 7067 7068 return 0; 7069 } 7070 7071 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 7072 7073 /* carve out the first eat bytes from skb's frag_list. May recurse into 7074 * pskb_carve() 7075 */ 7076 static int pskb_carve_frag_list(struct skb_shared_info *shinfo, int eat, 7077 gfp_t gfp_mask) 7078 { 7079 struct sk_buff *list = shinfo->frag_list; 7080 struct sk_buff *clone = NULL; 7081 struct sk_buff *insp = NULL; 7082 7083 do { 7084 if (!list) { 7085 pr_err("Not enough bytes to eat. Want %d\n", eat); 7086 return -EFAULT; 7087 } 7088 if (list->len <= eat) { 7089 /* Eaten as whole. */ 7090 eat -= list->len; 7091 list = list->next; 7092 insp = list; 7093 } else { 7094 /* Eaten partially. */ 7095 if (skb_shared(list)) { 7096 clone = skb_clone(list, gfp_mask); 7097 if (!clone) 7098 return -ENOMEM; 7099 insp = list->next; 7100 list = clone; 7101 } else { 7102 /* This may be pulled without problems. */ 7103 insp = list; 7104 } 7105 if (pskb_carve(list, eat, gfp_mask) < 0) { 7106 kfree_skb(clone); 7107 return -ENOMEM; 7108 } 7109 break; 7110 } 7111 } while (eat); 7112 7113 /* Free pulled out fragments. */ 7114 while ((list = shinfo->frag_list) != insp) { 7115 shinfo->frag_list = list->next; 7116 consume_skb(list); 7117 } 7118 /* And insert new clone at head. */ 7119 if (clone) { 7120 clone->next = list; 7121 shinfo->frag_list = clone; 7122 } 7123 return 0; 7124 } 7125 7126 /* carve off first len bytes from skb. Split line (off) is in the 7127 * non-linear part of skb 7128 */ 7129 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 7130 int pos, gfp_t gfp_mask) 7131 { 7132 int i, k = 0; 7133 unsigned int size = skb_end_offset(skb); 7134 u8 *data; 7135 const int nfrags = skb_shinfo(skb)->nr_frags; 7136 struct skb_shared_info *shinfo; 7137 7138 if (skb_pfmemalloc(skb)) 7139 gfp_mask |= __GFP_MEMALLOC; 7140 7141 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 7142 if (!data) 7143 return -ENOMEM; 7144 size = SKB_WITH_OVERHEAD(size); 7145 7146 /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it 7147 * while refcounting frags below. 7148 */ 7149 skb_zcopy_downgrade_managed(skb); 7150 7151 memcpy((struct skb_shared_info *)(data + size), 7152 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 7153 if (skb_orphan_frags(skb, gfp_mask)) { 7154 skb_kfree_head(data); 7155 return -ENOMEM; 7156 } 7157 shinfo = (struct skb_shared_info *)(data + size); 7158 for (i = 0; i < nfrags; i++) { 7159 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 7160 7161 if (pos + fsize > off) { 7162 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 7163 7164 if (pos < off) { 7165 /* Split frag. 7166 * We have two variants in this case: 7167 * 1. Move all the frag to the second 7168 * part, if it is possible. F.e. 7169 * this approach is mandatory for TUX, 7170 * where splitting is expensive. 7171 * 2. Split is accurately. We make this. 7172 */ 7173 skb_frag_off_add(&shinfo->frags[0], off - pos); 7174 skb_frag_size_sub(&shinfo->frags[0], off - pos); 7175 } 7176 skb_frag_ref(skb, i); 7177 k++; 7178 } 7179 pos += fsize; 7180 } 7181 shinfo->nr_frags = k; 7182 if (skb_has_frag_list(skb)) 7183 skb_clone_fraglist(skb); 7184 7185 /* split line is in frag list */ 7186 if (k == 0 && pskb_carve_frag_list(shinfo, off - pos, gfp_mask)) { 7187 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 7188 if (skb_has_frag_list(skb)) 7189 kfree_skb_list(skb_shinfo(skb)->frag_list); 7190 skb_kfree_head(data); 7191 return -ENOMEM; 7192 } 7193 if (skb_zcopy(skb)) 7194 net_zcopy_get(skb_uarg(skb)); 7195 skb_release_data(skb, SKB_CONSUMED); 7196 7197 skb->head = data; 7198 skb->head_frag = 0; 7199 skb->data = data; 7200 skb_set_end_offset(skb, size); 7201 skb_reset_tail_pointer(skb); 7202 skb_carve_reset_headers(skb); 7203 skb->cloned = 0; 7204 skb->hdr_len = 0; 7205 skb->nohdr = 0; 7206 skb->len -= off; 7207 skb->data_len = skb->len; 7208 atomic_set(&skb_shinfo(skb)->dataref, 1); 7209 return 0; 7210 } 7211 7212 /* remove len bytes from the beginning of the skb */ 7213 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 7214 { 7215 int headlen = skb_headlen(skb); 7216 7217 if (len < headlen) 7218 return pskb_carve_inside_header(skb, len, headlen, gfp); 7219 else 7220 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 7221 } 7222 7223 /* Extract to_copy bytes starting at off from skb, and return this in 7224 * a new skb 7225 */ 7226 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 7227 int to_copy, gfp_t gfp) 7228 { 7229 struct sk_buff *clone = skb_clone(skb, gfp); 7230 7231 if (!clone) 7232 return NULL; 7233 7234 if (pskb_carve(clone, off, gfp) < 0 || 7235 pskb_trim(clone, to_copy)) { 7236 kfree_skb(clone); 7237 return NULL; 7238 } 7239 return clone; 7240 } 7241 EXPORT_SYMBOL(pskb_extract); 7242 7243 /** 7244 * skb_condense - try to get rid of fragments/frag_list if possible 7245 * @skb: buffer 7246 * 7247 * Can be used to save memory before skb is added to a busy queue. 7248 * If packet has bytes in frags and enough tail room in skb->head, 7249 * pull all of them, so that we can free the frags right now and adjust 7250 * truesize. 7251 * Notes: 7252 * We do not reallocate skb->head thus can not fail. 7253 * Caller must re-evaluate skb->truesize if needed. 7254 */ 7255 void skb_condense(struct sk_buff *skb) 7256 { 7257 if (skb->data_len) { 7258 if (skb->data_len > skb->end - skb->tail || 7259 skb_cloned(skb) || !skb_frags_readable(skb)) 7260 return; 7261 7262 /* Nice, we can free page frag(s) right now */ 7263 __pskb_pull_tail(skb, skb->data_len); 7264 } 7265 /* At this point, skb->truesize might be over estimated, 7266 * because skb had a fragment, and fragments do not tell 7267 * their truesize. 7268 * When we pulled its content into skb->head, fragment 7269 * was freed, but __pskb_pull_tail() could not possibly 7270 * adjust skb->truesize, not knowing the frag truesize. 7271 */ 7272 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 7273 } 7274 EXPORT_SYMBOL(skb_condense); 7275 7276 #ifdef CONFIG_SKB_EXTENSIONS 7277 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 7278 { 7279 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 7280 } 7281 7282 /** 7283 * __skb_ext_alloc - allocate a new skb extensions storage 7284 * 7285 * @flags: See kmalloc(). 7286 * 7287 * Returns the newly allocated pointer. The pointer can later attached to a 7288 * skb via __skb_ext_set(). 7289 * Note: caller must handle the skb_ext as an opaque data. 7290 */ 7291 struct skb_ext *__skb_ext_alloc(gfp_t flags) 7292 { 7293 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 7294 7295 if (new) { 7296 memset(new->offset, 0, sizeof(new->offset)); 7297 refcount_set(&new->refcnt, 1); 7298 } 7299 7300 return new; 7301 } 7302 7303 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 7304 unsigned int old_active) 7305 { 7306 struct skb_ext *new; 7307 7308 if (refcount_read(&old->refcnt) == 1) 7309 return old; 7310 7311 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 7312 if (!new) 7313 return NULL; 7314 7315 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 7316 refcount_set(&new->refcnt, 1); 7317 7318 #ifdef CONFIG_XFRM 7319 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 7320 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 7321 unsigned int i; 7322 7323 for (i = 0; i < sp->len; i++) 7324 xfrm_state_hold(sp->xvec[i]); 7325 } 7326 #endif 7327 #ifdef CONFIG_MCTP_FLOWS 7328 if (old_active & (1 << SKB_EXT_MCTP)) { 7329 struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP); 7330 7331 if (flow->key) 7332 refcount_inc(&flow->key->refs); 7333 } 7334 #endif 7335 __skb_ext_put(old); 7336 return new; 7337 } 7338 7339 /** 7340 * __skb_ext_set - attach the specified extension storage to this skb 7341 * @skb: buffer 7342 * @id: extension id 7343 * @ext: extension storage previously allocated via __skb_ext_alloc() 7344 * 7345 * Existing extensions, if any, are cleared. 7346 * 7347 * Returns the pointer to the extension. 7348 */ 7349 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 7350 struct skb_ext *ext) 7351 { 7352 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 7353 7354 skb_ext_put(skb); 7355 newlen = newoff + skb_ext_type_len[id]; 7356 ext->chunks = newlen; 7357 ext->offset[id] = newoff; 7358 skb->extensions = ext; 7359 skb->active_extensions = 1 << id; 7360 return skb_ext_get_ptr(ext, id); 7361 } 7362 EXPORT_SYMBOL_NS_GPL(__skb_ext_set, "NETDEV_INTERNAL"); 7363 7364 /** 7365 * skb_ext_add - allocate space for given extension, COW if needed 7366 * @skb: buffer 7367 * @id: extension to allocate space for 7368 * 7369 * Allocates enough space for the given extension. 7370 * If the extension is already present, a pointer to that extension 7371 * is returned. 7372 * 7373 * If the skb was cloned, COW applies and the returned memory can be 7374 * modified without changing the extension space of clones buffers. 7375 * 7376 * Returns pointer to the extension or NULL on allocation failure. 7377 */ 7378 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 7379 { 7380 struct skb_ext *new, *old = NULL; 7381 unsigned int newlen, newoff; 7382 7383 if (skb->active_extensions) { 7384 old = skb->extensions; 7385 7386 new = skb_ext_maybe_cow(old, skb->active_extensions); 7387 if (!new) 7388 return NULL; 7389 7390 if (__skb_ext_exist(new, id)) 7391 goto set_active; 7392 7393 newoff = new->chunks; 7394 } else { 7395 newoff = SKB_EXT_CHUNKSIZEOF(*new); 7396 7397 new = __skb_ext_alloc(GFP_ATOMIC); 7398 if (!new) 7399 return NULL; 7400 } 7401 7402 newlen = newoff + skb_ext_type_len[id]; 7403 new->chunks = newlen; 7404 new->offset[id] = newoff; 7405 set_active: 7406 skb->slow_gro = 1; 7407 skb->extensions = new; 7408 skb->active_extensions |= 1 << id; 7409 return skb_ext_get_ptr(new, id); 7410 } 7411 EXPORT_SYMBOL(skb_ext_add); 7412 7413 #ifdef CONFIG_XFRM 7414 static void skb_ext_put_sp(struct sec_path *sp) 7415 { 7416 unsigned int i; 7417 7418 if (!sp->len) 7419 return; 7420 7421 for (i = 0; i < sp->len; i++) 7422 xfrm_state_put(sp->xvec[i]); 7423 sp->len = 0; 7424 } 7425 #endif 7426 7427 #ifdef CONFIG_MCTP_FLOWS 7428 static void skb_ext_put_mctp(struct mctp_flow *flow) 7429 { 7430 if (!flow->key) 7431 return; 7432 7433 mctp_key_unref(flow->key); 7434 flow->key = NULL; 7435 } 7436 #endif 7437 7438 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 7439 { 7440 struct skb_ext *ext = skb->extensions; 7441 7442 skb->active_extensions &= ~(1 << id); 7443 if (skb->active_extensions == 0) { 7444 skb->extensions = NULL; 7445 __skb_ext_put(ext); 7446 return; 7447 } 7448 7449 if (refcount_read(&ext->refcnt) > 1) 7450 return; 7451 7452 #ifdef CONFIG_XFRM 7453 if (id == SKB_EXT_SEC_PATH) 7454 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 7455 #endif 7456 #ifdef CONFIG_MCTP_FLOWS 7457 if (id == SKB_EXT_MCTP) 7458 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 7459 #endif 7460 } 7461 EXPORT_SYMBOL(__skb_ext_del); 7462 7463 void __skb_ext_put(struct skb_ext *ext) 7464 { 7465 /* If this is last clone, nothing can increment 7466 * it after check passes. Avoids one atomic op. 7467 */ 7468 if (refcount_read(&ext->refcnt) == 1) 7469 goto free_now; 7470 7471 if (!refcount_dec_and_test(&ext->refcnt)) 7472 return; 7473 free_now: 7474 #ifdef CONFIG_XFRM 7475 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 7476 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 7477 #endif 7478 #ifdef CONFIG_MCTP_FLOWS 7479 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 7480 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 7481 #endif 7482 7483 kmem_cache_free(skbuff_ext_cache, ext); 7484 } 7485 EXPORT_SYMBOL(__skb_ext_put); 7486 #endif /* CONFIG_SKB_EXTENSIONS */ 7487 7488 static void kfree_skb_napi_cache(struct sk_buff *skb) 7489 { 7490 /* if SKB is a clone, don't handle this case */ 7491 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 7492 __kfree_skb(skb); 7493 return; 7494 } 7495 7496 local_bh_disable(); 7497 __napi_kfree_skb(skb, SKB_CONSUMED); 7498 local_bh_enable(); 7499 } 7500 7501 DEFINE_STATIC_KEY_FALSE(skb_defer_disable_key); 7502 7503 /** 7504 * skb_attempt_defer_free - queue skb for remote freeing 7505 * @skb: buffer 7506 * 7507 * Put @skb in a per-cpu list, using the cpu which 7508 * allocated the skb/pages to reduce false sharing 7509 * and memory zone spinlock contention. 7510 */ 7511 void skb_attempt_defer_free(struct sk_buff *skb) 7512 { 7513 struct skb_defer_node *sdn; 7514 unsigned long defer_count; 7515 unsigned int defer_max; 7516 int cpu, my_cpu; 7517 bool kick; 7518 7519 if (static_branch_unlikely(&skb_defer_disable_key)) 7520 goto nodefer; 7521 7522 /* zero copy notifications should not be delayed. */ 7523 if (skb_zcopy(skb)) 7524 goto nodefer; 7525 7526 cpu = skb->alloc_cpu; 7527 my_cpu = raw_smp_processor_id(); 7528 if (cpu == my_cpu || 7529 WARN_ON_ONCE(cpu >= nr_cpu_ids) || 7530 !cpu_online(cpu)) { 7531 nodefer: kfree_skb_napi_cache(skb); 7532 return; 7533 } 7534 7535 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 7536 DEBUG_NET_WARN_ON_ONCE(skb->destructor); 7537 DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb)); 7538 7539 sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu); 7540 7541 defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); 7542 defer_count = atomic_long_inc_return(&sdn->defer_count); 7543 7544 if (defer_count > defer_max) { 7545 /* Undo the reservation. An llist_add() does not release one -- 7546 * the skb stays charged until a flush subtracts the batch that 7547 * contains it -- so a rejected free must undo its own. 7548 */ 7549 atomic_long_dec(&sdn->defer_count); 7550 goto nodefer; 7551 } 7552 7553 llist_add(&skb->ll_node, &sdn->defer_list); 7554 7555 if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) { 7556 skb_defer_node_flush(sdn); 7557 return; 7558 } 7559 7560 /* Half capacity, plus the skb that fills the queue. Equality alone can 7561 * be consumed early -- if the producer holding the kick value publishes 7562 * first, its purge drains only that skb and the reservations published 7563 * after it match nothing -- which would leave a full queue with nothing 7564 * pending. 7565 */ 7566 kick = (defer_count - 1) == (defer_max >> 1) || defer_count == defer_max; 7567 7568 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 7569 * if we are unlucky enough (this seems very unlikely). 7570 */ 7571 if (unlikely(kick)) 7572 kick_defer_list_purge(cpu); 7573 } 7574 7575 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, 7576 size_t offset, size_t len, 7577 unsigned int csum_offset) 7578 { 7579 const char *kaddr; 7580 __wsum csum; 7581 7582 kaddr = kmap_local_page(page); 7583 csum = csum_partial(kaddr + offset, len, 0); 7584 kunmap_local(kaddr); 7585 skb->csum = csum_block_add(skb->csum, csum, csum_offset); 7586 } 7587 7588 /** 7589 * skb_splice_from_iter - Splice (or copy) pages to skbuff 7590 * @skb: The buffer to add pages to 7591 * @iter: Iterator representing the pages to be added 7592 * @maxsize: Maximum amount of pages to be added 7593 * 7594 * This is a common helper function for supporting MSG_SPLICE_PAGES. It 7595 * extracts pages from an iterator and adds them to the socket buffer if 7596 * possible, copying them to fragments if not possible (such as if they're slab 7597 * pages). 7598 * 7599 * Returns the amount of data spliced/copied or -EMSGSIZE if there's 7600 * insufficient space in the buffer to transfer anything. 7601 */ 7602 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, 7603 ssize_t maxsize) 7604 { 7605 size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags); 7606 struct page *pages[8], **ppages = pages; 7607 ssize_t spliced = 0, ret = 0; 7608 unsigned int i; 7609 7610 while (iter->count > 0) { 7611 ssize_t space, nr, len; 7612 size_t off; 7613 7614 ret = -EMSGSIZE; 7615 space = frag_limit - skb_shinfo(skb)->nr_frags; 7616 if (space < 0) 7617 break; 7618 7619 /* We might be able to coalesce without increasing nr_frags */ 7620 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages)); 7621 7622 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off); 7623 if (len <= 0) { 7624 ret = len ?: -EIO; 7625 break; 7626 } 7627 7628 i = 0; 7629 do { 7630 struct page *page = pages[i++]; 7631 size_t part = min_t(size_t, PAGE_SIZE - off, len); 7632 7633 ret = -EIO; 7634 if (WARN_ON_ONCE(!sendpage_ok(page))) 7635 goto out; 7636 7637 ret = skb_append_pagefrags(skb, page, off, part, 7638 frag_limit); 7639 if (ret < 0) { 7640 iov_iter_revert(iter, len); 7641 goto out; 7642 } 7643 7644 if (skb->ip_summed == CHECKSUM_NONE) 7645 skb_splice_csum_page(skb, page, off, part, 7646 skb->len + spliced); 7647 7648 off = 0; 7649 spliced += part; 7650 maxsize -= part; 7651 len -= part; 7652 } while (len > 0); 7653 7654 if (maxsize <= 0) 7655 break; 7656 } 7657 7658 out: 7659 skb_len_add(skb, spliced); 7660 return spliced ?: ret; 7661 } 7662 EXPORT_SYMBOL(skb_splice_from_iter); 7663 7664 static __always_inline 7665 size_t memcpy_from_iter_csum(void *iter_from, size_t progress, 7666 size_t len, void *to, void *priv2) 7667 { 7668 __wsum *csum = priv2; 7669 __wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len); 7670 7671 *csum = csum_block_add(*csum, next, progress); 7672 return 0; 7673 } 7674 7675 static __always_inline 7676 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress, 7677 size_t len, void *to, void *priv2) 7678 { 7679 __wsum next, *csum = priv2; 7680 7681 next = csum_and_copy_from_user(iter_from, to + progress, len); 7682 *csum = csum_block_add(*csum, next, progress); 7683 return next ? 0 : len; 7684 } 7685 7686 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, 7687 __wsum *csum, struct iov_iter *i) 7688 { 7689 size_t copied; 7690 7691 if (WARN_ON_ONCE(!i->data_source)) 7692 return false; 7693 copied = iterate_and_advance2(i, bytes, addr, csum, 7694 copy_from_user_iter_csum, 7695 memcpy_from_iter_csum); 7696 if (likely(copied == bytes)) 7697 return true; 7698 iov_iter_revert(i, copied); 7699 return false; 7700 } 7701 EXPORT_SYMBOL(csum_and_copy_from_iter_full); 7702 7703 /** 7704 * net_iov_inc_ref - acquire a non-page_pool reference on a net_iov 7705 * @niov: net_iov to reference 7706 * 7707 * Acquires a non-pp backing reference (binding->ref) on @niov regardless of 7708 * page_pool membership, warning if @niov does not support non-pp references. 7709 * Counterpart to net_iov_dec_ref(). 7710 */ 7711 void net_iov_inc_ref(struct net_iov *niov) 7712 { 7713 if (net_is_devmem_iov(niov)) 7714 net_devmem_net_iov_inc_ref(niov); 7715 else 7716 DEBUG_NET_WARN_ON_ONCE(true); 7717 } 7718 EXPORT_SYMBOL(net_iov_inc_ref); 7719 7720 /** 7721 * net_iov_dec_ref - release a non-page_pool reference on a net_iov 7722 * @niov: net_iov to unreference 7723 * 7724 * Drops a non-pp backing reference (binding->ref) on @niov regardless of 7725 * page_pool membership, warning if @niov does not support non-pp references. 7726 * Counterpart to net_iov_inc_ref(). 7727 */ 7728 void net_iov_dec_ref(struct net_iov *niov) 7729 { 7730 if (net_is_devmem_iov(niov)) 7731 net_devmem_net_iov_dec_ref(niov); 7732 else 7733 DEBUG_NET_WARN_ON_ONCE(true); 7734 } 7735 EXPORT_SYMBOL(net_iov_dec_ref); 7736 7737 struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb, 7738 __be16 type, 7739 int mac_offset) 7740 { 7741 unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH; 7742 7743 /* if type is 802.1Q/AD then the header should already be 7744 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at 7745 * ETH_HLEN otherwise 7746 */ 7747 if (vlan_depth) { 7748 if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN)) 7749 return (struct vlan_type_depth) { 0 }; 7750 vlan_depth -= VLAN_HLEN; 7751 } else { 7752 vlan_depth = ETH_HLEN; 7753 } 7754 do { 7755 struct vlan_hdr vhdr, *vh; 7756 7757 vh = skb_header_pointer(skb, mac_offset + vlan_depth, 7758 sizeof(vhdr), &vhdr); 7759 if (unlikely(!vh || !--parse_depth)) 7760 return (struct vlan_type_depth) { 0 }; 7761 7762 type = vh->h_vlan_encapsulated_proto; 7763 vlan_depth += VLAN_HLEN; 7764 } while (eth_type_vlan(type)); 7765 7766 return (struct vlan_type_depth) { 7767 .type = type, 7768 .depth = vlan_depth 7769 }; 7770 } 7771 EXPORT_SYMBOL(__vlan_get_protocol_offset); 7772