1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/skbuff_ref.h> 55 #include <linux/splice.h> 56 #include <linux/cache.h> 57 #include <linux/rtnetlink.h> 58 #include <linux/init.h> 59 #include <linux/scatterlist.h> 60 #include <linux/errqueue.h> 61 #include <linux/prefetch.h> 62 #include <linux/bitfield.h> 63 #include <linux/if_vlan.h> 64 #include <linux/mpls.h> 65 #include <linux/kcov.h> 66 #include <linux/iov_iter.h> 67 #include <linux/crc32.h> 68 #include <kunit/visibility.h> 69 70 #include <net/protocol.h> 71 #include <net/dst.h> 72 #include <net/sock.h> 73 #include <net/checksum.h> 74 #include <net/gro.h> 75 #include <net/gso.h> 76 #include <net/hotdata.h> 77 #include <net/ip6_checksum.h> 78 #include <net/xfrm.h> 79 #include <net/mpls.h> 80 #include <net/mptcp.h> 81 #include <net/mctp.h> 82 #include <net/tcp.h> 83 #include <net/can.h> 84 #include <net/page_pool/helpers.h> 85 #include <net/psp/types.h> 86 #include <net/dropreason.h> 87 #include <net/xdp_sock.h> 88 89 #include <linux/uaccess.h> 90 #include <trace/events/skb.h> 91 #include <linux/highmem.h> 92 #include <linux/capability.h> 93 #include <linux/user_namespace.h> 94 #include <linux/indirect_call_wrapper.h> 95 #include <linux/textsearch.h> 96 97 #include "dev.h" 98 #include "devmem.h" 99 #include "net-sysfs.h" 100 #include "netmem_priv.h" 101 102 #ifdef CONFIG_SKB_EXTENSIONS 103 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 104 #endif 105 106 #define GRO_MAX_HEAD_PAD (GRO_MAX_HEAD + NET_SKB_PAD + NET_IP_ALIGN) 107 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(max(MAX_TCP_HEADER, \ 108 GRO_MAX_HEAD_PAD)) 109 110 /* SKB_SMALL_HEAD_CACHE_SIZE is the size used for the skbuff_small_head 111 * kmem_cache. The non-power-of-2 padding is kept for historical reasons and 112 * to avoid potential collisions with generic kmalloc bucket sizes. 113 */ 114 #define SKB_SMALL_HEAD_CACHE_SIZE \ 115 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \ 116 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \ 117 SKB_SMALL_HEAD_SIZE) 118 119 #define SKB_SMALL_HEAD_HEADROOM \ 120 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) 121 122 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use 123 * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the 124 * netmem is a page. 125 */ 126 static_assert(offsetof(struct bio_vec, bv_page) == 127 offsetof(skb_frag_t, netmem)); 128 static_assert(sizeof_field(struct bio_vec, bv_page) == 129 sizeof_field(skb_frag_t, netmem)); 130 131 static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len)); 132 static_assert(sizeof_field(struct bio_vec, bv_len) == 133 sizeof_field(skb_frag_t, len)); 134 135 static_assert(offsetof(struct bio_vec, bv_offset) == 136 offsetof(skb_frag_t, offset)); 137 static_assert(sizeof_field(struct bio_vec, bv_offset) == 138 sizeof_field(skb_frag_t, offset)); 139 140 #undef FN 141 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason, 142 static const char * const drop_reasons[] = { 143 [SKB_CONSUMED] = "CONSUMED", 144 DEFINE_DROP_REASON(FN, FN) 145 }; 146 147 static const struct drop_reason_list drop_reasons_core = { 148 .reasons = drop_reasons, 149 .n_reasons = ARRAY_SIZE(drop_reasons), 150 }; 151 152 const struct drop_reason_list __rcu * 153 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = { 154 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core), 155 }; 156 EXPORT_SYMBOL(drop_reasons_by_subsys); 157 158 /** 159 * drop_reasons_register_subsys - register another drop reason subsystem 160 * @subsys: the subsystem to register, must not be the core 161 * @list: the list of drop reasons within the subsystem, must point to 162 * a statically initialized list 163 */ 164 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys, 165 const struct drop_reason_list *list) 166 { 167 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 168 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 169 "invalid subsystem %d\n", subsys)) 170 return; 171 172 /* must point to statically allocated memory, so INIT is OK */ 173 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list); 174 } 175 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys); 176 177 /** 178 * drop_reasons_unregister_subsys - unregister a drop reason subsystem 179 * @subsys: the subsystem to remove, must not be the core 180 * 181 * Note: This will synchronize_rcu() to ensure no users when it returns. 182 */ 183 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys) 184 { 185 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE || 186 subsys >= ARRAY_SIZE(drop_reasons_by_subsys), 187 "invalid subsystem %d\n", subsys)) 188 return; 189 190 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL); 191 192 synchronize_rcu(); 193 } 194 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys); 195 196 /** 197 * skb_panic - private function for out-of-line support 198 * @skb: buffer 199 * @sz: size 200 * @addr: address 201 * @msg: skb_over_panic or skb_under_panic 202 * 203 * Out-of-line support for skb_put() and skb_push(). 204 * Called via the wrapper skb_over_panic() or skb_under_panic(). 205 * Keep out of line to prevent kernel bloat. 206 * __builtin_return_address is not used because it is not always reliable. 207 */ 208 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 209 const char msg[]) 210 { 211 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 212 msg, addr, skb->len, sz, skb->head, skb->data, 213 (unsigned long)skb->tail, (unsigned long)skb->end, 214 skb->dev ? skb->dev->name : "<NULL>"); 215 BUG(); 216 } 217 218 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 219 { 220 skb_panic(skb, sz, addr, __func__); 221 } 222 223 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 224 { 225 skb_panic(skb, sz, addr, __func__); 226 } 227 228 #define NAPI_SKB_CACHE_SIZE 128 229 #define NAPI_SKB_CACHE_BULK 32 230 #define NAPI_SKB_CACHE_FREE 32 231 232 struct napi_alloc_cache { 233 local_lock_t bh_lock; 234 struct page_frag_cache page; 235 unsigned int skb_count; 236 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 237 }; 238 239 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 240 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache) = { 241 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 242 }; 243 244 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 245 { 246 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 247 void *data; 248 249 fragsz = SKB_DATA_ALIGN(fragsz); 250 251 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 252 data = __page_frag_alloc_align(&nc->page, fragsz, 253 GFP_ATOMIC | __GFP_NOWARN, align_mask); 254 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 255 return data; 256 257 } 258 EXPORT_SYMBOL(__napi_alloc_frag_align); 259 260 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 261 { 262 void *data; 263 264 if (in_hardirq() || irqs_disabled()) { 265 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache); 266 267 fragsz = SKB_DATA_ALIGN(fragsz); 268 data = __page_frag_alloc_align(nc, fragsz, 269 GFP_ATOMIC | __GFP_NOWARN, 270 align_mask); 271 } else { 272 local_bh_disable(); 273 data = __napi_alloc_frag_align(fragsz, align_mask); 274 local_bh_enable(); 275 } 276 return data; 277 } 278 EXPORT_SYMBOL(__netdev_alloc_frag_align); 279 280 /* Cache kmem_cache_size(net_hotdata.skbuff_cache) to help the compiler 281 * remove dead code (and skbuff_cache_size) when CONFIG_KASAN is unset. 282 */ 283 static u32 skbuff_cache_size __read_mostly; 284 285 static inline struct sk_buff *napi_skb_cache_get(bool alloc) 286 { 287 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 288 struct sk_buff *skb; 289 290 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 291 if (unlikely(!nc->skb_count)) { 292 if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 293 GFP_ATOMIC | __GFP_NOWARN, 294 NAPI_SKB_CACHE_BULK, 295 nc->skb_cache)) 296 nc->skb_count = NAPI_SKB_CACHE_BULK; 297 if (unlikely(!nc->skb_count)) { 298 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 299 return NULL; 300 } 301 } 302 303 skb = nc->skb_cache[--nc->skb_count]; 304 if (nc->skb_count) 305 prefetch(nc->skb_cache[nc->skb_count - 1]); 306 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 307 kasan_mempool_unpoison_object(skb, skbuff_cache_size); 308 309 return skb; 310 } 311 312 /* 313 * Only clear those fields we need to clear, not those that we will 314 * actually initialise later. Hence, don't put any more fields after 315 * the tail pointer in struct sk_buff! 316 */ 317 static inline void skbuff_clear(struct sk_buff *skb) 318 { 319 /* Replace memset(skb, 0, offsetof(struct sk_buff, tail)) 320 * with two smaller memset(), with a barrier() between them. 321 * This forces the compiler to inline both calls. 322 */ 323 BUILD_BUG_ON(offsetof(struct sk_buff, tail) <= 128); 324 memset(skb, 0, 128); 325 barrier(); 326 memset((void *)skb + 128, 0, offsetof(struct sk_buff, tail) - 128); 327 } 328 329 /** 330 * napi_skb_cache_get_bulk - obtain a number of zeroed skb heads from the cache 331 * @skbs: pointer to an at least @n-sized array to fill with skb pointers 332 * @n: number of entries to provide 333 * 334 * Tries to obtain @n &sk_buff entries from the NAPI percpu cache and writes 335 * the pointers into the provided array @skbs. If there are less entries 336 * available, tries to replenish the cache and bulk-allocates the diff from 337 * the MM layer if needed. 338 * The heads are being zeroed with either memset() or %__GFP_ZERO, so they are 339 * ready for {,__}build_skb_around() and don't have any data buffers attached. 340 * Must be called *only* from the BH context. 341 * 342 * Return: number of successfully allocated skbs (@n if no actual allocation 343 * needed or kmem_cache_alloc_bulk() didn't fail). 344 */ 345 u32 napi_skb_cache_get_bulk(void **skbs, u32 n) 346 { 347 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 348 u32 bulk, total = n; 349 350 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 351 352 if (nc->skb_count >= n) 353 goto get; 354 355 /* No enough cached skbs. Try refilling the cache first */ 356 bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK); 357 if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 358 GFP_ATOMIC | __GFP_NOWARN, bulk, 359 &nc->skb_cache[nc->skb_count])) 360 nc->skb_count += bulk; 361 if (likely(nc->skb_count >= n)) 362 goto get; 363 364 /* Still not enough. Bulk-allocate the missing part directly, zeroed */ 365 if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, 366 GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, 367 n - nc->skb_count, &skbs[nc->skb_count])) 368 n = nc->skb_count; 369 if (likely(nc->skb_count >= n)) 370 goto get; 371 372 /* kmem_cache didn't allocate the number we need, limit the output */ 373 total -= n - nc->skb_count; 374 n = nc->skb_count; 375 376 get: 377 for (u32 base = nc->skb_count - n, i = 0; i < n; i++) { 378 skbs[i] = nc->skb_cache[base + i]; 379 380 kasan_mempool_unpoison_object(skbs[i], skbuff_cache_size); 381 skbuff_clear(skbs[i]); 382 } 383 384 nc->skb_count -= n; 385 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 386 387 return total; 388 } 389 EXPORT_SYMBOL_GPL(napi_skb_cache_get_bulk); 390 391 static inline void __finalize_skb_around(struct sk_buff *skb, void *data, 392 unsigned int size) 393 { 394 struct skb_shared_info *shinfo; 395 396 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 397 398 /* Assumes caller memset cleared SKB */ 399 skb->truesize = SKB_TRUESIZE(size); 400 refcount_set(&skb->users, 1); 401 skb->head = data; 402 skb->data = data; 403 skb_reset_tail_pointer(skb); 404 skb_set_end_offset(skb, size); 405 skb->mac_header = (typeof(skb->mac_header))~0U; 406 skb->transport_header = (typeof(skb->transport_header))~0U; 407 skb->alloc_cpu = raw_smp_processor_id(); 408 /* make sure we initialize shinfo sequentially */ 409 shinfo = skb_shinfo(skb); 410 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 411 atomic_set(&shinfo->dataref, 1); 412 413 skb_set_kcov_handle(skb, kcov_common_handle()); 414 } 415 416 static inline void *__slab_build_skb(void *data, unsigned int *size) 417 { 418 void *resized; 419 420 /* Must find the allocation size (and grow it to match). */ 421 *size = ksize(data); 422 /* krealloc() will immediately return "data" when 423 * "ksize(data)" is requested: it is the existing upper 424 * bounds. As a result, GFP_ATOMIC will be ignored. Note 425 * that this "new" pointer needs to be passed back to the 426 * caller for use so the __alloc_size hinting will be 427 * tracked correctly. 428 */ 429 resized = krealloc(data, *size, GFP_ATOMIC); 430 WARN_ON_ONCE(resized != data); 431 return resized; 432 } 433 434 /* build_skb() variant which can operate on slab buffers. 435 * Note that this should be used sparingly as slab buffers 436 * cannot be combined efficiently by GRO! 437 */ 438 struct sk_buff *slab_build_skb(void *data) 439 { 440 struct sk_buff *skb; 441 unsigned int size; 442 443 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, 444 GFP_ATOMIC | __GFP_NOWARN); 445 if (unlikely(!skb)) 446 return NULL; 447 448 skbuff_clear(skb); 449 data = __slab_build_skb(data, &size); 450 __finalize_skb_around(skb, data, size); 451 452 return skb; 453 } 454 EXPORT_SYMBOL(slab_build_skb); 455 456 /* Caller must provide SKB that is memset cleared */ 457 static void __build_skb_around(struct sk_buff *skb, void *data, 458 unsigned int frag_size) 459 { 460 unsigned int size = frag_size; 461 462 /* frag_size == 0 is considered deprecated now. Callers 463 * using slab buffer should use slab_build_skb() instead. 464 */ 465 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) 466 data = __slab_build_skb(data, &size); 467 468 __finalize_skb_around(skb, data, size); 469 } 470 471 /** 472 * __build_skb - build a network buffer 473 * @data: data buffer provided by caller 474 * @frag_size: size of data (must not be 0) 475 * 476 * Allocate a new &sk_buff. Caller provides space holding head and 477 * skb_shared_info. @data must have been allocated from the page 478 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc() 479 * allocation is deprecated, and callers should use slab_build_skb() 480 * instead.) 481 * The return is the new skb buffer. 482 * On a failure the return is %NULL, and @data is not freed. 483 * Notes : 484 * Before IO, driver allocates only data buffer where NIC put incoming frame 485 * Driver should add room at head (NET_SKB_PAD) and 486 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 487 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 488 * before giving packet to stack. 489 * RX rings only contains data buffers, not full skbs. 490 */ 491 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 492 { 493 struct sk_buff *skb; 494 495 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, 496 GFP_ATOMIC | __GFP_NOWARN); 497 if (unlikely(!skb)) 498 return NULL; 499 500 skbuff_clear(skb); 501 __build_skb_around(skb, data, frag_size); 502 503 return skb; 504 } 505 506 /* build_skb() is wrapper over __build_skb(), that specifically 507 * takes care of skb->head and skb->pfmemalloc 508 */ 509 struct sk_buff *build_skb(void *data, unsigned int frag_size) 510 { 511 struct sk_buff *skb = __build_skb(data, frag_size); 512 513 if (likely(skb && frag_size)) { 514 skb->head_frag = 1; 515 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 516 } 517 return skb; 518 } 519 EXPORT_SYMBOL(build_skb); 520 521 /** 522 * build_skb_around - build a network buffer around provided skb 523 * @skb: sk_buff provide by caller, must be memset cleared 524 * @data: data buffer provided by caller 525 * @frag_size: size of data 526 */ 527 struct sk_buff *build_skb_around(struct sk_buff *skb, 528 void *data, unsigned int frag_size) 529 { 530 if (unlikely(!skb)) 531 return NULL; 532 533 __build_skb_around(skb, data, frag_size); 534 535 if (frag_size) { 536 skb->head_frag = 1; 537 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 538 } 539 return skb; 540 } 541 EXPORT_SYMBOL(build_skb_around); 542 543 /** 544 * __napi_build_skb - build a network buffer 545 * @data: data buffer provided by caller 546 * @frag_size: size of data 547 * 548 * Version of __build_skb() that uses NAPI percpu caches to obtain 549 * skbuff_head instead of inplace allocation. 550 * 551 * Returns a new &sk_buff on success, %NULL on allocation failure. 552 */ 553 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 554 { 555 struct sk_buff *skb; 556 557 skb = napi_skb_cache_get(true); 558 if (unlikely(!skb)) 559 return NULL; 560 561 skbuff_clear(skb); 562 __build_skb_around(skb, data, frag_size); 563 564 return skb; 565 } 566 567 /** 568 * napi_build_skb - build a network buffer 569 * @data: data buffer provided by caller 570 * @frag_size: size of data 571 * 572 * Version of __napi_build_skb() that takes care of skb->head_frag 573 * and skb->pfmemalloc when the data is a page or page fragment. 574 * 575 * Returns a new &sk_buff on success, %NULL on allocation failure. 576 */ 577 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 578 { 579 struct sk_buff *skb = __napi_build_skb(data, frag_size); 580 581 if (likely(skb) && frag_size) { 582 skb->head_frag = 1; 583 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 584 } 585 586 return skb; 587 } 588 EXPORT_SYMBOL(napi_build_skb); 589 590 static void *kmalloc_pfmemalloc(size_t obj_size, gfp_t flags, int node) 591 { 592 if (!gfp_pfmemalloc_allowed(flags)) 593 return NULL; 594 if (!obj_size) 595 return kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, 596 flags, node); 597 return kmalloc_node_track_caller(obj_size, flags, node); 598 } 599 600 /* 601 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 602 * the caller if emergency pfmemalloc reserves are being used. If it is and 603 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 604 * may be used. Otherwise, the packet data may be discarded until enough 605 * memory is free 606 */ 607 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, 608 struct sk_buff *skb) 609 { 610 size_t obj_size; 611 void *obj; 612 613 obj_size = SKB_HEAD_ALIGN(*size); 614 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && 615 !(flags & KMALLOC_NOT_NORMAL_BITS)) { 616 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, 617 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 618 node); 619 *size = SKB_SMALL_HEAD_CACHE_SIZE; 620 if (likely(obj)) 621 goto out; 622 /* Try again but now we are using pfmemalloc reserves */ 623 if (skb) 624 skb->pfmemalloc = true; 625 return kmalloc_pfmemalloc(0, flags, node); 626 } 627 628 obj_size = kmalloc_size_roundup(obj_size); 629 /* The following cast might truncate high-order bits of obj_size, this 630 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway. 631 */ 632 *size = (unsigned int)obj_size; 633 634 /* 635 * Try a regular allocation, when that fails and we're not entitled 636 * to the reserves, fail. 637 */ 638 obj = kmalloc_node_track_caller(obj_size, 639 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 640 node); 641 if (likely(obj)) 642 goto out; 643 644 /* Try again but now we are using pfmemalloc reserves */ 645 if (skb) 646 skb->pfmemalloc = true; 647 obj = kmalloc_pfmemalloc(obj_size, flags, node); 648 out: 649 return obj; 650 } 651 652 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 653 * 'private' fields and also do memory statistics to find all the 654 * [BEEP] leaks. 655 * 656 */ 657 658 /** 659 * __alloc_skb - allocate a network buffer 660 * @size: size to allocate 661 * @gfp_mask: allocation mask 662 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 663 * instead of head cache and allocate a cloned (child) skb. 664 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 665 * allocations in case the data is required for writeback 666 * @node: numa node to allocate memory on 667 * 668 * Allocate a new &sk_buff. The returned buffer has no headroom and a 669 * tail room of at least size bytes. The object has a reference count 670 * of one. The return is the buffer. On a failure the return is %NULL. 671 * 672 * Buffers may only be allocated from interrupts using a @gfp_mask of 673 * %GFP_ATOMIC. 674 */ 675 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 676 int flags, int node) 677 { 678 struct sk_buff *skb = NULL; 679 struct kmem_cache *cache; 680 u8 *data; 681 682 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 683 gfp_mask |= __GFP_MEMALLOC; 684 685 if (flags & SKB_ALLOC_FCLONE) { 686 cache = net_hotdata.skbuff_fclone_cache; 687 goto fallback; 688 } 689 cache = net_hotdata.skbuff_cache; 690 if (unlikely(node != NUMA_NO_NODE && node != numa_mem_id())) 691 goto fallback; 692 693 if (flags & SKB_ALLOC_NAPI) { 694 skb = napi_skb_cache_get(true); 695 if (unlikely(!skb)) 696 return NULL; 697 } else if (!in_hardirq() && !irqs_disabled()) { 698 local_bh_disable(); 699 skb = napi_skb_cache_get(false); 700 local_bh_enable(); 701 } 702 703 if (!skb) { 704 fallback: 705 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 706 if (unlikely(!skb)) 707 return NULL; 708 } 709 skbuff_clear(skb); 710 711 /* We do our best to align skb_shared_info on a separate cache 712 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 713 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 714 * Both skb->head and skb_shared_info are cache line aligned. 715 */ 716 data = kmalloc_reserve(&size, gfp_mask, node, skb); 717 if (unlikely(!data)) 718 goto nodata; 719 /* kmalloc_size_roundup() might give us more room than requested. 720 * Put skb_shared_info exactly at the end of allocated zone, 721 * to allow max possible filling before reallocation. 722 */ 723 __finalize_skb_around(skb, data, size); 724 725 if (flags & SKB_ALLOC_FCLONE) { 726 struct sk_buff_fclones *fclones; 727 728 fclones = container_of(skb, struct sk_buff_fclones, skb1); 729 730 /* skb->fclone is a 2bits field. 731 * Replace expensive RMW (skb->fclone = SKB_FCLONE_ORIG) 732 * with a single OR. 733 */ 734 BUILD_BUG_ON(SKB_FCLONE_UNAVAILABLE != 0); 735 DEBUG_NET_WARN_ON_ONCE(skb->fclone != SKB_FCLONE_UNAVAILABLE); 736 skb->fclone |= SKB_FCLONE_ORIG; 737 738 refcount_set(&fclones->fclone_ref, 1); 739 } 740 741 return skb; 742 743 nodata: 744 kmem_cache_free(cache, skb); 745 return NULL; 746 } 747 EXPORT_SYMBOL(__alloc_skb); 748 749 /** 750 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 751 * @dev: network device to receive on 752 * @len: length to allocate 753 * @gfp_mask: get_free_pages mask, passed to alloc_skb 754 * 755 * Allocate a new &sk_buff and assign it a usage count of one. The 756 * buffer has NET_SKB_PAD headroom built in. Users should allocate 757 * the headroom they think they need without accounting for the 758 * built in space. The built in space is used for optimisations. 759 * 760 * %NULL is returned if there is no free memory. 761 */ 762 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 763 gfp_t gfp_mask) 764 { 765 struct page_frag_cache *nc; 766 struct sk_buff *skb; 767 bool pfmemalloc; 768 void *data; 769 770 len += NET_SKB_PAD; 771 772 /* If requested length is either too small or too big, 773 * we use kmalloc() for skb->head allocation. 774 */ 775 if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) || 776 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 777 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 778 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 779 if (!skb) 780 goto skb_fail; 781 goto skb_success; 782 } 783 784 len = SKB_HEAD_ALIGN(len); 785 786 if (sk_memalloc_socks()) 787 gfp_mask |= __GFP_MEMALLOC; 788 789 if (in_hardirq() || irqs_disabled()) { 790 nc = this_cpu_ptr(&netdev_alloc_cache); 791 data = page_frag_alloc(nc, len, gfp_mask); 792 pfmemalloc = page_frag_cache_is_pfmemalloc(nc); 793 } else { 794 local_bh_disable(); 795 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 796 797 nc = this_cpu_ptr(&napi_alloc_cache.page); 798 data = page_frag_alloc(nc, len, gfp_mask); 799 pfmemalloc = page_frag_cache_is_pfmemalloc(nc); 800 801 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 802 local_bh_enable(); 803 } 804 805 if (unlikely(!data)) 806 return NULL; 807 808 skb = __build_skb(data, len); 809 if (unlikely(!skb)) { 810 skb_free_frag(data); 811 return NULL; 812 } 813 814 if (pfmemalloc) 815 skb->pfmemalloc = 1; 816 skb->head_frag = 1; 817 818 skb_success: 819 skb_reserve(skb, NET_SKB_PAD); 820 skb->dev = dev; 821 822 skb_fail: 823 return skb; 824 } 825 EXPORT_SYMBOL(__netdev_alloc_skb); 826 827 /** 828 * napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 829 * @napi: napi instance this buffer was allocated for 830 * @len: length to allocate 831 * 832 * Allocate a new sk_buff for use in NAPI receive. This buffer will 833 * attempt to allocate the head from a special reserved region used 834 * only for NAPI Rx allocation. By doing this we can save several 835 * CPU cycles by avoiding having to disable and re-enable IRQs. 836 * 837 * %NULL is returned if there is no free memory. 838 */ 839 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int len) 840 { 841 gfp_t gfp_mask = GFP_ATOMIC | __GFP_NOWARN; 842 struct napi_alloc_cache *nc; 843 struct sk_buff *skb; 844 bool pfmemalloc; 845 void *data; 846 847 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 848 len += NET_SKB_PAD + NET_IP_ALIGN; 849 850 /* If requested length is either too small or too big, 851 * we use kmalloc() for skb->head allocation. 852 */ 853 if (len <= SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) || 854 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 855 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 856 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 857 NUMA_NO_NODE); 858 if (!skb) 859 goto skb_fail; 860 goto skb_success; 861 } 862 863 len = SKB_HEAD_ALIGN(len); 864 865 if (sk_memalloc_socks()) 866 gfp_mask |= __GFP_MEMALLOC; 867 868 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 869 nc = this_cpu_ptr(&napi_alloc_cache); 870 871 data = page_frag_alloc(&nc->page, len, gfp_mask); 872 pfmemalloc = page_frag_cache_is_pfmemalloc(&nc->page); 873 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 874 875 if (unlikely(!data)) 876 return NULL; 877 878 skb = __napi_build_skb(data, len); 879 if (unlikely(!skb)) { 880 skb_free_frag(data); 881 return NULL; 882 } 883 884 if (pfmemalloc) 885 skb->pfmemalloc = 1; 886 skb->head_frag = 1; 887 888 skb_success: 889 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 890 skb->dev = napi->dev; 891 892 skb_fail: 893 return skb; 894 } 895 EXPORT_SYMBOL(napi_alloc_skb); 896 897 898 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 899 unsigned int truesize) 900 { 901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 902 903 DEBUG_NET_WARN_ON_ONCE(size > truesize); 904 905 skb_frag_size_add(frag, size); 906 skb->len += size; 907 skb->data_len += size; 908 skb->truesize += truesize; 909 } 910 EXPORT_SYMBOL(skb_coalesce_rx_frag); 911 912 static void skb_drop_list(struct sk_buff **listp) 913 { 914 kfree_skb_list(*listp); 915 *listp = NULL; 916 } 917 918 static inline void skb_drop_fraglist(struct sk_buff *skb) 919 { 920 skb_drop_list(&skb_shinfo(skb)->frag_list); 921 } 922 923 static void skb_clone_fraglist(struct sk_buff *skb) 924 { 925 struct sk_buff *list; 926 927 skb_walk_frags(skb, list) 928 skb_get(list); 929 } 930 931 /** 932 * skb_pp_cow_data() - copy skb data into page-pool backed storage 933 * @pool: page pool to allocate from 934 * @pskb: pointer to skb pointer, replaced with the copied skb on success 935 * @headroom: headroom to reserve in the copied skb 936 * 937 * skb_copy_bits() handles both frags[] and frag_list input. If the copied 938 * skb remains non-linear, it uses frags[], which is the representation used 939 * by XDP multi-buffer. 940 * 941 * Return: 0 on success or a negative errno on failure. 942 */ 943 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, 944 unsigned int headroom) 945 { 946 #if IS_ENABLED(CONFIG_PAGE_POOL) 947 u32 size, truesize, len, max_head_size, off; 948 struct sk_buff *skb = *pskb, *nskb; 949 int err, i, head_off; 950 void *data; 951 952 max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); 953 if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) 954 return -ENOMEM; 955 956 size = min_t(u32, skb->len, max_head_size); 957 truesize = SKB_HEAD_ALIGN(size) + headroom; 958 data = page_pool_dev_alloc_va(pool, &truesize); 959 if (!data) 960 return -ENOMEM; 961 962 nskb = napi_build_skb(data, truesize); 963 if (!nskb) { 964 page_pool_free_va(pool, data, true); 965 return -ENOMEM; 966 } 967 968 skb_reserve(nskb, headroom); 969 skb_copy_header(nskb, skb); 970 nskb->mac_len = skb->mac_len; 971 skb_mark_for_recycle(nskb); 972 973 err = skb_copy_bits(skb, 0, nskb->data, size); 974 if (err) { 975 consume_skb(nskb); 976 return err; 977 } 978 skb_put(nskb, size); 979 980 head_off = skb_headroom(nskb) - skb_headroom(skb); 981 skb_headers_offset_update(nskb, head_off); 982 983 off = size; 984 len = skb->len - off; 985 for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) { 986 struct page *page; 987 u32 page_off; 988 989 size = min_t(u32, len, PAGE_SIZE); 990 truesize = size; 991 992 page = page_pool_dev_alloc(pool, &page_off, &truesize); 993 if (!page) { 994 consume_skb(nskb); 995 return -ENOMEM; 996 } 997 998 skb_add_rx_frag(nskb, i, page, page_off, size, truesize); 999 err = skb_copy_bits(skb, off, page_address(page) + page_off, 1000 size); 1001 if (err) { 1002 consume_skb(nskb); 1003 return err; 1004 } 1005 1006 len -= size; 1007 off += size; 1008 } 1009 1010 consume_skb(skb); 1011 *pskb = nskb; 1012 1013 return 0; 1014 #else 1015 return -EOPNOTSUPP; 1016 #endif 1017 } 1018 EXPORT_SYMBOL(skb_pp_cow_data); 1019 1020 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, 1021 const struct bpf_prog *prog) 1022 { 1023 if (!prog->aux->xdp_has_frags) 1024 return -EINVAL; 1025 1026 return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM); 1027 } 1028 EXPORT_SYMBOL(skb_cow_data_for_xdp); 1029 1030 #if IS_ENABLED(CONFIG_PAGE_POOL) 1031 /** 1032 * napi_pp_put_page - release a page_pool reference on a netmem 1033 * @netmem: netmem to unreference 1034 * 1035 * Drops a page_pool reference (pp_ref_count) on @netmem if it belongs to a 1036 * page_pool. Counterpart to napi_pp_get_page(). 1037 * 1038 * Return: true if @netmem belongs to a page_pool, false otherwise. 1039 */ 1040 bool napi_pp_put_page(netmem_ref netmem) 1041 { 1042 netmem = netmem_compound_head(netmem); 1043 1044 if (unlikely(!netmem_is_pp(netmem))) 1045 return false; 1046 1047 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false); 1048 1049 return true; 1050 } 1051 EXPORT_SYMBOL(napi_pp_put_page); 1052 1053 /** 1054 * napi_pp_get_page - acquire a page_pool reference on a netmem 1055 * @netmem: netmem to reference 1056 * 1057 * Acquires a page_pool reference (pp_ref_count) on @netmem if it belongs to a 1058 * page_pool. Counterpart to napi_pp_put_page(). 1059 * 1060 * Return: true if @netmem belongs to a page_pool, false otherwise. 1061 */ 1062 bool napi_pp_get_page(netmem_ref netmem) 1063 { 1064 netmem = netmem_compound_head(netmem); 1065 1066 if (unlikely(!netmem_is_pp(netmem))) 1067 return false; 1068 1069 page_pool_ref_netmem(netmem); 1070 1071 return true; 1072 } 1073 EXPORT_SYMBOL(napi_pp_get_page); 1074 #endif 1075 1076 static bool skb_pp_recycle(struct sk_buff *skb, void *data) 1077 { 1078 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) 1079 return false; 1080 return napi_pp_put_page(page_to_netmem(virt_to_page(data))); 1081 } 1082 1083 static void skb_kfree_head(void *head) 1084 { 1085 kfree(head); 1086 } 1087 1088 static void skb_free_head(struct sk_buff *skb) 1089 { 1090 unsigned char *head = skb->head; 1091 1092 if (skb->head_frag) { 1093 if (skb_pp_recycle(skb, head)) 1094 return; 1095 skb_free_frag(head); 1096 } else { 1097 skb_kfree_head(head); 1098 } 1099 } 1100 1101 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason) 1102 { 1103 struct skb_shared_info *shinfo = skb_shinfo(skb); 1104 int i; 1105 1106 if (!skb_data_unref(skb, shinfo)) 1107 return; 1108 1109 if (skb_zcopy(skb)) { 1110 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; 1111 1112 skb_zcopy_clear(skb, true); 1113 if (skip_unref) 1114 goto free_head; 1115 } 1116 1117 for (i = 0; i < shinfo->nr_frags; i++) 1118 skb_netmem_unref(skb_frag_netmem(&shinfo->frags[i]), 1119 skb->pp_recycle); 1120 1121 free_head: 1122 if (shinfo->frag_list) 1123 kfree_skb_list_reason(shinfo->frag_list, reason); 1124 1125 skb_free_head(skb); 1126 } 1127 1128 /* 1129 * Free an skbuff by memory without cleaning the state. 1130 */ 1131 static void kfree_skbmem(struct sk_buff *skb) 1132 { 1133 struct sk_buff_fclones *fclones; 1134 1135 switch (skb->fclone) { 1136 case SKB_FCLONE_UNAVAILABLE: 1137 kmem_cache_free(net_hotdata.skbuff_cache, skb); 1138 return; 1139 1140 case SKB_FCLONE_ORIG: 1141 fclones = container_of(skb, struct sk_buff_fclones, skb1); 1142 1143 /* We usually free the clone (TX completion) before original skb 1144 * This test would have no chance to be true for the clone, 1145 * while here, branch prediction will be good. 1146 */ 1147 if (refcount_read(&fclones->fclone_ref) == 1) 1148 goto fastpath; 1149 break; 1150 1151 default: /* SKB_FCLONE_CLONE */ 1152 fclones = container_of(skb, struct sk_buff_fclones, skb2); 1153 break; 1154 } 1155 if (!refcount_dec_and_test(&fclones->fclone_ref)) 1156 return; 1157 fastpath: 1158 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones); 1159 } 1160 1161 void skb_release_head_state(struct sk_buff *skb) 1162 { 1163 skb_dst_drop(skb); 1164 if (skb->destructor) { 1165 DEBUG_NET_WARN_ON_ONCE(in_hardirq()); 1166 #ifdef CONFIG_INET 1167 INDIRECT_CALL_4(skb->destructor, 1168 tcp_wfree, __sock_wfree, sock_wfree, 1169 xsk_destruct_skb, 1170 skb); 1171 #else 1172 INDIRECT_CALL_2(skb->destructor, 1173 sock_wfree, xsk_destruct_skb, 1174 skb); 1175 1176 #endif 1177 skb->destructor = NULL; 1178 skb->sk = NULL; 1179 } 1180 nf_reset_ct(skb); 1181 skb_ext_reset(skb); 1182 } 1183 1184 /* Free everything but the sk_buff shell. */ 1185 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason) 1186 { 1187 skb_release_head_state(skb); 1188 if (likely(skb->head)) 1189 skb_release_data(skb, reason); 1190 } 1191 1192 /** 1193 * __kfree_skb - private function 1194 * @skb: buffer 1195 * 1196 * Free an sk_buff. Release anything attached to the buffer. 1197 * Clean the state. This is an internal helper function. Users should 1198 * always call kfree_skb 1199 */ 1200 1201 void __kfree_skb(struct sk_buff *skb) 1202 { 1203 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED); 1204 kfree_skbmem(skb); 1205 } 1206 EXPORT_SYMBOL(__kfree_skb); 1207 1208 static __always_inline 1209 bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, 1210 enum skb_drop_reason reason) 1211 { 1212 if (unlikely(!skb_unref(skb))) 1213 return false; 1214 1215 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET || 1216 u32_get_bits(reason, 1217 SKB_DROP_REASON_SUBSYS_MASK) >= 1218 SKB_DROP_REASON_SUBSYS_NUM); 1219 1220 if (reason == SKB_CONSUMED) 1221 trace_consume_skb(skb, __builtin_return_address(0)); 1222 else 1223 trace_kfree_skb(skb, __builtin_return_address(0), reason, sk); 1224 return true; 1225 } 1226 1227 /** 1228 * sk_skb_reason_drop - free an sk_buff with special reason 1229 * @sk: the socket to receive @skb, or NULL if not applicable 1230 * @skb: buffer to free 1231 * @reason: reason why this skb is dropped 1232 * 1233 * Drop a reference to the buffer and free it if the usage count has hit 1234 * zero. Meanwhile, pass the receiving socket and drop reason to 1235 * 'kfree_skb' tracepoint. 1236 */ 1237 void __fix_address 1238 sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, 1239 enum skb_drop_reason reason) 1240 { 1241 if (__sk_skb_reason_drop(sk, skb, reason)) 1242 __kfree_skb(skb); 1243 } 1244 EXPORT_SYMBOL(sk_skb_reason_drop); 1245 1246 #define KFREE_SKB_BULK_SIZE 16 1247 1248 struct skb_free_array { 1249 unsigned int skb_count; 1250 void *skb_array[KFREE_SKB_BULK_SIZE]; 1251 }; 1252 1253 static void kfree_skb_add_bulk(struct sk_buff *skb, 1254 struct skb_free_array *sa, 1255 enum skb_drop_reason reason) 1256 { 1257 /* if SKB is a clone, don't handle this case */ 1258 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) { 1259 __kfree_skb(skb); 1260 return; 1261 } 1262 1263 skb_release_all(skb, reason); 1264 sa->skb_array[sa->skb_count++] = skb; 1265 1266 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { 1267 kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE, 1268 sa->skb_array); 1269 sa->skb_count = 0; 1270 } 1271 } 1272 1273 void __fix_address 1274 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) 1275 { 1276 struct skb_free_array sa; 1277 1278 sa.skb_count = 0; 1279 1280 while (segs) { 1281 struct sk_buff *next = segs->next; 1282 1283 if (__sk_skb_reason_drop(NULL, segs, reason)) { 1284 skb_poison_list(segs); 1285 kfree_skb_add_bulk(segs, &sa, reason); 1286 } 1287 1288 segs = next; 1289 } 1290 1291 if (sa.skb_count) 1292 kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array); 1293 } 1294 EXPORT_SYMBOL(kfree_skb_list_reason); 1295 1296 /* Dump skb information and contents. 1297 * 1298 * Must only be called from net_ratelimit()-ed paths. 1299 * 1300 * Dumps whole packets if full_pkt, only headers otherwise. 1301 */ 1302 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 1303 { 1304 struct skb_shared_info *sh = skb_shinfo(skb); 1305 struct net_device *dev = skb->dev; 1306 struct sock *sk = skb->sk; 1307 struct sk_buff *list_skb; 1308 bool has_mac, has_trans; 1309 int headroom, tailroom; 1310 int i, len, seg_len; 1311 1312 if (full_pkt) 1313 len = skb->len; 1314 else 1315 len = min_t(int, skb->len, MAX_HEADER + 128); 1316 1317 headroom = skb_headroom(skb); 1318 tailroom = skb_tailroom(skb); 1319 1320 has_mac = skb_mac_header_was_set(skb); 1321 has_trans = skb_transport_header_was_set(skb); 1322 1323 printk("%sskb len=%u data_len=%u headroom=%u headlen=%u tailroom=%u\n" 1324 "end-tail=%u mac=(%d,%d) mac_len=%u net=(%d,%d) trans=%d\n" 1325 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 1326 "csum(0x%x start=%u offset=%u ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 1327 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n" 1328 "priority=0x%x mark=0x%x alloc_cpu=%u vlan_all=0x%x\n" 1329 "encapsulation=%d inner(proto=0x%04x, mac=%u, net=%u, trans=%u)\n", 1330 level, skb->len, skb->data_len, headroom, skb_headlen(skb), 1331 tailroom, skb->end - skb->tail, 1332 has_mac ? skb->mac_header : -1, 1333 has_mac ? skb_mac_header_len(skb) : -1, 1334 skb->mac_len, 1335 skb->network_header, 1336 has_trans ? skb_network_header_len(skb) : -1, 1337 has_trans ? skb->transport_header : -1, 1338 sh->tx_flags, sh->nr_frags, 1339 sh->gso_size, sh->gso_type, sh->gso_segs, 1340 skb->csum, skb->csum_start, skb->csum_offset, skb->ip_summed, 1341 skb->csum_complete_sw, skb->csum_valid, skb->csum_level, 1342 skb->hash, skb->sw_hash, skb->l4_hash, 1343 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif, 1344 skb->priority, skb->mark, skb->alloc_cpu, skb->vlan_all, 1345 skb->encapsulation, skb->inner_protocol, skb->inner_mac_header, 1346 skb->inner_network_header, skb->inner_transport_header); 1347 1348 if (dev) 1349 printk("%sdev name=%s feat=%pNF\n", 1350 level, dev->name, &dev->features); 1351 if (sk) 1352 printk("%ssk family=%hu type=%u proto=%u\n", 1353 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 1354 1355 if (full_pkt && headroom) 1356 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 1357 16, 1, skb->head, headroom, false); 1358 1359 seg_len = min_t(int, skb_headlen(skb), len); 1360 if (seg_len) 1361 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 1362 16, 1, skb->data, seg_len, false); 1363 len -= seg_len; 1364 1365 if (full_pkt && tailroom) 1366 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 1367 16, 1, skb_tail_pointer(skb), tailroom, false); 1368 1369 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 1370 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1371 u32 p_off, p_len, copied; 1372 struct page *p; 1373 u8 *vaddr; 1374 1375 if (skb_frag_is_net_iov(frag)) { 1376 printk("%sskb frag %d: not readable\n", level, i); 1377 len -= skb_frag_size(frag); 1378 if (!len) 1379 break; 1380 continue; 1381 } 1382 1383 skb_frag_foreach_page(frag, skb_frag_off(frag), 1384 skb_frag_size(frag), p, p_off, p_len, 1385 copied) { 1386 seg_len = min_t(int, p_len, len); 1387 vaddr = kmap_atomic(p); 1388 print_hex_dump(level, "skb frag: ", 1389 DUMP_PREFIX_OFFSET, 1390 16, 1, vaddr + p_off, seg_len, false); 1391 kunmap_atomic(vaddr); 1392 len -= seg_len; 1393 if (!len) 1394 break; 1395 } 1396 } 1397 1398 if (full_pkt && skb_has_frag_list(skb)) { 1399 printk("skb fraglist:\n"); 1400 skb_walk_frags(skb, list_skb) 1401 skb_dump(level, list_skb, true); 1402 } 1403 } 1404 EXPORT_SYMBOL(skb_dump); 1405 1406 /** 1407 * skb_tx_error - report an sk_buff xmit error 1408 * @skb: buffer that triggered an error 1409 * 1410 * Report xmit error if a device callback is tracking this skb. 1411 * skb must be freed afterwards. 1412 * 1413 * Does nothing for a cloned skb: the zerocopy state lives in 1414 * skb_shinfo(), which the clones share. 1415 */ 1416 void skb_tx_error(struct sk_buff *skb) 1417 { 1418 if (skb && !skb_cloned(skb)) { 1419 skb_zcopy_downgrade_managed(skb); 1420 skb_zcopy_clear(skb, true); 1421 } 1422 } 1423 EXPORT_SYMBOL(skb_tx_error); 1424 1425 #ifdef CONFIG_TRACEPOINTS 1426 /** 1427 * consume_skb - free an skbuff 1428 * @skb: buffer to free 1429 * 1430 * Drop a ref to the buffer and free it if the usage count has hit zero 1431 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 1432 * is being dropped after a failure and notes that 1433 */ 1434 void consume_skb(struct sk_buff *skb) 1435 { 1436 if (!skb_unref(skb)) 1437 return; 1438 1439 trace_consume_skb(skb, __builtin_return_address(0)); 1440 __kfree_skb(skb); 1441 } 1442 EXPORT_SYMBOL(consume_skb); 1443 #endif 1444 1445 /** 1446 * __consume_stateless_skb - free an skbuff, assuming it is stateless 1447 * @skb: buffer to free 1448 * 1449 * Alike consume_skb(), but this variant assumes that this is the last 1450 * skb reference and all the head states have been already dropped 1451 */ 1452 void __consume_stateless_skb(struct sk_buff *skb) 1453 { 1454 trace_consume_skb(skb, __builtin_return_address(0)); 1455 skb_release_data(skb, SKB_CONSUMED); 1456 kfree_skbmem(skb); 1457 } 1458 1459 static void napi_skb_cache_put(struct sk_buff *skb) 1460 { 1461 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 1462 1463 if (!kasan_mempool_poison_object(skb)) 1464 return; 1465 1466 local_lock_nested_bh(&napi_alloc_cache.bh_lock); 1467 nc->skb_cache[nc->skb_count++] = skb; 1468 1469 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 1470 u32 i, remaining = NAPI_SKB_CACHE_SIZE - NAPI_SKB_CACHE_FREE; 1471 1472 for (i = remaining; i < NAPI_SKB_CACHE_SIZE; i++) 1473 kasan_mempool_unpoison_object(nc->skb_cache[i], 1474 skbuff_cache_size); 1475 1476 kmem_cache_free_bulk(net_hotdata.skbuff_cache, 1477 NAPI_SKB_CACHE_FREE, 1478 nc->skb_cache + remaining); 1479 nc->skb_count = remaining; 1480 } 1481 local_unlock_nested_bh(&napi_alloc_cache.bh_lock); 1482 } 1483 1484 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason) 1485 { 1486 skb_release_all(skb, reason); 1487 napi_skb_cache_put(skb); 1488 } 1489 1490 void napi_skb_free_stolen_head(struct sk_buff *skb) 1491 { 1492 if (unlikely(skb->slow_gro)) { 1493 nf_reset_ct(skb); 1494 skb_dst_drop(skb); 1495 skb_ext_put(skb); 1496 skb_orphan(skb); 1497 skb->slow_gro = 0; 1498 } 1499 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1500 kfree_skbmem(skb); 1501 return; 1502 } 1503 napi_skb_cache_put(skb); 1504 } 1505 1506 /** 1507 * napi_consume_skb() - consume skb in NAPI context, try to feed skb cache 1508 * @skb: buffer to free 1509 * @budget: NAPI budget 1510 * 1511 * Non-zero @budget must come from the @budget argument passed by the core 1512 * to a NAPI poll function. Note that core may pass budget of 0 to NAPI poll 1513 * for example when polling for netpoll / netconsole. 1514 * 1515 * Passing @budget of 0 is safe from any context, it turns this function 1516 * into dev_consume_skb_any(). 1517 */ 1518 void napi_consume_skb(struct sk_buff *skb, int budget) 1519 { 1520 if (unlikely(!budget || !skb)) { 1521 dev_consume_skb_any(skb); 1522 return; 1523 } 1524 1525 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 1526 1527 if (!static_branch_unlikely(&skb_defer_disable_key) && 1528 skb->alloc_cpu != smp_processor_id() && !skb_shared(skb)) { 1529 skb_release_head_state(skb); 1530 return skb_attempt_defer_free(skb); 1531 } 1532 1533 if (!skb_unref(skb)) 1534 return; 1535 1536 /* if reaching here SKB is ready to free */ 1537 trace_consume_skb(skb, __builtin_return_address(0)); 1538 1539 /* if SKB is a clone, don't handle this case */ 1540 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 1541 __kfree_skb(skb); 1542 return; 1543 } 1544 1545 skb_release_all(skb, SKB_CONSUMED); 1546 napi_skb_cache_put(skb); 1547 } 1548 EXPORT_SYMBOL(napi_consume_skb); 1549 1550 /* Make sure a field is contained by headers group */ 1551 #define CHECK_SKB_FIELD(field) \ 1552 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ 1553 offsetof(struct sk_buff, headers.field)); \ 1554 1555 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 1556 { 1557 new->tstamp = old->tstamp; 1558 /* We do not copy old->sk */ 1559 new->dev = old->dev; 1560 memcpy(new->cb, old->cb, sizeof(old->cb)); 1561 skb_dst_copy(new, old); 1562 __skb_ext_copy(new, old); 1563 __nf_copy(new, old, false); 1564 1565 /* Note : this field could be in the headers group. 1566 * It is not yet because we do not want to have a 16 bit hole 1567 */ 1568 new->queue_mapping = old->queue_mapping; 1569 1570 memcpy(&new->headers, &old->headers, sizeof(new->headers)); 1571 CHECK_SKB_FIELD(protocol); 1572 CHECK_SKB_FIELD(csum); 1573 CHECK_SKB_FIELD(hash); 1574 CHECK_SKB_FIELD(priority); 1575 CHECK_SKB_FIELD(skb_iif); 1576 CHECK_SKB_FIELD(vlan_proto); 1577 CHECK_SKB_FIELD(vlan_tci); 1578 CHECK_SKB_FIELD(transport_header); 1579 CHECK_SKB_FIELD(network_header); 1580 CHECK_SKB_FIELD(mac_header); 1581 CHECK_SKB_FIELD(inner_protocol); 1582 CHECK_SKB_FIELD(inner_transport_header); 1583 CHECK_SKB_FIELD(inner_network_header); 1584 CHECK_SKB_FIELD(inner_mac_header); 1585 CHECK_SKB_FIELD(mark); 1586 #ifdef CONFIG_NETWORK_SECMARK 1587 CHECK_SKB_FIELD(secmark); 1588 #endif 1589 #ifdef CONFIG_NET_RX_BUSY_POLL 1590 CHECK_SKB_FIELD(napi_id); 1591 #endif 1592 #ifdef CONFIG_XPS 1593 CHECK_SKB_FIELD(sender_cpu); 1594 #endif 1595 #ifdef CONFIG_NET_SCHED 1596 CHECK_SKB_FIELD(tc_index); 1597 #endif 1598 1599 } 1600 1601 /* 1602 * You should not add any new code to this function. Add it to 1603 * __copy_skb_header above instead. 1604 */ 1605 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1606 { 1607 #define C(x) n->x = skb->x 1608 1609 n->next = n->prev = NULL; 1610 n->sk = NULL; 1611 __copy_skb_header(n, skb); 1612 1613 C(len); 1614 C(data_len); 1615 C(mac_len); 1616 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1617 n->cloned = 1; 1618 n->nohdr = 0; 1619 n->peeked = 0; 1620 C(pfmemalloc); 1621 C(pp_recycle); 1622 n->destructor = NULL; 1623 C(alloc_cpu); 1624 C(tail); 1625 C(end); 1626 C(head); 1627 C(head_frag); 1628 C(data); 1629 C(truesize); 1630 refcount_set(&n->users, 1); 1631 1632 atomic_inc(&(skb_shinfo(skb)->dataref)); 1633 skb->cloned = 1; 1634 1635 return n; 1636 #undef C 1637 } 1638 1639 /** 1640 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1641 * @first: first sk_buff of the msg 1642 */ 1643 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1644 { 1645 struct sk_buff *n; 1646 1647 n = alloc_skb(0, GFP_ATOMIC); 1648 if (!n) 1649 return NULL; 1650 1651 n->len = first->len; 1652 n->data_len = first->len; 1653 n->truesize = first->truesize; 1654 1655 skb_shinfo(n)->frag_list = first; 1656 1657 __copy_skb_header(n, first); 1658 n->destructor = NULL; 1659 1660 return n; 1661 } 1662 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1663 1664 /** 1665 * skb_morph - morph one skb into another 1666 * @dst: the skb to receive the contents 1667 * @src: the skb to supply the contents 1668 * 1669 * This is identical to skb_clone except that the target skb is 1670 * supplied by the user. 1671 * 1672 * The target skb is returned upon exit. 1673 */ 1674 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1675 { 1676 skb_release_all(dst, SKB_CONSUMED); 1677 return __skb_clone(dst, src); 1678 } 1679 EXPORT_SYMBOL_GPL(skb_morph); 1680 1681 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1682 { 1683 unsigned long max_pg, num_pg, new_pg, old_pg, rlim; 1684 struct user_struct *user; 1685 1686 if (capable(CAP_IPC_LOCK) || !size) 1687 return 0; 1688 1689 rlim = rlimit(RLIMIT_MEMLOCK); 1690 if (rlim == RLIM_INFINITY) 1691 return 0; 1692 1693 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1694 max_pg = rlim >> PAGE_SHIFT; 1695 user = mmp->user ? : current_user(); 1696 1697 old_pg = atomic_long_read(&user->locked_vm); 1698 do { 1699 new_pg = old_pg + num_pg; 1700 if (new_pg > max_pg) 1701 return -ENOBUFS; 1702 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg)); 1703 1704 if (!mmp->user) { 1705 mmp->user = get_uid(user); 1706 mmp->num_pg = num_pg; 1707 } else { 1708 mmp->num_pg += num_pg; 1709 } 1710 1711 return 0; 1712 } 1713 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1714 1715 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1716 { 1717 if (mmp->user) { 1718 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1719 free_uid(mmp->user); 1720 } 1721 } 1722 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1723 1724 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size, 1725 bool devmem) 1726 { 1727 struct ubuf_info_msgzc *uarg; 1728 struct sk_buff *skb; 1729 1730 WARN_ON_ONCE(!in_task()); 1731 1732 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1733 if (!skb) 1734 return NULL; 1735 1736 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1737 uarg = (void *)skb->cb; 1738 uarg->mmp.user = NULL; 1739 1740 if (likely(!devmem) && mm_account_pinned_pages(&uarg->mmp, size)) { 1741 kfree_skb(skb); 1742 return NULL; 1743 } 1744 1745 uarg->ubuf.ops = &msg_zerocopy_ubuf_ops; 1746 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1747 uarg->len = 1; 1748 uarg->bytelen = size; 1749 uarg->zerocopy = 1; 1750 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 1751 refcount_set(&uarg->ubuf.refcnt, 1); 1752 sock_hold(sk); 1753 1754 return &uarg->ubuf; 1755 } 1756 1757 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) 1758 { 1759 return container_of((void *)uarg, struct sk_buff, cb); 1760 } 1761 1762 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1763 struct ubuf_info *uarg, bool devmem) 1764 { 1765 if (uarg) { 1766 struct ubuf_info_msgzc *uarg_zc; 1767 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1768 u32 bytelen, next; 1769 1770 /* there might be non MSG_ZEROCOPY users */ 1771 if (uarg->ops != &msg_zerocopy_ubuf_ops) 1772 return NULL; 1773 1774 /* realloc only when socket is locked (TCP, UDP cork), 1775 * so uarg->len and sk_zckey access is serialized 1776 */ 1777 if (!sock_owned_by_user(sk)) { 1778 WARN_ON_ONCE(1); 1779 return NULL; 1780 } 1781 1782 uarg_zc = uarg_to_msgzc(uarg); 1783 bytelen = uarg_zc->bytelen + size; 1784 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1785 /* TCP can create new skb to attach new uarg */ 1786 if (sk->sk_type == SOCK_STREAM) 1787 goto new_alloc; 1788 return NULL; 1789 } 1790 1791 next = (u32)atomic_read(&sk->sk_zckey); 1792 if ((u32)(uarg_zc->id + uarg_zc->len) == next) { 1793 if (likely(!devmem) && 1794 mm_account_pinned_pages(&uarg_zc->mmp, size)) 1795 return NULL; 1796 uarg_zc->len++; 1797 uarg_zc->bytelen = bytelen; 1798 atomic_set(&sk->sk_zckey, ++next); 1799 1800 /* no extra ref when appending to datagram (MSG_MORE) */ 1801 if (sk->sk_type == SOCK_STREAM) 1802 net_zcopy_get(uarg); 1803 1804 return uarg; 1805 } 1806 } 1807 1808 new_alloc: 1809 return msg_zerocopy_alloc(sk, size, devmem); 1810 } 1811 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1812 1813 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1814 { 1815 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1816 u32 old_lo, old_hi; 1817 u64 sum_len; 1818 1819 old_lo = serr->ee.ee_info; 1820 old_hi = serr->ee.ee_data; 1821 sum_len = old_hi - old_lo + 1ULL + len; 1822 1823 if (sum_len >= (1ULL << 32)) 1824 return false; 1825 1826 if (lo != old_hi + 1) 1827 return false; 1828 1829 serr->ee.ee_data += len; 1830 return true; 1831 } 1832 1833 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) 1834 { 1835 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1836 struct sock_exterr_skb *serr; 1837 struct sock *sk = skb->sk; 1838 struct sk_buff_head *q; 1839 unsigned long flags; 1840 bool is_zerocopy; 1841 u32 lo, hi; 1842 u16 len; 1843 1844 mm_unaccount_pinned_pages(&uarg->mmp); 1845 1846 /* if !len, there was only 1 call, and it was aborted 1847 * so do not queue a completion notification 1848 */ 1849 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1850 goto release; 1851 1852 len = uarg->len; 1853 lo = uarg->id; 1854 hi = uarg->id + len - 1; 1855 is_zerocopy = uarg->zerocopy; 1856 1857 serr = SKB_EXT_ERR(skb); 1858 memset(serr, 0, sizeof(*serr)); 1859 serr->ee.ee_errno = 0; 1860 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1861 serr->ee.ee_data = hi; 1862 serr->ee.ee_info = lo; 1863 if (!is_zerocopy) 1864 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1865 1866 q = &sk->sk_error_queue; 1867 spin_lock_irqsave(&q->lock, flags); 1868 tail = skb_peek_tail(q); 1869 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1870 !skb_zerocopy_notify_extend(tail, lo, len)) { 1871 __skb_queue_tail(q, skb); 1872 skb = NULL; 1873 } 1874 spin_unlock_irqrestore(&q->lock, flags); 1875 1876 sk_error_report(sk); 1877 1878 release: 1879 consume_skb(skb); 1880 sock_put(sk); 1881 } 1882 1883 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg, 1884 bool success) 1885 { 1886 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); 1887 1888 uarg_zc->zerocopy = uarg_zc->zerocopy & success; 1889 1890 if (refcount_dec_and_test(&uarg->refcnt)) 1891 __msg_zerocopy_callback(uarg_zc); 1892 } 1893 1894 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1895 { 1896 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; 1897 1898 atomic_dec(&sk->sk_zckey); 1899 uarg_to_msgzc(uarg)->len--; 1900 1901 if (have_uref) 1902 msg_zerocopy_complete(NULL, uarg, true); 1903 } 1904 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1905 1906 const struct ubuf_info_ops msg_zerocopy_ubuf_ops = { 1907 .complete = msg_zerocopy_complete, 1908 }; 1909 EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops); 1910 1911 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1912 struct msghdr *msg, int len, 1913 struct ubuf_info *uarg, 1914 struct net_devmem_dmabuf_binding *binding) 1915 { 1916 unsigned int orig_truesize = skb->truesize; 1917 int err, orig_len = skb->len; 1918 1919 if (uarg->ops->link_skb) { 1920 err = uarg->ops->link_skb(skb, uarg); 1921 if (err) 1922 return err; 1923 } else { 1924 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1925 1926 /* An skb can only point to one uarg. This edge case happens 1927 * when TCP appends to an skb, but zerocopy_realloc triggered 1928 * a new alloc. 1929 */ 1930 if (orig_uarg && uarg != orig_uarg) 1931 return -EEXIST; 1932 } 1933 1934 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len, 1935 binding); 1936 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1937 u32 delta_truesize = skb->truesize - orig_truesize; 1938 struct sock *save_sk = skb->sk; 1939 1940 /* Streams do not free skb on error. Reset to prev state. */ 1941 iov_iter_revert(&msg->msg_iter, skb->len - orig_len); 1942 skb->sk = sk; 1943 if (!___pskb_trim(skb, orig_len)) { 1944 skb->truesize -= delta_truesize; 1945 sk_wmem_queued_add(sk, -delta_truesize); 1946 if (!skb_zcopy_pure(skb)) 1947 sk_mem_uncharge(sk, delta_truesize); 1948 } 1949 skb->sk = save_sk; 1950 return err; 1951 } 1952 1953 skb_zcopy_set(skb, uarg, NULL); 1954 return skb->len - orig_len; 1955 } 1956 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1957 1958 void __skb_zcopy_downgrade_managed(struct sk_buff *skb) 1959 { 1960 int i; 1961 1962 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; 1963 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1964 skb_frag_ref(skb, i); 1965 } 1966 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); 1967 1968 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1969 gfp_t gfp_mask) 1970 { 1971 if (skb_zcopy(orig)) { 1972 if (skb_zcopy(nskb)) { 1973 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1974 if (!gfp_mask) { 1975 WARN_ON_ONCE(1); 1976 return -ENOMEM; 1977 } 1978 if (skb_uarg(nskb) == skb_uarg(orig)) 1979 return 0; 1980 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1981 return -EIO; 1982 } 1983 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1984 } 1985 return 0; 1986 } 1987 1988 /** 1989 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1990 * @skb: the skb to modify 1991 * @gfp_mask: allocation priority 1992 * 1993 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1994 * It will copy all frags into kernel and drop the reference 1995 * to userspace pages. 1996 * 1997 * If this function is called from an interrupt gfp_mask() must be 1998 * %GFP_ATOMIC. 1999 * 2000 * Returns 0 on success or a negative error code on failure 2001 * to allocate kernel memory to copy to. 2002 */ 2003 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 2004 { 2005 int num_frags = skb_shinfo(skb)->nr_frags; 2006 struct page *page, *head = NULL; 2007 int i, order, psize, new_frags; 2008 u32 d_off; 2009 2010 if (!skb_frags_readable(skb)) 2011 return -EFAULT; 2012 2013 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 2014 return -EINVAL; 2015 2016 if (!num_frags) 2017 goto release; 2018 2019 /* We might have to allocate high order pages, so compute what minimum 2020 * page order is needed. 2021 */ 2022 order = 0; 2023 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb)) 2024 order++; 2025 psize = (PAGE_SIZE << order); 2026 2027 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order); 2028 for (i = 0; i < new_frags; i++) { 2029 page = alloc_pages(gfp_mask | __GFP_COMP, order); 2030 if (!page) { 2031 while (head) { 2032 struct page *next = (struct page *)page_private(head); 2033 put_page(head); 2034 head = next; 2035 } 2036 return -ENOMEM; 2037 } 2038 set_page_private(page, (unsigned long)head); 2039 head = page; 2040 } 2041 2042 page = head; 2043 d_off = 0; 2044 for (i = 0; i < num_frags; i++) { 2045 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2046 u32 p_off, p_len, copied; 2047 struct page *p; 2048 u8 *vaddr; 2049 2050 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 2051 p, p_off, p_len, copied) { 2052 u32 copy, done = 0; 2053 vaddr = kmap_atomic(p); 2054 2055 while (done < p_len) { 2056 if (d_off == psize) { 2057 d_off = 0; 2058 page = (struct page *)page_private(page); 2059 } 2060 copy = min_t(u32, psize - d_off, p_len - done); 2061 memcpy(page_address(page) + d_off, 2062 vaddr + p_off + done, copy); 2063 done += copy; 2064 d_off += copy; 2065 } 2066 kunmap_atomic(vaddr); 2067 } 2068 } 2069 2070 /* skb frags release userspace buffers */ 2071 for (i = 0; i < num_frags; i++) 2072 skb_frag_unref(skb, i); 2073 2074 /* skb frags point to kernel buffers */ 2075 for (i = 0; i < new_frags - 1; i++) { 2076 __skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize); 2077 head = (struct page *)page_private(head); 2078 } 2079 __skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0, 2080 d_off); 2081 skb_shinfo(skb)->nr_frags = new_frags; 2082 2083 release: 2084 skb_zcopy_clear(skb, false); 2085 return 0; 2086 } 2087 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 2088 2089 /** 2090 * skb_clone - duplicate an sk_buff 2091 * @skb: buffer to clone 2092 * @gfp_mask: allocation priority 2093 * 2094 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 2095 * copies share the same packet data but not structure. The new 2096 * buffer has a reference count of 1. If the allocation fails the 2097 * function returns %NULL otherwise the new buffer is returned. 2098 * 2099 * If this function is called from an interrupt gfp_mask() must be 2100 * %GFP_ATOMIC. 2101 */ 2102 2103 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 2104 { 2105 struct sk_buff_fclones *fclones = container_of(skb, 2106 struct sk_buff_fclones, 2107 skb1); 2108 struct sk_buff *n; 2109 2110 if (skb_orphan_frags(skb, gfp_mask)) 2111 return NULL; 2112 2113 if (skb->fclone == SKB_FCLONE_ORIG && 2114 refcount_read(&fclones->fclone_ref) == 1) { 2115 n = &fclones->skb2; 2116 refcount_set(&fclones->fclone_ref, 2); 2117 n->fclone = SKB_FCLONE_CLONE; 2118 } else { 2119 if (skb_pfmemalloc(skb)) 2120 gfp_mask |= __GFP_MEMALLOC; 2121 2122 n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask); 2123 if (!n) 2124 return NULL; 2125 2126 n->fclone = SKB_FCLONE_UNAVAILABLE; 2127 } 2128 2129 return __skb_clone(n, skb); 2130 } 2131 EXPORT_SYMBOL(skb_clone); 2132 2133 void skb_headers_offset_update(struct sk_buff *skb, int off) 2134 { 2135 /* Only adjust this if it actually is csum_start rather than csum */ 2136 if (skb->ip_summed == CHECKSUM_PARTIAL) 2137 skb->csum_start += off; 2138 /* {transport,network,mac}_header and tail are relative to skb->head */ 2139 skb->transport_header += off; 2140 skb->network_header += off; 2141 if (skb_mac_header_was_set(skb)) 2142 skb->mac_header += off; 2143 skb->inner_transport_header += off; 2144 skb->inner_network_header += off; 2145 skb->inner_mac_header += off; 2146 } 2147 EXPORT_SYMBOL(skb_headers_offset_update); 2148 2149 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 2150 { 2151 __copy_skb_header(new, old); 2152 2153 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 2154 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 2155 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 2156 } 2157 EXPORT_SYMBOL(skb_copy_header); 2158 2159 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 2160 { 2161 if (skb_pfmemalloc(skb)) 2162 return SKB_ALLOC_RX; 2163 return 0; 2164 } 2165 2166 /** 2167 * skb_copy - create private copy of an sk_buff 2168 * @skb: buffer to copy 2169 * @gfp_mask: allocation priority 2170 * 2171 * Make a copy of both an &sk_buff and its data. This is used when the 2172 * caller wishes to modify the data and needs a private copy of the 2173 * data to alter. Returns %NULL on failure or the pointer to the buffer 2174 * on success. The returned buffer has a reference count of 1. 2175 * 2176 * As by-product this function converts non-linear &sk_buff to linear 2177 * one, so that &sk_buff becomes completely private and caller is allowed 2178 * to modify all the data of returned buffer. This means that this 2179 * function is not recommended for use in circumstances when only 2180 * header is going to be modified. Use pskb_copy() instead. 2181 */ 2182 2183 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 2184 { 2185 struct sk_buff *n; 2186 unsigned int size; 2187 int headerlen; 2188 2189 if (!skb_frags_readable(skb)) 2190 return NULL; 2191 2192 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)) 2193 return NULL; 2194 2195 headerlen = skb_headroom(skb); 2196 size = skb_end_offset(skb) + skb->data_len; 2197 n = __alloc_skb(size, gfp_mask, 2198 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 2199 if (!n) 2200 return NULL; 2201 2202 /* Set the data pointer */ 2203 skb_reserve(n, headerlen); 2204 /* Set the tail pointer and length */ 2205 skb_put(n, skb->len); 2206 2207 if (unlikely(skb_copy_bits(skb, -headerlen, n->head, 2208 headerlen + skb->len))) { 2209 DEBUG_NET_WARN_ON_ONCE(1); 2210 kfree_skb(n); 2211 return NULL; 2212 } 2213 2214 skb_copy_header(n, skb); 2215 return n; 2216 } 2217 EXPORT_SYMBOL(skb_copy); 2218 2219 /** 2220 * __pskb_copy_fclone - create copy of an sk_buff with private head. 2221 * @skb: buffer to copy 2222 * @headroom: headroom of new skb 2223 * @gfp_mask: allocation priority 2224 * @fclone: if true allocate the copy of the skb from the fclone 2225 * cache instead of the head cache; it is recommended to set this 2226 * to true for the cases where the copy will likely be cloned 2227 * 2228 * Make a copy of both an &sk_buff and part of its data, located 2229 * in header. Fragmented data remain shared. This is used when 2230 * the caller wishes to modify only header of &sk_buff and needs 2231 * private copy of the header to alter. Returns %NULL on failure 2232 * or the pointer to the buffer on success. 2233 * The returned buffer has a reference count of 1. 2234 */ 2235 2236 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 2237 gfp_t gfp_mask, bool fclone) 2238 { 2239 unsigned int size = skb_headlen(skb) + headroom; 2240 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 2241 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 2242 2243 if (!n) 2244 goto out; 2245 2246 /* Set the data pointer */ 2247 skb_reserve(n, headroom); 2248 /* Set the tail pointer and length */ 2249 skb_put(n, skb_headlen(skb)); 2250 /* Copy the bytes */ 2251 skb_copy_from_linear_data(skb, n->data, n->len); 2252 2253 n->truesize += skb->data_len; 2254 n->data_len = skb->data_len; 2255 n->len = skb->len; 2256 2257 if (skb_shinfo(skb)->nr_frags) { 2258 int i; 2259 2260 if (skb_orphan_frags(skb, gfp_mask) || 2261 skb_zerocopy_clone(n, skb, gfp_mask)) { 2262 kfree_skb(n); 2263 n = NULL; 2264 goto out; 2265 } 2266 n->pp_recycle = skb->pp_recycle; 2267 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2268 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 2269 skb_frag_ref(n, i); 2270 } 2271 skb_shinfo(n)->nr_frags = i; 2272 skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 2273 } 2274 2275 if (skb_has_frag_list(skb)) { 2276 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 2277 skb_clone_fraglist(n); 2278 } 2279 2280 skb_copy_header(n, skb); 2281 out: 2282 return n; 2283 } 2284 EXPORT_SYMBOL(__pskb_copy_fclone); 2285 2286 /** 2287 * pskb_expand_head - reallocate header of &sk_buff 2288 * @skb: buffer to reallocate 2289 * @nhead: room to add at head 2290 * @ntail: room to add at tail 2291 * @gfp_mask: allocation priority 2292 * 2293 * Expands (or creates identical copy, if @nhead and @ntail are zero) 2294 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 2295 * reference count of 1. Returns zero in the case of success or error, 2296 * if expansion failed. In the last case, &sk_buff is not changed. 2297 * 2298 * All the pointers pointing into skb header may change and must be 2299 * reloaded after call to this function. 2300 * 2301 * Note: If you skb_push() the start of the buffer after reallocating the 2302 * header, call skb_postpush_data_move() first to move the metadata out of 2303 * the way before writing to &sk_buff->data. 2304 */ 2305 2306 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 2307 gfp_t gfp_mask) 2308 { 2309 unsigned int osize = skb_end_offset(skb); 2310 unsigned int size = osize + nhead + ntail; 2311 long off; 2312 u8 *data; 2313 int i; 2314 2315 if (unlikely(nhead < 0)) { 2316 DEBUG_NET_WARN_ON_ONCE(1); 2317 return -EINVAL; 2318 } 2319 2320 if (unlikely(skb_shared(skb))) { 2321 DEBUG_NET_WARN_ON_ONCE(1); 2322 return -EINVAL; 2323 } 2324 2325 skb_zcopy_downgrade_managed(skb); 2326 2327 if (skb_pfmemalloc(skb)) 2328 gfp_mask |= __GFP_MEMALLOC; 2329 2330 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 2331 if (!data) 2332 goto nodata; 2333 size = SKB_WITH_OVERHEAD(size); 2334 2335 /* Copy only real data... and, alas, header. This should be 2336 * optimized for the cases when header is void. 2337 */ 2338 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 2339 2340 memcpy((struct skb_shared_info *)(data + size), 2341 skb_shinfo(skb), 2342 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 2343 2344 /* 2345 * if shinfo is shared we must drop the old head gracefully, but if it 2346 * is not we can just drop the old head and let the existing refcount 2347 * be since all we did is relocate the values 2348 */ 2349 if (skb_cloned(skb)) { 2350 if (skb_orphan_frags(skb, gfp_mask)) 2351 goto nofrags; 2352 if (skb_zcopy(skb)) 2353 net_zcopy_get(skb_uarg(skb)); 2354 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2355 skb_frag_ref(skb, i); 2356 2357 if (skb_has_frag_list(skb)) 2358 skb_clone_fraglist(skb); 2359 2360 skb_release_data(skb, SKB_CONSUMED); 2361 } else { 2362 skb_free_head(skb); 2363 } 2364 off = (data + nhead) - skb->head; 2365 2366 skb->head = data; 2367 skb->head_frag = 0; 2368 skb->data += off; 2369 2370 skb_set_end_offset(skb, size); 2371 #ifdef NET_SKBUFF_DATA_USES_OFFSET 2372 off = nhead; 2373 #endif 2374 skb->tail += off; 2375 skb_headers_offset_update(skb, nhead); 2376 skb->cloned = 0; 2377 skb->hdr_len = 0; 2378 skb->nohdr = 0; 2379 atomic_set(&skb_shinfo(skb)->dataref, 1); 2380 2381 /* It is not generally safe to change skb->truesize. 2382 * For the moment, we really care of rx path, or 2383 * when skb is orphaned (not attached to a socket). 2384 */ 2385 if (!skb->sk || skb->destructor == sock_edemux) 2386 skb->truesize += size - osize; 2387 2388 return 0; 2389 2390 nofrags: 2391 skb_kfree_head(data); 2392 nodata: 2393 return -ENOMEM; 2394 } 2395 EXPORT_SYMBOL(pskb_expand_head); 2396 2397 /* Make private copy of skb with writable head and some headroom */ 2398 2399 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 2400 { 2401 struct sk_buff *skb2; 2402 int delta = headroom - skb_headroom(skb); 2403 2404 if (delta <= 0) 2405 skb2 = pskb_copy(skb, GFP_ATOMIC); 2406 else { 2407 skb2 = skb_clone(skb, GFP_ATOMIC); 2408 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 2409 GFP_ATOMIC)) { 2410 kfree_skb(skb2); 2411 skb2 = NULL; 2412 } 2413 } 2414 return skb2; 2415 } 2416 EXPORT_SYMBOL(skb_realloc_headroom); 2417 2418 /* Note: We plan to rework this in linux-6.4 */ 2419 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) 2420 { 2421 unsigned int saved_end_offset, saved_truesize; 2422 struct skb_shared_info *shinfo; 2423 int res; 2424 2425 saved_end_offset = skb_end_offset(skb); 2426 saved_truesize = skb->truesize; 2427 2428 res = pskb_expand_head(skb, 0, 0, pri); 2429 if (res) 2430 return res; 2431 2432 skb->truesize = saved_truesize; 2433 2434 if (likely(skb_end_offset(skb) == saved_end_offset)) 2435 return 0; 2436 2437 shinfo = skb_shinfo(skb); 2438 2439 /* We are about to change back skb->end, 2440 * we need to move skb_shinfo() to its new location. 2441 */ 2442 memmove(skb->head + saved_end_offset, 2443 shinfo, 2444 offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); 2445 2446 skb_set_end_offset(skb, saved_end_offset); 2447 2448 return 0; 2449 } 2450 2451 /** 2452 * skb_expand_head - reallocate header of &sk_buff 2453 * @skb: buffer to reallocate 2454 * @headroom: needed headroom 2455 * 2456 * Unlike skb_realloc_headroom, this one does not allocate a new skb 2457 * if possible; copies skb->sk to new skb as needed 2458 * and frees original skb in case of failures. 2459 * 2460 * It expect increased headroom and generates warning otherwise. 2461 */ 2462 2463 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) 2464 { 2465 int delta = headroom - skb_headroom(skb); 2466 int osize = skb_end_offset(skb); 2467 struct sock *sk = skb->sk; 2468 2469 if (WARN_ONCE(delta <= 0, 2470 "%s is expecting an increase in the headroom", __func__)) 2471 return skb; 2472 2473 delta = SKB_DATA_ALIGN(delta); 2474 /* pskb_expand_head() might crash, if skb is shared. */ 2475 if (skb_shared(skb) || !is_skb_wmem(skb)) { 2476 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2477 2478 if (unlikely(!nskb)) 2479 goto fail; 2480 2481 if (sk) 2482 skb_set_owner_w(nskb, sk); 2483 consume_skb(skb); 2484 skb = nskb; 2485 } 2486 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) 2487 goto fail; 2488 2489 if (sk && is_skb_wmem(skb)) { 2490 delta = skb_end_offset(skb) - osize; 2491 refcount_add(delta, &sk->sk_wmem_alloc); 2492 skb->truesize += delta; 2493 } 2494 return skb; 2495 2496 fail: 2497 kfree_skb(skb); 2498 return NULL; 2499 } 2500 EXPORT_SYMBOL(skb_expand_head); 2501 2502 /** 2503 * skb_copy_expand - copy and expand sk_buff 2504 * @skb: buffer to copy 2505 * @newheadroom: new free bytes at head 2506 * @newtailroom: new free bytes at tail 2507 * @gfp_mask: allocation priority 2508 * 2509 * Make a copy of both an &sk_buff and its data and while doing so 2510 * allocate additional space. 2511 * 2512 * This is used when the caller wishes to modify the data and needs a 2513 * private copy of the data to alter as well as more space for new fields. 2514 * Returns %NULL on failure or the pointer to the buffer 2515 * on success. The returned buffer has a reference count of 1. 2516 * 2517 * You must pass %GFP_ATOMIC as the allocation priority if this function 2518 * is called from an interrupt. 2519 */ 2520 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 2521 int newheadroom, int newtailroom, 2522 gfp_t gfp_mask) 2523 { 2524 /* 2525 * Allocate the copy buffer 2526 */ 2527 int head_copy_len, head_copy_off; 2528 struct sk_buff *n; 2529 int oldheadroom; 2530 2531 if (!skb_frags_readable(skb)) 2532 return NULL; 2533 2534 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)) 2535 return NULL; 2536 2537 oldheadroom = skb_headroom(skb); 2538 n = __alloc_skb(newheadroom + skb->len + newtailroom, 2539 gfp_mask, skb_alloc_rx_flag(skb), 2540 NUMA_NO_NODE); 2541 if (!n) 2542 return NULL; 2543 2544 skb_reserve(n, newheadroom); 2545 2546 /* Set the tail pointer and length */ 2547 skb_put(n, skb->len); 2548 2549 head_copy_len = oldheadroom; 2550 head_copy_off = 0; 2551 if (newheadroom <= head_copy_len) 2552 head_copy_len = newheadroom; 2553 else 2554 head_copy_off = newheadroom - head_copy_len; 2555 2556 /* Copy the linear header and data. */ 2557 if (unlikely(skb_copy_bits(skb, -head_copy_len, 2558 n->head + head_copy_off, 2559 skb->len + head_copy_len))) { 2560 DEBUG_NET_WARN_ON_ONCE(1); 2561 kfree_skb(n); 2562 return NULL; 2563 } 2564 2565 skb_copy_header(n, skb); 2566 2567 skb_headers_offset_update(n, newheadroom - oldheadroom); 2568 2569 return n; 2570 } 2571 EXPORT_SYMBOL(skb_copy_expand); 2572 2573 /** 2574 * __skb_pad - zero pad the tail of an skb 2575 * @skb: buffer to pad 2576 * @pad: space to pad 2577 * @free_on_error: free buffer on error 2578 * 2579 * Ensure that a buffer is followed by a padding area that is zero 2580 * filled. Used by network drivers which may DMA or transfer data 2581 * beyond the buffer end onto the wire. 2582 * 2583 * May return error in out of memory cases. The skb is freed on error 2584 * if @free_on_error is true. 2585 */ 2586 2587 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 2588 { 2589 int err; 2590 int ntail; 2591 2592 /* If the skbuff is non linear tailroom is always zero.. */ 2593 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 2594 memset(skb->data+skb->len, 0, pad); 2595 return 0; 2596 } 2597 2598 ntail = skb->data_len + pad - (skb->end - skb->tail); 2599 if (likely(skb_cloned(skb) || ntail > 0)) { 2600 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 2601 if (unlikely(err)) 2602 goto free_skb; 2603 } 2604 2605 /* FIXME: The use of this function with non-linear skb's really needs 2606 * to be audited. 2607 */ 2608 err = skb_linearize(skb); 2609 if (unlikely(err)) 2610 goto free_skb; 2611 2612 memset(skb->data + skb->len, 0, pad); 2613 return 0; 2614 2615 free_skb: 2616 if (free_on_error) 2617 kfree_skb(skb); 2618 return err; 2619 } 2620 EXPORT_SYMBOL(__skb_pad); 2621 2622 /** 2623 * pskb_put - add data to the tail of a potentially fragmented buffer 2624 * @skb: start of the buffer to use 2625 * @tail: tail fragment of the buffer to use 2626 * @len: amount of data to add 2627 * 2628 * This function extends the used data area of the potentially 2629 * fragmented buffer. @tail must be the last fragment of @skb -- or 2630 * @skb itself. If this would exceed the total buffer size the kernel 2631 * will panic. A pointer to the first byte of the extra data is 2632 * returned. 2633 */ 2634 2635 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 2636 { 2637 if (tail != skb) { 2638 skb->data_len += len; 2639 skb->len += len; 2640 } 2641 return skb_put(tail, len); 2642 } 2643 EXPORT_SYMBOL_GPL(pskb_put); 2644 2645 /** 2646 * skb_put - add data to a buffer 2647 * @skb: buffer to use 2648 * @len: amount of data to add 2649 * 2650 * This function extends the used data area of the buffer. If this would 2651 * exceed the total buffer size the kernel will panic. A pointer to the 2652 * first byte of the extra data is returned. 2653 */ 2654 void *skb_put(struct sk_buff *skb, unsigned int len) 2655 { 2656 void *tmp = skb_tail_pointer(skb); 2657 SKB_LINEAR_ASSERT(skb); 2658 skb->tail += len; 2659 skb->len += len; 2660 if (unlikely(skb->tail > skb->end)) 2661 skb_over_panic(skb, len, __builtin_return_address(0)); 2662 return tmp; 2663 } 2664 EXPORT_SYMBOL(skb_put); 2665 2666 /** 2667 * skb_push - add data to the start of a buffer 2668 * @skb: buffer to use 2669 * @len: amount of data to add 2670 * 2671 * This function extends the used data area of the buffer at the buffer 2672 * start. If this would exceed the total buffer headroom the kernel will 2673 * panic. A pointer to the first byte of the extra data is returned. 2674 */ 2675 void *skb_push(struct sk_buff *skb, unsigned int len) 2676 { 2677 skb->data -= len; 2678 skb->len += len; 2679 if (unlikely(skb->data < skb->head)) 2680 skb_under_panic(skb, len, __builtin_return_address(0)); 2681 return skb->data; 2682 } 2683 EXPORT_SYMBOL(skb_push); 2684 2685 /** 2686 * skb_pull - remove data from the start of a buffer 2687 * @skb: buffer to use 2688 * @len: amount of data to remove 2689 * 2690 * This function removes data from the start of a buffer, returning 2691 * the memory to the headroom. A pointer to the next data in the buffer 2692 * is returned. Once the data has been pulled future pushes will overwrite 2693 * the old data. 2694 */ 2695 void *skb_pull(struct sk_buff *skb, unsigned int len) 2696 { 2697 return skb_pull_inline(skb, len); 2698 } 2699 EXPORT_SYMBOL(skb_pull); 2700 2701 /** 2702 * skb_pull_data - remove data from the start of a buffer returning its 2703 * original position. 2704 * @skb: buffer to use 2705 * @len: amount of data to remove 2706 * 2707 * This function removes data from the start of a buffer, returning 2708 * the memory to the headroom. A pointer to the original data in the buffer 2709 * is returned after checking if there is enough data to pull. Once the 2710 * data has been pulled future pushes will overwrite the old data. 2711 */ 2712 void *skb_pull_data(struct sk_buff *skb, size_t len) 2713 { 2714 void *data = skb->data; 2715 2716 if (skb->len < len) 2717 return NULL; 2718 2719 skb_pull(skb, len); 2720 2721 return data; 2722 } 2723 EXPORT_SYMBOL(skb_pull_data); 2724 2725 /** 2726 * skb_trim - remove end from a buffer 2727 * @skb: buffer to alter 2728 * @len: new length 2729 * 2730 * Cut the length of a buffer down by removing data from the tail. If 2731 * the buffer is already under the length specified it is not modified. 2732 * The skb must be linear. 2733 */ 2734 void skb_trim(struct sk_buff *skb, unsigned int len) 2735 { 2736 if (skb->len > len) 2737 __skb_trim(skb, len); 2738 } 2739 EXPORT_SYMBOL(skb_trim); 2740 2741 /* Trims skb to length len. It can change skb pointers. 2742 */ 2743 2744 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 2745 { 2746 struct sk_buff **fragp; 2747 struct sk_buff *frag; 2748 int offset = skb_headlen(skb); 2749 int nfrags = skb_shinfo(skb)->nr_frags; 2750 int i; 2751 int err; 2752 2753 if (skb_cloned(skb) && 2754 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 2755 return err; 2756 2757 i = 0; 2758 if (offset >= len) 2759 goto drop_pages; 2760 2761 for (; i < nfrags; i++) { 2762 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2763 2764 if (end < len) { 2765 offset = end; 2766 continue; 2767 } 2768 2769 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 2770 2771 drop_pages: 2772 skb_shinfo(skb)->nr_frags = i; 2773 2774 for (; i < nfrags; i++) 2775 skb_frag_unref(skb, i); 2776 2777 if (skb_has_frag_list(skb)) 2778 skb_drop_fraglist(skb); 2779 goto done; 2780 } 2781 2782 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2783 fragp = &frag->next) { 2784 int end = offset + frag->len; 2785 2786 if (skb_shared(frag)) { 2787 struct sk_buff *nfrag; 2788 2789 nfrag = skb_clone(frag, GFP_ATOMIC); 2790 if (unlikely(!nfrag)) 2791 return -ENOMEM; 2792 2793 nfrag->next = frag->next; 2794 consume_skb(frag); 2795 frag = nfrag; 2796 *fragp = frag; 2797 } 2798 2799 if (end < len) { 2800 offset = end; 2801 continue; 2802 } 2803 2804 if (end > len && 2805 unlikely((err = pskb_trim(frag, len - offset)))) 2806 return err; 2807 2808 if (frag->next) 2809 skb_drop_list(&frag->next); 2810 break; 2811 } 2812 2813 done: 2814 if (len > skb_headlen(skb)) { 2815 skb->data_len -= skb->len - len; 2816 skb->len = len; 2817 } else { 2818 skb->len = len; 2819 skb->data_len = 0; 2820 skb_set_tail_pointer(skb, len); 2821 } 2822 if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb)) 2823 skb->unreadable = 0; 2824 2825 if (!skb->sk || skb->destructor == sock_edemux) 2826 skb_condense(skb); 2827 return 0; 2828 } 2829 EXPORT_SYMBOL(___pskb_trim); 2830 2831 static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len) 2832 { 2833 int delta = skb->len - len; 2834 2835 if (skb_frags_readable(skb)) { 2836 skb->csum = csum_block_sub(skb->csum, 2837 skb_checksum(skb, len, delta, 0), 2838 len); 2839 return 0; 2840 } 2841 2842 if (len > skb_headlen(skb)) 2843 return -EFAULT; 2844 2845 /* The trimmed bytes are unreadable, but the remaining packet can be 2846 * checksummed by software after trimming. 2847 */ 2848 skb->ip_summed = CHECKSUM_NONE; 2849 return 0; 2850 } 2851 2852 /* Note : use pskb_trim_rcsum() instead of calling this directly 2853 */ 2854 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2855 { 2856 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2857 int err; 2858 2859 err = pskb_trim_rcsum_complete(skb, len); 2860 if (err) 2861 return err; 2862 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2863 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2864 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2865 2866 if (offset + sizeof(__sum16) > hdlen) 2867 return -EINVAL; 2868 } 2869 return __pskb_trim(skb, len); 2870 } 2871 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2872 2873 /** 2874 * __pskb_pull_tail - advance tail of skb header 2875 * @skb: buffer to reallocate 2876 * @delta: number of bytes to advance tail 2877 * 2878 * The function makes a sense only on a fragmented &sk_buff, 2879 * it expands header moving its tail forward and copying necessary 2880 * data from fragmented part. 2881 * 2882 * &sk_buff MUST have reference count of 1. 2883 * 2884 * Returns %NULL (and &sk_buff does not change) if pull failed 2885 * or value of new tail of skb in the case of success. 2886 * 2887 * All the pointers pointing into skb header may change and must be 2888 * reloaded after call to this function. 2889 */ 2890 2891 /* Moves tail of skb head forward, copying data from fragmented part, 2892 * when it is necessary. 2893 * 1. It may fail due to malloc failure. 2894 * 2. It may change skb pointers. 2895 * 2896 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2897 */ 2898 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2899 { 2900 /* If skb has not enough free space at tail, get new one 2901 * plus 128 bytes for future expansions. If we have enough 2902 * room at tail, reallocate without expansion only if skb is cloned. 2903 */ 2904 int i, k, eat = (skb->tail + delta) - skb->end; 2905 2906 if (!skb_frags_readable(skb)) 2907 return NULL; 2908 2909 if (eat > 0 || skb_cloned(skb)) { 2910 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2911 GFP_ATOMIC)) 2912 return NULL; 2913 } 2914 2915 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2916 skb_tail_pointer(skb), delta)); 2917 2918 /* Optimization: no fragments, no reasons to preestimate 2919 * size of pulled pages. Superb. 2920 */ 2921 if (!skb_has_frag_list(skb)) 2922 goto pull_pages; 2923 2924 /* Estimate size of pulled pages. */ 2925 eat = delta; 2926 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2927 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2928 2929 if (size >= eat) 2930 goto pull_pages; 2931 eat -= size; 2932 } 2933 2934 /* If we need update frag list, we are in troubles. 2935 * Certainly, it is possible to add an offset to skb data, 2936 * but taking into account that pulling is expected to 2937 * be very rare operation, it is worth to fight against 2938 * further bloating skb head and crucify ourselves here instead. 2939 * Pure masohism, indeed. 8)8) 2940 */ 2941 if (eat) { 2942 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2943 struct sk_buff *clone = NULL; 2944 struct sk_buff *insp = NULL; 2945 2946 do { 2947 if (list->len <= eat) { 2948 /* Eaten as whole. */ 2949 eat -= list->len; 2950 list = list->next; 2951 insp = list; 2952 } else { 2953 /* Eaten partially. */ 2954 if (skb_is_gso(skb) && !list->head_frag && 2955 skb_headlen(list)) 2956 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 2957 2958 if (skb_shared(list)) { 2959 /* Sucks! We need to fork list. :-( */ 2960 clone = skb_clone(list, GFP_ATOMIC); 2961 if (!clone) 2962 return NULL; 2963 insp = list->next; 2964 list = clone; 2965 } else { 2966 /* This may be pulled without 2967 * problems. */ 2968 insp = list; 2969 } 2970 if (!pskb_pull(list, eat)) { 2971 kfree_skb(clone); 2972 return NULL; 2973 } 2974 break; 2975 } 2976 } while (eat); 2977 2978 /* Free pulled out fragments. */ 2979 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2980 skb_shinfo(skb)->frag_list = list->next; 2981 consume_skb(list); 2982 } 2983 /* And insert new clone at head. */ 2984 if (clone) { 2985 clone->next = list; 2986 skb_shinfo(skb)->frag_list = clone; 2987 } 2988 } 2989 /* Success! Now we may commit changes to skb data. */ 2990 2991 pull_pages: 2992 eat = delta; 2993 k = 0; 2994 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2995 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2996 2997 if (size <= eat) { 2998 skb_frag_unref(skb, i); 2999 eat -= size; 3000 } else { 3001 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 3002 3003 *frag = skb_shinfo(skb)->frags[i]; 3004 if (eat) { 3005 skb_frag_off_add(frag, eat); 3006 skb_frag_size_sub(frag, eat); 3007 if (!i) 3008 goto end; 3009 eat = 0; 3010 } 3011 k++; 3012 } 3013 } 3014 skb_shinfo(skb)->nr_frags = k; 3015 3016 end: 3017 skb->tail += delta; 3018 skb->data_len -= delta; 3019 3020 if (!skb->data_len) 3021 skb_zcopy_clear(skb, false); 3022 3023 return skb_tail_pointer(skb); 3024 } 3025 EXPORT_SYMBOL(__pskb_pull_tail); 3026 3027 /** 3028 * skb_copy_bits - copy bits from skb to kernel buffer 3029 * @skb: source skb 3030 * @offset: offset in source 3031 * @to: destination buffer 3032 * @len: number of bytes to copy 3033 * 3034 * Copy the specified number of bytes from the source skb to the 3035 * destination buffer. 3036 * 3037 * CAUTION ! : 3038 * If its prototype is ever changed, 3039 * check arch/{*}/net/{*}.S files, 3040 * since it is called from BPF assembly code. 3041 */ 3042 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 3043 { 3044 int start = skb_headlen(skb); 3045 struct sk_buff *frag_iter; 3046 int i, copy; 3047 3048 if (offset > (int)skb->len - len) 3049 goto fault; 3050 3051 /* Copy header. */ 3052 if ((copy = start - offset) > 0) { 3053 if (copy > len) 3054 copy = len; 3055 skb_copy_from_linear_data_offset(skb, offset, to, copy); 3056 if ((len -= copy) == 0) 3057 return 0; 3058 offset += copy; 3059 to += copy; 3060 } 3061 3062 if (!skb_frags_readable(skb)) 3063 goto fault; 3064 3065 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3066 int end; 3067 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 3068 3069 WARN_ON(start > offset + len); 3070 3071 end = start + skb_frag_size(f); 3072 if ((copy = end - offset) > 0) { 3073 u32 p_off, p_len, copied; 3074 struct page *p; 3075 u8 *vaddr; 3076 3077 if (copy > len) 3078 copy = len; 3079 3080 skb_frag_foreach_page(f, 3081 skb_frag_off(f) + offset - start, 3082 copy, p, p_off, p_len, copied) { 3083 vaddr = kmap_atomic(p); 3084 memcpy(to + copied, vaddr + p_off, p_len); 3085 kunmap_atomic(vaddr); 3086 } 3087 3088 if ((len -= copy) == 0) 3089 return 0; 3090 offset += copy; 3091 to += copy; 3092 } 3093 start = end; 3094 } 3095 3096 skb_walk_frags(skb, frag_iter) { 3097 int end; 3098 3099 WARN_ON(start > offset + len); 3100 3101 end = start + frag_iter->len; 3102 if ((copy = end - offset) > 0) { 3103 if (copy > len) 3104 copy = len; 3105 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 3106 goto fault; 3107 if ((len -= copy) == 0) 3108 return 0; 3109 offset += copy; 3110 to += copy; 3111 } 3112 start = end; 3113 } 3114 3115 if (!len) 3116 return 0; 3117 3118 fault: 3119 return -EFAULT; 3120 } 3121 EXPORT_SYMBOL(skb_copy_bits); 3122 3123 /* 3124 * Callback from splice_to_pipe(), if we need to release some pages 3125 * at the end of the spd in case we error'ed out in filling the pipe. 3126 */ 3127 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 3128 { 3129 put_page(spd->pages[i]); 3130 } 3131 3132 static struct page *linear_to_page(struct page *page, unsigned int *len, 3133 unsigned int *offset, 3134 struct sock *sk) 3135 { 3136 struct page_frag *pfrag = sk_page_frag(sk); 3137 3138 if (!sk_page_frag_refill(sk, pfrag)) 3139 return NULL; 3140 3141 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 3142 3143 memcpy(page_address(pfrag->page) + pfrag->offset, 3144 page_address(page) + *offset, *len); 3145 *offset = pfrag->offset; 3146 pfrag->offset += *len; 3147 3148 return pfrag->page; 3149 } 3150 3151 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 3152 struct page *page, 3153 unsigned int offset) 3154 { 3155 return spd->nr_pages && 3156 spd->pages[spd->nr_pages - 1] == page && 3157 (spd->partial[spd->nr_pages - 1].offset + 3158 spd->partial[spd->nr_pages - 1].len == offset); 3159 } 3160 3161 /* 3162 * Fill page/offset/length into spd, if it can hold more pages. 3163 */ 3164 static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, 3165 unsigned int *len, unsigned int offset, bool linear, 3166 struct sock *sk) 3167 { 3168 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 3169 return true; 3170 3171 if (linear) { 3172 page = linear_to_page(page, len, &offset, sk); 3173 if (!page) 3174 return true; 3175 } 3176 if (spd_can_coalesce(spd, page, offset)) { 3177 spd->partial[spd->nr_pages - 1].len += *len; 3178 return false; 3179 } 3180 get_page(page); 3181 spd->pages[spd->nr_pages] = page; 3182 spd->partial[spd->nr_pages].len = *len; 3183 spd->partial[spd->nr_pages].offset = offset; 3184 spd->nr_pages++; 3185 3186 return false; 3187 } 3188 3189 static bool __splice_segment(struct page *page, unsigned int poff, 3190 unsigned int plen, unsigned int *off, 3191 unsigned int *len, 3192 struct splice_pipe_desc *spd, bool linear, 3193 struct sock *sk) 3194 { 3195 if (!*len) 3196 return true; 3197 3198 /* skip this segment if already processed */ 3199 if (*off >= plen) { 3200 *off -= plen; 3201 return false; 3202 } 3203 3204 /* ignore any bits we already processed */ 3205 poff += *off; 3206 plen -= *off; 3207 *off = 0; 3208 3209 do { 3210 unsigned int flen = min(*len, plen); 3211 3212 if (spd_fill_page(spd, page, &flen, poff, linear, sk)) 3213 return true; 3214 poff += flen; 3215 plen -= flen; 3216 *len -= flen; 3217 if (!*len) 3218 return true; 3219 } while (plen); 3220 3221 return false; 3222 } 3223 3224 /* 3225 * Map linear and fragment data from the skb to spd. It reports true if the 3226 * pipe is full or if we already spliced the requested length. 3227 */ 3228 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 3229 unsigned int *offset, unsigned int *len, 3230 struct splice_pipe_desc *spd, struct sock *sk) 3231 { 3232 struct sk_buff *iter; 3233 int seg; 3234 3235 /* map the linear part : 3236 * If skb->head_frag is set, this 'linear' part is backed by a 3237 * fragment, and if the head is not shared with any clones then 3238 * we can avoid a copy since we own the head portion of this page. 3239 */ 3240 if (__splice_segment(virt_to_page(skb->data), 3241 (unsigned long) skb->data & (PAGE_SIZE - 1), 3242 skb_headlen(skb), 3243 offset, len, spd, 3244 skb_head_is_locked(skb), 3245 sk)) 3246 return true; 3247 3248 /* 3249 * then map the fragments 3250 */ 3251 if (!skb_frags_readable(skb)) 3252 return false; 3253 3254 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 3255 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 3256 3257 if (WARN_ON_ONCE(!skb_frag_page(f))) 3258 return false; 3259 3260 if (__splice_segment(skb_frag_page(f), 3261 skb_frag_off(f), skb_frag_size(f), 3262 offset, len, spd, false, sk)) 3263 return true; 3264 } 3265 3266 skb_walk_frags(skb, iter) { 3267 if (*offset >= iter->len) { 3268 *offset -= iter->len; 3269 continue; 3270 } 3271 /* __skb_splice_bits() only fails if the output has no room 3272 * left, so no point in going over the frag_list for the error 3273 * case. 3274 */ 3275 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 3276 return true; 3277 } 3278 3279 return false; 3280 } 3281 3282 /* 3283 * Map data from the skb to a pipe. Should handle both the linear part, 3284 * the fragments, and the frag list. 3285 */ 3286 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 3287 struct pipe_inode_info *pipe, unsigned int tlen, 3288 unsigned int flags) 3289 { 3290 struct partial_page partial[MAX_SKB_FRAGS]; 3291 struct page *pages[MAX_SKB_FRAGS]; 3292 struct splice_pipe_desc spd = { 3293 .pages = pages, 3294 .partial = partial, 3295 .nr_pages_max = MAX_SKB_FRAGS, 3296 .ops = &default_pipe_buf_ops, 3297 .spd_release = sock_spd_release, 3298 }; 3299 int ret = 0; 3300 3301 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 3302 3303 if (spd.nr_pages) 3304 ret = splice_to_pipe(pipe, &spd); 3305 3306 return ret; 3307 } 3308 EXPORT_SYMBOL_GPL(skb_splice_bits); 3309 3310 static int sendmsg_locked(struct sock *sk, struct msghdr *msg) 3311 { 3312 struct socket *sock = sk->sk_socket; 3313 size_t size = msg_data_left(msg); 3314 3315 if (!sock) 3316 return -EINVAL; 3317 3318 if (!sock->ops->sendmsg_locked) 3319 return sock_no_sendmsg_locked(sk, msg, size); 3320 3321 return sock->ops->sendmsg_locked(sk, msg, size); 3322 } 3323 3324 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg) 3325 { 3326 struct socket *sock = sk->sk_socket; 3327 3328 if (!sock) 3329 return -EINVAL; 3330 return sock_sendmsg(sock, msg); 3331 } 3332 3333 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg); 3334 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, 3335 int len, sendmsg_func sendmsg, int flags) 3336 { 3337 int more_hint = sk_is_tcp(sk) ? MSG_MORE : 0; 3338 unsigned int orig_len = len; 3339 struct sk_buff *head = skb; 3340 unsigned short fragidx; 3341 int slen, ret; 3342 3343 do_frag_list: 3344 3345 /* Deal with head data */ 3346 while (offset < skb_headlen(skb) && len) { 3347 struct kvec kv; 3348 struct msghdr msg; 3349 3350 slen = min_t(int, len, skb_headlen(skb) - offset); 3351 kv.iov_base = skb->data + offset; 3352 kv.iov_len = slen; 3353 memset(&msg, 0, sizeof(msg)); 3354 msg.msg_flags = MSG_DONTWAIT | flags; 3355 if (slen < len) 3356 msg.msg_flags |= more_hint; 3357 3358 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen); 3359 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3360 sendmsg_unlocked, sk, &msg); 3361 if (ret <= 0) 3362 goto error; 3363 3364 offset += ret; 3365 len -= ret; 3366 } 3367 3368 /* All the data was skb head? */ 3369 if (!len) 3370 goto out; 3371 3372 /* Make offset relative to start of frags */ 3373 offset -= skb_headlen(skb); 3374 3375 /* Find where we are in frag list */ 3376 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3377 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3378 3379 if (offset < skb_frag_size(frag)) 3380 break; 3381 3382 offset -= skb_frag_size(frag); 3383 } 3384 3385 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 3386 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 3387 3388 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 3389 3390 while (slen) { 3391 struct bio_vec bvec; 3392 struct msghdr msg = { 3393 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | 3394 flags, 3395 }; 3396 3397 if (slen < len) 3398 msg.msg_flags |= more_hint; 3399 bvec_set_page(&bvec, skb_frag_page(frag), slen, 3400 skb_frag_off(frag) + offset); 3401 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, 3402 slen); 3403 3404 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, 3405 sendmsg_unlocked, sk, &msg); 3406 if (ret <= 0) 3407 goto error; 3408 3409 len -= ret; 3410 offset += ret; 3411 slen -= ret; 3412 } 3413 3414 offset = 0; 3415 } 3416 3417 if (len) { 3418 /* Process any frag lists */ 3419 3420 if (skb == head) { 3421 if (skb_has_frag_list(skb)) { 3422 skb = skb_shinfo(skb)->frag_list; 3423 goto do_frag_list; 3424 } 3425 } else if (skb->next) { 3426 skb = skb->next; 3427 goto do_frag_list; 3428 } 3429 } 3430 3431 out: 3432 return orig_len - len; 3433 3434 error: 3435 return orig_len == len ? ret : orig_len - len; 3436 } 3437 3438 /* Send skb data on a socket. Socket must be locked. */ 3439 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 3440 int len) 3441 { 3442 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, 0); 3443 } 3444 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 3445 3446 int skb_send_sock_locked_with_flags(struct sock *sk, struct sk_buff *skb, 3447 int offset, int len, int flags) 3448 { 3449 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked, flags); 3450 } 3451 EXPORT_SYMBOL_GPL(skb_send_sock_locked_with_flags); 3452 3453 /* Send skb data on a socket. Socket must be unlocked. */ 3454 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) 3455 { 3456 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, 0); 3457 } 3458 3459 /** 3460 * skb_store_bits - store bits from kernel buffer to skb 3461 * @skb: destination buffer 3462 * @offset: offset in destination 3463 * @from: source buffer 3464 * @len: number of bytes to copy 3465 * 3466 * Copy the specified number of bytes from the source buffer to the 3467 * destination skb. This function handles all the messy bits of 3468 * traversing fragment lists and such. 3469 */ 3470 3471 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 3472 { 3473 int start = skb_headlen(skb); 3474 struct sk_buff *frag_iter; 3475 int i, copy; 3476 3477 if (offset > (int)skb->len - len) 3478 goto fault; 3479 3480 if ((copy = start - offset) > 0) { 3481 if (copy > len) 3482 copy = len; 3483 skb_copy_to_linear_data_offset(skb, offset, from, copy); 3484 if ((len -= copy) == 0) 3485 return 0; 3486 offset += copy; 3487 from += copy; 3488 } 3489 3490 if (!skb_frags_readable(skb)) 3491 goto fault; 3492 3493 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3494 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3495 int end; 3496 3497 WARN_ON(start > offset + len); 3498 3499 end = start + skb_frag_size(frag); 3500 if ((copy = end - offset) > 0) { 3501 u32 p_off, p_len, copied; 3502 struct page *p; 3503 u8 *vaddr; 3504 3505 if (copy > len) 3506 copy = len; 3507 3508 skb_frag_foreach_page(frag, 3509 skb_frag_off(frag) + offset - start, 3510 copy, p, p_off, p_len, copied) { 3511 vaddr = kmap_atomic(p); 3512 memcpy(vaddr + p_off, from + copied, p_len); 3513 kunmap_atomic(vaddr); 3514 } 3515 3516 if ((len -= copy) == 0) 3517 return 0; 3518 offset += copy; 3519 from += copy; 3520 } 3521 start = end; 3522 } 3523 3524 skb_walk_frags(skb, frag_iter) { 3525 int end; 3526 3527 WARN_ON(start > offset + len); 3528 3529 end = start + frag_iter->len; 3530 if ((copy = end - offset) > 0) { 3531 if (copy > len) 3532 copy = len; 3533 if (skb_store_bits(frag_iter, offset - start, 3534 from, copy)) 3535 goto fault; 3536 if ((len -= copy) == 0) 3537 return 0; 3538 offset += copy; 3539 from += copy; 3540 } 3541 start = end; 3542 } 3543 if (!len) 3544 return 0; 3545 3546 fault: 3547 return -EFAULT; 3548 } 3549 EXPORT_SYMBOL(skb_store_bits); 3550 3551 /* Checksum skb data. */ 3552 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, __wsum csum) 3553 { 3554 int start = skb_headlen(skb); 3555 int i, copy = start - offset; 3556 struct sk_buff *frag_iter; 3557 int pos = 0; 3558 3559 /* Checksum header. */ 3560 if (copy > 0) { 3561 if (copy > len) 3562 copy = len; 3563 csum = csum_partial(skb->data + offset, copy, csum); 3564 if ((len -= copy) == 0) 3565 return csum; 3566 offset += copy; 3567 pos = copy; 3568 } 3569 3570 if (WARN_ON_ONCE(!skb_frags_readable(skb))) 3571 return 0; 3572 3573 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3574 int end; 3575 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3576 3577 WARN_ON(start > offset + len); 3578 3579 end = start + skb_frag_size(frag); 3580 if ((copy = end - offset) > 0) { 3581 u32 p_off, p_len, copied; 3582 struct page *p; 3583 __wsum csum2; 3584 u8 *vaddr; 3585 3586 if (copy > len) 3587 copy = len; 3588 3589 skb_frag_foreach_page(frag, 3590 skb_frag_off(frag) + offset - start, 3591 copy, p, p_off, p_len, copied) { 3592 vaddr = kmap_atomic(p); 3593 csum2 = csum_partial(vaddr + p_off, p_len, 0); 3594 kunmap_atomic(vaddr); 3595 csum = csum_block_add(csum, csum2, pos); 3596 pos += p_len; 3597 } 3598 3599 if (!(len -= copy)) 3600 return csum; 3601 offset += copy; 3602 } 3603 start = end; 3604 } 3605 3606 skb_walk_frags(skb, frag_iter) { 3607 int end; 3608 3609 WARN_ON(start > offset + len); 3610 3611 end = start + frag_iter->len; 3612 if ((copy = end - offset) > 0) { 3613 __wsum csum2; 3614 if (copy > len) 3615 copy = len; 3616 csum2 = skb_checksum(frag_iter, offset - start, copy, 3617 0); 3618 csum = csum_block_add(csum, csum2, pos); 3619 if ((len -= copy) == 0) 3620 return csum; 3621 offset += copy; 3622 pos += copy; 3623 } 3624 start = end; 3625 } 3626 if (unlikely(len)) { 3627 DEBUG_NET_WARN_ON_ONCE(1); 3628 return 0; 3629 } 3630 3631 return csum; 3632 } 3633 EXPORT_SYMBOL(skb_checksum); 3634 3635 /* Both of above in one bottle. */ 3636 3637 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 3638 u8 *to, int len) 3639 { 3640 int start = skb_headlen(skb); 3641 int i, copy = start - offset; 3642 struct sk_buff *frag_iter; 3643 int pos = 0; 3644 __wsum csum = 0; 3645 3646 /* Copy header. */ 3647 if (copy > 0) { 3648 if (copy > len) 3649 copy = len; 3650 csum = csum_partial_copy_nocheck(skb->data + offset, to, 3651 copy); 3652 if ((len -= copy) == 0) 3653 return csum; 3654 offset += copy; 3655 to += copy; 3656 pos = copy; 3657 } 3658 3659 if (!skb_frags_readable(skb)) { 3660 /* Don't hand the caller a buffer with stale bytes in it. */ 3661 if (len > 0) 3662 memset(to, 0, len); 3663 return 0; 3664 } 3665 3666 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3667 int end; 3668 3669 WARN_ON(start > offset + len); 3670 3671 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3672 if ((copy = end - offset) > 0) { 3673 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3674 u32 p_off, p_len, copied; 3675 struct page *p; 3676 __wsum csum2; 3677 u8 *vaddr; 3678 3679 if (copy > len) 3680 copy = len; 3681 3682 skb_frag_foreach_page(frag, 3683 skb_frag_off(frag) + offset - start, 3684 copy, p, p_off, p_len, copied) { 3685 vaddr = kmap_atomic(p); 3686 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 3687 to + copied, 3688 p_len); 3689 kunmap_atomic(vaddr); 3690 csum = csum_block_add(csum, csum2, pos); 3691 pos += p_len; 3692 } 3693 3694 if (!(len -= copy)) 3695 return csum; 3696 offset += copy; 3697 to += copy; 3698 } 3699 start = end; 3700 } 3701 3702 skb_walk_frags(skb, frag_iter) { 3703 __wsum csum2; 3704 int end; 3705 3706 WARN_ON(start > offset + len); 3707 3708 end = start + frag_iter->len; 3709 if ((copy = end - offset) > 0) { 3710 if (copy > len) 3711 copy = len; 3712 csum2 = skb_copy_and_csum_bits(frag_iter, 3713 offset - start, 3714 to, copy); 3715 csum = csum_block_add(csum, csum2, pos); 3716 if ((len -= copy) == 0) 3717 return csum; 3718 offset += copy; 3719 to += copy; 3720 pos += copy; 3721 } 3722 start = end; 3723 } 3724 if (unlikely(len)) { 3725 DEBUG_NET_WARN_ON_ONCE(1); 3726 /* Don't hand the caller a buffer with stale bytes in it. */ 3727 if (len > 0) 3728 memset(to, 0, len); 3729 return 0; 3730 } 3731 return csum; 3732 } 3733 EXPORT_SYMBOL(skb_copy_and_csum_bits); 3734 3735 #ifdef CONFIG_NET_CRC32C 3736 u32 skb_crc32c(const struct sk_buff *skb, int offset, int len, u32 crc) 3737 { 3738 int start = skb_headlen(skb); 3739 int i, copy = start - offset; 3740 struct sk_buff *frag_iter; 3741 3742 if (copy > 0) { 3743 copy = min(copy, len); 3744 crc = crc32c(crc, skb->data + offset, copy); 3745 len -= copy; 3746 if (len == 0) 3747 return crc; 3748 offset += copy; 3749 } 3750 3751 if (WARN_ON_ONCE(!skb_frags_readable(skb))) 3752 return 0; 3753 3754 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3755 int end; 3756 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3757 3758 WARN_ON(start > offset + len); 3759 3760 end = start + skb_frag_size(frag); 3761 copy = end - offset; 3762 if (copy > 0) { 3763 u32 p_off, p_len, copied; 3764 struct page *p; 3765 u8 *vaddr; 3766 3767 copy = min(copy, len); 3768 skb_frag_foreach_page(frag, 3769 skb_frag_off(frag) + offset - start, 3770 copy, p, p_off, p_len, copied) { 3771 vaddr = kmap_atomic(p); 3772 crc = crc32c(crc, vaddr + p_off, p_len); 3773 kunmap_atomic(vaddr); 3774 } 3775 len -= copy; 3776 if (len == 0) 3777 return crc; 3778 offset += copy; 3779 } 3780 start = end; 3781 } 3782 3783 skb_walk_frags(skb, frag_iter) { 3784 int end; 3785 3786 WARN_ON(start > offset + len); 3787 3788 end = start + frag_iter->len; 3789 copy = end - offset; 3790 if (copy > 0) { 3791 copy = min(copy, len); 3792 crc = skb_crc32c(frag_iter, offset - start, copy, crc); 3793 len -= copy; 3794 if (len == 0) 3795 return crc; 3796 offset += copy; 3797 } 3798 start = end; 3799 } 3800 if (unlikely(len)) { 3801 DEBUG_NET_WARN_ON_ONCE(1); 3802 return 0; 3803 } 3804 3805 return crc; 3806 } 3807 EXPORT_SYMBOL(skb_crc32c); 3808 #endif /* CONFIG_NET_CRC32C */ 3809 3810 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 3811 { 3812 __sum16 sum; 3813 3814 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 3815 /* See comments in __skb_checksum_complete(). */ 3816 if (likely(!sum)) { 3817 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3818 !skb->csum_complete_sw) 3819 netdev_rx_csum_fault(skb->dev, skb); 3820 } 3821 if (!skb_shared(skb)) 3822 skb->csum_valid = !sum; 3823 return sum; 3824 } 3825 EXPORT_SYMBOL(__skb_checksum_complete_head); 3826 3827 /* This function assumes skb->csum already holds pseudo header's checksum, 3828 * which has been changed from the hardware checksum, for example, by 3829 * __skb_checksum_validate_complete(). And, the original skb->csum must 3830 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 3831 * 3832 * It returns non-zero if the recomputed checksum is still invalid, otherwise 3833 * zero. The new checksum is stored back into skb->csum unless the skb is 3834 * shared. 3835 */ 3836 __sum16 __skb_checksum_complete(struct sk_buff *skb) 3837 { 3838 __wsum csum; 3839 __sum16 sum; 3840 3841 csum = skb_checksum(skb, 0, skb->len, 0); 3842 3843 sum = csum_fold(csum_add(skb->csum, csum)); 3844 /* This check is inverted, because we already knew the hardware 3845 * checksum is invalid before calling this function. So, if the 3846 * re-computed checksum is valid instead, then we have a mismatch 3847 * between the original skb->csum and skb_checksum(). This means either 3848 * the original hardware checksum is incorrect or we screw up skb->csum 3849 * when moving skb->data around. 3850 */ 3851 if (likely(!sum)) { 3852 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 3853 !skb->csum_complete_sw) 3854 netdev_rx_csum_fault(skb->dev, skb); 3855 } 3856 3857 if (!skb_shared(skb)) { 3858 /* Save full packet checksum */ 3859 skb->csum = csum; 3860 skb->ip_summed = CHECKSUM_COMPLETE; 3861 skb->csum_complete_sw = 1; 3862 skb->csum_valid = !sum; 3863 } 3864 3865 return sum; 3866 } 3867 EXPORT_SYMBOL(__skb_checksum_complete); 3868 3869 /** 3870 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 3871 * @from: source buffer 3872 * 3873 * Calculates the amount of linear headroom needed in the 'to' skb passed 3874 * into skb_zerocopy(). 3875 */ 3876 unsigned int 3877 skb_zerocopy_headlen(const struct sk_buff *from) 3878 { 3879 unsigned int hlen = 0; 3880 3881 if (!from->head_frag || 3882 skb_headlen(from) < L1_CACHE_BYTES || 3883 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { 3884 hlen = skb_headlen(from); 3885 if (!hlen) 3886 hlen = from->len; 3887 } 3888 3889 if (skb_has_frag_list(from)) 3890 hlen = from->len; 3891 3892 return hlen; 3893 } 3894 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 3895 3896 /** 3897 * skb_zerocopy - Zero copy skb to skb 3898 * @to: destination buffer 3899 * @from: source buffer 3900 * @len: number of bytes to copy from source buffer 3901 * @hlen: size of linear headroom in destination buffer 3902 * 3903 * Copies up to `len` bytes from `from` to `to` by creating references 3904 * to the frags in the source buffer. 3905 * 3906 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 3907 * headroom in the `to` buffer. 3908 * 3909 * Return value: 3910 * 0: everything is OK 3911 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 3912 * -EFAULT: skb_copy_bits() found some problem with skb geometry, or readable head 3913 * payload would be mixed with unreadable frags. 3914 */ 3915 int 3916 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 3917 { 3918 int i, j = 0; 3919 int plen = 0; /* length of skb->head fragment */ 3920 int ret; 3921 struct page *page; 3922 unsigned int offset; 3923 3924 if (unlikely(!from->head_frag && !hlen)) { 3925 DEBUG_NET_WARN_ON_ONCE(1); 3926 return -EFAULT; 3927 } 3928 3929 /* dont bother with small payloads */ 3930 if (len <= skb_tailroom(to)) 3931 return skb_copy_bits(from, 0, skb_put(to, len), len); 3932 3933 to->pp_recycle = from->pp_recycle; 3934 if (hlen) { 3935 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3936 if (unlikely(ret)) 3937 return ret; 3938 len -= hlen; 3939 } else { 3940 plen = min_t(int, skb_headlen(from), len); 3941 if (plen) { 3942 page = virt_to_head_page(from->head); 3943 offset = from->data - (unsigned char *)page_address(page); 3944 __skb_fill_netmem_desc(to, 0, page_to_netmem(page), 3945 offset, plen); 3946 skb_frag_ref(to, 0); 3947 j = 1; 3948 len -= plen; 3949 } 3950 } 3951 3952 if (!skb_frags_readable(from) && j > 0 && len) { 3953 skb_frag_unref(to, 0); 3954 return -EFAULT; 3955 } 3956 3957 skb_len_add(to, len + plen); 3958 3959 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3960 if (j > 0) 3961 skb_frag_unref(to, 0); 3962 return -ENOMEM; 3963 } 3964 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3965 3966 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3967 int size; 3968 3969 if (!len) 3970 break; 3971 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3972 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3973 len); 3974 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3975 len -= size; 3976 skb_frag_ref(to, j); 3977 j++; 3978 } 3979 skb_shinfo(to)->nr_frags = j; 3980 3981 skb_shinfo(to)->flags |= skb_shinfo(from)->flags & SKBFL_SHARED_FRAG; 3982 3983 if (i > 0 && from->unreadable) 3984 to->unreadable = 1; 3985 3986 return 0; 3987 } 3988 EXPORT_SYMBOL_GPL(skb_zerocopy); 3989 3990 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3991 { 3992 __wsum csum; 3993 long csstart; 3994 3995 if (skb->ip_summed == CHECKSUM_PARTIAL) 3996 csstart = skb_checksum_start_offset(skb); 3997 else 3998 csstart = skb_headlen(skb); 3999 4000 if (unlikely(csstart < 0 || csstart > skb_headlen(skb) || 4001 (skb->ip_summed == CHECKSUM_PARTIAL && 4002 csstart + skb->csum_offset + sizeof(__sum16) > 4003 skb->len))) { 4004 DEBUG_NET_WARN_ON_ONCE(1); 4005 /* Send the frame without the checksum filled in, or send 4006 * zeroes if we can't even copy it. 4007 */ 4008 if (skb_copy_bits(skb, 0, to, skb->len)) 4009 memset(to, 0, skb->len); 4010 return; 4011 } 4012 4013 skb_copy_from_linear_data(skb, to, csstart); 4014 4015 csum = 0; 4016 if (csstart != skb->len) 4017 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 4018 skb->len - csstart); 4019 4020 if (skb->ip_summed == CHECKSUM_PARTIAL) { 4021 long csstuff = csstart + skb->csum_offset; 4022 4023 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 4024 } 4025 } 4026 EXPORT_SYMBOL(skb_copy_and_csum_dev); 4027 4028 /** 4029 * skb_dequeue - remove from the head of the queue 4030 * @list: list to dequeue from 4031 * 4032 * Remove the head of the list. The list lock is taken so the function 4033 * may be used safely with other locking list functions. The head item is 4034 * returned or %NULL if the list is empty. 4035 */ 4036 4037 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 4038 { 4039 unsigned long flags; 4040 struct sk_buff *result; 4041 4042 spin_lock_irqsave(&list->lock, flags); 4043 result = __skb_dequeue(list); 4044 spin_unlock_irqrestore(&list->lock, flags); 4045 return result; 4046 } 4047 EXPORT_SYMBOL(skb_dequeue); 4048 4049 /** 4050 * skb_dequeue_tail - remove from the tail of the queue 4051 * @list: list to dequeue from 4052 * 4053 * Remove the tail of the list. The list lock is taken so the function 4054 * may be used safely with other locking list functions. The tail item is 4055 * returned or %NULL if the list is empty. 4056 */ 4057 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 4058 { 4059 unsigned long flags; 4060 struct sk_buff *result; 4061 4062 spin_lock_irqsave(&list->lock, flags); 4063 result = __skb_dequeue_tail(list); 4064 spin_unlock_irqrestore(&list->lock, flags); 4065 return result; 4066 } 4067 EXPORT_SYMBOL(skb_dequeue_tail); 4068 4069 /** 4070 * skb_queue_purge_reason - empty a list 4071 * @list: list to empty 4072 * @reason: drop reason 4073 * 4074 * Delete all buffers on an &sk_buff list. Each buffer is removed from 4075 * the list and one reference dropped. This function takes the list 4076 * lock and is atomic with respect to other list locking functions. 4077 */ 4078 void skb_queue_purge_reason(struct sk_buff_head *list, 4079 enum skb_drop_reason reason) 4080 { 4081 struct sk_buff_head tmp; 4082 unsigned long flags; 4083 4084 if (skb_queue_empty_lockless(list)) 4085 return; 4086 4087 __skb_queue_head_init(&tmp); 4088 4089 spin_lock_irqsave(&list->lock, flags); 4090 skb_queue_splice_init(list, &tmp); 4091 spin_unlock_irqrestore(&list->lock, flags); 4092 4093 __skb_queue_purge_reason(&tmp, reason); 4094 } 4095 EXPORT_SYMBOL(skb_queue_purge_reason); 4096 4097 /** 4098 * skb_rbtree_purge - empty a skb rbtree 4099 * @root: root of the rbtree to empty 4100 * Return value: the sum of truesizes of all purged skbs. 4101 * 4102 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 4103 * the list and one reference dropped. This function does not take 4104 * any lock. Synchronization should be handled by the caller (e.g., TCP 4105 * out-of-order queue is protected by the socket lock). 4106 */ 4107 unsigned int skb_rbtree_purge(struct rb_root *root) 4108 { 4109 struct rb_node *p = rb_first(root); 4110 unsigned int sum = 0; 4111 4112 while (p) { 4113 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 4114 4115 p = rb_next(p); 4116 rb_erase(&skb->rbnode, root); 4117 sum += skb->truesize; 4118 kfree_skb(skb); 4119 } 4120 return sum; 4121 } 4122 4123 void skb_errqueue_purge(struct sk_buff_head *list) 4124 { 4125 struct sk_buff *skb, *next; 4126 struct sk_buff_head kill; 4127 unsigned long flags; 4128 4129 __skb_queue_head_init(&kill); 4130 4131 spin_lock_irqsave(&list->lock, flags); 4132 skb_queue_walk_safe(list, skb, next) { 4133 if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY || 4134 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) 4135 continue; 4136 __skb_unlink(skb, list); 4137 __skb_queue_tail(&kill, skb); 4138 } 4139 spin_unlock_irqrestore(&list->lock, flags); 4140 __skb_queue_purge(&kill); 4141 } 4142 EXPORT_SYMBOL(skb_errqueue_purge); 4143 4144 /** 4145 * skb_queue_head - queue a buffer at the list head 4146 * @list: list to use 4147 * @newsk: buffer to queue 4148 * 4149 * Queue a buffer at the start of the list. This function takes the 4150 * list lock and can be used safely with other locking &sk_buff functions 4151 * safely. 4152 * 4153 * A buffer cannot be placed on two lists at the same time. 4154 */ 4155 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 4156 { 4157 unsigned long flags; 4158 4159 spin_lock_irqsave(&list->lock, flags); 4160 __skb_queue_head(list, newsk); 4161 spin_unlock_irqrestore(&list->lock, flags); 4162 } 4163 EXPORT_SYMBOL(skb_queue_head); 4164 4165 /** 4166 * skb_queue_tail - queue a buffer at the list tail 4167 * @list: list to use 4168 * @newsk: buffer to queue 4169 * 4170 * Queue a buffer at the tail of the list. This function takes the 4171 * list lock and can be used safely with other locking &sk_buff functions 4172 * safely. 4173 * 4174 * A buffer cannot be placed on two lists at the same time. 4175 */ 4176 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 4177 { 4178 unsigned long flags; 4179 4180 spin_lock_irqsave(&list->lock, flags); 4181 __skb_queue_tail(list, newsk); 4182 spin_unlock_irqrestore(&list->lock, flags); 4183 } 4184 EXPORT_SYMBOL(skb_queue_tail); 4185 4186 /** 4187 * skb_unlink - remove a buffer from a list 4188 * @skb: buffer to remove 4189 * @list: list to use 4190 * 4191 * Remove a packet from a list. The list locks are taken and this 4192 * function is atomic with respect to other list locked calls 4193 * 4194 * You must know what list the SKB is on. 4195 */ 4196 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 4197 { 4198 unsigned long flags; 4199 4200 spin_lock_irqsave(&list->lock, flags); 4201 __skb_unlink(skb, list); 4202 spin_unlock_irqrestore(&list->lock, flags); 4203 } 4204 EXPORT_SYMBOL(skb_unlink); 4205 4206 /** 4207 * skb_append - append a buffer 4208 * @old: buffer to insert after 4209 * @newsk: buffer to insert 4210 * @list: list to use 4211 * 4212 * Place a packet after a given packet in a list. The list locks are taken 4213 * and this function is atomic with respect to other list locked calls. 4214 * A buffer cannot be placed on two lists at the same time. 4215 */ 4216 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 4217 { 4218 unsigned long flags; 4219 4220 spin_lock_irqsave(&list->lock, flags); 4221 __skb_queue_after(list, old, newsk); 4222 spin_unlock_irqrestore(&list->lock, flags); 4223 } 4224 EXPORT_SYMBOL(skb_append); 4225 4226 static inline void skb_split_inside_header(struct sk_buff *skb, 4227 struct sk_buff* skb1, 4228 const u32 len, const int pos) 4229 { 4230 int i; 4231 4232 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 4233 pos - len); 4234 /* And move data appendix as is. */ 4235 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 4236 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 4237 4238 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 4239 skb1->unreadable = skb->unreadable; 4240 skb_shinfo(skb)->nr_frags = 0; 4241 skb1->data_len = skb->data_len; 4242 skb1->len += skb1->data_len; 4243 skb->data_len = 0; 4244 skb->len = len; 4245 skb_set_tail_pointer(skb, len); 4246 } 4247 4248 static inline void skb_split_no_header(struct sk_buff *skb, 4249 struct sk_buff* skb1, 4250 const u32 len, int pos) 4251 { 4252 int i, k = 0; 4253 const int nfrags = skb_shinfo(skb)->nr_frags; 4254 4255 skb_shinfo(skb)->nr_frags = 0; 4256 skb1->len = skb1->data_len = skb->len - len; 4257 skb->len = len; 4258 skb->data_len = len - pos; 4259 4260 for (i = 0; i < nfrags; i++) { 4261 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 4262 4263 if (pos + size > len) { 4264 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 4265 4266 if (pos < len) { 4267 /* Split frag. 4268 * We have two variants in this case: 4269 * 1. Move all the frag to the second 4270 * part, if it is possible. F.e. 4271 * this approach is mandatory for TUX, 4272 * where splitting is expensive. 4273 * 2. Split is accurately. We make this. 4274 */ 4275 skb_frag_ref(skb1, k); 4276 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 4277 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 4278 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 4279 skb_shinfo(skb)->nr_frags++; 4280 } 4281 k++; 4282 } else 4283 skb_shinfo(skb)->nr_frags++; 4284 pos += size; 4285 } 4286 skb_shinfo(skb1)->nr_frags = k; 4287 4288 skb1->unreadable = skb->unreadable; 4289 } 4290 4291 /** 4292 * skb_split - Split fragmented skb to two parts at length len. 4293 * @skb: the buffer to split 4294 * @skb1: the buffer to receive the second part 4295 * @len: new length for skb 4296 */ 4297 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 4298 { 4299 int pos = skb_headlen(skb); 4300 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; 4301 4302 skb_zcopy_downgrade_managed(skb); 4303 4304 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; 4305 skb_zerocopy_clone(skb1, skb, 0); 4306 skb1->pp_recycle = skb->pp_recycle; 4307 if (len < pos) /* Split line is inside header. */ 4308 skb_split_inside_header(skb, skb1, len, pos); 4309 else /* Second chunk has no header, nothing to copy. */ 4310 skb_split_no_header(skb, skb1, len, pos); 4311 } 4312 EXPORT_SYMBOL(skb_split); 4313 4314 /* Shifting from/to a cloned skb is a no-go. 4315 * 4316 * Caller cannot keep skb_shinfo related pointers past calling here! 4317 */ 4318 static int skb_prepare_for_shift(struct sk_buff *skb) 4319 { 4320 return skb_unclone_keeptruesize(skb, GFP_ATOMIC); 4321 } 4322 4323 /** 4324 * skb_shift - Shifts paged data partially from skb to another 4325 * @tgt: buffer into which tail data gets added 4326 * @skb: buffer from which the paged data comes from 4327 * @shiftlen: shift up to this many bytes 4328 * 4329 * Attempts to shift up to shiftlen worth of bytes, which may be less than 4330 * the length of the skb, from skb to tgt. Returns number bytes shifted. 4331 * It's up to caller to free skb if everything was shifted. 4332 * 4333 * If @tgt runs out of frags, the whole operation is aborted. 4334 * 4335 * Skb cannot include anything else but paged data while tgt is allowed 4336 * to have non-paged data as well. 4337 * 4338 * TODO: full sized shift could be optimized but that would need 4339 * specialized skb free'er to handle frags without up-to-date nr_frags. 4340 */ 4341 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 4342 { 4343 int from, to, merge, todo; 4344 skb_frag_t *fragfrom, *fragto; 4345 4346 if (unlikely(shiftlen > skb->len)) { 4347 DEBUG_NET_WARN_ON_ONCE(1); 4348 return 0; 4349 } 4350 4351 if (skb_headlen(skb)) 4352 return 0; 4353 if (skb_zcopy(tgt) || skb_zcopy(skb)) 4354 return 0; 4355 4356 DEBUG_NET_WARN_ON_ONCE(tgt->pp_recycle != skb->pp_recycle); 4357 DEBUG_NET_WARN_ON_ONCE(skb_cmp_decrypted(tgt, skb)); 4358 4359 todo = shiftlen; 4360 from = 0; 4361 to = skb_shinfo(tgt)->nr_frags; 4362 fragfrom = &skb_shinfo(skb)->frags[from]; 4363 4364 /* Actual merge is delayed until the point when we know we can 4365 * commit all, so that we don't have to undo partial changes 4366 */ 4367 if (!skb_can_coalesce_netmem(tgt, to, skb_frag_netmem(fragfrom), 4368 skb_frag_off(fragfrom))) { 4369 merge = -1; 4370 } else { 4371 merge = to - 1; 4372 4373 todo -= skb_frag_size(fragfrom); 4374 if (todo < 0) { 4375 if (skb_prepare_for_shift(skb) || 4376 skb_prepare_for_shift(tgt)) 4377 return 0; 4378 4379 /* All previous frag pointers might be stale! */ 4380 fragfrom = &skb_shinfo(skb)->frags[from]; 4381 fragto = &skb_shinfo(tgt)->frags[merge]; 4382 4383 skb_frag_size_add(fragto, shiftlen); 4384 skb_frag_size_sub(fragfrom, shiftlen); 4385 skb_frag_off_add(fragfrom, shiftlen); 4386 4387 goto onlymerged; 4388 } 4389 4390 from++; 4391 } 4392 4393 /* Skip full, not-fitting skb to avoid expensive operations */ 4394 if ((shiftlen == skb->len) && 4395 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 4396 return 0; 4397 4398 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 4399 return 0; 4400 4401 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 4402 if (to == MAX_SKB_FRAGS) 4403 return 0; 4404 4405 fragfrom = &skb_shinfo(skb)->frags[from]; 4406 fragto = &skb_shinfo(tgt)->frags[to]; 4407 4408 if (todo >= skb_frag_size(fragfrom)) { 4409 *fragto = *fragfrom; 4410 todo -= skb_frag_size(fragfrom); 4411 from++; 4412 to++; 4413 4414 } else { 4415 skb_frag_page_copy(fragto, fragfrom); 4416 skb_frag_ref(tgt, to); 4417 skb_frag_off_copy(fragto, fragfrom); 4418 skb_frag_size_set(fragto, todo); 4419 4420 skb_frag_off_add(fragfrom, todo); 4421 skb_frag_size_sub(fragfrom, todo); 4422 todo = 0; 4423 4424 to++; 4425 break; 4426 } 4427 } 4428 4429 /* The frags ran out before shiftlen did. Nothing has been committed 4430 * yet, so back out. 4431 */ 4432 if (unlikely(todo > 0)) { 4433 DEBUG_NET_WARN_ON_ONCE(1); 4434 return 0; 4435 } 4436 4437 /* Ready to "commit" this state change to tgt */ 4438 skb_shinfo(tgt)->nr_frags = to; 4439 4440 if (merge >= 0) { 4441 fragfrom = &skb_shinfo(skb)->frags[0]; 4442 fragto = &skb_shinfo(tgt)->frags[merge]; 4443 4444 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 4445 skb_frag_unref(skb, 0); 4446 } 4447 4448 /* Reposition in the original skb */ 4449 to = 0; 4450 while (from < skb_shinfo(skb)->nr_frags) 4451 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 4452 skb_shinfo(skb)->nr_frags = to; 4453 4454 onlymerged: 4455 /* Most likely the tgt won't ever need its checksum anymore, skb on 4456 * the other hand might need it if it needs to be resent 4457 */ 4458 tgt->ip_summed = CHECKSUM_PARTIAL; 4459 skb->ip_summed = CHECKSUM_PARTIAL; 4460 4461 skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 4462 4463 skb_len_add(skb, -shiftlen); 4464 skb_len_add(tgt, shiftlen); 4465 4466 return shiftlen; 4467 } 4468 EXPORT_SYMBOL_IF_KUNIT(skb_shift); 4469 4470 /** 4471 * skb_prepare_seq_read - Prepare a sequential read of skb data 4472 * @skb: the buffer to read 4473 * @from: lower offset of data to be read 4474 * @to: upper offset of data to be read 4475 * @st: state variable 4476 * 4477 * Initializes the specified state variable. Must be called before 4478 * invoking skb_seq_read() for the first time. 4479 */ 4480 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 4481 unsigned int to, struct skb_seq_state *st) 4482 { 4483 st->lower_offset = from; 4484 st->upper_offset = to; 4485 st->root_skb = st->cur_skb = skb; 4486 st->frag_idx = st->stepped_offset = 0; 4487 st->frag_data = NULL; 4488 st->frag_off = 0; 4489 } 4490 EXPORT_SYMBOL(skb_prepare_seq_read); 4491 4492 /** 4493 * skb_seq_read - Sequentially read skb data 4494 * @consumed: number of bytes consumed by the caller so far 4495 * @data: destination pointer for data to be returned 4496 * @st: state variable 4497 * 4498 * Reads a block of skb data at @consumed relative to the 4499 * lower offset specified to skb_prepare_seq_read(). Assigns 4500 * the head of the data block to @data and returns the length 4501 * of the block or 0 if the end of the skb data or the upper 4502 * offset has been reached. 4503 * 4504 * The caller is not required to consume all of the data 4505 * returned, i.e. @consumed is typically set to the number 4506 * of bytes already consumed and the next call to 4507 * skb_seq_read() will return the remaining part of the block. 4508 * 4509 * Note 1: The size of each block of data returned can be arbitrary, 4510 * this limitation is the cost for zerocopy sequential 4511 * reads of potentially non linear data. 4512 * 4513 * Note 2: Fragment lists within fragments are not implemented 4514 * at the moment, state->root_skb could be replaced with 4515 * a stack for this purpose. 4516 */ 4517 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 4518 struct skb_seq_state *st) 4519 { 4520 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 4521 skb_frag_t *frag; 4522 4523 if (unlikely(abs_offset >= st->upper_offset)) { 4524 if (st->frag_data) { 4525 kunmap_atomic(st->frag_data); 4526 st->frag_data = NULL; 4527 } 4528 return 0; 4529 } 4530 4531 next_skb: 4532 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 4533 4534 if (abs_offset < block_limit && !st->frag_data) { 4535 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 4536 return block_limit - abs_offset; 4537 } 4538 4539 if (!skb_frags_readable(st->cur_skb)) 4540 return 0; 4541 4542 if (st->frag_idx == 0 && !st->frag_data) 4543 st->stepped_offset += skb_headlen(st->cur_skb); 4544 4545 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 4546 unsigned int pg_idx, pg_off, pg_sz; 4547 4548 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 4549 4550 pg_idx = 0; 4551 pg_off = skb_frag_off(frag); 4552 pg_sz = skb_frag_size(frag); 4553 4554 if (skb_frag_must_loop(skb_frag_page(frag))) { 4555 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 4556 pg_off = offset_in_page(pg_off + st->frag_off); 4557 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 4558 PAGE_SIZE - pg_off); 4559 } 4560 4561 block_limit = pg_sz + st->stepped_offset; 4562 if (abs_offset < block_limit) { 4563 if (!st->frag_data) 4564 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 4565 4566 *data = (u8 *)st->frag_data + pg_off + 4567 (abs_offset - st->stepped_offset); 4568 4569 return block_limit - abs_offset; 4570 } 4571 4572 if (st->frag_data) { 4573 kunmap_atomic(st->frag_data); 4574 st->frag_data = NULL; 4575 } 4576 4577 st->stepped_offset += pg_sz; 4578 st->frag_off += pg_sz; 4579 if (st->frag_off == skb_frag_size(frag)) { 4580 st->frag_off = 0; 4581 st->frag_idx++; 4582 } 4583 } 4584 4585 if (st->frag_data) { 4586 kunmap_atomic(st->frag_data); 4587 st->frag_data = NULL; 4588 } 4589 4590 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 4591 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 4592 st->frag_idx = 0; 4593 goto next_skb; 4594 } else if (st->cur_skb->next) { 4595 st->cur_skb = st->cur_skb->next; 4596 st->frag_idx = 0; 4597 goto next_skb; 4598 } 4599 4600 return 0; 4601 } 4602 EXPORT_SYMBOL(skb_seq_read); 4603 4604 /** 4605 * skb_abort_seq_read - Abort a sequential read of skb data 4606 * @st: state variable 4607 * 4608 * Must be called if skb_seq_read() was not called until it 4609 * returned 0. 4610 */ 4611 void skb_abort_seq_read(struct skb_seq_state *st) 4612 { 4613 if (st->frag_data) 4614 kunmap_atomic(st->frag_data); 4615 } 4616 EXPORT_SYMBOL(skb_abort_seq_read); 4617 4618 /** 4619 * skb_copy_seq_read() - copy from a skb_seq_state to a buffer 4620 * @st: source skb_seq_state 4621 * @offset: offset in source 4622 * @to: destination buffer 4623 * @len: number of bytes to copy 4624 * 4625 * Copy @len bytes from @offset bytes into the source @st to the destination 4626 * buffer @to. `offset` should increase (or be unchanged) with each subsequent 4627 * call to this function. If offset needs to decrease from the previous use `st` 4628 * should be reset first. 4629 * 4630 * Return: 0 on success or -EINVAL if the copy ended early 4631 */ 4632 int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len) 4633 { 4634 const u8 *data; 4635 u32 sqlen; 4636 4637 for (;;) { 4638 sqlen = skb_seq_read(offset, &data, st); 4639 if (sqlen == 0) 4640 return -EINVAL; 4641 if (sqlen >= len) { 4642 memcpy(to, data, len); 4643 return 0; 4644 } 4645 memcpy(to, data, sqlen); 4646 to += sqlen; 4647 offset += sqlen; 4648 len -= sqlen; 4649 } 4650 } 4651 EXPORT_SYMBOL(skb_copy_seq_read); 4652 4653 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 4654 4655 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 4656 struct ts_config *conf, 4657 struct ts_state *state) 4658 { 4659 return skb_seq_read(offset, text, TS_SKB_CB(state)); 4660 } 4661 4662 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 4663 { 4664 skb_abort_seq_read(TS_SKB_CB(state)); 4665 } 4666 4667 /** 4668 * skb_find_text - Find a text pattern in skb data 4669 * @skb: the buffer to look in 4670 * @from: search offset 4671 * @to: search limit 4672 * @config: textsearch configuration 4673 * 4674 * Finds a pattern in the skb data according to the specified 4675 * textsearch configuration. Use textsearch_next() to retrieve 4676 * subsequent occurrences of the pattern. Returns the offset 4677 * to the first occurrence or UINT_MAX if no match was found. 4678 */ 4679 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 4680 unsigned int to, struct ts_config *config) 4681 { 4682 unsigned int patlen = config->ops->get_pattern_len(config); 4683 struct ts_state state; 4684 unsigned int ret; 4685 4686 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 4687 4688 config->get_next_block = skb_ts_get_next_block; 4689 config->finish = skb_ts_finish; 4690 4691 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 4692 4693 ret = textsearch_find(config, &state); 4694 return (ret + patlen <= to - from ? ret : UINT_MAX); 4695 } 4696 EXPORT_SYMBOL(skb_find_text); 4697 4698 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 4699 int offset, size_t size, size_t max_frags) 4700 { 4701 int i = skb_shinfo(skb)->nr_frags; 4702 4703 if (skb_can_coalesce(skb, i, page, offset)) { 4704 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 4705 } else if (i < max_frags) { 4706 skb_zcopy_downgrade_managed(skb); 4707 get_page(page); 4708 skb_fill_page_desc_noacc(skb, i, page, offset, size); 4709 } else { 4710 return -EMSGSIZE; 4711 } 4712 4713 return 0; 4714 } 4715 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 4716 4717 /** 4718 * skb_pull_rcsum - pull skb and update receive checksum 4719 * @skb: buffer to update 4720 * @len: length of data pulled 4721 * 4722 * This function performs an skb_pull on the packet and updates 4723 * the CHECKSUM_COMPLETE checksum. It should be used on 4724 * receive path processing instead of skb_pull unless you know 4725 * that the checksum difference is zero (e.g., a valid IP header) 4726 * or you are setting ip_summed to CHECKSUM_NONE. 4727 */ 4728 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 4729 { 4730 unsigned char *data = skb->data; 4731 4732 BUG_ON(len > skb->len); 4733 __skb_pull(skb, len); 4734 skb_postpull_rcsum(skb, data, len); 4735 return skb->data; 4736 } 4737 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 4738 4739 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 4740 { 4741 skb_frag_t head_frag; 4742 struct page *page; 4743 4744 page = virt_to_head_page(frag_skb->head); 4745 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data - 4746 (unsigned char *)page_address(page), 4747 skb_headlen(frag_skb)); 4748 return head_frag; 4749 } 4750 4751 struct sk_buff *skb_segment_list(struct sk_buff *skb, 4752 netdev_features_t features, 4753 unsigned int offset) 4754 { 4755 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 4756 unsigned int tnl_hlen = skb_tnl_header_len(skb); 4757 unsigned int delta_len = 0; 4758 struct sk_buff *tail = NULL; 4759 struct sk_buff *nskb, *tmp; 4760 int len_diff, err; 4761 4762 /* Only skb_gro_receive_list generated skbs arrive here */ 4763 DEBUG_NET_WARN_ON_ONCE(!(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)); 4764 4765 skb_push(skb, -skb_network_offset(skb) + offset); 4766 4767 /* Ensure the head is writeable before touching the shared info */ 4768 err = skb_unclone(skb, GFP_ATOMIC); 4769 if (err) 4770 goto err_linearize; 4771 4772 skb_shinfo(skb)->frag_list = NULL; 4773 4774 while (list_skb) { 4775 nskb = list_skb; 4776 list_skb = list_skb->next; 4777 4778 DEBUG_NET_WARN_ON_ONCE(nskb->sk); 4779 4780 err = 0; 4781 if (skb_shared(nskb)) { 4782 tmp = skb_clone(nskb, GFP_ATOMIC); 4783 if (tmp) { 4784 consume_skb(nskb); 4785 nskb = tmp; 4786 err = skb_unclone(nskb, GFP_ATOMIC); 4787 } else { 4788 err = -ENOMEM; 4789 } 4790 } 4791 4792 if (!tail) 4793 skb->next = nskb; 4794 else 4795 tail->next = nskb; 4796 4797 /* nskb may be a clone with a NULL ->next: keep the remaining 4798 * members linked so that err_linearize frees them. 4799 */ 4800 nskb->next = list_skb; 4801 4802 if (unlikely(err)) 4803 goto err_linearize; 4804 4805 tail = nskb; 4806 4807 delta_len += nskb->len; 4808 4809 skb_push(nskb, -skb_network_offset(nskb) + offset); 4810 4811 skb_release_head_state(nskb); 4812 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); 4813 __copy_skb_header(nskb, skb); 4814 4815 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 4816 nskb->transport_header += len_diff; 4817 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 4818 nskb->data - tnl_hlen, 4819 offset + tnl_hlen); 4820 4821 if (skb_needs_linearize(nskb, features) && 4822 __skb_linearize(nskb)) 4823 goto err_linearize; 4824 } 4825 4826 skb->data_len = skb->data_len - delta_len; 4827 skb->len = skb->len - delta_len; 4828 4829 skb_gso_reset(skb); 4830 4831 skb->prev = tail; 4832 4833 if (skb_needs_linearize(skb, features) && 4834 __skb_linearize(skb)) 4835 goto err_linearize; 4836 4837 skb_get(skb); 4838 4839 return skb; 4840 4841 err_linearize: 4842 kfree_skb_list(skb->next); 4843 skb->next = NULL; 4844 return ERR_PTR(-ENOMEM); 4845 } 4846 EXPORT_SYMBOL_GPL(skb_segment_list); 4847 4848 /** 4849 * skb_segment - Perform protocol segmentation on skb. 4850 * @head_skb: buffer to segment 4851 * @features: features for the output path (see dev->features) 4852 * 4853 * This function performs segmentation on the given skb. It returns 4854 * a pointer to the first in a list of new skbs for the segments. 4855 * In case of error it returns ERR_PTR(err). 4856 */ 4857 struct sk_buff *skb_segment(struct sk_buff *head_skb, 4858 netdev_features_t features) 4859 { 4860 struct sk_buff *segs = NULL; 4861 struct sk_buff *tail = NULL; 4862 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 4863 unsigned int max_segs = SKB_GSO_CB(head_skb)->max_segs; 4864 unsigned int mss = skb_shinfo(head_skb)->gso_size; 4865 bool gso_by_frags = mss == GSO_BY_FRAGS; 4866 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 4867 unsigned int offset = doffset; 4868 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 4869 unsigned int partial_segs = 0; 4870 unsigned int headroom; 4871 unsigned int len = head_skb->len; 4872 struct sk_buff *frag_skb; 4873 skb_frag_t *frag; 4874 __be16 proto; 4875 bool csum, sg; 4876 int err = -ENOMEM; 4877 int i = 0; 4878 int nfrags, pos; 4879 4880 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && 4881 !gso_by_frags && mss != skb_headlen(head_skb)) { 4882 struct sk_buff *check_skb; 4883 4884 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { 4885 if (skb_headlen(check_skb) && !check_skb->head_frag) { 4886 /* gso_size is untrusted, and we have a frag_list with 4887 * a linear non head_frag item. 4888 * 4889 * If head_skb's headlen does not fit requested gso_size, 4890 * it means that the frag_list members do NOT terminate 4891 * on exact gso_size boundaries. Hence we cannot perform 4892 * skb_frag_t page sharing. Therefore we must fallback to 4893 * copying the frag_list skbs; we do so by disabling SG. 4894 */ 4895 features &= ~NETIF_F_SG; 4896 break; 4897 } 4898 } 4899 } 4900 4901 __skb_push(head_skb, doffset); 4902 proto = skb_network_protocol(head_skb, NULL); 4903 if (unlikely(!proto)) 4904 return ERR_PTR(-EINVAL); 4905 4906 sg = !!(features & NETIF_F_SG); 4907 csum = !!can_checksum_protocol(features, proto); 4908 4909 if (sg && csum && !gso_by_frags) { 4910 if (!max_segs && !(features & NETIF_F_GSO_PARTIAL)) { 4911 struct sk_buff *iter; 4912 unsigned int frag_len; 4913 4914 if (!list_skb || 4915 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 4916 goto normal; 4917 4918 /* If we get here then all the required 4919 * GSO features except frag_list are supported. 4920 * Try to split the SKB to multiple GSO SKBs 4921 * with no frag_list. 4922 * Currently we can do that only when the buffers don't 4923 * have a linear part and all the buffers except 4924 * the last are of the same length. 4925 */ 4926 frag_len = list_skb->len; 4927 skb_walk_frags(head_skb, iter) { 4928 if (frag_len != iter->len && iter->next) 4929 goto normal; 4930 if (skb_headlen(iter) && !iter->head_frag) 4931 goto normal; 4932 4933 len -= iter->len; 4934 } 4935 4936 if (len != frag_len) 4937 goto normal; 4938 } 4939 4940 /* GSO partial only requires that we trim off any excess that 4941 * doesn't fit into an MSS sized block, so take care of that 4942 * now. 4943 */ 4944 DEBUG_NET_WARN_ON_ONCE(!max_segs && len / mss > GSO_MAX_SEGS); 4945 if (max_segs) 4946 partial_segs = min(len / mss, max_segs); 4947 else 4948 partial_segs = min(len / mss, GSO_MAX_SEGS); 4949 if (partial_segs > 1) 4950 mss *= partial_segs; 4951 else 4952 partial_segs = 0; 4953 } 4954 4955 normal: 4956 headroom = skb_headroom(head_skb); 4957 pos = skb_headlen(head_skb); 4958 4959 if (skb_orphan_frags(head_skb, GFP_ATOMIC)) 4960 return ERR_PTR(-ENOMEM); 4961 4962 nfrags = skb_shinfo(head_skb)->nr_frags; 4963 frag = skb_shinfo(head_skb)->frags; 4964 frag_skb = head_skb; 4965 4966 do { 4967 struct sk_buff *nskb; 4968 skb_frag_t *nskb_frag; 4969 int hsize; 4970 int size; 4971 4972 if (unlikely(gso_by_frags)) { 4973 len = list_skb->len; 4974 } else { 4975 len = head_skb->len - offset; 4976 if (len > mss) 4977 len = mss; 4978 } 4979 4980 hsize = skb_headlen(head_skb) - offset; 4981 4982 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 4983 (skb_headlen(list_skb) == len || sg)) { 4984 if (unlikely(skb_headlen(list_skb) > len)) { 4985 DEBUG_NET_WARN_ON_ONCE(1); 4986 err = -EINVAL; 4987 goto err; 4988 } 4989 4990 nskb = skb_clone(list_skb, GFP_ATOMIC); 4991 if (unlikely(!nskb)) 4992 goto err; 4993 4994 i = 0; 4995 nfrags = skb_shinfo(list_skb)->nr_frags; 4996 frag = skb_shinfo(list_skb)->frags; 4997 frag_skb = list_skb; 4998 pos += skb_headlen(list_skb); 4999 5000 while (pos < offset + len) { 5001 if (unlikely(i >= nfrags)) { 5002 DEBUG_NET_WARN_ON_ONCE(1); 5003 kfree_skb(nskb); 5004 err = -EINVAL; 5005 goto err; 5006 } 5007 5008 size = skb_frag_size(frag); 5009 if (pos + size > offset + len) 5010 break; 5011 5012 i++; 5013 pos += size; 5014 frag++; 5015 } 5016 5017 list_skb = list_skb->next; 5018 5019 if (unlikely(pskb_trim(nskb, len))) { 5020 kfree_skb(nskb); 5021 goto err; 5022 } 5023 5024 hsize = skb_end_offset(nskb); 5025 if (skb_cow_head(nskb, doffset + headroom)) { 5026 kfree_skb(nskb); 5027 goto err; 5028 } 5029 5030 nskb->truesize += skb_end_offset(nskb) - hsize; 5031 skb_release_head_state(nskb); 5032 __skb_push(nskb, doffset); 5033 } else { 5034 if (hsize < 0) 5035 hsize = 0; 5036 if (hsize > len || !sg) 5037 hsize = len; 5038 5039 nskb = __alloc_skb(hsize + doffset + headroom, 5040 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 5041 NUMA_NO_NODE); 5042 5043 if (unlikely(!nskb)) 5044 goto err; 5045 5046 skb_reserve(nskb, headroom); 5047 __skb_put(nskb, doffset); 5048 } 5049 5050 if (segs) 5051 tail->next = nskb; 5052 else 5053 segs = nskb; 5054 tail = nskb; 5055 5056 __copy_skb_header(nskb, head_skb); 5057 5058 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 5059 skb_reset_mac_len(nskb); 5060 5061 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 5062 nskb->data - tnl_hlen, 5063 doffset + tnl_hlen); 5064 5065 if (nskb->len == len + doffset) 5066 goto perform_csum_check; 5067 5068 if (!sg) { 5069 if (!csum) { 5070 if (!nskb->remcsum_offload) 5071 nskb->ip_summed = CHECKSUM_NONE; 5072 SKB_GSO_CB(nskb)->csum = 5073 skb_copy_and_csum_bits(head_skb, offset, 5074 skb_put(nskb, 5075 len), 5076 len); 5077 SKB_GSO_CB(nskb)->csum_start = 5078 skb_headroom(nskb) + doffset; 5079 } else { 5080 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) 5081 goto err; 5082 } 5083 continue; 5084 } 5085 5086 nskb_frag = skb_shinfo(nskb)->frags; 5087 5088 skb_copy_from_linear_data_offset(head_skb, offset, 5089 skb_put(nskb, hsize), hsize); 5090 5091 skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags | 5092 skb_shinfo(frag_skb)->flags) & 5093 SKBFL_SHARED_FRAG; 5094 5095 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 5096 goto err; 5097 5098 while (pos < offset + len) { 5099 if (i >= nfrags) { 5100 if (skb_shinfo(nskb)->nr_frags && 5101 nskb->pp_recycle != list_skb->pp_recycle) { 5102 nskb->data_len = pos - offset - hsize; 5103 nskb->len += nskb->data_len; 5104 nskb->truesize += nskb->data_len; 5105 if (__skb_linearize(nskb)) 5106 goto err; 5107 hsize = pos - offset; 5108 nskb_frag = skb_shinfo(nskb)->frags; 5109 } 5110 if (skb_orphan_frags(list_skb, GFP_ATOMIC) || 5111 skb_zerocopy_clone(nskb, list_skb, 5112 GFP_ATOMIC)) 5113 goto err; 5114 5115 i = 0; 5116 nfrags = skb_shinfo(list_skb)->nr_frags; 5117 frag = skb_shinfo(list_skb)->frags; 5118 frag_skb = list_skb; 5119 5120 skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG; 5121 5122 if (!skb_headlen(list_skb)) { 5123 if (unlikely(!nfrags)) { 5124 DEBUG_NET_WARN_ON_ONCE(1); 5125 err = -EINVAL; 5126 goto err; 5127 } 5128 } else { 5129 if (unlikely(!list_skb->head_frag)) { 5130 DEBUG_NET_WARN_ON_ONCE(1); 5131 err = -EINVAL; 5132 goto err; 5133 } 5134 5135 /* to make room for head_frag. */ 5136 i--; 5137 frag--; 5138 } 5139 5140 list_skb = list_skb->next; 5141 } 5142 5143 if (unlikely(skb_shinfo(nskb)->nr_frags >= 5144 MAX_SKB_FRAGS)) { 5145 net_warn_ratelimited( 5146 "skb_segment: too many frags: %u %u\n", 5147 pos, mss); 5148 err = -EINVAL; 5149 goto err; 5150 } 5151 5152 if (!skb_shinfo(nskb)->nr_frags) 5153 nskb->pp_recycle = frag_skb->pp_recycle; 5154 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 5155 skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags); 5156 size = skb_frag_size(nskb_frag); 5157 5158 if (pos < offset) { 5159 skb_frag_off_add(nskb_frag, offset - pos); 5160 skb_frag_size_sub(nskb_frag, offset - pos); 5161 } 5162 5163 skb_shinfo(nskb)->nr_frags++; 5164 5165 if (pos + size <= offset + len) { 5166 i++; 5167 frag++; 5168 pos += size; 5169 } else { 5170 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 5171 goto skip_fraglist; 5172 } 5173 5174 nskb_frag++; 5175 } 5176 5177 skip_fraglist: 5178 nskb->data_len = len - hsize; 5179 nskb->len += nskb->data_len; 5180 nskb->truesize += nskb->data_len; 5181 5182 perform_csum_check: 5183 if (!csum) { 5184 if (skb_has_shared_frag(nskb) && 5185 __skb_linearize(nskb)) 5186 goto err; 5187 5188 if (!nskb->remcsum_offload) 5189 nskb->ip_summed = CHECKSUM_NONE; 5190 SKB_GSO_CB(nskb)->csum = 5191 skb_checksum(nskb, doffset, 5192 nskb->len - doffset, 0); 5193 SKB_GSO_CB(nskb)->csum_start = 5194 skb_headroom(nskb) + doffset; 5195 } 5196 } while ((offset += len) < head_skb->len); 5197 5198 /* Some callers want to get the end of the list. 5199 * Put it in segs->prev to avoid walking the list. 5200 * (see validate_xmit_skb_list() for example) 5201 */ 5202 segs->prev = tail; 5203 5204 if (partial_segs) { 5205 struct sk_buff *iter; 5206 int type = skb_shinfo(head_skb)->gso_type; 5207 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 5208 5209 /* Update type to add partial and then remove dodgy if set */ 5210 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 5211 type &= ~SKB_GSO_DODGY; 5212 5213 /* Update GSO info and prepare to start updating headers on 5214 * our way back down the stack of protocols. 5215 */ 5216 for (iter = segs; iter; iter = iter->next) { 5217 skb_shinfo(iter)->gso_size = gso_size; 5218 skb_shinfo(iter)->gso_segs = partial_segs; 5219 skb_shinfo(iter)->gso_type = type; 5220 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 5221 } 5222 5223 if (tail->len - doffset <= gso_size) 5224 skb_gso_reset(tail); 5225 else if (tail != segs) 5226 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 5227 } 5228 5229 /* Following permits correct backpressure, for protocols 5230 * using skb_set_owner_w(). 5231 * Idea is to tranfert ownership from head_skb to last segment. 5232 */ 5233 if (head_skb->destructor == sock_wfree) { 5234 swap(tail->truesize, head_skb->truesize); 5235 swap(tail->destructor, head_skb->destructor); 5236 swap(tail->sk, head_skb->sk); 5237 } 5238 return segs; 5239 5240 err: 5241 kfree_skb_list(segs); 5242 return ERR_PTR(err); 5243 } 5244 EXPORT_SYMBOL_GPL(skb_segment); 5245 5246 #ifdef CONFIG_SKB_EXTENSIONS 5247 #define SKB_EXT_ALIGN_VALUE 8 5248 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 5249 5250 static const u8 skb_ext_type_len[] = { 5251 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 5252 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 5253 #endif 5254 #ifdef CONFIG_XFRM 5255 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 5256 #endif 5257 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 5258 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 5259 #endif 5260 #if IS_ENABLED(CONFIG_MPTCP) 5261 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 5262 #endif 5263 #if IS_ENABLED(CONFIG_MCTP_FLOWS) 5264 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow), 5265 #endif 5266 #if IS_ENABLED(CONFIG_INET_PSP) 5267 [SKB_EXT_PSP] = SKB_EXT_CHUNKSIZEOF(struct psp_skb_ext), 5268 #endif 5269 #if IS_ENABLED(CONFIG_CAN) 5270 [SKB_EXT_CAN] = SKB_EXT_CHUNKSIZEOF(struct can_skb_ext), 5271 #endif 5272 }; 5273 5274 static __always_inline __no_profile unsigned int skb_ext_total_length(void) 5275 { 5276 unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext); 5277 int i; 5278 5279 for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++) 5280 l += skb_ext_type_len[i]; 5281 5282 return l; 5283 } 5284 5285 static noinline void __init __no_profile skb_extensions_init(void) 5286 { 5287 BUILD_BUG_ON(SKB_EXT_NUM > 8); 5288 BUILD_BUG_ON(skb_ext_total_length() > 255); 5289 5290 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 5291 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 5292 0, 5293 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 5294 NULL); 5295 } 5296 #else 5297 static void skb_extensions_init(void) {} 5298 #endif 5299 5300 /* The SKB kmem_cache slab is critical for network performance. Never 5301 * merge/alias the slab with similar sized objects. This avoids fragmentation 5302 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs. 5303 */ 5304 #ifndef CONFIG_SLUB_TINY 5305 #define FLAG_SKB_NO_MERGE SLAB_NO_MERGE 5306 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */ 5307 #define FLAG_SKB_NO_MERGE 0 5308 #endif 5309 5310 void __init skb_init(void) 5311 { 5312 net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", 5313 sizeof(struct sk_buff), 5314 0, 5315 SLAB_HWCACHE_ALIGN|SLAB_PANIC| 5316 FLAG_SKB_NO_MERGE, 5317 offsetof(struct sk_buff, cb), 5318 sizeof_field(struct sk_buff, cb), 5319 NULL); 5320 skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache); 5321 5322 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 5323 sizeof(struct sk_buff_fclones), 5324 0, 5325 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 5326 NULL); 5327 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes. 5328 * struct skb_shared_info is located at the end of skb->head, 5329 * and should not be copied to/from user. 5330 */ 5331 net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", 5332 SKB_SMALL_HEAD_CACHE_SIZE, 5333 0, 5334 SLAB_HWCACHE_ALIGN | SLAB_PANIC, 5335 0, 5336 SKB_SMALL_HEAD_HEADROOM, 5337 NULL); 5338 skb_extensions_init(); 5339 } 5340 5341 static int 5342 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 5343 unsigned int recursion_level) 5344 { 5345 int start = skb_headlen(skb); 5346 int i, copy = start - offset; 5347 struct sk_buff *frag_iter; 5348 int elt = 0; 5349 5350 if (unlikely(recursion_level >= 24)) 5351 return -EMSGSIZE; 5352 5353 if (copy > 0) { 5354 if (copy > len) 5355 copy = len; 5356 sg_set_buf(sg, skb->data + offset, copy); 5357 elt++; 5358 if ((len -= copy) == 0) 5359 return elt; 5360 offset += copy; 5361 } 5362 5363 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5364 int end; 5365 5366 WARN_ON(start > offset + len); 5367 5368 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 5369 if ((copy = end - offset) > 0) { 5370 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5371 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5372 return -EMSGSIZE; 5373 5374 if (copy > len) 5375 copy = len; 5376 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 5377 skb_frag_off(frag) + offset - start); 5378 elt++; 5379 if (!(len -= copy)) 5380 return elt; 5381 offset += copy; 5382 } 5383 start = end; 5384 } 5385 5386 skb_walk_frags(skb, frag_iter) { 5387 int end, ret; 5388 5389 WARN_ON(start > offset + len); 5390 5391 end = start + frag_iter->len; 5392 if ((copy = end - offset) > 0) { 5393 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 5394 return -EMSGSIZE; 5395 5396 if (copy > len) 5397 copy = len; 5398 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 5399 copy, recursion_level + 1); 5400 if (unlikely(ret < 0)) 5401 return ret; 5402 elt += ret; 5403 if ((len -= copy) == 0) 5404 return elt; 5405 offset += copy; 5406 } 5407 start = end; 5408 } 5409 if (unlikely(len)) { 5410 DEBUG_NET_WARN_ON_ONCE(1); 5411 return -EINVAL; 5412 } 5413 return elt; 5414 } 5415 5416 /** 5417 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 5418 * @skb: Socket buffer containing the buffers to be mapped 5419 * @sg: The scatter-gather list to map into 5420 * @offset: The offset into the buffer's contents to start mapping 5421 * @len: Length of buffer space to be mapped 5422 * 5423 * Fill the specified scatter-gather list with mappings/pointers into a 5424 * region of the buffer space attached to a socket buffer. Returns either 5425 * the number of scatterlist items used, or -EMSGSIZE if the contents 5426 * could not fit. 5427 */ 5428 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 5429 { 5430 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 5431 5432 if (nsg <= 0) 5433 return nsg; 5434 5435 sg_mark_end(&sg[nsg - 1]); 5436 5437 return nsg; 5438 } 5439 EXPORT_SYMBOL_GPL(skb_to_sgvec); 5440 5441 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 5442 * sglist without mark the sg which contain last skb data as the end. 5443 * So the caller can mannipulate sg list as will when padding new data after 5444 * the first call without calling sg_unmark_end to expend sg list. 5445 * 5446 * Scenario to use skb_to_sgvec_nomark: 5447 * 1. sg_init_table 5448 * 2. skb_to_sgvec_nomark(payload1) 5449 * 3. skb_to_sgvec_nomark(payload2) 5450 * 5451 * This is equivalent to: 5452 * 1. sg_init_table 5453 * 2. skb_to_sgvec(payload1) 5454 * 3. sg_unmark_end 5455 * 4. skb_to_sgvec(payload2) 5456 * 5457 * When mapping multiple payload conditionally, skb_to_sgvec_nomark 5458 * is more preferable. 5459 */ 5460 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 5461 int offset, int len) 5462 { 5463 return __skb_to_sgvec(skb, sg, offset, len, 0); 5464 } 5465 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 5466 5467 5468 5469 /** 5470 * skb_cow_data - Check that a socket buffer's data buffers are writable 5471 * @skb: The socket buffer to check. 5472 * @tailbits: Amount of trailing space to be added 5473 * @trailer: Returned pointer to the skb where the @tailbits space begins 5474 * 5475 * Make sure that the data buffers attached to a socket buffer are 5476 * writable. If they are not, private copies are made of the data buffers 5477 * and the socket buffer is set to use these instead. 5478 * 5479 * If @tailbits is given, make sure that there is space to write @tailbits 5480 * bytes of data beyond current end of socket buffer. @trailer will be 5481 * set to point to the skb in which this space begins. 5482 * 5483 * The number of scatterlist elements required to completely map the 5484 * COW'd and extended socket buffer will be returned. 5485 */ 5486 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 5487 { 5488 int copyflag; 5489 int elt; 5490 struct sk_buff *skb1, **skb_p; 5491 5492 /* If skb is cloned or its head is paged, reallocate 5493 * head pulling out all the pages (pages are considered not writable 5494 * at the moment even if they are anonymous). 5495 */ 5496 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 5497 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 5498 return -ENOMEM; 5499 5500 /* Easy case. Most of packets will go this way. */ 5501 if (!skb_has_frag_list(skb)) { 5502 /* A little of trouble, not enough of space for trailer. 5503 * This should not happen, when stack is tuned to generate 5504 * good frames. OK, on miss we reallocate and reserve even more 5505 * space, 128 bytes is fair. */ 5506 5507 if (skb_tailroom(skb) < tailbits && 5508 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 5509 return -ENOMEM; 5510 5511 /* Voila! */ 5512 *trailer = skb; 5513 return 1; 5514 } 5515 5516 /* Misery. We are in troubles, going to mincer fragments... */ 5517 5518 elt = 1; 5519 skb_p = &skb_shinfo(skb)->frag_list; 5520 copyflag = 0; 5521 5522 while ((skb1 = *skb_p) != NULL) { 5523 int ntail = 0; 5524 5525 /* The fragment is partially pulled by someone, 5526 * this can happen on input. Copy it and everything 5527 * after it. */ 5528 5529 if (skb_shared(skb1)) 5530 copyflag = 1; 5531 5532 /* If the skb is the last, worry about trailer. */ 5533 5534 if (skb1->next == NULL && tailbits) { 5535 if (skb_shinfo(skb1)->nr_frags || 5536 skb_has_frag_list(skb1) || 5537 skb_tailroom(skb1) < tailbits) 5538 ntail = tailbits + 128; 5539 } 5540 5541 if (copyflag || 5542 skb_cloned(skb1) || 5543 ntail || 5544 skb_shinfo(skb1)->nr_frags || 5545 skb_has_frag_list(skb1)) { 5546 struct sk_buff *skb2; 5547 5548 /* Fuck, we are miserable poor guys... */ 5549 if (ntail == 0) 5550 skb2 = skb_copy(skb1, GFP_ATOMIC); 5551 else 5552 skb2 = skb_copy_expand(skb1, 5553 skb_headroom(skb1), 5554 ntail, 5555 GFP_ATOMIC); 5556 if (unlikely(skb2 == NULL)) 5557 return -ENOMEM; 5558 5559 if (skb1->sk) 5560 skb_set_owner_w(skb2, skb1->sk); 5561 5562 /* Looking around. Are we still alive? 5563 * OK, link new skb, drop old one */ 5564 5565 skb2->next = skb1->next; 5566 *skb_p = skb2; 5567 kfree_skb(skb1); 5568 skb1 = skb2; 5569 } 5570 elt++; 5571 *trailer = skb1; 5572 skb_p = &skb1->next; 5573 } 5574 5575 return elt; 5576 } 5577 EXPORT_SYMBOL_GPL(skb_cow_data); 5578 5579 void sock_rmem_free(struct sk_buff *skb) 5580 { 5581 struct sock *sk = skb->sk; 5582 5583 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 5584 } 5585 5586 static void skb_set_err_queue(struct sk_buff *skb) 5587 { 5588 /* The error-queue test in skb_is_err_queue() matches this marker 5589 * with the sock_rmem_free destructor installed by sock_queue_err_skb(). 5590 */ 5591 skb->pkt_type = PACKET_OUTGOING; 5592 BUILD_BUG_ON(PACKET_OUTGOING == 0); 5593 } 5594 5595 /* 5596 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 5597 */ 5598 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 5599 { 5600 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 5601 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 5602 return -ENOMEM; 5603 5604 skb_orphan(skb); 5605 skb->sk = sk; 5606 skb->destructor = sock_rmem_free; 5607 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 5608 skb_set_err_queue(skb); 5609 5610 /* before exiting rcu section, make sure dst is refcounted */ 5611 skb_dst_force(skb); 5612 5613 skb_queue_tail(&sk->sk_error_queue, skb); 5614 if (!sock_flag(sk, SOCK_DEAD)) 5615 sk_error_report(sk); 5616 return 0; 5617 } 5618 EXPORT_SYMBOL(sock_queue_err_skb); 5619 5620 static bool is_icmp_err_skb(const struct sk_buff *skb) 5621 { 5622 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 5623 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 5624 } 5625 5626 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 5627 { 5628 struct sk_buff_head *q = &sk->sk_error_queue; 5629 struct sk_buff *skb, *skb_next = NULL; 5630 bool icmp_next = false; 5631 unsigned long flags; 5632 5633 if (skb_queue_empty_lockless(q)) 5634 return NULL; 5635 5636 spin_lock_irqsave(&q->lock, flags); 5637 skb = __skb_dequeue(q); 5638 if (skb && (skb_next = skb_peek(q))) { 5639 icmp_next = is_icmp_err_skb(skb_next); 5640 if (icmp_next) 5641 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 5642 } 5643 spin_unlock_irqrestore(&q->lock, flags); 5644 5645 if (is_icmp_err_skb(skb) && !icmp_next) 5646 sk->sk_err = 0; 5647 5648 if (skb_next) 5649 sk_error_report(sk); 5650 5651 return skb; 5652 } 5653 EXPORT_SYMBOL(sock_dequeue_err_skb); 5654 5655 /** 5656 * skb_clone_sk - create clone of skb, and take reference to socket 5657 * @skb: the skb to clone 5658 * 5659 * This function creates a clone of a buffer that holds a reference on 5660 * sk_refcnt. Buffers created via this function are meant to be 5661 * returned using sock_queue_err_skb, or free via kfree_skb. 5662 * 5663 * When passing buffers allocated with this function to sock_queue_err_skb 5664 * it is necessary to wrap the call with sock_hold/sock_put in order to 5665 * prevent the socket from being released prior to being enqueued on 5666 * the sk_error_queue. 5667 */ 5668 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 5669 { 5670 struct sock *sk = skb->sk; 5671 struct sk_buff *clone; 5672 5673 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 5674 return NULL; 5675 5676 clone = skb_clone(skb, GFP_ATOMIC); 5677 if (!clone) { 5678 sock_put(sk); 5679 return NULL; 5680 } 5681 5682 clone->sk = sk; 5683 clone->destructor = sock_efree; 5684 5685 return clone; 5686 } 5687 EXPORT_SYMBOL(skb_clone_sk); 5688 5689 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 5690 struct sock *sk, 5691 int tstype, 5692 bool opt_stats) 5693 { 5694 struct sock_exterr_skb *serr; 5695 int err; 5696 5697 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 5698 5699 serr = SKB_EXT_ERR(skb); 5700 memset(serr, 0, sizeof(*serr)); 5701 serr->ee.ee_errno = ENOMSG; 5702 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 5703 serr->ee.ee_info = tstype; 5704 serr->opt_stats = opt_stats; 5705 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 5706 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) { 5707 serr->ee.ee_data = skb_shinfo(skb)->tskey; 5708 if (sk_is_tcp(sk)) 5709 serr->ee.ee_data -= atomic_read(&sk->sk_tskey); 5710 } 5711 5712 err = sock_queue_err_skb(sk, skb); 5713 5714 if (err) 5715 kfree_skb(skb); 5716 } 5717 5718 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 5719 { 5720 struct socket *sock; 5721 struct file *file; 5722 bool ret = false; 5723 5724 if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data))) 5725 return true; 5726 5727 /* The sk pointer remains valid as long as the skb is. The sk_socket and 5728 * file pointer may become NULL if the socket is closed. Both structures 5729 * (including file->cred) are RCU freed which means they can be accessed 5730 * within a RCU read section. 5731 */ 5732 rcu_read_lock(); 5733 sock = READ_ONCE(sk->sk_socket); 5734 if (!sock) 5735 goto out; 5736 file = READ_ONCE(sock->file); 5737 if (!file) 5738 goto out; 5739 ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW); 5740 out: 5741 rcu_read_unlock(); 5742 return ret; 5743 } 5744 5745 void skb_complete_tx_timestamp(struct sk_buff *skb, 5746 struct skb_shared_hwtstamps *hwtstamps) 5747 { 5748 struct sock *sk = skb->sk; 5749 5750 if (!skb_may_tx_timestamp(sk, false)) 5751 goto err; 5752 5753 /* Take a reference to prevent skb_orphan() from freeing the socket, 5754 * but only if the socket refcount is not zero. 5755 */ 5756 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5757 *skb_hwtstamps(skb) = *hwtstamps; 5758 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 5759 sock_put(sk); 5760 return; 5761 } 5762 5763 err: 5764 kfree_skb(skb); 5765 } 5766 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 5767 5768 static bool skb_tstamp_tx_report_so_timestamping(struct sk_buff *skb, 5769 struct skb_shared_hwtstamps *hwtstamps, 5770 int tstype) 5771 { 5772 switch (tstype) { 5773 case SCM_TSTAMP_SCHED: 5774 return skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP; 5775 case SCM_TSTAMP_SND: 5776 return skb_shinfo(skb)->tx_flags & (hwtstamps ? SKBTX_HW_TSTAMP_NOBPF : 5777 SKBTX_SW_TSTAMP); 5778 case SCM_TSTAMP_ACK: 5779 return TCP_SKB_CB(skb)->txstamp_ack & TSTAMP_ACK_SK; 5780 case SCM_TSTAMP_COMPLETION: 5781 return skb_shinfo(skb)->tx_flags & SKBTX_COMPLETION_TSTAMP; 5782 } 5783 5784 return false; 5785 } 5786 5787 static void skb_tstamp_tx_report_bpf_timestamping(struct sk_buff *skb, 5788 struct skb_shared_hwtstamps *hwtstamps, 5789 struct sock *sk, 5790 int tstype) 5791 { 5792 int op; 5793 5794 switch (tstype) { 5795 case SCM_TSTAMP_SCHED: 5796 op = BPF_SOCK_OPS_TSTAMP_SCHED_CB; 5797 break; 5798 case SCM_TSTAMP_SND: 5799 if (hwtstamps) { 5800 op = BPF_SOCK_OPS_TSTAMP_SND_HW_CB; 5801 *skb_hwtstamps(skb) = *hwtstamps; 5802 } else { 5803 op = BPF_SOCK_OPS_TSTAMP_SND_SW_CB; 5804 } 5805 break; 5806 case SCM_TSTAMP_ACK: 5807 op = BPF_SOCK_OPS_TSTAMP_ACK_CB; 5808 break; 5809 default: 5810 return; 5811 } 5812 5813 bpf_skops_tx_timestamping(sk, skb, op); 5814 } 5815 5816 void __skb_tstamp_tx(struct sk_buff *orig_skb, 5817 const struct sk_buff *ack_skb, 5818 struct skb_shared_hwtstamps *hwtstamps, 5819 struct sock *sk, int tstype) 5820 { 5821 struct sk_buff *skb; 5822 bool tsonly, opt_stats = false; 5823 u32 tsflags; 5824 5825 if (!sk) 5826 return; 5827 5828 if (skb_shinfo(orig_skb)->tx_flags & SKBTX_BPF) 5829 skb_tstamp_tx_report_bpf_timestamping(orig_skb, hwtstamps, 5830 sk, tstype); 5831 5832 if (!skb_tstamp_tx_report_so_timestamping(orig_skb, hwtstamps, tstype)) 5833 return; 5834 5835 tsflags = READ_ONCE(sk->sk_tsflags); 5836 if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 5837 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 5838 return; 5839 5840 tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 5841 if (!skb_may_tx_timestamp(sk, tsonly)) 5842 return; 5843 5844 if (tsonly) { 5845 #ifdef CONFIG_INET 5846 if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) && 5847 sk_is_tcp(sk)) { 5848 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 5849 ack_skb); 5850 opt_stats = true; 5851 } else 5852 #endif 5853 skb = alloc_skb(0, GFP_ATOMIC); 5854 } else { 5855 skb = skb_clone(orig_skb, GFP_ATOMIC); 5856 5857 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) { 5858 kfree_skb(skb); 5859 return; 5860 } 5861 } 5862 if (!skb) 5863 return; 5864 5865 if (tsonly) { 5866 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 5867 SKBTX_ANY_TSTAMP; 5868 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 5869 } 5870 5871 if (hwtstamps) 5872 *skb_hwtstamps(skb) = *hwtstamps; 5873 else 5874 __net_timestamp(skb); 5875 5876 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 5877 } 5878 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 5879 5880 void skb_tstamp_tx(struct sk_buff *orig_skb, 5881 struct skb_shared_hwtstamps *hwtstamps) 5882 { 5883 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 5884 SCM_TSTAMP_SND); 5885 } 5886 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 5887 5888 #ifdef CONFIG_WIRELESS 5889 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 5890 { 5891 struct sock *sk = skb->sk; 5892 struct sock_exterr_skb *serr; 5893 int err = 1; 5894 5895 skb->wifi_acked_valid = 1; 5896 skb->wifi_acked = acked; 5897 5898 serr = SKB_EXT_ERR(skb); 5899 memset(serr, 0, sizeof(*serr)); 5900 serr->ee.ee_errno = ENOMSG; 5901 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 5902 5903 /* Take a reference to prevent skb_orphan() from freeing the socket, 5904 * but only if the socket refcount is not zero. 5905 */ 5906 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 5907 err = sock_queue_err_skb(sk, skb); 5908 sock_put(sk); 5909 } 5910 if (err) 5911 kfree_skb(skb); 5912 } 5913 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 5914 #endif /* CONFIG_WIRELESS */ 5915 5916 /** 5917 * skb_partial_csum_set - set up and verify partial csum values for packet 5918 * @skb: the skb to set 5919 * @start: the number of bytes after skb->data to start checksumming. 5920 * @off: the offset from start to place the checksum. 5921 * 5922 * For untrusted partially-checksummed packets, we need to make sure the values 5923 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 5924 * 5925 * This function checks and sets those values and skb->ip_summed: if this 5926 * returns false you should drop the packet. 5927 */ 5928 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 5929 { 5930 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 5931 u32 csum_start = skb_headroom(skb) + (u32)start; 5932 5933 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) { 5934 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 5935 start, off, skb_headroom(skb), skb_headlen(skb)); 5936 return false; 5937 } 5938 skb->ip_summed = CHECKSUM_PARTIAL; 5939 skb->csum_start = csum_start; 5940 skb->csum_offset = off; 5941 skb->transport_header = csum_start; 5942 return true; 5943 } 5944 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 5945 5946 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 5947 unsigned int max) 5948 { 5949 if (skb_headlen(skb) >= len) 5950 return 0; 5951 5952 /* If we need to pullup then pullup to the max, so we 5953 * won't need to do it again. 5954 */ 5955 if (max > skb->len) 5956 max = skb->len; 5957 5958 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 5959 return -ENOMEM; 5960 5961 if (skb_headlen(skb) < len) 5962 return -EPROTO; 5963 5964 return 0; 5965 } 5966 5967 #define MAX_TCP_HDR_LEN (15 * 4) 5968 5969 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 5970 typeof(IPPROTO_IP) proto, 5971 unsigned int off) 5972 { 5973 int err; 5974 5975 switch (proto) { 5976 case IPPROTO_TCP: 5977 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 5978 off + MAX_TCP_HDR_LEN); 5979 if (!err && !skb_partial_csum_set(skb, off, 5980 offsetof(struct tcphdr, 5981 check))) 5982 err = -EPROTO; 5983 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 5984 5985 case IPPROTO_UDP: 5986 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 5987 off + sizeof(struct udphdr)); 5988 if (!err && !skb_partial_csum_set(skb, off, 5989 offsetof(struct udphdr, 5990 check))) 5991 err = -EPROTO; 5992 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 5993 } 5994 5995 return ERR_PTR(-EPROTO); 5996 } 5997 5998 /* This value should be large enough to cover a tagged ethernet header plus 5999 * maximally sized IP and TCP or UDP headers. 6000 */ 6001 #define MAX_IP_HDR_LEN 128 6002 6003 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 6004 { 6005 unsigned int off; 6006 bool fragment; 6007 __sum16 *csum; 6008 int err; 6009 6010 fragment = false; 6011 6012 err = skb_maybe_pull_tail(skb, 6013 sizeof(struct iphdr), 6014 MAX_IP_HDR_LEN); 6015 if (err < 0) 6016 goto out; 6017 6018 if (ip_is_fragment(ip_hdr(skb))) 6019 fragment = true; 6020 6021 off = ip_hdrlen(skb); 6022 6023 err = -EPROTO; 6024 6025 if (fragment) 6026 goto out; 6027 6028 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 6029 if (IS_ERR(csum)) 6030 return PTR_ERR(csum); 6031 6032 if (recalculate) 6033 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 6034 ip_hdr(skb)->daddr, 6035 skb->len - off, 6036 ip_hdr(skb)->protocol, 0); 6037 err = 0; 6038 6039 out: 6040 return err; 6041 } 6042 6043 /* This value should be large enough to cover a tagged ethernet header plus 6044 * an IPv6 header, all options, and a maximal TCP or UDP header. 6045 */ 6046 #define MAX_IPV6_HDR_LEN 256 6047 6048 #define OPT_HDR(type, skb, off) \ 6049 (type *)(skb_network_header(skb) + (off)) 6050 6051 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 6052 { 6053 int err; 6054 u8 nexthdr; 6055 unsigned int off; 6056 unsigned int len; 6057 bool fragment; 6058 bool done; 6059 __sum16 *csum; 6060 6061 fragment = false; 6062 done = false; 6063 6064 off = sizeof(struct ipv6hdr); 6065 6066 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 6067 if (err < 0) 6068 goto out; 6069 6070 nexthdr = ipv6_hdr(skb)->nexthdr; 6071 6072 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 6073 while (off <= len && !done) { 6074 switch (nexthdr) { 6075 case IPPROTO_DSTOPTS: 6076 case IPPROTO_HOPOPTS: 6077 case IPPROTO_ROUTING: { 6078 struct ipv6_opt_hdr *hp; 6079 6080 err = skb_maybe_pull_tail(skb, 6081 off + 6082 sizeof(struct ipv6_opt_hdr), 6083 off + 6084 sizeof(struct ipv6_opt_hdr)); 6085 if (err < 0) 6086 goto out; 6087 6088 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 6089 nexthdr = hp->nexthdr; 6090 off += ipv6_optlen(hp); 6091 break; 6092 } 6093 case IPPROTO_AH: { 6094 struct ip_auth_hdr *hp; 6095 6096 err = skb_maybe_pull_tail(skb, 6097 off + 6098 sizeof(struct ip_auth_hdr), 6099 off + 6100 sizeof(struct ip_auth_hdr)); 6101 if (err < 0) 6102 goto out; 6103 6104 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 6105 nexthdr = hp->nexthdr; 6106 off += ipv6_authlen(hp); 6107 break; 6108 } 6109 case IPPROTO_FRAGMENT: { 6110 struct frag_hdr *hp; 6111 6112 err = skb_maybe_pull_tail(skb, 6113 off + 6114 sizeof(struct frag_hdr), 6115 off + 6116 sizeof(struct frag_hdr)); 6117 if (err < 0) 6118 goto out; 6119 6120 hp = OPT_HDR(struct frag_hdr, skb, off); 6121 6122 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 6123 fragment = true; 6124 6125 nexthdr = hp->nexthdr; 6126 off += sizeof(struct frag_hdr); 6127 break; 6128 } 6129 default: 6130 done = true; 6131 break; 6132 } 6133 } 6134 6135 err = -EPROTO; 6136 6137 if (!done || fragment) 6138 goto out; 6139 6140 csum = skb_checksum_setup_ip(skb, nexthdr, off); 6141 if (IS_ERR(csum)) 6142 return PTR_ERR(csum); 6143 6144 if (recalculate) 6145 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 6146 &ipv6_hdr(skb)->daddr, 6147 skb->len - off, nexthdr, 0); 6148 err = 0; 6149 6150 out: 6151 return err; 6152 } 6153 6154 /** 6155 * skb_checksum_setup - set up partial checksum offset 6156 * @skb: the skb to set up 6157 * @recalculate: if true the pseudo-header checksum will be recalculated 6158 */ 6159 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 6160 { 6161 int err; 6162 6163 switch (skb->protocol) { 6164 case htons(ETH_P_IP): 6165 err = skb_checksum_setup_ipv4(skb, recalculate); 6166 break; 6167 6168 case htons(ETH_P_IPV6): 6169 err = skb_checksum_setup_ipv6(skb, recalculate); 6170 break; 6171 6172 default: 6173 err = -EPROTO; 6174 break; 6175 } 6176 6177 return err; 6178 } 6179 EXPORT_SYMBOL(skb_checksum_setup); 6180 6181 /** 6182 * skb_checksum_maybe_trim - maybe trims the given skb 6183 * @skb: the skb to check 6184 * @transport_len: the data length beyond the network header 6185 * 6186 * Checks whether the given skb has data beyond the given transport length. 6187 * If so, returns a cloned skb trimmed to this transport length. 6188 * Otherwise returns the provided skb. Returns NULL in error cases 6189 * (e.g. transport_len exceeds skb length or out-of-memory). 6190 * 6191 * Caller needs to set the skb transport header and free any returned skb if it 6192 * differs from the provided skb. 6193 */ 6194 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 6195 unsigned int transport_len) 6196 { 6197 struct sk_buff *skb_chk; 6198 unsigned int len = skb_transport_offset(skb) + transport_len; 6199 int ret; 6200 6201 if (skb->len < len) 6202 return NULL; 6203 else if (skb->len == len) 6204 return skb; 6205 6206 skb_chk = skb_clone(skb, GFP_ATOMIC); 6207 if (!skb_chk) 6208 return NULL; 6209 6210 ret = pskb_trim_rcsum(skb_chk, len); 6211 if (ret) { 6212 kfree_skb(skb_chk); 6213 return NULL; 6214 } 6215 6216 return skb_chk; 6217 } 6218 6219 /** 6220 * skb_checksum_trimmed - validate checksum of an skb 6221 * @skb: the skb to check 6222 * @transport_len: the data length beyond the network header 6223 * @skb_chkf: checksum function to use 6224 * 6225 * Applies the given checksum function skb_chkf to the provided skb. 6226 * Returns a checked and maybe trimmed skb. Returns NULL on error. 6227 * 6228 * If the skb has data beyond the given transport length, then a 6229 * trimmed & cloned skb is checked and returned. 6230 * 6231 * Caller needs to set the skb transport header and free any returned skb if it 6232 * differs from the provided skb. 6233 */ 6234 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 6235 unsigned int transport_len, 6236 __sum16(*skb_chkf)(struct sk_buff *skb)) 6237 { 6238 struct sk_buff *skb_chk; 6239 unsigned int offset = skb_transport_offset(skb); 6240 __sum16 ret; 6241 6242 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 6243 if (!skb_chk) 6244 goto err; 6245 6246 if (!pskb_may_pull(skb_chk, offset)) 6247 goto err; 6248 6249 skb_pull_rcsum(skb_chk, offset); 6250 ret = skb_chkf(skb_chk); 6251 skb_push_rcsum(skb_chk, offset); 6252 6253 if (ret) 6254 goto err; 6255 6256 return skb_chk; 6257 6258 err: 6259 if (skb_chk && skb_chk != skb) 6260 kfree_skb(skb_chk); 6261 6262 return NULL; 6263 6264 } 6265 EXPORT_SYMBOL(skb_checksum_trimmed); 6266 6267 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 6268 { 6269 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 6270 skb->dev->name); 6271 } 6272 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 6273 6274 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 6275 { 6276 if (head_stolen) { 6277 skb_release_head_state(skb); 6278 kfree_skbmem(skb); 6279 } else { 6280 __kfree_skb(skb); 6281 } 6282 } 6283 EXPORT_SYMBOL(kfree_skb_partial); 6284 6285 /** 6286 * skb_try_coalesce - try to merge skb to prior one 6287 * @to: prior buffer 6288 * @from: buffer to add 6289 * @fragstolen: pointer to boolean 6290 * @delta_truesize: how much more was allocated than was requested 6291 */ 6292 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 6293 bool *fragstolen, int *delta_truesize) 6294 { 6295 struct skb_shared_info *to_shinfo, *from_shinfo; 6296 int i, delta, len = from->len; 6297 6298 *fragstolen = false; 6299 6300 if (skb_cloned(to)) 6301 return false; 6302 6303 /* In general, avoid mixing page_pool and non-page_pool allocated 6304 * pages within the same SKB. In theory we could take full 6305 * references if @from is cloned and !@to->pp_recycle but its 6306 * tricky (due to potential race with the clone disappearing) and 6307 * rare, so not worth dealing with. 6308 */ 6309 if (to->pp_recycle != from->pp_recycle) 6310 return false; 6311 6312 /* All frags in an skb must either all be struct page-backed or all 6313 * belong to the same memory provider instance; do not coalesce skbs 6314 * that would mix them. 6315 */ 6316 if (skb_frags_readable(from) != skb_frags_readable(to)) 6317 return false; 6318 6319 if (len <= skb_tailroom(to) && skb_frags_readable(from)) { 6320 if (len) { 6321 if (unlikely(skb_copy_bits(from, 0, 6322 skb_tail_pointer(to), 6323 len))) { 6324 DEBUG_NET_WARN_ON_ONCE(1); 6325 return false; 6326 } 6327 skb_put(to, len); 6328 } 6329 *delta_truesize = 0; 6330 return true; 6331 } 6332 6333 to_shinfo = skb_shinfo(to); 6334 from_shinfo = skb_shinfo(from); 6335 if (to_shinfo->frag_list || from_shinfo->frag_list) 6336 return false; 6337 if (skb_zcopy(to) || skb_zcopy(from)) 6338 return false; 6339 6340 if (skb_headlen(from) != 0) { 6341 struct page *page; 6342 unsigned int offset; 6343 6344 if (to_shinfo->nr_frags + 6345 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 6346 return false; 6347 6348 if (skb_head_is_locked(from)) 6349 return false; 6350 6351 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 6352 6353 page = virt_to_head_page(from->head); 6354 offset = from->data - (unsigned char *)page_address(page); 6355 6356 skb_fill_page_desc(to, to_shinfo->nr_frags, 6357 page, offset, skb_headlen(from)); 6358 *fragstolen = true; 6359 } else { 6360 if (to_shinfo->nr_frags + 6361 from_shinfo->nr_frags > MAX_SKB_FRAGS) 6362 return false; 6363 6364 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 6365 } 6366 6367 WARN_ON_ONCE(delta < len); 6368 6369 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 6370 from_shinfo->frags, 6371 from_shinfo->nr_frags * sizeof(skb_frag_t)); 6372 to_shinfo->nr_frags += from_shinfo->nr_frags; 6373 if (from_shinfo->nr_frags) 6374 to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG; 6375 6376 if (!skb_cloned(from)) 6377 from_shinfo->nr_frags = 0; 6378 6379 /* if the skb is not cloned this does nothing 6380 * since we set nr_frags to 0. 6381 */ 6382 for (i = 0; i < from_shinfo->nr_frags; i++) 6383 skb_frag_ref(from, i); 6384 6385 to->truesize += delta; 6386 to->len += len; 6387 to->data_len += len; 6388 6389 *delta_truesize = delta; 6390 return true; 6391 } 6392 EXPORT_SYMBOL(skb_try_coalesce); 6393 6394 static void skb_ext_scrub(struct sk_buff *skb) 6395 { 6396 #if IS_ENABLED(CONFIG_INET_PSP) 6397 struct psp_skb_ext *pse = skb_ext_find(skb, SKB_EXT_PSP); 6398 6399 if (pse) { 6400 struct skb_ext *ext = skb->extensions; 6401 struct psp_skb_ext tmp; 6402 6403 /* Nothing else to scrub if PSP is the only extension in ext. */ 6404 if (ext->chunks == SKB_EXT_CHUNKSIZEOF(*ext) + 6405 skb_ext_type_len[SKB_EXT_PSP]) 6406 return; 6407 6408 tmp = *pse; 6409 skb_ext_reset(skb); 6410 pse = skb_ext_add(skb, SKB_EXT_PSP); 6411 if (pse) 6412 memcpy(pse, &tmp, sizeof(*pse)); 6413 return; 6414 } 6415 #endif 6416 skb_ext_reset(skb); 6417 } 6418 6419 /** 6420 * skb_scrub_packet - scrub an skb 6421 * 6422 * @skb: buffer to clean 6423 * @xnet: packet is crossing netns 6424 * 6425 * skb_scrub_packet can be used after encapsulating or decapsulating a packet 6426 * into/from a tunnel. Some information have to be cleared during these 6427 * operations. 6428 * skb_scrub_packet can also be used to clean a skb before injecting it in 6429 * another namespace (@xnet == true). We have to clear all information in the 6430 * skb that could impact namespace isolation. Note that PSP skb extension is 6431 * preserved across packet scrubbing by design. 6432 */ 6433 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 6434 { 6435 skb->pkt_type = PACKET_HOST; 6436 skb->skb_iif = 0; 6437 skb->ignore_df = 0; 6438 skb_dst_drop(skb); 6439 skb_ext_scrub(skb); 6440 nf_reset_ct(skb); 6441 nf_reset_trace(skb); 6442 6443 #ifdef CONFIG_NET_SWITCHDEV 6444 skb->offload_fwd_mark = 0; 6445 skb->offload_l3_fwd_mark = 0; 6446 #endif 6447 ipvs_reset(skb); 6448 6449 if (!xnet) 6450 return; 6451 6452 skb->mark = 0; 6453 skb_clear_tstamp(skb); 6454 } 6455 EXPORT_SYMBOL_GPL(skb_scrub_packet); 6456 6457 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 6458 { 6459 int mac_len, meta_len; 6460 void *meta; 6461 6462 if (skb_cow(skb, skb_headroom(skb)) < 0) { 6463 kfree_skb(skb); 6464 return NULL; 6465 } 6466 6467 mac_len = skb->data - skb_mac_header(skb); 6468 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 6469 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 6470 mac_len - VLAN_HLEN - ETH_TLEN); 6471 } 6472 6473 meta_len = skb_metadata_len(skb); 6474 if (meta_len) { 6475 meta = skb_metadata_end(skb) - meta_len; 6476 memmove(meta + VLAN_HLEN, meta, meta_len); 6477 } 6478 6479 skb->mac_header += VLAN_HLEN; 6480 return skb; 6481 } 6482 6483 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 6484 { 6485 struct vlan_hdr *vhdr; 6486 u16 vlan_tci; 6487 6488 if (unlikely(skb_vlan_tag_present(skb))) { 6489 /* vlan_tci is already set-up so leave this for another time */ 6490 return skb; 6491 } 6492 6493 skb = skb_share_check(skb, GFP_ATOMIC); 6494 if (unlikely(!skb)) 6495 goto err_free; 6496 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 6497 goto err_free; 6498 6499 vhdr = (struct vlan_hdr *)skb->data; 6500 vlan_tci = ntohs(vhdr->h_vlan_TCI); 6501 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 6502 6503 skb_pull_rcsum(skb, VLAN_HLEN); 6504 vlan_set_encap_proto(skb, vhdr); 6505 6506 skb = skb_reorder_vlan_header(skb); 6507 if (unlikely(!skb)) 6508 goto err_free; 6509 6510 skb_reset_network_header(skb); 6511 if (!skb_transport_header_was_set(skb)) 6512 skb_reset_transport_header(skb); 6513 skb_reset_mac_len(skb); 6514 6515 return skb; 6516 6517 err_free: 6518 kfree_skb(skb); 6519 return NULL; 6520 } 6521 EXPORT_SYMBOL(skb_vlan_untag); 6522 6523 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) 6524 { 6525 if (!pskb_may_pull(skb, write_len)) 6526 return -ENOMEM; 6527 6528 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 6529 return 0; 6530 6531 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 6532 } 6533 EXPORT_SYMBOL(skb_ensure_writable); 6534 6535 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev) 6536 { 6537 int needed_headroom = dev->needed_headroom; 6538 int needed_tailroom = dev->needed_tailroom; 6539 6540 /* For tail taggers, we need to pad short frames ourselves, to ensure 6541 * that the tail tag does not fail at its role of being at the end of 6542 * the packet, once the conduit interface pads the frame. Account for 6543 * that pad length here, and pad later. 6544 */ 6545 if (unlikely(needed_tailroom && skb->len < ETH_ZLEN)) 6546 needed_tailroom += ETH_ZLEN - skb->len; 6547 /* skb_headroom() returns unsigned int... */ 6548 needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0); 6549 needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0); 6550 6551 if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb))) 6552 /* No reallocation needed, yay! */ 6553 return 0; 6554 6555 return pskb_expand_head(skb, needed_headroom, needed_tailroom, 6556 GFP_ATOMIC); 6557 } 6558 EXPORT_SYMBOL(skb_ensure_writable_head_tail); 6559 6560 /* remove VLAN header from packet and update csum accordingly. 6561 * expects a non skb_vlan_tag_present skb with a vlan tag payload 6562 */ 6563 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 6564 { 6565 int offset = skb->data - skb_mac_header(skb); 6566 int err; 6567 6568 if (WARN_ONCE(offset, 6569 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 6570 offset)) { 6571 return -EINVAL; 6572 } 6573 6574 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 6575 if (unlikely(err)) 6576 return err; 6577 6578 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6579 6580 vlan_remove_tag(skb, vlan_tci); 6581 6582 skb->mac_header += VLAN_HLEN; 6583 6584 if (skb_network_offset(skb) < ETH_HLEN) 6585 skb_set_network_header(skb, ETH_HLEN); 6586 6587 skb_reset_mac_len(skb); 6588 6589 return err; 6590 } 6591 EXPORT_SYMBOL(__skb_vlan_pop); 6592 6593 /* Pop a vlan tag either from hwaccel or from payload. 6594 * Expects skb->data at mac header. 6595 */ 6596 int skb_vlan_pop(struct sk_buff *skb) 6597 { 6598 u16 vlan_tci; 6599 __be16 vlan_proto; 6600 int err; 6601 6602 if (likely(skb_vlan_tag_present(skb))) { 6603 __vlan_hwaccel_clear_tag(skb); 6604 } else { 6605 if (unlikely(!eth_type_vlan(skb->protocol))) 6606 return 0; 6607 6608 err = __skb_vlan_pop(skb, &vlan_tci); 6609 if (err) 6610 return err; 6611 } 6612 /* move next vlan tag to hw accel tag */ 6613 if (likely(!eth_type_vlan(skb->protocol))) 6614 return 0; 6615 6616 vlan_proto = skb->protocol; 6617 err = __skb_vlan_pop(skb, &vlan_tci); 6618 if (unlikely(err)) 6619 return err; 6620 6621 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6622 return 0; 6623 } 6624 EXPORT_SYMBOL(skb_vlan_pop); 6625 6626 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 6627 * Expects skb->data at mac header. 6628 */ 6629 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 6630 { 6631 if (skb_vlan_tag_present(skb)) { 6632 int offset = skb->data - skb_mac_header(skb); 6633 int err; 6634 6635 if (WARN_ONCE(offset, 6636 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 6637 offset)) { 6638 return -EINVAL; 6639 } 6640 6641 err = __vlan_insert_tag(skb, skb->vlan_proto, 6642 skb_vlan_tag_get(skb)); 6643 if (err) 6644 return err; 6645 6646 skb->protocol = skb->vlan_proto; 6647 skb->network_header -= VLAN_HLEN; 6648 6649 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 6650 } 6651 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 6652 return 0; 6653 } 6654 EXPORT_SYMBOL(skb_vlan_push); 6655 6656 /** 6657 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 6658 * 6659 * @skb: Socket buffer to modify 6660 * 6661 * Drop the Ethernet header of @skb. 6662 * 6663 * Expects that skb->data points to the mac header and that no VLAN tags are 6664 * present. 6665 * 6666 * Returns 0 on success, -errno otherwise. 6667 */ 6668 int skb_eth_pop(struct sk_buff *skb) 6669 { 6670 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 6671 skb_network_offset(skb) < ETH_HLEN) 6672 return -EPROTO; 6673 6674 skb_pull_rcsum(skb, ETH_HLEN); 6675 skb_reset_mac_header(skb); 6676 skb_reset_mac_len(skb); 6677 6678 return 0; 6679 } 6680 EXPORT_SYMBOL(skb_eth_pop); 6681 6682 /** 6683 * skb_eth_push() - Add a new Ethernet header at the head of a packet 6684 * 6685 * @skb: Socket buffer to modify 6686 * @dst: Destination MAC address of the new header 6687 * @src: Source MAC address of the new header 6688 * 6689 * Prepend @skb with a new Ethernet header. 6690 * 6691 * Expects that skb->data points to the mac header, which must be empty. 6692 * 6693 * Returns 0 on success, -errno otherwise. 6694 */ 6695 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 6696 const unsigned char *src) 6697 { 6698 struct ethhdr *eth; 6699 int err; 6700 6701 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 6702 return -EPROTO; 6703 6704 err = skb_cow_head(skb, sizeof(*eth)); 6705 if (err < 0) 6706 return err; 6707 6708 skb_push(skb, sizeof(*eth)); 6709 skb_reset_mac_header(skb); 6710 skb_reset_mac_len(skb); 6711 6712 eth = eth_hdr(skb); 6713 ether_addr_copy(eth->h_dest, dst); 6714 ether_addr_copy(eth->h_source, src); 6715 eth->h_proto = skb->protocol; 6716 6717 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 6718 6719 return 0; 6720 } 6721 EXPORT_SYMBOL(skb_eth_push); 6722 6723 /* Update the ethertype of hdr and the skb csum value if required. */ 6724 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 6725 __be16 ethertype) 6726 { 6727 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6728 __be16 diff[] = { ~hdr->h_proto, ethertype }; 6729 6730 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6731 } 6732 6733 hdr->h_proto = ethertype; 6734 } 6735 6736 /** 6737 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 6738 * the packet 6739 * 6740 * @skb: buffer 6741 * @mpls_lse: MPLS label stack entry to push 6742 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 6743 * @mac_len: length of the MAC header 6744 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 6745 * ethernet 6746 * 6747 * Expects skb->data at mac header. 6748 * 6749 * Returns 0 on success, -errno otherwise. 6750 */ 6751 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 6752 int mac_len, bool ethernet) 6753 { 6754 struct mpls_shim_hdr *lse; 6755 int err; 6756 6757 if (unlikely(!eth_p_mpls(mpls_proto))) 6758 return -EINVAL; 6759 6760 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 6761 if (skb->encapsulation) 6762 return -EINVAL; 6763 6764 err = skb_cow_head(skb, MPLS_HLEN); 6765 if (unlikely(err)) 6766 return err; 6767 6768 if (!skb->inner_protocol) { 6769 skb_set_inner_network_header(skb, skb_network_offset(skb)); 6770 skb_set_inner_protocol(skb, skb->protocol); 6771 } 6772 6773 skb_push(skb, MPLS_HLEN); 6774 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 6775 mac_len); 6776 skb_reset_mac_header(skb); 6777 skb_set_network_header(skb, mac_len); 6778 skb_reset_mac_len(skb); 6779 6780 lse = mpls_hdr(skb); 6781 lse->label_stack_entry = mpls_lse; 6782 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 6783 6784 if (ethernet && mac_len >= ETH_HLEN) 6785 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 6786 skb->protocol = mpls_proto; 6787 6788 return 0; 6789 } 6790 EXPORT_SYMBOL_GPL(skb_mpls_push); 6791 6792 /** 6793 * skb_mpls_pop() - pop the outermost MPLS header 6794 * 6795 * @skb: buffer 6796 * @next_proto: ethertype of header after popped MPLS header 6797 * @mac_len: length of the MAC header 6798 * @ethernet: flag to indicate if the packet is ethernet 6799 * 6800 * Expects skb->data at mac header. 6801 * 6802 * Returns 0 on success, -errno otherwise. 6803 */ 6804 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 6805 bool ethernet) 6806 { 6807 int err; 6808 6809 if (unlikely(!eth_p_mpls(skb->protocol))) 6810 return 0; 6811 6812 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 6813 if (unlikely(err)) 6814 return err; 6815 6816 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 6817 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 6818 mac_len); 6819 6820 __skb_pull(skb, MPLS_HLEN); 6821 skb_reset_mac_header(skb); 6822 skb_set_network_header(skb, mac_len); 6823 6824 if (ethernet && mac_len >= ETH_HLEN) { 6825 struct ethhdr *hdr; 6826 6827 /* use mpls_hdr() to get ethertype to account for VLANs. */ 6828 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 6829 skb_mod_eth_type(skb, hdr, next_proto); 6830 } 6831 skb->protocol = next_proto; 6832 6833 /* The last label is gone, so the inner header recorded by 6834 * skb_mpls_push() no longer describes this packet. Drop it, or a 6835 * later push keeps the stale offset. 6836 */ 6837 if (!eth_p_mpls(next_proto)) 6838 skb->inner_protocol = 0; 6839 6840 return 0; 6841 } 6842 EXPORT_SYMBOL_GPL(skb_mpls_pop); 6843 6844 /** 6845 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 6846 * 6847 * @skb: buffer 6848 * @mpls_lse: new MPLS label stack entry to update to 6849 * 6850 * Expects skb->data at mac header. 6851 * 6852 * Returns 0 on success, -errno otherwise. 6853 */ 6854 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 6855 { 6856 int err; 6857 6858 if (unlikely(!eth_p_mpls(skb->protocol))) 6859 return -EINVAL; 6860 6861 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 6862 if (unlikely(err)) 6863 return err; 6864 6865 if (skb->ip_summed == CHECKSUM_COMPLETE) { 6866 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 6867 6868 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 6869 } 6870 6871 mpls_hdr(skb)->label_stack_entry = mpls_lse; 6872 6873 return 0; 6874 } 6875 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 6876 6877 /** 6878 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 6879 * 6880 * @skb: buffer 6881 * 6882 * Expects skb->data at mac header. 6883 * 6884 * Returns 0 on success, -errno otherwise. 6885 */ 6886 int skb_mpls_dec_ttl(struct sk_buff *skb) 6887 { 6888 u32 lse; 6889 u8 ttl; 6890 6891 if (unlikely(!eth_p_mpls(skb->protocol))) 6892 return -EINVAL; 6893 6894 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 6895 return -ENOMEM; 6896 6897 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 6898 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 6899 if (!ttl || !--ttl) 6900 return -EINVAL; 6901 6902 lse &= ~MPLS_LS_TTL_MASK; 6903 lse |= ttl << MPLS_LS_TTL_SHIFT; 6904 6905 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 6906 } 6907 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 6908 6909 /** 6910 * alloc_skb_with_frags - allocate skb with page frags 6911 * 6912 * @header_len: size of linear part 6913 * @data_len: needed length in frags 6914 * @order: max page order desired. 6915 * @errcode: pointer to error code if any 6916 * @gfp_mask: allocation mask 6917 * 6918 * This can be used to allocate a paged skb, given a maximal order for frags. 6919 */ 6920 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 6921 unsigned long data_len, 6922 int order, 6923 int *errcode, 6924 gfp_t gfp_mask) 6925 { 6926 unsigned long chunk; 6927 struct sk_buff *skb; 6928 struct page *page; 6929 int nr_frags = 0; 6930 6931 *errcode = -EMSGSIZE; 6932 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order))) 6933 return NULL; 6934 6935 *errcode = -ENOBUFS; 6936 skb = alloc_skb(header_len, gfp_mask); 6937 if (!skb) 6938 return NULL; 6939 6940 while (data_len) { 6941 if (nr_frags == MAX_SKB_FRAGS) 6942 goto failure; 6943 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order)) 6944 order--; 6945 6946 if (order) { 6947 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 6948 __GFP_COMP | 6949 __GFP_NOWARN, 6950 order); 6951 if (!page) { 6952 order--; 6953 continue; 6954 } 6955 } else { 6956 page = alloc_page(gfp_mask); 6957 if (!page) 6958 goto failure; 6959 } 6960 chunk = min_t(unsigned long, data_len, 6961 PAGE_SIZE << order); 6962 skb_fill_page_desc(skb, nr_frags, page, 0, chunk); 6963 nr_frags++; 6964 skb->truesize += (PAGE_SIZE << order); 6965 data_len -= chunk; 6966 } 6967 return skb; 6968 6969 failure: 6970 kfree_skb(skb); 6971 return NULL; 6972 } 6973 EXPORT_SYMBOL(alloc_skb_with_frags); 6974 6975 /* pskb_carve_inside_header() and pskb_carve_inside_nonlinear() 6976 * remove the first bytes of a packet and reallocate skb->head. 6977 * 6978 * Whatever headers were present before the operation are gone, 6979 * we must not leave stale offsets, otherwise users of this skb 6980 * (skb_dump(), drop_monitor, taps, ...) would read or pull garbage. 6981 */ 6982 static void skb_carve_reset_headers(struct sk_buff *skb) 6983 { 6984 skb_unset_mac_header(skb); 6985 skb_unset_transport_header(skb); 6986 skb_reset_network_header(skb); 6987 skb->mac_len = 0; 6988 6989 /* Inner offsets have no "unset" marker, zero them so that 6990 * skb_inner_network_header_was_set() becomes false and no 6991 * consumer mistakes them for a real (and long gone) header. 6992 */ 6993 skb->inner_mac_header = 0; 6994 skb->inner_network_header = 0; 6995 skb->inner_transport_header = 0; 6996 skb->inner_protocol = 0; 6997 skb->encapsulation = 0; 6998 6999 if (skb->ip_summed == CHECKSUM_PARTIAL) 7000 skb->ip_summed = CHECKSUM_NONE; 7001 } 7002 7003 /* carve out the first off bytes from skb when off < headlen */ 7004 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 7005 const int headlen, gfp_t gfp_mask) 7006 { 7007 int i; 7008 unsigned int size = skb_end_offset(skb); 7009 int new_hlen = headlen - off; 7010 u8 *data; 7011 7012 if (skb_pfmemalloc(skb)) 7013 gfp_mask |= __GFP_MEMALLOC; 7014 7015 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 7016 if (!data) 7017 return -ENOMEM; 7018 size = SKB_WITH_OVERHEAD(size); 7019 7020 /* Copy real data, and all frags */ 7021 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 7022 skb->len -= off; 7023 7024 /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it 7025 * while refcounting frags below. 7026 */ 7027 skb_zcopy_downgrade_managed(skb); 7028 7029 memcpy((struct skb_shared_info *)(data + size), 7030 skb_shinfo(skb), 7031 offsetof(struct skb_shared_info, 7032 frags[skb_shinfo(skb)->nr_frags])); 7033 if (skb_cloned(skb)) { 7034 /* drop the old head gracefully */ 7035 if (skb_orphan_frags(skb, gfp_mask)) { 7036 skb_kfree_head(data); 7037 return -ENOMEM; 7038 } 7039 if (skb_zcopy(skb)) 7040 net_zcopy_get(skb_uarg(skb)); 7041 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 7042 skb_frag_ref(skb, i); 7043 if (skb_has_frag_list(skb)) 7044 skb_clone_fraglist(skb); 7045 skb_release_data(skb, SKB_CONSUMED); 7046 } else { 7047 /* we can reuse existing recount- all we did was 7048 * relocate values 7049 */ 7050 skb_free_head(skb); 7051 } 7052 7053 skb->head = data; 7054 skb->data = data; 7055 skb->head_frag = 0; 7056 skb_set_end_offset(skb, size); 7057 skb_set_tail_pointer(skb, skb_headlen(skb)); 7058 skb_carve_reset_headers(skb); 7059 skb->cloned = 0; 7060 skb->hdr_len = 0; 7061 skb->nohdr = 0; 7062 atomic_set(&skb_shinfo(skb)->dataref, 1); 7063 7064 return 0; 7065 } 7066 7067 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 7068 7069 /* carve out the first eat bytes from skb's frag_list. May recurse into 7070 * pskb_carve() 7071 */ 7072 static int pskb_carve_frag_list(struct skb_shared_info *shinfo, int eat, 7073 gfp_t gfp_mask) 7074 { 7075 struct sk_buff *list = shinfo->frag_list; 7076 struct sk_buff *clone = NULL; 7077 struct sk_buff *insp = NULL; 7078 7079 do { 7080 if (!list) { 7081 pr_err("Not enough bytes to eat. Want %d\n", eat); 7082 return -EFAULT; 7083 } 7084 if (list->len <= eat) { 7085 /* Eaten as whole. */ 7086 eat -= list->len; 7087 list = list->next; 7088 insp = list; 7089 } else { 7090 /* Eaten partially. */ 7091 if (skb_shared(list)) { 7092 clone = skb_clone(list, gfp_mask); 7093 if (!clone) 7094 return -ENOMEM; 7095 insp = list->next; 7096 list = clone; 7097 } else { 7098 /* This may be pulled without problems. */ 7099 insp = list; 7100 } 7101 if (pskb_carve(list, eat, gfp_mask) < 0) { 7102 kfree_skb(clone); 7103 return -ENOMEM; 7104 } 7105 break; 7106 } 7107 } while (eat); 7108 7109 /* Free pulled out fragments. */ 7110 while ((list = shinfo->frag_list) != insp) { 7111 shinfo->frag_list = list->next; 7112 consume_skb(list); 7113 } 7114 /* And insert new clone at head. */ 7115 if (clone) { 7116 clone->next = list; 7117 shinfo->frag_list = clone; 7118 } 7119 return 0; 7120 } 7121 7122 /* carve off first len bytes from skb. Split line (off) is in the 7123 * non-linear part of skb 7124 */ 7125 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 7126 int pos, gfp_t gfp_mask) 7127 { 7128 int i, k = 0; 7129 unsigned int size = skb_end_offset(skb); 7130 u8 *data; 7131 const int nfrags = skb_shinfo(skb)->nr_frags; 7132 struct skb_shared_info *shinfo; 7133 7134 if (skb_pfmemalloc(skb)) 7135 gfp_mask |= __GFP_MEMALLOC; 7136 7137 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL); 7138 if (!data) 7139 return -ENOMEM; 7140 size = SKB_WITH_OVERHEAD(size); 7141 7142 /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it 7143 * while refcounting frags below. 7144 */ 7145 skb_zcopy_downgrade_managed(skb); 7146 7147 memcpy((struct skb_shared_info *)(data + size), 7148 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 7149 if (skb_orphan_frags(skb, gfp_mask)) { 7150 skb_kfree_head(data); 7151 return -ENOMEM; 7152 } 7153 shinfo = (struct skb_shared_info *)(data + size); 7154 for (i = 0; i < nfrags; i++) { 7155 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 7156 7157 if (pos + fsize > off) { 7158 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 7159 7160 if (pos < off) { 7161 /* Split frag. 7162 * We have two variants in this case: 7163 * 1. Move all the frag to the second 7164 * part, if it is possible. F.e. 7165 * this approach is mandatory for TUX, 7166 * where splitting is expensive. 7167 * 2. Split is accurately. We make this. 7168 */ 7169 skb_frag_off_add(&shinfo->frags[0], off - pos); 7170 skb_frag_size_sub(&shinfo->frags[0], off - pos); 7171 } 7172 skb_frag_ref(skb, i); 7173 k++; 7174 } 7175 pos += fsize; 7176 } 7177 shinfo->nr_frags = k; 7178 if (skb_has_frag_list(skb)) 7179 skb_clone_fraglist(skb); 7180 7181 /* split line is in frag list */ 7182 if (k == 0 && pskb_carve_frag_list(shinfo, off - pos, gfp_mask)) { 7183 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 7184 if (skb_has_frag_list(skb)) 7185 kfree_skb_list(skb_shinfo(skb)->frag_list); 7186 skb_kfree_head(data); 7187 return -ENOMEM; 7188 } 7189 if (skb_zcopy(skb)) 7190 net_zcopy_get(skb_uarg(skb)); 7191 skb_release_data(skb, SKB_CONSUMED); 7192 7193 skb->head = data; 7194 skb->head_frag = 0; 7195 skb->data = data; 7196 skb_set_end_offset(skb, size); 7197 skb_reset_tail_pointer(skb); 7198 skb_carve_reset_headers(skb); 7199 skb->cloned = 0; 7200 skb->hdr_len = 0; 7201 skb->nohdr = 0; 7202 skb->len -= off; 7203 skb->data_len = skb->len; 7204 atomic_set(&skb_shinfo(skb)->dataref, 1); 7205 return 0; 7206 } 7207 7208 /* remove len bytes from the beginning of the skb */ 7209 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 7210 { 7211 int headlen = skb_headlen(skb); 7212 7213 if (len < headlen) 7214 return pskb_carve_inside_header(skb, len, headlen, gfp); 7215 else 7216 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 7217 } 7218 7219 /* Extract to_copy bytes starting at off from skb, and return this in 7220 * a new skb 7221 */ 7222 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 7223 int to_copy, gfp_t gfp) 7224 { 7225 struct sk_buff *clone = skb_clone(skb, gfp); 7226 7227 if (!clone) 7228 return NULL; 7229 7230 if (pskb_carve(clone, off, gfp) < 0 || 7231 pskb_trim(clone, to_copy)) { 7232 kfree_skb(clone); 7233 return NULL; 7234 } 7235 return clone; 7236 } 7237 EXPORT_SYMBOL(pskb_extract); 7238 7239 /** 7240 * skb_condense - try to get rid of fragments/frag_list if possible 7241 * @skb: buffer 7242 * 7243 * Can be used to save memory before skb is added to a busy queue. 7244 * If packet has bytes in frags and enough tail room in skb->head, 7245 * pull all of them, so that we can free the frags right now and adjust 7246 * truesize. 7247 * Notes: 7248 * We do not reallocate skb->head thus can not fail. 7249 * Caller must re-evaluate skb->truesize if needed. 7250 */ 7251 void skb_condense(struct sk_buff *skb) 7252 { 7253 if (skb->data_len) { 7254 if (skb->data_len > skb->end - skb->tail || 7255 skb_cloned(skb) || !skb_frags_readable(skb)) 7256 return; 7257 7258 /* Nice, we can free page frag(s) right now */ 7259 __pskb_pull_tail(skb, skb->data_len); 7260 } 7261 /* At this point, skb->truesize might be over estimated, 7262 * because skb had a fragment, and fragments do not tell 7263 * their truesize. 7264 * When we pulled its content into skb->head, fragment 7265 * was freed, but __pskb_pull_tail() could not possibly 7266 * adjust skb->truesize, not knowing the frag truesize. 7267 */ 7268 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 7269 } 7270 EXPORT_SYMBOL(skb_condense); 7271 7272 #ifdef CONFIG_SKB_EXTENSIONS 7273 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 7274 { 7275 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 7276 } 7277 7278 /** 7279 * __skb_ext_alloc - allocate a new skb extensions storage 7280 * 7281 * @flags: See kmalloc(). 7282 * 7283 * Returns the newly allocated pointer. The pointer can later attached to a 7284 * skb via __skb_ext_set(). 7285 * Note: caller must handle the skb_ext as an opaque data. 7286 */ 7287 struct skb_ext *__skb_ext_alloc(gfp_t flags) 7288 { 7289 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 7290 7291 if (new) { 7292 memset(new->offset, 0, sizeof(new->offset)); 7293 refcount_set(&new->refcnt, 1); 7294 } 7295 7296 return new; 7297 } 7298 7299 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 7300 unsigned int old_active) 7301 { 7302 struct skb_ext *new; 7303 7304 if (refcount_read(&old->refcnt) == 1) 7305 return old; 7306 7307 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 7308 if (!new) 7309 return NULL; 7310 7311 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 7312 refcount_set(&new->refcnt, 1); 7313 7314 #ifdef CONFIG_XFRM 7315 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 7316 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 7317 unsigned int i; 7318 7319 for (i = 0; i < sp->len; i++) 7320 xfrm_state_hold(sp->xvec[i]); 7321 } 7322 #endif 7323 #ifdef CONFIG_MCTP_FLOWS 7324 if (old_active & (1 << SKB_EXT_MCTP)) { 7325 struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP); 7326 7327 if (flow->key) 7328 refcount_inc(&flow->key->refs); 7329 } 7330 #endif 7331 __skb_ext_put(old); 7332 return new; 7333 } 7334 7335 /** 7336 * __skb_ext_set - attach the specified extension storage to this skb 7337 * @skb: buffer 7338 * @id: extension id 7339 * @ext: extension storage previously allocated via __skb_ext_alloc() 7340 * 7341 * Existing extensions, if any, are cleared. 7342 * 7343 * Returns the pointer to the extension. 7344 */ 7345 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 7346 struct skb_ext *ext) 7347 { 7348 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 7349 7350 skb_ext_put(skb); 7351 newlen = newoff + skb_ext_type_len[id]; 7352 ext->chunks = newlen; 7353 ext->offset[id] = newoff; 7354 skb->extensions = ext; 7355 skb->active_extensions = 1 << id; 7356 return skb_ext_get_ptr(ext, id); 7357 } 7358 EXPORT_SYMBOL_NS_GPL(__skb_ext_set, "NETDEV_INTERNAL"); 7359 7360 /** 7361 * skb_ext_add - allocate space for given extension, COW if needed 7362 * @skb: buffer 7363 * @id: extension to allocate space for 7364 * 7365 * Allocates enough space for the given extension. 7366 * If the extension is already present, a pointer to that extension 7367 * is returned. 7368 * 7369 * If the skb was cloned, COW applies and the returned memory can be 7370 * modified without changing the extension space of clones buffers. 7371 * 7372 * Returns pointer to the extension or NULL on allocation failure. 7373 */ 7374 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 7375 { 7376 struct skb_ext *new, *old = NULL; 7377 unsigned int newlen, newoff; 7378 7379 if (skb->active_extensions) { 7380 old = skb->extensions; 7381 7382 new = skb_ext_maybe_cow(old, skb->active_extensions); 7383 if (!new) 7384 return NULL; 7385 7386 if (__skb_ext_exist(new, id)) 7387 goto set_active; 7388 7389 newoff = new->chunks; 7390 } else { 7391 newoff = SKB_EXT_CHUNKSIZEOF(*new); 7392 7393 new = __skb_ext_alloc(GFP_ATOMIC); 7394 if (!new) 7395 return NULL; 7396 } 7397 7398 newlen = newoff + skb_ext_type_len[id]; 7399 new->chunks = newlen; 7400 new->offset[id] = newoff; 7401 set_active: 7402 skb->slow_gro = 1; 7403 skb->extensions = new; 7404 skb->active_extensions |= 1 << id; 7405 return skb_ext_get_ptr(new, id); 7406 } 7407 EXPORT_SYMBOL(skb_ext_add); 7408 7409 #ifdef CONFIG_XFRM 7410 static void skb_ext_put_sp(struct sec_path *sp) 7411 { 7412 unsigned int i; 7413 7414 if (!sp->len) 7415 return; 7416 7417 for (i = 0; i < sp->len; i++) 7418 xfrm_state_put(sp->xvec[i]); 7419 sp->len = 0; 7420 } 7421 #endif 7422 7423 #ifdef CONFIG_MCTP_FLOWS 7424 static void skb_ext_put_mctp(struct mctp_flow *flow) 7425 { 7426 if (!flow->key) 7427 return; 7428 7429 mctp_key_unref(flow->key); 7430 flow->key = NULL; 7431 } 7432 #endif 7433 7434 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 7435 { 7436 struct skb_ext *ext = skb->extensions; 7437 7438 skb->active_extensions &= ~(1 << id); 7439 if (skb->active_extensions == 0) { 7440 skb->extensions = NULL; 7441 __skb_ext_put(ext); 7442 return; 7443 } 7444 7445 if (refcount_read(&ext->refcnt) > 1) 7446 return; 7447 7448 #ifdef CONFIG_XFRM 7449 if (id == SKB_EXT_SEC_PATH) 7450 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 7451 #endif 7452 #ifdef CONFIG_MCTP_FLOWS 7453 if (id == SKB_EXT_MCTP) 7454 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 7455 #endif 7456 } 7457 EXPORT_SYMBOL(__skb_ext_del); 7458 7459 void __skb_ext_put(struct skb_ext *ext) 7460 { 7461 /* If this is last clone, nothing can increment 7462 * it after check passes. Avoids one atomic op. 7463 */ 7464 if (refcount_read(&ext->refcnt) == 1) 7465 goto free_now; 7466 7467 if (!refcount_dec_and_test(&ext->refcnt)) 7468 return; 7469 free_now: 7470 #ifdef CONFIG_XFRM 7471 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 7472 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 7473 #endif 7474 #ifdef CONFIG_MCTP_FLOWS 7475 if (__skb_ext_exist(ext, SKB_EXT_MCTP)) 7476 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP)); 7477 #endif 7478 7479 kmem_cache_free(skbuff_ext_cache, ext); 7480 } 7481 EXPORT_SYMBOL(__skb_ext_put); 7482 #endif /* CONFIG_SKB_EXTENSIONS */ 7483 7484 static void kfree_skb_napi_cache(struct sk_buff *skb) 7485 { 7486 /* if SKB is a clone, don't handle this case */ 7487 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 7488 __kfree_skb(skb); 7489 return; 7490 } 7491 7492 local_bh_disable(); 7493 __napi_kfree_skb(skb, SKB_CONSUMED); 7494 local_bh_enable(); 7495 } 7496 7497 DEFINE_STATIC_KEY_FALSE(skb_defer_disable_key); 7498 7499 /** 7500 * skb_attempt_defer_free - queue skb for remote freeing 7501 * @skb: buffer 7502 * 7503 * Put @skb in a per-cpu list, using the cpu which 7504 * allocated the skb/pages to reduce false sharing 7505 * and memory zone spinlock contention. 7506 */ 7507 void skb_attempt_defer_free(struct sk_buff *skb) 7508 { 7509 struct skb_defer_node *sdn; 7510 unsigned long defer_count; 7511 unsigned int defer_max; 7512 int cpu, my_cpu; 7513 bool kick; 7514 7515 if (static_branch_unlikely(&skb_defer_disable_key)) 7516 goto nodefer; 7517 7518 /* zero copy notifications should not be delayed. */ 7519 if (skb_zcopy(skb)) 7520 goto nodefer; 7521 7522 cpu = skb->alloc_cpu; 7523 my_cpu = raw_smp_processor_id(); 7524 if (cpu == my_cpu || 7525 WARN_ON_ONCE(cpu >= nr_cpu_ids) || 7526 !cpu_online(cpu)) { 7527 nodefer: kfree_skb_napi_cache(skb); 7528 return; 7529 } 7530 7531 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 7532 DEBUG_NET_WARN_ON_ONCE(skb->destructor); 7533 DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb)); 7534 7535 sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu); 7536 7537 defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); 7538 defer_count = atomic_long_inc_return(&sdn->defer_count); 7539 7540 if (defer_count > defer_max) { 7541 /* Undo the reservation. An llist_add() does not release one -- 7542 * the skb stays charged until a flush subtracts the batch that 7543 * contains it -- so a rejected free must undo its own. 7544 */ 7545 atomic_long_dec(&sdn->defer_count); 7546 goto nodefer; 7547 } 7548 7549 llist_add(&skb->ll_node, &sdn->defer_list); 7550 7551 if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) { 7552 skb_defer_node_flush(sdn); 7553 return; 7554 } 7555 7556 /* Half capacity, plus the skb that fills the queue. Equality alone can 7557 * be consumed early -- if the producer holding the kick value publishes 7558 * first, its purge drains only that skb and the reservations published 7559 * after it match nothing -- which would leave a full queue with nothing 7560 * pending. 7561 */ 7562 kick = (defer_count - 1) == (defer_max >> 1) || defer_count == defer_max; 7563 7564 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU 7565 * if we are unlucky enough (this seems very unlikely). 7566 */ 7567 if (unlikely(kick)) 7568 kick_defer_list_purge(cpu); 7569 } 7570 7571 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, 7572 size_t offset, size_t len, 7573 unsigned int csum_offset) 7574 { 7575 const char *kaddr; 7576 __wsum csum; 7577 7578 kaddr = kmap_local_page(page); 7579 csum = csum_partial(kaddr + offset, len, 0); 7580 kunmap_local(kaddr); 7581 skb->csum = csum_block_add(skb->csum, csum, csum_offset); 7582 } 7583 7584 /** 7585 * skb_splice_from_iter - Splice (or copy) pages to skbuff 7586 * @skb: The buffer to add pages to 7587 * @iter: Iterator representing the pages to be added 7588 * @maxsize: Maximum amount of pages to be added 7589 * 7590 * This is a common helper function for supporting MSG_SPLICE_PAGES. It 7591 * extracts pages from an iterator and adds them to the socket buffer if 7592 * possible, copying them to fragments if not possible (such as if they're slab 7593 * pages). 7594 * 7595 * Returns the amount of data spliced/copied or -EMSGSIZE if there's 7596 * insufficient space in the buffer to transfer anything. 7597 */ 7598 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, 7599 ssize_t maxsize) 7600 { 7601 size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags); 7602 struct page *pages[8], **ppages = pages; 7603 ssize_t spliced = 0, ret = 0; 7604 unsigned int i; 7605 7606 while (iter->count > 0) { 7607 ssize_t space, nr, len; 7608 size_t off; 7609 7610 ret = -EMSGSIZE; 7611 space = frag_limit - skb_shinfo(skb)->nr_frags; 7612 if (space < 0) 7613 break; 7614 7615 /* We might be able to coalesce without increasing nr_frags */ 7616 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages)); 7617 7618 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off); 7619 if (len <= 0) { 7620 ret = len ?: -EIO; 7621 break; 7622 } 7623 7624 i = 0; 7625 do { 7626 struct page *page = pages[i++]; 7627 size_t part = min_t(size_t, PAGE_SIZE - off, len); 7628 7629 ret = -EIO; 7630 if (WARN_ON_ONCE(!sendpage_ok(page))) 7631 goto out; 7632 7633 ret = skb_append_pagefrags(skb, page, off, part, 7634 frag_limit); 7635 if (ret < 0) { 7636 iov_iter_revert(iter, len); 7637 goto out; 7638 } 7639 7640 if (skb->ip_summed == CHECKSUM_NONE) 7641 skb_splice_csum_page(skb, page, off, part, 7642 skb->len + spliced); 7643 7644 off = 0; 7645 spliced += part; 7646 maxsize -= part; 7647 len -= part; 7648 } while (len > 0); 7649 7650 if (maxsize <= 0) 7651 break; 7652 } 7653 7654 out: 7655 skb_len_add(skb, spliced); 7656 return spliced ?: ret; 7657 } 7658 EXPORT_SYMBOL(skb_splice_from_iter); 7659 7660 static __always_inline 7661 size_t memcpy_from_iter_csum(void *iter_from, size_t progress, 7662 size_t len, void *to, void *priv2) 7663 { 7664 __wsum *csum = priv2; 7665 __wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len); 7666 7667 *csum = csum_block_add(*csum, next, progress); 7668 return 0; 7669 } 7670 7671 static __always_inline 7672 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress, 7673 size_t len, void *to, void *priv2) 7674 { 7675 __wsum next, *csum = priv2; 7676 7677 next = csum_and_copy_from_user(iter_from, to + progress, len); 7678 *csum = csum_block_add(*csum, next, progress); 7679 return next ? 0 : len; 7680 } 7681 7682 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, 7683 __wsum *csum, struct iov_iter *i) 7684 { 7685 size_t copied; 7686 7687 if (WARN_ON_ONCE(!i->data_source)) 7688 return false; 7689 copied = iterate_and_advance2(i, bytes, addr, csum, 7690 copy_from_user_iter_csum, 7691 memcpy_from_iter_csum); 7692 if (likely(copied == bytes)) 7693 return true; 7694 iov_iter_revert(i, copied); 7695 return false; 7696 } 7697 EXPORT_SYMBOL(csum_and_copy_from_iter_full); 7698 7699 /** 7700 * net_iov_inc_ref - acquire a non-page_pool reference on a net_iov 7701 * @niov: net_iov to reference 7702 * 7703 * Acquires a non-pp backing reference (binding->ref) on @niov regardless of 7704 * page_pool membership, warning if @niov does not support non-pp references. 7705 * Counterpart to net_iov_dec_ref(). 7706 */ 7707 void net_iov_inc_ref(struct net_iov *niov) 7708 { 7709 if (net_is_devmem_iov(niov)) 7710 net_devmem_net_iov_inc_ref(niov); 7711 else 7712 DEBUG_NET_WARN_ON_ONCE(true); 7713 } 7714 EXPORT_SYMBOL(net_iov_inc_ref); 7715 7716 /** 7717 * net_iov_dec_ref - release a non-page_pool reference on a net_iov 7718 * @niov: net_iov to unreference 7719 * 7720 * Drops a non-pp backing reference (binding->ref) on @niov regardless of 7721 * page_pool membership, warning if @niov does not support non-pp references. 7722 * Counterpart to net_iov_inc_ref(). 7723 */ 7724 void net_iov_dec_ref(struct net_iov *niov) 7725 { 7726 if (net_is_devmem_iov(niov)) 7727 net_devmem_net_iov_dec_ref(niov); 7728 else 7729 DEBUG_NET_WARN_ON_ONCE(true); 7730 } 7731 EXPORT_SYMBOL(net_iov_dec_ref); 7732 7733 struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb, 7734 __be16 type, 7735 int mac_offset) 7736 { 7737 unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH; 7738 7739 /* if type is 802.1Q/AD then the header should already be 7740 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at 7741 * ETH_HLEN otherwise 7742 */ 7743 if (vlan_depth) { 7744 if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN)) 7745 return (struct vlan_type_depth) { 0 }; 7746 vlan_depth -= VLAN_HLEN; 7747 } else { 7748 vlan_depth = ETH_HLEN; 7749 } 7750 do { 7751 struct vlan_hdr vhdr, *vh; 7752 7753 vh = skb_header_pointer(skb, mac_offset + vlan_depth, 7754 sizeof(vhdr), &vhdr); 7755 if (unlikely(!vh || !--parse_depth)) 7756 return (struct vlan_type_depth) { 0 }; 7757 7758 type = vh->h_vlan_encapsulated_proto; 7759 vlan_depth += VLAN_HLEN; 7760 } while (eth_type_vlan(type)); 7761 7762 return (struct vlan_type_depth) { 7763 .type = type, 7764 .depth = vlan_depth 7765 }; 7766 } 7767 EXPORT_SYMBOL(__vlan_get_protocol_offset); 7768