1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Routines having to do with the 'struct sk_buff' memory handlers. 4 * 5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 6 * Florian La Roche <rzsfl@rz.uni-sb.de> 7 * 8 * Fixes: 9 * Alan Cox : Fixed the worst of the load 10 * balancer bugs. 11 * Dave Platt : Interrupt stacking fix. 12 * Richard Kooijman : Timestamp fixes. 13 * Alan Cox : Changed buffer format. 14 * Alan Cox : destructor hook for AF_UNIX etc. 15 * Linus Torvalds : Better skb_clone. 16 * Alan Cox : Added skb_copy. 17 * Alan Cox : Added all the changed routines Linus 18 * only put in the headers 19 * Ray VanTassle : Fixed --skb->lock in free 20 * Alan Cox : skb_copy copy arp field 21 * Andi Kleen : slabified it. 22 * Robert Olsson : Removed skb_head_pool 23 * 24 * NOTE: 25 * The __skb_ routines should be called with interrupts 26 * disabled, or you better be *real* sure that the operation is atomic 27 * with respect to whatever list is being frobbed (e.g. via lock_sock() 28 * or via disabling bottom half handlers, etc). 29 */ 30 31 /* 32 * The functions in this file will not compile correctly with gcc 2.4.x 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/module.h> 38 #include <linux/types.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/interrupt.h> 42 #include <linux/in.h> 43 #include <linux/inet.h> 44 #include <linux/slab.h> 45 #include <linux/tcp.h> 46 #include <linux/udp.h> 47 #include <linux/sctp.h> 48 #include <linux/netdevice.h> 49 #ifdef CONFIG_NET_CLS_ACT 50 #include <net/pkt_sched.h> 51 #endif 52 #include <linux/string.h> 53 #include <linux/skbuff.h> 54 #include <linux/splice.h> 55 #include <linux/cache.h> 56 #include <linux/rtnetlink.h> 57 #include <linux/init.h> 58 #include <linux/scatterlist.h> 59 #include <linux/errqueue.h> 60 #include <linux/prefetch.h> 61 #include <linux/if_vlan.h> 62 #include <linux/mpls.h> 63 64 #include <net/protocol.h> 65 #include <net/dst.h> 66 #include <net/sock.h> 67 #include <net/checksum.h> 68 #include <net/ip6_checksum.h> 69 #include <net/xfrm.h> 70 #include <net/mpls.h> 71 #include <net/mptcp.h> 72 73 #include <linux/uaccess.h> 74 #include <trace/events/skb.h> 75 #include <linux/highmem.h> 76 #include <linux/capability.h> 77 #include <linux/user_namespace.h> 78 #include <linux/indirect_call_wrapper.h> 79 80 #include "datagram.h" 81 82 struct kmem_cache *skbuff_head_cache __ro_after_init; 83 static struct kmem_cache *skbuff_fclone_cache __ro_after_init; 84 #ifdef CONFIG_SKB_EXTENSIONS 85 static struct kmem_cache *skbuff_ext_cache __ro_after_init; 86 #endif 87 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; 88 EXPORT_SYMBOL(sysctl_max_skb_frags); 89 90 /** 91 * skb_panic - private function for out-of-line support 92 * @skb: buffer 93 * @sz: size 94 * @addr: address 95 * @msg: skb_over_panic or skb_under_panic 96 * 97 * Out-of-line support for skb_put() and skb_push(). 98 * Called via the wrapper skb_over_panic() or skb_under_panic(). 99 * Keep out of line to prevent kernel bloat. 100 * __builtin_return_address is not used because it is not always reliable. 101 */ 102 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 103 const char msg[]) 104 { 105 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", 106 msg, addr, skb->len, sz, skb->head, skb->data, 107 (unsigned long)skb->tail, (unsigned long)skb->end, 108 skb->dev ? skb->dev->name : "<NULL>"); 109 BUG(); 110 } 111 112 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 113 { 114 skb_panic(skb, sz, addr, __func__); 115 } 116 117 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 118 { 119 skb_panic(skb, sz, addr, __func__); 120 } 121 122 #define NAPI_SKB_CACHE_SIZE 64 123 #define NAPI_SKB_CACHE_BULK 16 124 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) 125 126 struct napi_alloc_cache { 127 struct page_frag_cache page; 128 unsigned int skb_count; 129 void *skb_cache[NAPI_SKB_CACHE_SIZE]; 130 }; 131 132 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); 133 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); 134 135 static void *__alloc_frag_align(unsigned int fragsz, gfp_t gfp_mask, 136 unsigned int align_mask) 137 { 138 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 139 140 return page_frag_alloc_align(&nc->page, fragsz, gfp_mask, align_mask); 141 } 142 143 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 144 { 145 fragsz = SKB_DATA_ALIGN(fragsz); 146 147 return __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); 148 } 149 EXPORT_SYMBOL(__napi_alloc_frag_align); 150 151 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) 152 { 153 struct page_frag_cache *nc; 154 void *data; 155 156 fragsz = SKB_DATA_ALIGN(fragsz); 157 if (in_irq() || irqs_disabled()) { 158 nc = this_cpu_ptr(&netdev_alloc_cache); 159 data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); 160 } else { 161 local_bh_disable(); 162 data = __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); 163 local_bh_enable(); 164 } 165 return data; 166 } 167 EXPORT_SYMBOL(__netdev_alloc_frag_align); 168 169 static struct sk_buff *napi_skb_cache_get(void) 170 { 171 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 172 struct sk_buff *skb; 173 174 if (unlikely(!nc->skb_count)) 175 nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache, 176 GFP_ATOMIC, 177 NAPI_SKB_CACHE_BULK, 178 nc->skb_cache); 179 if (unlikely(!nc->skb_count)) 180 return NULL; 181 182 skb = nc->skb_cache[--nc->skb_count]; 183 kasan_unpoison_object_data(skbuff_head_cache, skb); 184 185 return skb; 186 } 187 188 /* Caller must provide SKB that is memset cleared */ 189 static void __build_skb_around(struct sk_buff *skb, void *data, 190 unsigned int frag_size) 191 { 192 struct skb_shared_info *shinfo; 193 unsigned int size = frag_size ? : ksize(data); 194 195 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 196 197 /* Assumes caller memset cleared SKB */ 198 skb->truesize = SKB_TRUESIZE(size); 199 refcount_set(&skb->users, 1); 200 skb->head = data; 201 skb->data = data; 202 skb_reset_tail_pointer(skb); 203 skb->end = skb->tail + size; 204 skb->mac_header = (typeof(skb->mac_header))~0U; 205 skb->transport_header = (typeof(skb->transport_header))~0U; 206 207 /* make sure we initialize shinfo sequentially */ 208 shinfo = skb_shinfo(skb); 209 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 210 atomic_set(&shinfo->dataref, 1); 211 212 skb_set_kcov_handle(skb, kcov_common_handle()); 213 } 214 215 /** 216 * __build_skb - build a network buffer 217 * @data: data buffer provided by caller 218 * @frag_size: size of data, or 0 if head was kmalloced 219 * 220 * Allocate a new &sk_buff. Caller provides space holding head and 221 * skb_shared_info. @data must have been allocated by kmalloc() only if 222 * @frag_size is 0, otherwise data should come from the page allocator 223 * or vmalloc() 224 * The return is the new skb buffer. 225 * On a failure the return is %NULL, and @data is not freed. 226 * Notes : 227 * Before IO, driver allocates only data buffer where NIC put incoming frame 228 * Driver should add room at head (NET_SKB_PAD) and 229 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 230 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 231 * before giving packet to stack. 232 * RX rings only contains data buffers, not full skbs. 233 */ 234 struct sk_buff *__build_skb(void *data, unsigned int frag_size) 235 { 236 struct sk_buff *skb; 237 238 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 239 if (unlikely(!skb)) 240 return NULL; 241 242 memset(skb, 0, offsetof(struct sk_buff, tail)); 243 __build_skb_around(skb, data, frag_size); 244 245 return skb; 246 } 247 248 /* build_skb() is wrapper over __build_skb(), that specifically 249 * takes care of skb->head and skb->pfmemalloc 250 * This means that if @frag_size is not zero, then @data must be backed 251 * by a page fragment, not kmalloc() or vmalloc() 252 */ 253 struct sk_buff *build_skb(void *data, unsigned int frag_size) 254 { 255 struct sk_buff *skb = __build_skb(data, frag_size); 256 257 if (skb && frag_size) { 258 skb->head_frag = 1; 259 if (page_is_pfmemalloc(virt_to_head_page(data))) 260 skb->pfmemalloc = 1; 261 } 262 return skb; 263 } 264 EXPORT_SYMBOL(build_skb); 265 266 /** 267 * build_skb_around - build a network buffer around provided skb 268 * @skb: sk_buff provide by caller, must be memset cleared 269 * @data: data buffer provided by caller 270 * @frag_size: size of data, or 0 if head was kmalloced 271 */ 272 struct sk_buff *build_skb_around(struct sk_buff *skb, 273 void *data, unsigned int frag_size) 274 { 275 if (unlikely(!skb)) 276 return NULL; 277 278 __build_skb_around(skb, data, frag_size); 279 280 if (frag_size) { 281 skb->head_frag = 1; 282 if (page_is_pfmemalloc(virt_to_head_page(data))) 283 skb->pfmemalloc = 1; 284 } 285 return skb; 286 } 287 EXPORT_SYMBOL(build_skb_around); 288 289 /** 290 * __napi_build_skb - build a network buffer 291 * @data: data buffer provided by caller 292 * @frag_size: size of data, or 0 if head was kmalloced 293 * 294 * Version of __build_skb() that uses NAPI percpu caches to obtain 295 * skbuff_head instead of inplace allocation. 296 * 297 * Returns a new &sk_buff on success, %NULL on allocation failure. 298 */ 299 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) 300 { 301 struct sk_buff *skb; 302 303 skb = napi_skb_cache_get(); 304 if (unlikely(!skb)) 305 return NULL; 306 307 memset(skb, 0, offsetof(struct sk_buff, tail)); 308 __build_skb_around(skb, data, frag_size); 309 310 return skb; 311 } 312 313 /** 314 * napi_build_skb - build a network buffer 315 * @data: data buffer provided by caller 316 * @frag_size: size of data, or 0 if head was kmalloced 317 * 318 * Version of __napi_build_skb() that takes care of skb->head_frag 319 * and skb->pfmemalloc when the data is a page or page fragment. 320 * 321 * Returns a new &sk_buff on success, %NULL on allocation failure. 322 */ 323 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) 324 { 325 struct sk_buff *skb = __napi_build_skb(data, frag_size); 326 327 if (likely(skb) && frag_size) { 328 skb->head_frag = 1; 329 skb_propagate_pfmemalloc(virt_to_head_page(data), skb); 330 } 331 332 return skb; 333 } 334 EXPORT_SYMBOL(napi_build_skb); 335 336 /* 337 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 338 * the caller if emergency pfmemalloc reserves are being used. If it is and 339 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 340 * may be used. Otherwise, the packet data may be discarded until enough 341 * memory is free 342 */ 343 static void *kmalloc_reserve(size_t size, gfp_t flags, int node, 344 bool *pfmemalloc) 345 { 346 void *obj; 347 bool ret_pfmemalloc = false; 348 349 /* 350 * Try a regular allocation, when that fails and we're not entitled 351 * to the reserves, fail. 352 */ 353 obj = kmalloc_node_track_caller(size, 354 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 355 node); 356 if (obj || !(gfp_pfmemalloc_allowed(flags))) 357 goto out; 358 359 /* Try again but now we are using pfmemalloc reserves */ 360 ret_pfmemalloc = true; 361 obj = kmalloc_node_track_caller(size, flags, node); 362 363 out: 364 if (pfmemalloc) 365 *pfmemalloc = ret_pfmemalloc; 366 367 return obj; 368 } 369 370 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 371 * 'private' fields and also do memory statistics to find all the 372 * [BEEP] leaks. 373 * 374 */ 375 376 /** 377 * __alloc_skb - allocate a network buffer 378 * @size: size to allocate 379 * @gfp_mask: allocation mask 380 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 381 * instead of head cache and allocate a cloned (child) skb. 382 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 383 * allocations in case the data is required for writeback 384 * @node: numa node to allocate memory on 385 * 386 * Allocate a new &sk_buff. The returned buffer has no headroom and a 387 * tail room of at least size bytes. The object has a reference count 388 * of one. The return is the buffer. On a failure the return is %NULL. 389 * 390 * Buffers may only be allocated from interrupts using a @gfp_mask of 391 * %GFP_ATOMIC. 392 */ 393 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 394 int flags, int node) 395 { 396 struct kmem_cache *cache; 397 struct sk_buff *skb; 398 u8 *data; 399 bool pfmemalloc; 400 401 cache = (flags & SKB_ALLOC_FCLONE) 402 ? skbuff_fclone_cache : skbuff_head_cache; 403 404 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 405 gfp_mask |= __GFP_MEMALLOC; 406 407 /* Get the HEAD */ 408 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && 409 likely(node == NUMA_NO_NODE || node == numa_mem_id())) 410 skb = napi_skb_cache_get(); 411 else 412 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); 413 if (unlikely(!skb)) 414 return NULL; 415 prefetchw(skb); 416 417 /* We do our best to align skb_shared_info on a separate cache 418 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 419 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 420 * Both skb->head and skb_shared_info are cache line aligned. 421 */ 422 size = SKB_DATA_ALIGN(size); 423 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 424 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 425 if (unlikely(!data)) 426 goto nodata; 427 /* kmalloc(size) might give us more room than requested. 428 * Put skb_shared_info exactly at the end of allocated zone, 429 * to allow max possible filling before reallocation. 430 */ 431 size = SKB_WITH_OVERHEAD(ksize(data)); 432 prefetchw(data + size); 433 434 /* 435 * Only clear those fields we need to clear, not those that we will 436 * actually initialise below. Hence, don't put any more fields after 437 * the tail pointer in struct sk_buff! 438 */ 439 memset(skb, 0, offsetof(struct sk_buff, tail)); 440 __build_skb_around(skb, data, 0); 441 skb->pfmemalloc = pfmemalloc; 442 443 if (flags & SKB_ALLOC_FCLONE) { 444 struct sk_buff_fclones *fclones; 445 446 fclones = container_of(skb, struct sk_buff_fclones, skb1); 447 448 skb->fclone = SKB_FCLONE_ORIG; 449 refcount_set(&fclones->fclone_ref, 1); 450 451 fclones->skb2.fclone = SKB_FCLONE_CLONE; 452 } 453 454 return skb; 455 456 nodata: 457 kmem_cache_free(cache, skb); 458 return NULL; 459 } 460 EXPORT_SYMBOL(__alloc_skb); 461 462 /** 463 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 464 * @dev: network device to receive on 465 * @len: length to allocate 466 * @gfp_mask: get_free_pages mask, passed to alloc_skb 467 * 468 * Allocate a new &sk_buff and assign it a usage count of one. The 469 * buffer has NET_SKB_PAD headroom built in. Users should allocate 470 * the headroom they think they need without accounting for the 471 * built in space. The built in space is used for optimisations. 472 * 473 * %NULL is returned if there is no free memory. 474 */ 475 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, 476 gfp_t gfp_mask) 477 { 478 struct page_frag_cache *nc; 479 struct sk_buff *skb; 480 bool pfmemalloc; 481 void *data; 482 483 len += NET_SKB_PAD; 484 485 /* If requested length is either too small or too big, 486 * we use kmalloc() for skb->head allocation. 487 */ 488 if (len <= SKB_WITH_OVERHEAD(1024) || 489 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 490 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 491 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); 492 if (!skb) 493 goto skb_fail; 494 goto skb_success; 495 } 496 497 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 498 len = SKB_DATA_ALIGN(len); 499 500 if (sk_memalloc_socks()) 501 gfp_mask |= __GFP_MEMALLOC; 502 503 if (in_irq() || irqs_disabled()) { 504 nc = this_cpu_ptr(&netdev_alloc_cache); 505 data = page_frag_alloc(nc, len, gfp_mask); 506 pfmemalloc = nc->pfmemalloc; 507 } else { 508 local_bh_disable(); 509 nc = this_cpu_ptr(&napi_alloc_cache.page); 510 data = page_frag_alloc(nc, len, gfp_mask); 511 pfmemalloc = nc->pfmemalloc; 512 local_bh_enable(); 513 } 514 515 if (unlikely(!data)) 516 return NULL; 517 518 skb = __build_skb(data, len); 519 if (unlikely(!skb)) { 520 skb_free_frag(data); 521 return NULL; 522 } 523 524 if (pfmemalloc) 525 skb->pfmemalloc = 1; 526 skb->head_frag = 1; 527 528 skb_success: 529 skb_reserve(skb, NET_SKB_PAD); 530 skb->dev = dev; 531 532 skb_fail: 533 return skb; 534 } 535 EXPORT_SYMBOL(__netdev_alloc_skb); 536 537 /** 538 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance 539 * @napi: napi instance this buffer was allocated for 540 * @len: length to allocate 541 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages 542 * 543 * Allocate a new sk_buff for use in NAPI receive. This buffer will 544 * attempt to allocate the head from a special reserved region used 545 * only for NAPI Rx allocation. By doing this we can save several 546 * CPU cycles by avoiding having to disable and re-enable IRQs. 547 * 548 * %NULL is returned if there is no free memory. 549 */ 550 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, 551 gfp_t gfp_mask) 552 { 553 struct napi_alloc_cache *nc; 554 struct sk_buff *skb; 555 void *data; 556 557 len += NET_SKB_PAD + NET_IP_ALIGN; 558 559 /* If requested length is either too small or too big, 560 * we use kmalloc() for skb->head allocation. 561 */ 562 if (len <= SKB_WITH_OVERHEAD(1024) || 563 len > SKB_WITH_OVERHEAD(PAGE_SIZE) || 564 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { 565 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, 566 NUMA_NO_NODE); 567 if (!skb) 568 goto skb_fail; 569 goto skb_success; 570 } 571 572 nc = this_cpu_ptr(&napi_alloc_cache); 573 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 574 len = SKB_DATA_ALIGN(len); 575 576 if (sk_memalloc_socks()) 577 gfp_mask |= __GFP_MEMALLOC; 578 579 data = page_frag_alloc(&nc->page, len, gfp_mask); 580 if (unlikely(!data)) 581 return NULL; 582 583 skb = __napi_build_skb(data, len); 584 if (unlikely(!skb)) { 585 skb_free_frag(data); 586 return NULL; 587 } 588 589 if (nc->page.pfmemalloc) 590 skb->pfmemalloc = 1; 591 skb->head_frag = 1; 592 593 skb_success: 594 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); 595 skb->dev = napi->dev; 596 597 skb_fail: 598 return skb; 599 } 600 EXPORT_SYMBOL(__napi_alloc_skb); 601 602 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 603 int size, unsigned int truesize) 604 { 605 skb_fill_page_desc(skb, i, page, off, size); 606 skb->len += size; 607 skb->data_len += size; 608 skb->truesize += truesize; 609 } 610 EXPORT_SYMBOL(skb_add_rx_frag); 611 612 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 613 unsigned int truesize) 614 { 615 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 616 617 skb_frag_size_add(frag, size); 618 skb->len += size; 619 skb->data_len += size; 620 skb->truesize += truesize; 621 } 622 EXPORT_SYMBOL(skb_coalesce_rx_frag); 623 624 static void skb_drop_list(struct sk_buff **listp) 625 { 626 kfree_skb_list(*listp); 627 *listp = NULL; 628 } 629 630 static inline void skb_drop_fraglist(struct sk_buff *skb) 631 { 632 skb_drop_list(&skb_shinfo(skb)->frag_list); 633 } 634 635 static void skb_clone_fraglist(struct sk_buff *skb) 636 { 637 struct sk_buff *list; 638 639 skb_walk_frags(skb, list) 640 skb_get(list); 641 } 642 643 static void skb_free_head(struct sk_buff *skb) 644 { 645 unsigned char *head = skb->head; 646 647 if (skb->head_frag) 648 skb_free_frag(head); 649 else 650 kfree(head); 651 } 652 653 static void skb_release_data(struct sk_buff *skb) 654 { 655 struct skb_shared_info *shinfo = skb_shinfo(skb); 656 int i; 657 658 if (skb->cloned && 659 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 660 &shinfo->dataref)) 661 return; 662 663 skb_zcopy_clear(skb, true); 664 665 for (i = 0; i < shinfo->nr_frags; i++) 666 __skb_frag_unref(&shinfo->frags[i]); 667 668 if (shinfo->frag_list) 669 kfree_skb_list(shinfo->frag_list); 670 671 skb_free_head(skb); 672 } 673 674 /* 675 * Free an skbuff by memory without cleaning the state. 676 */ 677 static void kfree_skbmem(struct sk_buff *skb) 678 { 679 struct sk_buff_fclones *fclones; 680 681 switch (skb->fclone) { 682 case SKB_FCLONE_UNAVAILABLE: 683 kmem_cache_free(skbuff_head_cache, skb); 684 return; 685 686 case SKB_FCLONE_ORIG: 687 fclones = container_of(skb, struct sk_buff_fclones, skb1); 688 689 /* We usually free the clone (TX completion) before original skb 690 * This test would have no chance to be true for the clone, 691 * while here, branch prediction will be good. 692 */ 693 if (refcount_read(&fclones->fclone_ref) == 1) 694 goto fastpath; 695 break; 696 697 default: /* SKB_FCLONE_CLONE */ 698 fclones = container_of(skb, struct sk_buff_fclones, skb2); 699 break; 700 } 701 if (!refcount_dec_and_test(&fclones->fclone_ref)) 702 return; 703 fastpath: 704 kmem_cache_free(skbuff_fclone_cache, fclones); 705 } 706 707 void skb_release_head_state(struct sk_buff *skb) 708 { 709 skb_dst_drop(skb); 710 if (skb->destructor) { 711 WARN_ON(in_irq()); 712 skb->destructor(skb); 713 } 714 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 715 nf_conntrack_put(skb_nfct(skb)); 716 #endif 717 skb_ext_put(skb); 718 } 719 720 /* Free everything but the sk_buff shell. */ 721 static void skb_release_all(struct sk_buff *skb) 722 { 723 skb_release_head_state(skb); 724 if (likely(skb->head)) 725 skb_release_data(skb); 726 } 727 728 /** 729 * __kfree_skb - private function 730 * @skb: buffer 731 * 732 * Free an sk_buff. Release anything attached to the buffer. 733 * Clean the state. This is an internal helper function. Users should 734 * always call kfree_skb 735 */ 736 737 void __kfree_skb(struct sk_buff *skb) 738 { 739 skb_release_all(skb); 740 kfree_skbmem(skb); 741 } 742 EXPORT_SYMBOL(__kfree_skb); 743 744 /** 745 * kfree_skb - free an sk_buff 746 * @skb: buffer to free 747 * 748 * Drop a reference to the buffer and free it if the usage count has 749 * hit zero. 750 */ 751 void kfree_skb(struct sk_buff *skb) 752 { 753 if (!skb_unref(skb)) 754 return; 755 756 trace_kfree_skb(skb, __builtin_return_address(0)); 757 __kfree_skb(skb); 758 } 759 EXPORT_SYMBOL(kfree_skb); 760 761 void kfree_skb_list(struct sk_buff *segs) 762 { 763 while (segs) { 764 struct sk_buff *next = segs->next; 765 766 kfree_skb(segs); 767 segs = next; 768 } 769 } 770 EXPORT_SYMBOL(kfree_skb_list); 771 772 /* Dump skb information and contents. 773 * 774 * Must only be called from net_ratelimit()-ed paths. 775 * 776 * Dumps whole packets if full_pkt, only headers otherwise. 777 */ 778 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) 779 { 780 struct skb_shared_info *sh = skb_shinfo(skb); 781 struct net_device *dev = skb->dev; 782 struct sock *sk = skb->sk; 783 struct sk_buff *list_skb; 784 bool has_mac, has_trans; 785 int headroom, tailroom; 786 int i, len, seg_len; 787 788 if (full_pkt) 789 len = skb->len; 790 else 791 len = min_t(int, skb->len, MAX_HEADER + 128); 792 793 headroom = skb_headroom(skb); 794 tailroom = skb_tailroom(skb); 795 796 has_mac = skb_mac_header_was_set(skb); 797 has_trans = skb_transport_header_was_set(skb); 798 799 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" 800 "mac=(%d,%d) net=(%d,%d) trans=%d\n" 801 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" 802 "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" 803 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", 804 level, skb->len, headroom, skb_headlen(skb), tailroom, 805 has_mac ? skb->mac_header : -1, 806 has_mac ? skb_mac_header_len(skb) : -1, 807 skb->network_header, 808 has_trans ? skb_network_header_len(skb) : -1, 809 has_trans ? skb->transport_header : -1, 810 sh->tx_flags, sh->nr_frags, 811 sh->gso_size, sh->gso_type, sh->gso_segs, 812 skb->csum, skb->ip_summed, skb->csum_complete_sw, 813 skb->csum_valid, skb->csum_level, 814 skb->hash, skb->sw_hash, skb->l4_hash, 815 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); 816 817 if (dev) 818 printk("%sdev name=%s feat=0x%pNF\n", 819 level, dev->name, &dev->features); 820 if (sk) 821 printk("%ssk family=%hu type=%u proto=%u\n", 822 level, sk->sk_family, sk->sk_type, sk->sk_protocol); 823 824 if (full_pkt && headroom) 825 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, 826 16, 1, skb->head, headroom, false); 827 828 seg_len = min_t(int, skb_headlen(skb), len); 829 if (seg_len) 830 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, 831 16, 1, skb->data, seg_len, false); 832 len -= seg_len; 833 834 if (full_pkt && tailroom) 835 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, 836 16, 1, skb_tail_pointer(skb), tailroom, false); 837 838 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { 839 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 840 u32 p_off, p_len, copied; 841 struct page *p; 842 u8 *vaddr; 843 844 skb_frag_foreach_page(frag, skb_frag_off(frag), 845 skb_frag_size(frag), p, p_off, p_len, 846 copied) { 847 seg_len = min_t(int, p_len, len); 848 vaddr = kmap_atomic(p); 849 print_hex_dump(level, "skb frag: ", 850 DUMP_PREFIX_OFFSET, 851 16, 1, vaddr + p_off, seg_len, false); 852 kunmap_atomic(vaddr); 853 len -= seg_len; 854 if (!len) 855 break; 856 } 857 } 858 859 if (full_pkt && skb_has_frag_list(skb)) { 860 printk("skb fraglist:\n"); 861 skb_walk_frags(skb, list_skb) 862 skb_dump(level, list_skb, true); 863 } 864 } 865 EXPORT_SYMBOL(skb_dump); 866 867 /** 868 * skb_tx_error - report an sk_buff xmit error 869 * @skb: buffer that triggered an error 870 * 871 * Report xmit error if a device callback is tracking this skb. 872 * skb must be freed afterwards. 873 */ 874 void skb_tx_error(struct sk_buff *skb) 875 { 876 skb_zcopy_clear(skb, true); 877 } 878 EXPORT_SYMBOL(skb_tx_error); 879 880 #ifdef CONFIG_TRACEPOINTS 881 /** 882 * consume_skb - free an skbuff 883 * @skb: buffer to free 884 * 885 * Drop a ref to the buffer and free it if the usage count has hit zero 886 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 887 * is being dropped after a failure and notes that 888 */ 889 void consume_skb(struct sk_buff *skb) 890 { 891 if (!skb_unref(skb)) 892 return; 893 894 trace_consume_skb(skb); 895 __kfree_skb(skb); 896 } 897 EXPORT_SYMBOL(consume_skb); 898 #endif 899 900 /** 901 * __consume_stateless_skb - free an skbuff, assuming it is stateless 902 * @skb: buffer to free 903 * 904 * Alike consume_skb(), but this variant assumes that this is the last 905 * skb reference and all the head states have been already dropped 906 */ 907 void __consume_stateless_skb(struct sk_buff *skb) 908 { 909 trace_consume_skb(skb); 910 skb_release_data(skb); 911 kfree_skbmem(skb); 912 } 913 914 static void napi_skb_cache_put(struct sk_buff *skb) 915 { 916 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); 917 u32 i; 918 919 kasan_poison_object_data(skbuff_head_cache, skb); 920 nc->skb_cache[nc->skb_count++] = skb; 921 922 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { 923 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) 924 kasan_unpoison_object_data(skbuff_head_cache, 925 nc->skb_cache[i]); 926 927 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF, 928 nc->skb_cache + NAPI_SKB_CACHE_HALF); 929 nc->skb_count = NAPI_SKB_CACHE_HALF; 930 } 931 } 932 933 void __kfree_skb_defer(struct sk_buff *skb) 934 { 935 skb_release_all(skb); 936 napi_skb_cache_put(skb); 937 } 938 939 void napi_skb_free_stolen_head(struct sk_buff *skb) 940 { 941 skb_dst_drop(skb); 942 skb_ext_put(skb); 943 napi_skb_cache_put(skb); 944 } 945 946 void napi_consume_skb(struct sk_buff *skb, int budget) 947 { 948 /* Zero budget indicate non-NAPI context called us, like netpoll */ 949 if (unlikely(!budget)) { 950 dev_consume_skb_any(skb); 951 return; 952 } 953 954 lockdep_assert_in_softirq(); 955 956 if (!skb_unref(skb)) 957 return; 958 959 /* if reaching here SKB is ready to free */ 960 trace_consume_skb(skb); 961 962 /* if SKB is a clone, don't handle this case */ 963 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { 964 __kfree_skb(skb); 965 return; 966 } 967 968 skb_release_all(skb); 969 napi_skb_cache_put(skb); 970 } 971 EXPORT_SYMBOL(napi_consume_skb); 972 973 /* Make sure a field is enclosed inside headers_start/headers_end section */ 974 #define CHECK_SKB_FIELD(field) \ 975 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 976 offsetof(struct sk_buff, headers_start)); \ 977 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 978 offsetof(struct sk_buff, headers_end)); \ 979 980 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 981 { 982 new->tstamp = old->tstamp; 983 /* We do not copy old->sk */ 984 new->dev = old->dev; 985 memcpy(new->cb, old->cb, sizeof(old->cb)); 986 skb_dst_copy(new, old); 987 __skb_ext_copy(new, old); 988 __nf_copy(new, old, false); 989 990 /* Note : this field could be in headers_start/headers_end section 991 * It is not yet because we do not want to have a 16 bit hole 992 */ 993 new->queue_mapping = old->queue_mapping; 994 995 memcpy(&new->headers_start, &old->headers_start, 996 offsetof(struct sk_buff, headers_end) - 997 offsetof(struct sk_buff, headers_start)); 998 CHECK_SKB_FIELD(protocol); 999 CHECK_SKB_FIELD(csum); 1000 CHECK_SKB_FIELD(hash); 1001 CHECK_SKB_FIELD(priority); 1002 CHECK_SKB_FIELD(skb_iif); 1003 CHECK_SKB_FIELD(vlan_proto); 1004 CHECK_SKB_FIELD(vlan_tci); 1005 CHECK_SKB_FIELD(transport_header); 1006 CHECK_SKB_FIELD(network_header); 1007 CHECK_SKB_FIELD(mac_header); 1008 CHECK_SKB_FIELD(inner_protocol); 1009 CHECK_SKB_FIELD(inner_transport_header); 1010 CHECK_SKB_FIELD(inner_network_header); 1011 CHECK_SKB_FIELD(inner_mac_header); 1012 CHECK_SKB_FIELD(mark); 1013 #ifdef CONFIG_NETWORK_SECMARK 1014 CHECK_SKB_FIELD(secmark); 1015 #endif 1016 #ifdef CONFIG_NET_RX_BUSY_POLL 1017 CHECK_SKB_FIELD(napi_id); 1018 #endif 1019 #ifdef CONFIG_XPS 1020 CHECK_SKB_FIELD(sender_cpu); 1021 #endif 1022 #ifdef CONFIG_NET_SCHED 1023 CHECK_SKB_FIELD(tc_index); 1024 #endif 1025 1026 } 1027 1028 /* 1029 * You should not add any new code to this function. Add it to 1030 * __copy_skb_header above instead. 1031 */ 1032 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 1033 { 1034 #define C(x) n->x = skb->x 1035 1036 n->next = n->prev = NULL; 1037 n->sk = NULL; 1038 __copy_skb_header(n, skb); 1039 1040 C(len); 1041 C(data_len); 1042 C(mac_len); 1043 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 1044 n->cloned = 1; 1045 n->nohdr = 0; 1046 n->peeked = 0; 1047 C(pfmemalloc); 1048 n->destructor = NULL; 1049 C(tail); 1050 C(end); 1051 C(head); 1052 C(head_frag); 1053 C(data); 1054 C(truesize); 1055 refcount_set(&n->users, 1); 1056 1057 atomic_inc(&(skb_shinfo(skb)->dataref)); 1058 skb->cloned = 1; 1059 1060 return n; 1061 #undef C 1062 } 1063 1064 /** 1065 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg 1066 * @first: first sk_buff of the msg 1067 */ 1068 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) 1069 { 1070 struct sk_buff *n; 1071 1072 n = alloc_skb(0, GFP_ATOMIC); 1073 if (!n) 1074 return NULL; 1075 1076 n->len = first->len; 1077 n->data_len = first->len; 1078 n->truesize = first->truesize; 1079 1080 skb_shinfo(n)->frag_list = first; 1081 1082 __copy_skb_header(n, first); 1083 n->destructor = NULL; 1084 1085 return n; 1086 } 1087 EXPORT_SYMBOL_GPL(alloc_skb_for_msg); 1088 1089 /** 1090 * skb_morph - morph one skb into another 1091 * @dst: the skb to receive the contents 1092 * @src: the skb to supply the contents 1093 * 1094 * This is identical to skb_clone except that the target skb is 1095 * supplied by the user. 1096 * 1097 * The target skb is returned upon exit. 1098 */ 1099 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 1100 { 1101 skb_release_all(dst); 1102 return __skb_clone(dst, src); 1103 } 1104 EXPORT_SYMBOL_GPL(skb_morph); 1105 1106 int mm_account_pinned_pages(struct mmpin *mmp, size_t size) 1107 { 1108 unsigned long max_pg, num_pg, new_pg, old_pg; 1109 struct user_struct *user; 1110 1111 if (capable(CAP_IPC_LOCK) || !size) 1112 return 0; 1113 1114 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ 1115 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 1116 user = mmp->user ? : current_user(); 1117 1118 do { 1119 old_pg = atomic_long_read(&user->locked_vm); 1120 new_pg = old_pg + num_pg; 1121 if (new_pg > max_pg) 1122 return -ENOBUFS; 1123 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) != 1124 old_pg); 1125 1126 if (!mmp->user) { 1127 mmp->user = get_uid(user); 1128 mmp->num_pg = num_pg; 1129 } else { 1130 mmp->num_pg += num_pg; 1131 } 1132 1133 return 0; 1134 } 1135 EXPORT_SYMBOL_GPL(mm_account_pinned_pages); 1136 1137 void mm_unaccount_pinned_pages(struct mmpin *mmp) 1138 { 1139 if (mmp->user) { 1140 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); 1141 free_uid(mmp->user); 1142 } 1143 } 1144 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); 1145 1146 struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) 1147 { 1148 struct ubuf_info *uarg; 1149 struct sk_buff *skb; 1150 1151 WARN_ON_ONCE(!in_task()); 1152 1153 skb = sock_omalloc(sk, 0, GFP_KERNEL); 1154 if (!skb) 1155 return NULL; 1156 1157 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); 1158 uarg = (void *)skb->cb; 1159 uarg->mmp.user = NULL; 1160 1161 if (mm_account_pinned_pages(&uarg->mmp, size)) { 1162 kfree_skb(skb); 1163 return NULL; 1164 } 1165 1166 uarg->callback = msg_zerocopy_callback; 1167 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; 1168 uarg->len = 1; 1169 uarg->bytelen = size; 1170 uarg->zerocopy = 1; 1171 uarg->flags = SKBFL_ZEROCOPY_FRAG; 1172 refcount_set(&uarg->refcnt, 1); 1173 sock_hold(sk); 1174 1175 return uarg; 1176 } 1177 EXPORT_SYMBOL_GPL(msg_zerocopy_alloc); 1178 1179 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) 1180 { 1181 return container_of((void *)uarg, struct sk_buff, cb); 1182 } 1183 1184 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, 1185 struct ubuf_info *uarg) 1186 { 1187 if (uarg) { 1188 const u32 byte_limit = 1 << 19; /* limit to a few TSO */ 1189 u32 bytelen, next; 1190 1191 /* realloc only when socket is locked (TCP, UDP cork), 1192 * so uarg->len and sk_zckey access is serialized 1193 */ 1194 if (!sock_owned_by_user(sk)) { 1195 WARN_ON_ONCE(1); 1196 return NULL; 1197 } 1198 1199 bytelen = uarg->bytelen + size; 1200 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { 1201 /* TCP can create new skb to attach new uarg */ 1202 if (sk->sk_type == SOCK_STREAM) 1203 goto new_alloc; 1204 return NULL; 1205 } 1206 1207 next = (u32)atomic_read(&sk->sk_zckey); 1208 if ((u32)(uarg->id + uarg->len) == next) { 1209 if (mm_account_pinned_pages(&uarg->mmp, size)) 1210 return NULL; 1211 uarg->len++; 1212 uarg->bytelen = bytelen; 1213 atomic_set(&sk->sk_zckey, ++next); 1214 1215 /* no extra ref when appending to datagram (MSG_MORE) */ 1216 if (sk->sk_type == SOCK_STREAM) 1217 net_zcopy_get(uarg); 1218 1219 return uarg; 1220 } 1221 } 1222 1223 new_alloc: 1224 return msg_zerocopy_alloc(sk, size); 1225 } 1226 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); 1227 1228 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) 1229 { 1230 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); 1231 u32 old_lo, old_hi; 1232 u64 sum_len; 1233 1234 old_lo = serr->ee.ee_info; 1235 old_hi = serr->ee.ee_data; 1236 sum_len = old_hi - old_lo + 1ULL + len; 1237 1238 if (sum_len >= (1ULL << 32)) 1239 return false; 1240 1241 if (lo != old_hi + 1) 1242 return false; 1243 1244 serr->ee.ee_data += len; 1245 return true; 1246 } 1247 1248 static void __msg_zerocopy_callback(struct ubuf_info *uarg) 1249 { 1250 struct sk_buff *tail, *skb = skb_from_uarg(uarg); 1251 struct sock_exterr_skb *serr; 1252 struct sock *sk = skb->sk; 1253 struct sk_buff_head *q; 1254 unsigned long flags; 1255 u32 lo, hi; 1256 u16 len; 1257 1258 mm_unaccount_pinned_pages(&uarg->mmp); 1259 1260 /* if !len, there was only 1 call, and it was aborted 1261 * so do not queue a completion notification 1262 */ 1263 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) 1264 goto release; 1265 1266 len = uarg->len; 1267 lo = uarg->id; 1268 hi = uarg->id + len - 1; 1269 1270 serr = SKB_EXT_ERR(skb); 1271 memset(serr, 0, sizeof(*serr)); 1272 serr->ee.ee_errno = 0; 1273 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; 1274 serr->ee.ee_data = hi; 1275 serr->ee.ee_info = lo; 1276 if (!uarg->zerocopy) 1277 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; 1278 1279 q = &sk->sk_error_queue; 1280 spin_lock_irqsave(&q->lock, flags); 1281 tail = skb_peek_tail(q); 1282 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || 1283 !skb_zerocopy_notify_extend(tail, lo, len)) { 1284 __skb_queue_tail(q, skb); 1285 skb = NULL; 1286 } 1287 spin_unlock_irqrestore(&q->lock, flags); 1288 1289 sk->sk_error_report(sk); 1290 1291 release: 1292 consume_skb(skb); 1293 sock_put(sk); 1294 } 1295 1296 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, 1297 bool success) 1298 { 1299 uarg->zerocopy = uarg->zerocopy & success; 1300 1301 if (refcount_dec_and_test(&uarg->refcnt)) 1302 __msg_zerocopy_callback(uarg); 1303 } 1304 EXPORT_SYMBOL_GPL(msg_zerocopy_callback); 1305 1306 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) 1307 { 1308 struct sock *sk = skb_from_uarg(uarg)->sk; 1309 1310 atomic_dec(&sk->sk_zckey); 1311 uarg->len--; 1312 1313 if (have_uref) 1314 msg_zerocopy_callback(NULL, uarg, true); 1315 } 1316 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); 1317 1318 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) 1319 { 1320 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); 1321 } 1322 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram); 1323 1324 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, 1325 struct msghdr *msg, int len, 1326 struct ubuf_info *uarg) 1327 { 1328 struct ubuf_info *orig_uarg = skb_zcopy(skb); 1329 struct iov_iter orig_iter = msg->msg_iter; 1330 int err, orig_len = skb->len; 1331 1332 /* An skb can only point to one uarg. This edge case happens when 1333 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. 1334 */ 1335 if (orig_uarg && uarg != orig_uarg) 1336 return -EEXIST; 1337 1338 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); 1339 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { 1340 struct sock *save_sk = skb->sk; 1341 1342 /* Streams do not free skb on error. Reset to prev state. */ 1343 msg->msg_iter = orig_iter; 1344 skb->sk = sk; 1345 ___pskb_trim(skb, orig_len); 1346 skb->sk = save_sk; 1347 return err; 1348 } 1349 1350 skb_zcopy_set(skb, uarg, NULL); 1351 return skb->len - orig_len; 1352 } 1353 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); 1354 1355 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, 1356 gfp_t gfp_mask) 1357 { 1358 if (skb_zcopy(orig)) { 1359 if (skb_zcopy(nskb)) { 1360 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ 1361 if (!gfp_mask) { 1362 WARN_ON_ONCE(1); 1363 return -ENOMEM; 1364 } 1365 if (skb_uarg(nskb) == skb_uarg(orig)) 1366 return 0; 1367 if (skb_copy_ubufs(nskb, GFP_ATOMIC)) 1368 return -EIO; 1369 } 1370 skb_zcopy_set(nskb, skb_uarg(orig), NULL); 1371 } 1372 return 0; 1373 } 1374 1375 /** 1376 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 1377 * @skb: the skb to modify 1378 * @gfp_mask: allocation priority 1379 * 1380 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. 1381 * It will copy all frags into kernel and drop the reference 1382 * to userspace pages. 1383 * 1384 * If this function is called from an interrupt gfp_mask() must be 1385 * %GFP_ATOMIC. 1386 * 1387 * Returns 0 on success or a negative error code on failure 1388 * to allocate kernel memory to copy to. 1389 */ 1390 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 1391 { 1392 int num_frags = skb_shinfo(skb)->nr_frags; 1393 struct page *page, *head = NULL; 1394 int i, new_frags; 1395 u32 d_off; 1396 1397 if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) 1398 return -EINVAL; 1399 1400 if (!num_frags) 1401 goto release; 1402 1403 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; 1404 for (i = 0; i < new_frags; i++) { 1405 page = alloc_page(gfp_mask); 1406 if (!page) { 1407 while (head) { 1408 struct page *next = (struct page *)page_private(head); 1409 put_page(head); 1410 head = next; 1411 } 1412 return -ENOMEM; 1413 } 1414 set_page_private(page, (unsigned long)head); 1415 head = page; 1416 } 1417 1418 page = head; 1419 d_off = 0; 1420 for (i = 0; i < num_frags; i++) { 1421 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1422 u32 p_off, p_len, copied; 1423 struct page *p; 1424 u8 *vaddr; 1425 1426 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), 1427 p, p_off, p_len, copied) { 1428 u32 copy, done = 0; 1429 vaddr = kmap_atomic(p); 1430 1431 while (done < p_len) { 1432 if (d_off == PAGE_SIZE) { 1433 d_off = 0; 1434 page = (struct page *)page_private(page); 1435 } 1436 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); 1437 memcpy(page_address(page) + d_off, 1438 vaddr + p_off + done, copy); 1439 done += copy; 1440 d_off += copy; 1441 } 1442 kunmap_atomic(vaddr); 1443 } 1444 } 1445 1446 /* skb frags release userspace buffers */ 1447 for (i = 0; i < num_frags; i++) 1448 skb_frag_unref(skb, i); 1449 1450 /* skb frags point to kernel buffers */ 1451 for (i = 0; i < new_frags - 1; i++) { 1452 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); 1453 head = (struct page *)page_private(head); 1454 } 1455 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); 1456 skb_shinfo(skb)->nr_frags = new_frags; 1457 1458 release: 1459 skb_zcopy_clear(skb, false); 1460 return 0; 1461 } 1462 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 1463 1464 /** 1465 * skb_clone - duplicate an sk_buff 1466 * @skb: buffer to clone 1467 * @gfp_mask: allocation priority 1468 * 1469 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 1470 * copies share the same packet data but not structure. The new 1471 * buffer has a reference count of 1. If the allocation fails the 1472 * function returns %NULL otherwise the new buffer is returned. 1473 * 1474 * If this function is called from an interrupt gfp_mask() must be 1475 * %GFP_ATOMIC. 1476 */ 1477 1478 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 1479 { 1480 struct sk_buff_fclones *fclones = container_of(skb, 1481 struct sk_buff_fclones, 1482 skb1); 1483 struct sk_buff *n; 1484 1485 if (skb_orphan_frags(skb, gfp_mask)) 1486 return NULL; 1487 1488 if (skb->fclone == SKB_FCLONE_ORIG && 1489 refcount_read(&fclones->fclone_ref) == 1) { 1490 n = &fclones->skb2; 1491 refcount_set(&fclones->fclone_ref, 2); 1492 } else { 1493 if (skb_pfmemalloc(skb)) 1494 gfp_mask |= __GFP_MEMALLOC; 1495 1496 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 1497 if (!n) 1498 return NULL; 1499 1500 n->fclone = SKB_FCLONE_UNAVAILABLE; 1501 } 1502 1503 return __skb_clone(n, skb); 1504 } 1505 EXPORT_SYMBOL(skb_clone); 1506 1507 void skb_headers_offset_update(struct sk_buff *skb, int off) 1508 { 1509 /* Only adjust this if it actually is csum_start rather than csum */ 1510 if (skb->ip_summed == CHECKSUM_PARTIAL) 1511 skb->csum_start += off; 1512 /* {transport,network,mac}_header and tail are relative to skb->head */ 1513 skb->transport_header += off; 1514 skb->network_header += off; 1515 if (skb_mac_header_was_set(skb)) 1516 skb->mac_header += off; 1517 skb->inner_transport_header += off; 1518 skb->inner_network_header += off; 1519 skb->inner_mac_header += off; 1520 } 1521 EXPORT_SYMBOL(skb_headers_offset_update); 1522 1523 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) 1524 { 1525 __copy_skb_header(new, old); 1526 1527 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 1528 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 1529 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 1530 } 1531 EXPORT_SYMBOL(skb_copy_header); 1532 1533 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 1534 { 1535 if (skb_pfmemalloc(skb)) 1536 return SKB_ALLOC_RX; 1537 return 0; 1538 } 1539 1540 /** 1541 * skb_copy - create private copy of an sk_buff 1542 * @skb: buffer to copy 1543 * @gfp_mask: allocation priority 1544 * 1545 * Make a copy of both an &sk_buff and its data. This is used when the 1546 * caller wishes to modify the data and needs a private copy of the 1547 * data to alter. Returns %NULL on failure or the pointer to the buffer 1548 * on success. The returned buffer has a reference count of 1. 1549 * 1550 * As by-product this function converts non-linear &sk_buff to linear 1551 * one, so that &sk_buff becomes completely private and caller is allowed 1552 * to modify all the data of returned buffer. This means that this 1553 * function is not recommended for use in circumstances when only 1554 * header is going to be modified. Use pskb_copy() instead. 1555 */ 1556 1557 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 1558 { 1559 int headerlen = skb_headroom(skb); 1560 unsigned int size = skb_end_offset(skb) + skb->data_len; 1561 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1562 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1563 1564 if (!n) 1565 return NULL; 1566 1567 /* Set the data pointer */ 1568 skb_reserve(n, headerlen); 1569 /* Set the tail pointer and length */ 1570 skb_put(n, skb->len); 1571 1572 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); 1573 1574 skb_copy_header(n, skb); 1575 return n; 1576 } 1577 EXPORT_SYMBOL(skb_copy); 1578 1579 /** 1580 * __pskb_copy_fclone - create copy of an sk_buff with private head. 1581 * @skb: buffer to copy 1582 * @headroom: headroom of new skb 1583 * @gfp_mask: allocation priority 1584 * @fclone: if true allocate the copy of the skb from the fclone 1585 * cache instead of the head cache; it is recommended to set this 1586 * to true for the cases where the copy will likely be cloned 1587 * 1588 * Make a copy of both an &sk_buff and part of its data, located 1589 * in header. Fragmented data remain shared. This is used when 1590 * the caller wishes to modify only header of &sk_buff and needs 1591 * private copy of the header to alter. Returns %NULL on failure 1592 * or the pointer to the buffer on success. 1593 * The returned buffer has a reference count of 1. 1594 */ 1595 1596 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 1597 gfp_t gfp_mask, bool fclone) 1598 { 1599 unsigned int size = skb_headlen(skb) + headroom; 1600 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 1601 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 1602 1603 if (!n) 1604 goto out; 1605 1606 /* Set the data pointer */ 1607 skb_reserve(n, headroom); 1608 /* Set the tail pointer and length */ 1609 skb_put(n, skb_headlen(skb)); 1610 /* Copy the bytes */ 1611 skb_copy_from_linear_data(skb, n->data, n->len); 1612 1613 n->truesize += skb->data_len; 1614 n->data_len = skb->data_len; 1615 n->len = skb->len; 1616 1617 if (skb_shinfo(skb)->nr_frags) { 1618 int i; 1619 1620 if (skb_orphan_frags(skb, gfp_mask) || 1621 skb_zerocopy_clone(n, skb, gfp_mask)) { 1622 kfree_skb(n); 1623 n = NULL; 1624 goto out; 1625 } 1626 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1627 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1628 skb_frag_ref(skb, i); 1629 } 1630 skb_shinfo(n)->nr_frags = i; 1631 } 1632 1633 if (skb_has_frag_list(skb)) { 1634 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1635 skb_clone_fraglist(n); 1636 } 1637 1638 skb_copy_header(n, skb); 1639 out: 1640 return n; 1641 } 1642 EXPORT_SYMBOL(__pskb_copy_fclone); 1643 1644 /** 1645 * pskb_expand_head - reallocate header of &sk_buff 1646 * @skb: buffer to reallocate 1647 * @nhead: room to add at head 1648 * @ntail: room to add at tail 1649 * @gfp_mask: allocation priority 1650 * 1651 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1652 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1653 * reference count of 1. Returns zero in the case of success or error, 1654 * if expansion failed. In the last case, &sk_buff is not changed. 1655 * 1656 * All the pointers pointing into skb header may change and must be 1657 * reloaded after call to this function. 1658 */ 1659 1660 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1661 gfp_t gfp_mask) 1662 { 1663 int i, osize = skb_end_offset(skb); 1664 int size = osize + nhead + ntail; 1665 long off; 1666 u8 *data; 1667 1668 BUG_ON(nhead < 0); 1669 1670 BUG_ON(skb_shared(skb)); 1671 1672 size = SKB_DATA_ALIGN(size); 1673 1674 if (skb_pfmemalloc(skb)) 1675 gfp_mask |= __GFP_MEMALLOC; 1676 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1677 gfp_mask, NUMA_NO_NODE, NULL); 1678 if (!data) 1679 goto nodata; 1680 size = SKB_WITH_OVERHEAD(ksize(data)); 1681 1682 /* Copy only real data... and, alas, header. This should be 1683 * optimized for the cases when header is void. 1684 */ 1685 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1686 1687 memcpy((struct skb_shared_info *)(data + size), 1688 skb_shinfo(skb), 1689 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1690 1691 /* 1692 * if shinfo is shared we must drop the old head gracefully, but if it 1693 * is not we can just drop the old head and let the existing refcount 1694 * be since all we did is relocate the values 1695 */ 1696 if (skb_cloned(skb)) { 1697 if (skb_orphan_frags(skb, gfp_mask)) 1698 goto nofrags; 1699 if (skb_zcopy(skb)) 1700 refcount_inc(&skb_uarg(skb)->refcnt); 1701 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1702 skb_frag_ref(skb, i); 1703 1704 if (skb_has_frag_list(skb)) 1705 skb_clone_fraglist(skb); 1706 1707 skb_release_data(skb); 1708 } else { 1709 skb_free_head(skb); 1710 } 1711 off = (data + nhead) - skb->head; 1712 1713 skb->head = data; 1714 skb->head_frag = 0; 1715 skb->data += off; 1716 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1717 skb->end = size; 1718 off = nhead; 1719 #else 1720 skb->end = skb->head + size; 1721 #endif 1722 skb->tail += off; 1723 skb_headers_offset_update(skb, nhead); 1724 skb->cloned = 0; 1725 skb->hdr_len = 0; 1726 skb->nohdr = 0; 1727 atomic_set(&skb_shinfo(skb)->dataref, 1); 1728 1729 skb_metadata_clear(skb); 1730 1731 /* It is not generally safe to change skb->truesize. 1732 * For the moment, we really care of rx path, or 1733 * when skb is orphaned (not attached to a socket). 1734 */ 1735 if (!skb->sk || skb->destructor == sock_edemux) 1736 skb->truesize += size - osize; 1737 1738 return 0; 1739 1740 nofrags: 1741 kfree(data); 1742 nodata: 1743 return -ENOMEM; 1744 } 1745 EXPORT_SYMBOL(pskb_expand_head); 1746 1747 /* Make private copy of skb with writable head and some headroom */ 1748 1749 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1750 { 1751 struct sk_buff *skb2; 1752 int delta = headroom - skb_headroom(skb); 1753 1754 if (delta <= 0) 1755 skb2 = pskb_copy(skb, GFP_ATOMIC); 1756 else { 1757 skb2 = skb_clone(skb, GFP_ATOMIC); 1758 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1759 GFP_ATOMIC)) { 1760 kfree_skb(skb2); 1761 skb2 = NULL; 1762 } 1763 } 1764 return skb2; 1765 } 1766 EXPORT_SYMBOL(skb_realloc_headroom); 1767 1768 /** 1769 * skb_copy_expand - copy and expand sk_buff 1770 * @skb: buffer to copy 1771 * @newheadroom: new free bytes at head 1772 * @newtailroom: new free bytes at tail 1773 * @gfp_mask: allocation priority 1774 * 1775 * Make a copy of both an &sk_buff and its data and while doing so 1776 * allocate additional space. 1777 * 1778 * This is used when the caller wishes to modify the data and needs a 1779 * private copy of the data to alter as well as more space for new fields. 1780 * Returns %NULL on failure or the pointer to the buffer 1781 * on success. The returned buffer has a reference count of 1. 1782 * 1783 * You must pass %GFP_ATOMIC as the allocation priority if this function 1784 * is called from an interrupt. 1785 */ 1786 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1787 int newheadroom, int newtailroom, 1788 gfp_t gfp_mask) 1789 { 1790 /* 1791 * Allocate the copy buffer 1792 */ 1793 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1794 gfp_mask, skb_alloc_rx_flag(skb), 1795 NUMA_NO_NODE); 1796 int oldheadroom = skb_headroom(skb); 1797 int head_copy_len, head_copy_off; 1798 1799 if (!n) 1800 return NULL; 1801 1802 skb_reserve(n, newheadroom); 1803 1804 /* Set the tail pointer and length */ 1805 skb_put(n, skb->len); 1806 1807 head_copy_len = oldheadroom; 1808 head_copy_off = 0; 1809 if (newheadroom <= head_copy_len) 1810 head_copy_len = newheadroom; 1811 else 1812 head_copy_off = newheadroom - head_copy_len; 1813 1814 /* Copy the linear header and data. */ 1815 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1816 skb->len + head_copy_len)); 1817 1818 skb_copy_header(n, skb); 1819 1820 skb_headers_offset_update(n, newheadroom - oldheadroom); 1821 1822 return n; 1823 } 1824 EXPORT_SYMBOL(skb_copy_expand); 1825 1826 /** 1827 * __skb_pad - zero pad the tail of an skb 1828 * @skb: buffer to pad 1829 * @pad: space to pad 1830 * @free_on_error: free buffer on error 1831 * 1832 * Ensure that a buffer is followed by a padding area that is zero 1833 * filled. Used by network drivers which may DMA or transfer data 1834 * beyond the buffer end onto the wire. 1835 * 1836 * May return error in out of memory cases. The skb is freed on error 1837 * if @free_on_error is true. 1838 */ 1839 1840 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) 1841 { 1842 int err; 1843 int ntail; 1844 1845 /* If the skbuff is non linear tailroom is always zero.. */ 1846 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1847 memset(skb->data+skb->len, 0, pad); 1848 return 0; 1849 } 1850 1851 ntail = skb->data_len + pad - (skb->end - skb->tail); 1852 if (likely(skb_cloned(skb) || ntail > 0)) { 1853 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1854 if (unlikely(err)) 1855 goto free_skb; 1856 } 1857 1858 /* FIXME: The use of this function with non-linear skb's really needs 1859 * to be audited. 1860 */ 1861 err = skb_linearize(skb); 1862 if (unlikely(err)) 1863 goto free_skb; 1864 1865 memset(skb->data + skb->len, 0, pad); 1866 return 0; 1867 1868 free_skb: 1869 if (free_on_error) 1870 kfree_skb(skb); 1871 return err; 1872 } 1873 EXPORT_SYMBOL(__skb_pad); 1874 1875 /** 1876 * pskb_put - add data to the tail of a potentially fragmented buffer 1877 * @skb: start of the buffer to use 1878 * @tail: tail fragment of the buffer to use 1879 * @len: amount of data to add 1880 * 1881 * This function extends the used data area of the potentially 1882 * fragmented buffer. @tail must be the last fragment of @skb -- or 1883 * @skb itself. If this would exceed the total buffer size the kernel 1884 * will panic. A pointer to the first byte of the extra data is 1885 * returned. 1886 */ 1887 1888 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1889 { 1890 if (tail != skb) { 1891 skb->data_len += len; 1892 skb->len += len; 1893 } 1894 return skb_put(tail, len); 1895 } 1896 EXPORT_SYMBOL_GPL(pskb_put); 1897 1898 /** 1899 * skb_put - add data to a buffer 1900 * @skb: buffer to use 1901 * @len: amount of data to add 1902 * 1903 * This function extends the used data area of the buffer. If this would 1904 * exceed the total buffer size the kernel will panic. A pointer to the 1905 * first byte of the extra data is returned. 1906 */ 1907 void *skb_put(struct sk_buff *skb, unsigned int len) 1908 { 1909 void *tmp = skb_tail_pointer(skb); 1910 SKB_LINEAR_ASSERT(skb); 1911 skb->tail += len; 1912 skb->len += len; 1913 if (unlikely(skb->tail > skb->end)) 1914 skb_over_panic(skb, len, __builtin_return_address(0)); 1915 return tmp; 1916 } 1917 EXPORT_SYMBOL(skb_put); 1918 1919 /** 1920 * skb_push - add data to the start of a buffer 1921 * @skb: buffer to use 1922 * @len: amount of data to add 1923 * 1924 * This function extends the used data area of the buffer at the buffer 1925 * start. If this would exceed the total buffer headroom the kernel will 1926 * panic. A pointer to the first byte of the extra data is returned. 1927 */ 1928 void *skb_push(struct sk_buff *skb, unsigned int len) 1929 { 1930 skb->data -= len; 1931 skb->len += len; 1932 if (unlikely(skb->data < skb->head)) 1933 skb_under_panic(skb, len, __builtin_return_address(0)); 1934 return skb->data; 1935 } 1936 EXPORT_SYMBOL(skb_push); 1937 1938 /** 1939 * skb_pull - remove data from the start of a buffer 1940 * @skb: buffer to use 1941 * @len: amount of data to remove 1942 * 1943 * This function removes data from the start of a buffer, returning 1944 * the memory to the headroom. A pointer to the next data in the buffer 1945 * is returned. Once the data has been pulled future pushes will overwrite 1946 * the old data. 1947 */ 1948 void *skb_pull(struct sk_buff *skb, unsigned int len) 1949 { 1950 return skb_pull_inline(skb, len); 1951 } 1952 EXPORT_SYMBOL(skb_pull); 1953 1954 /** 1955 * skb_trim - remove end from a buffer 1956 * @skb: buffer to alter 1957 * @len: new length 1958 * 1959 * Cut the length of a buffer down by removing data from the tail. If 1960 * the buffer is already under the length specified it is not modified. 1961 * The skb must be linear. 1962 */ 1963 void skb_trim(struct sk_buff *skb, unsigned int len) 1964 { 1965 if (skb->len > len) 1966 __skb_trim(skb, len); 1967 } 1968 EXPORT_SYMBOL(skb_trim); 1969 1970 /* Trims skb to length len. It can change skb pointers. 1971 */ 1972 1973 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1974 { 1975 struct sk_buff **fragp; 1976 struct sk_buff *frag; 1977 int offset = skb_headlen(skb); 1978 int nfrags = skb_shinfo(skb)->nr_frags; 1979 int i; 1980 int err; 1981 1982 if (skb_cloned(skb) && 1983 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1984 return err; 1985 1986 i = 0; 1987 if (offset >= len) 1988 goto drop_pages; 1989 1990 for (; i < nfrags; i++) { 1991 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1992 1993 if (end < len) { 1994 offset = end; 1995 continue; 1996 } 1997 1998 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1999 2000 drop_pages: 2001 skb_shinfo(skb)->nr_frags = i; 2002 2003 for (; i < nfrags; i++) 2004 skb_frag_unref(skb, i); 2005 2006 if (skb_has_frag_list(skb)) 2007 skb_drop_fraglist(skb); 2008 goto done; 2009 } 2010 2011 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 2012 fragp = &frag->next) { 2013 int end = offset + frag->len; 2014 2015 if (skb_shared(frag)) { 2016 struct sk_buff *nfrag; 2017 2018 nfrag = skb_clone(frag, GFP_ATOMIC); 2019 if (unlikely(!nfrag)) 2020 return -ENOMEM; 2021 2022 nfrag->next = frag->next; 2023 consume_skb(frag); 2024 frag = nfrag; 2025 *fragp = frag; 2026 } 2027 2028 if (end < len) { 2029 offset = end; 2030 continue; 2031 } 2032 2033 if (end > len && 2034 unlikely((err = pskb_trim(frag, len - offset)))) 2035 return err; 2036 2037 if (frag->next) 2038 skb_drop_list(&frag->next); 2039 break; 2040 } 2041 2042 done: 2043 if (len > skb_headlen(skb)) { 2044 skb->data_len -= skb->len - len; 2045 skb->len = len; 2046 } else { 2047 skb->len = len; 2048 skb->data_len = 0; 2049 skb_set_tail_pointer(skb, len); 2050 } 2051 2052 if (!skb->sk || skb->destructor == sock_edemux) 2053 skb_condense(skb); 2054 return 0; 2055 } 2056 EXPORT_SYMBOL(___pskb_trim); 2057 2058 /* Note : use pskb_trim_rcsum() instead of calling this directly 2059 */ 2060 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) 2061 { 2062 if (skb->ip_summed == CHECKSUM_COMPLETE) { 2063 int delta = skb->len - len; 2064 2065 skb->csum = csum_block_sub(skb->csum, 2066 skb_checksum(skb, len, delta, 0), 2067 len); 2068 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 2069 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; 2070 int offset = skb_checksum_start_offset(skb) + skb->csum_offset; 2071 2072 if (offset + sizeof(__sum16) > hdlen) 2073 return -EINVAL; 2074 } 2075 return __pskb_trim(skb, len); 2076 } 2077 EXPORT_SYMBOL(pskb_trim_rcsum_slow); 2078 2079 /** 2080 * __pskb_pull_tail - advance tail of skb header 2081 * @skb: buffer to reallocate 2082 * @delta: number of bytes to advance tail 2083 * 2084 * The function makes a sense only on a fragmented &sk_buff, 2085 * it expands header moving its tail forward and copying necessary 2086 * data from fragmented part. 2087 * 2088 * &sk_buff MUST have reference count of 1. 2089 * 2090 * Returns %NULL (and &sk_buff does not change) if pull failed 2091 * or value of new tail of skb in the case of success. 2092 * 2093 * All the pointers pointing into skb header may change and must be 2094 * reloaded after call to this function. 2095 */ 2096 2097 /* Moves tail of skb head forward, copying data from fragmented part, 2098 * when it is necessary. 2099 * 1. It may fail due to malloc failure. 2100 * 2. It may change skb pointers. 2101 * 2102 * It is pretty complicated. Luckily, it is called only in exceptional cases. 2103 */ 2104 void *__pskb_pull_tail(struct sk_buff *skb, int delta) 2105 { 2106 /* If skb has not enough free space at tail, get new one 2107 * plus 128 bytes for future expansions. If we have enough 2108 * room at tail, reallocate without expansion only if skb is cloned. 2109 */ 2110 int i, k, eat = (skb->tail + delta) - skb->end; 2111 2112 if (eat > 0 || skb_cloned(skb)) { 2113 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 2114 GFP_ATOMIC)) 2115 return NULL; 2116 } 2117 2118 BUG_ON(skb_copy_bits(skb, skb_headlen(skb), 2119 skb_tail_pointer(skb), delta)); 2120 2121 /* Optimization: no fragments, no reasons to preestimate 2122 * size of pulled pages. Superb. 2123 */ 2124 if (!skb_has_frag_list(skb)) 2125 goto pull_pages; 2126 2127 /* Estimate size of pulled pages. */ 2128 eat = delta; 2129 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2130 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2131 2132 if (size >= eat) 2133 goto pull_pages; 2134 eat -= size; 2135 } 2136 2137 /* If we need update frag list, we are in troubles. 2138 * Certainly, it is possible to add an offset to skb data, 2139 * but taking into account that pulling is expected to 2140 * be very rare operation, it is worth to fight against 2141 * further bloating skb head and crucify ourselves here instead. 2142 * Pure masohism, indeed. 8)8) 2143 */ 2144 if (eat) { 2145 struct sk_buff *list = skb_shinfo(skb)->frag_list; 2146 struct sk_buff *clone = NULL; 2147 struct sk_buff *insp = NULL; 2148 2149 do { 2150 if (list->len <= eat) { 2151 /* Eaten as whole. */ 2152 eat -= list->len; 2153 list = list->next; 2154 insp = list; 2155 } else { 2156 /* Eaten partially. */ 2157 2158 if (skb_shared(list)) { 2159 /* Sucks! We need to fork list. :-( */ 2160 clone = skb_clone(list, GFP_ATOMIC); 2161 if (!clone) 2162 return NULL; 2163 insp = list->next; 2164 list = clone; 2165 } else { 2166 /* This may be pulled without 2167 * problems. */ 2168 insp = list; 2169 } 2170 if (!pskb_pull(list, eat)) { 2171 kfree_skb(clone); 2172 return NULL; 2173 } 2174 break; 2175 } 2176 } while (eat); 2177 2178 /* Free pulled out fragments. */ 2179 while ((list = skb_shinfo(skb)->frag_list) != insp) { 2180 skb_shinfo(skb)->frag_list = list->next; 2181 kfree_skb(list); 2182 } 2183 /* And insert new clone at head. */ 2184 if (clone) { 2185 clone->next = list; 2186 skb_shinfo(skb)->frag_list = clone; 2187 } 2188 } 2189 /* Success! Now we may commit changes to skb data. */ 2190 2191 pull_pages: 2192 eat = delta; 2193 k = 0; 2194 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2195 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2196 2197 if (size <= eat) { 2198 skb_frag_unref(skb, i); 2199 eat -= size; 2200 } else { 2201 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 2202 2203 *frag = skb_shinfo(skb)->frags[i]; 2204 if (eat) { 2205 skb_frag_off_add(frag, eat); 2206 skb_frag_size_sub(frag, eat); 2207 if (!i) 2208 goto end; 2209 eat = 0; 2210 } 2211 k++; 2212 } 2213 } 2214 skb_shinfo(skb)->nr_frags = k; 2215 2216 end: 2217 skb->tail += delta; 2218 skb->data_len -= delta; 2219 2220 if (!skb->data_len) 2221 skb_zcopy_clear(skb, false); 2222 2223 return skb_tail_pointer(skb); 2224 } 2225 EXPORT_SYMBOL(__pskb_pull_tail); 2226 2227 /** 2228 * skb_copy_bits - copy bits from skb to kernel buffer 2229 * @skb: source skb 2230 * @offset: offset in source 2231 * @to: destination buffer 2232 * @len: number of bytes to copy 2233 * 2234 * Copy the specified number of bytes from the source skb to the 2235 * destination buffer. 2236 * 2237 * CAUTION ! : 2238 * If its prototype is ever changed, 2239 * check arch/{*}/net/{*}.S files, 2240 * since it is called from BPF assembly code. 2241 */ 2242 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 2243 { 2244 int start = skb_headlen(skb); 2245 struct sk_buff *frag_iter; 2246 int i, copy; 2247 2248 if (offset > (int)skb->len - len) 2249 goto fault; 2250 2251 /* Copy header. */ 2252 if ((copy = start - offset) > 0) { 2253 if (copy > len) 2254 copy = len; 2255 skb_copy_from_linear_data_offset(skb, offset, to, copy); 2256 if ((len -= copy) == 0) 2257 return 0; 2258 offset += copy; 2259 to += copy; 2260 } 2261 2262 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2263 int end; 2264 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 2265 2266 WARN_ON(start > offset + len); 2267 2268 end = start + skb_frag_size(f); 2269 if ((copy = end - offset) > 0) { 2270 u32 p_off, p_len, copied; 2271 struct page *p; 2272 u8 *vaddr; 2273 2274 if (copy > len) 2275 copy = len; 2276 2277 skb_frag_foreach_page(f, 2278 skb_frag_off(f) + offset - start, 2279 copy, p, p_off, p_len, copied) { 2280 vaddr = kmap_atomic(p); 2281 memcpy(to + copied, vaddr + p_off, p_len); 2282 kunmap_atomic(vaddr); 2283 } 2284 2285 if ((len -= copy) == 0) 2286 return 0; 2287 offset += copy; 2288 to += copy; 2289 } 2290 start = end; 2291 } 2292 2293 skb_walk_frags(skb, frag_iter) { 2294 int end; 2295 2296 WARN_ON(start > offset + len); 2297 2298 end = start + frag_iter->len; 2299 if ((copy = end - offset) > 0) { 2300 if (copy > len) 2301 copy = len; 2302 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 2303 goto fault; 2304 if ((len -= copy) == 0) 2305 return 0; 2306 offset += copy; 2307 to += copy; 2308 } 2309 start = end; 2310 } 2311 2312 if (!len) 2313 return 0; 2314 2315 fault: 2316 return -EFAULT; 2317 } 2318 EXPORT_SYMBOL(skb_copy_bits); 2319 2320 /* 2321 * Callback from splice_to_pipe(), if we need to release some pages 2322 * at the end of the spd in case we error'ed out in filling the pipe. 2323 */ 2324 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 2325 { 2326 put_page(spd->pages[i]); 2327 } 2328 2329 static struct page *linear_to_page(struct page *page, unsigned int *len, 2330 unsigned int *offset, 2331 struct sock *sk) 2332 { 2333 struct page_frag *pfrag = sk_page_frag(sk); 2334 2335 if (!sk_page_frag_refill(sk, pfrag)) 2336 return NULL; 2337 2338 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 2339 2340 memcpy(page_address(pfrag->page) + pfrag->offset, 2341 page_address(page) + *offset, *len); 2342 *offset = pfrag->offset; 2343 pfrag->offset += *len; 2344 2345 return pfrag->page; 2346 } 2347 2348 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 2349 struct page *page, 2350 unsigned int offset) 2351 { 2352 return spd->nr_pages && 2353 spd->pages[spd->nr_pages - 1] == page && 2354 (spd->partial[spd->nr_pages - 1].offset + 2355 spd->partial[spd->nr_pages - 1].len == offset); 2356 } 2357 2358 /* 2359 * Fill page/offset/length into spd, if it can hold more pages. 2360 */ 2361 static bool spd_fill_page(struct splice_pipe_desc *spd, 2362 struct pipe_inode_info *pipe, struct page *page, 2363 unsigned int *len, unsigned int offset, 2364 bool linear, 2365 struct sock *sk) 2366 { 2367 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 2368 return true; 2369 2370 if (linear) { 2371 page = linear_to_page(page, len, &offset, sk); 2372 if (!page) 2373 return true; 2374 } 2375 if (spd_can_coalesce(spd, page, offset)) { 2376 spd->partial[spd->nr_pages - 1].len += *len; 2377 return false; 2378 } 2379 get_page(page); 2380 spd->pages[spd->nr_pages] = page; 2381 spd->partial[spd->nr_pages].len = *len; 2382 spd->partial[spd->nr_pages].offset = offset; 2383 spd->nr_pages++; 2384 2385 return false; 2386 } 2387 2388 static bool __splice_segment(struct page *page, unsigned int poff, 2389 unsigned int plen, unsigned int *off, 2390 unsigned int *len, 2391 struct splice_pipe_desc *spd, bool linear, 2392 struct sock *sk, 2393 struct pipe_inode_info *pipe) 2394 { 2395 if (!*len) 2396 return true; 2397 2398 /* skip this segment if already processed */ 2399 if (*off >= plen) { 2400 *off -= plen; 2401 return false; 2402 } 2403 2404 /* ignore any bits we already processed */ 2405 poff += *off; 2406 plen -= *off; 2407 *off = 0; 2408 2409 do { 2410 unsigned int flen = min(*len, plen); 2411 2412 if (spd_fill_page(spd, pipe, page, &flen, poff, 2413 linear, sk)) 2414 return true; 2415 poff += flen; 2416 plen -= flen; 2417 *len -= flen; 2418 } while (*len && plen); 2419 2420 return false; 2421 } 2422 2423 /* 2424 * Map linear and fragment data from the skb to spd. It reports true if the 2425 * pipe is full or if we already spliced the requested length. 2426 */ 2427 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 2428 unsigned int *offset, unsigned int *len, 2429 struct splice_pipe_desc *spd, struct sock *sk) 2430 { 2431 int seg; 2432 struct sk_buff *iter; 2433 2434 /* map the linear part : 2435 * If skb->head_frag is set, this 'linear' part is backed by a 2436 * fragment, and if the head is not shared with any clones then 2437 * we can avoid a copy since we own the head portion of this page. 2438 */ 2439 if (__splice_segment(virt_to_page(skb->data), 2440 (unsigned long) skb->data & (PAGE_SIZE - 1), 2441 skb_headlen(skb), 2442 offset, len, spd, 2443 skb_head_is_locked(skb), 2444 sk, pipe)) 2445 return true; 2446 2447 /* 2448 * then map the fragments 2449 */ 2450 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 2451 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 2452 2453 if (__splice_segment(skb_frag_page(f), 2454 skb_frag_off(f), skb_frag_size(f), 2455 offset, len, spd, false, sk, pipe)) 2456 return true; 2457 } 2458 2459 skb_walk_frags(skb, iter) { 2460 if (*offset >= iter->len) { 2461 *offset -= iter->len; 2462 continue; 2463 } 2464 /* __skb_splice_bits() only fails if the output has no room 2465 * left, so no point in going over the frag_list for the error 2466 * case. 2467 */ 2468 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) 2469 return true; 2470 } 2471 2472 return false; 2473 } 2474 2475 /* 2476 * Map data from the skb to a pipe. Should handle both the linear part, 2477 * the fragments, and the frag list. 2478 */ 2479 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, 2480 struct pipe_inode_info *pipe, unsigned int tlen, 2481 unsigned int flags) 2482 { 2483 struct partial_page partial[MAX_SKB_FRAGS]; 2484 struct page *pages[MAX_SKB_FRAGS]; 2485 struct splice_pipe_desc spd = { 2486 .pages = pages, 2487 .partial = partial, 2488 .nr_pages_max = MAX_SKB_FRAGS, 2489 .ops = &nosteal_pipe_buf_ops, 2490 .spd_release = sock_spd_release, 2491 }; 2492 int ret = 0; 2493 2494 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); 2495 2496 if (spd.nr_pages) 2497 ret = splice_to_pipe(pipe, &spd); 2498 2499 return ret; 2500 } 2501 EXPORT_SYMBOL_GPL(skb_splice_bits); 2502 2503 /* Send skb data on a socket. Socket must be locked. */ 2504 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, 2505 int len) 2506 { 2507 unsigned int orig_len = len; 2508 struct sk_buff *head = skb; 2509 unsigned short fragidx; 2510 int slen, ret; 2511 2512 do_frag_list: 2513 2514 /* Deal with head data */ 2515 while (offset < skb_headlen(skb) && len) { 2516 struct kvec kv; 2517 struct msghdr msg; 2518 2519 slen = min_t(int, len, skb_headlen(skb) - offset); 2520 kv.iov_base = skb->data + offset; 2521 kv.iov_len = slen; 2522 memset(&msg, 0, sizeof(msg)); 2523 msg.msg_flags = MSG_DONTWAIT; 2524 2525 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen); 2526 if (ret <= 0) 2527 goto error; 2528 2529 offset += ret; 2530 len -= ret; 2531 } 2532 2533 /* All the data was skb head? */ 2534 if (!len) 2535 goto out; 2536 2537 /* Make offset relative to start of frags */ 2538 offset -= skb_headlen(skb); 2539 2540 /* Find where we are in frag list */ 2541 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2542 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2543 2544 if (offset < skb_frag_size(frag)) 2545 break; 2546 2547 offset -= skb_frag_size(frag); 2548 } 2549 2550 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { 2551 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; 2552 2553 slen = min_t(size_t, len, skb_frag_size(frag) - offset); 2554 2555 while (slen) { 2556 ret = kernel_sendpage_locked(sk, skb_frag_page(frag), 2557 skb_frag_off(frag) + offset, 2558 slen, MSG_DONTWAIT); 2559 if (ret <= 0) 2560 goto error; 2561 2562 len -= ret; 2563 offset += ret; 2564 slen -= ret; 2565 } 2566 2567 offset = 0; 2568 } 2569 2570 if (len) { 2571 /* Process any frag lists */ 2572 2573 if (skb == head) { 2574 if (skb_has_frag_list(skb)) { 2575 skb = skb_shinfo(skb)->frag_list; 2576 goto do_frag_list; 2577 } 2578 } else if (skb->next) { 2579 skb = skb->next; 2580 goto do_frag_list; 2581 } 2582 } 2583 2584 out: 2585 return orig_len - len; 2586 2587 error: 2588 return orig_len == len ? ret : orig_len - len; 2589 } 2590 EXPORT_SYMBOL_GPL(skb_send_sock_locked); 2591 2592 /** 2593 * skb_store_bits - store bits from kernel buffer to skb 2594 * @skb: destination buffer 2595 * @offset: offset in destination 2596 * @from: source buffer 2597 * @len: number of bytes to copy 2598 * 2599 * Copy the specified number of bytes from the source buffer to the 2600 * destination skb. This function handles all the messy bits of 2601 * traversing fragment lists and such. 2602 */ 2603 2604 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 2605 { 2606 int start = skb_headlen(skb); 2607 struct sk_buff *frag_iter; 2608 int i, copy; 2609 2610 if (offset > (int)skb->len - len) 2611 goto fault; 2612 2613 if ((copy = start - offset) > 0) { 2614 if (copy > len) 2615 copy = len; 2616 skb_copy_to_linear_data_offset(skb, offset, from, copy); 2617 if ((len -= copy) == 0) 2618 return 0; 2619 offset += copy; 2620 from += copy; 2621 } 2622 2623 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2624 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2625 int end; 2626 2627 WARN_ON(start > offset + len); 2628 2629 end = start + skb_frag_size(frag); 2630 if ((copy = end - offset) > 0) { 2631 u32 p_off, p_len, copied; 2632 struct page *p; 2633 u8 *vaddr; 2634 2635 if (copy > len) 2636 copy = len; 2637 2638 skb_frag_foreach_page(frag, 2639 skb_frag_off(frag) + offset - start, 2640 copy, p, p_off, p_len, copied) { 2641 vaddr = kmap_atomic(p); 2642 memcpy(vaddr + p_off, from + copied, p_len); 2643 kunmap_atomic(vaddr); 2644 } 2645 2646 if ((len -= copy) == 0) 2647 return 0; 2648 offset += copy; 2649 from += copy; 2650 } 2651 start = end; 2652 } 2653 2654 skb_walk_frags(skb, frag_iter) { 2655 int end; 2656 2657 WARN_ON(start > offset + len); 2658 2659 end = start + frag_iter->len; 2660 if ((copy = end - offset) > 0) { 2661 if (copy > len) 2662 copy = len; 2663 if (skb_store_bits(frag_iter, offset - start, 2664 from, copy)) 2665 goto fault; 2666 if ((len -= copy) == 0) 2667 return 0; 2668 offset += copy; 2669 from += copy; 2670 } 2671 start = end; 2672 } 2673 if (!len) 2674 return 0; 2675 2676 fault: 2677 return -EFAULT; 2678 } 2679 EXPORT_SYMBOL(skb_store_bits); 2680 2681 /* Checksum skb data. */ 2682 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 2683 __wsum csum, const struct skb_checksum_ops *ops) 2684 { 2685 int start = skb_headlen(skb); 2686 int i, copy = start - offset; 2687 struct sk_buff *frag_iter; 2688 int pos = 0; 2689 2690 /* Checksum header. */ 2691 if (copy > 0) { 2692 if (copy > len) 2693 copy = len; 2694 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, 2695 skb->data + offset, copy, csum); 2696 if ((len -= copy) == 0) 2697 return csum; 2698 offset += copy; 2699 pos = copy; 2700 } 2701 2702 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2703 int end; 2704 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2705 2706 WARN_ON(start > offset + len); 2707 2708 end = start + skb_frag_size(frag); 2709 if ((copy = end - offset) > 0) { 2710 u32 p_off, p_len, copied; 2711 struct page *p; 2712 __wsum csum2; 2713 u8 *vaddr; 2714 2715 if (copy > len) 2716 copy = len; 2717 2718 skb_frag_foreach_page(frag, 2719 skb_frag_off(frag) + offset - start, 2720 copy, p, p_off, p_len, copied) { 2721 vaddr = kmap_atomic(p); 2722 csum2 = INDIRECT_CALL_1(ops->update, 2723 csum_partial_ext, 2724 vaddr + p_off, p_len, 0); 2725 kunmap_atomic(vaddr); 2726 csum = INDIRECT_CALL_1(ops->combine, 2727 csum_block_add_ext, csum, 2728 csum2, pos, p_len); 2729 pos += p_len; 2730 } 2731 2732 if (!(len -= copy)) 2733 return csum; 2734 offset += copy; 2735 } 2736 start = end; 2737 } 2738 2739 skb_walk_frags(skb, frag_iter) { 2740 int end; 2741 2742 WARN_ON(start > offset + len); 2743 2744 end = start + frag_iter->len; 2745 if ((copy = end - offset) > 0) { 2746 __wsum csum2; 2747 if (copy > len) 2748 copy = len; 2749 csum2 = __skb_checksum(frag_iter, offset - start, 2750 copy, 0, ops); 2751 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, 2752 csum, csum2, pos, copy); 2753 if ((len -= copy) == 0) 2754 return csum; 2755 offset += copy; 2756 pos += copy; 2757 } 2758 start = end; 2759 } 2760 BUG_ON(len); 2761 2762 return csum; 2763 } 2764 EXPORT_SYMBOL(__skb_checksum); 2765 2766 __wsum skb_checksum(const struct sk_buff *skb, int offset, 2767 int len, __wsum csum) 2768 { 2769 const struct skb_checksum_ops ops = { 2770 .update = csum_partial_ext, 2771 .combine = csum_block_add_ext, 2772 }; 2773 2774 return __skb_checksum(skb, offset, len, csum, &ops); 2775 } 2776 EXPORT_SYMBOL(skb_checksum); 2777 2778 /* Both of above in one bottle. */ 2779 2780 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2781 u8 *to, int len) 2782 { 2783 int start = skb_headlen(skb); 2784 int i, copy = start - offset; 2785 struct sk_buff *frag_iter; 2786 int pos = 0; 2787 __wsum csum = 0; 2788 2789 /* Copy header. */ 2790 if (copy > 0) { 2791 if (copy > len) 2792 copy = len; 2793 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2794 copy); 2795 if ((len -= copy) == 0) 2796 return csum; 2797 offset += copy; 2798 to += copy; 2799 pos = copy; 2800 } 2801 2802 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2803 int end; 2804 2805 WARN_ON(start > offset + len); 2806 2807 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2808 if ((copy = end - offset) > 0) { 2809 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2810 u32 p_off, p_len, copied; 2811 struct page *p; 2812 __wsum csum2; 2813 u8 *vaddr; 2814 2815 if (copy > len) 2816 copy = len; 2817 2818 skb_frag_foreach_page(frag, 2819 skb_frag_off(frag) + offset - start, 2820 copy, p, p_off, p_len, copied) { 2821 vaddr = kmap_atomic(p); 2822 csum2 = csum_partial_copy_nocheck(vaddr + p_off, 2823 to + copied, 2824 p_len); 2825 kunmap_atomic(vaddr); 2826 csum = csum_block_add(csum, csum2, pos); 2827 pos += p_len; 2828 } 2829 2830 if (!(len -= copy)) 2831 return csum; 2832 offset += copy; 2833 to += copy; 2834 } 2835 start = end; 2836 } 2837 2838 skb_walk_frags(skb, frag_iter) { 2839 __wsum csum2; 2840 int end; 2841 2842 WARN_ON(start > offset + len); 2843 2844 end = start + frag_iter->len; 2845 if ((copy = end - offset) > 0) { 2846 if (copy > len) 2847 copy = len; 2848 csum2 = skb_copy_and_csum_bits(frag_iter, 2849 offset - start, 2850 to, copy); 2851 csum = csum_block_add(csum, csum2, pos); 2852 if ((len -= copy) == 0) 2853 return csum; 2854 offset += copy; 2855 to += copy; 2856 pos += copy; 2857 } 2858 start = end; 2859 } 2860 BUG_ON(len); 2861 return csum; 2862 } 2863 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2864 2865 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 2866 { 2867 __sum16 sum; 2868 2869 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 2870 /* See comments in __skb_checksum_complete(). */ 2871 if (likely(!sum)) { 2872 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 2873 !skb->csum_complete_sw) 2874 netdev_rx_csum_fault(skb->dev, skb); 2875 } 2876 if (!skb_shared(skb)) 2877 skb->csum_valid = !sum; 2878 return sum; 2879 } 2880 EXPORT_SYMBOL(__skb_checksum_complete_head); 2881 2882 /* This function assumes skb->csum already holds pseudo header's checksum, 2883 * which has been changed from the hardware checksum, for example, by 2884 * __skb_checksum_validate_complete(). And, the original skb->csum must 2885 * have been validated unsuccessfully for CHECKSUM_COMPLETE case. 2886 * 2887 * It returns non-zero if the recomputed checksum is still invalid, otherwise 2888 * zero. The new checksum is stored back into skb->csum unless the skb is 2889 * shared. 2890 */ 2891 __sum16 __skb_checksum_complete(struct sk_buff *skb) 2892 { 2893 __wsum csum; 2894 __sum16 sum; 2895 2896 csum = skb_checksum(skb, 0, skb->len, 0); 2897 2898 sum = csum_fold(csum_add(skb->csum, csum)); 2899 /* This check is inverted, because we already knew the hardware 2900 * checksum is invalid before calling this function. So, if the 2901 * re-computed checksum is valid instead, then we have a mismatch 2902 * between the original skb->csum and skb_checksum(). This means either 2903 * the original hardware checksum is incorrect or we screw up skb->csum 2904 * when moving skb->data around. 2905 */ 2906 if (likely(!sum)) { 2907 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 2908 !skb->csum_complete_sw) 2909 netdev_rx_csum_fault(skb->dev, skb); 2910 } 2911 2912 if (!skb_shared(skb)) { 2913 /* Save full packet checksum */ 2914 skb->csum = csum; 2915 skb->ip_summed = CHECKSUM_COMPLETE; 2916 skb->csum_complete_sw = 1; 2917 skb->csum_valid = !sum; 2918 } 2919 2920 return sum; 2921 } 2922 EXPORT_SYMBOL(__skb_checksum_complete); 2923 2924 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) 2925 { 2926 net_warn_ratelimited( 2927 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2928 __func__); 2929 return 0; 2930 } 2931 2932 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, 2933 int offset, int len) 2934 { 2935 net_warn_ratelimited( 2936 "%s: attempt to compute crc32c without libcrc32c.ko\n", 2937 __func__); 2938 return 0; 2939 } 2940 2941 static const struct skb_checksum_ops default_crc32c_ops = { 2942 .update = warn_crc32c_csum_update, 2943 .combine = warn_crc32c_csum_combine, 2944 }; 2945 2946 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = 2947 &default_crc32c_ops; 2948 EXPORT_SYMBOL(crc32c_csum_stub); 2949 2950 /** 2951 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 2952 * @from: source buffer 2953 * 2954 * Calculates the amount of linear headroom needed in the 'to' skb passed 2955 * into skb_zerocopy(). 2956 */ 2957 unsigned int 2958 skb_zerocopy_headlen(const struct sk_buff *from) 2959 { 2960 unsigned int hlen = 0; 2961 2962 if (!from->head_frag || 2963 skb_headlen(from) < L1_CACHE_BYTES || 2964 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 2965 hlen = skb_headlen(from); 2966 2967 if (skb_has_frag_list(from)) 2968 hlen = from->len; 2969 2970 return hlen; 2971 } 2972 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 2973 2974 /** 2975 * skb_zerocopy - Zero copy skb to skb 2976 * @to: destination buffer 2977 * @from: source buffer 2978 * @len: number of bytes to copy from source buffer 2979 * @hlen: size of linear headroom in destination buffer 2980 * 2981 * Copies up to `len` bytes from `from` to `to` by creating references 2982 * to the frags in the source buffer. 2983 * 2984 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 2985 * headroom in the `to` buffer. 2986 * 2987 * Return value: 2988 * 0: everything is OK 2989 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 2990 * -EFAULT: skb_copy_bits() found some problem with skb geometry 2991 */ 2992 int 2993 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 2994 { 2995 int i, j = 0; 2996 int plen = 0; /* length of skb->head fragment */ 2997 int ret; 2998 struct page *page; 2999 unsigned int offset; 3000 3001 BUG_ON(!from->head_frag && !hlen); 3002 3003 /* dont bother with small payloads */ 3004 if (len <= skb_tailroom(to)) 3005 return skb_copy_bits(from, 0, skb_put(to, len), len); 3006 3007 if (hlen) { 3008 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 3009 if (unlikely(ret)) 3010 return ret; 3011 len -= hlen; 3012 } else { 3013 plen = min_t(int, skb_headlen(from), len); 3014 if (plen) { 3015 page = virt_to_head_page(from->head); 3016 offset = from->data - (unsigned char *)page_address(page); 3017 __skb_fill_page_desc(to, 0, page, offset, plen); 3018 get_page(page); 3019 j = 1; 3020 len -= plen; 3021 } 3022 } 3023 3024 to->truesize += len + plen; 3025 to->len += len + plen; 3026 to->data_len += len + plen; 3027 3028 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 3029 skb_tx_error(from); 3030 return -ENOMEM; 3031 } 3032 skb_zerocopy_clone(to, from, GFP_ATOMIC); 3033 3034 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 3035 int size; 3036 3037 if (!len) 3038 break; 3039 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 3040 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]), 3041 len); 3042 skb_frag_size_set(&skb_shinfo(to)->frags[j], size); 3043 len -= size; 3044 skb_frag_ref(to, j); 3045 j++; 3046 } 3047 skb_shinfo(to)->nr_frags = j; 3048 3049 return 0; 3050 } 3051 EXPORT_SYMBOL_GPL(skb_zerocopy); 3052 3053 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 3054 { 3055 __wsum csum; 3056 long csstart; 3057 3058 if (skb->ip_summed == CHECKSUM_PARTIAL) 3059 csstart = skb_checksum_start_offset(skb); 3060 else 3061 csstart = skb_headlen(skb); 3062 3063 BUG_ON(csstart > skb_headlen(skb)); 3064 3065 skb_copy_from_linear_data(skb, to, csstart); 3066 3067 csum = 0; 3068 if (csstart != skb->len) 3069 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 3070 skb->len - csstart); 3071 3072 if (skb->ip_summed == CHECKSUM_PARTIAL) { 3073 long csstuff = csstart + skb->csum_offset; 3074 3075 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 3076 } 3077 } 3078 EXPORT_SYMBOL(skb_copy_and_csum_dev); 3079 3080 /** 3081 * skb_dequeue - remove from the head of the queue 3082 * @list: list to dequeue from 3083 * 3084 * Remove the head of the list. The list lock is taken so the function 3085 * may be used safely with other locking list functions. The head item is 3086 * returned or %NULL if the list is empty. 3087 */ 3088 3089 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 3090 { 3091 unsigned long flags; 3092 struct sk_buff *result; 3093 3094 spin_lock_irqsave(&list->lock, flags); 3095 result = __skb_dequeue(list); 3096 spin_unlock_irqrestore(&list->lock, flags); 3097 return result; 3098 } 3099 EXPORT_SYMBOL(skb_dequeue); 3100 3101 /** 3102 * skb_dequeue_tail - remove from the tail of the queue 3103 * @list: list to dequeue from 3104 * 3105 * Remove the tail of the list. The list lock is taken so the function 3106 * may be used safely with other locking list functions. The tail item is 3107 * returned or %NULL if the list is empty. 3108 */ 3109 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 3110 { 3111 unsigned long flags; 3112 struct sk_buff *result; 3113 3114 spin_lock_irqsave(&list->lock, flags); 3115 result = __skb_dequeue_tail(list); 3116 spin_unlock_irqrestore(&list->lock, flags); 3117 return result; 3118 } 3119 EXPORT_SYMBOL(skb_dequeue_tail); 3120 3121 /** 3122 * skb_queue_purge - empty a list 3123 * @list: list to empty 3124 * 3125 * Delete all buffers on an &sk_buff list. Each buffer is removed from 3126 * the list and one reference dropped. This function takes the list 3127 * lock and is atomic with respect to other list locking functions. 3128 */ 3129 void skb_queue_purge(struct sk_buff_head *list) 3130 { 3131 struct sk_buff *skb; 3132 while ((skb = skb_dequeue(list)) != NULL) 3133 kfree_skb(skb); 3134 } 3135 EXPORT_SYMBOL(skb_queue_purge); 3136 3137 /** 3138 * skb_rbtree_purge - empty a skb rbtree 3139 * @root: root of the rbtree to empty 3140 * Return value: the sum of truesizes of all purged skbs. 3141 * 3142 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from 3143 * the list and one reference dropped. This function does not take 3144 * any lock. Synchronization should be handled by the caller (e.g., TCP 3145 * out-of-order queue is protected by the socket lock). 3146 */ 3147 unsigned int skb_rbtree_purge(struct rb_root *root) 3148 { 3149 struct rb_node *p = rb_first(root); 3150 unsigned int sum = 0; 3151 3152 while (p) { 3153 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode); 3154 3155 p = rb_next(p); 3156 rb_erase(&skb->rbnode, root); 3157 sum += skb->truesize; 3158 kfree_skb(skb); 3159 } 3160 return sum; 3161 } 3162 3163 /** 3164 * skb_queue_head - queue a buffer at the list head 3165 * @list: list to use 3166 * @newsk: buffer to queue 3167 * 3168 * Queue a buffer at the start of the list. This function takes the 3169 * list lock and can be used safely with other locking &sk_buff functions 3170 * safely. 3171 * 3172 * A buffer cannot be placed on two lists at the same time. 3173 */ 3174 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 3175 { 3176 unsigned long flags; 3177 3178 spin_lock_irqsave(&list->lock, flags); 3179 __skb_queue_head(list, newsk); 3180 spin_unlock_irqrestore(&list->lock, flags); 3181 } 3182 EXPORT_SYMBOL(skb_queue_head); 3183 3184 /** 3185 * skb_queue_tail - queue a buffer at the list tail 3186 * @list: list to use 3187 * @newsk: buffer to queue 3188 * 3189 * Queue a buffer at the tail of the list. This function takes the 3190 * list lock and can be used safely with other locking &sk_buff functions 3191 * safely. 3192 * 3193 * A buffer cannot be placed on two lists at the same time. 3194 */ 3195 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 3196 { 3197 unsigned long flags; 3198 3199 spin_lock_irqsave(&list->lock, flags); 3200 __skb_queue_tail(list, newsk); 3201 spin_unlock_irqrestore(&list->lock, flags); 3202 } 3203 EXPORT_SYMBOL(skb_queue_tail); 3204 3205 /** 3206 * skb_unlink - remove a buffer from a list 3207 * @skb: buffer to remove 3208 * @list: list to use 3209 * 3210 * Remove a packet from a list. The list locks are taken and this 3211 * function is atomic with respect to other list locked calls 3212 * 3213 * You must know what list the SKB is on. 3214 */ 3215 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 3216 { 3217 unsigned long flags; 3218 3219 spin_lock_irqsave(&list->lock, flags); 3220 __skb_unlink(skb, list); 3221 spin_unlock_irqrestore(&list->lock, flags); 3222 } 3223 EXPORT_SYMBOL(skb_unlink); 3224 3225 /** 3226 * skb_append - append a buffer 3227 * @old: buffer to insert after 3228 * @newsk: buffer to insert 3229 * @list: list to use 3230 * 3231 * Place a packet after a given packet in a list. The list locks are taken 3232 * and this function is atomic with respect to other list locked calls. 3233 * A buffer cannot be placed on two lists at the same time. 3234 */ 3235 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 3236 { 3237 unsigned long flags; 3238 3239 spin_lock_irqsave(&list->lock, flags); 3240 __skb_queue_after(list, old, newsk); 3241 spin_unlock_irqrestore(&list->lock, flags); 3242 } 3243 EXPORT_SYMBOL(skb_append); 3244 3245 static inline void skb_split_inside_header(struct sk_buff *skb, 3246 struct sk_buff* skb1, 3247 const u32 len, const int pos) 3248 { 3249 int i; 3250 3251 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 3252 pos - len); 3253 /* And move data appendix as is. */ 3254 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 3255 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 3256 3257 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 3258 skb_shinfo(skb)->nr_frags = 0; 3259 skb1->data_len = skb->data_len; 3260 skb1->len += skb1->data_len; 3261 skb->data_len = 0; 3262 skb->len = len; 3263 skb_set_tail_pointer(skb, len); 3264 } 3265 3266 static inline void skb_split_no_header(struct sk_buff *skb, 3267 struct sk_buff* skb1, 3268 const u32 len, int pos) 3269 { 3270 int i, k = 0; 3271 const int nfrags = skb_shinfo(skb)->nr_frags; 3272 3273 skb_shinfo(skb)->nr_frags = 0; 3274 skb1->len = skb1->data_len = skb->len - len; 3275 skb->len = len; 3276 skb->data_len = len - pos; 3277 3278 for (i = 0; i < nfrags; i++) { 3279 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 3280 3281 if (pos + size > len) { 3282 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 3283 3284 if (pos < len) { 3285 /* Split frag. 3286 * We have two variants in this case: 3287 * 1. Move all the frag to the second 3288 * part, if it is possible. F.e. 3289 * this approach is mandatory for TUX, 3290 * where splitting is expensive. 3291 * 2. Split is accurately. We make this. 3292 */ 3293 skb_frag_ref(skb, i); 3294 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos); 3295 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 3296 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 3297 skb_shinfo(skb)->nr_frags++; 3298 } 3299 k++; 3300 } else 3301 skb_shinfo(skb)->nr_frags++; 3302 pos += size; 3303 } 3304 skb_shinfo(skb1)->nr_frags = k; 3305 } 3306 3307 /** 3308 * skb_split - Split fragmented skb to two parts at length len. 3309 * @skb: the buffer to split 3310 * @skb1: the buffer to receive the second part 3311 * @len: new length for skb 3312 */ 3313 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 3314 { 3315 int pos = skb_headlen(skb); 3316 3317 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; 3318 skb_zerocopy_clone(skb1, skb, 0); 3319 if (len < pos) /* Split line is inside header. */ 3320 skb_split_inside_header(skb, skb1, len, pos); 3321 else /* Second chunk has no header, nothing to copy. */ 3322 skb_split_no_header(skb, skb1, len, pos); 3323 } 3324 EXPORT_SYMBOL(skb_split); 3325 3326 /* Shifting from/to a cloned skb is a no-go. 3327 * 3328 * Caller cannot keep skb_shinfo related pointers past calling here! 3329 */ 3330 static int skb_prepare_for_shift(struct sk_buff *skb) 3331 { 3332 int ret = 0; 3333 3334 if (skb_cloned(skb)) { 3335 /* Save and restore truesize: pskb_expand_head() may reallocate 3336 * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we 3337 * cannot change truesize at this point. 3338 */ 3339 unsigned int save_truesize = skb->truesize; 3340 3341 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 3342 skb->truesize = save_truesize; 3343 } 3344 return ret; 3345 } 3346 3347 /** 3348 * skb_shift - Shifts paged data partially from skb to another 3349 * @tgt: buffer into which tail data gets added 3350 * @skb: buffer from which the paged data comes from 3351 * @shiftlen: shift up to this many bytes 3352 * 3353 * Attempts to shift up to shiftlen worth of bytes, which may be less than 3354 * the length of the skb, from skb to tgt. Returns number bytes shifted. 3355 * It's up to caller to free skb if everything was shifted. 3356 * 3357 * If @tgt runs out of frags, the whole operation is aborted. 3358 * 3359 * Skb cannot include anything else but paged data while tgt is allowed 3360 * to have non-paged data as well. 3361 * 3362 * TODO: full sized shift could be optimized but that would need 3363 * specialized skb free'er to handle frags without up-to-date nr_frags. 3364 */ 3365 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 3366 { 3367 int from, to, merge, todo; 3368 skb_frag_t *fragfrom, *fragto; 3369 3370 BUG_ON(shiftlen > skb->len); 3371 3372 if (skb_headlen(skb)) 3373 return 0; 3374 if (skb_zcopy(tgt) || skb_zcopy(skb)) 3375 return 0; 3376 3377 todo = shiftlen; 3378 from = 0; 3379 to = skb_shinfo(tgt)->nr_frags; 3380 fragfrom = &skb_shinfo(skb)->frags[from]; 3381 3382 /* Actual merge is delayed until the point when we know we can 3383 * commit all, so that we don't have to undo partial changes 3384 */ 3385 if (!to || 3386 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 3387 skb_frag_off(fragfrom))) { 3388 merge = -1; 3389 } else { 3390 merge = to - 1; 3391 3392 todo -= skb_frag_size(fragfrom); 3393 if (todo < 0) { 3394 if (skb_prepare_for_shift(skb) || 3395 skb_prepare_for_shift(tgt)) 3396 return 0; 3397 3398 /* All previous frag pointers might be stale! */ 3399 fragfrom = &skb_shinfo(skb)->frags[from]; 3400 fragto = &skb_shinfo(tgt)->frags[merge]; 3401 3402 skb_frag_size_add(fragto, shiftlen); 3403 skb_frag_size_sub(fragfrom, shiftlen); 3404 skb_frag_off_add(fragfrom, shiftlen); 3405 3406 goto onlymerged; 3407 } 3408 3409 from++; 3410 } 3411 3412 /* Skip full, not-fitting skb to avoid expensive operations */ 3413 if ((shiftlen == skb->len) && 3414 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 3415 return 0; 3416 3417 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 3418 return 0; 3419 3420 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 3421 if (to == MAX_SKB_FRAGS) 3422 return 0; 3423 3424 fragfrom = &skb_shinfo(skb)->frags[from]; 3425 fragto = &skb_shinfo(tgt)->frags[to]; 3426 3427 if (todo >= skb_frag_size(fragfrom)) { 3428 *fragto = *fragfrom; 3429 todo -= skb_frag_size(fragfrom); 3430 from++; 3431 to++; 3432 3433 } else { 3434 __skb_frag_ref(fragfrom); 3435 skb_frag_page_copy(fragto, fragfrom); 3436 skb_frag_off_copy(fragto, fragfrom); 3437 skb_frag_size_set(fragto, todo); 3438 3439 skb_frag_off_add(fragfrom, todo); 3440 skb_frag_size_sub(fragfrom, todo); 3441 todo = 0; 3442 3443 to++; 3444 break; 3445 } 3446 } 3447 3448 /* Ready to "commit" this state change to tgt */ 3449 skb_shinfo(tgt)->nr_frags = to; 3450 3451 if (merge >= 0) { 3452 fragfrom = &skb_shinfo(skb)->frags[0]; 3453 fragto = &skb_shinfo(tgt)->frags[merge]; 3454 3455 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 3456 __skb_frag_unref(fragfrom); 3457 } 3458 3459 /* Reposition in the original skb */ 3460 to = 0; 3461 while (from < skb_shinfo(skb)->nr_frags) 3462 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 3463 skb_shinfo(skb)->nr_frags = to; 3464 3465 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 3466 3467 onlymerged: 3468 /* Most likely the tgt won't ever need its checksum anymore, skb on 3469 * the other hand might need it if it needs to be resent 3470 */ 3471 tgt->ip_summed = CHECKSUM_PARTIAL; 3472 skb->ip_summed = CHECKSUM_PARTIAL; 3473 3474 /* Yak, is it really working this way? Some helper please? */ 3475 skb->len -= shiftlen; 3476 skb->data_len -= shiftlen; 3477 skb->truesize -= shiftlen; 3478 tgt->len += shiftlen; 3479 tgt->data_len += shiftlen; 3480 tgt->truesize += shiftlen; 3481 3482 return shiftlen; 3483 } 3484 3485 /** 3486 * skb_prepare_seq_read - Prepare a sequential read of skb data 3487 * @skb: the buffer to read 3488 * @from: lower offset of data to be read 3489 * @to: upper offset of data to be read 3490 * @st: state variable 3491 * 3492 * Initializes the specified state variable. Must be called before 3493 * invoking skb_seq_read() for the first time. 3494 */ 3495 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 3496 unsigned int to, struct skb_seq_state *st) 3497 { 3498 st->lower_offset = from; 3499 st->upper_offset = to; 3500 st->root_skb = st->cur_skb = skb; 3501 st->frag_idx = st->stepped_offset = 0; 3502 st->frag_data = NULL; 3503 st->frag_off = 0; 3504 } 3505 EXPORT_SYMBOL(skb_prepare_seq_read); 3506 3507 /** 3508 * skb_seq_read - Sequentially read skb data 3509 * @consumed: number of bytes consumed by the caller so far 3510 * @data: destination pointer for data to be returned 3511 * @st: state variable 3512 * 3513 * Reads a block of skb data at @consumed relative to the 3514 * lower offset specified to skb_prepare_seq_read(). Assigns 3515 * the head of the data block to @data and returns the length 3516 * of the block or 0 if the end of the skb data or the upper 3517 * offset has been reached. 3518 * 3519 * The caller is not required to consume all of the data 3520 * returned, i.e. @consumed is typically set to the number 3521 * of bytes already consumed and the next call to 3522 * skb_seq_read() will return the remaining part of the block. 3523 * 3524 * Note 1: The size of each block of data returned can be arbitrary, 3525 * this limitation is the cost for zerocopy sequential 3526 * reads of potentially non linear data. 3527 * 3528 * Note 2: Fragment lists within fragments are not implemented 3529 * at the moment, state->root_skb could be replaced with 3530 * a stack for this purpose. 3531 */ 3532 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 3533 struct skb_seq_state *st) 3534 { 3535 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 3536 skb_frag_t *frag; 3537 3538 if (unlikely(abs_offset >= st->upper_offset)) { 3539 if (st->frag_data) { 3540 kunmap_atomic(st->frag_data); 3541 st->frag_data = NULL; 3542 } 3543 return 0; 3544 } 3545 3546 next_skb: 3547 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 3548 3549 if (abs_offset < block_limit && !st->frag_data) { 3550 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 3551 return block_limit - abs_offset; 3552 } 3553 3554 if (st->frag_idx == 0 && !st->frag_data) 3555 st->stepped_offset += skb_headlen(st->cur_skb); 3556 3557 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 3558 unsigned int pg_idx, pg_off, pg_sz; 3559 3560 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 3561 3562 pg_idx = 0; 3563 pg_off = skb_frag_off(frag); 3564 pg_sz = skb_frag_size(frag); 3565 3566 if (skb_frag_must_loop(skb_frag_page(frag))) { 3567 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT; 3568 pg_off = offset_in_page(pg_off + st->frag_off); 3569 pg_sz = min_t(unsigned int, pg_sz - st->frag_off, 3570 PAGE_SIZE - pg_off); 3571 } 3572 3573 block_limit = pg_sz + st->stepped_offset; 3574 if (abs_offset < block_limit) { 3575 if (!st->frag_data) 3576 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx); 3577 3578 *data = (u8 *)st->frag_data + pg_off + 3579 (abs_offset - st->stepped_offset); 3580 3581 return block_limit - abs_offset; 3582 } 3583 3584 if (st->frag_data) { 3585 kunmap_atomic(st->frag_data); 3586 st->frag_data = NULL; 3587 } 3588 3589 st->stepped_offset += pg_sz; 3590 st->frag_off += pg_sz; 3591 if (st->frag_off == skb_frag_size(frag)) { 3592 st->frag_off = 0; 3593 st->frag_idx++; 3594 } 3595 } 3596 3597 if (st->frag_data) { 3598 kunmap_atomic(st->frag_data); 3599 st->frag_data = NULL; 3600 } 3601 3602 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 3603 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 3604 st->frag_idx = 0; 3605 goto next_skb; 3606 } else if (st->cur_skb->next) { 3607 st->cur_skb = st->cur_skb->next; 3608 st->frag_idx = 0; 3609 goto next_skb; 3610 } 3611 3612 return 0; 3613 } 3614 EXPORT_SYMBOL(skb_seq_read); 3615 3616 /** 3617 * skb_abort_seq_read - Abort a sequential read of skb data 3618 * @st: state variable 3619 * 3620 * Must be called if skb_seq_read() was not called until it 3621 * returned 0. 3622 */ 3623 void skb_abort_seq_read(struct skb_seq_state *st) 3624 { 3625 if (st->frag_data) 3626 kunmap_atomic(st->frag_data); 3627 } 3628 EXPORT_SYMBOL(skb_abort_seq_read); 3629 3630 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 3631 3632 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 3633 struct ts_config *conf, 3634 struct ts_state *state) 3635 { 3636 return skb_seq_read(offset, text, TS_SKB_CB(state)); 3637 } 3638 3639 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 3640 { 3641 skb_abort_seq_read(TS_SKB_CB(state)); 3642 } 3643 3644 /** 3645 * skb_find_text - Find a text pattern in skb data 3646 * @skb: the buffer to look in 3647 * @from: search offset 3648 * @to: search limit 3649 * @config: textsearch configuration 3650 * 3651 * Finds a pattern in the skb data according to the specified 3652 * textsearch configuration. Use textsearch_next() to retrieve 3653 * subsequent occurrences of the pattern. Returns the offset 3654 * to the first occurrence or UINT_MAX if no match was found. 3655 */ 3656 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 3657 unsigned int to, struct ts_config *config) 3658 { 3659 struct ts_state state; 3660 unsigned int ret; 3661 3662 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb)); 3663 3664 config->get_next_block = skb_ts_get_next_block; 3665 config->finish = skb_ts_finish; 3666 3667 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state)); 3668 3669 ret = textsearch_find(config, &state); 3670 return (ret <= to - from ? ret : UINT_MAX); 3671 } 3672 EXPORT_SYMBOL(skb_find_text); 3673 3674 int skb_append_pagefrags(struct sk_buff *skb, struct page *page, 3675 int offset, size_t size) 3676 { 3677 int i = skb_shinfo(skb)->nr_frags; 3678 3679 if (skb_can_coalesce(skb, i, page, offset)) { 3680 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); 3681 } else if (i < MAX_SKB_FRAGS) { 3682 get_page(page); 3683 skb_fill_page_desc(skb, i, page, offset, size); 3684 } else { 3685 return -EMSGSIZE; 3686 } 3687 3688 return 0; 3689 } 3690 EXPORT_SYMBOL_GPL(skb_append_pagefrags); 3691 3692 /** 3693 * skb_pull_rcsum - pull skb and update receive checksum 3694 * @skb: buffer to update 3695 * @len: length of data pulled 3696 * 3697 * This function performs an skb_pull on the packet and updates 3698 * the CHECKSUM_COMPLETE checksum. It should be used on 3699 * receive path processing instead of skb_pull unless you know 3700 * that the checksum difference is zero (e.g., a valid IP header) 3701 * or you are setting ip_summed to CHECKSUM_NONE. 3702 */ 3703 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 3704 { 3705 unsigned char *data = skb->data; 3706 3707 BUG_ON(len > skb->len); 3708 __skb_pull(skb, len); 3709 skb_postpull_rcsum(skb, data, len); 3710 return skb->data; 3711 } 3712 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 3713 3714 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb) 3715 { 3716 skb_frag_t head_frag; 3717 struct page *page; 3718 3719 page = virt_to_head_page(frag_skb->head); 3720 __skb_frag_set_page(&head_frag, page); 3721 skb_frag_off_set(&head_frag, frag_skb->data - 3722 (unsigned char *)page_address(page)); 3723 skb_frag_size_set(&head_frag, skb_headlen(frag_skb)); 3724 return head_frag; 3725 } 3726 3727 struct sk_buff *skb_segment_list(struct sk_buff *skb, 3728 netdev_features_t features, 3729 unsigned int offset) 3730 { 3731 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list; 3732 unsigned int tnl_hlen = skb_tnl_header_len(skb); 3733 unsigned int delta_truesize = 0; 3734 unsigned int delta_len = 0; 3735 struct sk_buff *nskb, *tmp; 3736 int err; 3737 3738 skb_push(skb, -skb_network_offset(skb) + offset); 3739 3740 skb_shinfo(skb)->frag_list = NULL; 3741 skb->next = list_skb; 3742 3743 do { 3744 nskb = list_skb; 3745 list_skb = list_skb->next; 3746 3747 err = 0; 3748 if (skb_shared(nskb)) { 3749 tmp = skb_clone(nskb, GFP_ATOMIC); 3750 if (tmp) { 3751 consume_skb(nskb); 3752 nskb = tmp; 3753 err = skb_unclone(nskb, GFP_ATOMIC); 3754 } else { 3755 err = -ENOMEM; 3756 } 3757 } 3758 3759 if (unlikely(err)) 3760 goto err_linearize; 3761 3762 delta_len += nskb->len; 3763 delta_truesize += nskb->truesize; 3764 3765 skb_push(nskb, -skb_network_offset(nskb) + offset); 3766 3767 skb_release_head_state(nskb); 3768 __copy_skb_header(nskb, skb); 3769 3770 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); 3771 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 3772 nskb->data - tnl_hlen, 3773 offset + tnl_hlen); 3774 3775 if (skb_needs_linearize(nskb, features) && 3776 __skb_linearize(nskb)) 3777 goto err_linearize; 3778 3779 } while (list_skb); 3780 3781 skb->truesize = skb->truesize - delta_truesize; 3782 skb->data_len = skb->data_len - delta_len; 3783 skb->len = skb->len - delta_len; 3784 3785 skb_gso_reset(skb); 3786 3787 skb->prev = nskb; 3788 3789 if (skb_needs_linearize(skb, features) && 3790 __skb_linearize(skb)) 3791 goto err_linearize; 3792 3793 skb_get(skb); 3794 3795 return skb; 3796 3797 err_linearize: 3798 kfree_skb_list(skb->next); 3799 skb->next = NULL; 3800 return ERR_PTR(-ENOMEM); 3801 } 3802 EXPORT_SYMBOL_GPL(skb_segment_list); 3803 3804 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) 3805 { 3806 if (unlikely(p->len + skb->len >= 65536)) 3807 return -E2BIG; 3808 3809 if (NAPI_GRO_CB(p)->last == p) 3810 skb_shinfo(p)->frag_list = skb; 3811 else 3812 NAPI_GRO_CB(p)->last->next = skb; 3813 3814 skb_pull(skb, skb_gro_offset(skb)); 3815 3816 NAPI_GRO_CB(p)->last = skb; 3817 NAPI_GRO_CB(p)->count++; 3818 p->data_len += skb->len; 3819 p->truesize += skb->truesize; 3820 p->len += skb->len; 3821 3822 NAPI_GRO_CB(skb)->same_flow = 1; 3823 3824 return 0; 3825 } 3826 3827 /** 3828 * skb_segment - Perform protocol segmentation on skb. 3829 * @head_skb: buffer to segment 3830 * @features: features for the output path (see dev->features) 3831 * 3832 * This function performs segmentation on the given skb. It returns 3833 * a pointer to the first in a list of new skbs for the segments. 3834 * In case of error it returns ERR_PTR(err). 3835 */ 3836 struct sk_buff *skb_segment(struct sk_buff *head_skb, 3837 netdev_features_t features) 3838 { 3839 struct sk_buff *segs = NULL; 3840 struct sk_buff *tail = NULL; 3841 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 3842 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 3843 unsigned int mss = skb_shinfo(head_skb)->gso_size; 3844 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 3845 struct sk_buff *frag_skb = head_skb; 3846 unsigned int offset = doffset; 3847 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 3848 unsigned int partial_segs = 0; 3849 unsigned int headroom; 3850 unsigned int len = head_skb->len; 3851 __be16 proto; 3852 bool csum, sg; 3853 int nfrags = skb_shinfo(head_skb)->nr_frags; 3854 int err = -ENOMEM; 3855 int i = 0; 3856 int pos; 3857 3858 if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) && 3859 (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) { 3860 /* gso_size is untrusted, and we have a frag_list with a linear 3861 * non head_frag head. 3862 * 3863 * (we assume checking the first list_skb member suffices; 3864 * i.e if either of the list_skb members have non head_frag 3865 * head, then the first one has too). 3866 * 3867 * If head_skb's headlen does not fit requested gso_size, it 3868 * means that the frag_list members do NOT terminate on exact 3869 * gso_size boundaries. Hence we cannot perform skb_frag_t page 3870 * sharing. Therefore we must fallback to copying the frag_list 3871 * skbs; we do so by disabling SG. 3872 */ 3873 if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) 3874 features &= ~NETIF_F_SG; 3875 } 3876 3877 __skb_push(head_skb, doffset); 3878 proto = skb_network_protocol(head_skb, NULL); 3879 if (unlikely(!proto)) 3880 return ERR_PTR(-EINVAL); 3881 3882 sg = !!(features & NETIF_F_SG); 3883 csum = !!can_checksum_protocol(features, proto); 3884 3885 if (sg && csum && (mss != GSO_BY_FRAGS)) { 3886 if (!(features & NETIF_F_GSO_PARTIAL)) { 3887 struct sk_buff *iter; 3888 unsigned int frag_len; 3889 3890 if (!list_skb || 3891 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type)) 3892 goto normal; 3893 3894 /* If we get here then all the required 3895 * GSO features except frag_list are supported. 3896 * Try to split the SKB to multiple GSO SKBs 3897 * with no frag_list. 3898 * Currently we can do that only when the buffers don't 3899 * have a linear part and all the buffers except 3900 * the last are of the same length. 3901 */ 3902 frag_len = list_skb->len; 3903 skb_walk_frags(head_skb, iter) { 3904 if (frag_len != iter->len && iter->next) 3905 goto normal; 3906 if (skb_headlen(iter) && !iter->head_frag) 3907 goto normal; 3908 3909 len -= iter->len; 3910 } 3911 3912 if (len != frag_len) 3913 goto normal; 3914 } 3915 3916 /* GSO partial only requires that we trim off any excess that 3917 * doesn't fit into an MSS sized block, so take care of that 3918 * now. 3919 */ 3920 partial_segs = len / mss; 3921 if (partial_segs > 1) 3922 mss *= partial_segs; 3923 else 3924 partial_segs = 0; 3925 } 3926 3927 normal: 3928 headroom = skb_headroom(head_skb); 3929 pos = skb_headlen(head_skb); 3930 3931 do { 3932 struct sk_buff *nskb; 3933 skb_frag_t *nskb_frag; 3934 int hsize; 3935 int size; 3936 3937 if (unlikely(mss == GSO_BY_FRAGS)) { 3938 len = list_skb->len; 3939 } else { 3940 len = head_skb->len - offset; 3941 if (len > mss) 3942 len = mss; 3943 } 3944 3945 hsize = skb_headlen(head_skb) - offset; 3946 3947 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) && 3948 (skb_headlen(list_skb) == len || sg)) { 3949 BUG_ON(skb_headlen(list_skb) > len); 3950 3951 i = 0; 3952 nfrags = skb_shinfo(list_skb)->nr_frags; 3953 frag = skb_shinfo(list_skb)->frags; 3954 frag_skb = list_skb; 3955 pos += skb_headlen(list_skb); 3956 3957 while (pos < offset + len) { 3958 BUG_ON(i >= nfrags); 3959 3960 size = skb_frag_size(frag); 3961 if (pos + size > offset + len) 3962 break; 3963 3964 i++; 3965 pos += size; 3966 frag++; 3967 } 3968 3969 nskb = skb_clone(list_skb, GFP_ATOMIC); 3970 list_skb = list_skb->next; 3971 3972 if (unlikely(!nskb)) 3973 goto err; 3974 3975 if (unlikely(pskb_trim(nskb, len))) { 3976 kfree_skb(nskb); 3977 goto err; 3978 } 3979 3980 hsize = skb_end_offset(nskb); 3981 if (skb_cow_head(nskb, doffset + headroom)) { 3982 kfree_skb(nskb); 3983 goto err; 3984 } 3985 3986 nskb->truesize += skb_end_offset(nskb) - hsize; 3987 skb_release_head_state(nskb); 3988 __skb_push(nskb, doffset); 3989 } else { 3990 if (hsize < 0) 3991 hsize = 0; 3992 if (hsize > len || !sg) 3993 hsize = len; 3994 3995 nskb = __alloc_skb(hsize + doffset + headroom, 3996 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 3997 NUMA_NO_NODE); 3998 3999 if (unlikely(!nskb)) 4000 goto err; 4001 4002 skb_reserve(nskb, headroom); 4003 __skb_put(nskb, doffset); 4004 } 4005 4006 if (segs) 4007 tail->next = nskb; 4008 else 4009 segs = nskb; 4010 tail = nskb; 4011 4012 __copy_skb_header(nskb, head_skb); 4013 4014 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 4015 skb_reset_mac_len(nskb); 4016 4017 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 4018 nskb->data - tnl_hlen, 4019 doffset + tnl_hlen); 4020 4021 if (nskb->len == len + doffset) 4022 goto perform_csum_check; 4023 4024 if (!sg) { 4025 if (!csum) { 4026 if (!nskb->remcsum_offload) 4027 nskb->ip_summed = CHECKSUM_NONE; 4028 SKB_GSO_CB(nskb)->csum = 4029 skb_copy_and_csum_bits(head_skb, offset, 4030 skb_put(nskb, 4031 len), 4032 len); 4033 SKB_GSO_CB(nskb)->csum_start = 4034 skb_headroom(nskb) + doffset; 4035 } else { 4036 skb_copy_bits(head_skb, offset, 4037 skb_put(nskb, len), 4038 len); 4039 } 4040 continue; 4041 } 4042 4043 nskb_frag = skb_shinfo(nskb)->frags; 4044 4045 skb_copy_from_linear_data_offset(head_skb, offset, 4046 skb_put(nskb, hsize), hsize); 4047 4048 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & 4049 SKBFL_SHARED_FRAG; 4050 4051 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4052 skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) 4053 goto err; 4054 4055 while (pos < offset + len) { 4056 if (i >= nfrags) { 4057 i = 0; 4058 nfrags = skb_shinfo(list_skb)->nr_frags; 4059 frag = skb_shinfo(list_skb)->frags; 4060 frag_skb = list_skb; 4061 if (!skb_headlen(list_skb)) { 4062 BUG_ON(!nfrags); 4063 } else { 4064 BUG_ON(!list_skb->head_frag); 4065 4066 /* to make room for head_frag. */ 4067 i--; 4068 frag--; 4069 } 4070 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || 4071 skb_zerocopy_clone(nskb, frag_skb, 4072 GFP_ATOMIC)) 4073 goto err; 4074 4075 list_skb = list_skb->next; 4076 } 4077 4078 if (unlikely(skb_shinfo(nskb)->nr_frags >= 4079 MAX_SKB_FRAGS)) { 4080 net_warn_ratelimited( 4081 "skb_segment: too many frags: %u %u\n", 4082 pos, mss); 4083 err = -EINVAL; 4084 goto err; 4085 } 4086 4087 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; 4088 __skb_frag_ref(nskb_frag); 4089 size = skb_frag_size(nskb_frag); 4090 4091 if (pos < offset) { 4092 skb_frag_off_add(nskb_frag, offset - pos); 4093 skb_frag_size_sub(nskb_frag, offset - pos); 4094 } 4095 4096 skb_shinfo(nskb)->nr_frags++; 4097 4098 if (pos + size <= offset + len) { 4099 i++; 4100 frag++; 4101 pos += size; 4102 } else { 4103 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 4104 goto skip_fraglist; 4105 } 4106 4107 nskb_frag++; 4108 } 4109 4110 skip_fraglist: 4111 nskb->data_len = len - hsize; 4112 nskb->len += nskb->data_len; 4113 nskb->truesize += nskb->data_len; 4114 4115 perform_csum_check: 4116 if (!csum) { 4117 if (skb_has_shared_frag(nskb) && 4118 __skb_linearize(nskb)) 4119 goto err; 4120 4121 if (!nskb->remcsum_offload) 4122 nskb->ip_summed = CHECKSUM_NONE; 4123 SKB_GSO_CB(nskb)->csum = 4124 skb_checksum(nskb, doffset, 4125 nskb->len - doffset, 0); 4126 SKB_GSO_CB(nskb)->csum_start = 4127 skb_headroom(nskb) + doffset; 4128 } 4129 } while ((offset += len) < head_skb->len); 4130 4131 /* Some callers want to get the end of the list. 4132 * Put it in segs->prev to avoid walking the list. 4133 * (see validate_xmit_skb_list() for example) 4134 */ 4135 segs->prev = tail; 4136 4137 if (partial_segs) { 4138 struct sk_buff *iter; 4139 int type = skb_shinfo(head_skb)->gso_type; 4140 unsigned short gso_size = skb_shinfo(head_skb)->gso_size; 4141 4142 /* Update type to add partial and then remove dodgy if set */ 4143 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL; 4144 type &= ~SKB_GSO_DODGY; 4145 4146 /* Update GSO info and prepare to start updating headers on 4147 * our way back down the stack of protocols. 4148 */ 4149 for (iter = segs; iter; iter = iter->next) { 4150 skb_shinfo(iter)->gso_size = gso_size; 4151 skb_shinfo(iter)->gso_segs = partial_segs; 4152 skb_shinfo(iter)->gso_type = type; 4153 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset; 4154 } 4155 4156 if (tail->len - doffset <= gso_size) 4157 skb_shinfo(tail)->gso_size = 0; 4158 else if (tail != segs) 4159 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size); 4160 } 4161 4162 /* Following permits correct backpressure, for protocols 4163 * using skb_set_owner_w(). 4164 * Idea is to tranfert ownership from head_skb to last segment. 4165 */ 4166 if (head_skb->destructor == sock_wfree) { 4167 swap(tail->truesize, head_skb->truesize); 4168 swap(tail->destructor, head_skb->destructor); 4169 swap(tail->sk, head_skb->sk); 4170 } 4171 return segs; 4172 4173 err: 4174 kfree_skb_list(segs); 4175 return ERR_PTR(err); 4176 } 4177 EXPORT_SYMBOL_GPL(skb_segment); 4178 4179 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) 4180 { 4181 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 4182 unsigned int offset = skb_gro_offset(skb); 4183 unsigned int headlen = skb_headlen(skb); 4184 unsigned int len = skb_gro_len(skb); 4185 unsigned int delta_truesize; 4186 struct sk_buff *lp; 4187 4188 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) 4189 return -E2BIG; 4190 4191 lp = NAPI_GRO_CB(p)->last; 4192 pinfo = skb_shinfo(lp); 4193 4194 if (headlen <= offset) { 4195 skb_frag_t *frag; 4196 skb_frag_t *frag2; 4197 int i = skbinfo->nr_frags; 4198 int nr_frags = pinfo->nr_frags + i; 4199 4200 if (nr_frags > MAX_SKB_FRAGS) 4201 goto merge; 4202 4203 offset -= headlen; 4204 pinfo->nr_frags = nr_frags; 4205 skbinfo->nr_frags = 0; 4206 4207 frag = pinfo->frags + nr_frags; 4208 frag2 = skbinfo->frags + i; 4209 do { 4210 *--frag = *--frag2; 4211 } while (--i); 4212 4213 skb_frag_off_add(frag, offset); 4214 skb_frag_size_sub(frag, offset); 4215 4216 /* all fragments truesize : remove (head size + sk_buff) */ 4217 delta_truesize = skb->truesize - 4218 SKB_TRUESIZE(skb_end_offset(skb)); 4219 4220 skb->truesize -= skb->data_len; 4221 skb->len -= skb->data_len; 4222 skb->data_len = 0; 4223 4224 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 4225 goto done; 4226 } else if (skb->head_frag) { 4227 int nr_frags = pinfo->nr_frags; 4228 skb_frag_t *frag = pinfo->frags + nr_frags; 4229 struct page *page = virt_to_head_page(skb->head); 4230 unsigned int first_size = headlen - offset; 4231 unsigned int first_offset; 4232 4233 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 4234 goto merge; 4235 4236 first_offset = skb->data - 4237 (unsigned char *)page_address(page) + 4238 offset; 4239 4240 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 4241 4242 __skb_frag_set_page(frag, page); 4243 skb_frag_off_set(frag, first_offset); 4244 skb_frag_size_set(frag, first_size); 4245 4246 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 4247 /* We dont need to clear skbinfo->nr_frags here */ 4248 4249 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 4250 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 4251 goto done; 4252 } 4253 4254 merge: 4255 delta_truesize = skb->truesize; 4256 if (offset > headlen) { 4257 unsigned int eat = offset - headlen; 4258 4259 skb_frag_off_add(&skbinfo->frags[0], eat); 4260 skb_frag_size_sub(&skbinfo->frags[0], eat); 4261 skb->data_len -= eat; 4262 skb->len -= eat; 4263 offset = headlen; 4264 } 4265 4266 __skb_pull(skb, offset); 4267 4268 if (NAPI_GRO_CB(p)->last == p) 4269 skb_shinfo(p)->frag_list = skb; 4270 else 4271 NAPI_GRO_CB(p)->last->next = skb; 4272 NAPI_GRO_CB(p)->last = skb; 4273 __skb_header_release(skb); 4274 lp = p; 4275 4276 done: 4277 NAPI_GRO_CB(p)->count++; 4278 p->data_len += len; 4279 p->truesize += delta_truesize; 4280 p->len += len; 4281 if (lp != p) { 4282 lp->data_len += len; 4283 lp->truesize += delta_truesize; 4284 lp->len += len; 4285 } 4286 NAPI_GRO_CB(skb)->same_flow = 1; 4287 return 0; 4288 } 4289 4290 #ifdef CONFIG_SKB_EXTENSIONS 4291 #define SKB_EXT_ALIGN_VALUE 8 4292 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) 4293 4294 static const u8 skb_ext_type_len[] = { 4295 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4296 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info), 4297 #endif 4298 #ifdef CONFIG_XFRM 4299 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path), 4300 #endif 4301 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4302 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext), 4303 #endif 4304 #if IS_ENABLED(CONFIG_MPTCP) 4305 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext), 4306 #endif 4307 }; 4308 4309 static __always_inline unsigned int skb_ext_total_length(void) 4310 { 4311 return SKB_EXT_CHUNKSIZEOF(struct skb_ext) + 4312 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 4313 skb_ext_type_len[SKB_EXT_BRIDGE_NF] + 4314 #endif 4315 #ifdef CONFIG_XFRM 4316 skb_ext_type_len[SKB_EXT_SEC_PATH] + 4317 #endif 4318 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 4319 skb_ext_type_len[TC_SKB_EXT] + 4320 #endif 4321 #if IS_ENABLED(CONFIG_MPTCP) 4322 skb_ext_type_len[SKB_EXT_MPTCP] + 4323 #endif 4324 0; 4325 } 4326 4327 static void skb_extensions_init(void) 4328 { 4329 BUILD_BUG_ON(SKB_EXT_NUM >= 8); 4330 BUILD_BUG_ON(skb_ext_total_length() > 255); 4331 4332 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", 4333 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), 4334 0, 4335 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4336 NULL); 4337 } 4338 #else 4339 static void skb_extensions_init(void) {} 4340 #endif 4341 4342 void __init skb_init(void) 4343 { 4344 skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache", 4345 sizeof(struct sk_buff), 4346 0, 4347 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4348 offsetof(struct sk_buff, cb), 4349 sizeof_field(struct sk_buff, cb), 4350 NULL); 4351 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 4352 sizeof(struct sk_buff_fclones), 4353 0, 4354 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 4355 NULL); 4356 skb_extensions_init(); 4357 } 4358 4359 static int 4360 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len, 4361 unsigned int recursion_level) 4362 { 4363 int start = skb_headlen(skb); 4364 int i, copy = start - offset; 4365 struct sk_buff *frag_iter; 4366 int elt = 0; 4367 4368 if (unlikely(recursion_level >= 24)) 4369 return -EMSGSIZE; 4370 4371 if (copy > 0) { 4372 if (copy > len) 4373 copy = len; 4374 sg_set_buf(sg, skb->data + offset, copy); 4375 elt++; 4376 if ((len -= copy) == 0) 4377 return elt; 4378 offset += copy; 4379 } 4380 4381 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4382 int end; 4383 4384 WARN_ON(start > offset + len); 4385 4386 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 4387 if ((copy = end - offset) > 0) { 4388 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4389 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4390 return -EMSGSIZE; 4391 4392 if (copy > len) 4393 copy = len; 4394 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 4395 skb_frag_off(frag) + offset - start); 4396 elt++; 4397 if (!(len -= copy)) 4398 return elt; 4399 offset += copy; 4400 } 4401 start = end; 4402 } 4403 4404 skb_walk_frags(skb, frag_iter) { 4405 int end, ret; 4406 4407 WARN_ON(start > offset + len); 4408 4409 end = start + frag_iter->len; 4410 if ((copy = end - offset) > 0) { 4411 if (unlikely(elt && sg_is_last(&sg[elt - 1]))) 4412 return -EMSGSIZE; 4413 4414 if (copy > len) 4415 copy = len; 4416 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start, 4417 copy, recursion_level + 1); 4418 if (unlikely(ret < 0)) 4419 return ret; 4420 elt += ret; 4421 if ((len -= copy) == 0) 4422 return elt; 4423 offset += copy; 4424 } 4425 start = end; 4426 } 4427 BUG_ON(len); 4428 return elt; 4429 } 4430 4431 /** 4432 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 4433 * @skb: Socket buffer containing the buffers to be mapped 4434 * @sg: The scatter-gather list to map into 4435 * @offset: The offset into the buffer's contents to start mapping 4436 * @len: Length of buffer space to be mapped 4437 * 4438 * Fill the specified scatter-gather list with mappings/pointers into a 4439 * region of the buffer space attached to a socket buffer. Returns either 4440 * the number of scatterlist items used, or -EMSGSIZE if the contents 4441 * could not fit. 4442 */ 4443 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 4444 { 4445 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0); 4446 4447 if (nsg <= 0) 4448 return nsg; 4449 4450 sg_mark_end(&sg[nsg - 1]); 4451 4452 return nsg; 4453 } 4454 EXPORT_SYMBOL_GPL(skb_to_sgvec); 4455 4456 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 4457 * sglist without mark the sg which contain last skb data as the end. 4458 * So the caller can mannipulate sg list as will when padding new data after 4459 * the first call without calling sg_unmark_end to expend sg list. 4460 * 4461 * Scenario to use skb_to_sgvec_nomark: 4462 * 1. sg_init_table 4463 * 2. skb_to_sgvec_nomark(payload1) 4464 * 3. skb_to_sgvec_nomark(payload2) 4465 * 4466 * This is equivalent to: 4467 * 1. sg_init_table 4468 * 2. skb_to_sgvec(payload1) 4469 * 3. sg_unmark_end 4470 * 4. skb_to_sgvec(payload2) 4471 * 4472 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 4473 * is more preferable. 4474 */ 4475 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 4476 int offset, int len) 4477 { 4478 return __skb_to_sgvec(skb, sg, offset, len, 0); 4479 } 4480 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 4481 4482 4483 4484 /** 4485 * skb_cow_data - Check that a socket buffer's data buffers are writable 4486 * @skb: The socket buffer to check. 4487 * @tailbits: Amount of trailing space to be added 4488 * @trailer: Returned pointer to the skb where the @tailbits space begins 4489 * 4490 * Make sure that the data buffers attached to a socket buffer are 4491 * writable. If they are not, private copies are made of the data buffers 4492 * and the socket buffer is set to use these instead. 4493 * 4494 * If @tailbits is given, make sure that there is space to write @tailbits 4495 * bytes of data beyond current end of socket buffer. @trailer will be 4496 * set to point to the skb in which this space begins. 4497 * 4498 * The number of scatterlist elements required to completely map the 4499 * COW'd and extended socket buffer will be returned. 4500 */ 4501 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 4502 { 4503 int copyflag; 4504 int elt; 4505 struct sk_buff *skb1, **skb_p; 4506 4507 /* If skb is cloned or its head is paged, reallocate 4508 * head pulling out all the pages (pages are considered not writable 4509 * at the moment even if they are anonymous). 4510 */ 4511 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 4512 !__pskb_pull_tail(skb, __skb_pagelen(skb))) 4513 return -ENOMEM; 4514 4515 /* Easy case. Most of packets will go this way. */ 4516 if (!skb_has_frag_list(skb)) { 4517 /* A little of trouble, not enough of space for trailer. 4518 * This should not happen, when stack is tuned to generate 4519 * good frames. OK, on miss we reallocate and reserve even more 4520 * space, 128 bytes is fair. */ 4521 4522 if (skb_tailroom(skb) < tailbits && 4523 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 4524 return -ENOMEM; 4525 4526 /* Voila! */ 4527 *trailer = skb; 4528 return 1; 4529 } 4530 4531 /* Misery. We are in troubles, going to mincer fragments... */ 4532 4533 elt = 1; 4534 skb_p = &skb_shinfo(skb)->frag_list; 4535 copyflag = 0; 4536 4537 while ((skb1 = *skb_p) != NULL) { 4538 int ntail = 0; 4539 4540 /* The fragment is partially pulled by someone, 4541 * this can happen on input. Copy it and everything 4542 * after it. */ 4543 4544 if (skb_shared(skb1)) 4545 copyflag = 1; 4546 4547 /* If the skb is the last, worry about trailer. */ 4548 4549 if (skb1->next == NULL && tailbits) { 4550 if (skb_shinfo(skb1)->nr_frags || 4551 skb_has_frag_list(skb1) || 4552 skb_tailroom(skb1) < tailbits) 4553 ntail = tailbits + 128; 4554 } 4555 4556 if (copyflag || 4557 skb_cloned(skb1) || 4558 ntail || 4559 skb_shinfo(skb1)->nr_frags || 4560 skb_has_frag_list(skb1)) { 4561 struct sk_buff *skb2; 4562 4563 /* Fuck, we are miserable poor guys... */ 4564 if (ntail == 0) 4565 skb2 = skb_copy(skb1, GFP_ATOMIC); 4566 else 4567 skb2 = skb_copy_expand(skb1, 4568 skb_headroom(skb1), 4569 ntail, 4570 GFP_ATOMIC); 4571 if (unlikely(skb2 == NULL)) 4572 return -ENOMEM; 4573 4574 if (skb1->sk) 4575 skb_set_owner_w(skb2, skb1->sk); 4576 4577 /* Looking around. Are we still alive? 4578 * OK, link new skb, drop old one */ 4579 4580 skb2->next = skb1->next; 4581 *skb_p = skb2; 4582 kfree_skb(skb1); 4583 skb1 = skb2; 4584 } 4585 elt++; 4586 *trailer = skb1; 4587 skb_p = &skb1->next; 4588 } 4589 4590 return elt; 4591 } 4592 EXPORT_SYMBOL_GPL(skb_cow_data); 4593 4594 static void sock_rmem_free(struct sk_buff *skb) 4595 { 4596 struct sock *sk = skb->sk; 4597 4598 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 4599 } 4600 4601 static void skb_set_err_queue(struct sk_buff *skb) 4602 { 4603 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. 4604 * So, it is safe to (mis)use it to mark skbs on the error queue. 4605 */ 4606 skb->pkt_type = PACKET_OUTGOING; 4607 BUILD_BUG_ON(PACKET_OUTGOING == 0); 4608 } 4609 4610 /* 4611 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 4612 */ 4613 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 4614 { 4615 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 4616 (unsigned int)READ_ONCE(sk->sk_rcvbuf)) 4617 return -ENOMEM; 4618 4619 skb_orphan(skb); 4620 skb->sk = sk; 4621 skb->destructor = sock_rmem_free; 4622 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 4623 skb_set_err_queue(skb); 4624 4625 /* before exiting rcu section, make sure dst is refcounted */ 4626 skb_dst_force(skb); 4627 4628 skb_queue_tail(&sk->sk_error_queue, skb); 4629 if (!sock_flag(sk, SOCK_DEAD)) 4630 sk->sk_error_report(sk); 4631 return 0; 4632 } 4633 EXPORT_SYMBOL(sock_queue_err_skb); 4634 4635 static bool is_icmp_err_skb(const struct sk_buff *skb) 4636 { 4637 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP || 4638 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6); 4639 } 4640 4641 struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 4642 { 4643 struct sk_buff_head *q = &sk->sk_error_queue; 4644 struct sk_buff *skb, *skb_next = NULL; 4645 bool icmp_next = false; 4646 unsigned long flags; 4647 4648 spin_lock_irqsave(&q->lock, flags); 4649 skb = __skb_dequeue(q); 4650 if (skb && (skb_next = skb_peek(q))) { 4651 icmp_next = is_icmp_err_skb(skb_next); 4652 if (icmp_next) 4653 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 4654 } 4655 spin_unlock_irqrestore(&q->lock, flags); 4656 4657 if (is_icmp_err_skb(skb) && !icmp_next) 4658 sk->sk_err = 0; 4659 4660 if (skb_next) 4661 sk->sk_error_report(sk); 4662 4663 return skb; 4664 } 4665 EXPORT_SYMBOL(sock_dequeue_err_skb); 4666 4667 /** 4668 * skb_clone_sk - create clone of skb, and take reference to socket 4669 * @skb: the skb to clone 4670 * 4671 * This function creates a clone of a buffer that holds a reference on 4672 * sk_refcnt. Buffers created via this function are meant to be 4673 * returned using sock_queue_err_skb, or free via kfree_skb. 4674 * 4675 * When passing buffers allocated with this function to sock_queue_err_skb 4676 * it is necessary to wrap the call with sock_hold/sock_put in order to 4677 * prevent the socket from being released prior to being enqueued on 4678 * the sk_error_queue. 4679 */ 4680 struct sk_buff *skb_clone_sk(struct sk_buff *skb) 4681 { 4682 struct sock *sk = skb->sk; 4683 struct sk_buff *clone; 4684 4685 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) 4686 return NULL; 4687 4688 clone = skb_clone(skb, GFP_ATOMIC); 4689 if (!clone) { 4690 sock_put(sk); 4691 return NULL; 4692 } 4693 4694 clone->sk = sk; 4695 clone->destructor = sock_efree; 4696 4697 return clone; 4698 } 4699 EXPORT_SYMBOL(skb_clone_sk); 4700 4701 static void __skb_complete_tx_timestamp(struct sk_buff *skb, 4702 struct sock *sk, 4703 int tstype, 4704 bool opt_stats) 4705 { 4706 struct sock_exterr_skb *serr; 4707 int err; 4708 4709 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb)); 4710 4711 serr = SKB_EXT_ERR(skb); 4712 memset(serr, 0, sizeof(*serr)); 4713 serr->ee.ee_errno = ENOMSG; 4714 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 4715 serr->ee.ee_info = tstype; 4716 serr->opt_stats = opt_stats; 4717 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; 4718 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 4719 serr->ee.ee_data = skb_shinfo(skb)->tskey; 4720 if (sk->sk_protocol == IPPROTO_TCP && 4721 sk->sk_type == SOCK_STREAM) 4722 serr->ee.ee_data -= sk->sk_tskey; 4723 } 4724 4725 err = sock_queue_err_skb(sk, skb); 4726 4727 if (err) 4728 kfree_skb(skb); 4729 } 4730 4731 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) 4732 { 4733 bool ret; 4734 4735 if (likely(sysctl_tstamp_allow_data || tsonly)) 4736 return true; 4737 4738 read_lock_bh(&sk->sk_callback_lock); 4739 ret = sk->sk_socket && sk->sk_socket->file && 4740 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); 4741 read_unlock_bh(&sk->sk_callback_lock); 4742 return ret; 4743 } 4744 4745 void skb_complete_tx_timestamp(struct sk_buff *skb, 4746 struct skb_shared_hwtstamps *hwtstamps) 4747 { 4748 struct sock *sk = skb->sk; 4749 4750 if (!skb_may_tx_timestamp(sk, false)) 4751 goto err; 4752 4753 /* Take a reference to prevent skb_orphan() from freeing the socket, 4754 * but only if the socket refcount is not zero. 4755 */ 4756 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4757 *skb_hwtstamps(skb) = *hwtstamps; 4758 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false); 4759 sock_put(sk); 4760 return; 4761 } 4762 4763 err: 4764 kfree_skb(skb); 4765 } 4766 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 4767 4768 void __skb_tstamp_tx(struct sk_buff *orig_skb, 4769 const struct sk_buff *ack_skb, 4770 struct skb_shared_hwtstamps *hwtstamps, 4771 struct sock *sk, int tstype) 4772 { 4773 struct sk_buff *skb; 4774 bool tsonly, opt_stats = false; 4775 4776 if (!sk) 4777 return; 4778 4779 if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) && 4780 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS) 4781 return; 4782 4783 tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY; 4784 if (!skb_may_tx_timestamp(sk, tsonly)) 4785 return; 4786 4787 if (tsonly) { 4788 #ifdef CONFIG_INET 4789 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && 4790 sk->sk_protocol == IPPROTO_TCP && 4791 sk->sk_type == SOCK_STREAM) { 4792 skb = tcp_get_timestamping_opt_stats(sk, orig_skb, 4793 ack_skb); 4794 opt_stats = true; 4795 } else 4796 #endif 4797 skb = alloc_skb(0, GFP_ATOMIC); 4798 } else { 4799 skb = skb_clone(orig_skb, GFP_ATOMIC); 4800 } 4801 if (!skb) 4802 return; 4803 4804 if (tsonly) { 4805 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags & 4806 SKBTX_ANY_TSTAMP; 4807 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey; 4808 } 4809 4810 if (hwtstamps) 4811 *skb_hwtstamps(skb) = *hwtstamps; 4812 else 4813 skb->tstamp = ktime_get_real(); 4814 4815 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats); 4816 } 4817 EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 4818 4819 void skb_tstamp_tx(struct sk_buff *orig_skb, 4820 struct skb_shared_hwtstamps *hwtstamps) 4821 { 4822 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, 4823 SCM_TSTAMP_SND); 4824 } 4825 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 4826 4827 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 4828 { 4829 struct sock *sk = skb->sk; 4830 struct sock_exterr_skb *serr; 4831 int err = 1; 4832 4833 skb->wifi_acked_valid = 1; 4834 skb->wifi_acked = acked; 4835 4836 serr = SKB_EXT_ERR(skb); 4837 memset(serr, 0, sizeof(*serr)); 4838 serr->ee.ee_errno = ENOMSG; 4839 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 4840 4841 /* Take a reference to prevent skb_orphan() from freeing the socket, 4842 * but only if the socket refcount is not zero. 4843 */ 4844 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) { 4845 err = sock_queue_err_skb(sk, skb); 4846 sock_put(sk); 4847 } 4848 if (err) 4849 kfree_skb(skb); 4850 } 4851 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 4852 4853 /** 4854 * skb_partial_csum_set - set up and verify partial csum values for packet 4855 * @skb: the skb to set 4856 * @start: the number of bytes after skb->data to start checksumming. 4857 * @off: the offset from start to place the checksum. 4858 * 4859 * For untrusted partially-checksummed packets, we need to make sure the values 4860 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 4861 * 4862 * This function checks and sets those values and skb->ip_summed: if this 4863 * returns false you should drop the packet. 4864 */ 4865 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 4866 { 4867 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16); 4868 u32 csum_start = skb_headroom(skb) + (u32)start; 4869 4870 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) { 4871 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n", 4872 start, off, skb_headroom(skb), skb_headlen(skb)); 4873 return false; 4874 } 4875 skb->ip_summed = CHECKSUM_PARTIAL; 4876 skb->csum_start = csum_start; 4877 skb->csum_offset = off; 4878 skb_set_transport_header(skb, start); 4879 return true; 4880 } 4881 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 4882 4883 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 4884 unsigned int max) 4885 { 4886 if (skb_headlen(skb) >= len) 4887 return 0; 4888 4889 /* If we need to pullup then pullup to the max, so we 4890 * won't need to do it again. 4891 */ 4892 if (max > skb->len) 4893 max = skb->len; 4894 4895 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 4896 return -ENOMEM; 4897 4898 if (skb_headlen(skb) < len) 4899 return -EPROTO; 4900 4901 return 0; 4902 } 4903 4904 #define MAX_TCP_HDR_LEN (15 * 4) 4905 4906 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 4907 typeof(IPPROTO_IP) proto, 4908 unsigned int off) 4909 { 4910 int err; 4911 4912 switch (proto) { 4913 case IPPROTO_TCP: 4914 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 4915 off + MAX_TCP_HDR_LEN); 4916 if (!err && !skb_partial_csum_set(skb, off, 4917 offsetof(struct tcphdr, 4918 check))) 4919 err = -EPROTO; 4920 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 4921 4922 case IPPROTO_UDP: 4923 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 4924 off + sizeof(struct udphdr)); 4925 if (!err && !skb_partial_csum_set(skb, off, 4926 offsetof(struct udphdr, 4927 check))) 4928 err = -EPROTO; 4929 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 4930 } 4931 4932 return ERR_PTR(-EPROTO); 4933 } 4934 4935 /* This value should be large enough to cover a tagged ethernet header plus 4936 * maximally sized IP and TCP or UDP headers. 4937 */ 4938 #define MAX_IP_HDR_LEN 128 4939 4940 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 4941 { 4942 unsigned int off; 4943 bool fragment; 4944 __sum16 *csum; 4945 int err; 4946 4947 fragment = false; 4948 4949 err = skb_maybe_pull_tail(skb, 4950 sizeof(struct iphdr), 4951 MAX_IP_HDR_LEN); 4952 if (err < 0) 4953 goto out; 4954 4955 if (ip_is_fragment(ip_hdr(skb))) 4956 fragment = true; 4957 4958 off = ip_hdrlen(skb); 4959 4960 err = -EPROTO; 4961 4962 if (fragment) 4963 goto out; 4964 4965 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 4966 if (IS_ERR(csum)) 4967 return PTR_ERR(csum); 4968 4969 if (recalculate) 4970 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 4971 ip_hdr(skb)->daddr, 4972 skb->len - off, 4973 ip_hdr(skb)->protocol, 0); 4974 err = 0; 4975 4976 out: 4977 return err; 4978 } 4979 4980 /* This value should be large enough to cover a tagged ethernet header plus 4981 * an IPv6 header, all options, and a maximal TCP or UDP header. 4982 */ 4983 #define MAX_IPV6_HDR_LEN 256 4984 4985 #define OPT_HDR(type, skb, off) \ 4986 (type *)(skb_network_header(skb) + (off)) 4987 4988 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 4989 { 4990 int err; 4991 u8 nexthdr; 4992 unsigned int off; 4993 unsigned int len; 4994 bool fragment; 4995 bool done; 4996 __sum16 *csum; 4997 4998 fragment = false; 4999 done = false; 5000 5001 off = sizeof(struct ipv6hdr); 5002 5003 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 5004 if (err < 0) 5005 goto out; 5006 5007 nexthdr = ipv6_hdr(skb)->nexthdr; 5008 5009 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 5010 while (off <= len && !done) { 5011 switch (nexthdr) { 5012 case IPPROTO_DSTOPTS: 5013 case IPPROTO_HOPOPTS: 5014 case IPPROTO_ROUTING: { 5015 struct ipv6_opt_hdr *hp; 5016 5017 err = skb_maybe_pull_tail(skb, 5018 off + 5019 sizeof(struct ipv6_opt_hdr), 5020 MAX_IPV6_HDR_LEN); 5021 if (err < 0) 5022 goto out; 5023 5024 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 5025 nexthdr = hp->nexthdr; 5026 off += ipv6_optlen(hp); 5027 break; 5028 } 5029 case IPPROTO_AH: { 5030 struct ip_auth_hdr *hp; 5031 5032 err = skb_maybe_pull_tail(skb, 5033 off + 5034 sizeof(struct ip_auth_hdr), 5035 MAX_IPV6_HDR_LEN); 5036 if (err < 0) 5037 goto out; 5038 5039 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 5040 nexthdr = hp->nexthdr; 5041 off += ipv6_authlen(hp); 5042 break; 5043 } 5044 case IPPROTO_FRAGMENT: { 5045 struct frag_hdr *hp; 5046 5047 err = skb_maybe_pull_tail(skb, 5048 off + 5049 sizeof(struct frag_hdr), 5050 MAX_IPV6_HDR_LEN); 5051 if (err < 0) 5052 goto out; 5053 5054 hp = OPT_HDR(struct frag_hdr, skb, off); 5055 5056 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 5057 fragment = true; 5058 5059 nexthdr = hp->nexthdr; 5060 off += sizeof(struct frag_hdr); 5061 break; 5062 } 5063 default: 5064 done = true; 5065 break; 5066 } 5067 } 5068 5069 err = -EPROTO; 5070 5071 if (!done || fragment) 5072 goto out; 5073 5074 csum = skb_checksum_setup_ip(skb, nexthdr, off); 5075 if (IS_ERR(csum)) 5076 return PTR_ERR(csum); 5077 5078 if (recalculate) 5079 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5080 &ipv6_hdr(skb)->daddr, 5081 skb->len - off, nexthdr, 0); 5082 err = 0; 5083 5084 out: 5085 return err; 5086 } 5087 5088 /** 5089 * skb_checksum_setup - set up partial checksum offset 5090 * @skb: the skb to set up 5091 * @recalculate: if true the pseudo-header checksum will be recalculated 5092 */ 5093 int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 5094 { 5095 int err; 5096 5097 switch (skb->protocol) { 5098 case htons(ETH_P_IP): 5099 err = skb_checksum_setup_ipv4(skb, recalculate); 5100 break; 5101 5102 case htons(ETH_P_IPV6): 5103 err = skb_checksum_setup_ipv6(skb, recalculate); 5104 break; 5105 5106 default: 5107 err = -EPROTO; 5108 break; 5109 } 5110 5111 return err; 5112 } 5113 EXPORT_SYMBOL(skb_checksum_setup); 5114 5115 /** 5116 * skb_checksum_maybe_trim - maybe trims the given skb 5117 * @skb: the skb to check 5118 * @transport_len: the data length beyond the network header 5119 * 5120 * Checks whether the given skb has data beyond the given transport length. 5121 * If so, returns a cloned skb trimmed to this transport length. 5122 * Otherwise returns the provided skb. Returns NULL in error cases 5123 * (e.g. transport_len exceeds skb length or out-of-memory). 5124 * 5125 * Caller needs to set the skb transport header and free any returned skb if it 5126 * differs from the provided skb. 5127 */ 5128 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb, 5129 unsigned int transport_len) 5130 { 5131 struct sk_buff *skb_chk; 5132 unsigned int len = skb_transport_offset(skb) + transport_len; 5133 int ret; 5134 5135 if (skb->len < len) 5136 return NULL; 5137 else if (skb->len == len) 5138 return skb; 5139 5140 skb_chk = skb_clone(skb, GFP_ATOMIC); 5141 if (!skb_chk) 5142 return NULL; 5143 5144 ret = pskb_trim_rcsum(skb_chk, len); 5145 if (ret) { 5146 kfree_skb(skb_chk); 5147 return NULL; 5148 } 5149 5150 return skb_chk; 5151 } 5152 5153 /** 5154 * skb_checksum_trimmed - validate checksum of an skb 5155 * @skb: the skb to check 5156 * @transport_len: the data length beyond the network header 5157 * @skb_chkf: checksum function to use 5158 * 5159 * Applies the given checksum function skb_chkf to the provided skb. 5160 * Returns a checked and maybe trimmed skb. Returns NULL on error. 5161 * 5162 * If the skb has data beyond the given transport length, then a 5163 * trimmed & cloned skb is checked and returned. 5164 * 5165 * Caller needs to set the skb transport header and free any returned skb if it 5166 * differs from the provided skb. 5167 */ 5168 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb, 5169 unsigned int transport_len, 5170 __sum16(*skb_chkf)(struct sk_buff *skb)) 5171 { 5172 struct sk_buff *skb_chk; 5173 unsigned int offset = skb_transport_offset(skb); 5174 __sum16 ret; 5175 5176 skb_chk = skb_checksum_maybe_trim(skb, transport_len); 5177 if (!skb_chk) 5178 goto err; 5179 5180 if (!pskb_may_pull(skb_chk, offset)) 5181 goto err; 5182 5183 skb_pull_rcsum(skb_chk, offset); 5184 ret = skb_chkf(skb_chk); 5185 skb_push_rcsum(skb_chk, offset); 5186 5187 if (ret) 5188 goto err; 5189 5190 return skb_chk; 5191 5192 err: 5193 if (skb_chk && skb_chk != skb) 5194 kfree_skb(skb_chk); 5195 5196 return NULL; 5197 5198 } 5199 EXPORT_SYMBOL(skb_checksum_trimmed); 5200 5201 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 5202 { 5203 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 5204 skb->dev->name); 5205 } 5206 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 5207 5208 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 5209 { 5210 if (head_stolen) { 5211 skb_release_head_state(skb); 5212 kmem_cache_free(skbuff_head_cache, skb); 5213 } else { 5214 __kfree_skb(skb); 5215 } 5216 } 5217 EXPORT_SYMBOL(kfree_skb_partial); 5218 5219 /** 5220 * skb_try_coalesce - try to merge skb to prior one 5221 * @to: prior buffer 5222 * @from: buffer to add 5223 * @fragstolen: pointer to boolean 5224 * @delta_truesize: how much more was allocated than was requested 5225 */ 5226 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 5227 bool *fragstolen, int *delta_truesize) 5228 { 5229 struct skb_shared_info *to_shinfo, *from_shinfo; 5230 int i, delta, len = from->len; 5231 5232 *fragstolen = false; 5233 5234 if (skb_cloned(to)) 5235 return false; 5236 5237 if (len <= skb_tailroom(to)) { 5238 if (len) 5239 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 5240 *delta_truesize = 0; 5241 return true; 5242 } 5243 5244 to_shinfo = skb_shinfo(to); 5245 from_shinfo = skb_shinfo(from); 5246 if (to_shinfo->frag_list || from_shinfo->frag_list) 5247 return false; 5248 if (skb_zcopy(to) || skb_zcopy(from)) 5249 return false; 5250 5251 if (skb_headlen(from) != 0) { 5252 struct page *page; 5253 unsigned int offset; 5254 5255 if (to_shinfo->nr_frags + 5256 from_shinfo->nr_frags >= MAX_SKB_FRAGS) 5257 return false; 5258 5259 if (skb_head_is_locked(from)) 5260 return false; 5261 5262 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 5263 5264 page = virt_to_head_page(from->head); 5265 offset = from->data - (unsigned char *)page_address(page); 5266 5267 skb_fill_page_desc(to, to_shinfo->nr_frags, 5268 page, offset, skb_headlen(from)); 5269 *fragstolen = true; 5270 } else { 5271 if (to_shinfo->nr_frags + 5272 from_shinfo->nr_frags > MAX_SKB_FRAGS) 5273 return false; 5274 5275 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 5276 } 5277 5278 WARN_ON_ONCE(delta < len); 5279 5280 memcpy(to_shinfo->frags + to_shinfo->nr_frags, 5281 from_shinfo->frags, 5282 from_shinfo->nr_frags * sizeof(skb_frag_t)); 5283 to_shinfo->nr_frags += from_shinfo->nr_frags; 5284 5285 if (!skb_cloned(from)) 5286 from_shinfo->nr_frags = 0; 5287 5288 /* if the skb is not cloned this does nothing 5289 * since we set nr_frags to 0. 5290 */ 5291 for (i = 0; i < from_shinfo->nr_frags; i++) 5292 __skb_frag_ref(&from_shinfo->frags[i]); 5293 5294 to->truesize += delta; 5295 to->len += len; 5296 to->data_len += len; 5297 5298 *delta_truesize = delta; 5299 return true; 5300 } 5301 EXPORT_SYMBOL(skb_try_coalesce); 5302 5303 /** 5304 * skb_scrub_packet - scrub an skb 5305 * 5306 * @skb: buffer to clean 5307 * @xnet: packet is crossing netns 5308 * 5309 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 5310 * into/from a tunnel. Some information have to be cleared during these 5311 * operations. 5312 * skb_scrub_packet can also be used to clean a skb before injecting it in 5313 * another namespace (@xnet == true). We have to clear all information in the 5314 * skb that could impact namespace isolation. 5315 */ 5316 void skb_scrub_packet(struct sk_buff *skb, bool xnet) 5317 { 5318 skb->pkt_type = PACKET_HOST; 5319 skb->skb_iif = 0; 5320 skb->ignore_df = 0; 5321 skb_dst_drop(skb); 5322 skb_ext_reset(skb); 5323 nf_reset_ct(skb); 5324 nf_reset_trace(skb); 5325 5326 #ifdef CONFIG_NET_SWITCHDEV 5327 skb->offload_fwd_mark = 0; 5328 skb->offload_l3_fwd_mark = 0; 5329 #endif 5330 5331 if (!xnet) 5332 return; 5333 5334 ipvs_reset(skb); 5335 skb->mark = 0; 5336 skb->tstamp = 0; 5337 } 5338 EXPORT_SYMBOL_GPL(skb_scrub_packet); 5339 5340 /** 5341 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 5342 * 5343 * @skb: GSO skb 5344 * 5345 * skb_gso_transport_seglen is used to determine the real size of the 5346 * individual segments, including Layer4 headers (TCP/UDP). 5347 * 5348 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 5349 */ 5350 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 5351 { 5352 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5353 unsigned int thlen = 0; 5354 5355 if (skb->encapsulation) { 5356 thlen = skb_inner_transport_header(skb) - 5357 skb_transport_header(skb); 5358 5359 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 5360 thlen += inner_tcp_hdrlen(skb); 5361 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 5362 thlen = tcp_hdrlen(skb); 5363 } else if (unlikely(skb_is_gso_sctp(skb))) { 5364 thlen = sizeof(struct sctphdr); 5365 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 5366 thlen = sizeof(struct udphdr); 5367 } 5368 /* UFO sets gso_size to the size of the fragmentation 5369 * payload, i.e. the size of the L4 (UDP) header is already 5370 * accounted for. 5371 */ 5372 return thlen + shinfo->gso_size; 5373 } 5374 5375 /** 5376 * skb_gso_network_seglen - Return length of individual segments of a gso packet 5377 * 5378 * @skb: GSO skb 5379 * 5380 * skb_gso_network_seglen is used to determine the real size of the 5381 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP). 5382 * 5383 * The MAC/L2 header is not accounted for. 5384 */ 5385 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb) 5386 { 5387 unsigned int hdr_len = skb_transport_header(skb) - 5388 skb_network_header(skb); 5389 5390 return hdr_len + skb_gso_transport_seglen(skb); 5391 } 5392 5393 /** 5394 * skb_gso_mac_seglen - Return length of individual segments of a gso packet 5395 * 5396 * @skb: GSO skb 5397 * 5398 * skb_gso_mac_seglen is used to determine the real size of the 5399 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4 5400 * headers (TCP/UDP). 5401 */ 5402 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb) 5403 { 5404 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb); 5405 5406 return hdr_len + skb_gso_transport_seglen(skb); 5407 } 5408 5409 /** 5410 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS 5411 * 5412 * There are a couple of instances where we have a GSO skb, and we 5413 * want to determine what size it would be after it is segmented. 5414 * 5415 * We might want to check: 5416 * - L3+L4+payload size (e.g. IP forwarding) 5417 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver) 5418 * 5419 * This is a helper to do that correctly considering GSO_BY_FRAGS. 5420 * 5421 * @skb: GSO skb 5422 * 5423 * @seg_len: The segmented length (from skb_gso_*_seglen). In the 5424 * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS]. 5425 * 5426 * @max_len: The maximum permissible length. 5427 * 5428 * Returns true if the segmented length <= max length. 5429 */ 5430 static inline bool skb_gso_size_check(const struct sk_buff *skb, 5431 unsigned int seg_len, 5432 unsigned int max_len) { 5433 const struct skb_shared_info *shinfo = skb_shinfo(skb); 5434 const struct sk_buff *iter; 5435 5436 if (shinfo->gso_size != GSO_BY_FRAGS) 5437 return seg_len <= max_len; 5438 5439 /* Undo this so we can re-use header sizes */ 5440 seg_len -= GSO_BY_FRAGS; 5441 5442 skb_walk_frags(skb, iter) { 5443 if (seg_len + skb_headlen(iter) > max_len) 5444 return false; 5445 } 5446 5447 return true; 5448 } 5449 5450 /** 5451 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU? 5452 * 5453 * @skb: GSO skb 5454 * @mtu: MTU to validate against 5455 * 5456 * skb_gso_validate_network_len validates if a given skb will fit a 5457 * wanted MTU once split. It considers L3 headers, L4 headers, and the 5458 * payload. 5459 */ 5460 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu) 5461 { 5462 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu); 5463 } 5464 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len); 5465 5466 /** 5467 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length? 5468 * 5469 * @skb: GSO skb 5470 * @len: length to validate against 5471 * 5472 * skb_gso_validate_mac_len validates if a given skb will fit a wanted 5473 * length once split, including L2, L3 and L4 headers and the payload. 5474 */ 5475 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len) 5476 { 5477 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len); 5478 } 5479 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len); 5480 5481 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 5482 { 5483 int mac_len, meta_len; 5484 void *meta; 5485 5486 if (skb_cow(skb, skb_headroom(skb)) < 0) { 5487 kfree_skb(skb); 5488 return NULL; 5489 } 5490 5491 mac_len = skb->data - skb_mac_header(skb); 5492 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) { 5493 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb), 5494 mac_len - VLAN_HLEN - ETH_TLEN); 5495 } 5496 5497 meta_len = skb_metadata_len(skb); 5498 if (meta_len) { 5499 meta = skb_metadata_end(skb) - meta_len; 5500 memmove(meta + VLAN_HLEN, meta, meta_len); 5501 } 5502 5503 skb->mac_header += VLAN_HLEN; 5504 return skb; 5505 } 5506 5507 struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 5508 { 5509 struct vlan_hdr *vhdr; 5510 u16 vlan_tci; 5511 5512 if (unlikely(skb_vlan_tag_present(skb))) { 5513 /* vlan_tci is already set-up so leave this for another time */ 5514 return skb; 5515 } 5516 5517 skb = skb_share_check(skb, GFP_ATOMIC); 5518 if (unlikely(!skb)) 5519 goto err_free; 5520 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */ 5521 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short)))) 5522 goto err_free; 5523 5524 vhdr = (struct vlan_hdr *)skb->data; 5525 vlan_tci = ntohs(vhdr->h_vlan_TCI); 5526 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 5527 5528 skb_pull_rcsum(skb, VLAN_HLEN); 5529 vlan_set_encap_proto(skb, vhdr); 5530 5531 skb = skb_reorder_vlan_header(skb); 5532 if (unlikely(!skb)) 5533 goto err_free; 5534 5535 skb_reset_network_header(skb); 5536 if (!skb_transport_header_was_set(skb)) 5537 skb_reset_transport_header(skb); 5538 skb_reset_mac_len(skb); 5539 5540 return skb; 5541 5542 err_free: 5543 kfree_skb(skb); 5544 return NULL; 5545 } 5546 EXPORT_SYMBOL(skb_vlan_untag); 5547 5548 int skb_ensure_writable(struct sk_buff *skb, int write_len) 5549 { 5550 if (!pskb_may_pull(skb, write_len)) 5551 return -ENOMEM; 5552 5553 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) 5554 return 0; 5555 5556 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5557 } 5558 EXPORT_SYMBOL(skb_ensure_writable); 5559 5560 /* remove VLAN header from packet and update csum accordingly. 5561 * expects a non skb_vlan_tag_present skb with a vlan tag payload 5562 */ 5563 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci) 5564 { 5565 struct vlan_hdr *vhdr; 5566 int offset = skb->data - skb_mac_header(skb); 5567 int err; 5568 5569 if (WARN_ONCE(offset, 5570 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n", 5571 offset)) { 5572 return -EINVAL; 5573 } 5574 5575 err = skb_ensure_writable(skb, VLAN_ETH_HLEN); 5576 if (unlikely(err)) 5577 return err; 5578 5579 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5580 5581 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN); 5582 *vlan_tci = ntohs(vhdr->h_vlan_TCI); 5583 5584 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN); 5585 __skb_pull(skb, VLAN_HLEN); 5586 5587 vlan_set_encap_proto(skb, vhdr); 5588 skb->mac_header += VLAN_HLEN; 5589 5590 if (skb_network_offset(skb) < ETH_HLEN) 5591 skb_set_network_header(skb, ETH_HLEN); 5592 5593 skb_reset_mac_len(skb); 5594 5595 return err; 5596 } 5597 EXPORT_SYMBOL(__skb_vlan_pop); 5598 5599 /* Pop a vlan tag either from hwaccel or from payload. 5600 * Expects skb->data at mac header. 5601 */ 5602 int skb_vlan_pop(struct sk_buff *skb) 5603 { 5604 u16 vlan_tci; 5605 __be16 vlan_proto; 5606 int err; 5607 5608 if (likely(skb_vlan_tag_present(skb))) { 5609 __vlan_hwaccel_clear_tag(skb); 5610 } else { 5611 if (unlikely(!eth_type_vlan(skb->protocol))) 5612 return 0; 5613 5614 err = __skb_vlan_pop(skb, &vlan_tci); 5615 if (err) 5616 return err; 5617 } 5618 /* move next vlan tag to hw accel tag */ 5619 if (likely(!eth_type_vlan(skb->protocol))) 5620 return 0; 5621 5622 vlan_proto = skb->protocol; 5623 err = __skb_vlan_pop(skb, &vlan_tci); 5624 if (unlikely(err)) 5625 return err; 5626 5627 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5628 return 0; 5629 } 5630 EXPORT_SYMBOL(skb_vlan_pop); 5631 5632 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present). 5633 * Expects skb->data at mac header. 5634 */ 5635 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) 5636 { 5637 if (skb_vlan_tag_present(skb)) { 5638 int offset = skb->data - skb_mac_header(skb); 5639 int err; 5640 5641 if (WARN_ONCE(offset, 5642 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n", 5643 offset)) { 5644 return -EINVAL; 5645 } 5646 5647 err = __vlan_insert_tag(skb, skb->vlan_proto, 5648 skb_vlan_tag_get(skb)); 5649 if (err) 5650 return err; 5651 5652 skb->protocol = skb->vlan_proto; 5653 skb->mac_len += VLAN_HLEN; 5654 5655 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN); 5656 } 5657 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci); 5658 return 0; 5659 } 5660 EXPORT_SYMBOL(skb_vlan_push); 5661 5662 /** 5663 * skb_eth_pop() - Drop the Ethernet header at the head of a packet 5664 * 5665 * @skb: Socket buffer to modify 5666 * 5667 * Drop the Ethernet header of @skb. 5668 * 5669 * Expects that skb->data points to the mac header and that no VLAN tags are 5670 * present. 5671 * 5672 * Returns 0 on success, -errno otherwise. 5673 */ 5674 int skb_eth_pop(struct sk_buff *skb) 5675 { 5676 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) || 5677 skb_network_offset(skb) < ETH_HLEN) 5678 return -EPROTO; 5679 5680 skb_pull_rcsum(skb, ETH_HLEN); 5681 skb_reset_mac_header(skb); 5682 skb_reset_mac_len(skb); 5683 5684 return 0; 5685 } 5686 EXPORT_SYMBOL(skb_eth_pop); 5687 5688 /** 5689 * skb_eth_push() - Add a new Ethernet header at the head of a packet 5690 * 5691 * @skb: Socket buffer to modify 5692 * @dst: Destination MAC address of the new header 5693 * @src: Source MAC address of the new header 5694 * 5695 * Prepend @skb with a new Ethernet header. 5696 * 5697 * Expects that skb->data points to the mac header, which must be empty. 5698 * 5699 * Returns 0 on success, -errno otherwise. 5700 */ 5701 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst, 5702 const unsigned char *src) 5703 { 5704 struct ethhdr *eth; 5705 int err; 5706 5707 if (skb_network_offset(skb) || skb_vlan_tag_present(skb)) 5708 return -EPROTO; 5709 5710 err = skb_cow_head(skb, sizeof(*eth)); 5711 if (err < 0) 5712 return err; 5713 5714 skb_push(skb, sizeof(*eth)); 5715 skb_reset_mac_header(skb); 5716 skb_reset_mac_len(skb); 5717 5718 eth = eth_hdr(skb); 5719 ether_addr_copy(eth->h_dest, dst); 5720 ether_addr_copy(eth->h_source, src); 5721 eth->h_proto = skb->protocol; 5722 5723 skb_postpush_rcsum(skb, eth, sizeof(*eth)); 5724 5725 return 0; 5726 } 5727 EXPORT_SYMBOL(skb_eth_push); 5728 5729 /* Update the ethertype of hdr and the skb csum value if required. */ 5730 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr, 5731 __be16 ethertype) 5732 { 5733 if (skb->ip_summed == CHECKSUM_COMPLETE) { 5734 __be16 diff[] = { ~hdr->h_proto, ethertype }; 5735 5736 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 5737 } 5738 5739 hdr->h_proto = ethertype; 5740 } 5741 5742 /** 5743 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of 5744 * the packet 5745 * 5746 * @skb: buffer 5747 * @mpls_lse: MPLS label stack entry to push 5748 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848) 5749 * @mac_len: length of the MAC header 5750 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is 5751 * ethernet 5752 * 5753 * Expects skb->data at mac header. 5754 * 5755 * Returns 0 on success, -errno otherwise. 5756 */ 5757 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto, 5758 int mac_len, bool ethernet) 5759 { 5760 struct mpls_shim_hdr *lse; 5761 int err; 5762 5763 if (unlikely(!eth_p_mpls(mpls_proto))) 5764 return -EINVAL; 5765 5766 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */ 5767 if (skb->encapsulation) 5768 return -EINVAL; 5769 5770 err = skb_cow_head(skb, MPLS_HLEN); 5771 if (unlikely(err)) 5772 return err; 5773 5774 if (!skb->inner_protocol) { 5775 skb_set_inner_network_header(skb, skb_network_offset(skb)); 5776 skb_set_inner_protocol(skb, skb->protocol); 5777 } 5778 5779 skb_push(skb, MPLS_HLEN); 5780 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb), 5781 mac_len); 5782 skb_reset_mac_header(skb); 5783 skb_set_network_header(skb, mac_len); 5784 skb_reset_mac_len(skb); 5785 5786 lse = mpls_hdr(skb); 5787 lse->label_stack_entry = mpls_lse; 5788 skb_postpush_rcsum(skb, lse, MPLS_HLEN); 5789 5790 if (ethernet && mac_len >= ETH_HLEN) 5791 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto); 5792 skb->protocol = mpls_proto; 5793 5794 return 0; 5795 } 5796 EXPORT_SYMBOL_GPL(skb_mpls_push); 5797 5798 /** 5799 * skb_mpls_pop() - pop the outermost MPLS header 5800 * 5801 * @skb: buffer 5802 * @next_proto: ethertype of header after popped MPLS header 5803 * @mac_len: length of the MAC header 5804 * @ethernet: flag to indicate if the packet is ethernet 5805 * 5806 * Expects skb->data at mac header. 5807 * 5808 * Returns 0 on success, -errno otherwise. 5809 */ 5810 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, 5811 bool ethernet) 5812 { 5813 int err; 5814 5815 if (unlikely(!eth_p_mpls(skb->protocol))) 5816 return 0; 5817 5818 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN); 5819 if (unlikely(err)) 5820 return err; 5821 5822 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN); 5823 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb), 5824 mac_len); 5825 5826 __skb_pull(skb, MPLS_HLEN); 5827 skb_reset_mac_header(skb); 5828 skb_set_network_header(skb, mac_len); 5829 5830 if (ethernet && mac_len >= ETH_HLEN) { 5831 struct ethhdr *hdr; 5832 5833 /* use mpls_hdr() to get ethertype to account for VLANs. */ 5834 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN); 5835 skb_mod_eth_type(skb, hdr, next_proto); 5836 } 5837 skb->protocol = next_proto; 5838 5839 return 0; 5840 } 5841 EXPORT_SYMBOL_GPL(skb_mpls_pop); 5842 5843 /** 5844 * skb_mpls_update_lse() - modify outermost MPLS header and update csum 5845 * 5846 * @skb: buffer 5847 * @mpls_lse: new MPLS label stack entry to update to 5848 * 5849 * Expects skb->data at mac header. 5850 * 5851 * Returns 0 on success, -errno otherwise. 5852 */ 5853 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse) 5854 { 5855 int err; 5856 5857 if (unlikely(!eth_p_mpls(skb->protocol))) 5858 return -EINVAL; 5859 5860 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN); 5861 if (unlikely(err)) 5862 return err; 5863 5864 if (skb->ip_summed == CHECKSUM_COMPLETE) { 5865 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse }; 5866 5867 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum); 5868 } 5869 5870 mpls_hdr(skb)->label_stack_entry = mpls_lse; 5871 5872 return 0; 5873 } 5874 EXPORT_SYMBOL_GPL(skb_mpls_update_lse); 5875 5876 /** 5877 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header 5878 * 5879 * @skb: buffer 5880 * 5881 * Expects skb->data at mac header. 5882 * 5883 * Returns 0 on success, -errno otherwise. 5884 */ 5885 int skb_mpls_dec_ttl(struct sk_buff *skb) 5886 { 5887 u32 lse; 5888 u8 ttl; 5889 5890 if (unlikely(!eth_p_mpls(skb->protocol))) 5891 return -EINVAL; 5892 5893 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN)) 5894 return -ENOMEM; 5895 5896 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry); 5897 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT; 5898 if (!--ttl) 5899 return -EINVAL; 5900 5901 lse &= ~MPLS_LS_TTL_MASK; 5902 lse |= ttl << MPLS_LS_TTL_SHIFT; 5903 5904 return skb_mpls_update_lse(skb, cpu_to_be32(lse)); 5905 } 5906 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl); 5907 5908 /** 5909 * alloc_skb_with_frags - allocate skb with page frags 5910 * 5911 * @header_len: size of linear part 5912 * @data_len: needed length in frags 5913 * @max_page_order: max page order desired. 5914 * @errcode: pointer to error code if any 5915 * @gfp_mask: allocation mask 5916 * 5917 * This can be used to allocate a paged skb, given a maximal order for frags. 5918 */ 5919 struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 5920 unsigned long data_len, 5921 int max_page_order, 5922 int *errcode, 5923 gfp_t gfp_mask) 5924 { 5925 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 5926 unsigned long chunk; 5927 struct sk_buff *skb; 5928 struct page *page; 5929 int i; 5930 5931 *errcode = -EMSGSIZE; 5932 /* Note this test could be relaxed, if we succeed to allocate 5933 * high order pages... 5934 */ 5935 if (npages > MAX_SKB_FRAGS) 5936 return NULL; 5937 5938 *errcode = -ENOBUFS; 5939 skb = alloc_skb(header_len, gfp_mask); 5940 if (!skb) 5941 return NULL; 5942 5943 skb->truesize += npages << PAGE_SHIFT; 5944 5945 for (i = 0; npages > 0; i++) { 5946 int order = max_page_order; 5947 5948 while (order) { 5949 if (npages >= 1 << order) { 5950 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) | 5951 __GFP_COMP | 5952 __GFP_NOWARN, 5953 order); 5954 if (page) 5955 goto fill_page; 5956 /* Do not retry other high order allocations */ 5957 order = 1; 5958 max_page_order = 0; 5959 } 5960 order--; 5961 } 5962 page = alloc_page(gfp_mask); 5963 if (!page) 5964 goto failure; 5965 fill_page: 5966 chunk = min_t(unsigned long, data_len, 5967 PAGE_SIZE << order); 5968 skb_fill_page_desc(skb, i, page, 0, chunk); 5969 data_len -= chunk; 5970 npages -= 1 << order; 5971 } 5972 return skb; 5973 5974 failure: 5975 kfree_skb(skb); 5976 return NULL; 5977 } 5978 EXPORT_SYMBOL(alloc_skb_with_frags); 5979 5980 /* carve out the first off bytes from skb when off < headlen */ 5981 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, 5982 const int headlen, gfp_t gfp_mask) 5983 { 5984 int i; 5985 int size = skb_end_offset(skb); 5986 int new_hlen = headlen - off; 5987 u8 *data; 5988 5989 size = SKB_DATA_ALIGN(size); 5990 5991 if (skb_pfmemalloc(skb)) 5992 gfp_mask |= __GFP_MEMALLOC; 5993 data = kmalloc_reserve(size + 5994 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 5995 gfp_mask, NUMA_NO_NODE, NULL); 5996 if (!data) 5997 return -ENOMEM; 5998 5999 size = SKB_WITH_OVERHEAD(ksize(data)); 6000 6001 /* Copy real data, and all frags */ 6002 skb_copy_from_linear_data_offset(skb, off, data, new_hlen); 6003 skb->len -= off; 6004 6005 memcpy((struct skb_shared_info *)(data + size), 6006 skb_shinfo(skb), 6007 offsetof(struct skb_shared_info, 6008 frags[skb_shinfo(skb)->nr_frags])); 6009 if (skb_cloned(skb)) { 6010 /* drop the old head gracefully */ 6011 if (skb_orphan_frags(skb, gfp_mask)) { 6012 kfree(data); 6013 return -ENOMEM; 6014 } 6015 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 6016 skb_frag_ref(skb, i); 6017 if (skb_has_frag_list(skb)) 6018 skb_clone_fraglist(skb); 6019 skb_release_data(skb); 6020 } else { 6021 /* we can reuse existing recount- all we did was 6022 * relocate values 6023 */ 6024 skb_free_head(skb); 6025 } 6026 6027 skb->head = data; 6028 skb->data = data; 6029 skb->head_frag = 0; 6030 #ifdef NET_SKBUFF_DATA_USES_OFFSET 6031 skb->end = size; 6032 #else 6033 skb->end = skb->head + size; 6034 #endif 6035 skb_set_tail_pointer(skb, skb_headlen(skb)); 6036 skb_headers_offset_update(skb, 0); 6037 skb->cloned = 0; 6038 skb->hdr_len = 0; 6039 skb->nohdr = 0; 6040 atomic_set(&skb_shinfo(skb)->dataref, 1); 6041 6042 return 0; 6043 } 6044 6045 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp); 6046 6047 /* carve out the first eat bytes from skb's frag_list. May recurse into 6048 * pskb_carve() 6049 */ 6050 static int pskb_carve_frag_list(struct sk_buff *skb, 6051 struct skb_shared_info *shinfo, int eat, 6052 gfp_t gfp_mask) 6053 { 6054 struct sk_buff *list = shinfo->frag_list; 6055 struct sk_buff *clone = NULL; 6056 struct sk_buff *insp = NULL; 6057 6058 do { 6059 if (!list) { 6060 pr_err("Not enough bytes to eat. Want %d\n", eat); 6061 return -EFAULT; 6062 } 6063 if (list->len <= eat) { 6064 /* Eaten as whole. */ 6065 eat -= list->len; 6066 list = list->next; 6067 insp = list; 6068 } else { 6069 /* Eaten partially. */ 6070 if (skb_shared(list)) { 6071 clone = skb_clone(list, gfp_mask); 6072 if (!clone) 6073 return -ENOMEM; 6074 insp = list->next; 6075 list = clone; 6076 } else { 6077 /* This may be pulled without problems. */ 6078 insp = list; 6079 } 6080 if (pskb_carve(list, eat, gfp_mask) < 0) { 6081 kfree_skb(clone); 6082 return -ENOMEM; 6083 } 6084 break; 6085 } 6086 } while (eat); 6087 6088 /* Free pulled out fragments. */ 6089 while ((list = shinfo->frag_list) != insp) { 6090 shinfo->frag_list = list->next; 6091 kfree_skb(list); 6092 } 6093 /* And insert new clone at head. */ 6094 if (clone) { 6095 clone->next = list; 6096 shinfo->frag_list = clone; 6097 } 6098 return 0; 6099 } 6100 6101 /* carve off first len bytes from skb. Split line (off) is in the 6102 * non-linear part of skb 6103 */ 6104 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, 6105 int pos, gfp_t gfp_mask) 6106 { 6107 int i, k = 0; 6108 int size = skb_end_offset(skb); 6109 u8 *data; 6110 const int nfrags = skb_shinfo(skb)->nr_frags; 6111 struct skb_shared_info *shinfo; 6112 6113 size = SKB_DATA_ALIGN(size); 6114 6115 if (skb_pfmemalloc(skb)) 6116 gfp_mask |= __GFP_MEMALLOC; 6117 data = kmalloc_reserve(size + 6118 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 6119 gfp_mask, NUMA_NO_NODE, NULL); 6120 if (!data) 6121 return -ENOMEM; 6122 6123 size = SKB_WITH_OVERHEAD(ksize(data)); 6124 6125 memcpy((struct skb_shared_info *)(data + size), 6126 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); 6127 if (skb_orphan_frags(skb, gfp_mask)) { 6128 kfree(data); 6129 return -ENOMEM; 6130 } 6131 shinfo = (struct skb_shared_info *)(data + size); 6132 for (i = 0; i < nfrags; i++) { 6133 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]); 6134 6135 if (pos + fsize > off) { 6136 shinfo->frags[k] = skb_shinfo(skb)->frags[i]; 6137 6138 if (pos < off) { 6139 /* Split frag. 6140 * We have two variants in this case: 6141 * 1. Move all the frag to the second 6142 * part, if it is possible. F.e. 6143 * this approach is mandatory for TUX, 6144 * where splitting is expensive. 6145 * 2. Split is accurately. We make this. 6146 */ 6147 skb_frag_off_add(&shinfo->frags[0], off - pos); 6148 skb_frag_size_sub(&shinfo->frags[0], off - pos); 6149 } 6150 skb_frag_ref(skb, i); 6151 k++; 6152 } 6153 pos += fsize; 6154 } 6155 shinfo->nr_frags = k; 6156 if (skb_has_frag_list(skb)) 6157 skb_clone_fraglist(skb); 6158 6159 /* split line is in frag list */ 6160 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) { 6161 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */ 6162 if (skb_has_frag_list(skb)) 6163 kfree_skb_list(skb_shinfo(skb)->frag_list); 6164 kfree(data); 6165 return -ENOMEM; 6166 } 6167 skb_release_data(skb); 6168 6169 skb->head = data; 6170 skb->head_frag = 0; 6171 skb->data = data; 6172 #ifdef NET_SKBUFF_DATA_USES_OFFSET 6173 skb->end = size; 6174 #else 6175 skb->end = skb->head + size; 6176 #endif 6177 skb_reset_tail_pointer(skb); 6178 skb_headers_offset_update(skb, 0); 6179 skb->cloned = 0; 6180 skb->hdr_len = 0; 6181 skb->nohdr = 0; 6182 skb->len -= off; 6183 skb->data_len = skb->len; 6184 atomic_set(&skb_shinfo(skb)->dataref, 1); 6185 return 0; 6186 } 6187 6188 /* remove len bytes from the beginning of the skb */ 6189 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp) 6190 { 6191 int headlen = skb_headlen(skb); 6192 6193 if (len < headlen) 6194 return pskb_carve_inside_header(skb, len, headlen, gfp); 6195 else 6196 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp); 6197 } 6198 6199 /* Extract to_copy bytes starting at off from skb, and return this in 6200 * a new skb 6201 */ 6202 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, 6203 int to_copy, gfp_t gfp) 6204 { 6205 struct sk_buff *clone = skb_clone(skb, gfp); 6206 6207 if (!clone) 6208 return NULL; 6209 6210 if (pskb_carve(clone, off, gfp) < 0 || 6211 pskb_trim(clone, to_copy)) { 6212 kfree_skb(clone); 6213 return NULL; 6214 } 6215 return clone; 6216 } 6217 EXPORT_SYMBOL(pskb_extract); 6218 6219 /** 6220 * skb_condense - try to get rid of fragments/frag_list if possible 6221 * @skb: buffer 6222 * 6223 * Can be used to save memory before skb is added to a busy queue. 6224 * If packet has bytes in frags and enough tail room in skb->head, 6225 * pull all of them, so that we can free the frags right now and adjust 6226 * truesize. 6227 * Notes: 6228 * We do not reallocate skb->head thus can not fail. 6229 * Caller must re-evaluate skb->truesize if needed. 6230 */ 6231 void skb_condense(struct sk_buff *skb) 6232 { 6233 if (skb->data_len) { 6234 if (skb->data_len > skb->end - skb->tail || 6235 skb_cloned(skb)) 6236 return; 6237 6238 /* Nice, we can free page frag(s) right now */ 6239 __pskb_pull_tail(skb, skb->data_len); 6240 } 6241 /* At this point, skb->truesize might be over estimated, 6242 * because skb had a fragment, and fragments do not tell 6243 * their truesize. 6244 * When we pulled its content into skb->head, fragment 6245 * was freed, but __pskb_pull_tail() could not possibly 6246 * adjust skb->truesize, not knowing the frag truesize. 6247 */ 6248 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); 6249 } 6250 6251 #ifdef CONFIG_SKB_EXTENSIONS 6252 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id) 6253 { 6254 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE); 6255 } 6256 6257 /** 6258 * __skb_ext_alloc - allocate a new skb extensions storage 6259 * 6260 * @flags: See kmalloc(). 6261 * 6262 * Returns the newly allocated pointer. The pointer can later attached to a 6263 * skb via __skb_ext_set(). 6264 * Note: caller must handle the skb_ext as an opaque data. 6265 */ 6266 struct skb_ext *__skb_ext_alloc(gfp_t flags) 6267 { 6268 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags); 6269 6270 if (new) { 6271 memset(new->offset, 0, sizeof(new->offset)); 6272 refcount_set(&new->refcnt, 1); 6273 } 6274 6275 return new; 6276 } 6277 6278 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old, 6279 unsigned int old_active) 6280 { 6281 struct skb_ext *new; 6282 6283 if (refcount_read(&old->refcnt) == 1) 6284 return old; 6285 6286 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC); 6287 if (!new) 6288 return NULL; 6289 6290 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE); 6291 refcount_set(&new->refcnt, 1); 6292 6293 #ifdef CONFIG_XFRM 6294 if (old_active & (1 << SKB_EXT_SEC_PATH)) { 6295 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH); 6296 unsigned int i; 6297 6298 for (i = 0; i < sp->len; i++) 6299 xfrm_state_hold(sp->xvec[i]); 6300 } 6301 #endif 6302 __skb_ext_put(old); 6303 return new; 6304 } 6305 6306 /** 6307 * __skb_ext_set - attach the specified extension storage to this skb 6308 * @skb: buffer 6309 * @id: extension id 6310 * @ext: extension storage previously allocated via __skb_ext_alloc() 6311 * 6312 * Existing extensions, if any, are cleared. 6313 * 6314 * Returns the pointer to the extension. 6315 */ 6316 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id, 6317 struct skb_ext *ext) 6318 { 6319 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext); 6320 6321 skb_ext_put(skb); 6322 newlen = newoff + skb_ext_type_len[id]; 6323 ext->chunks = newlen; 6324 ext->offset[id] = newoff; 6325 skb->extensions = ext; 6326 skb->active_extensions = 1 << id; 6327 return skb_ext_get_ptr(ext, id); 6328 } 6329 6330 /** 6331 * skb_ext_add - allocate space for given extension, COW if needed 6332 * @skb: buffer 6333 * @id: extension to allocate space for 6334 * 6335 * Allocates enough space for the given extension. 6336 * If the extension is already present, a pointer to that extension 6337 * is returned. 6338 * 6339 * If the skb was cloned, COW applies and the returned memory can be 6340 * modified without changing the extension space of clones buffers. 6341 * 6342 * Returns pointer to the extension or NULL on allocation failure. 6343 */ 6344 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id) 6345 { 6346 struct skb_ext *new, *old = NULL; 6347 unsigned int newlen, newoff; 6348 6349 if (skb->active_extensions) { 6350 old = skb->extensions; 6351 6352 new = skb_ext_maybe_cow(old, skb->active_extensions); 6353 if (!new) 6354 return NULL; 6355 6356 if (__skb_ext_exist(new, id)) 6357 goto set_active; 6358 6359 newoff = new->chunks; 6360 } else { 6361 newoff = SKB_EXT_CHUNKSIZEOF(*new); 6362 6363 new = __skb_ext_alloc(GFP_ATOMIC); 6364 if (!new) 6365 return NULL; 6366 } 6367 6368 newlen = newoff + skb_ext_type_len[id]; 6369 new->chunks = newlen; 6370 new->offset[id] = newoff; 6371 set_active: 6372 skb->extensions = new; 6373 skb->active_extensions |= 1 << id; 6374 return skb_ext_get_ptr(new, id); 6375 } 6376 EXPORT_SYMBOL(skb_ext_add); 6377 6378 #ifdef CONFIG_XFRM 6379 static void skb_ext_put_sp(struct sec_path *sp) 6380 { 6381 unsigned int i; 6382 6383 for (i = 0; i < sp->len; i++) 6384 xfrm_state_put(sp->xvec[i]); 6385 } 6386 #endif 6387 6388 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id) 6389 { 6390 struct skb_ext *ext = skb->extensions; 6391 6392 skb->active_extensions &= ~(1 << id); 6393 if (skb->active_extensions == 0) { 6394 skb->extensions = NULL; 6395 __skb_ext_put(ext); 6396 #ifdef CONFIG_XFRM 6397 } else if (id == SKB_EXT_SEC_PATH && 6398 refcount_read(&ext->refcnt) == 1) { 6399 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH); 6400 6401 skb_ext_put_sp(sp); 6402 sp->len = 0; 6403 #endif 6404 } 6405 } 6406 EXPORT_SYMBOL(__skb_ext_del); 6407 6408 void __skb_ext_put(struct skb_ext *ext) 6409 { 6410 /* If this is last clone, nothing can increment 6411 * it after check passes. Avoids one atomic op. 6412 */ 6413 if (refcount_read(&ext->refcnt) == 1) 6414 goto free_now; 6415 6416 if (!refcount_dec_and_test(&ext->refcnt)) 6417 return; 6418 free_now: 6419 #ifdef CONFIG_XFRM 6420 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH)) 6421 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH)); 6422 #endif 6423 6424 kmem_cache_free(skbuff_ext_cache, ext); 6425 } 6426 EXPORT_SYMBOL(__skb_ext_put); 6427 #endif /* CONFIG_SKB_EXTENSIONS */ 6428