1 /* 2 * Routines having to do with the 'struct sk_buff' memory handlers. 3 * 4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 5 * Florian La Roche <rzsfl@rz.uni-sb.de> 6 * 7 * Fixes: 8 * Alan Cox : Fixed the worst of the load 9 * balancer bugs. 10 * Dave Platt : Interrupt stacking fix. 11 * Richard Kooijman : Timestamp fixes. 12 * Alan Cox : Changed buffer format. 13 * Alan Cox : destructor hook for AF_UNIX etc. 14 * Linus Torvalds : Better skb_clone. 15 * Alan Cox : Added skb_copy. 16 * Alan Cox : Added all the changed routines Linus 17 * only put in the headers 18 * Ray VanTassle : Fixed --skb->lock in free 19 * Alan Cox : skb_copy copy arp field 20 * Andi Kleen : slabified it. 21 * Robert Olsson : Removed skb_head_pool 22 * 23 * NOTE: 24 * The __skb_ routines should be called with interrupts 25 * disabled, or you better be *real* sure that the operation is atomic 26 * with respect to whatever list is being frobbed (e.g. via lock_sock() 27 * or via disabling bottom half handlers, etc). 28 * 29 * This program is free software; you can redistribute it and/or 30 * modify it under the terms of the GNU General Public License 31 * as published by the Free Software Foundation; either version 32 * 2 of the License, or (at your option) any later version. 33 */ 34 35 /* 36 * The functions in this file will not compile correctly with gcc 2.4.x 37 */ 38 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 40 41 #include <linux/module.h> 42 #include <linux/types.h> 43 #include <linux/kernel.h> 44 #include <linux/kmemcheck.h> 45 #include <linux/mm.h> 46 #include <linux/interrupt.h> 47 #include <linux/in.h> 48 #include <linux/inet.h> 49 #include <linux/slab.h> 50 #include <linux/netdevice.h> 51 #ifdef CONFIG_NET_CLS_ACT 52 #include <net/pkt_sched.h> 53 #endif 54 #include <linux/string.h> 55 #include <linux/skbuff.h> 56 #include <linux/splice.h> 57 #include <linux/cache.h> 58 #include <linux/rtnetlink.h> 59 #include <linux/init.h> 60 #include <linux/scatterlist.h> 61 #include <linux/errqueue.h> 62 #include <linux/prefetch.h> 63 64 #include <net/protocol.h> 65 #include <net/dst.h> 66 #include <net/sock.h> 67 #include <net/checksum.h> 68 #include <net/xfrm.h> 69 70 #include <asm/uaccess.h> 71 #include <trace/events/skb.h> 72 #include <linux/highmem.h> 73 74 struct kmem_cache *skbuff_head_cache __read_mostly; 75 static struct kmem_cache *skbuff_fclone_cache __read_mostly; 76 77 static void sock_pipe_buf_release(struct pipe_inode_info *pipe, 78 struct pipe_buffer *buf) 79 { 80 put_page(buf->page); 81 } 82 83 static void sock_pipe_buf_get(struct pipe_inode_info *pipe, 84 struct pipe_buffer *buf) 85 { 86 get_page(buf->page); 87 } 88 89 static int sock_pipe_buf_steal(struct pipe_inode_info *pipe, 90 struct pipe_buffer *buf) 91 { 92 return 1; 93 } 94 95 96 /* Pipe buffer operations for a socket. */ 97 static const struct pipe_buf_operations sock_pipe_buf_ops = { 98 .can_merge = 0, 99 .map = generic_pipe_buf_map, 100 .unmap = generic_pipe_buf_unmap, 101 .confirm = generic_pipe_buf_confirm, 102 .release = sock_pipe_buf_release, 103 .steal = sock_pipe_buf_steal, 104 .get = sock_pipe_buf_get, 105 }; 106 107 /** 108 * skb_panic - private function for out-of-line support 109 * @skb: buffer 110 * @sz: size 111 * @addr: address 112 * @msg: skb_over_panic or skb_under_panic 113 * 114 * Out-of-line support for skb_put() and skb_push(). 115 * Called via the wrapper skb_over_panic() or skb_under_panic(). 116 * Keep out of line to prevent kernel bloat. 117 * __builtin_return_address is not used because it is not always reliable. 118 */ 119 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 120 const char msg[]) 121 { 122 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", 123 msg, addr, skb->len, sz, skb->head, skb->data, 124 (unsigned long)skb->tail, (unsigned long)skb->end, 125 skb->dev ? skb->dev->name : "<NULL>"); 126 BUG(); 127 } 128 129 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 130 { 131 skb_panic(skb, sz, addr, __func__); 132 } 133 134 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 135 { 136 skb_panic(skb, sz, addr, __func__); 137 } 138 139 /* 140 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 141 * the caller if emergency pfmemalloc reserves are being used. If it is and 142 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 143 * may be used. Otherwise, the packet data may be discarded until enough 144 * memory is free 145 */ 146 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \ 147 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc) 148 149 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, 150 unsigned long ip, bool *pfmemalloc) 151 { 152 void *obj; 153 bool ret_pfmemalloc = false; 154 155 /* 156 * Try a regular allocation, when that fails and we're not entitled 157 * to the reserves, fail. 158 */ 159 obj = kmalloc_node_track_caller(size, 160 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 161 node); 162 if (obj || !(gfp_pfmemalloc_allowed(flags))) 163 goto out; 164 165 /* Try again but now we are using pfmemalloc reserves */ 166 ret_pfmemalloc = true; 167 obj = kmalloc_node_track_caller(size, flags, node); 168 169 out: 170 if (pfmemalloc) 171 *pfmemalloc = ret_pfmemalloc; 172 173 return obj; 174 } 175 176 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 177 * 'private' fields and also do memory statistics to find all the 178 * [BEEP] leaks. 179 * 180 */ 181 182 struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node) 183 { 184 struct sk_buff *skb; 185 186 /* Get the HEAD */ 187 skb = kmem_cache_alloc_node(skbuff_head_cache, 188 gfp_mask & ~__GFP_DMA, node); 189 if (!skb) 190 goto out; 191 192 /* 193 * Only clear those fields we need to clear, not those that we will 194 * actually initialise below. Hence, don't put any more fields after 195 * the tail pointer in struct sk_buff! 196 */ 197 memset(skb, 0, offsetof(struct sk_buff, tail)); 198 skb->head = NULL; 199 skb->truesize = sizeof(struct sk_buff); 200 atomic_set(&skb->users, 1); 201 202 #ifdef NET_SKBUFF_DATA_USES_OFFSET 203 skb->mac_header = ~0U; 204 #endif 205 out: 206 return skb; 207 } 208 209 /** 210 * __alloc_skb - allocate a network buffer 211 * @size: size to allocate 212 * @gfp_mask: allocation mask 213 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 214 * instead of head cache and allocate a cloned (child) skb. 215 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 216 * allocations in case the data is required for writeback 217 * @node: numa node to allocate memory on 218 * 219 * Allocate a new &sk_buff. The returned buffer has no headroom and a 220 * tail room of at least size bytes. The object has a reference count 221 * of one. The return is the buffer. On a failure the return is %NULL. 222 * 223 * Buffers may only be allocated from interrupts using a @gfp_mask of 224 * %GFP_ATOMIC. 225 */ 226 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 227 int flags, int node) 228 { 229 struct kmem_cache *cache; 230 struct skb_shared_info *shinfo; 231 struct sk_buff *skb; 232 u8 *data; 233 bool pfmemalloc; 234 235 cache = (flags & SKB_ALLOC_FCLONE) 236 ? skbuff_fclone_cache : skbuff_head_cache; 237 238 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 239 gfp_mask |= __GFP_MEMALLOC; 240 241 /* Get the HEAD */ 242 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 243 if (!skb) 244 goto out; 245 prefetchw(skb); 246 247 /* We do our best to align skb_shared_info on a separate cache 248 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 249 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 250 * Both skb->head and skb_shared_info are cache line aligned. 251 */ 252 size = SKB_DATA_ALIGN(size); 253 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 254 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 255 if (!data) 256 goto nodata; 257 /* kmalloc(size) might give us more room than requested. 258 * Put skb_shared_info exactly at the end of allocated zone, 259 * to allow max possible filling before reallocation. 260 */ 261 size = SKB_WITH_OVERHEAD(ksize(data)); 262 prefetchw(data + size); 263 264 /* 265 * Only clear those fields we need to clear, not those that we will 266 * actually initialise below. Hence, don't put any more fields after 267 * the tail pointer in struct sk_buff! 268 */ 269 memset(skb, 0, offsetof(struct sk_buff, tail)); 270 /* Account for allocated memory : skb + skb->head */ 271 skb->truesize = SKB_TRUESIZE(size); 272 skb->pfmemalloc = pfmemalloc; 273 atomic_set(&skb->users, 1); 274 skb->head = data; 275 skb->data = data; 276 skb_reset_tail_pointer(skb); 277 skb->end = skb->tail + size; 278 #ifdef NET_SKBUFF_DATA_USES_OFFSET 279 skb->mac_header = ~0U; 280 skb->transport_header = ~0U; 281 #endif 282 283 /* make sure we initialize shinfo sequentially */ 284 shinfo = skb_shinfo(skb); 285 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 286 atomic_set(&shinfo->dataref, 1); 287 kmemcheck_annotate_variable(shinfo->destructor_arg); 288 289 if (flags & SKB_ALLOC_FCLONE) { 290 struct sk_buff *child = skb + 1; 291 atomic_t *fclone_ref = (atomic_t *) (child + 1); 292 293 kmemcheck_annotate_bitfield(child, flags1); 294 kmemcheck_annotate_bitfield(child, flags2); 295 skb->fclone = SKB_FCLONE_ORIG; 296 atomic_set(fclone_ref, 1); 297 298 child->fclone = SKB_FCLONE_UNAVAILABLE; 299 child->pfmemalloc = pfmemalloc; 300 } 301 out: 302 return skb; 303 nodata: 304 kmem_cache_free(cache, skb); 305 skb = NULL; 306 goto out; 307 } 308 EXPORT_SYMBOL(__alloc_skb); 309 310 /** 311 * build_skb - build a network buffer 312 * @data: data buffer provided by caller 313 * @frag_size: size of fragment, or 0 if head was kmalloced 314 * 315 * Allocate a new &sk_buff. Caller provides space holding head and 316 * skb_shared_info. @data must have been allocated by kmalloc() 317 * The return is the new skb buffer. 318 * On a failure the return is %NULL, and @data is not freed. 319 * Notes : 320 * Before IO, driver allocates only data buffer where NIC put incoming frame 321 * Driver should add room at head (NET_SKB_PAD) and 322 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 323 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 324 * before giving packet to stack. 325 * RX rings only contains data buffers, not full skbs. 326 */ 327 struct sk_buff *build_skb(void *data, unsigned int frag_size) 328 { 329 struct skb_shared_info *shinfo; 330 struct sk_buff *skb; 331 unsigned int size = frag_size ? : ksize(data); 332 333 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 334 if (!skb) 335 return NULL; 336 337 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 338 339 memset(skb, 0, offsetof(struct sk_buff, tail)); 340 skb->truesize = SKB_TRUESIZE(size); 341 skb->head_frag = frag_size != 0; 342 atomic_set(&skb->users, 1); 343 skb->head = data; 344 skb->data = data; 345 skb_reset_tail_pointer(skb); 346 skb->end = skb->tail + size; 347 #ifdef NET_SKBUFF_DATA_USES_OFFSET 348 skb->mac_header = ~0U; 349 skb->transport_header = ~0U; 350 #endif 351 352 /* make sure we initialize shinfo sequentially */ 353 shinfo = skb_shinfo(skb); 354 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 355 atomic_set(&shinfo->dataref, 1); 356 kmemcheck_annotate_variable(shinfo->destructor_arg); 357 358 return skb; 359 } 360 EXPORT_SYMBOL(build_skb); 361 362 struct netdev_alloc_cache { 363 struct page_frag frag; 364 /* we maintain a pagecount bias, so that we dont dirty cache line 365 * containing page->_count every time we allocate a fragment. 366 */ 367 unsigned int pagecnt_bias; 368 }; 369 static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache); 370 371 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 372 { 373 struct netdev_alloc_cache *nc; 374 void *data = NULL; 375 int order; 376 unsigned long flags; 377 378 local_irq_save(flags); 379 nc = &__get_cpu_var(netdev_alloc_cache); 380 if (unlikely(!nc->frag.page)) { 381 refill: 382 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) { 383 gfp_t gfp = gfp_mask; 384 385 if (order) 386 gfp |= __GFP_COMP | __GFP_NOWARN; 387 nc->frag.page = alloc_pages(gfp, order); 388 if (likely(nc->frag.page)) 389 break; 390 if (--order < 0) 391 goto end; 392 } 393 nc->frag.size = PAGE_SIZE << order; 394 recycle: 395 atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS); 396 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS; 397 nc->frag.offset = 0; 398 } 399 400 if (nc->frag.offset + fragsz > nc->frag.size) { 401 /* avoid unnecessary locked operations if possible */ 402 if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) || 403 atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count)) 404 goto recycle; 405 goto refill; 406 } 407 408 data = page_address(nc->frag.page) + nc->frag.offset; 409 nc->frag.offset += fragsz; 410 nc->pagecnt_bias--; 411 end: 412 local_irq_restore(flags); 413 return data; 414 } 415 416 /** 417 * netdev_alloc_frag - allocate a page fragment 418 * @fragsz: fragment size 419 * 420 * Allocates a frag from a page for receive buffer. 421 * Uses GFP_ATOMIC allocations. 422 */ 423 void *netdev_alloc_frag(unsigned int fragsz) 424 { 425 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); 426 } 427 EXPORT_SYMBOL(netdev_alloc_frag); 428 429 /** 430 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 431 * @dev: network device to receive on 432 * @length: length to allocate 433 * @gfp_mask: get_free_pages mask, passed to alloc_skb 434 * 435 * Allocate a new &sk_buff and assign it a usage count of one. The 436 * buffer has unspecified headroom built in. Users should allocate 437 * the headroom they think they need without accounting for the 438 * built in space. The built in space is used for optimisations. 439 * 440 * %NULL is returned if there is no free memory. 441 */ 442 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 443 unsigned int length, gfp_t gfp_mask) 444 { 445 struct sk_buff *skb = NULL; 446 unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) + 447 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 448 449 if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) { 450 void *data; 451 452 if (sk_memalloc_socks()) 453 gfp_mask |= __GFP_MEMALLOC; 454 455 data = __netdev_alloc_frag(fragsz, gfp_mask); 456 457 if (likely(data)) { 458 skb = build_skb(data, fragsz); 459 if (unlikely(!skb)) 460 put_page(virt_to_head_page(data)); 461 } 462 } else { 463 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 464 SKB_ALLOC_RX, NUMA_NO_NODE); 465 } 466 if (likely(skb)) { 467 skb_reserve(skb, NET_SKB_PAD); 468 skb->dev = dev; 469 } 470 return skb; 471 } 472 EXPORT_SYMBOL(__netdev_alloc_skb); 473 474 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 475 int size, unsigned int truesize) 476 { 477 skb_fill_page_desc(skb, i, page, off, size); 478 skb->len += size; 479 skb->data_len += size; 480 skb->truesize += truesize; 481 } 482 EXPORT_SYMBOL(skb_add_rx_frag); 483 484 static void skb_drop_list(struct sk_buff **listp) 485 { 486 kfree_skb_list(*listp); 487 *listp = NULL; 488 } 489 490 static inline void skb_drop_fraglist(struct sk_buff *skb) 491 { 492 skb_drop_list(&skb_shinfo(skb)->frag_list); 493 } 494 495 static void skb_clone_fraglist(struct sk_buff *skb) 496 { 497 struct sk_buff *list; 498 499 skb_walk_frags(skb, list) 500 skb_get(list); 501 } 502 503 static void skb_free_head(struct sk_buff *skb) 504 { 505 if (skb->head_frag) 506 put_page(virt_to_head_page(skb->head)); 507 else 508 kfree(skb->head); 509 } 510 511 static void skb_release_data(struct sk_buff *skb) 512 { 513 if (!skb->cloned || 514 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 515 &skb_shinfo(skb)->dataref)) { 516 if (skb_shinfo(skb)->nr_frags) { 517 int i; 518 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 519 skb_frag_unref(skb, i); 520 } 521 522 /* 523 * If skb buf is from userspace, we need to notify the caller 524 * the lower device DMA has done; 525 */ 526 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) { 527 struct ubuf_info *uarg; 528 529 uarg = skb_shinfo(skb)->destructor_arg; 530 if (uarg->callback) 531 uarg->callback(uarg, true); 532 } 533 534 if (skb_has_frag_list(skb)) 535 skb_drop_fraglist(skb); 536 537 skb_free_head(skb); 538 } 539 } 540 541 /* 542 * Free an skbuff by memory without cleaning the state. 543 */ 544 static void kfree_skbmem(struct sk_buff *skb) 545 { 546 struct sk_buff *other; 547 atomic_t *fclone_ref; 548 549 switch (skb->fclone) { 550 case SKB_FCLONE_UNAVAILABLE: 551 kmem_cache_free(skbuff_head_cache, skb); 552 break; 553 554 case SKB_FCLONE_ORIG: 555 fclone_ref = (atomic_t *) (skb + 2); 556 if (atomic_dec_and_test(fclone_ref)) 557 kmem_cache_free(skbuff_fclone_cache, skb); 558 break; 559 560 case SKB_FCLONE_CLONE: 561 fclone_ref = (atomic_t *) (skb + 1); 562 other = skb - 1; 563 564 /* The clone portion is available for 565 * fast-cloning again. 566 */ 567 skb->fclone = SKB_FCLONE_UNAVAILABLE; 568 569 if (atomic_dec_and_test(fclone_ref)) 570 kmem_cache_free(skbuff_fclone_cache, other); 571 break; 572 } 573 } 574 575 static void skb_release_head_state(struct sk_buff *skb) 576 { 577 skb_dst_drop(skb); 578 #ifdef CONFIG_XFRM 579 secpath_put(skb->sp); 580 #endif 581 if (skb->destructor) { 582 WARN_ON(in_irq()); 583 skb->destructor(skb); 584 } 585 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 586 nf_conntrack_put(skb->nfct); 587 #endif 588 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED 589 nf_conntrack_put_reasm(skb->nfct_reasm); 590 #endif 591 #ifdef CONFIG_BRIDGE_NETFILTER 592 nf_bridge_put(skb->nf_bridge); 593 #endif 594 /* XXX: IS this still necessary? - JHS */ 595 #ifdef CONFIG_NET_SCHED 596 skb->tc_index = 0; 597 #ifdef CONFIG_NET_CLS_ACT 598 skb->tc_verd = 0; 599 #endif 600 #endif 601 } 602 603 /* Free everything but the sk_buff shell. */ 604 static void skb_release_all(struct sk_buff *skb) 605 { 606 skb_release_head_state(skb); 607 if (likely(skb->head)) 608 skb_release_data(skb); 609 } 610 611 /** 612 * __kfree_skb - private function 613 * @skb: buffer 614 * 615 * Free an sk_buff. Release anything attached to the buffer. 616 * Clean the state. This is an internal helper function. Users should 617 * always call kfree_skb 618 */ 619 620 void __kfree_skb(struct sk_buff *skb) 621 { 622 skb_release_all(skb); 623 kfree_skbmem(skb); 624 } 625 EXPORT_SYMBOL(__kfree_skb); 626 627 /** 628 * kfree_skb - free an sk_buff 629 * @skb: buffer to free 630 * 631 * Drop a reference to the buffer and free it if the usage count has 632 * hit zero. 633 */ 634 void kfree_skb(struct sk_buff *skb) 635 { 636 if (unlikely(!skb)) 637 return; 638 if (likely(atomic_read(&skb->users) == 1)) 639 smp_rmb(); 640 else if (likely(!atomic_dec_and_test(&skb->users))) 641 return; 642 trace_kfree_skb(skb, __builtin_return_address(0)); 643 __kfree_skb(skb); 644 } 645 EXPORT_SYMBOL(kfree_skb); 646 647 void kfree_skb_list(struct sk_buff *segs) 648 { 649 while (segs) { 650 struct sk_buff *next = segs->next; 651 652 kfree_skb(segs); 653 segs = next; 654 } 655 } 656 EXPORT_SYMBOL(kfree_skb_list); 657 658 /** 659 * skb_tx_error - report an sk_buff xmit error 660 * @skb: buffer that triggered an error 661 * 662 * Report xmit error if a device callback is tracking this skb. 663 * skb must be freed afterwards. 664 */ 665 void skb_tx_error(struct sk_buff *skb) 666 { 667 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) { 668 struct ubuf_info *uarg; 669 670 uarg = skb_shinfo(skb)->destructor_arg; 671 if (uarg->callback) 672 uarg->callback(uarg, false); 673 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 674 } 675 } 676 EXPORT_SYMBOL(skb_tx_error); 677 678 /** 679 * consume_skb - free an skbuff 680 * @skb: buffer to free 681 * 682 * Drop a ref to the buffer and free it if the usage count has hit zero 683 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 684 * is being dropped after a failure and notes that 685 */ 686 void consume_skb(struct sk_buff *skb) 687 { 688 if (unlikely(!skb)) 689 return; 690 if (likely(atomic_read(&skb->users) == 1)) 691 smp_rmb(); 692 else if (likely(!atomic_dec_and_test(&skb->users))) 693 return; 694 trace_consume_skb(skb); 695 __kfree_skb(skb); 696 } 697 EXPORT_SYMBOL(consume_skb); 698 699 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 700 { 701 new->tstamp = old->tstamp; 702 new->dev = old->dev; 703 new->transport_header = old->transport_header; 704 new->network_header = old->network_header; 705 new->mac_header = old->mac_header; 706 new->inner_transport_header = old->inner_transport_header; 707 new->inner_network_header = old->inner_network_header; 708 new->inner_mac_header = old->inner_mac_header; 709 skb_dst_copy(new, old); 710 new->rxhash = old->rxhash; 711 new->ooo_okay = old->ooo_okay; 712 new->l4_rxhash = old->l4_rxhash; 713 new->no_fcs = old->no_fcs; 714 new->encapsulation = old->encapsulation; 715 #ifdef CONFIG_XFRM 716 new->sp = secpath_get(old->sp); 717 #endif 718 memcpy(new->cb, old->cb, sizeof(old->cb)); 719 new->csum = old->csum; 720 new->local_df = old->local_df; 721 new->pkt_type = old->pkt_type; 722 new->ip_summed = old->ip_summed; 723 skb_copy_queue_mapping(new, old); 724 new->priority = old->priority; 725 #if IS_ENABLED(CONFIG_IP_VS) 726 new->ipvs_property = old->ipvs_property; 727 #endif 728 new->pfmemalloc = old->pfmemalloc; 729 new->protocol = old->protocol; 730 new->mark = old->mark; 731 new->skb_iif = old->skb_iif; 732 __nf_copy(new, old); 733 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) 734 new->nf_trace = old->nf_trace; 735 #endif 736 #ifdef CONFIG_NET_SCHED 737 new->tc_index = old->tc_index; 738 #ifdef CONFIG_NET_CLS_ACT 739 new->tc_verd = old->tc_verd; 740 #endif 741 #endif 742 new->vlan_proto = old->vlan_proto; 743 new->vlan_tci = old->vlan_tci; 744 745 skb_copy_secmark(new, old); 746 } 747 748 /* 749 * You should not add any new code to this function. Add it to 750 * __copy_skb_header above instead. 751 */ 752 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 753 { 754 #define C(x) n->x = skb->x 755 756 n->next = n->prev = NULL; 757 n->sk = NULL; 758 __copy_skb_header(n, skb); 759 760 C(len); 761 C(data_len); 762 C(mac_len); 763 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 764 n->cloned = 1; 765 n->nohdr = 0; 766 n->destructor = NULL; 767 C(tail); 768 C(end); 769 C(head); 770 C(head_frag); 771 C(data); 772 C(truesize); 773 atomic_set(&n->users, 1); 774 775 atomic_inc(&(skb_shinfo(skb)->dataref)); 776 skb->cloned = 1; 777 778 return n; 779 #undef C 780 } 781 782 /** 783 * skb_morph - morph one skb into another 784 * @dst: the skb to receive the contents 785 * @src: the skb to supply the contents 786 * 787 * This is identical to skb_clone except that the target skb is 788 * supplied by the user. 789 * 790 * The target skb is returned upon exit. 791 */ 792 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 793 { 794 skb_release_all(dst); 795 return __skb_clone(dst, src); 796 } 797 EXPORT_SYMBOL_GPL(skb_morph); 798 799 /** 800 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 801 * @skb: the skb to modify 802 * @gfp_mask: allocation priority 803 * 804 * This must be called on SKBTX_DEV_ZEROCOPY skb. 805 * It will copy all frags into kernel and drop the reference 806 * to userspace pages. 807 * 808 * If this function is called from an interrupt gfp_mask() must be 809 * %GFP_ATOMIC. 810 * 811 * Returns 0 on success or a negative error code on failure 812 * to allocate kernel memory to copy to. 813 */ 814 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 815 { 816 int i; 817 int num_frags = skb_shinfo(skb)->nr_frags; 818 struct page *page, *head = NULL; 819 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg; 820 821 for (i = 0; i < num_frags; i++) { 822 u8 *vaddr; 823 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 824 825 page = alloc_page(gfp_mask); 826 if (!page) { 827 while (head) { 828 struct page *next = (struct page *)head->private; 829 put_page(head); 830 head = next; 831 } 832 return -ENOMEM; 833 } 834 vaddr = kmap_atomic(skb_frag_page(f)); 835 memcpy(page_address(page), 836 vaddr + f->page_offset, skb_frag_size(f)); 837 kunmap_atomic(vaddr); 838 page->private = (unsigned long)head; 839 head = page; 840 } 841 842 /* skb frags release userspace buffers */ 843 for (i = 0; i < num_frags; i++) 844 skb_frag_unref(skb, i); 845 846 uarg->callback(uarg, false); 847 848 /* skb frags point to kernel buffers */ 849 for (i = num_frags - 1; i >= 0; i--) { 850 __skb_fill_page_desc(skb, i, head, 0, 851 skb_shinfo(skb)->frags[i].size); 852 head = (struct page *)head->private; 853 } 854 855 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 856 return 0; 857 } 858 EXPORT_SYMBOL_GPL(skb_copy_ubufs); 859 860 /** 861 * skb_clone - duplicate an sk_buff 862 * @skb: buffer to clone 863 * @gfp_mask: allocation priority 864 * 865 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 866 * copies share the same packet data but not structure. The new 867 * buffer has a reference count of 1. If the allocation fails the 868 * function returns %NULL otherwise the new buffer is returned. 869 * 870 * If this function is called from an interrupt gfp_mask() must be 871 * %GFP_ATOMIC. 872 */ 873 874 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 875 { 876 struct sk_buff *n; 877 878 if (skb_orphan_frags(skb, gfp_mask)) 879 return NULL; 880 881 n = skb + 1; 882 if (skb->fclone == SKB_FCLONE_ORIG && 883 n->fclone == SKB_FCLONE_UNAVAILABLE) { 884 atomic_t *fclone_ref = (atomic_t *) (n + 1); 885 n->fclone = SKB_FCLONE_CLONE; 886 atomic_inc(fclone_ref); 887 } else { 888 if (skb_pfmemalloc(skb)) 889 gfp_mask |= __GFP_MEMALLOC; 890 891 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 892 if (!n) 893 return NULL; 894 895 kmemcheck_annotate_bitfield(n, flags1); 896 kmemcheck_annotate_bitfield(n, flags2); 897 n->fclone = SKB_FCLONE_UNAVAILABLE; 898 } 899 900 return __skb_clone(n, skb); 901 } 902 EXPORT_SYMBOL(skb_clone); 903 904 static void skb_headers_offset_update(struct sk_buff *skb, int off) 905 { 906 /* {transport,network,mac}_header and tail are relative to skb->head */ 907 skb->transport_header += off; 908 skb->network_header += off; 909 if (skb_mac_header_was_set(skb)) 910 skb->mac_header += off; 911 skb->inner_transport_header += off; 912 skb->inner_network_header += off; 913 skb->inner_mac_header += off; 914 } 915 916 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 917 { 918 #ifndef NET_SKBUFF_DATA_USES_OFFSET 919 /* 920 * Shift between the two data areas in bytes 921 */ 922 unsigned long offset = new->data - old->data; 923 #endif 924 925 __copy_skb_header(new, old); 926 927 #ifndef NET_SKBUFF_DATA_USES_OFFSET 928 skb_headers_offset_update(new, offset); 929 #endif 930 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 931 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 932 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 933 } 934 935 static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 936 { 937 if (skb_pfmemalloc(skb)) 938 return SKB_ALLOC_RX; 939 return 0; 940 } 941 942 /** 943 * skb_copy - create private copy of an sk_buff 944 * @skb: buffer to copy 945 * @gfp_mask: allocation priority 946 * 947 * Make a copy of both an &sk_buff and its data. This is used when the 948 * caller wishes to modify the data and needs a private copy of the 949 * data to alter. Returns %NULL on failure or the pointer to the buffer 950 * on success. The returned buffer has a reference count of 1. 951 * 952 * As by-product this function converts non-linear &sk_buff to linear 953 * one, so that &sk_buff becomes completely private and caller is allowed 954 * to modify all the data of returned buffer. This means that this 955 * function is not recommended for use in circumstances when only 956 * header is going to be modified. Use pskb_copy() instead. 957 */ 958 959 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 960 { 961 int headerlen = skb_headroom(skb); 962 unsigned int size = skb_end_offset(skb) + skb->data_len; 963 struct sk_buff *n = __alloc_skb(size, gfp_mask, 964 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 965 966 if (!n) 967 return NULL; 968 969 /* Set the data pointer */ 970 skb_reserve(n, headerlen); 971 /* Set the tail pointer and length */ 972 skb_put(n, skb->len); 973 974 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)) 975 BUG(); 976 977 copy_skb_header(n, skb); 978 return n; 979 } 980 EXPORT_SYMBOL(skb_copy); 981 982 /** 983 * __pskb_copy - create copy of an sk_buff with private head. 984 * @skb: buffer to copy 985 * @headroom: headroom of new skb 986 * @gfp_mask: allocation priority 987 * 988 * Make a copy of both an &sk_buff and part of its data, located 989 * in header. Fragmented data remain shared. This is used when 990 * the caller wishes to modify only header of &sk_buff and needs 991 * private copy of the header to alter. Returns %NULL on failure 992 * or the pointer to the buffer on success. 993 * The returned buffer has a reference count of 1. 994 */ 995 996 struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask) 997 { 998 unsigned int size = skb_headlen(skb) + headroom; 999 struct sk_buff *n = __alloc_skb(size, gfp_mask, 1000 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 1001 1002 if (!n) 1003 goto out; 1004 1005 /* Set the data pointer */ 1006 skb_reserve(n, headroom); 1007 /* Set the tail pointer and length */ 1008 skb_put(n, skb_headlen(skb)); 1009 /* Copy the bytes */ 1010 skb_copy_from_linear_data(skb, n->data, n->len); 1011 1012 n->truesize += skb->data_len; 1013 n->data_len = skb->data_len; 1014 n->len = skb->len; 1015 1016 if (skb_shinfo(skb)->nr_frags) { 1017 int i; 1018 1019 if (skb_orphan_frags(skb, gfp_mask)) { 1020 kfree_skb(n); 1021 n = NULL; 1022 goto out; 1023 } 1024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1025 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1026 skb_frag_ref(skb, i); 1027 } 1028 skb_shinfo(n)->nr_frags = i; 1029 } 1030 1031 if (skb_has_frag_list(skb)) { 1032 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1033 skb_clone_fraglist(n); 1034 } 1035 1036 copy_skb_header(n, skb); 1037 out: 1038 return n; 1039 } 1040 EXPORT_SYMBOL(__pskb_copy); 1041 1042 /** 1043 * pskb_expand_head - reallocate header of &sk_buff 1044 * @skb: buffer to reallocate 1045 * @nhead: room to add at head 1046 * @ntail: room to add at tail 1047 * @gfp_mask: allocation priority 1048 * 1049 * Expands (or creates identical copy, if &nhead and &ntail are zero) 1050 * header of skb. &sk_buff itself is not changed. &sk_buff MUST have 1051 * reference count of 1. Returns zero in the case of success or error, 1052 * if expansion failed. In the last case, &sk_buff is not changed. 1053 * 1054 * All the pointers pointing into skb header may change and must be 1055 * reloaded after call to this function. 1056 */ 1057 1058 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1059 gfp_t gfp_mask) 1060 { 1061 int i; 1062 u8 *data; 1063 int size = nhead + skb_end_offset(skb) + ntail; 1064 long off; 1065 1066 BUG_ON(nhead < 0); 1067 1068 if (skb_shared(skb)) 1069 BUG(); 1070 1071 size = SKB_DATA_ALIGN(size); 1072 1073 if (skb_pfmemalloc(skb)) 1074 gfp_mask |= __GFP_MEMALLOC; 1075 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1076 gfp_mask, NUMA_NO_NODE, NULL); 1077 if (!data) 1078 goto nodata; 1079 size = SKB_WITH_OVERHEAD(ksize(data)); 1080 1081 /* Copy only real data... and, alas, header. This should be 1082 * optimized for the cases when header is void. 1083 */ 1084 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1085 1086 memcpy((struct skb_shared_info *)(data + size), 1087 skb_shinfo(skb), 1088 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1089 1090 /* 1091 * if shinfo is shared we must drop the old head gracefully, but if it 1092 * is not we can just drop the old head and let the existing refcount 1093 * be since all we did is relocate the values 1094 */ 1095 if (skb_cloned(skb)) { 1096 /* copy this zero copy skb frags */ 1097 if (skb_orphan_frags(skb, gfp_mask)) 1098 goto nofrags; 1099 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1100 skb_frag_ref(skb, i); 1101 1102 if (skb_has_frag_list(skb)) 1103 skb_clone_fraglist(skb); 1104 1105 skb_release_data(skb); 1106 } else { 1107 skb_free_head(skb); 1108 } 1109 off = (data + nhead) - skb->head; 1110 1111 skb->head = data; 1112 skb->head_frag = 0; 1113 skb->data += off; 1114 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1115 skb->end = size; 1116 off = nhead; 1117 #else 1118 skb->end = skb->head + size; 1119 #endif 1120 skb->tail += off; 1121 skb_headers_offset_update(skb, off); 1122 /* Only adjust this if it actually is csum_start rather than csum */ 1123 if (skb->ip_summed == CHECKSUM_PARTIAL) 1124 skb->csum_start += nhead; 1125 skb->cloned = 0; 1126 skb->hdr_len = 0; 1127 skb->nohdr = 0; 1128 atomic_set(&skb_shinfo(skb)->dataref, 1); 1129 return 0; 1130 1131 nofrags: 1132 kfree(data); 1133 nodata: 1134 return -ENOMEM; 1135 } 1136 EXPORT_SYMBOL(pskb_expand_head); 1137 1138 /* Make private copy of skb with writable head and some headroom */ 1139 1140 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1141 { 1142 struct sk_buff *skb2; 1143 int delta = headroom - skb_headroom(skb); 1144 1145 if (delta <= 0) 1146 skb2 = pskb_copy(skb, GFP_ATOMIC); 1147 else { 1148 skb2 = skb_clone(skb, GFP_ATOMIC); 1149 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1150 GFP_ATOMIC)) { 1151 kfree_skb(skb2); 1152 skb2 = NULL; 1153 } 1154 } 1155 return skb2; 1156 } 1157 EXPORT_SYMBOL(skb_realloc_headroom); 1158 1159 /** 1160 * skb_copy_expand - copy and expand sk_buff 1161 * @skb: buffer to copy 1162 * @newheadroom: new free bytes at head 1163 * @newtailroom: new free bytes at tail 1164 * @gfp_mask: allocation priority 1165 * 1166 * Make a copy of both an &sk_buff and its data and while doing so 1167 * allocate additional space. 1168 * 1169 * This is used when the caller wishes to modify the data and needs a 1170 * private copy of the data to alter as well as more space for new fields. 1171 * Returns %NULL on failure or the pointer to the buffer 1172 * on success. The returned buffer has a reference count of 1. 1173 * 1174 * You must pass %GFP_ATOMIC as the allocation priority if this function 1175 * is called from an interrupt. 1176 */ 1177 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1178 int newheadroom, int newtailroom, 1179 gfp_t gfp_mask) 1180 { 1181 /* 1182 * Allocate the copy buffer 1183 */ 1184 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1185 gfp_mask, skb_alloc_rx_flag(skb), 1186 NUMA_NO_NODE); 1187 int oldheadroom = skb_headroom(skb); 1188 int head_copy_len, head_copy_off; 1189 int off; 1190 1191 if (!n) 1192 return NULL; 1193 1194 skb_reserve(n, newheadroom); 1195 1196 /* Set the tail pointer and length */ 1197 skb_put(n, skb->len); 1198 1199 head_copy_len = oldheadroom; 1200 head_copy_off = 0; 1201 if (newheadroom <= head_copy_len) 1202 head_copy_len = newheadroom; 1203 else 1204 head_copy_off = newheadroom - head_copy_len; 1205 1206 /* Copy the linear header and data. */ 1207 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1208 skb->len + head_copy_len)) 1209 BUG(); 1210 1211 copy_skb_header(n, skb); 1212 1213 off = newheadroom - oldheadroom; 1214 if (n->ip_summed == CHECKSUM_PARTIAL) 1215 n->csum_start += off; 1216 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1217 skb_headers_offset_update(n, off); 1218 #endif 1219 1220 return n; 1221 } 1222 EXPORT_SYMBOL(skb_copy_expand); 1223 1224 /** 1225 * skb_pad - zero pad the tail of an skb 1226 * @skb: buffer to pad 1227 * @pad: space to pad 1228 * 1229 * Ensure that a buffer is followed by a padding area that is zero 1230 * filled. Used by network drivers which may DMA or transfer data 1231 * beyond the buffer end onto the wire. 1232 * 1233 * May return error in out of memory cases. The skb is freed on error. 1234 */ 1235 1236 int skb_pad(struct sk_buff *skb, int pad) 1237 { 1238 int err; 1239 int ntail; 1240 1241 /* If the skbuff is non linear tailroom is always zero.. */ 1242 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1243 memset(skb->data+skb->len, 0, pad); 1244 return 0; 1245 } 1246 1247 ntail = skb->data_len + pad - (skb->end - skb->tail); 1248 if (likely(skb_cloned(skb) || ntail > 0)) { 1249 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1250 if (unlikely(err)) 1251 goto free_skb; 1252 } 1253 1254 /* FIXME: The use of this function with non-linear skb's really needs 1255 * to be audited. 1256 */ 1257 err = skb_linearize(skb); 1258 if (unlikely(err)) 1259 goto free_skb; 1260 1261 memset(skb->data + skb->len, 0, pad); 1262 return 0; 1263 1264 free_skb: 1265 kfree_skb(skb); 1266 return err; 1267 } 1268 EXPORT_SYMBOL(skb_pad); 1269 1270 /** 1271 * skb_put - add data to a buffer 1272 * @skb: buffer to use 1273 * @len: amount of data to add 1274 * 1275 * This function extends the used data area of the buffer. If this would 1276 * exceed the total buffer size the kernel will panic. A pointer to the 1277 * first byte of the extra data is returned. 1278 */ 1279 unsigned char *skb_put(struct sk_buff *skb, unsigned int len) 1280 { 1281 unsigned char *tmp = skb_tail_pointer(skb); 1282 SKB_LINEAR_ASSERT(skb); 1283 skb->tail += len; 1284 skb->len += len; 1285 if (unlikely(skb->tail > skb->end)) 1286 skb_over_panic(skb, len, __builtin_return_address(0)); 1287 return tmp; 1288 } 1289 EXPORT_SYMBOL(skb_put); 1290 1291 /** 1292 * skb_push - add data to the start of a buffer 1293 * @skb: buffer to use 1294 * @len: amount of data to add 1295 * 1296 * This function extends the used data area of the buffer at the buffer 1297 * start. If this would exceed the total buffer headroom the kernel will 1298 * panic. A pointer to the first byte of the extra data is returned. 1299 */ 1300 unsigned char *skb_push(struct sk_buff *skb, unsigned int len) 1301 { 1302 skb->data -= len; 1303 skb->len += len; 1304 if (unlikely(skb->data<skb->head)) 1305 skb_under_panic(skb, len, __builtin_return_address(0)); 1306 return skb->data; 1307 } 1308 EXPORT_SYMBOL(skb_push); 1309 1310 /** 1311 * skb_pull - remove data from the start of a buffer 1312 * @skb: buffer to use 1313 * @len: amount of data to remove 1314 * 1315 * This function removes data from the start of a buffer, returning 1316 * the memory to the headroom. A pointer to the next data in the buffer 1317 * is returned. Once the data has been pulled future pushes will overwrite 1318 * the old data. 1319 */ 1320 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len) 1321 { 1322 return skb_pull_inline(skb, len); 1323 } 1324 EXPORT_SYMBOL(skb_pull); 1325 1326 /** 1327 * skb_trim - remove end from a buffer 1328 * @skb: buffer to alter 1329 * @len: new length 1330 * 1331 * Cut the length of a buffer down by removing data from the tail. If 1332 * the buffer is already under the length specified it is not modified. 1333 * The skb must be linear. 1334 */ 1335 void skb_trim(struct sk_buff *skb, unsigned int len) 1336 { 1337 if (skb->len > len) 1338 __skb_trim(skb, len); 1339 } 1340 EXPORT_SYMBOL(skb_trim); 1341 1342 /* Trims skb to length len. It can change skb pointers. 1343 */ 1344 1345 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1346 { 1347 struct sk_buff **fragp; 1348 struct sk_buff *frag; 1349 int offset = skb_headlen(skb); 1350 int nfrags = skb_shinfo(skb)->nr_frags; 1351 int i; 1352 int err; 1353 1354 if (skb_cloned(skb) && 1355 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1356 return err; 1357 1358 i = 0; 1359 if (offset >= len) 1360 goto drop_pages; 1361 1362 for (; i < nfrags; i++) { 1363 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1364 1365 if (end < len) { 1366 offset = end; 1367 continue; 1368 } 1369 1370 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1371 1372 drop_pages: 1373 skb_shinfo(skb)->nr_frags = i; 1374 1375 for (; i < nfrags; i++) 1376 skb_frag_unref(skb, i); 1377 1378 if (skb_has_frag_list(skb)) 1379 skb_drop_fraglist(skb); 1380 goto done; 1381 } 1382 1383 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1384 fragp = &frag->next) { 1385 int end = offset + frag->len; 1386 1387 if (skb_shared(frag)) { 1388 struct sk_buff *nfrag; 1389 1390 nfrag = skb_clone(frag, GFP_ATOMIC); 1391 if (unlikely(!nfrag)) 1392 return -ENOMEM; 1393 1394 nfrag->next = frag->next; 1395 consume_skb(frag); 1396 frag = nfrag; 1397 *fragp = frag; 1398 } 1399 1400 if (end < len) { 1401 offset = end; 1402 continue; 1403 } 1404 1405 if (end > len && 1406 unlikely((err = pskb_trim(frag, len - offset)))) 1407 return err; 1408 1409 if (frag->next) 1410 skb_drop_list(&frag->next); 1411 break; 1412 } 1413 1414 done: 1415 if (len > skb_headlen(skb)) { 1416 skb->data_len -= skb->len - len; 1417 skb->len = len; 1418 } else { 1419 skb->len = len; 1420 skb->data_len = 0; 1421 skb_set_tail_pointer(skb, len); 1422 } 1423 1424 return 0; 1425 } 1426 EXPORT_SYMBOL(___pskb_trim); 1427 1428 /** 1429 * __pskb_pull_tail - advance tail of skb header 1430 * @skb: buffer to reallocate 1431 * @delta: number of bytes to advance tail 1432 * 1433 * The function makes a sense only on a fragmented &sk_buff, 1434 * it expands header moving its tail forward and copying necessary 1435 * data from fragmented part. 1436 * 1437 * &sk_buff MUST have reference count of 1. 1438 * 1439 * Returns %NULL (and &sk_buff does not change) if pull failed 1440 * or value of new tail of skb in the case of success. 1441 * 1442 * All the pointers pointing into skb header may change and must be 1443 * reloaded after call to this function. 1444 */ 1445 1446 /* Moves tail of skb head forward, copying data from fragmented part, 1447 * when it is necessary. 1448 * 1. It may fail due to malloc failure. 1449 * 2. It may change skb pointers. 1450 * 1451 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1452 */ 1453 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) 1454 { 1455 /* If skb has not enough free space at tail, get new one 1456 * plus 128 bytes for future expansions. If we have enough 1457 * room at tail, reallocate without expansion only if skb is cloned. 1458 */ 1459 int i, k, eat = (skb->tail + delta) - skb->end; 1460 1461 if (eat > 0 || skb_cloned(skb)) { 1462 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1463 GFP_ATOMIC)) 1464 return NULL; 1465 } 1466 1467 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta)) 1468 BUG(); 1469 1470 /* Optimization: no fragments, no reasons to preestimate 1471 * size of pulled pages. Superb. 1472 */ 1473 if (!skb_has_frag_list(skb)) 1474 goto pull_pages; 1475 1476 /* Estimate size of pulled pages. */ 1477 eat = delta; 1478 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1479 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1480 1481 if (size >= eat) 1482 goto pull_pages; 1483 eat -= size; 1484 } 1485 1486 /* If we need update frag list, we are in troubles. 1487 * Certainly, it possible to add an offset to skb data, 1488 * but taking into account that pulling is expected to 1489 * be very rare operation, it is worth to fight against 1490 * further bloating skb head and crucify ourselves here instead. 1491 * Pure masohism, indeed. 8)8) 1492 */ 1493 if (eat) { 1494 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1495 struct sk_buff *clone = NULL; 1496 struct sk_buff *insp = NULL; 1497 1498 do { 1499 BUG_ON(!list); 1500 1501 if (list->len <= eat) { 1502 /* Eaten as whole. */ 1503 eat -= list->len; 1504 list = list->next; 1505 insp = list; 1506 } else { 1507 /* Eaten partially. */ 1508 1509 if (skb_shared(list)) { 1510 /* Sucks! We need to fork list. :-( */ 1511 clone = skb_clone(list, GFP_ATOMIC); 1512 if (!clone) 1513 return NULL; 1514 insp = list->next; 1515 list = clone; 1516 } else { 1517 /* This may be pulled without 1518 * problems. */ 1519 insp = list; 1520 } 1521 if (!pskb_pull(list, eat)) { 1522 kfree_skb(clone); 1523 return NULL; 1524 } 1525 break; 1526 } 1527 } while (eat); 1528 1529 /* Free pulled out fragments. */ 1530 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1531 skb_shinfo(skb)->frag_list = list->next; 1532 kfree_skb(list); 1533 } 1534 /* And insert new clone at head. */ 1535 if (clone) { 1536 clone->next = list; 1537 skb_shinfo(skb)->frag_list = clone; 1538 } 1539 } 1540 /* Success! Now we may commit changes to skb data. */ 1541 1542 pull_pages: 1543 eat = delta; 1544 k = 0; 1545 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1546 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1547 1548 if (size <= eat) { 1549 skb_frag_unref(skb, i); 1550 eat -= size; 1551 } else { 1552 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 1553 if (eat) { 1554 skb_shinfo(skb)->frags[k].page_offset += eat; 1555 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); 1556 eat = 0; 1557 } 1558 k++; 1559 } 1560 } 1561 skb_shinfo(skb)->nr_frags = k; 1562 1563 skb->tail += delta; 1564 skb->data_len -= delta; 1565 1566 return skb_tail_pointer(skb); 1567 } 1568 EXPORT_SYMBOL(__pskb_pull_tail); 1569 1570 /** 1571 * skb_copy_bits - copy bits from skb to kernel buffer 1572 * @skb: source skb 1573 * @offset: offset in source 1574 * @to: destination buffer 1575 * @len: number of bytes to copy 1576 * 1577 * Copy the specified number of bytes from the source skb to the 1578 * destination buffer. 1579 * 1580 * CAUTION ! : 1581 * If its prototype is ever changed, 1582 * check arch/{*}/net/{*}.S files, 1583 * since it is called from BPF assembly code. 1584 */ 1585 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 1586 { 1587 int start = skb_headlen(skb); 1588 struct sk_buff *frag_iter; 1589 int i, copy; 1590 1591 if (offset > (int)skb->len - len) 1592 goto fault; 1593 1594 /* Copy header. */ 1595 if ((copy = start - offset) > 0) { 1596 if (copy > len) 1597 copy = len; 1598 skb_copy_from_linear_data_offset(skb, offset, to, copy); 1599 if ((len -= copy) == 0) 1600 return 0; 1601 offset += copy; 1602 to += copy; 1603 } 1604 1605 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1606 int end; 1607 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1608 1609 WARN_ON(start > offset + len); 1610 1611 end = start + skb_frag_size(f); 1612 if ((copy = end - offset) > 0) { 1613 u8 *vaddr; 1614 1615 if (copy > len) 1616 copy = len; 1617 1618 vaddr = kmap_atomic(skb_frag_page(f)); 1619 memcpy(to, 1620 vaddr + f->page_offset + offset - start, 1621 copy); 1622 kunmap_atomic(vaddr); 1623 1624 if ((len -= copy) == 0) 1625 return 0; 1626 offset += copy; 1627 to += copy; 1628 } 1629 start = end; 1630 } 1631 1632 skb_walk_frags(skb, frag_iter) { 1633 int end; 1634 1635 WARN_ON(start > offset + len); 1636 1637 end = start + frag_iter->len; 1638 if ((copy = end - offset) > 0) { 1639 if (copy > len) 1640 copy = len; 1641 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 1642 goto fault; 1643 if ((len -= copy) == 0) 1644 return 0; 1645 offset += copy; 1646 to += copy; 1647 } 1648 start = end; 1649 } 1650 1651 if (!len) 1652 return 0; 1653 1654 fault: 1655 return -EFAULT; 1656 } 1657 EXPORT_SYMBOL(skb_copy_bits); 1658 1659 /* 1660 * Callback from splice_to_pipe(), if we need to release some pages 1661 * at the end of the spd in case we error'ed out in filling the pipe. 1662 */ 1663 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 1664 { 1665 put_page(spd->pages[i]); 1666 } 1667 1668 static struct page *linear_to_page(struct page *page, unsigned int *len, 1669 unsigned int *offset, 1670 struct sock *sk) 1671 { 1672 struct page_frag *pfrag = sk_page_frag(sk); 1673 1674 if (!sk_page_frag_refill(sk, pfrag)) 1675 return NULL; 1676 1677 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 1678 1679 memcpy(page_address(pfrag->page) + pfrag->offset, 1680 page_address(page) + *offset, *len); 1681 *offset = pfrag->offset; 1682 pfrag->offset += *len; 1683 1684 return pfrag->page; 1685 } 1686 1687 static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 1688 struct page *page, 1689 unsigned int offset) 1690 { 1691 return spd->nr_pages && 1692 spd->pages[spd->nr_pages - 1] == page && 1693 (spd->partial[spd->nr_pages - 1].offset + 1694 spd->partial[spd->nr_pages - 1].len == offset); 1695 } 1696 1697 /* 1698 * Fill page/offset/length into spd, if it can hold more pages. 1699 */ 1700 static bool spd_fill_page(struct splice_pipe_desc *spd, 1701 struct pipe_inode_info *pipe, struct page *page, 1702 unsigned int *len, unsigned int offset, 1703 bool linear, 1704 struct sock *sk) 1705 { 1706 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 1707 return true; 1708 1709 if (linear) { 1710 page = linear_to_page(page, len, &offset, sk); 1711 if (!page) 1712 return true; 1713 } 1714 if (spd_can_coalesce(spd, page, offset)) { 1715 spd->partial[spd->nr_pages - 1].len += *len; 1716 return false; 1717 } 1718 get_page(page); 1719 spd->pages[spd->nr_pages] = page; 1720 spd->partial[spd->nr_pages].len = *len; 1721 spd->partial[spd->nr_pages].offset = offset; 1722 spd->nr_pages++; 1723 1724 return false; 1725 } 1726 1727 static bool __splice_segment(struct page *page, unsigned int poff, 1728 unsigned int plen, unsigned int *off, 1729 unsigned int *len, 1730 struct splice_pipe_desc *spd, bool linear, 1731 struct sock *sk, 1732 struct pipe_inode_info *pipe) 1733 { 1734 if (!*len) 1735 return true; 1736 1737 /* skip this segment if already processed */ 1738 if (*off >= plen) { 1739 *off -= plen; 1740 return false; 1741 } 1742 1743 /* ignore any bits we already processed */ 1744 poff += *off; 1745 plen -= *off; 1746 *off = 0; 1747 1748 do { 1749 unsigned int flen = min(*len, plen); 1750 1751 if (spd_fill_page(spd, pipe, page, &flen, poff, 1752 linear, sk)) 1753 return true; 1754 poff += flen; 1755 plen -= flen; 1756 *len -= flen; 1757 } while (*len && plen); 1758 1759 return false; 1760 } 1761 1762 /* 1763 * Map linear and fragment data from the skb to spd. It reports true if the 1764 * pipe is full or if we already spliced the requested length. 1765 */ 1766 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 1767 unsigned int *offset, unsigned int *len, 1768 struct splice_pipe_desc *spd, struct sock *sk) 1769 { 1770 int seg; 1771 1772 /* map the linear part : 1773 * If skb->head_frag is set, this 'linear' part is backed by a 1774 * fragment, and if the head is not shared with any clones then 1775 * we can avoid a copy since we own the head portion of this page. 1776 */ 1777 if (__splice_segment(virt_to_page(skb->data), 1778 (unsigned long) skb->data & (PAGE_SIZE - 1), 1779 skb_headlen(skb), 1780 offset, len, spd, 1781 skb_head_is_locked(skb), 1782 sk, pipe)) 1783 return true; 1784 1785 /* 1786 * then map the fragments 1787 */ 1788 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 1789 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 1790 1791 if (__splice_segment(skb_frag_page(f), 1792 f->page_offset, skb_frag_size(f), 1793 offset, len, spd, false, sk, pipe)) 1794 return true; 1795 } 1796 1797 return false; 1798 } 1799 1800 /* 1801 * Map data from the skb to a pipe. Should handle both the linear part, 1802 * the fragments, and the frag list. It does NOT handle frag lists within 1803 * the frag list, if such a thing exists. We'd probably need to recurse to 1804 * handle that cleanly. 1805 */ 1806 int skb_splice_bits(struct sk_buff *skb, unsigned int offset, 1807 struct pipe_inode_info *pipe, unsigned int tlen, 1808 unsigned int flags) 1809 { 1810 struct partial_page partial[MAX_SKB_FRAGS]; 1811 struct page *pages[MAX_SKB_FRAGS]; 1812 struct splice_pipe_desc spd = { 1813 .pages = pages, 1814 .partial = partial, 1815 .nr_pages_max = MAX_SKB_FRAGS, 1816 .flags = flags, 1817 .ops = &sock_pipe_buf_ops, 1818 .spd_release = sock_spd_release, 1819 }; 1820 struct sk_buff *frag_iter; 1821 struct sock *sk = skb->sk; 1822 int ret = 0; 1823 1824 /* 1825 * __skb_splice_bits() only fails if the output has no room left, 1826 * so no point in going over the frag_list for the error case. 1827 */ 1828 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk)) 1829 goto done; 1830 else if (!tlen) 1831 goto done; 1832 1833 /* 1834 * now see if we have a frag_list to map 1835 */ 1836 skb_walk_frags(skb, frag_iter) { 1837 if (!tlen) 1838 break; 1839 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk)) 1840 break; 1841 } 1842 1843 done: 1844 if (spd.nr_pages) { 1845 /* 1846 * Drop the socket lock, otherwise we have reverse 1847 * locking dependencies between sk_lock and i_mutex 1848 * here as compared to sendfile(). We enter here 1849 * with the socket lock held, and splice_to_pipe() will 1850 * grab the pipe inode lock. For sendfile() emulation, 1851 * we call into ->sendpage() with the i_mutex lock held 1852 * and networking will grab the socket lock. 1853 */ 1854 release_sock(sk); 1855 ret = splice_to_pipe(pipe, &spd); 1856 lock_sock(sk); 1857 } 1858 1859 return ret; 1860 } 1861 1862 /** 1863 * skb_store_bits - store bits from kernel buffer to skb 1864 * @skb: destination buffer 1865 * @offset: offset in destination 1866 * @from: source buffer 1867 * @len: number of bytes to copy 1868 * 1869 * Copy the specified number of bytes from the source buffer to the 1870 * destination skb. This function handles all the messy bits of 1871 * traversing fragment lists and such. 1872 */ 1873 1874 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 1875 { 1876 int start = skb_headlen(skb); 1877 struct sk_buff *frag_iter; 1878 int i, copy; 1879 1880 if (offset > (int)skb->len - len) 1881 goto fault; 1882 1883 if ((copy = start - offset) > 0) { 1884 if (copy > len) 1885 copy = len; 1886 skb_copy_to_linear_data_offset(skb, offset, from, copy); 1887 if ((len -= copy) == 0) 1888 return 0; 1889 offset += copy; 1890 from += copy; 1891 } 1892 1893 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1894 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1895 int end; 1896 1897 WARN_ON(start > offset + len); 1898 1899 end = start + skb_frag_size(frag); 1900 if ((copy = end - offset) > 0) { 1901 u8 *vaddr; 1902 1903 if (copy > len) 1904 copy = len; 1905 1906 vaddr = kmap_atomic(skb_frag_page(frag)); 1907 memcpy(vaddr + frag->page_offset + offset - start, 1908 from, copy); 1909 kunmap_atomic(vaddr); 1910 1911 if ((len -= copy) == 0) 1912 return 0; 1913 offset += copy; 1914 from += copy; 1915 } 1916 start = end; 1917 } 1918 1919 skb_walk_frags(skb, frag_iter) { 1920 int end; 1921 1922 WARN_ON(start > offset + len); 1923 1924 end = start + frag_iter->len; 1925 if ((copy = end - offset) > 0) { 1926 if (copy > len) 1927 copy = len; 1928 if (skb_store_bits(frag_iter, offset - start, 1929 from, copy)) 1930 goto fault; 1931 if ((len -= copy) == 0) 1932 return 0; 1933 offset += copy; 1934 from += copy; 1935 } 1936 start = end; 1937 } 1938 if (!len) 1939 return 0; 1940 1941 fault: 1942 return -EFAULT; 1943 } 1944 EXPORT_SYMBOL(skb_store_bits); 1945 1946 /* Checksum skb data. */ 1947 1948 __wsum skb_checksum(const struct sk_buff *skb, int offset, 1949 int len, __wsum csum) 1950 { 1951 int start = skb_headlen(skb); 1952 int i, copy = start - offset; 1953 struct sk_buff *frag_iter; 1954 int pos = 0; 1955 1956 /* Checksum header. */ 1957 if (copy > 0) { 1958 if (copy > len) 1959 copy = len; 1960 csum = csum_partial(skb->data + offset, copy, csum); 1961 if ((len -= copy) == 0) 1962 return csum; 1963 offset += copy; 1964 pos = copy; 1965 } 1966 1967 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1968 int end; 1969 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1970 1971 WARN_ON(start > offset + len); 1972 1973 end = start + skb_frag_size(frag); 1974 if ((copy = end - offset) > 0) { 1975 __wsum csum2; 1976 u8 *vaddr; 1977 1978 if (copy > len) 1979 copy = len; 1980 vaddr = kmap_atomic(skb_frag_page(frag)); 1981 csum2 = csum_partial(vaddr + frag->page_offset + 1982 offset - start, copy, 0); 1983 kunmap_atomic(vaddr); 1984 csum = csum_block_add(csum, csum2, pos); 1985 if (!(len -= copy)) 1986 return csum; 1987 offset += copy; 1988 pos += copy; 1989 } 1990 start = end; 1991 } 1992 1993 skb_walk_frags(skb, frag_iter) { 1994 int end; 1995 1996 WARN_ON(start > offset + len); 1997 1998 end = start + frag_iter->len; 1999 if ((copy = end - offset) > 0) { 2000 __wsum csum2; 2001 if (copy > len) 2002 copy = len; 2003 csum2 = skb_checksum(frag_iter, offset - start, 2004 copy, 0); 2005 csum = csum_block_add(csum, csum2, pos); 2006 if ((len -= copy) == 0) 2007 return csum; 2008 offset += copy; 2009 pos += copy; 2010 } 2011 start = end; 2012 } 2013 BUG_ON(len); 2014 2015 return csum; 2016 } 2017 EXPORT_SYMBOL(skb_checksum); 2018 2019 /* Both of above in one bottle. */ 2020 2021 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2022 u8 *to, int len, __wsum csum) 2023 { 2024 int start = skb_headlen(skb); 2025 int i, copy = start - offset; 2026 struct sk_buff *frag_iter; 2027 int pos = 0; 2028 2029 /* Copy header. */ 2030 if (copy > 0) { 2031 if (copy > len) 2032 copy = len; 2033 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2034 copy, csum); 2035 if ((len -= copy) == 0) 2036 return csum; 2037 offset += copy; 2038 to += copy; 2039 pos = copy; 2040 } 2041 2042 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2043 int end; 2044 2045 WARN_ON(start > offset + len); 2046 2047 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2048 if ((copy = end - offset) > 0) { 2049 __wsum csum2; 2050 u8 *vaddr; 2051 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2052 2053 if (copy > len) 2054 copy = len; 2055 vaddr = kmap_atomic(skb_frag_page(frag)); 2056 csum2 = csum_partial_copy_nocheck(vaddr + 2057 frag->page_offset + 2058 offset - start, to, 2059 copy, 0); 2060 kunmap_atomic(vaddr); 2061 csum = csum_block_add(csum, csum2, pos); 2062 if (!(len -= copy)) 2063 return csum; 2064 offset += copy; 2065 to += copy; 2066 pos += copy; 2067 } 2068 start = end; 2069 } 2070 2071 skb_walk_frags(skb, frag_iter) { 2072 __wsum csum2; 2073 int end; 2074 2075 WARN_ON(start > offset + len); 2076 2077 end = start + frag_iter->len; 2078 if ((copy = end - offset) > 0) { 2079 if (copy > len) 2080 copy = len; 2081 csum2 = skb_copy_and_csum_bits(frag_iter, 2082 offset - start, 2083 to, copy, 0); 2084 csum = csum_block_add(csum, csum2, pos); 2085 if ((len -= copy) == 0) 2086 return csum; 2087 offset += copy; 2088 to += copy; 2089 pos += copy; 2090 } 2091 start = end; 2092 } 2093 BUG_ON(len); 2094 return csum; 2095 } 2096 EXPORT_SYMBOL(skb_copy_and_csum_bits); 2097 2098 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 2099 { 2100 __wsum csum; 2101 long csstart; 2102 2103 if (skb->ip_summed == CHECKSUM_PARTIAL) 2104 csstart = skb_checksum_start_offset(skb); 2105 else 2106 csstart = skb_headlen(skb); 2107 2108 BUG_ON(csstart > skb_headlen(skb)); 2109 2110 skb_copy_from_linear_data(skb, to, csstart); 2111 2112 csum = 0; 2113 if (csstart != skb->len) 2114 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 2115 skb->len - csstart, 0); 2116 2117 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2118 long csstuff = csstart + skb->csum_offset; 2119 2120 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 2121 } 2122 } 2123 EXPORT_SYMBOL(skb_copy_and_csum_dev); 2124 2125 /** 2126 * skb_dequeue - remove from the head of the queue 2127 * @list: list to dequeue from 2128 * 2129 * Remove the head of the list. The list lock is taken so the function 2130 * may be used safely with other locking list functions. The head item is 2131 * returned or %NULL if the list is empty. 2132 */ 2133 2134 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 2135 { 2136 unsigned long flags; 2137 struct sk_buff *result; 2138 2139 spin_lock_irqsave(&list->lock, flags); 2140 result = __skb_dequeue(list); 2141 spin_unlock_irqrestore(&list->lock, flags); 2142 return result; 2143 } 2144 EXPORT_SYMBOL(skb_dequeue); 2145 2146 /** 2147 * skb_dequeue_tail - remove from the tail of the queue 2148 * @list: list to dequeue from 2149 * 2150 * Remove the tail of the list. The list lock is taken so the function 2151 * may be used safely with other locking list functions. The tail item is 2152 * returned or %NULL if the list is empty. 2153 */ 2154 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 2155 { 2156 unsigned long flags; 2157 struct sk_buff *result; 2158 2159 spin_lock_irqsave(&list->lock, flags); 2160 result = __skb_dequeue_tail(list); 2161 spin_unlock_irqrestore(&list->lock, flags); 2162 return result; 2163 } 2164 EXPORT_SYMBOL(skb_dequeue_tail); 2165 2166 /** 2167 * skb_queue_purge - empty a list 2168 * @list: list to empty 2169 * 2170 * Delete all buffers on an &sk_buff list. Each buffer is removed from 2171 * the list and one reference dropped. This function takes the list 2172 * lock and is atomic with respect to other list locking functions. 2173 */ 2174 void skb_queue_purge(struct sk_buff_head *list) 2175 { 2176 struct sk_buff *skb; 2177 while ((skb = skb_dequeue(list)) != NULL) 2178 kfree_skb(skb); 2179 } 2180 EXPORT_SYMBOL(skb_queue_purge); 2181 2182 /** 2183 * skb_queue_head - queue a buffer at the list head 2184 * @list: list to use 2185 * @newsk: buffer to queue 2186 * 2187 * Queue a buffer at the start of the list. This function takes the 2188 * list lock and can be used safely with other locking &sk_buff functions 2189 * safely. 2190 * 2191 * A buffer cannot be placed on two lists at the same time. 2192 */ 2193 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 2194 { 2195 unsigned long flags; 2196 2197 spin_lock_irqsave(&list->lock, flags); 2198 __skb_queue_head(list, newsk); 2199 spin_unlock_irqrestore(&list->lock, flags); 2200 } 2201 EXPORT_SYMBOL(skb_queue_head); 2202 2203 /** 2204 * skb_queue_tail - queue a buffer at the list tail 2205 * @list: list to use 2206 * @newsk: buffer to queue 2207 * 2208 * Queue a buffer at the tail of the list. This function takes the 2209 * list lock and can be used safely with other locking &sk_buff functions 2210 * safely. 2211 * 2212 * A buffer cannot be placed on two lists at the same time. 2213 */ 2214 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 2215 { 2216 unsigned long flags; 2217 2218 spin_lock_irqsave(&list->lock, flags); 2219 __skb_queue_tail(list, newsk); 2220 spin_unlock_irqrestore(&list->lock, flags); 2221 } 2222 EXPORT_SYMBOL(skb_queue_tail); 2223 2224 /** 2225 * skb_unlink - remove a buffer from a list 2226 * @skb: buffer to remove 2227 * @list: list to use 2228 * 2229 * Remove a packet from a list. The list locks are taken and this 2230 * function is atomic with respect to other list locked calls 2231 * 2232 * You must know what list the SKB is on. 2233 */ 2234 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 2235 { 2236 unsigned long flags; 2237 2238 spin_lock_irqsave(&list->lock, flags); 2239 __skb_unlink(skb, list); 2240 spin_unlock_irqrestore(&list->lock, flags); 2241 } 2242 EXPORT_SYMBOL(skb_unlink); 2243 2244 /** 2245 * skb_append - append a buffer 2246 * @old: buffer to insert after 2247 * @newsk: buffer to insert 2248 * @list: list to use 2249 * 2250 * Place a packet after a given packet in a list. The list locks are taken 2251 * and this function is atomic with respect to other list locked calls. 2252 * A buffer cannot be placed on two lists at the same time. 2253 */ 2254 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2255 { 2256 unsigned long flags; 2257 2258 spin_lock_irqsave(&list->lock, flags); 2259 __skb_queue_after(list, old, newsk); 2260 spin_unlock_irqrestore(&list->lock, flags); 2261 } 2262 EXPORT_SYMBOL(skb_append); 2263 2264 /** 2265 * skb_insert - insert a buffer 2266 * @old: buffer to insert before 2267 * @newsk: buffer to insert 2268 * @list: list to use 2269 * 2270 * Place a packet before a given packet in a list. The list locks are 2271 * taken and this function is atomic with respect to other list locked 2272 * calls. 2273 * 2274 * A buffer cannot be placed on two lists at the same time. 2275 */ 2276 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2277 { 2278 unsigned long flags; 2279 2280 spin_lock_irqsave(&list->lock, flags); 2281 __skb_insert(newsk, old->prev, old, list); 2282 spin_unlock_irqrestore(&list->lock, flags); 2283 } 2284 EXPORT_SYMBOL(skb_insert); 2285 2286 static inline void skb_split_inside_header(struct sk_buff *skb, 2287 struct sk_buff* skb1, 2288 const u32 len, const int pos) 2289 { 2290 int i; 2291 2292 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 2293 pos - len); 2294 /* And move data appendix as is. */ 2295 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2296 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 2297 2298 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 2299 skb_shinfo(skb)->nr_frags = 0; 2300 skb1->data_len = skb->data_len; 2301 skb1->len += skb1->data_len; 2302 skb->data_len = 0; 2303 skb->len = len; 2304 skb_set_tail_pointer(skb, len); 2305 } 2306 2307 static inline void skb_split_no_header(struct sk_buff *skb, 2308 struct sk_buff* skb1, 2309 const u32 len, int pos) 2310 { 2311 int i, k = 0; 2312 const int nfrags = skb_shinfo(skb)->nr_frags; 2313 2314 skb_shinfo(skb)->nr_frags = 0; 2315 skb1->len = skb1->data_len = skb->len - len; 2316 skb->len = len; 2317 skb->data_len = len - pos; 2318 2319 for (i = 0; i < nfrags; i++) { 2320 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2321 2322 if (pos + size > len) { 2323 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 2324 2325 if (pos < len) { 2326 /* Split frag. 2327 * We have two variants in this case: 2328 * 1. Move all the frag to the second 2329 * part, if it is possible. F.e. 2330 * this approach is mandatory for TUX, 2331 * where splitting is expensive. 2332 * 2. Split is accurately. We make this. 2333 */ 2334 skb_frag_ref(skb, i); 2335 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 2336 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 2337 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 2338 skb_shinfo(skb)->nr_frags++; 2339 } 2340 k++; 2341 } else 2342 skb_shinfo(skb)->nr_frags++; 2343 pos += size; 2344 } 2345 skb_shinfo(skb1)->nr_frags = k; 2346 } 2347 2348 /** 2349 * skb_split - Split fragmented skb to two parts at length len. 2350 * @skb: the buffer to split 2351 * @skb1: the buffer to receive the second part 2352 * @len: new length for skb 2353 */ 2354 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 2355 { 2356 int pos = skb_headlen(skb); 2357 2358 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; 2359 if (len < pos) /* Split line is inside header. */ 2360 skb_split_inside_header(skb, skb1, len, pos); 2361 else /* Second chunk has no header, nothing to copy. */ 2362 skb_split_no_header(skb, skb1, len, pos); 2363 } 2364 EXPORT_SYMBOL(skb_split); 2365 2366 /* Shifting from/to a cloned skb is a no-go. 2367 * 2368 * Caller cannot keep skb_shinfo related pointers past calling here! 2369 */ 2370 static int skb_prepare_for_shift(struct sk_buff *skb) 2371 { 2372 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2373 } 2374 2375 /** 2376 * skb_shift - Shifts paged data partially from skb to another 2377 * @tgt: buffer into which tail data gets added 2378 * @skb: buffer from which the paged data comes from 2379 * @shiftlen: shift up to this many bytes 2380 * 2381 * Attempts to shift up to shiftlen worth of bytes, which may be less than 2382 * the length of the skb, from skb to tgt. Returns number bytes shifted. 2383 * It's up to caller to free skb if everything was shifted. 2384 * 2385 * If @tgt runs out of frags, the whole operation is aborted. 2386 * 2387 * Skb cannot include anything else but paged data while tgt is allowed 2388 * to have non-paged data as well. 2389 * 2390 * TODO: full sized shift could be optimized but that would need 2391 * specialized skb free'er to handle frags without up-to-date nr_frags. 2392 */ 2393 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 2394 { 2395 int from, to, merge, todo; 2396 struct skb_frag_struct *fragfrom, *fragto; 2397 2398 BUG_ON(shiftlen > skb->len); 2399 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */ 2400 2401 todo = shiftlen; 2402 from = 0; 2403 to = skb_shinfo(tgt)->nr_frags; 2404 fragfrom = &skb_shinfo(skb)->frags[from]; 2405 2406 /* Actual merge is delayed until the point when we know we can 2407 * commit all, so that we don't have to undo partial changes 2408 */ 2409 if (!to || 2410 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 2411 fragfrom->page_offset)) { 2412 merge = -1; 2413 } else { 2414 merge = to - 1; 2415 2416 todo -= skb_frag_size(fragfrom); 2417 if (todo < 0) { 2418 if (skb_prepare_for_shift(skb) || 2419 skb_prepare_for_shift(tgt)) 2420 return 0; 2421 2422 /* All previous frag pointers might be stale! */ 2423 fragfrom = &skb_shinfo(skb)->frags[from]; 2424 fragto = &skb_shinfo(tgt)->frags[merge]; 2425 2426 skb_frag_size_add(fragto, shiftlen); 2427 skb_frag_size_sub(fragfrom, shiftlen); 2428 fragfrom->page_offset += shiftlen; 2429 2430 goto onlymerged; 2431 } 2432 2433 from++; 2434 } 2435 2436 /* Skip full, not-fitting skb to avoid expensive operations */ 2437 if ((shiftlen == skb->len) && 2438 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 2439 return 0; 2440 2441 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 2442 return 0; 2443 2444 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 2445 if (to == MAX_SKB_FRAGS) 2446 return 0; 2447 2448 fragfrom = &skb_shinfo(skb)->frags[from]; 2449 fragto = &skb_shinfo(tgt)->frags[to]; 2450 2451 if (todo >= skb_frag_size(fragfrom)) { 2452 *fragto = *fragfrom; 2453 todo -= skb_frag_size(fragfrom); 2454 from++; 2455 to++; 2456 2457 } else { 2458 __skb_frag_ref(fragfrom); 2459 fragto->page = fragfrom->page; 2460 fragto->page_offset = fragfrom->page_offset; 2461 skb_frag_size_set(fragto, todo); 2462 2463 fragfrom->page_offset += todo; 2464 skb_frag_size_sub(fragfrom, todo); 2465 todo = 0; 2466 2467 to++; 2468 break; 2469 } 2470 } 2471 2472 /* Ready to "commit" this state change to tgt */ 2473 skb_shinfo(tgt)->nr_frags = to; 2474 2475 if (merge >= 0) { 2476 fragfrom = &skb_shinfo(skb)->frags[0]; 2477 fragto = &skb_shinfo(tgt)->frags[merge]; 2478 2479 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 2480 __skb_frag_unref(fragfrom); 2481 } 2482 2483 /* Reposition in the original skb */ 2484 to = 0; 2485 while (from < skb_shinfo(skb)->nr_frags) 2486 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 2487 skb_shinfo(skb)->nr_frags = to; 2488 2489 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 2490 2491 onlymerged: 2492 /* Most likely the tgt won't ever need its checksum anymore, skb on 2493 * the other hand might need it if it needs to be resent 2494 */ 2495 tgt->ip_summed = CHECKSUM_PARTIAL; 2496 skb->ip_summed = CHECKSUM_PARTIAL; 2497 2498 /* Yak, is it really working this way? Some helper please? */ 2499 skb->len -= shiftlen; 2500 skb->data_len -= shiftlen; 2501 skb->truesize -= shiftlen; 2502 tgt->len += shiftlen; 2503 tgt->data_len += shiftlen; 2504 tgt->truesize += shiftlen; 2505 2506 return shiftlen; 2507 } 2508 2509 /** 2510 * skb_prepare_seq_read - Prepare a sequential read of skb data 2511 * @skb: the buffer to read 2512 * @from: lower offset of data to be read 2513 * @to: upper offset of data to be read 2514 * @st: state variable 2515 * 2516 * Initializes the specified state variable. Must be called before 2517 * invoking skb_seq_read() for the first time. 2518 */ 2519 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 2520 unsigned int to, struct skb_seq_state *st) 2521 { 2522 st->lower_offset = from; 2523 st->upper_offset = to; 2524 st->root_skb = st->cur_skb = skb; 2525 st->frag_idx = st->stepped_offset = 0; 2526 st->frag_data = NULL; 2527 } 2528 EXPORT_SYMBOL(skb_prepare_seq_read); 2529 2530 /** 2531 * skb_seq_read - Sequentially read skb data 2532 * @consumed: number of bytes consumed by the caller so far 2533 * @data: destination pointer for data to be returned 2534 * @st: state variable 2535 * 2536 * Reads a block of skb data at &consumed relative to the 2537 * lower offset specified to skb_prepare_seq_read(). Assigns 2538 * the head of the data block to &data and returns the length 2539 * of the block or 0 if the end of the skb data or the upper 2540 * offset has been reached. 2541 * 2542 * The caller is not required to consume all of the data 2543 * returned, i.e. &consumed is typically set to the number 2544 * of bytes already consumed and the next call to 2545 * skb_seq_read() will return the remaining part of the block. 2546 * 2547 * Note 1: The size of each block of data returned can be arbitrary, 2548 * this limitation is the cost for zerocopy seqeuental 2549 * reads of potentially non linear data. 2550 * 2551 * Note 2: Fragment lists within fragments are not implemented 2552 * at the moment, state->root_skb could be replaced with 2553 * a stack for this purpose. 2554 */ 2555 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 2556 struct skb_seq_state *st) 2557 { 2558 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 2559 skb_frag_t *frag; 2560 2561 if (unlikely(abs_offset >= st->upper_offset)) 2562 return 0; 2563 2564 next_skb: 2565 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 2566 2567 if (abs_offset < block_limit && !st->frag_data) { 2568 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 2569 return block_limit - abs_offset; 2570 } 2571 2572 if (st->frag_idx == 0 && !st->frag_data) 2573 st->stepped_offset += skb_headlen(st->cur_skb); 2574 2575 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 2576 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 2577 block_limit = skb_frag_size(frag) + st->stepped_offset; 2578 2579 if (abs_offset < block_limit) { 2580 if (!st->frag_data) 2581 st->frag_data = kmap_atomic(skb_frag_page(frag)); 2582 2583 *data = (u8 *) st->frag_data + frag->page_offset + 2584 (abs_offset - st->stepped_offset); 2585 2586 return block_limit - abs_offset; 2587 } 2588 2589 if (st->frag_data) { 2590 kunmap_atomic(st->frag_data); 2591 st->frag_data = NULL; 2592 } 2593 2594 st->frag_idx++; 2595 st->stepped_offset += skb_frag_size(frag); 2596 } 2597 2598 if (st->frag_data) { 2599 kunmap_atomic(st->frag_data); 2600 st->frag_data = NULL; 2601 } 2602 2603 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 2604 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 2605 st->frag_idx = 0; 2606 goto next_skb; 2607 } else if (st->cur_skb->next) { 2608 st->cur_skb = st->cur_skb->next; 2609 st->frag_idx = 0; 2610 goto next_skb; 2611 } 2612 2613 return 0; 2614 } 2615 EXPORT_SYMBOL(skb_seq_read); 2616 2617 /** 2618 * skb_abort_seq_read - Abort a sequential read of skb data 2619 * @st: state variable 2620 * 2621 * Must be called if skb_seq_read() was not called until it 2622 * returned 0. 2623 */ 2624 void skb_abort_seq_read(struct skb_seq_state *st) 2625 { 2626 if (st->frag_data) 2627 kunmap_atomic(st->frag_data); 2628 } 2629 EXPORT_SYMBOL(skb_abort_seq_read); 2630 2631 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 2632 2633 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 2634 struct ts_config *conf, 2635 struct ts_state *state) 2636 { 2637 return skb_seq_read(offset, text, TS_SKB_CB(state)); 2638 } 2639 2640 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 2641 { 2642 skb_abort_seq_read(TS_SKB_CB(state)); 2643 } 2644 2645 /** 2646 * skb_find_text - Find a text pattern in skb data 2647 * @skb: the buffer to look in 2648 * @from: search offset 2649 * @to: search limit 2650 * @config: textsearch configuration 2651 * @state: uninitialized textsearch state variable 2652 * 2653 * Finds a pattern in the skb data according to the specified 2654 * textsearch configuration. Use textsearch_next() to retrieve 2655 * subsequent occurrences of the pattern. Returns the offset 2656 * to the first occurrence or UINT_MAX if no match was found. 2657 */ 2658 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 2659 unsigned int to, struct ts_config *config, 2660 struct ts_state *state) 2661 { 2662 unsigned int ret; 2663 2664 config->get_next_block = skb_ts_get_next_block; 2665 config->finish = skb_ts_finish; 2666 2667 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state)); 2668 2669 ret = textsearch_find(config, state); 2670 return (ret <= to - from ? ret : UINT_MAX); 2671 } 2672 EXPORT_SYMBOL(skb_find_text); 2673 2674 /** 2675 * skb_append_datato_frags - append the user data to a skb 2676 * @sk: sock structure 2677 * @skb: skb structure to be appened with user data. 2678 * @getfrag: call back function to be used for getting the user data 2679 * @from: pointer to user message iov 2680 * @length: length of the iov message 2681 * 2682 * Description: This procedure append the user data in the fragment part 2683 * of the skb if any page alloc fails user this procedure returns -ENOMEM 2684 */ 2685 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 2686 int (*getfrag)(void *from, char *to, int offset, 2687 int len, int odd, struct sk_buff *skb), 2688 void *from, int length) 2689 { 2690 int frg_cnt = skb_shinfo(skb)->nr_frags; 2691 int copy; 2692 int offset = 0; 2693 int ret; 2694 struct page_frag *pfrag = ¤t->task_frag; 2695 2696 do { 2697 /* Return error if we don't have space for new frag */ 2698 if (frg_cnt >= MAX_SKB_FRAGS) 2699 return -EMSGSIZE; 2700 2701 if (!sk_page_frag_refill(sk, pfrag)) 2702 return -ENOMEM; 2703 2704 /* copy the user data to page */ 2705 copy = min_t(int, length, pfrag->size - pfrag->offset); 2706 2707 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset, 2708 offset, copy, 0, skb); 2709 if (ret < 0) 2710 return -EFAULT; 2711 2712 /* copy was successful so update the size parameters */ 2713 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset, 2714 copy); 2715 frg_cnt++; 2716 pfrag->offset += copy; 2717 get_page(pfrag->page); 2718 2719 skb->truesize += copy; 2720 atomic_add(copy, &sk->sk_wmem_alloc); 2721 skb->len += copy; 2722 skb->data_len += copy; 2723 offset += copy; 2724 length -= copy; 2725 2726 } while (length > 0); 2727 2728 return 0; 2729 } 2730 EXPORT_SYMBOL(skb_append_datato_frags); 2731 2732 /** 2733 * skb_pull_rcsum - pull skb and update receive checksum 2734 * @skb: buffer to update 2735 * @len: length of data pulled 2736 * 2737 * This function performs an skb_pull on the packet and updates 2738 * the CHECKSUM_COMPLETE checksum. It should be used on 2739 * receive path processing instead of skb_pull unless you know 2740 * that the checksum difference is zero (e.g., a valid IP header) 2741 * or you are setting ip_summed to CHECKSUM_NONE. 2742 */ 2743 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 2744 { 2745 BUG_ON(len > skb->len); 2746 skb->len -= len; 2747 BUG_ON(skb->len < skb->data_len); 2748 skb_postpull_rcsum(skb, skb->data, len); 2749 return skb->data += len; 2750 } 2751 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 2752 2753 /** 2754 * skb_segment - Perform protocol segmentation on skb. 2755 * @skb: buffer to segment 2756 * @features: features for the output path (see dev->features) 2757 * 2758 * This function performs segmentation on the given skb. It returns 2759 * a pointer to the first in a list of new skbs for the segments. 2760 * In case of error it returns ERR_PTR(err). 2761 */ 2762 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features) 2763 { 2764 struct sk_buff *segs = NULL; 2765 struct sk_buff *tail = NULL; 2766 struct sk_buff *fskb = skb_shinfo(skb)->frag_list; 2767 unsigned int mss = skb_shinfo(skb)->gso_size; 2768 unsigned int doffset = skb->data - skb_mac_header(skb); 2769 unsigned int offset = doffset; 2770 unsigned int tnl_hlen = skb_tnl_header_len(skb); 2771 unsigned int headroom; 2772 unsigned int len; 2773 __be16 proto; 2774 bool csum; 2775 int sg = !!(features & NETIF_F_SG); 2776 int nfrags = skb_shinfo(skb)->nr_frags; 2777 int err = -ENOMEM; 2778 int i = 0; 2779 int pos; 2780 2781 proto = skb_network_protocol(skb); 2782 if (unlikely(!proto)) 2783 return ERR_PTR(-EINVAL); 2784 2785 csum = !!can_checksum_protocol(features, proto); 2786 __skb_push(skb, doffset); 2787 headroom = skb_headroom(skb); 2788 pos = skb_headlen(skb); 2789 2790 do { 2791 struct sk_buff *nskb; 2792 skb_frag_t *frag; 2793 int hsize; 2794 int size; 2795 2796 len = skb->len - offset; 2797 if (len > mss) 2798 len = mss; 2799 2800 hsize = skb_headlen(skb) - offset; 2801 if (hsize < 0) 2802 hsize = 0; 2803 if (hsize > len || !sg) 2804 hsize = len; 2805 2806 if (!hsize && i >= nfrags) { 2807 BUG_ON(fskb->len != len); 2808 2809 pos += len; 2810 nskb = skb_clone(fskb, GFP_ATOMIC); 2811 fskb = fskb->next; 2812 2813 if (unlikely(!nskb)) 2814 goto err; 2815 2816 hsize = skb_end_offset(nskb); 2817 if (skb_cow_head(nskb, doffset + headroom)) { 2818 kfree_skb(nskb); 2819 goto err; 2820 } 2821 2822 nskb->truesize += skb_end_offset(nskb) - hsize; 2823 skb_release_head_state(nskb); 2824 __skb_push(nskb, doffset); 2825 } else { 2826 nskb = __alloc_skb(hsize + doffset + headroom, 2827 GFP_ATOMIC, skb_alloc_rx_flag(skb), 2828 NUMA_NO_NODE); 2829 2830 if (unlikely(!nskb)) 2831 goto err; 2832 2833 skb_reserve(nskb, headroom); 2834 __skb_put(nskb, doffset); 2835 } 2836 2837 if (segs) 2838 tail->next = nskb; 2839 else 2840 segs = nskb; 2841 tail = nskb; 2842 2843 __copy_skb_header(nskb, skb); 2844 nskb->mac_len = skb->mac_len; 2845 2846 /* nskb and skb might have different headroom */ 2847 if (nskb->ip_summed == CHECKSUM_PARTIAL) 2848 nskb->csum_start += skb_headroom(nskb) - headroom; 2849 2850 skb_reset_mac_header(nskb); 2851 skb_set_network_header(nskb, skb->mac_len); 2852 nskb->transport_header = (nskb->network_header + 2853 skb_network_header_len(skb)); 2854 2855 skb_copy_from_linear_data_offset(skb, -tnl_hlen, 2856 nskb->data - tnl_hlen, 2857 doffset + tnl_hlen); 2858 2859 if (fskb != skb_shinfo(skb)->frag_list) 2860 continue; 2861 2862 if (!sg) { 2863 nskb->ip_summed = CHECKSUM_NONE; 2864 nskb->csum = skb_copy_and_csum_bits(skb, offset, 2865 skb_put(nskb, len), 2866 len, 0); 2867 continue; 2868 } 2869 2870 frag = skb_shinfo(nskb)->frags; 2871 2872 skb_copy_from_linear_data_offset(skb, offset, 2873 skb_put(nskb, hsize), hsize); 2874 2875 skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; 2876 2877 while (pos < offset + len && i < nfrags) { 2878 *frag = skb_shinfo(skb)->frags[i]; 2879 __skb_frag_ref(frag); 2880 size = skb_frag_size(frag); 2881 2882 if (pos < offset) { 2883 frag->page_offset += offset - pos; 2884 skb_frag_size_sub(frag, offset - pos); 2885 } 2886 2887 skb_shinfo(nskb)->nr_frags++; 2888 2889 if (pos + size <= offset + len) { 2890 i++; 2891 pos += size; 2892 } else { 2893 skb_frag_size_sub(frag, pos + size - (offset + len)); 2894 goto skip_fraglist; 2895 } 2896 2897 frag++; 2898 } 2899 2900 if (pos < offset + len) { 2901 struct sk_buff *fskb2 = fskb; 2902 2903 BUG_ON(pos + fskb->len != offset + len); 2904 2905 pos += fskb->len; 2906 fskb = fskb->next; 2907 2908 if (fskb2->next) { 2909 fskb2 = skb_clone(fskb2, GFP_ATOMIC); 2910 if (!fskb2) 2911 goto err; 2912 } else 2913 skb_get(fskb2); 2914 2915 SKB_FRAG_ASSERT(nskb); 2916 skb_shinfo(nskb)->frag_list = fskb2; 2917 } 2918 2919 skip_fraglist: 2920 nskb->data_len = len - hsize; 2921 nskb->len += nskb->data_len; 2922 nskb->truesize += nskb->data_len; 2923 2924 if (!csum) { 2925 nskb->csum = skb_checksum(nskb, doffset, 2926 nskb->len - doffset, 0); 2927 nskb->ip_summed = CHECKSUM_NONE; 2928 } 2929 } while ((offset += len) < skb->len); 2930 2931 return segs; 2932 2933 err: 2934 while ((skb = segs)) { 2935 segs = skb->next; 2936 kfree_skb(skb); 2937 } 2938 return ERR_PTR(err); 2939 } 2940 EXPORT_SYMBOL_GPL(skb_segment); 2941 2942 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 2943 { 2944 struct sk_buff *p = *head; 2945 struct sk_buff *nskb; 2946 struct skb_shared_info *skbinfo = skb_shinfo(skb); 2947 struct skb_shared_info *pinfo = skb_shinfo(p); 2948 unsigned int headroom; 2949 unsigned int len = skb_gro_len(skb); 2950 unsigned int offset = skb_gro_offset(skb); 2951 unsigned int headlen = skb_headlen(skb); 2952 unsigned int delta_truesize; 2953 2954 if (p->len + len >= 65536) 2955 return -E2BIG; 2956 2957 if (pinfo->frag_list) 2958 goto merge; 2959 else if (headlen <= offset) { 2960 skb_frag_t *frag; 2961 skb_frag_t *frag2; 2962 int i = skbinfo->nr_frags; 2963 int nr_frags = pinfo->nr_frags + i; 2964 2965 offset -= headlen; 2966 2967 if (nr_frags > MAX_SKB_FRAGS) 2968 return -E2BIG; 2969 2970 pinfo->nr_frags = nr_frags; 2971 skbinfo->nr_frags = 0; 2972 2973 frag = pinfo->frags + nr_frags; 2974 frag2 = skbinfo->frags + i; 2975 do { 2976 *--frag = *--frag2; 2977 } while (--i); 2978 2979 frag->page_offset += offset; 2980 skb_frag_size_sub(frag, offset); 2981 2982 /* all fragments truesize : remove (head size + sk_buff) */ 2983 delta_truesize = skb->truesize - 2984 SKB_TRUESIZE(skb_end_offset(skb)); 2985 2986 skb->truesize -= skb->data_len; 2987 skb->len -= skb->data_len; 2988 skb->data_len = 0; 2989 2990 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 2991 goto done; 2992 } else if (skb->head_frag) { 2993 int nr_frags = pinfo->nr_frags; 2994 skb_frag_t *frag = pinfo->frags + nr_frags; 2995 struct page *page = virt_to_head_page(skb->head); 2996 unsigned int first_size = headlen - offset; 2997 unsigned int first_offset; 2998 2999 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 3000 return -E2BIG; 3001 3002 first_offset = skb->data - 3003 (unsigned char *)page_address(page) + 3004 offset; 3005 3006 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 3007 3008 frag->page.p = page; 3009 frag->page_offset = first_offset; 3010 skb_frag_size_set(frag, first_size); 3011 3012 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 3013 /* We dont need to clear skbinfo->nr_frags here */ 3014 3015 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3016 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 3017 goto done; 3018 } else if (skb_gro_len(p) != pinfo->gso_size) 3019 return -E2BIG; 3020 3021 headroom = skb_headroom(p); 3022 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC); 3023 if (unlikely(!nskb)) 3024 return -ENOMEM; 3025 3026 __copy_skb_header(nskb, p); 3027 nskb->mac_len = p->mac_len; 3028 3029 skb_reserve(nskb, headroom); 3030 __skb_put(nskb, skb_gro_offset(p)); 3031 3032 skb_set_mac_header(nskb, skb_mac_header(p) - p->data); 3033 skb_set_network_header(nskb, skb_network_offset(p)); 3034 skb_set_transport_header(nskb, skb_transport_offset(p)); 3035 3036 __skb_pull(p, skb_gro_offset(p)); 3037 memcpy(skb_mac_header(nskb), skb_mac_header(p), 3038 p->data - skb_mac_header(p)); 3039 3040 skb_shinfo(nskb)->frag_list = p; 3041 skb_shinfo(nskb)->gso_size = pinfo->gso_size; 3042 pinfo->gso_size = 0; 3043 skb_header_release(p); 3044 NAPI_GRO_CB(nskb)->last = p; 3045 3046 nskb->data_len += p->len; 3047 nskb->truesize += p->truesize; 3048 nskb->len += p->len; 3049 3050 *head = nskb; 3051 nskb->next = p->next; 3052 p->next = NULL; 3053 3054 p = nskb; 3055 3056 merge: 3057 delta_truesize = skb->truesize; 3058 if (offset > headlen) { 3059 unsigned int eat = offset - headlen; 3060 3061 skbinfo->frags[0].page_offset += eat; 3062 skb_frag_size_sub(&skbinfo->frags[0], eat); 3063 skb->data_len -= eat; 3064 skb->len -= eat; 3065 offset = headlen; 3066 } 3067 3068 __skb_pull(skb, offset); 3069 3070 NAPI_GRO_CB(p)->last->next = skb; 3071 NAPI_GRO_CB(p)->last = skb; 3072 skb_header_release(skb); 3073 3074 done: 3075 NAPI_GRO_CB(p)->count++; 3076 p->data_len += len; 3077 p->truesize += delta_truesize; 3078 p->len += len; 3079 3080 NAPI_GRO_CB(skb)->same_flow = 1; 3081 return 0; 3082 } 3083 EXPORT_SYMBOL_GPL(skb_gro_receive); 3084 3085 void __init skb_init(void) 3086 { 3087 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 3088 sizeof(struct sk_buff), 3089 0, 3090 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3091 NULL); 3092 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 3093 (2*sizeof(struct sk_buff)) + 3094 sizeof(atomic_t), 3095 0, 3096 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3097 NULL); 3098 } 3099 3100 /** 3101 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 3102 * @skb: Socket buffer containing the buffers to be mapped 3103 * @sg: The scatter-gather list to map into 3104 * @offset: The offset into the buffer's contents to start mapping 3105 * @len: Length of buffer space to be mapped 3106 * 3107 * Fill the specified scatter-gather list with mappings/pointers into a 3108 * region of the buffer space attached to a socket buffer. 3109 */ 3110 static int 3111 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3112 { 3113 int start = skb_headlen(skb); 3114 int i, copy = start - offset; 3115 struct sk_buff *frag_iter; 3116 int elt = 0; 3117 3118 if (copy > 0) { 3119 if (copy > len) 3120 copy = len; 3121 sg_set_buf(sg, skb->data + offset, copy); 3122 elt++; 3123 if ((len -= copy) == 0) 3124 return elt; 3125 offset += copy; 3126 } 3127 3128 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3129 int end; 3130 3131 WARN_ON(start > offset + len); 3132 3133 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3134 if ((copy = end - offset) > 0) { 3135 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3136 3137 if (copy > len) 3138 copy = len; 3139 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 3140 frag->page_offset+offset-start); 3141 elt++; 3142 if (!(len -= copy)) 3143 return elt; 3144 offset += copy; 3145 } 3146 start = end; 3147 } 3148 3149 skb_walk_frags(skb, frag_iter) { 3150 int end; 3151 3152 WARN_ON(start > offset + len); 3153 3154 end = start + frag_iter->len; 3155 if ((copy = end - offset) > 0) { 3156 if (copy > len) 3157 copy = len; 3158 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start, 3159 copy); 3160 if ((len -= copy) == 0) 3161 return elt; 3162 offset += copy; 3163 } 3164 start = end; 3165 } 3166 BUG_ON(len); 3167 return elt; 3168 } 3169 3170 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3171 { 3172 int nsg = __skb_to_sgvec(skb, sg, offset, len); 3173 3174 sg_mark_end(&sg[nsg - 1]); 3175 3176 return nsg; 3177 } 3178 EXPORT_SYMBOL_GPL(skb_to_sgvec); 3179 3180 /** 3181 * skb_cow_data - Check that a socket buffer's data buffers are writable 3182 * @skb: The socket buffer to check. 3183 * @tailbits: Amount of trailing space to be added 3184 * @trailer: Returned pointer to the skb where the @tailbits space begins 3185 * 3186 * Make sure that the data buffers attached to a socket buffer are 3187 * writable. If they are not, private copies are made of the data buffers 3188 * and the socket buffer is set to use these instead. 3189 * 3190 * If @tailbits is given, make sure that there is space to write @tailbits 3191 * bytes of data beyond current end of socket buffer. @trailer will be 3192 * set to point to the skb in which this space begins. 3193 * 3194 * The number of scatterlist elements required to completely map the 3195 * COW'd and extended socket buffer will be returned. 3196 */ 3197 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 3198 { 3199 int copyflag; 3200 int elt; 3201 struct sk_buff *skb1, **skb_p; 3202 3203 /* If skb is cloned or its head is paged, reallocate 3204 * head pulling out all the pages (pages are considered not writable 3205 * at the moment even if they are anonymous). 3206 */ 3207 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 3208 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 3209 return -ENOMEM; 3210 3211 /* Easy case. Most of packets will go this way. */ 3212 if (!skb_has_frag_list(skb)) { 3213 /* A little of trouble, not enough of space for trailer. 3214 * This should not happen, when stack is tuned to generate 3215 * good frames. OK, on miss we reallocate and reserve even more 3216 * space, 128 bytes is fair. */ 3217 3218 if (skb_tailroom(skb) < tailbits && 3219 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 3220 return -ENOMEM; 3221 3222 /* Voila! */ 3223 *trailer = skb; 3224 return 1; 3225 } 3226 3227 /* Misery. We are in troubles, going to mincer fragments... */ 3228 3229 elt = 1; 3230 skb_p = &skb_shinfo(skb)->frag_list; 3231 copyflag = 0; 3232 3233 while ((skb1 = *skb_p) != NULL) { 3234 int ntail = 0; 3235 3236 /* The fragment is partially pulled by someone, 3237 * this can happen on input. Copy it and everything 3238 * after it. */ 3239 3240 if (skb_shared(skb1)) 3241 copyflag = 1; 3242 3243 /* If the skb is the last, worry about trailer. */ 3244 3245 if (skb1->next == NULL && tailbits) { 3246 if (skb_shinfo(skb1)->nr_frags || 3247 skb_has_frag_list(skb1) || 3248 skb_tailroom(skb1) < tailbits) 3249 ntail = tailbits + 128; 3250 } 3251 3252 if (copyflag || 3253 skb_cloned(skb1) || 3254 ntail || 3255 skb_shinfo(skb1)->nr_frags || 3256 skb_has_frag_list(skb1)) { 3257 struct sk_buff *skb2; 3258 3259 /* Fuck, we are miserable poor guys... */ 3260 if (ntail == 0) 3261 skb2 = skb_copy(skb1, GFP_ATOMIC); 3262 else 3263 skb2 = skb_copy_expand(skb1, 3264 skb_headroom(skb1), 3265 ntail, 3266 GFP_ATOMIC); 3267 if (unlikely(skb2 == NULL)) 3268 return -ENOMEM; 3269 3270 if (skb1->sk) 3271 skb_set_owner_w(skb2, skb1->sk); 3272 3273 /* Looking around. Are we still alive? 3274 * OK, link new skb, drop old one */ 3275 3276 skb2->next = skb1->next; 3277 *skb_p = skb2; 3278 kfree_skb(skb1); 3279 skb1 = skb2; 3280 } 3281 elt++; 3282 *trailer = skb1; 3283 skb_p = &skb1->next; 3284 } 3285 3286 return elt; 3287 } 3288 EXPORT_SYMBOL_GPL(skb_cow_data); 3289 3290 static void sock_rmem_free(struct sk_buff *skb) 3291 { 3292 struct sock *sk = skb->sk; 3293 3294 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 3295 } 3296 3297 /* 3298 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 3299 */ 3300 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 3301 { 3302 int len = skb->len; 3303 3304 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 3305 (unsigned int)sk->sk_rcvbuf) 3306 return -ENOMEM; 3307 3308 skb_orphan(skb); 3309 skb->sk = sk; 3310 skb->destructor = sock_rmem_free; 3311 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 3312 3313 /* before exiting rcu section, make sure dst is refcounted */ 3314 skb_dst_force(skb); 3315 3316 skb_queue_tail(&sk->sk_error_queue, skb); 3317 if (!sock_flag(sk, SOCK_DEAD)) 3318 sk->sk_data_ready(sk, len); 3319 return 0; 3320 } 3321 EXPORT_SYMBOL(sock_queue_err_skb); 3322 3323 void skb_tstamp_tx(struct sk_buff *orig_skb, 3324 struct skb_shared_hwtstamps *hwtstamps) 3325 { 3326 struct sock *sk = orig_skb->sk; 3327 struct sock_exterr_skb *serr; 3328 struct sk_buff *skb; 3329 int err; 3330 3331 if (!sk) 3332 return; 3333 3334 if (hwtstamps) { 3335 *skb_hwtstamps(orig_skb) = 3336 *hwtstamps; 3337 } else { 3338 /* 3339 * no hardware time stamps available, 3340 * so keep the shared tx_flags and only 3341 * store software time stamp 3342 */ 3343 orig_skb->tstamp = ktime_get_real(); 3344 } 3345 3346 skb = skb_clone(orig_skb, GFP_ATOMIC); 3347 if (!skb) 3348 return; 3349 3350 serr = SKB_EXT_ERR(skb); 3351 memset(serr, 0, sizeof(*serr)); 3352 serr->ee.ee_errno = ENOMSG; 3353 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 3354 3355 err = sock_queue_err_skb(sk, skb); 3356 3357 if (err) 3358 kfree_skb(skb); 3359 } 3360 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 3361 3362 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 3363 { 3364 struct sock *sk = skb->sk; 3365 struct sock_exterr_skb *serr; 3366 int err; 3367 3368 skb->wifi_acked_valid = 1; 3369 skb->wifi_acked = acked; 3370 3371 serr = SKB_EXT_ERR(skb); 3372 memset(serr, 0, sizeof(*serr)); 3373 serr->ee.ee_errno = ENOMSG; 3374 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 3375 3376 err = sock_queue_err_skb(sk, skb); 3377 if (err) 3378 kfree_skb(skb); 3379 } 3380 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 3381 3382 3383 /** 3384 * skb_partial_csum_set - set up and verify partial csum values for packet 3385 * @skb: the skb to set 3386 * @start: the number of bytes after skb->data to start checksumming. 3387 * @off: the offset from start to place the checksum. 3388 * 3389 * For untrusted partially-checksummed packets, we need to make sure the values 3390 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 3391 * 3392 * This function checks and sets those values and skb->ip_summed: if this 3393 * returns false you should drop the packet. 3394 */ 3395 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 3396 { 3397 if (unlikely(start > skb_headlen(skb)) || 3398 unlikely((int)start + off > skb_headlen(skb) - 2)) { 3399 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n", 3400 start, off, skb_headlen(skb)); 3401 return false; 3402 } 3403 skb->ip_summed = CHECKSUM_PARTIAL; 3404 skb->csum_start = skb_headroom(skb) + start; 3405 skb->csum_offset = off; 3406 skb_set_transport_header(skb, start); 3407 return true; 3408 } 3409 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 3410 3411 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 3412 { 3413 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 3414 skb->dev->name); 3415 } 3416 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 3417 3418 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 3419 { 3420 if (head_stolen) { 3421 skb_release_head_state(skb); 3422 kmem_cache_free(skbuff_head_cache, skb); 3423 } else { 3424 __kfree_skb(skb); 3425 } 3426 } 3427 EXPORT_SYMBOL(kfree_skb_partial); 3428 3429 /** 3430 * skb_try_coalesce - try to merge skb to prior one 3431 * @to: prior buffer 3432 * @from: buffer to add 3433 * @fragstolen: pointer to boolean 3434 * @delta_truesize: how much more was allocated than was requested 3435 */ 3436 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 3437 bool *fragstolen, int *delta_truesize) 3438 { 3439 int i, delta, len = from->len; 3440 3441 *fragstolen = false; 3442 3443 if (skb_cloned(to)) 3444 return false; 3445 3446 if (len <= skb_tailroom(to)) { 3447 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 3448 *delta_truesize = 0; 3449 return true; 3450 } 3451 3452 if (skb_has_frag_list(to) || skb_has_frag_list(from)) 3453 return false; 3454 3455 if (skb_headlen(from) != 0) { 3456 struct page *page; 3457 unsigned int offset; 3458 3459 if (skb_shinfo(to)->nr_frags + 3460 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 3461 return false; 3462 3463 if (skb_head_is_locked(from)) 3464 return false; 3465 3466 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3467 3468 page = virt_to_head_page(from->head); 3469 offset = from->data - (unsigned char *)page_address(page); 3470 3471 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags, 3472 page, offset, skb_headlen(from)); 3473 *fragstolen = true; 3474 } else { 3475 if (skb_shinfo(to)->nr_frags + 3476 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS) 3477 return false; 3478 3479 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 3480 } 3481 3482 WARN_ON_ONCE(delta < len); 3483 3484 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags, 3485 skb_shinfo(from)->frags, 3486 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t)); 3487 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags; 3488 3489 if (!skb_cloned(from)) 3490 skb_shinfo(from)->nr_frags = 0; 3491 3492 /* if the skb is not cloned this does nothing 3493 * since we set nr_frags to 0. 3494 */ 3495 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) 3496 skb_frag_ref(from, i); 3497 3498 to->truesize += delta; 3499 to->len += len; 3500 to->data_len += len; 3501 3502 *delta_truesize = delta; 3503 return true; 3504 } 3505 EXPORT_SYMBOL(skb_try_coalesce); 3506