1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <net/psp.h>
3 #include <net/gro.h>
4 #include <net/dst_metadata.h>
5 #include <net/busy_poll.h>
6 #include <trace/events/net.h>
7 #include <linux/skbuff_ref.h>
8
9 #define MAX_GRO_SKBS 8
10
11 static DEFINE_SPINLOCK(offload_lock);
12
13 /**
14 * dev_add_offload - register offload handlers
15 * @po: protocol offload declaration
16 *
17 * Add protocol offload handlers to the networking stack. The passed
18 * &proto_offload is linked into kernel lists and may not be freed until
19 * it has been removed from the kernel lists.
20 *
21 * This call does not sleep therefore it can not
22 * guarantee all CPU's that are in middle of receiving packets
23 * will see the new offload handlers (until the next received packet).
24 */
dev_add_offload(struct packet_offload * po)25 void dev_add_offload(struct packet_offload *po)
26 {
27 struct packet_offload *elem;
28
29 spin_lock(&offload_lock);
30 list_for_each_entry(elem, &net_hotdata.offload_base, list) {
31 if (po->priority < elem->priority)
32 break;
33 }
34 list_add_rcu(&po->list, elem->list.prev);
35 spin_unlock(&offload_lock);
36 }
37 EXPORT_SYMBOL(dev_add_offload);
38
39 /**
40 * __dev_remove_offload - remove offload handler
41 * @po: packet offload declaration
42 *
43 * Remove a protocol offload handler that was previously added to the
44 * kernel offload handlers by dev_add_offload(). The passed &offload_type
45 * is removed from the kernel lists and can be freed or reused once this
46 * function returns.
47 *
48 * The packet type might still be in use by receivers
49 * and must not be freed until after all the CPU's have gone
50 * through a quiescent state.
51 */
__dev_remove_offload(struct packet_offload * po)52 static void __dev_remove_offload(struct packet_offload *po)
53 {
54 struct list_head *head = &net_hotdata.offload_base;
55 struct packet_offload *po1;
56
57 spin_lock(&offload_lock);
58
59 list_for_each_entry(po1, head, list) {
60 if (po == po1) {
61 list_del_rcu(&po->list);
62 goto out;
63 }
64 }
65
66 pr_warn("dev_remove_offload: %p not found\n", po);
67 out:
68 spin_unlock(&offload_lock);
69 }
70
71 /**
72 * dev_remove_offload - remove packet offload handler
73 * @po: packet offload declaration
74 *
75 * Remove a packet offload handler that was previously added to the kernel
76 * offload handlers by dev_add_offload(). The passed &offload_type is
77 * removed from the kernel lists and can be freed or reused once this
78 * function returns.
79 *
80 * This call sleeps to guarantee that no CPU is looking at the packet
81 * type after return.
82 */
dev_remove_offload(struct packet_offload * po)83 void dev_remove_offload(struct packet_offload *po)
84 {
85 __dev_remove_offload(po);
86
87 synchronize_net();
88 }
89 EXPORT_SYMBOL(dev_remove_offload);
90
91
skb_gro_receive(struct sk_buff * p,struct sk_buff * skb)92 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
93 {
94 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
95 unsigned int offset = skb_gro_offset(skb);
96 unsigned int headlen = skb_headlen(skb);
97 unsigned int len = skb_gro_len(skb);
98 unsigned int delta_truesize;
99 unsigned int new_truesize;
100 struct sk_buff *lp;
101 int segs;
102
103 /* Do not splice page pool based packets w/ non-page pool
104 * packets. This can result in reference count issues as page
105 * pool pages will not decrement the reference count and will
106 * instead be immediately returned to the pool or have frag
107 * count decremented.
108 */
109 if (p->pp_recycle != skb->pp_recycle)
110 return -ETOOMANYREFS;
111
112 if (skb_zcopy(p) || skb_zcopy(skb))
113 return -ETOOMANYREFS;
114
115 if (unlikely(p->len + len >= netif_get_gro_max_size(p->dev, p) ||
116 NAPI_GRO_CB(skb)->flush))
117 return -E2BIG;
118
119 if (unlikely(p->len + len >= GRO_LEGACY_MAX_SIZE)) {
120 if (NAPI_GRO_CB(skb)->proto != IPPROTO_TCP ||
121 p->encapsulation)
122 return -E2BIG;
123 }
124
125 segs = NAPI_GRO_CB(skb)->count;
126 lp = NAPI_GRO_CB(p)->last;
127 pinfo = skb_shinfo(lp);
128
129 if (headlen <= offset) {
130 skb_frag_t *frag;
131 skb_frag_t *frag2;
132 int i = skbinfo->nr_frags;
133 int nr_frags = pinfo->nr_frags + i;
134
135 if (nr_frags > MAX_SKB_FRAGS)
136 goto merge;
137
138 offset -= headlen;
139 pinfo->nr_frags = nr_frags;
140 skbinfo->nr_frags = 0;
141
142 frag = pinfo->frags + nr_frags;
143 frag2 = skbinfo->frags + i;
144 do {
145 *--frag = *--frag2;
146 } while (--i);
147
148 skb_frag_off_add(frag, offset);
149 skb_frag_size_sub(frag, offset);
150
151 /* all fragments truesize : remove (head size + sk_buff) */
152 new_truesize = SKB_TRUESIZE(skb_end_offset(skb));
153 delta_truesize = skb->truesize - new_truesize;
154
155 skb->truesize = new_truesize;
156 skb->len -= skb->data_len;
157 skb->data_len = 0;
158
159 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
160 goto done;
161 } else if (skb->head_frag) {
162 int nr_frags = pinfo->nr_frags;
163 skb_frag_t *frag = pinfo->frags + nr_frags;
164 struct page *page = virt_to_head_page(skb->head);
165 unsigned int first_size = headlen - offset;
166 unsigned int first_offset;
167
168 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
169 goto merge;
170
171 first_offset = skb->data -
172 (unsigned char *)page_address(page) +
173 offset;
174
175 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
176
177 skb_frag_fill_page_desc(frag, page, first_offset, first_size);
178
179 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
180 /* We dont need to clear skbinfo->nr_frags here */
181
182 new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
183 delta_truesize = skb->truesize - new_truesize;
184 skb->truesize = new_truesize;
185 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
186 goto done;
187 }
188
189 merge:
190 /* sk ownership - if any - completely transferred to the aggregated packet */
191 skb->destructor = NULL;
192 skb->sk = NULL;
193 delta_truesize = skb->truesize;
194 if (offset > headlen) {
195 unsigned int eat = offset - headlen;
196
197 skb_frag_off_add(&skbinfo->frags[0], eat);
198 skb_frag_size_sub(&skbinfo->frags[0], eat);
199 skb->data_len -= eat;
200 skb->len -= eat;
201 offset = headlen;
202 }
203
204 __skb_pull(skb, offset);
205
206 if (NAPI_GRO_CB(p)->last == p)
207 skb_shinfo(p)->frag_list = skb;
208 else
209 NAPI_GRO_CB(p)->last->next = skb;
210 NAPI_GRO_CB(p)->last = skb;
211 __skb_header_release(skb);
212 lp = p;
213
214 done:
215 NAPI_GRO_CB(p)->count += segs;
216 p->data_len += len;
217 p->truesize += delta_truesize;
218 p->len += len;
219 skb_shinfo(p)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
220 if (lp != p) {
221 lp->data_len += len;
222 lp->truesize += delta_truesize;
223 lp->len += len;
224 skb_shinfo(lp)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG;
225 }
226 NAPI_GRO_CB(skb)->same_flow = 1;
227 return 0;
228 }
229
skb_gro_receive_list(struct sk_buff * p,struct sk_buff * skb)230 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
231 {
232 if (unlikely(p->len + skb->len >= 65536))
233 return -E2BIG;
234
235 if (NAPI_GRO_CB(p)->last == p)
236 skb_shinfo(p)->frag_list = skb;
237 else
238 NAPI_GRO_CB(p)->last->next = skb;
239
240 skb_pull(skb, skb_gro_offset(skb));
241
242 NAPI_GRO_CB(p)->last = skb;
243 NAPI_GRO_CB(p)->count++;
244 p->data_len += skb->len;
245
246 /* sk ownership - if any - completely transferred to the aggregated packet */
247 skb->destructor = NULL;
248 skb->sk = NULL;
249 p->truesize += skb->truesize;
250 p->len += skb->len;
251
252 skb_shinfo(p)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
253
254 NAPI_GRO_CB(skb)->same_flow = 1;
255
256 return 0;
257 }
258
gro_complete(struct gro_node * gro,struct sk_buff * skb)259 static void gro_complete(struct gro_node *gro, struct sk_buff *skb)
260 {
261 struct list_head *head = &net_hotdata.offload_base;
262 struct packet_offload *ptype;
263 __be16 type = skb->protocol;
264 int err = -ENOENT;
265
266 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
267
268 if (NAPI_GRO_CB(skb)->count == 1) {
269 skb_shinfo(skb)->gso_size = 0;
270 goto out;
271 }
272
273 /* NICs can feed encapsulated packets into GRO */
274 skb->encapsulation = 0;
275 rcu_read_lock();
276 list_for_each_entry_rcu(ptype, head, list) {
277 if (ptype->type != type || !ptype->callbacks.gro_complete)
278 continue;
279
280 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
281 ipv6_gro_complete, inet_gro_complete,
282 skb, 0);
283 break;
284 }
285 rcu_read_unlock();
286
287 if (err) {
288 WARN_ON(&ptype->list == head);
289 kfree_skb(skb);
290 return;
291 }
292
293 out:
294 gro_normal_one(gro, skb, NAPI_GRO_CB(skb)->count);
295 }
296
__gro_flush_chain(struct gro_node * gro,u32 index,bool flush_old)297 static void __gro_flush_chain(struct gro_node *gro, u32 index, bool flush_old)
298 {
299 struct list_head *head = &gro->hash[index].list;
300 struct sk_buff *skb, *p;
301
302 list_for_each_entry_safe_reverse(skb, p, head, list) {
303 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
304 return;
305 skb_list_del_init(skb);
306 gro_complete(gro, skb);
307 gro->hash[index].count--;
308 }
309
310 if (!gro->hash[index].count)
311 __clear_bit(index, &gro->bitmask);
312 }
313
314 /*
315 * gro->hash[].list contains packets ordered by age.
316 * youngest packets at the head of it.
317 * Complete skbs in reverse order to reduce latencies.
318 */
__gro_flush(struct gro_node * gro,bool flush_old)319 void __gro_flush(struct gro_node *gro, bool flush_old)
320 {
321 unsigned long bitmask = gro->bitmask;
322 unsigned int i, base = ~0U;
323
324 while ((i = ffs(bitmask)) != 0) {
325 bitmask >>= i;
326 base += i;
327 __gro_flush_chain(gro, base, flush_old);
328 }
329 }
330 EXPORT_SYMBOL(__gro_flush);
331
gro_list_prepare_tc_ext(const struct sk_buff * skb,const struct sk_buff * p,unsigned long diffs)332 static unsigned long gro_list_prepare_tc_ext(const struct sk_buff *skb,
333 const struct sk_buff *p,
334 unsigned long diffs)
335 {
336 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
337 struct tc_skb_ext *skb_ext;
338 struct tc_skb_ext *p_ext;
339
340 skb_ext = skb_ext_find(skb, TC_SKB_EXT);
341 p_ext = skb_ext_find(p, TC_SKB_EXT);
342
343 diffs |= (!!p_ext) ^ (!!skb_ext);
344 if (!diffs && unlikely(skb_ext))
345 diffs |= p_ext->chain ^ skb_ext->chain;
346 #endif
347 return diffs;
348 }
349
gro_list_prepare(const struct list_head * head,const struct sk_buff * skb)350 static void gro_list_prepare(const struct list_head *head,
351 const struct sk_buff *skb)
352 {
353 unsigned int maclen = skb->dev->hard_header_len;
354 u32 hash = skb_get_hash_raw(skb);
355 struct sk_buff *p;
356
357 list_for_each_entry(p, head, list) {
358 unsigned long diffs;
359
360 if (hash != skb_get_hash_raw(p)) {
361 NAPI_GRO_CB(p)->same_flow = 0;
362 continue;
363 }
364
365 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
366 diffs |= p->vlan_all ^ skb->vlan_all;
367 diffs |= skb_metadata_differs(p, skb);
368 if (maclen == ETH_HLEN)
369 diffs |= compare_ether_header(skb_mac_header(p),
370 skb_mac_header(skb));
371 else if (!diffs)
372 diffs = memcmp(skb_mac_header(p),
373 skb_mac_header(skb),
374 maclen);
375
376 /* in most common scenarios 'slow_gro' is 0
377 * otherwise we are already on some slower paths
378 * either skip all the infrequent tests altogether or
379 * avoid trying too hard to skip each of them individually
380 */
381 if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
382 diffs |= p->sk != skb->sk;
383 diffs |= skb_metadata_dst_cmp(p, skb);
384 diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
385
386 diffs |= gro_list_prepare_tc_ext(skb, p, diffs);
387 diffs |= __psp_skb_coalesce_diff(skb, p, diffs);
388 }
389
390 NAPI_GRO_CB(p)->same_flow = !diffs;
391 }
392 }
393
skb_gro_reset_offset(struct sk_buff * skb,u32 nhoff)394 static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
395 {
396 const struct skb_shared_info *pinfo;
397 const skb_frag_t *frag0;
398 unsigned int headlen;
399
400 NAPI_GRO_CB(skb)->network_offset = 0;
401 NAPI_GRO_CB(skb)->data_offset = 0;
402 headlen = skb_headlen(skb);
403 NAPI_GRO_CB(skb)->frag0 = skb->data;
404 NAPI_GRO_CB(skb)->frag0_len = headlen;
405 if (headlen)
406 return;
407
408 pinfo = skb_shinfo(skb);
409 frag0 = &pinfo->frags[0];
410
411 if (pinfo->nr_frags && skb_frag_page(frag0) &&
412 !PageHighMem(skb_frag_page(frag0)) &&
413 (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
414 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
415 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
416 skb_frag_size(frag0),
417 skb->end - skb->tail);
418 }
419 }
420
gro_pull_from_frag0(struct sk_buff * skb,int grow)421 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
422 {
423 struct skb_shared_info *pinfo = skb_shinfo(skb);
424
425 DEBUG_NET_WARN_ON_ONCE(skb->end - skb->tail < grow);
426
427 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
428
429 skb->data_len -= grow;
430 skb->tail += grow;
431
432 skb_frag_off_add(&pinfo->frags[0], grow);
433 skb_frag_size_sub(&pinfo->frags[0], grow);
434
435 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
436 skb_frag_unref(skb, 0);
437 memmove(pinfo->frags, pinfo->frags + 1,
438 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
439 }
440 }
441
gro_try_pull_from_frag0(struct sk_buff * skb)442 static void gro_try_pull_from_frag0(struct sk_buff *skb)
443 {
444 int grow = skb_gro_offset(skb) - skb_headlen(skb);
445
446 if (grow > 0)
447 gro_pull_from_frag0(skb, grow);
448 }
449
gro_flush_oldest(struct gro_node * gro,struct list_head * head)450 static void gro_flush_oldest(struct gro_node *gro, struct list_head *head)
451 {
452 struct sk_buff *oldest;
453
454 oldest = list_last_entry(head, struct sk_buff, list);
455
456 /* We are called with head length >= MAX_GRO_SKBS, so this is
457 * impossible.
458 */
459 if (WARN_ON_ONCE(!oldest))
460 return;
461
462 /* Do not adjust napi->gro_hash[].count, caller is adding a new
463 * SKB to the chain.
464 */
465 skb_list_del_init(oldest);
466 gro_complete(gro, oldest);
467 }
468
dev_gro_receive(struct gro_node * gro,struct sk_buff * skb)469 static enum gro_result dev_gro_receive(struct gro_node *gro,
470 struct sk_buff *skb)
471 {
472 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
473 struct list_head *head = &net_hotdata.offload_base;
474 struct gro_list *gro_list = &gro->hash[bucket];
475 struct packet_offload *ptype;
476 __be16 type = skb->protocol;
477 struct sk_buff *pp = NULL;
478 enum gro_result ret;
479 int same_flow;
480
481 if (netif_elide_gro(skb->dev))
482 goto normal;
483
484 gro_list_prepare(&gro_list->list, skb);
485
486 rcu_read_lock();
487 list_for_each_entry_rcu(ptype, head, list) {
488 if (ptype->type == type && ptype->callbacks.gro_receive)
489 goto found_ptype;
490 }
491 rcu_read_unlock();
492 goto normal;
493
494 found_ptype:
495 skb_set_network_header(skb, skb_gro_offset(skb));
496 skb_reset_mac_len(skb);
497 BUILD_BUG_ON(sizeof_field(struct napi_gro_cb, zeroed) != sizeof(u32));
498 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed),
499 sizeof(u32))); /* Avoid slow unaligned acc */
500 *(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0;
501 NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb);
502 NAPI_GRO_CB(skb)->count = 1;
503 if (unlikely(skb_is_gso(skb))) {
504 NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs;
505 /* Only support TCP and non DODGY users. */
506 if (!skb_is_gso_tcp(skb) ||
507 (skb_shinfo(skb)->gso_type & SKB_GSO_DODGY))
508 NAPI_GRO_CB(skb)->flush = 1;
509 }
510
511 /* Setup for GRO checksum validation */
512 switch (skb->ip_summed) {
513 case CHECKSUM_COMPLETE:
514 NAPI_GRO_CB(skb)->csum = skb->csum;
515 NAPI_GRO_CB(skb)->csum_valid = 1;
516 break;
517 case CHECKSUM_UNNECESSARY:
518 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
519 break;
520 }
521
522 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
523 ipv6_gro_receive, inet_gro_receive,
524 &gro_list->list, skb);
525
526 rcu_read_unlock();
527
528 if (PTR_ERR(pp) == -EINPROGRESS) {
529 ret = GRO_CONSUMED;
530 goto ok;
531 }
532
533 same_flow = NAPI_GRO_CB(skb)->same_flow;
534 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
535
536 if (pp) {
537 skb_list_del_init(pp);
538 gro_complete(gro, pp);
539 gro_list->count--;
540 }
541
542 if (same_flow)
543 goto ok;
544
545 if (NAPI_GRO_CB(skb)->flush)
546 goto normal;
547
548 if (unlikely(gro_list->count >= MAX_GRO_SKBS))
549 gro_flush_oldest(gro, &gro_list->list);
550 else
551 gro_list->count++;
552
553 /* Must be called before setting NAPI_GRO_CB(skb)->{age|last} */
554 gro_try_pull_from_frag0(skb);
555 NAPI_GRO_CB(skb)->age = jiffies;
556 NAPI_GRO_CB(skb)->last = skb;
557 if (!skb_is_gso(skb))
558 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
559 list_add(&skb->list, &gro_list->list);
560 ret = GRO_HELD;
561 ok:
562 if (gro_list->count) {
563 if (!test_bit(bucket, &gro->bitmask))
564 __set_bit(bucket, &gro->bitmask);
565 } else if (test_bit(bucket, &gro->bitmask)) {
566 __clear_bit(bucket, &gro->bitmask);
567 }
568
569 return ret;
570
571 normal:
572 ret = GRO_NORMAL;
573 gro_try_pull_from_frag0(skb);
574 goto ok;
575 }
576
gro_find_receive_by_type(__be16 type)577 struct packet_offload *gro_find_receive_by_type(__be16 type)
578 {
579 struct list_head *offload_head = &net_hotdata.offload_base;
580 struct packet_offload *ptype;
581
582 list_for_each_entry_rcu(ptype, offload_head, list) {
583 if (ptype->type != type || !ptype->callbacks.gro_receive)
584 continue;
585 return ptype;
586 }
587 return NULL;
588 }
589 EXPORT_SYMBOL(gro_find_receive_by_type);
590
gro_find_complete_by_type(__be16 type)591 struct packet_offload *gro_find_complete_by_type(__be16 type)
592 {
593 struct list_head *offload_head = &net_hotdata.offload_base;
594 struct packet_offload *ptype;
595
596 list_for_each_entry_rcu(ptype, offload_head, list) {
597 if (ptype->type != type || !ptype->callbacks.gro_complete)
598 continue;
599 return ptype;
600 }
601 return NULL;
602 }
603 EXPORT_SYMBOL(gro_find_complete_by_type);
604
gro_skb_finish(struct gro_node * gro,struct sk_buff * skb,gro_result_t ret)605 static gro_result_t gro_skb_finish(struct gro_node *gro, struct sk_buff *skb,
606 gro_result_t ret)
607 {
608 switch (ret) {
609 case GRO_NORMAL:
610 gro_normal_one(gro, skb, 1);
611 break;
612
613 case GRO_MERGED_FREE:
614 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
615 napi_skb_free_stolen_head(skb);
616 else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
617 __kfree_skb(skb);
618 else
619 __napi_kfree_skb(skb, SKB_CONSUMED);
620 break;
621
622 case GRO_HELD:
623 case GRO_MERGED:
624 case GRO_CONSUMED:
625 break;
626 }
627
628 return ret;
629 }
630
gro_receive_skb(struct gro_node * gro,struct sk_buff * skb)631 gro_result_t gro_receive_skb(struct gro_node *gro, struct sk_buff *skb)
632 {
633 gro_result_t ret;
634
635 __skb_mark_napi_id(skb, gro);
636 trace_napi_gro_receive_entry(skb);
637
638 skb_gro_reset_offset(skb, 0);
639
640 ret = gro_skb_finish(gro, skb, dev_gro_receive(gro, skb));
641 trace_napi_gro_receive_exit(ret);
642
643 return ret;
644 }
645 EXPORT_SYMBOL(gro_receive_skb);
646
napi_reuse_skb(struct napi_struct * napi,struct sk_buff * skb)647 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
648 {
649 struct skb_shared_info *shinfo;
650
651 if (unlikely(skb->pfmemalloc)) {
652 consume_skb(skb);
653 return;
654 }
655 __skb_pull(skb, skb_headlen(skb));
656 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
657 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
658 __vlan_hwaccel_clear_tag(skb);
659 skb->dev = napi->dev;
660 skb->skb_iif = 0;
661
662 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
663 skb->pkt_type = PACKET_HOST;
664
665 skb->encapsulation = 0;
666 skb->ip_summed = CHECKSUM_NONE;
667
668 shinfo = skb_shinfo(skb);
669 shinfo->gso_type = 0;
670 shinfo->gso_size = 0;
671 shinfo->hwtstamps.hwtstamp = 0;
672
673 if (unlikely(skb->slow_gro)) {
674 skb_orphan(skb);
675 skb_ext_reset(skb);
676 nf_reset_ct(skb);
677 skb->slow_gro = 0;
678 }
679
680 napi->skb = skb;
681 }
682
napi_get_frags(struct napi_struct * napi)683 struct sk_buff *napi_get_frags(struct napi_struct *napi)
684 {
685 struct sk_buff *skb = napi->skb;
686
687 if (!skb) {
688 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
689 if (skb) {
690 napi->skb = skb;
691 skb_mark_napi_id(skb, napi);
692 }
693 }
694 return skb;
695 }
696 EXPORT_SYMBOL(napi_get_frags);
697
napi_frags_finish(struct napi_struct * napi,struct sk_buff * skb,gro_result_t ret)698 static gro_result_t napi_frags_finish(struct napi_struct *napi,
699 struct sk_buff *skb,
700 gro_result_t ret)
701 {
702 switch (ret) {
703 case GRO_NORMAL:
704 case GRO_HELD:
705 __skb_push(skb, ETH_HLEN);
706 skb->protocol = eth_type_trans(skb, skb->dev);
707 if (ret == GRO_NORMAL)
708 gro_normal_one(&napi->gro, skb, 1);
709 break;
710
711 case GRO_MERGED_FREE:
712 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
713 napi_skb_free_stolen_head(skb);
714 else
715 napi_reuse_skb(napi, skb);
716 break;
717
718 case GRO_MERGED:
719 case GRO_CONSUMED:
720 break;
721 }
722
723 return ret;
724 }
725
726 /* Upper GRO stack assumes network header starts at gro_offset=0
727 * Drivers could call both napi_gro_frags() and napi_gro_receive()
728 * We copy ethernet header into skb->data to have a common layout.
729 */
napi_frags_skb(struct napi_struct * napi)730 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
731 {
732 struct sk_buff *skb = napi->skb;
733 const struct ethhdr *eth;
734 unsigned int hlen = sizeof(*eth);
735
736 napi->skb = NULL;
737
738 skb_reset_mac_header(skb);
739 skb_gro_reset_offset(skb, hlen);
740
741 if (unlikely(!skb_gro_may_pull(skb, hlen))) {
742 eth = skb_gro_header_slow(skb, hlen, 0);
743 if (unlikely(!eth)) {
744 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
745 __func__, napi->dev->name);
746 napi_reuse_skb(napi, skb);
747 return NULL;
748 }
749 } else {
750 eth = (const struct ethhdr *)skb->data;
751
752 if (NAPI_GRO_CB(skb)->frag0 != skb->data)
753 gro_pull_from_frag0(skb, hlen);
754
755 NAPI_GRO_CB(skb)->frag0 += hlen;
756 NAPI_GRO_CB(skb)->frag0_len -= hlen;
757 }
758 __skb_pull(skb, hlen);
759
760 /*
761 * This works because the only protocols we care about don't require
762 * special handling.
763 * We'll fix it up properly in napi_frags_finish()
764 */
765 skb->protocol = eth->h_proto;
766
767 return skb;
768 }
769
napi_gro_frags(struct napi_struct * napi)770 gro_result_t napi_gro_frags(struct napi_struct *napi)
771 {
772 gro_result_t ret;
773 struct sk_buff *skb = napi_frags_skb(napi);
774
775 trace_napi_gro_frags_entry(skb);
776
777 ret = napi_frags_finish(napi, skb, dev_gro_receive(&napi->gro, skb));
778 trace_napi_gro_frags_exit(ret);
779
780 return ret;
781 }
782 EXPORT_SYMBOL(napi_gro_frags);
783
784 /* Compute the checksum from gro_offset and return the folded value
785 * after adding in any pseudo checksum.
786 */
__skb_gro_checksum_complete(struct sk_buff * skb)787 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
788 {
789 __wsum wsum;
790 __sum16 sum;
791
792 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
793
794 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
795 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
796 /* See comments in __skb_checksum_complete(). */
797 if (likely(!sum)) {
798 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
799 !skb->csum_complete_sw)
800 netdev_rx_csum_fault(skb->dev, skb);
801 }
802
803 NAPI_GRO_CB(skb)->csum = wsum;
804 NAPI_GRO_CB(skb)->csum_valid = 1;
805
806 return sum;
807 }
808 EXPORT_SYMBOL(__skb_gro_checksum_complete);
809
gro_init(struct gro_node * gro)810 void gro_init(struct gro_node *gro)
811 {
812 for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) {
813 INIT_LIST_HEAD(&gro->hash[i].list);
814 gro->hash[i].count = 0;
815 }
816
817 gro->bitmask = 0;
818 gro->cached_napi_id = 0;
819
820 INIT_LIST_HEAD(&gro->rx_list);
821 gro->rx_count = 0;
822 }
823
gro_cleanup(struct gro_node * gro)824 void gro_cleanup(struct gro_node *gro)
825 {
826 struct sk_buff *skb, *n;
827
828 for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) {
829 list_for_each_entry_safe(skb, n, &gro->hash[i].list, list)
830 kfree_skb(skb);
831
832 gro->hash[i].count = 0;
833 }
834
835 gro->bitmask = 0;
836 gro->cached_napi_id = 0;
837
838 list_for_each_entry_safe(skb, n, &gro->rx_list, list)
839 kfree_skb(skb);
840
841 gro->rx_count = 0;
842 }
843