1 #ifndef _NET_GRO_CELLS_H 2 #define _NET_GRO_CELLS_H 3 4 #include <linux/skbuff.h> 5 #include <linux/slab.h> 6 #include <linux/netdevice.h> 7 8 struct gro_cell { 9 struct sk_buff_head napi_skbs; 10 struct napi_struct napi; 11 } ____cacheline_aligned_in_smp; 12 13 struct gro_cells { 14 unsigned int gro_cells_mask; 15 struct gro_cell *cells; 16 }; 17 18 static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) 19 { 20 unsigned long flags; 21 struct gro_cell *cell = gcells->cells; 22 struct net_device *dev = skb->dev; 23 24 if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) { 25 netif_rx(skb); 26 return; 27 } 28 29 if (skb_rx_queue_recorded(skb)) 30 cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask; 31 32 if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { 33 atomic_long_inc(&dev->rx_dropped); 34 kfree_skb(skb); 35 return; 36 } 37 38 spin_lock_irqsave(&cell->napi_skbs.lock, flags); 39 40 __skb_queue_tail(&cell->napi_skbs, skb); 41 if (skb_queue_len(&cell->napi_skbs) == 1) 42 napi_schedule(&cell->napi); 43 44 spin_unlock_irqrestore(&cell->napi_skbs.lock, flags); 45 } 46 47 static inline int gro_cell_poll(struct napi_struct *napi, int budget) 48 { 49 struct gro_cell *cell = container_of(napi, struct gro_cell, napi); 50 struct sk_buff *skb; 51 int work_done = 0; 52 53 while (work_done < budget) { 54 skb = skb_dequeue(&cell->napi_skbs); 55 if (!skb) 56 break; 57 58 napi_gro_receive(napi, skb); 59 work_done++; 60 } 61 62 if (work_done < budget) 63 napi_complete(napi); 64 return work_done; 65 } 66 67 static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) 68 { 69 int i; 70 71 gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1; 72 gcells->cells = kcalloc(sizeof(struct gro_cell), 73 gcells->gro_cells_mask + 1, 74 GFP_KERNEL); 75 if (!gcells->cells) 76 return -ENOMEM; 77 78 for (i = 0; i <= gcells->gro_cells_mask; i++) { 79 struct gro_cell *cell = gcells->cells + i; 80 81 skb_queue_head_init(&cell->napi_skbs); 82 netif_napi_add(dev, &cell->napi, gro_cell_poll, 64); 83 napi_enable(&cell->napi); 84 } 85 return 0; 86 } 87 88 static inline void gro_cells_destroy(struct gro_cells *gcells) 89 { 90 struct gro_cell *cell = gcells->cells; 91 int i; 92 93 if (!cell) 94 return; 95 for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) { 96 netif_napi_del(&cell->napi); 97 skb_queue_purge(&cell->napi_skbs); 98 } 99 kfree(gcells->cells); 100 gcells->cells = NULL; 101 } 102 103 #endif 104