include/net/gro_cells.h

   1 #ifndef _NET_GRO_CELLS_H
   2 #define _NET_GRO_CELLS_H
   3
   4 #include <linux/skbuff.h>
   5 #include <linux/slab.h>
   6 #include <linux/netdevice.h>
   7
   8 struct gro_cell {
   9         struct sk_buff_head     napi_skbs;
  10         struct napi_struct      napi;
  11 } ____cacheline_aligned_in_smp;
  12
  13 struct gro_cells {
  14         unsigned int            gro_cells_mask;
  15         struct gro_cell         *cells;
  16 };
  17
  18 static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb)
  19 {
  20         unsigned long flags;
  21         struct gro_cell *cell = gcells->cells;
  22         struct net_device *dev = skb->dev;
  23
  24         if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) {
  25                 netif_rx(skb);
  26                 return;
  27         }
  28
  29         if (skb_rx_queue_recorded(skb))
  30                 cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask;
  31
  32         if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) {
  33                 atomic_long_inc(&dev->rx_dropped);
  34                 kfree_skb(skb);
  35                 return;
  36         }
  37
  38         spin_lock_irqsave(&cell->napi_skbs.lock, flags);
  39
  40         __skb_queue_tail(&cell->napi_skbs, skb);
  41         if (skb_queue_len(&cell->napi_skbs) == 1)
  42                 napi_schedule(&cell->napi);
  43
  44         spin_unlock_irqrestore(&cell->napi_skbs.lock, flags);
  45 }
  46
  47 static inline int gro_cell_poll(struct napi_struct *napi, int budget)
  48 {
  49         struct gro_cell *cell = container_of(napi, struct gro_cell, napi);
  50         struct sk_buff *skb;
  51         int work_done = 0;
  52
  53         while (work_done < budget) {
  54                 skb = skb_dequeue(&cell->napi_skbs);
  55                 if (!skb)
  56                         break;
  57
  58                 napi_gro_receive(napi, skb);
  59                 work_done++;
  60         }
  61
  62         if (work_done < budget)
  63                 napi_complete(napi);
  64         return work_done;
  65 }
  66
  67 static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev)
  68 {
  69         int i;
  70
  71         gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1;
  72         gcells->cells = kcalloc(sizeof(struct gro_cell),
  73                                 gcells->gro_cells_mask + 1,
  74                                 GFP_KERNEL);
  75         if (!gcells->cells)
  76                 return -ENOMEM;
  77
  78         for (i = 0; i <= gcells->gro_cells_mask; i++) {
  79                 struct gro_cell *cell = gcells->cells + i;
  80
  81                 skb_queue_head_init(&cell->napi_skbs);
  82                 netif_napi_add(dev, &cell->napi, gro_cell_poll, 64);
  83                 napi_enable(&cell->napi);
  84         }
  85         return 0;
  86 }
  87
  88 static inline void gro_cells_destroy(struct gro_cells *gcells)
  89 {
  90         struct gro_cell *cell = gcells->cells;
  91         int i;
  92
  93         if (!cell)
  94                 return;
  95         for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) {
  96                 netif_napi_del(&cell->napi);
  97                 skb_queue_purge(&cell->napi_skbs);
  98         }
  99         kfree(gcells->cells);
 100         gcells->cells = NULL;
 101 }
 102
 103 #endif