drivers/net/ethernet/sfc/rx.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2005-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/socket.h>
  12 #include <linux/in.h>
  13 #include <linux/slab.h>
  14 #include <linux/ip.h>
  15 #include <linux/tcp.h>
  16 #include <linux/udp.h>
  17 #include <linux/prefetch.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/iommu.h>
  20 #include <net/ip.h>
  21 #include <net/checksum.h>
  22 #include "net_driver.h"
  23 #include "efx.h"
  24 #include "nic.h"
  25 #include "selftest.h"
  26 #include "workarounds.h"
  27
  28 /* Number of RX descriptors pushed at once. */
  29 #define EFX_RX_BATCH  8
  30
  31 /* Number of RX buffers to recycle pages for.  When creating the RX page recycle
  32  * ring, this number is divided by the number of buffers per page to calculate
  33  * the number of pages to store in the RX page recycle ring.
  34  */
  35 #define EFX_RECYCLE_RING_SIZE_IOMMU 4096
  36 #define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_BATCH)
  37
  38 /* Maximum length for an RX descriptor sharing a page */
  39 #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state) \
  40                           - EFX_PAGE_IP_ALIGN)
  41
  42 /* Size of buffer allocated for skb header area. */
  43 #define EFX_SKB_HEADERS  64u
  44
  45 /* This is the percentage fill level below which new RX descriptors
  46  * will be added to the RX descriptor ring.
  47  */
  48 static unsigned int rx_refill_threshold;
  49
  50 /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */
  51 #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \
  52                                       EFX_RX_USR_BUF_SIZE)
  53
  54 /*
  55  * RX maximum head room required.
  56  *
  57  * This must be at least 1 to prevent overflow, plus one packet-worth
  58  * to allow pipelined receives.
  59  */
  60 #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)
  61
  62 static inline u8 *efx_rx_buf_va(struct efx_rx_buffer *buf)
  63 {
  64         return page_address(buf->page) + buf->page_offset;
  65 }
  66
  67 static inline u32 efx_rx_buf_hash(const u8 *eh)
  68 {
  69         /* The ethernet header is always directly after any hash. */
  70 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0
  71         return __le32_to_cpup((const __le32 *)(eh - 4));
  72 #else
  73         const u8 *data = eh - 4;
  74         return (u32)data[0]       |
  75                (u32)data[1] << 8  |
  76                (u32)data[2] << 16 |
  77                (u32)data[3] << 24;
  78 #endif
  79 }
  80
  81 static inline struct efx_rx_buffer *
  82 efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf)
  83 {
  84         if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))
  85                 return efx_rx_buffer(rx_queue, 0);
  86         else
  87                 return rx_buf + 1;
  88 }
  89
  90 static inline void efx_sync_rx_buffer(struct efx_nic *efx,
  91                                       struct efx_rx_buffer *rx_buf,
  92                                       unsigned int len)
  93 {
  94         dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len,
  95                                 DMA_FROM_DEVICE);
  96 }
  97
  98 /* Check the RX page recycle ring for a page that can be reused. */
  99 static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
 100 {
 101         struct efx_nic *efx = rx_queue->efx;
 102         struct page *page;
 103         struct efx_rx_page_state *state;
 104         unsigned index;
 105
 106         index = rx_queue->page_remove & rx_queue->page_ptr_mask;
 107         page = rx_queue->page_ring[index];
 108         if (page == NULL)
 109                 return NULL;
 110
 111         rx_queue->page_ring[index] = NULL;
 112         /* page_remove cannot exceed page_add. */
 113         if (rx_queue->page_remove != rx_queue->page_add)
 114                 ++rx_queue->page_remove;
 115
 116         /* If page_count is 1 then we hold the only reference to this page. */
 117         if (page_count(page) == 1) {
 118                 ++rx_queue->page_recycle_count;
 119                 return page;
 120         } else {
 121                 state = page_address(page);
 122                 dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
 123                                PAGE_SIZE << efx->rx_buffer_order,
 124                                DMA_FROM_DEVICE);
 125                 put_page(page);
 126                 ++rx_queue->page_recycle_failed;
 127         }
 128
 129         return NULL;
 130 }
 131
 132 /**
 133  * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
 134  *
 135  * @rx_queue:           Efx RX queue
 136  *
 137  * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
 138  * and populates struct efx_rx_buffers for each one. Return a negative error
 139  * code or 0 on success. If a single page can be split between two buffers,
 140  * then the page will either be inserted fully, or not at at all.
 141  */
 142 static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
 143 {
 144         struct efx_nic *efx = rx_queue->efx;
 145         struct efx_rx_buffer *rx_buf;
 146         struct page *page;
 147         unsigned int page_offset;
 148         struct efx_rx_page_state *state;
 149         dma_addr_t dma_addr;
 150         unsigned index, count;
 151
 152         /* We can split a page between two buffers */
 153         BUILD_BUG_ON(EFX_RX_BATCH & 1);
 154
 155         for (count = 0; count < EFX_RX_BATCH; ++count) {
 156                 page = efx_reuse_page(rx_queue);
 157                 if (page == NULL) {
 158                         page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC,
 159                                            efx->rx_buffer_order);
 160                         if (unlikely(page == NULL))
 161                                 return -ENOMEM;
 162                         dma_addr =
 163                                 dma_map_page(&efx->pci_dev->dev, page, 0,
 164                                              PAGE_SIZE << efx->rx_buffer_order,
 165                                              DMA_FROM_DEVICE);
 166                         if (unlikely(dma_mapping_error(&efx->pci_dev->dev,
 167                                                        dma_addr))) {
 168                                 __free_pages(page, efx->rx_buffer_order);
 169                                 return -EIO;
 170                         }
 171                         state = page_address(page);
 172                         state->dma_addr = dma_addr;
 173                 } else {
 174                         state = page_address(page);
 175                         dma_addr = state->dma_addr;
 176                 }
 177                 get_page(page);
 178
 179                 dma_addr += sizeof(struct efx_rx_page_state);
 180                 page_offset = sizeof(struct efx_rx_page_state);
 181
 182         split:
 183                 index = rx_queue->added_count & rx_queue->ptr_mask;
 184                 rx_buf = efx_rx_buffer(rx_queue, index);
 185                 rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN;
 186                 rx_buf->page = page;
 187                 rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN;
 188                 rx_buf->len = efx->rx_dma_len;
 189                 ++rx_queue->added_count;
 190
 191                 if ((~count & 1) && (efx->rx_dma_len <= EFX_RX_HALF_PAGE)) {
 192                         /* Use the second half of the page */
 193                         get_page(page);
 194                         rx_buf->flags = 0;
 195                         dma_addr += (PAGE_SIZE >> 1);
 196                         page_offset += (PAGE_SIZE >> 1);
 197                         ++count;
 198                         goto split;
 199                 }
 200
 201                 rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;
 202         }
 203
 204         return 0;
 205 }
 206
 207 /* Unmap a DMA-mapped page.  This function is only called for the final RX
 208  * buffer in a page.
 209  */
 210 static void efx_unmap_rx_buffer(struct efx_nic *efx,
 211                                 struct efx_rx_buffer *rx_buf)
 212 {
 213         struct page *page = rx_buf->page;
 214
 215         if (page) {
 216                 struct efx_rx_page_state *state = page_address(page);
 217                 dma_unmap_page(&efx->pci_dev->dev,
 218                                state->dma_addr,
 219                                PAGE_SIZE << efx->rx_buffer_order,
 220                                DMA_FROM_DEVICE);
 221         }
 222 }
 223
 224 static void efx_free_rx_buffer(struct efx_rx_buffer *rx_buf)
 225 {
 226         if (rx_buf->page) {
 227                 put_page(rx_buf->page);
 228                 rx_buf->page = NULL;
 229         }
 230 }
 231
 232 /* Attempt to recycle the page if there is an RX recycle ring; the page can
 233  * only be added if this is the final RX buffer, to prevent pages being used in
 234  * the descriptor ring and appearing in the recycle ring simultaneously.
 235  */
 236 static void efx_recycle_rx_page(struct efx_channel *channel,
 237                                 struct efx_rx_buffer *rx_buf)
 238 {
 239         struct page *page = rx_buf->page;
 240         struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 241         struct efx_nic *efx = rx_queue->efx;
 242         unsigned index;
 243
 244         /* Only recycle the page after processing the final buffer. */
 245         if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))
 246                 return;
 247
 248         index = rx_queue->page_add & rx_queue->page_ptr_mask;
 249         if (rx_queue->page_ring[index] == NULL) {
 250                 unsigned read_index = rx_queue->page_remove &
 251                         rx_queue->page_ptr_mask;
 252
 253                 /* The next slot in the recycle ring is available, but
 254                  * increment page_remove if the read pointer currently
 255                  * points here.
 256                  */
 257                 if (read_index == index)
 258                         ++rx_queue->page_remove;
 259                 rx_queue->page_ring[index] = page;
 260                 ++rx_queue->page_add;
 261                 return;
 262         }
 263         ++rx_queue->page_recycle_full;
 264         efx_unmap_rx_buffer(efx, rx_buf);
 265         put_page(rx_buf->page);
 266 }
 267
 268 static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
 269                                struct efx_rx_buffer *rx_buf)
 270 {
 271         /* Release the page reference we hold for the buffer. */
 272         if (rx_buf->page)
 273                 put_page(rx_buf->page);
 274
 275         /* If this is the last buffer in a page, unmap and free it. */
 276         if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {
 277                 efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
 278                 efx_free_rx_buffer(rx_buf);
 279         }
 280         rx_buf->page = NULL;
 281 }
 282
 283 /* Recycle the pages that are used by buffers that have just been received. */
 284 static void efx_recycle_rx_buffers(struct efx_channel *channel,
 285                                    struct efx_rx_buffer *rx_buf,
 286                                    unsigned int n_frags)
 287 {
 288         struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 289
 290         do {
 291                 efx_recycle_rx_page(channel, rx_buf);
 292                 rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 293         } while (--n_frags);
 294 }
 295
 296 /**
 297  * efx_fast_push_rx_descriptors - push new RX descriptors quickly
 298  * @rx_queue:           RX descriptor queue
 299  *
 300  * This will aim to fill the RX descriptor queue up to
 301  * @rx_queue->@max_fill. If there is insufficient atomic
 302  * memory to do so, a slow fill will be scheduled.
 303  *
 304  * The caller must provide serialisation (none is used here). In practise,
 305  * this means this function must run from the NAPI handler, or be called
 306  * when NAPI is disabled.
 307  */
 308 void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
 309 {
 310         unsigned fill_level;
 311         int space, rc = 0;
 312
 313         /* Calculate current fill level, and exit if we don't need to fill */
 314         fill_level = (rx_queue->added_count - rx_queue->removed_count);
 315         EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);
 316         if (fill_level >= rx_queue->fast_fill_trigger)
 317                 goto out;
 318
 319         /* Record minimum fill level */
 320         if (unlikely(fill_level < rx_queue->min_fill)) {
 321                 if (fill_level)
 322                         rx_queue->min_fill = fill_level;
 323         }
 324
 325         space = rx_queue->max_fill - fill_level;
 326         EFX_BUG_ON_PARANOID(space < EFX_RX_BATCH);
 327
 328         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 329                    "RX queue %d fast-filling descriptor ring from"
 330                    " level %d to level %d\n",
 331                    efx_rx_queue_index(rx_queue), fill_level,
 332                    rx_queue->max_fill);
 333
 334
 335         do {
 336                 rc = efx_init_rx_buffers(rx_queue);
 337                 if (unlikely(rc)) {
 338                         /* Ensure that we don't leave the rx queue empty */
 339                         if (rx_queue->added_count == rx_queue->removed_count)
 340                                 efx_schedule_slow_fill(rx_queue);
 341                         goto out;
 342                 }
 343         } while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH);
 344
 345         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 346                    "RX queue %d fast-filled descriptor ring "
 347                    "to level %d\n", efx_rx_queue_index(rx_queue),
 348                    rx_queue->added_count - rx_queue->removed_count);
 349
 350  out:
 351         if (rx_queue->notified_count != rx_queue->added_count)
 352                 efx_nic_notify_rx_desc(rx_queue);
 353 }
 354
 355 void efx_rx_slow_fill(unsigned long context)
 356 {
 357         struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context;
 358
 359         /* Post an event to cause NAPI to run and refill the queue */
 360         efx_nic_generate_fill_event(rx_queue);
 361         ++rx_queue->slow_fill_count;
 362 }
 363
 364 static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue,
 365                                      struct efx_rx_buffer *rx_buf,
 366                                      int len)
 367 {
 368         struct efx_nic *efx = rx_queue->efx;
 369         unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;
 370
 371         if (likely(len <= max_len))
 372                 return;
 373
 374         /* The packet must be discarded, but this is only a fatal error
 375          * if the caller indicated it was
 376          */
 377         rx_buf->flags |= EFX_RX_PKT_DISCARD;
 378
 379         if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) {
 380                 if (net_ratelimit())
 381                         netif_err(efx, rx_err, efx->net_dev,
 382                                   " RX queue %d seriously overlength "
 383                                   "RX event (0x%x > 0x%x+0x%x). Leaking\n",
 384                                   efx_rx_queue_index(rx_queue), len, max_len,
 385                                   efx->type->rx_buffer_padding);
 386                 efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY);
 387         } else {
 388                 if (net_ratelimit())
 389                         netif_err(efx, rx_err, efx->net_dev,
 390                                   " RX queue %d overlength RX event "
 391                                   "(0x%x > 0x%x)\n",
 392                                   efx_rx_queue_index(rx_queue), len, max_len);
 393         }
 394
 395         efx_rx_queue_channel(rx_queue)->n_rx_overlength++;
 396 }
 397
 398 /* Pass a received packet up through GRO.  GRO can handle pages
 399  * regardless of checksum state and skbs with a good checksum.
 400  */
 401 static void
 402 efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
 403                   unsigned int n_frags, u8 *eh)
 404 {
 405         struct napi_struct *napi = &channel->napi_str;
 406         gro_result_t gro_result;
 407         struct efx_nic *efx = channel->efx;
 408         struct sk_buff *skb;
 409
 410         skb = napi_get_frags(napi);
 411         if (unlikely(!skb)) {
 412                 while (n_frags--) {
 413                         put_page(rx_buf->page);
 414                         rx_buf->page = NULL;
 415                         rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 416                 }
 417                 return;
 418         }
 419
 420         if (efx->net_dev->features & NETIF_F_RXHASH)
 421                 skb->rxhash = efx_rx_buf_hash(eh);
 422         skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
 423                           CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
 424
 425         for (;;) {
 426                 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 427                                    rx_buf->page, rx_buf->page_offset,
 428                                    rx_buf->len);
 429                 rx_buf->page = NULL;
 430                 skb->len += rx_buf->len;
 431                 if (skb_shinfo(skb)->nr_frags == n_frags)
 432                         break;
 433
 434                 rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 435         }
 436
 437         skb->data_len = skb->len;
 438         skb->truesize += n_frags * efx->rx_buffer_truesize;
 439
 440         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 441
 442         gro_result = napi_gro_frags(napi);
 443         if (gro_result != GRO_DROP)
 444                 channel->irq_mod_score += 2;
 445 }
 446
 447 /* Allocate and construct an SKB around page fragments */
 448 static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
 449                                      struct efx_rx_buffer *rx_buf,
 450                                      unsigned int n_frags,
 451                                      u8 *eh, int hdr_len)
 452 {
 453         struct efx_nic *efx = channel->efx;
 454         struct sk_buff *skb;
 455
 456         /* Allocate an SKB to store the headers */
 457         skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN);
 458         if (unlikely(skb == NULL))
 459                 return NULL;
 460
 461         EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
 462
 463         skb_reserve(skb, EFX_PAGE_SKB_ALIGN);
 464         memcpy(__skb_put(skb, hdr_len), eh, hdr_len);
 465
 466         /* Append the remaining page(s) onto the frag list */
 467         if (rx_buf->len > hdr_len) {
 468                 rx_buf->page_offset += hdr_len;
 469                 rx_buf->len -= hdr_len;
 470
 471                 for (;;) {
 472                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 473                                            rx_buf->page, rx_buf->page_offset,
 474                                            rx_buf->len);
 475                         rx_buf->page = NULL;
 476                         skb->len += rx_buf->len;
 477                         skb->data_len += rx_buf->len;
 478                         if (skb_shinfo(skb)->nr_frags == n_frags)
 479                                 break;
 480
 481                         rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 482                 }
 483         } else {
 484                 __free_pages(rx_buf->page, efx->rx_buffer_order);
 485                 rx_buf->page = NULL;
 486                 n_frags = 0;
 487         }
 488
 489         skb->truesize += n_frags * efx->rx_buffer_truesize;
 490
 491         /* Move past the ethernet header */
 492         skb->protocol = eth_type_trans(skb, efx->net_dev);
 493
 494         return skb;
 495 }
 496
 497 void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
 498                    unsigned int n_frags, unsigned int len, u16 flags)
 499 {
 500         struct efx_nic *efx = rx_queue->efx;
 501         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 502         struct efx_rx_buffer *rx_buf;
 503
 504         rx_buf = efx_rx_buffer(rx_queue, index);
 505         rx_buf->flags |= flags;
 506
 507         /* Validate the number of fragments and completed length */
 508         if (n_frags == 1) {
 509                 efx_rx_packet__check_len(rx_queue, rx_buf, len);
 510         } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) ||
 511                    unlikely(len <= (n_frags - 1) * EFX_RX_USR_BUF_SIZE) ||
 512                    unlikely(len > n_frags * EFX_RX_USR_BUF_SIZE) ||
 513                    unlikely(!efx->rx_scatter)) {
 514                 /* If this isn't an explicit discard request, either
 515                  * the hardware or the driver is broken.
 516                  */
 517                 WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD));
 518                 rx_buf->flags |= EFX_RX_PKT_DISCARD;
 519         }
 520
 521         netif_vdbg(efx, rx_status, efx->net_dev,
 522                    "RX queue %d received ids %x-%x len %d %s%s\n",
 523                    efx_rx_queue_index(rx_queue), index,
 524                    (index + n_frags - 1) & rx_queue->ptr_mask, len,
 525                    (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "",
 526                    (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : "");
 527
 528         /* Discard packet, if instructed to do so.  Process the
 529          * previous receive first.
 530          */
 531         if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) {
 532                 efx_rx_flush_packet(channel);
 533                 put_page(rx_buf->page);
 534                 efx_recycle_rx_buffers(channel, rx_buf, n_frags);
 535                 return;
 536         }
 537
 538         if (n_frags == 1)
 539                 rx_buf->len = len;
 540
 541         /* Release and/or sync the DMA mapping - assumes all RX buffers
 542          * consumed in-order per RX queue.
 543          */
 544         efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);
 545
 546         /* Prefetch nice and early so data will (hopefully) be in cache by
 547          * the time we look at it.
 548          */
 549         prefetch(efx_rx_buf_va(rx_buf));
 550
 551         rx_buf->page_offset += efx->type->rx_buffer_hash_size;
 552         rx_buf->len -= efx->type->rx_buffer_hash_size;
 553
 554         if (n_frags > 1) {
 555                 /* Release/sync DMA mapping for additional fragments.
 556                  * Fix length for last fragment.
 557                  */
 558                 unsigned int tail_frags = n_frags - 1;
 559
 560                 for (;;) {
 561                         rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 562                         if (--tail_frags == 0)
 563                                 break;
 564                         efx_sync_rx_buffer(efx, rx_buf, EFX_RX_USR_BUF_SIZE);
 565                 }
 566                 rx_buf->len = len - (n_frags - 1) * EFX_RX_USR_BUF_SIZE;
 567                 efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);
 568         }
 569
 570         /* All fragments have been DMA-synced, so recycle buffers and pages. */
 571         rx_buf = efx_rx_buffer(rx_queue, index);
 572         efx_recycle_rx_buffers(channel, rx_buf, n_frags);
 573
 574         /* Pipeline receives so that we give time for packet headers to be
 575          * prefetched into cache.
 576          */
 577         efx_rx_flush_packet(channel);
 578         channel->rx_pkt_n_frags = n_frags;
 579         channel->rx_pkt_index = index;
 580 }
 581
 582 static void efx_rx_deliver(struct efx_channel *channel, u8 *eh,
 583                            struct efx_rx_buffer *rx_buf,
 584                            unsigned int n_frags)
 585 {
 586         struct sk_buff *skb;
 587         u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS);
 588
 589         skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len);
 590         if (unlikely(skb == NULL)) {
 591                 efx_free_rx_buffer(rx_buf);
 592                 return;
 593         }
 594         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 595
 596         /* Set the SKB flags */
 597         skb_checksum_none_assert(skb);
 598
 599         if (channel->type->receive_skb)
 600                 if (channel->type->receive_skb(channel, skb))
 601                         return;
 602
 603         /* Pass the packet up */
 604         netif_receive_skb(skb);
 605 }
 606
 607 /* Handle a received packet.  Second half: Touches packet payload. */
 608 void __efx_rx_packet(struct efx_channel *channel)
 609 {
 610         struct efx_nic *efx = channel->efx;
 611         struct efx_rx_buffer *rx_buf =
 612                 efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index);
 613         u8 *eh = efx_rx_buf_va(rx_buf);
 614
 615         /* If we're in loopback test, then pass the packet directly to the
 616          * loopback layer, and free the rx_buf here
 617          */
 618         if (unlikely(efx->loopback_selftest)) {
 619                 efx_loopback_rx_packet(efx, eh, rx_buf->len);
 620                 efx_free_rx_buffer(rx_buf);
 621                 goto out;
 622         }
 623
 624         if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
 625                 rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;
 626
 627         if (!channel->type->receive_skb)
 628                 efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh);
 629         else
 630                 efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);
 631 out:
 632         channel->rx_pkt_n_frags = 0;
 633 }
 634
 635 int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
 636 {
 637         struct efx_nic *efx = rx_queue->efx;
 638         unsigned int entries;
 639         int rc;
 640
 641         /* Create the smallest power-of-two aligned ring */
 642         entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
 643         EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
 644         rx_queue->ptr_mask = entries - 1;
 645
 646         netif_dbg(efx, probe, efx->net_dev,
 647                   "creating RX queue %d size %#x mask %#x\n",
 648                   efx_rx_queue_index(rx_queue), efx->rxq_entries,
 649                   rx_queue->ptr_mask);
 650
 651         /* Allocate RX buffers */
 652         rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
 653                                    GFP_KERNEL);
 654         if (!rx_queue->buffer)
 655                 return -ENOMEM;
 656
 657         rc = efx_nic_probe_rx(rx_queue);
 658         if (rc) {
 659                 kfree(rx_queue->buffer);
 660                 rx_queue->buffer = NULL;
 661         }
 662
 663         return rc;
 664 }
 665
 666 void efx_init_rx_recycle_ring(struct efx_nic *efx,
 667                               struct efx_rx_queue *rx_queue)
 668 {
 669         unsigned int bufs_in_recycle_ring, page_ring_size;
 670
 671         /* Set the RX recycle ring size */
 672 #ifdef CONFIG_PPC64
 673         bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU;
 674 #else
 675         if (efx->pci_dev->dev.iommu_group)
 676                 bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU;
 677         else
 678                 bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_NOIOMMU;
 679 #endif /* CONFIG_PPC64 */
 680
 681         page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /
 682                                             efx->rx_bufs_per_page);
 683         rx_queue->page_ring = kcalloc(page_ring_size,
 684                                       sizeof(*rx_queue->page_ring), GFP_KERNEL);
 685         rx_queue->page_ptr_mask = page_ring_size - 1;
 686 }
 687
 688 void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
 689 {
 690         struct efx_nic *efx = rx_queue->efx;
 691         unsigned int max_fill, trigger, max_trigger;
 692
 693         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 694                   "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
 695
 696         /* Initialise ptr fields */
 697         rx_queue->added_count = 0;
 698         rx_queue->notified_count = 0;
 699         rx_queue->removed_count = 0;
 700         rx_queue->min_fill = -1U;
 701         efx_init_rx_recycle_ring(efx, rx_queue);
 702
 703         rx_queue->page_remove = 0;
 704         rx_queue->page_add = rx_queue->page_ptr_mask + 1;
 705         rx_queue->page_recycle_count = 0;
 706         rx_queue->page_recycle_failed = 0;
 707         rx_queue->page_recycle_full = 0;
 708
 709         /* Initialise limit fields */
 710         max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
 711         max_trigger = max_fill - EFX_RX_BATCH;
 712         if (rx_refill_threshold != 0) {
 713                 trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
 714                 if (trigger > max_trigger)
 715                         trigger = max_trigger;
 716         } else {
 717                 trigger = max_trigger;
 718         }
 719
 720         rx_queue->max_fill = max_fill;
 721         rx_queue->fast_fill_trigger = trigger;
 722
 723         /* Set up RX descriptor ring */
 724         rx_queue->enabled = true;
 725         efx_nic_init_rx(rx_queue);
 726 }
 727
 728 void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
 729 {
 730         int i;
 731         struct efx_nic *efx = rx_queue->efx;
 732         struct efx_rx_buffer *rx_buf;
 733
 734         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 735                   "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
 736
 737         /* A flush failure might have left rx_queue->enabled */
 738         rx_queue->enabled = false;
 739
 740         del_timer_sync(&rx_queue->slow_fill);
 741         efx_nic_fini_rx(rx_queue);
 742
 743         /* Release RX buffers from the current read ptr to the write ptr */
 744         if (rx_queue->buffer) {
 745                 for (i = rx_queue->removed_count; i < rx_queue->added_count;
 746                      i++) {
 747                         unsigned index = i & rx_queue->ptr_mask;
 748                         rx_buf = efx_rx_buffer(rx_queue, index);
 749                         efx_fini_rx_buffer(rx_queue, rx_buf);
 750                 }
 751         }
 752
 753         /* Unmap and release the pages in the recycle ring. Remove the ring. */
 754         for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
 755                 struct page *page = rx_queue->page_ring[i];
 756                 struct efx_rx_page_state *state;
 757
 758                 if (page == NULL)
 759                         continue;
 760
 761                 state = page_address(page);
 762                 dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
 763                                PAGE_SIZE << efx->rx_buffer_order,
 764                                DMA_FROM_DEVICE);
 765                 put_page(page);
 766         }
 767         kfree(rx_queue->page_ring);
 768         rx_queue->page_ring = NULL;
 769 }
 770
 771 void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)
 772 {
 773         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 774                   "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
 775
 776         efx_nic_remove_rx(rx_queue);
 777
 778         kfree(rx_queue->buffer);
 779         rx_queue->buffer = NULL;
 780 }
 781
 782
 783 module_param(rx_refill_threshold, uint, 0444);
 784 MODULE_PARM_DESC(rx_refill_threshold,
 785                  "RX descriptor ring refill threshold (%)");
 786