1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2008 Solarflare Communications Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include <linux/socket.h>
14 #include <linux/tcp.h>
15 #include <linux/udp.h>
17 #include <net/checksum.h>
18 #include "net_driver.h"
23 #include "workarounds.h"
25 /* Number of RX descriptors pushed at once. */
26 #define EFX_RX_BATCH 8
28 /* Size of buffer allocated for skb header area. */
29 #define EFX_SKB_HEADERS 64u
32 * rx_alloc_method - RX buffer allocation method
34 * This driver supports two methods for allocating and using RX buffers:
35 * each RX buffer may be backed by an skb or by an order-n page.
37 * When LRO is in use then the second method has a lower overhead,
38 * since we don't have to allocate then free skbs on reassembled frames.
41 * - RX_ALLOC_METHOD_AUTO = 0
42 * - RX_ALLOC_METHOD_SKB = 1
43 * - RX_ALLOC_METHOD_PAGE = 2
45 * The heuristic for %RX_ALLOC_METHOD_AUTO is a simple hysteresis count
46 * controlled by the parameters below.
48 * - Since pushing and popping descriptors are separated by the rx_queue
49 * size, so the watermarks should be ~rxd_size.
50 * - The performance win by using page-based allocation for LRO is less
51 * than the performance hit of using page-based allocation of non-LRO,
52 * so the watermarks should reflect this.
54 * Per channel we maintain a single variable, updated by each channel:
56 * rx_alloc_level += (lro_performed ? RX_ALLOC_FACTOR_LRO :
57 * RX_ALLOC_FACTOR_SKB)
58 * Per NAPI poll interval, we constrain rx_alloc_level to 0..MAX (which
59 * limits the hysteresis), and update the allocation strategy:
61 * rx_alloc_method = (rx_alloc_level > RX_ALLOC_LEVEL_LRO ?
62 * RX_ALLOC_METHOD_PAGE : RX_ALLOC_METHOD_SKB)
64 static int rx_alloc_method
= RX_ALLOC_METHOD_PAGE
;
66 #define RX_ALLOC_LEVEL_LRO 0x2000
67 #define RX_ALLOC_LEVEL_MAX 0x3000
68 #define RX_ALLOC_FACTOR_LRO 1
69 #define RX_ALLOC_FACTOR_SKB (-2)
71 /* This is the percentage fill level below which new RX descriptors
72 * will be added to the RX descriptor ring.
74 static unsigned int rx_refill_threshold
= 90;
76 /* This is the percentage fill level to which an RX queue will be refilled
77 * when the "RX refill threshold" is reached.
79 static unsigned int rx_refill_limit
= 95;
82 * RX maximum head room required.
84 * This must be at least 1 to prevent overflow and at least 2 to allow
87 #define EFX_RXD_HEAD_ROOM 2
89 static inline unsigned int efx_rx_buf_offset(struct efx_rx_buffer
*buf
)
91 /* Offset is always within one page, so we don't need to consider
94 return (__force
unsigned long) buf
->data
& (PAGE_SIZE
- 1);
96 static inline unsigned int efx_rx_buf_size(struct efx_nic
*efx
)
98 return PAGE_SIZE
<< efx
->rx_buffer_order
;
103 * efx_init_rx_buffer_skb - create new RX buffer using skb-based allocation
105 * @rx_queue: Efx RX queue
106 * @rx_buf: RX buffer structure to populate
108 * This allocates memory for a new receive buffer, maps it for DMA,
109 * and populates a struct efx_rx_buffer with the relevant
110 * information. Return a negative error code or 0 on success.
112 static int efx_init_rx_buffer_skb(struct efx_rx_queue
*rx_queue
,
113 struct efx_rx_buffer
*rx_buf
)
115 struct efx_nic
*efx
= rx_queue
->efx
;
116 struct net_device
*net_dev
= efx
->net_dev
;
117 int skb_len
= efx
->rx_buffer_len
;
119 rx_buf
->skb
= netdev_alloc_skb(net_dev
, skb_len
);
120 if (unlikely(!rx_buf
->skb
))
123 /* Adjust the SKB for padding and checksum */
124 skb_reserve(rx_buf
->skb
, NET_IP_ALIGN
);
125 rx_buf
->len
= skb_len
- NET_IP_ALIGN
;
126 rx_buf
->data
= (char *)rx_buf
->skb
->data
;
127 rx_buf
->skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
129 rx_buf
->dma_addr
= pci_map_single(efx
->pci_dev
,
130 rx_buf
->data
, rx_buf
->len
,
133 if (unlikely(pci_dma_mapping_error(efx
->pci_dev
, rx_buf
->dma_addr
))) {
134 dev_kfree_skb_any(rx_buf
->skb
);
143 * efx_init_rx_buffer_page - create new RX buffer using page-based allocation
145 * @rx_queue: Efx RX queue
146 * @rx_buf: RX buffer structure to populate
148 * This allocates memory for a new receive buffer, maps it for DMA,
149 * and populates a struct efx_rx_buffer with the relevant
150 * information. Return a negative error code or 0 on success.
152 static int efx_init_rx_buffer_page(struct efx_rx_queue
*rx_queue
,
153 struct efx_rx_buffer
*rx_buf
)
155 struct efx_nic
*efx
= rx_queue
->efx
;
156 int bytes
, space
, offset
;
158 bytes
= efx
->rx_buffer_len
- EFX_PAGE_IP_ALIGN
;
160 /* If there is space left in the previously allocated page,
161 * then use it. Otherwise allocate a new one */
162 rx_buf
->page
= rx_queue
->buf_page
;
163 if (rx_buf
->page
== NULL
) {
166 rx_buf
->page
= alloc_pages(__GFP_COLD
| __GFP_COMP
| GFP_ATOMIC
,
167 efx
->rx_buffer_order
);
168 if (unlikely(rx_buf
->page
== NULL
))
171 dma_addr
= pci_map_page(efx
->pci_dev
, rx_buf
->page
,
172 0, efx_rx_buf_size(efx
),
175 if (unlikely(pci_dma_mapping_error(efx
->pci_dev
, dma_addr
))) {
176 __free_pages(rx_buf
->page
, efx
->rx_buffer_order
);
181 rx_queue
->buf_page
= rx_buf
->page
;
182 rx_queue
->buf_dma_addr
= dma_addr
;
183 rx_queue
->buf_data
= (page_address(rx_buf
->page
) +
188 rx_buf
->data
= rx_queue
->buf_data
;
189 offset
= efx_rx_buf_offset(rx_buf
);
190 rx_buf
->dma_addr
= rx_queue
->buf_dma_addr
+ offset
;
192 /* Try to pack multiple buffers per page */
193 if (efx
->rx_buffer_order
== 0) {
194 /* The next buffer starts on the next 512 byte boundary */
195 rx_queue
->buf_data
+= ((bytes
+ 0x1ff) & ~0x1ff);
196 offset
+= ((bytes
+ 0x1ff) & ~0x1ff);
198 space
= efx_rx_buf_size(efx
) - offset
;
199 if (space
>= bytes
) {
200 /* Refs dropped on kernel releasing each skb */
201 get_page(rx_queue
->buf_page
);
206 /* This is the final RX buffer for this page, so mark it for
208 rx_queue
->buf_page
= NULL
;
209 rx_buf
->unmap_addr
= rx_queue
->buf_dma_addr
;
215 /* This allocates memory for a new receive buffer, maps it for DMA,
216 * and populates a struct efx_rx_buffer with the relevant
219 static int efx_init_rx_buffer(struct efx_rx_queue
*rx_queue
,
220 struct efx_rx_buffer
*new_rx_buf
)
224 if (rx_queue
->channel
->rx_alloc_push_pages
) {
225 new_rx_buf
->skb
= NULL
;
226 rc
= efx_init_rx_buffer_page(rx_queue
, new_rx_buf
);
227 rx_queue
->alloc_page_count
++;
229 new_rx_buf
->page
= NULL
;
230 rc
= efx_init_rx_buffer_skb(rx_queue
, new_rx_buf
);
231 rx_queue
->alloc_skb_count
++;
234 if (unlikely(rc
< 0))
235 EFX_LOG_RL(rx_queue
->efx
, "%s RXQ[%d] =%d\n", __func__
,
236 rx_queue
->queue
, rc
);
240 static void efx_unmap_rx_buffer(struct efx_nic
*efx
,
241 struct efx_rx_buffer
*rx_buf
)
244 EFX_BUG_ON_PARANOID(rx_buf
->skb
);
245 if (rx_buf
->unmap_addr
) {
246 pci_unmap_page(efx
->pci_dev
, rx_buf
->unmap_addr
,
247 efx_rx_buf_size(efx
),
249 rx_buf
->unmap_addr
= 0;
251 } else if (likely(rx_buf
->skb
)) {
252 pci_unmap_single(efx
->pci_dev
, rx_buf
->dma_addr
,
253 rx_buf
->len
, PCI_DMA_FROMDEVICE
);
257 static void efx_free_rx_buffer(struct efx_nic
*efx
,
258 struct efx_rx_buffer
*rx_buf
)
261 __free_pages(rx_buf
->page
, efx
->rx_buffer_order
);
263 } else if (likely(rx_buf
->skb
)) {
264 dev_kfree_skb_any(rx_buf
->skb
);
269 static void efx_fini_rx_buffer(struct efx_rx_queue
*rx_queue
,
270 struct efx_rx_buffer
*rx_buf
)
272 efx_unmap_rx_buffer(rx_queue
->efx
, rx_buf
);
273 efx_free_rx_buffer(rx_queue
->efx
, rx_buf
);
277 * efx_fast_push_rx_descriptors - push new RX descriptors quickly
278 * @rx_queue: RX descriptor queue
279 * @retry: Recheck the fill level
280 * This will aim to fill the RX descriptor queue up to
281 * @rx_queue->@fast_fill_limit. If there is insufficient atomic
282 * memory to do so, the caller should retry.
284 static int __efx_fast_push_rx_descriptors(struct efx_rx_queue
*rx_queue
,
287 struct efx_rx_buffer
*rx_buf
;
288 unsigned fill_level
, index
;
289 int i
, space
, rc
= 0;
291 /* Calculate current fill level. Do this outside the lock,
292 * because most of the time we'll end up not wanting to do the
295 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
);
296 EFX_BUG_ON_PARANOID(fill_level
> EFX_RXQ_SIZE
);
298 /* Don't fill if we don't need to */
299 if (fill_level
>= rx_queue
->fast_fill_trigger
)
302 /* Record minimum fill level */
303 if (unlikely(fill_level
< rx_queue
->min_fill
)) {
305 rx_queue
->min_fill
= fill_level
;
308 /* Acquire RX add lock. If this lock is contended, then a fast
309 * fill must already be in progress (e.g. in the refill
310 * tasklet), so we don't need to do anything
312 if (!spin_trylock_bh(&rx_queue
->add_lock
))
316 /* Recalculate current fill level now that we have the lock */
317 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
);
318 EFX_BUG_ON_PARANOID(fill_level
> EFX_RXQ_SIZE
);
319 space
= rx_queue
->fast_fill_limit
- fill_level
;
320 if (space
< EFX_RX_BATCH
)
323 EFX_TRACE(rx_queue
->efx
, "RX queue %d fast-filling descriptor ring from"
324 " level %d to level %d using %s allocation\n",
325 rx_queue
->queue
, fill_level
, rx_queue
->fast_fill_limit
,
326 rx_queue
->channel
->rx_alloc_push_pages
? "page" : "skb");
329 for (i
= 0; i
< EFX_RX_BATCH
; ++i
) {
330 index
= rx_queue
->added_count
& EFX_RXQ_MASK
;
331 rx_buf
= efx_rx_buffer(rx_queue
, index
);
332 rc
= efx_init_rx_buffer(rx_queue
, rx_buf
);
335 ++rx_queue
->added_count
;
337 } while ((space
-= EFX_RX_BATCH
) >= EFX_RX_BATCH
);
339 EFX_TRACE(rx_queue
->efx
, "RX queue %d fast-filled descriptor ring "
340 "to level %d\n", rx_queue
->queue
,
341 rx_queue
->added_count
- rx_queue
->removed_count
);
344 /* Send write pointer to card. */
345 falcon_notify_rx_desc(rx_queue
);
347 /* If the fast fill is running inside from the refill tasklet, then
348 * for SMP systems it may be running on a different CPU to
349 * RX event processing, which means that the fill level may now be
351 if (unlikely(retry
&& (rc
== 0)))
355 spin_unlock_bh(&rx_queue
->add_lock
);
361 * efx_fast_push_rx_descriptors - push new RX descriptors quickly
362 * @rx_queue: RX descriptor queue
364 * This will aim to fill the RX descriptor queue up to
365 * @rx_queue->@fast_fill_limit. If there is insufficient memory to do so,
366 * it will schedule a work item to immediately continue the fast fill
368 void efx_fast_push_rx_descriptors(struct efx_rx_queue
*rx_queue
)
372 rc
= __efx_fast_push_rx_descriptors(rx_queue
, 0);
374 /* Schedule the work item to run immediately. The hope is
375 * that work is immediately pending to free some memory
376 * (e.g. an RX event or TX completion)
378 efx_schedule_slow_fill(rx_queue
, 0);
382 void efx_rx_work(struct work_struct
*data
)
384 struct efx_rx_queue
*rx_queue
;
387 rx_queue
= container_of(data
, struct efx_rx_queue
, work
.work
);
389 if (unlikely(!rx_queue
->channel
->enabled
))
392 EFX_TRACE(rx_queue
->efx
, "RX queue %d worker thread executing on CPU "
393 "%d\n", rx_queue
->queue
, raw_smp_processor_id());
395 ++rx_queue
->slow_fill_count
;
396 /* Push new RX descriptors, allowing at least 1 jiffy for
397 * the kernel to free some more memory. */
398 rc
= __efx_fast_push_rx_descriptors(rx_queue
, 1);
400 efx_schedule_slow_fill(rx_queue
, 1);
403 static void efx_rx_packet__check_len(struct efx_rx_queue
*rx_queue
,
404 struct efx_rx_buffer
*rx_buf
,
405 int len
, bool *discard
,
408 struct efx_nic
*efx
= rx_queue
->efx
;
409 unsigned max_len
= rx_buf
->len
- efx
->type
->rx_buffer_padding
;
411 if (likely(len
<= max_len
))
414 /* The packet must be discarded, but this is only a fatal error
415 * if the caller indicated it was
419 if ((len
> rx_buf
->len
) && EFX_WORKAROUND_8071(efx
)) {
420 EFX_ERR_RL(efx
, " RX queue %d seriously overlength "
421 "RX event (0x%x > 0x%x+0x%x). Leaking\n",
422 rx_queue
->queue
, len
, max_len
,
423 efx
->type
->rx_buffer_padding
);
424 /* If this buffer was skb-allocated, then the meta
425 * data at the end of the skb will be trashed. So
426 * we have no choice but to leak the fragment.
428 *leak_packet
= (rx_buf
->skb
!= NULL
);
429 efx_schedule_reset(efx
, RESET_TYPE_RX_RECOVERY
);
431 EFX_ERR_RL(efx
, " RX queue %d overlength RX event "
432 "(0x%x > 0x%x)\n", rx_queue
->queue
, len
, max_len
);
435 rx_queue
->channel
->n_rx_overlength
++;
438 /* Pass a received packet up through the generic LRO stack
440 * Handles driverlink veto, and passes the fragment up via
441 * the appropriate LRO method
443 static void efx_rx_packet_lro(struct efx_channel
*channel
,
444 struct efx_rx_buffer
*rx_buf
,
447 struct napi_struct
*napi
= &channel
->napi_str
;
448 gro_result_t gro_result
;
450 /* Pass the skb/page into the LRO engine */
452 struct sk_buff
*skb
= napi_get_frags(napi
);
455 put_page(rx_buf
->page
);
456 gro_result
= GRO_DROP
;
460 skb_shinfo(skb
)->frags
[0].page
= rx_buf
->page
;
461 skb_shinfo(skb
)->frags
[0].page_offset
=
462 efx_rx_buf_offset(rx_buf
);
463 skb_shinfo(skb
)->frags
[0].size
= rx_buf
->len
;
464 skb_shinfo(skb
)->nr_frags
= 1;
466 skb
->len
= rx_buf
->len
;
467 skb
->data_len
= rx_buf
->len
;
468 skb
->truesize
+= rx_buf
->len
;
470 checksummed
? CHECKSUM_UNNECESSARY
: CHECKSUM_NONE
;
472 gro_result
= napi_gro_frags(napi
);
475 EFX_BUG_ON_PARANOID(rx_buf
->skb
);
478 EFX_BUG_ON_PARANOID(!rx_buf
->skb
);
479 EFX_BUG_ON_PARANOID(!checksummed
);
481 gro_result
= napi_gro_receive(napi
, rx_buf
->skb
);
485 if (gro_result
== GRO_NORMAL
) {
486 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
487 } else if (gro_result
!= GRO_DROP
) {
488 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_LRO
;
489 channel
->irq_mod_score
+= 2;
493 void efx_rx_packet(struct efx_rx_queue
*rx_queue
, unsigned int index
,
494 unsigned int len
, bool checksummed
, bool discard
)
496 struct efx_nic
*efx
= rx_queue
->efx
;
497 struct efx_rx_buffer
*rx_buf
;
498 bool leak_packet
= false;
500 rx_buf
= efx_rx_buffer(rx_queue
, index
);
501 EFX_BUG_ON_PARANOID(!rx_buf
->data
);
502 EFX_BUG_ON_PARANOID(rx_buf
->skb
&& rx_buf
->page
);
503 EFX_BUG_ON_PARANOID(!(rx_buf
->skb
|| rx_buf
->page
));
505 /* This allows the refill path to post another buffer.
506 * EFX_RXD_HEAD_ROOM ensures that the slot we are using
507 * isn't overwritten yet.
509 rx_queue
->removed_count
++;
511 /* Validate the length encoded in the event vs the descriptor pushed */
512 efx_rx_packet__check_len(rx_queue
, rx_buf
, len
,
513 &discard
, &leak_packet
);
515 EFX_TRACE(efx
, "RX queue %d received id %x at %llx+%x %s%s\n",
516 rx_queue
->queue
, index
,
517 (unsigned long long)rx_buf
->dma_addr
, len
,
518 (checksummed
? " [SUMMED]" : ""),
519 (discard
? " [DISCARD]" : ""));
521 /* Discard packet, if instructed to do so */
522 if (unlikely(discard
)) {
523 if (unlikely(leak_packet
))
524 rx_queue
->channel
->n_skbuff_leaks
++;
526 /* We haven't called efx_unmap_rx_buffer yet,
527 * so fini the entire rx_buffer here */
528 efx_fini_rx_buffer(rx_queue
, rx_buf
);
532 /* Release card resources - assumes all RX buffers consumed in-order
535 efx_unmap_rx_buffer(efx
, rx_buf
);
537 /* Prefetch nice and early so data will (hopefully) be in cache by
538 * the time we look at it.
540 prefetch(rx_buf
->data
);
542 /* Pipeline receives so that we give time for packet headers to be
543 * prefetched into cache.
546 if (rx_queue
->channel
->rx_pkt
)
547 __efx_rx_packet(rx_queue
->channel
,
548 rx_queue
->channel
->rx_pkt
,
549 rx_queue
->channel
->rx_pkt_csummed
);
550 rx_queue
->channel
->rx_pkt
= rx_buf
;
551 rx_queue
->channel
->rx_pkt_csummed
= checksummed
;
554 /* Handle a received packet. Second half: Touches packet payload. */
555 void __efx_rx_packet(struct efx_channel
*channel
,
556 struct efx_rx_buffer
*rx_buf
, bool checksummed
)
558 struct efx_nic
*efx
= channel
->efx
;
561 /* If we're in loopback test, then pass the packet directly to the
562 * loopback layer, and free the rx_buf here
564 if (unlikely(efx
->loopback_selftest
)) {
565 efx_loopback_rx_packet(efx
, rx_buf
->data
, rx_buf
->len
);
566 efx_free_rx_buffer(efx
, rx_buf
);
571 prefetch(skb_shinfo(rx_buf
->skb
));
573 skb_put(rx_buf
->skb
, rx_buf
->len
);
575 /* Move past the ethernet header. rx_buf->data still points
576 * at the ethernet header */
577 rx_buf
->skb
->protocol
= eth_type_trans(rx_buf
->skb
,
581 if (likely(checksummed
|| rx_buf
->page
)) {
582 efx_rx_packet_lro(channel
, rx_buf
, checksummed
);
586 /* We now own the SKB */
590 EFX_BUG_ON_PARANOID(rx_buf
->page
);
591 EFX_BUG_ON_PARANOID(rx_buf
->skb
);
592 EFX_BUG_ON_PARANOID(!skb
);
594 /* Set the SKB flags */
595 skb
->ip_summed
= CHECKSUM_NONE
;
597 skb_record_rx_queue(skb
, channel
->channel
);
599 /* Pass the packet up */
600 netif_receive_skb(skb
);
602 /* Update allocation strategy method */
603 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
609 void efx_rx_strategy(struct efx_channel
*channel
)
611 enum efx_rx_alloc_method method
= rx_alloc_method
;
613 /* Only makes sense to use page based allocation if LRO is enabled */
614 if (!(channel
->efx
->net_dev
->features
& NETIF_F_GRO
)) {
615 method
= RX_ALLOC_METHOD_SKB
;
616 } else if (method
== RX_ALLOC_METHOD_AUTO
) {
617 /* Constrain the rx_alloc_level */
618 if (channel
->rx_alloc_level
< 0)
619 channel
->rx_alloc_level
= 0;
620 else if (channel
->rx_alloc_level
> RX_ALLOC_LEVEL_MAX
)
621 channel
->rx_alloc_level
= RX_ALLOC_LEVEL_MAX
;
623 /* Decide on the allocation method */
624 method
= ((channel
->rx_alloc_level
> RX_ALLOC_LEVEL_LRO
) ?
625 RX_ALLOC_METHOD_PAGE
: RX_ALLOC_METHOD_SKB
);
628 /* Push the option */
629 channel
->rx_alloc_push_pages
= (method
== RX_ALLOC_METHOD_PAGE
);
632 int efx_probe_rx_queue(struct efx_rx_queue
*rx_queue
)
634 struct efx_nic
*efx
= rx_queue
->efx
;
635 unsigned int rxq_size
;
638 EFX_LOG(efx
, "creating RX queue %d\n", rx_queue
->queue
);
640 /* Allocate RX buffers */
641 rxq_size
= EFX_RXQ_SIZE
* sizeof(*rx_queue
->buffer
);
642 rx_queue
->buffer
= kzalloc(rxq_size
, GFP_KERNEL
);
643 if (!rx_queue
->buffer
)
646 rc
= falcon_probe_rx(rx_queue
);
648 kfree(rx_queue
->buffer
);
649 rx_queue
->buffer
= NULL
;
654 void efx_init_rx_queue(struct efx_rx_queue
*rx_queue
)
656 unsigned int max_fill
, trigger
, limit
;
658 EFX_LOG(rx_queue
->efx
, "initialising RX queue %d\n", rx_queue
->queue
);
660 /* Initialise ptr fields */
661 rx_queue
->added_count
= 0;
662 rx_queue
->notified_count
= 0;
663 rx_queue
->removed_count
= 0;
664 rx_queue
->min_fill
= -1U;
665 rx_queue
->min_overfill
= -1U;
667 /* Initialise limit fields */
668 max_fill
= EFX_RXQ_SIZE
- EFX_RXD_HEAD_ROOM
;
669 trigger
= max_fill
* min(rx_refill_threshold
, 100U) / 100U;
670 limit
= max_fill
* min(rx_refill_limit
, 100U) / 100U;
672 rx_queue
->max_fill
= max_fill
;
673 rx_queue
->fast_fill_trigger
= trigger
;
674 rx_queue
->fast_fill_limit
= limit
;
676 /* Set up RX descriptor ring */
677 falcon_init_rx(rx_queue
);
680 void efx_fini_rx_queue(struct efx_rx_queue
*rx_queue
)
683 struct efx_rx_buffer
*rx_buf
;
685 EFX_LOG(rx_queue
->efx
, "shutting down RX queue %d\n", rx_queue
->queue
);
687 falcon_fini_rx(rx_queue
);
689 /* Release RX buffers NB start at index 0 not current HW ptr */
690 if (rx_queue
->buffer
) {
691 for (i
= 0; i
<= EFX_RXQ_MASK
; i
++) {
692 rx_buf
= efx_rx_buffer(rx_queue
, i
);
693 efx_fini_rx_buffer(rx_queue
, rx_buf
);
697 /* For a page that is part-way through splitting into RX buffers */
698 if (rx_queue
->buf_page
!= NULL
) {
699 pci_unmap_page(rx_queue
->efx
->pci_dev
, rx_queue
->buf_dma_addr
,
700 efx_rx_buf_size(rx_queue
->efx
),
702 __free_pages(rx_queue
->buf_page
,
703 rx_queue
->efx
->rx_buffer_order
);
704 rx_queue
->buf_page
= NULL
;
708 void efx_remove_rx_queue(struct efx_rx_queue
*rx_queue
)
710 EFX_LOG(rx_queue
->efx
, "destroying RX queue %d\n", rx_queue
->queue
);
712 falcon_remove_rx(rx_queue
);
714 kfree(rx_queue
->buffer
);
715 rx_queue
->buffer
= NULL
;
719 module_param(rx_alloc_method
, int, 0644);
720 MODULE_PARM_DESC(rx_alloc_method
, "Allocation method used for RX buffers");
722 module_param(rx_refill_threshold
, uint
, 0444);
723 MODULE_PARM_DESC(rx_refill_threshold
,
724 "RX descriptor ring fast/slow fill threshold (%)");