2 * Copyright (c) 2005-2007 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/skbuff.h>
33 #include <linux/netdevice.h>
34 #include <linux/etherdevice.h>
35 #include <linux/if_vlan.h>
37 #include <linux/tcp.h>
38 #include <linux/dma-mapping.h>
43 #include "firmware_exports.h"
47 #define SGE_RX_SM_BUF_SIZE 1536
49 #define SGE_RX_COPY_THRES 256
50 #define SGE_RX_PULL_LEN 128
53 * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
54 * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
57 #define FL0_PG_CHUNK_SIZE 2048
58 #define FL0_PG_ORDER 0
59 #define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
60 #define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
62 #define SGE_RX_DROP_THRES 16
65 * Period of the Tx buffer reclaim timer. This timer does not need to run
66 * frequently as Tx buffers are usually reclaimed by new Tx packets.
68 #define TX_RECLAIM_PERIOD (HZ / 4)
70 /* WR size in bytes */
71 #define WR_LEN (WR_FLITS * 8)
74 * Types of Tx queues in each queue set. Order here matters, do not change.
76 enum { TXQ_ETH
, TXQ_OFLD
, TXQ_CTRL
};
78 /* Values for sge_txq.flags */
80 TXQ_RUNNING
= 1 << 0, /* fetch engine is running */
81 TXQ_LAST_PKT_DB
= 1 << 1, /* last packet rang the doorbell */
85 __be64 flit
[TX_DESC_FLITS
];
95 struct tx_sw_desc
{ /* SW state per Tx descriptor */
97 u8 eop
; /* set if last descriptor for packet */
98 u8 addr_idx
; /* buffer index of first SGL entry in descriptor */
99 u8 fragidx
; /* first page fragment associated with descriptor */
100 s8 sflit
; /* start flit of first SGL entry in descriptor */
103 struct rx_sw_desc
{ /* SW state per Rx descriptor */
106 struct fl_pg_chunk pg_chunk
;
108 DECLARE_PCI_UNMAP_ADDR(dma_addr
);
111 struct rsp_desc
{ /* response queue descriptor */
112 struct rss_header rss_hdr
;
120 * Holds unmapping information for Tx packets that need deferred unmapping.
121 * This structure lives at skb->head and must be allocated by callers.
123 struct deferred_unmap_info
{
124 struct pci_dev
*pdev
;
125 dma_addr_t addr
[MAX_SKB_FRAGS
+ 1];
129 * Maps a number of flits to the number of Tx descriptors that can hold them.
132 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
134 * HW allows up to 4 descriptors to be combined into a WR.
136 static u8 flit_desc_map
[] = {
138 #if SGE_NUM_GENBITS == 1
139 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
140 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
141 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
142 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
143 #elif SGE_NUM_GENBITS == 2
144 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
145 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
146 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
147 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
149 # error "SGE_NUM_GENBITS must be 1 or 2"
153 static inline struct sge_qset
*fl_to_qset(const struct sge_fl
*q
, int qidx
)
155 return container_of(q
, struct sge_qset
, fl
[qidx
]);
158 static inline struct sge_qset
*rspq_to_qset(const struct sge_rspq
*q
)
160 return container_of(q
, struct sge_qset
, rspq
);
163 static inline struct sge_qset
*txq_to_qset(const struct sge_txq
*q
, int qidx
)
165 return container_of(q
, struct sge_qset
, txq
[qidx
]);
169 * refill_rspq - replenish an SGE response queue
170 * @adapter: the adapter
171 * @q: the response queue to replenish
172 * @credits: how many new responses to make available
174 * Replenishes a response queue by making the supplied number of responses
177 static inline void refill_rspq(struct adapter
*adapter
,
178 const struct sge_rspq
*q
, unsigned int credits
)
181 t3_write_reg(adapter
, A_SG_RSPQ_CREDIT_RETURN
,
182 V_RSPQ(q
->cntxt_id
) | V_CREDITS(credits
));
186 * need_skb_unmap - does the platform need unmapping of sk_buffs?
188 * Returns true if the platfrom needs sk_buff unmapping. The compiler
189 * optimizes away unecessary code if this returns true.
191 static inline int need_skb_unmap(void)
194 * This structure is used to tell if the platfrom needs buffer
195 * unmapping by checking if DECLARE_PCI_UNMAP_ADDR defines anything.
198 DECLARE_PCI_UNMAP_ADDR(addr
);
201 return sizeof(struct dummy
) != 0;
205 * unmap_skb - unmap a packet main body and its page fragments
207 * @q: the Tx queue containing Tx descriptors for the packet
208 * @cidx: index of Tx descriptor
209 * @pdev: the PCI device
211 * Unmap the main body of an sk_buff and its page fragments, if any.
212 * Because of the fairly complicated structure of our SGLs and the desire
213 * to conserve space for metadata, the information necessary to unmap an
214 * sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx
215 * descriptors (the physical addresses of the various data buffers), and
216 * the SW descriptor state (assorted indices). The send functions
217 * initialize the indices for the first packet descriptor so we can unmap
218 * the buffers held in the first Tx descriptor here, and we have enough
219 * information at this point to set the state for the next Tx descriptor.
221 * Note that it is possible to clean up the first descriptor of a packet
222 * before the send routines have written the next descriptors, but this
223 * race does not cause any problem. We just end up writing the unmapping
224 * info for the descriptor first.
226 static inline void unmap_skb(struct sk_buff
*skb
, struct sge_txq
*q
,
227 unsigned int cidx
, struct pci_dev
*pdev
)
229 const struct sg_ent
*sgp
;
230 struct tx_sw_desc
*d
= &q
->sdesc
[cidx
];
231 int nfrags
, frag_idx
, curflit
, j
= d
->addr_idx
;
233 sgp
= (struct sg_ent
*)&q
->desc
[cidx
].flit
[d
->sflit
];
234 frag_idx
= d
->fragidx
;
236 if (frag_idx
== 0 && skb_headlen(skb
)) {
237 pci_unmap_single(pdev
, be64_to_cpu(sgp
->addr
[0]),
238 skb_headlen(skb
), PCI_DMA_TODEVICE
);
242 curflit
= d
->sflit
+ 1 + j
;
243 nfrags
= skb_shinfo(skb
)->nr_frags
;
245 while (frag_idx
< nfrags
&& curflit
< WR_FLITS
) {
246 pci_unmap_page(pdev
, be64_to_cpu(sgp
->addr
[j
]),
247 skb_shinfo(skb
)->frags
[frag_idx
].size
,
258 if (frag_idx
< nfrags
) { /* SGL continues into next Tx descriptor */
259 d
= cidx
+ 1 == q
->size
? q
->sdesc
: d
+ 1;
260 d
->fragidx
= frag_idx
;
262 d
->sflit
= curflit
- WR_FLITS
- j
; /* sflit can be -1 */
267 * free_tx_desc - reclaims Tx descriptors and their buffers
268 * @adapter: the adapter
269 * @q: the Tx queue to reclaim descriptors from
270 * @n: the number of descriptors to reclaim
272 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
273 * Tx buffers. Called with the Tx queue lock held.
275 static void free_tx_desc(struct adapter
*adapter
, struct sge_txq
*q
,
278 struct tx_sw_desc
*d
;
279 struct pci_dev
*pdev
= adapter
->pdev
;
280 unsigned int cidx
= q
->cidx
;
282 const int need_unmap
= need_skb_unmap() &&
283 q
->cntxt_id
>= FW_TUNNEL_SGEEC_START
;
287 if (d
->skb
) { /* an SGL is present */
289 unmap_skb(d
->skb
, q
, cidx
, pdev
);
294 if (++cidx
== q
->size
) {
303 * reclaim_completed_tx - reclaims completed Tx descriptors
304 * @adapter: the adapter
305 * @q: the Tx queue to reclaim completed descriptors from
307 * Reclaims Tx descriptors that the SGE has indicated it has processed,
308 * and frees the associated buffers if possible. Called with the Tx
311 static inline void reclaim_completed_tx(struct adapter
*adapter
,
314 unsigned int reclaim
= q
->processed
- q
->cleaned
;
317 free_tx_desc(adapter
, q
, reclaim
);
318 q
->cleaned
+= reclaim
;
319 q
->in_use
-= reclaim
;
324 * should_restart_tx - are there enough resources to restart a Tx queue?
327 * Checks if there are enough descriptors to restart a suspended Tx queue.
329 static inline int should_restart_tx(const struct sge_txq
*q
)
331 unsigned int r
= q
->processed
- q
->cleaned
;
333 return q
->in_use
- r
< (q
->size
>> 1);
337 * free_rx_bufs - free the Rx buffers on an SGE free list
338 * @pdev: the PCI device associated with the adapter
339 * @rxq: the SGE free list to clean up
341 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
342 * this queue should be stopped before calling this function.
344 static void free_rx_bufs(struct pci_dev
*pdev
, struct sge_fl
*q
)
346 unsigned int cidx
= q
->cidx
;
348 while (q
->credits
--) {
349 struct rx_sw_desc
*d
= &q
->sdesc
[cidx
];
351 pci_unmap_single(pdev
, pci_unmap_addr(d
, dma_addr
),
352 q
->buf_size
, PCI_DMA_FROMDEVICE
);
354 put_page(d
->pg_chunk
.page
);
355 d
->pg_chunk
.page
= NULL
;
360 if (++cidx
== q
->size
)
364 if (q
->pg_chunk
.page
) {
365 __free_pages(q
->pg_chunk
.page
, q
->order
);
366 q
->pg_chunk
.page
= NULL
;
371 * add_one_rx_buf - add a packet buffer to a free-buffer list
372 * @va: buffer start VA
373 * @len: the buffer length
374 * @d: the HW Rx descriptor to write
375 * @sd: the SW Rx descriptor to write
376 * @gen: the generation bit value
377 * @pdev: the PCI device associated with the adapter
379 * Add a buffer of the given length to the supplied HW and SW Rx
382 static inline int add_one_rx_buf(void *va
, unsigned int len
,
383 struct rx_desc
*d
, struct rx_sw_desc
*sd
,
384 unsigned int gen
, struct pci_dev
*pdev
)
388 mapping
= pci_map_single(pdev
, va
, len
, PCI_DMA_FROMDEVICE
);
389 if (unlikely(pci_dma_mapping_error(mapping
)))
392 pci_unmap_addr_set(sd
, dma_addr
, mapping
);
394 d
->addr_lo
= cpu_to_be32(mapping
);
395 d
->addr_hi
= cpu_to_be32((u64
) mapping
>> 32);
397 d
->len_gen
= cpu_to_be32(V_FLD_GEN1(gen
));
398 d
->gen2
= cpu_to_be32(V_FLD_GEN2(gen
));
402 static int alloc_pg_chunk(struct sge_fl
*q
, struct rx_sw_desc
*sd
, gfp_t gfp
,
405 if (!q
->pg_chunk
.page
) {
406 q
->pg_chunk
.page
= alloc_pages(gfp
, order
);
407 if (unlikely(!q
->pg_chunk
.page
))
409 q
->pg_chunk
.va
= page_address(q
->pg_chunk
.page
);
410 q
->pg_chunk
.offset
= 0;
412 sd
->pg_chunk
= q
->pg_chunk
;
414 q
->pg_chunk
.offset
+= q
->buf_size
;
415 if (q
->pg_chunk
.offset
== (PAGE_SIZE
<< order
))
416 q
->pg_chunk
.page
= NULL
;
418 q
->pg_chunk
.va
+= q
->buf_size
;
419 get_page(q
->pg_chunk
.page
);
425 * refill_fl - refill an SGE free-buffer list
426 * @adapter: the adapter
427 * @q: the free-list to refill
428 * @n: the number of new buffers to allocate
429 * @gfp: the gfp flags for allocating new buffers
431 * (Re)populate an SGE free-buffer list with up to @n new packet buffers,
432 * allocated with the supplied gfp flags. The caller must assure that
433 * @n does not exceed the queue's capacity.
435 static int refill_fl(struct adapter
*adap
, struct sge_fl
*q
, int n
, gfp_t gfp
)
438 struct rx_sw_desc
*sd
= &q
->sdesc
[q
->pidx
];
439 struct rx_desc
*d
= &q
->desc
[q
->pidx
];
440 unsigned int count
= 0;
446 if (unlikely(alloc_pg_chunk(q
, sd
, gfp
, q
->order
))) {
447 nomem
: q
->alloc_failed
++;
450 buf_start
= sd
->pg_chunk
.va
;
452 struct sk_buff
*skb
= alloc_skb(q
->buf_size
, gfp
);
458 buf_start
= skb
->data
;
461 err
= add_one_rx_buf(buf_start
, q
->buf_size
, d
, sd
, q
->gen
,
473 if (++q
->pidx
== q
->size
) {
484 t3_write_reg(adap
, A_SG_KDOORBELL
, V_EGRCNTX(q
->cntxt_id
));
489 static inline void __refill_fl(struct adapter
*adap
, struct sge_fl
*fl
)
491 refill_fl(adap
, fl
, min(16U, fl
->size
- fl
->credits
),
492 GFP_ATOMIC
| __GFP_COMP
);
496 * recycle_rx_buf - recycle a receive buffer
497 * @adapter: the adapter
498 * @q: the SGE free list
499 * @idx: index of buffer to recycle
501 * Recycles the specified buffer on the given free list by adding it at
502 * the next available slot on the list.
504 static void recycle_rx_buf(struct adapter
*adap
, struct sge_fl
*q
,
507 struct rx_desc
*from
= &q
->desc
[idx
];
508 struct rx_desc
*to
= &q
->desc
[q
->pidx
];
510 q
->sdesc
[q
->pidx
] = q
->sdesc
[idx
];
511 to
->addr_lo
= from
->addr_lo
; /* already big endian */
512 to
->addr_hi
= from
->addr_hi
; /* likewise */
514 to
->len_gen
= cpu_to_be32(V_FLD_GEN1(q
->gen
));
515 to
->gen2
= cpu_to_be32(V_FLD_GEN2(q
->gen
));
518 if (++q
->pidx
== q
->size
) {
522 t3_write_reg(adap
, A_SG_KDOORBELL
, V_EGRCNTX(q
->cntxt_id
));
526 * alloc_ring - allocate resources for an SGE descriptor ring
527 * @pdev: the PCI device
528 * @nelem: the number of descriptors
529 * @elem_size: the size of each descriptor
530 * @sw_size: the size of the SW state associated with each ring element
531 * @phys: the physical address of the allocated ring
532 * @metadata: address of the array holding the SW state for the ring
534 * Allocates resources for an SGE descriptor ring, such as Tx queues,
535 * free buffer lists, or response queues. Each SGE ring requires
536 * space for its HW descriptors plus, optionally, space for the SW state
537 * associated with each HW entry (the metadata). The function returns
538 * three values: the virtual address for the HW ring (the return value
539 * of the function), the physical address of the HW ring, and the address
542 static void *alloc_ring(struct pci_dev
*pdev
, size_t nelem
, size_t elem_size
,
543 size_t sw_size
, dma_addr_t
* phys
, void *metadata
)
545 size_t len
= nelem
* elem_size
;
547 void *p
= dma_alloc_coherent(&pdev
->dev
, len
, phys
, GFP_KERNEL
);
552 s
= kcalloc(nelem
, sw_size
, GFP_KERNEL
);
555 dma_free_coherent(&pdev
->dev
, len
, p
, *phys
);
560 *(void **)metadata
= s
;
566 * t3_reset_qset - reset a sge qset
569 * Reset the qset structure.
570 * the NAPI structure is preserved in the event of
571 * the qset's reincarnation, for example during EEH recovery.
573 static void t3_reset_qset(struct sge_qset
*q
)
576 !(q
->adap
->flags
& NAPI_INIT
)) {
577 memset(q
, 0, sizeof(*q
));
582 memset(&q
->rspq
, 0, sizeof(q
->rspq
));
583 memset(q
->fl
, 0, sizeof(struct sge_fl
) * SGE_RXQ_PER_SET
);
584 memset(q
->txq
, 0, sizeof(struct sge_txq
) * SGE_TXQ_PER_SET
);
586 memset(&q
->tx_reclaim_timer
, 0, sizeof(q
->tx_reclaim_timer
));
591 * free_qset - free the resources of an SGE queue set
592 * @adapter: the adapter owning the queue set
595 * Release the HW and SW resources associated with an SGE queue set, such
596 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
597 * queue set must be quiesced prior to calling this.
599 static void t3_free_qset(struct adapter
*adapter
, struct sge_qset
*q
)
602 struct pci_dev
*pdev
= adapter
->pdev
;
604 if (q
->tx_reclaim_timer
.function
)
605 del_timer_sync(&q
->tx_reclaim_timer
);
607 for (i
= 0; i
< SGE_RXQ_PER_SET
; ++i
)
609 spin_lock_irq(&adapter
->sge
.reg_lock
);
610 t3_sge_disable_fl(adapter
, q
->fl
[i
].cntxt_id
);
611 spin_unlock_irq(&adapter
->sge
.reg_lock
);
612 free_rx_bufs(pdev
, &q
->fl
[i
]);
613 kfree(q
->fl
[i
].sdesc
);
614 dma_free_coherent(&pdev
->dev
,
616 sizeof(struct rx_desc
), q
->fl
[i
].desc
,
620 for (i
= 0; i
< SGE_TXQ_PER_SET
; ++i
)
621 if (q
->txq
[i
].desc
) {
622 spin_lock_irq(&adapter
->sge
.reg_lock
);
623 t3_sge_enable_ecntxt(adapter
, q
->txq
[i
].cntxt_id
, 0);
624 spin_unlock_irq(&adapter
->sge
.reg_lock
);
625 if (q
->txq
[i
].sdesc
) {
626 free_tx_desc(adapter
, &q
->txq
[i
],
628 kfree(q
->txq
[i
].sdesc
);
630 dma_free_coherent(&pdev
->dev
,
632 sizeof(struct tx_desc
),
633 q
->txq
[i
].desc
, q
->txq
[i
].phys_addr
);
634 __skb_queue_purge(&q
->txq
[i
].sendq
);
638 spin_lock_irq(&adapter
->sge
.reg_lock
);
639 t3_sge_disable_rspcntxt(adapter
, q
->rspq
.cntxt_id
);
640 spin_unlock_irq(&adapter
->sge
.reg_lock
);
641 dma_free_coherent(&pdev
->dev
,
642 q
->rspq
.size
* sizeof(struct rsp_desc
),
643 q
->rspq
.desc
, q
->rspq
.phys_addr
);
650 * init_qset_cntxt - initialize an SGE queue set context info
652 * @id: the queue set id
654 * Initializes the TIDs and context ids for the queues of a queue set.
656 static void init_qset_cntxt(struct sge_qset
*qs
, unsigned int id
)
658 qs
->rspq
.cntxt_id
= id
;
659 qs
->fl
[0].cntxt_id
= 2 * id
;
660 qs
->fl
[1].cntxt_id
= 2 * id
+ 1;
661 qs
->txq
[TXQ_ETH
].cntxt_id
= FW_TUNNEL_SGEEC_START
+ id
;
662 qs
->txq
[TXQ_ETH
].token
= FW_TUNNEL_TID_START
+ id
;
663 qs
->txq
[TXQ_OFLD
].cntxt_id
= FW_OFLD_SGEEC_START
+ id
;
664 qs
->txq
[TXQ_CTRL
].cntxt_id
= FW_CTRL_SGEEC_START
+ id
;
665 qs
->txq
[TXQ_CTRL
].token
= FW_CTRL_TID_START
+ id
;
669 * sgl_len - calculates the size of an SGL of the given capacity
670 * @n: the number of SGL entries
672 * Calculates the number of flits needed for a scatter/gather list that
673 * can hold the given number of entries.
675 static inline unsigned int sgl_len(unsigned int n
)
677 /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
678 return (3 * n
) / 2 + (n
& 1);
682 * flits_to_desc - returns the num of Tx descriptors for the given flits
683 * @n: the number of flits
685 * Calculates the number of Tx descriptors needed for the supplied number
688 static inline unsigned int flits_to_desc(unsigned int n
)
690 BUG_ON(n
>= ARRAY_SIZE(flit_desc_map
));
691 return flit_desc_map
[n
];
695 * get_packet - return the next ingress packet buffer from a free list
696 * @adap: the adapter that received the packet
697 * @fl: the SGE free list holding the packet
698 * @len: the packet length including any SGE padding
699 * @drop_thres: # of remaining buffers before we start dropping packets
701 * Get the next packet from a free list and complete setup of the
702 * sk_buff. If the packet is small we make a copy and recycle the
703 * original buffer, otherwise we use the original buffer itself. If a
704 * positive drop threshold is supplied packets are dropped and their
705 * buffers recycled if (a) the number of remaining buffers is under the
706 * threshold and the packet is too big to copy, or (b) the packet should
707 * be copied but there is no memory for the copy.
709 static struct sk_buff
*get_packet(struct adapter
*adap
, struct sge_fl
*fl
,
710 unsigned int len
, unsigned int drop_thres
)
712 struct sk_buff
*skb
= NULL
;
713 struct rx_sw_desc
*sd
= &fl
->sdesc
[fl
->cidx
];
715 prefetch(sd
->skb
->data
);
718 if (len
<= SGE_RX_COPY_THRES
) {
719 skb
= alloc_skb(len
, GFP_ATOMIC
);
720 if (likely(skb
!= NULL
)) {
722 pci_dma_sync_single_for_cpu(adap
->pdev
,
723 pci_unmap_addr(sd
, dma_addr
), len
,
725 memcpy(skb
->data
, sd
->skb
->data
, len
);
726 pci_dma_sync_single_for_device(adap
->pdev
,
727 pci_unmap_addr(sd
, dma_addr
), len
,
729 } else if (!drop_thres
)
732 recycle_rx_buf(adap
, fl
, fl
->cidx
);
736 if (unlikely(fl
->credits
< drop_thres
))
740 pci_unmap_single(adap
->pdev
, pci_unmap_addr(sd
, dma_addr
),
741 fl
->buf_size
, PCI_DMA_FROMDEVICE
);
744 __refill_fl(adap
, fl
);
749 * get_packet_pg - return the next ingress packet buffer from a free list
750 * @adap: the adapter that received the packet
751 * @fl: the SGE free list holding the packet
752 * @len: the packet length including any SGE padding
753 * @drop_thres: # of remaining buffers before we start dropping packets
755 * Get the next packet from a free list populated with page chunks.
756 * If the packet is small we make a copy and recycle the original buffer,
757 * otherwise we attach the original buffer as a page fragment to a fresh
758 * sk_buff. If a positive drop threshold is supplied packets are dropped
759 * and their buffers recycled if (a) the number of remaining buffers is
760 * under the threshold and the packet is too big to copy, or (b) there's
763 * Note: this function is similar to @get_packet but deals with Rx buffers
764 * that are page chunks rather than sk_buffs.
766 static struct sk_buff
*get_packet_pg(struct adapter
*adap
, struct sge_fl
*fl
,
767 struct sge_rspq
*q
, unsigned int len
,
768 unsigned int drop_thres
)
770 struct sk_buff
*newskb
, *skb
;
771 struct rx_sw_desc
*sd
= &fl
->sdesc
[fl
->cidx
];
773 newskb
= skb
= q
->pg_skb
;
775 if (!skb
&& (len
<= SGE_RX_COPY_THRES
)) {
776 newskb
= alloc_skb(len
, GFP_ATOMIC
);
777 if (likely(newskb
!= NULL
)) {
778 __skb_put(newskb
, len
);
779 pci_dma_sync_single_for_cpu(adap
->pdev
,
780 pci_unmap_addr(sd
, dma_addr
), len
,
782 memcpy(newskb
->data
, sd
->pg_chunk
.va
, len
);
783 pci_dma_sync_single_for_device(adap
->pdev
,
784 pci_unmap_addr(sd
, dma_addr
), len
,
786 } else if (!drop_thres
)
790 recycle_rx_buf(adap
, fl
, fl
->cidx
);
795 if (unlikely(q
->rx_recycle_buf
|| (!skb
&& fl
->credits
<= drop_thres
)))
799 newskb
= alloc_skb(SGE_RX_PULL_LEN
, GFP_ATOMIC
);
800 if (unlikely(!newskb
)) {
806 pci_unmap_single(adap
->pdev
, pci_unmap_addr(sd
, dma_addr
),
807 fl
->buf_size
, PCI_DMA_FROMDEVICE
);
809 __skb_put(newskb
, SGE_RX_PULL_LEN
);
810 memcpy(newskb
->data
, sd
->pg_chunk
.va
, SGE_RX_PULL_LEN
);
811 skb_fill_page_desc(newskb
, 0, sd
->pg_chunk
.page
,
812 sd
->pg_chunk
.offset
+ SGE_RX_PULL_LEN
,
813 len
- SGE_RX_PULL_LEN
);
815 newskb
->data_len
= len
- SGE_RX_PULL_LEN
;
817 skb_fill_page_desc(newskb
, skb_shinfo(newskb
)->nr_frags
,
819 sd
->pg_chunk
.offset
, len
);
821 newskb
->data_len
+= len
;
823 newskb
->truesize
+= newskb
->data_len
;
827 * We do not refill FLs here, we let the caller do it to overlap a
834 * get_imm_packet - return the next ingress packet buffer from a response
835 * @resp: the response descriptor containing the packet data
837 * Return a packet containing the immediate data of the given response.
839 static inline struct sk_buff
*get_imm_packet(const struct rsp_desc
*resp
)
841 struct sk_buff
*skb
= alloc_skb(IMMED_PKT_SIZE
, GFP_ATOMIC
);
844 __skb_put(skb
, IMMED_PKT_SIZE
);
845 skb_copy_to_linear_data(skb
, resp
->imm_data
, IMMED_PKT_SIZE
);
851 * calc_tx_descs - calculate the number of Tx descriptors for a packet
854 * Returns the number of Tx descriptors needed for the given Ethernet
855 * packet. Ethernet packets require addition of WR and CPL headers.
857 static inline unsigned int calc_tx_descs(const struct sk_buff
*skb
)
861 if (skb
->len
<= WR_LEN
- sizeof(struct cpl_tx_pkt
))
864 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1) + 2;
865 if (skb_shinfo(skb
)->gso_size
)
867 return flits_to_desc(flits
);
871 * make_sgl - populate a scatter/gather list for a packet
873 * @sgp: the SGL to populate
874 * @start: start address of skb main body data to include in the SGL
875 * @len: length of skb main body data to include in the SGL
876 * @pdev: the PCI device
878 * Generates a scatter/gather list for the buffers that make up a packet
879 * and returns the SGL size in 8-byte words. The caller must size the SGL
882 static inline unsigned int make_sgl(const struct sk_buff
*skb
,
883 struct sg_ent
*sgp
, unsigned char *start
,
884 unsigned int len
, struct pci_dev
*pdev
)
887 unsigned int i
, j
= 0, nfrags
;
890 mapping
= pci_map_single(pdev
, start
, len
, PCI_DMA_TODEVICE
);
891 sgp
->len
[0] = cpu_to_be32(len
);
892 sgp
->addr
[0] = cpu_to_be64(mapping
);
896 nfrags
= skb_shinfo(skb
)->nr_frags
;
897 for (i
= 0; i
< nfrags
; i
++) {
898 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
900 mapping
= pci_map_page(pdev
, frag
->page
, frag
->page_offset
,
901 frag
->size
, PCI_DMA_TODEVICE
);
902 sgp
->len
[j
] = cpu_to_be32(frag
->size
);
903 sgp
->addr
[j
] = cpu_to_be64(mapping
);
910 return ((nfrags
+ (len
!= 0)) * 3) / 2 + j
;
914 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
918 * Ring the doorbel if a Tx queue is asleep. There is a natural race,
919 * where the HW is going to sleep just after we checked, however,
920 * then the interrupt handler will detect the outstanding TX packet
921 * and ring the doorbell for us.
923 * When GTS is disabled we unconditionally ring the doorbell.
925 static inline void check_ring_tx_db(struct adapter
*adap
, struct sge_txq
*q
)
928 clear_bit(TXQ_LAST_PKT_DB
, &q
->flags
);
929 if (test_and_set_bit(TXQ_RUNNING
, &q
->flags
) == 0) {
930 set_bit(TXQ_LAST_PKT_DB
, &q
->flags
);
931 t3_write_reg(adap
, A_SG_KDOORBELL
,
932 F_SELEGRCNTX
| V_EGRCNTX(q
->cntxt_id
));
935 wmb(); /* write descriptors before telling HW */
936 t3_write_reg(adap
, A_SG_KDOORBELL
,
937 F_SELEGRCNTX
| V_EGRCNTX(q
->cntxt_id
));
941 static inline void wr_gen2(struct tx_desc
*d
, unsigned int gen
)
943 #if SGE_NUM_GENBITS == 2
944 d
->flit
[TX_DESC_FLITS
- 1] = cpu_to_be64(gen
);
949 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
950 * @ndesc: number of Tx descriptors spanned by the SGL
951 * @skb: the packet corresponding to the WR
952 * @d: first Tx descriptor to be written
953 * @pidx: index of above descriptors
954 * @q: the SGE Tx queue
956 * @flits: number of flits to the start of the SGL in the first descriptor
957 * @sgl_flits: the SGL size in flits
958 * @gen: the Tx descriptor generation
959 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
960 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
962 * Write a work request header and an associated SGL. If the SGL is
963 * small enough to fit into one Tx descriptor it has already been written
964 * and we just need to write the WR header. Otherwise we distribute the
965 * SGL across the number of descriptors it spans.
967 static void write_wr_hdr_sgl(unsigned int ndesc
, struct sk_buff
*skb
,
968 struct tx_desc
*d
, unsigned int pidx
,
969 const struct sge_txq
*q
,
970 const struct sg_ent
*sgl
,
971 unsigned int flits
, unsigned int sgl_flits
,
972 unsigned int gen
, __be32 wr_hi
,
975 struct work_request_hdr
*wrp
= (struct work_request_hdr
*)d
;
976 struct tx_sw_desc
*sd
= &q
->sdesc
[pidx
];
979 if (need_skb_unmap()) {
985 if (likely(ndesc
== 1)) {
987 wrp
->wr_hi
= htonl(F_WR_SOP
| F_WR_EOP
| V_WR_DATATYPE(1) |
988 V_WR_SGLSFLT(flits
)) | wr_hi
;
990 wrp
->wr_lo
= htonl(V_WR_LEN(flits
+ sgl_flits
) |
991 V_WR_GEN(gen
)) | wr_lo
;
994 unsigned int ogen
= gen
;
995 const u64
*fp
= (const u64
*)sgl
;
996 struct work_request_hdr
*wp
= wrp
;
998 wrp
->wr_hi
= htonl(F_WR_SOP
| V_WR_DATATYPE(1) |
999 V_WR_SGLSFLT(flits
)) | wr_hi
;
1002 unsigned int avail
= WR_FLITS
- flits
;
1004 if (avail
> sgl_flits
)
1006 memcpy(&d
->flit
[flits
], fp
, avail
* sizeof(*fp
));
1016 if (++pidx
== q
->size
) {
1024 wrp
= (struct work_request_hdr
*)d
;
1025 wrp
->wr_hi
= htonl(V_WR_DATATYPE(1) |
1026 V_WR_SGLSFLT(1)) | wr_hi
;
1027 wrp
->wr_lo
= htonl(V_WR_LEN(min(WR_FLITS
,
1029 V_WR_GEN(gen
)) | wr_lo
;
1034 wrp
->wr_hi
|= htonl(F_WR_EOP
);
1036 wp
->wr_lo
= htonl(V_WR_LEN(WR_FLITS
) | V_WR_GEN(ogen
)) | wr_lo
;
1037 wr_gen2((struct tx_desc
*)wp
, ogen
);
1038 WARN_ON(ndesc
!= 0);
1043 * write_tx_pkt_wr - write a TX_PKT work request
1044 * @adap: the adapter
1045 * @skb: the packet to send
1046 * @pi: the egress interface
1047 * @pidx: index of the first Tx descriptor to write
1048 * @gen: the generation value to use
1050 * @ndesc: number of descriptors the packet will occupy
1051 * @compl: the value of the COMPL bit to use
1053 * Generate a TX_PKT work request to send the supplied packet.
1055 static void write_tx_pkt_wr(struct adapter
*adap
, struct sk_buff
*skb
,
1056 const struct port_info
*pi
,
1057 unsigned int pidx
, unsigned int gen
,
1058 struct sge_txq
*q
, unsigned int ndesc
,
1061 unsigned int flits
, sgl_flits
, cntrl
, tso_info
;
1062 struct sg_ent
*sgp
, sgl
[MAX_SKB_FRAGS
/ 2 + 1];
1063 struct tx_desc
*d
= &q
->desc
[pidx
];
1064 struct cpl_tx_pkt
*cpl
= (struct cpl_tx_pkt
*)d
;
1066 cpl
->len
= htonl(skb
->len
| 0x80000000);
1067 cntrl
= V_TXPKT_INTF(pi
->port_id
);
1069 if (vlan_tx_tag_present(skb
) && pi
->vlan_grp
)
1070 cntrl
|= F_TXPKT_VLAN_VLD
| V_TXPKT_VLAN(vlan_tx_tag_get(skb
));
1072 tso_info
= V_LSO_MSS(skb_shinfo(skb
)->gso_size
);
1075 struct cpl_tx_pkt_lso
*hdr
= (struct cpl_tx_pkt_lso
*)cpl
;
1078 cntrl
|= V_TXPKT_OPCODE(CPL_TX_PKT_LSO
);
1079 hdr
->cntrl
= htonl(cntrl
);
1080 eth_type
= skb_network_offset(skb
) == ETH_HLEN
?
1081 CPL_ETH_II
: CPL_ETH_II_VLAN
;
1082 tso_info
|= V_LSO_ETH_TYPE(eth_type
) |
1083 V_LSO_IPHDR_WORDS(ip_hdr(skb
)->ihl
) |
1084 V_LSO_TCPHDR_WORDS(tcp_hdr(skb
)->doff
);
1085 hdr
->lso_info
= htonl(tso_info
);
1088 cntrl
|= V_TXPKT_OPCODE(CPL_TX_PKT
);
1089 cntrl
|= F_TXPKT_IPCSUM_DIS
; /* SW calculates IP csum */
1090 cntrl
|= V_TXPKT_L4CSUM_DIS(skb
->ip_summed
!= CHECKSUM_PARTIAL
);
1091 cpl
->cntrl
= htonl(cntrl
);
1093 if (skb
->len
<= WR_LEN
- sizeof(*cpl
)) {
1094 q
->sdesc
[pidx
].skb
= NULL
;
1096 skb_copy_from_linear_data(skb
, &d
->flit
[2],
1099 skb_copy_bits(skb
, 0, &d
->flit
[2], skb
->len
);
1101 flits
= (skb
->len
+ 7) / 8 + 2;
1102 cpl
->wr
.wr_hi
= htonl(V_WR_BCNTLFLT(skb
->len
& 7) |
1103 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT
)
1104 | F_WR_SOP
| F_WR_EOP
| compl);
1106 cpl
->wr
.wr_lo
= htonl(V_WR_LEN(flits
) | V_WR_GEN(gen
) |
1107 V_WR_TID(q
->token
));
1116 sgp
= ndesc
== 1 ? (struct sg_ent
*)&d
->flit
[flits
] : sgl
;
1117 sgl_flits
= make_sgl(skb
, sgp
, skb
->data
, skb_headlen(skb
), adap
->pdev
);
1119 write_wr_hdr_sgl(ndesc
, skb
, d
, pidx
, q
, sgl
, flits
, sgl_flits
, gen
,
1120 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT
) | compl),
1121 htonl(V_WR_TID(q
->token
)));
1124 static inline void t3_stop_queue(struct net_device
*dev
, struct sge_qset
*qs
,
1127 netif_stop_queue(dev
);
1128 set_bit(TXQ_ETH
, &qs
->txq_stopped
);
1133 * eth_xmit - add a packet to the Ethernet Tx queue
1135 * @dev: the egress net device
1137 * Add a packet to an SGE Tx queue. Runs with softirqs disabled.
1139 int t3_eth_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1141 unsigned int ndesc
, pidx
, credits
, gen
, compl;
1142 const struct port_info
*pi
= netdev_priv(dev
);
1143 struct adapter
*adap
= pi
->adapter
;
1144 struct sge_qset
*qs
= pi
->qs
;
1145 struct sge_txq
*q
= &qs
->txq
[TXQ_ETH
];
1148 * The chip min packet length is 9 octets but play safe and reject
1149 * anything shorter than an Ethernet header.
1151 if (unlikely(skb
->len
< ETH_HLEN
)) {
1153 return NETDEV_TX_OK
;
1156 spin_lock(&q
->lock
);
1157 reclaim_completed_tx(adap
, q
);
1159 credits
= q
->size
- q
->in_use
;
1160 ndesc
= calc_tx_descs(skb
);
1162 if (unlikely(credits
< ndesc
)) {
1163 t3_stop_queue(dev
, qs
, q
);
1164 dev_err(&adap
->pdev
->dev
,
1165 "%s: Tx ring %u full while queue awake!\n",
1166 dev
->name
, q
->cntxt_id
& 7);
1167 spin_unlock(&q
->lock
);
1168 return NETDEV_TX_BUSY
;
1172 if (unlikely(credits
- ndesc
< q
->stop_thres
)) {
1173 t3_stop_queue(dev
, qs
, q
);
1175 if (should_restart_tx(q
) &&
1176 test_and_clear_bit(TXQ_ETH
, &qs
->txq_stopped
)) {
1178 netif_wake_queue(dev
);
1183 q
->unacked
+= ndesc
;
1184 compl = (q
->unacked
& 8) << (S_WR_COMPL
- 3);
1188 if (q
->pidx
>= q
->size
) {
1193 /* update port statistics */
1194 if (skb
->ip_summed
== CHECKSUM_COMPLETE
)
1195 qs
->port_stats
[SGE_PSTAT_TX_CSUM
]++;
1196 if (skb_shinfo(skb
)->gso_size
)
1197 qs
->port_stats
[SGE_PSTAT_TSO
]++;
1198 if (vlan_tx_tag_present(skb
) && pi
->vlan_grp
)
1199 qs
->port_stats
[SGE_PSTAT_VLANINS
]++;
1201 dev
->trans_start
= jiffies
;
1202 spin_unlock(&q
->lock
);
1205 * We do not use Tx completion interrupts to free DMAd Tx packets.
1206 * This is good for performamce but means that we rely on new Tx
1207 * packets arriving to run the destructors of completed packets,
1208 * which open up space in their sockets' send queues. Sometimes
1209 * we do not get such new packets causing Tx to stall. A single
1210 * UDP transmitter is a good example of this situation. We have
1211 * a clean up timer that periodically reclaims completed packets
1212 * but it doesn't run often enough (nor do we want it to) to prevent
1213 * lengthy stalls. A solution to this problem is to run the
1214 * destructor early, after the packet is queued but before it's DMAd.
1215 * A cons is that we lie to socket memory accounting, but the amount
1216 * of extra memory is reasonable (limited by the number of Tx
1217 * descriptors), the packets do actually get freed quickly by new
1218 * packets almost always, and for protocols like TCP that wait for
1219 * acks to really free up the data the extra memory is even less.
1220 * On the positive side we run the destructors on the sending CPU
1221 * rather than on a potentially different completing CPU, usually a
1222 * good thing. We also run them without holding our Tx queue lock,
1223 * unlike what reclaim_completed_tx() would otherwise do.
1225 * Run the destructor before telling the DMA engine about the packet
1226 * to make sure it doesn't complete and get freed prematurely.
1228 if (likely(!skb_shared(skb
)))
1231 write_tx_pkt_wr(adap
, skb
, pi
, pidx
, gen
, q
, ndesc
, compl);
1232 check_ring_tx_db(adap
, q
);
1233 return NETDEV_TX_OK
;
1237 * write_imm - write a packet into a Tx descriptor as immediate data
1238 * @d: the Tx descriptor to write
1240 * @len: the length of packet data to write as immediate data
1241 * @gen: the generation bit value to write
1243 * Writes a packet as immediate data into a Tx descriptor. The packet
1244 * contains a work request at its beginning. We must write the packet
1245 * carefully so the SGE doesn't read it accidentally before it's written
1248 static inline void write_imm(struct tx_desc
*d
, struct sk_buff
*skb
,
1249 unsigned int len
, unsigned int gen
)
1251 struct work_request_hdr
*from
= (struct work_request_hdr
*)skb
->data
;
1252 struct work_request_hdr
*to
= (struct work_request_hdr
*)d
;
1254 if (likely(!skb
->data_len
))
1255 memcpy(&to
[1], &from
[1], len
- sizeof(*from
));
1257 skb_copy_bits(skb
, sizeof(*from
), &to
[1], len
- sizeof(*from
));
1259 to
->wr_hi
= from
->wr_hi
| htonl(F_WR_SOP
| F_WR_EOP
|
1260 V_WR_BCNTLFLT(len
& 7));
1262 to
->wr_lo
= from
->wr_lo
| htonl(V_WR_GEN(gen
) |
1263 V_WR_LEN((len
+ 7) / 8));
1269 * check_desc_avail - check descriptor availability on a send queue
1270 * @adap: the adapter
1271 * @q: the send queue
1272 * @skb: the packet needing the descriptors
1273 * @ndesc: the number of Tx descriptors needed
1274 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1276 * Checks if the requested number of Tx descriptors is available on an
1277 * SGE send queue. If the queue is already suspended or not enough
1278 * descriptors are available the packet is queued for later transmission.
1279 * Must be called with the Tx queue locked.
1281 * Returns 0 if enough descriptors are available, 1 if there aren't
1282 * enough descriptors and the packet has been queued, and 2 if the caller
1283 * needs to retry because there weren't enough descriptors at the
1284 * beginning of the call but some freed up in the mean time.
1286 static inline int check_desc_avail(struct adapter
*adap
, struct sge_txq
*q
,
1287 struct sk_buff
*skb
, unsigned int ndesc
,
1290 if (unlikely(!skb_queue_empty(&q
->sendq
))) {
1291 addq_exit
:__skb_queue_tail(&q
->sendq
, skb
);
1294 if (unlikely(q
->size
- q
->in_use
< ndesc
)) {
1295 struct sge_qset
*qs
= txq_to_qset(q
, qid
);
1297 set_bit(qid
, &qs
->txq_stopped
);
1298 smp_mb__after_clear_bit();
1300 if (should_restart_tx(q
) &&
1301 test_and_clear_bit(qid
, &qs
->txq_stopped
))
1311 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1312 * @q: the SGE control Tx queue
1314 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1315 * that send only immediate data (presently just the control queues) and
1316 * thus do not have any sk_buffs to release.
1318 static inline void reclaim_completed_tx_imm(struct sge_txq
*q
)
1320 unsigned int reclaim
= q
->processed
- q
->cleaned
;
1322 q
->in_use
-= reclaim
;
1323 q
->cleaned
+= reclaim
;
1326 static inline int immediate(const struct sk_buff
*skb
)
1328 return skb
->len
<= WR_LEN
;
1332 * ctrl_xmit - send a packet through an SGE control Tx queue
1333 * @adap: the adapter
1334 * @q: the control queue
1337 * Send a packet through an SGE control Tx queue. Packets sent through
1338 * a control queue must fit entirely as immediate data in a single Tx
1339 * descriptor and have no page fragments.
1341 static int ctrl_xmit(struct adapter
*adap
, struct sge_txq
*q
,
1342 struct sk_buff
*skb
)
1345 struct work_request_hdr
*wrp
= (struct work_request_hdr
*)skb
->data
;
1347 if (unlikely(!immediate(skb
))) {
1350 return NET_XMIT_SUCCESS
;
1353 wrp
->wr_hi
|= htonl(F_WR_SOP
| F_WR_EOP
);
1354 wrp
->wr_lo
= htonl(V_WR_TID(q
->token
));
1356 spin_lock(&q
->lock
);
1357 again
:reclaim_completed_tx_imm(q
);
1359 ret
= check_desc_avail(adap
, q
, skb
, 1, TXQ_CTRL
);
1360 if (unlikely(ret
)) {
1362 spin_unlock(&q
->lock
);
1368 write_imm(&q
->desc
[q
->pidx
], skb
, skb
->len
, q
->gen
);
1371 if (++q
->pidx
>= q
->size
) {
1375 spin_unlock(&q
->lock
);
1377 t3_write_reg(adap
, A_SG_KDOORBELL
,
1378 F_SELEGRCNTX
| V_EGRCNTX(q
->cntxt_id
));
1379 return NET_XMIT_SUCCESS
;
1383 * restart_ctrlq - restart a suspended control queue
1384 * @qs: the queue set cotaining the control queue
1386 * Resumes transmission on a suspended Tx control queue.
1388 static void restart_ctrlq(unsigned long data
)
1390 struct sk_buff
*skb
;
1391 struct sge_qset
*qs
= (struct sge_qset
*)data
;
1392 struct sge_txq
*q
= &qs
->txq
[TXQ_CTRL
];
1394 spin_lock(&q
->lock
);
1395 again
:reclaim_completed_tx_imm(q
);
1397 while (q
->in_use
< q
->size
&&
1398 (skb
= __skb_dequeue(&q
->sendq
)) != NULL
) {
1400 write_imm(&q
->desc
[q
->pidx
], skb
, skb
->len
, q
->gen
);
1402 if (++q
->pidx
>= q
->size
) {
1409 if (!skb_queue_empty(&q
->sendq
)) {
1410 set_bit(TXQ_CTRL
, &qs
->txq_stopped
);
1411 smp_mb__after_clear_bit();
1413 if (should_restart_tx(q
) &&
1414 test_and_clear_bit(TXQ_CTRL
, &qs
->txq_stopped
))
1419 spin_unlock(&q
->lock
);
1421 t3_write_reg(qs
->adap
, A_SG_KDOORBELL
,
1422 F_SELEGRCNTX
| V_EGRCNTX(q
->cntxt_id
));
1426 * Send a management message through control queue 0
1428 int t3_mgmt_tx(struct adapter
*adap
, struct sk_buff
*skb
)
1432 ret
= ctrl_xmit(adap
, &adap
->sge
.qs
[0].txq
[TXQ_CTRL
], skb
);
1439 * deferred_unmap_destructor - unmap a packet when it is freed
1442 * This is the packet destructor used for Tx packets that need to remain
1443 * mapped until they are freed rather than until their Tx descriptors are
1446 static void deferred_unmap_destructor(struct sk_buff
*skb
)
1449 const dma_addr_t
*p
;
1450 const struct skb_shared_info
*si
;
1451 const struct deferred_unmap_info
*dui
;
1453 dui
= (struct deferred_unmap_info
*)skb
->head
;
1456 if (skb
->tail
- skb
->transport_header
)
1457 pci_unmap_single(dui
->pdev
, *p
++,
1458 skb
->tail
- skb
->transport_header
,
1461 si
= skb_shinfo(skb
);
1462 for (i
= 0; i
< si
->nr_frags
; i
++)
1463 pci_unmap_page(dui
->pdev
, *p
++, si
->frags
[i
].size
,
1467 static void setup_deferred_unmapping(struct sk_buff
*skb
, struct pci_dev
*pdev
,
1468 const struct sg_ent
*sgl
, int sgl_flits
)
1471 struct deferred_unmap_info
*dui
;
1473 dui
= (struct deferred_unmap_info
*)skb
->head
;
1475 for (p
= dui
->addr
; sgl_flits
>= 3; sgl
++, sgl_flits
-= 3) {
1476 *p
++ = be64_to_cpu(sgl
->addr
[0]);
1477 *p
++ = be64_to_cpu(sgl
->addr
[1]);
1480 *p
= be64_to_cpu(sgl
->addr
[0]);
1484 * write_ofld_wr - write an offload work request
1485 * @adap: the adapter
1486 * @skb: the packet to send
1488 * @pidx: index of the first Tx descriptor to write
1489 * @gen: the generation value to use
1490 * @ndesc: number of descriptors the packet will occupy
1492 * Write an offload work request to send the supplied packet. The packet
1493 * data already carry the work request with most fields populated.
1495 static void write_ofld_wr(struct adapter
*adap
, struct sk_buff
*skb
,
1496 struct sge_txq
*q
, unsigned int pidx
,
1497 unsigned int gen
, unsigned int ndesc
)
1499 unsigned int sgl_flits
, flits
;
1500 struct work_request_hdr
*from
;
1501 struct sg_ent
*sgp
, sgl
[MAX_SKB_FRAGS
/ 2 + 1];
1502 struct tx_desc
*d
= &q
->desc
[pidx
];
1504 if (immediate(skb
)) {
1505 q
->sdesc
[pidx
].skb
= NULL
;
1506 write_imm(d
, skb
, skb
->len
, gen
);
1510 /* Only TX_DATA builds SGLs */
1512 from
= (struct work_request_hdr
*)skb
->data
;
1513 memcpy(&d
->flit
[1], &from
[1],
1514 skb_transport_offset(skb
) - sizeof(*from
));
1516 flits
= skb_transport_offset(skb
) / 8;
1517 sgp
= ndesc
== 1 ? (struct sg_ent
*)&d
->flit
[flits
] : sgl
;
1518 sgl_flits
= make_sgl(skb
, sgp
, skb_transport_header(skb
),
1519 skb
->tail
- skb
->transport_header
,
1521 if (need_skb_unmap()) {
1522 setup_deferred_unmapping(skb
, adap
->pdev
, sgp
, sgl_flits
);
1523 skb
->destructor
= deferred_unmap_destructor
;
1526 write_wr_hdr_sgl(ndesc
, skb
, d
, pidx
, q
, sgl
, flits
, sgl_flits
,
1527 gen
, from
->wr_hi
, from
->wr_lo
);
1531 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1534 * Returns the number of Tx descriptors needed for the given offload
1535 * packet. These packets are already fully constructed.
1537 static inline unsigned int calc_tx_descs_ofld(const struct sk_buff
*skb
)
1539 unsigned int flits
, cnt
;
1541 if (skb
->len
<= WR_LEN
)
1542 return 1; /* packet fits as immediate data */
1544 flits
= skb_transport_offset(skb
) / 8; /* headers */
1545 cnt
= skb_shinfo(skb
)->nr_frags
;
1546 if (skb
->tail
!= skb
->transport_header
)
1548 return flits_to_desc(flits
+ sgl_len(cnt
));
1552 * ofld_xmit - send a packet through an offload queue
1553 * @adap: the adapter
1554 * @q: the Tx offload queue
1557 * Send an offload packet through an SGE offload queue.
1559 static int ofld_xmit(struct adapter
*adap
, struct sge_txq
*q
,
1560 struct sk_buff
*skb
)
1563 unsigned int ndesc
= calc_tx_descs_ofld(skb
), pidx
, gen
;
1565 spin_lock(&q
->lock
);
1566 again
:reclaim_completed_tx(adap
, q
);
1568 ret
= check_desc_avail(adap
, q
, skb
, ndesc
, TXQ_OFLD
);
1569 if (unlikely(ret
)) {
1571 skb
->priority
= ndesc
; /* save for restart */
1572 spin_unlock(&q
->lock
);
1582 if (q
->pidx
>= q
->size
) {
1586 spin_unlock(&q
->lock
);
1588 write_ofld_wr(adap
, skb
, q
, pidx
, gen
, ndesc
);
1589 check_ring_tx_db(adap
, q
);
1590 return NET_XMIT_SUCCESS
;
1594 * restart_offloadq - restart a suspended offload queue
1595 * @qs: the queue set cotaining the offload queue
1597 * Resumes transmission on a suspended Tx offload queue.
1599 static void restart_offloadq(unsigned long data
)
1601 struct sk_buff
*skb
;
1602 struct sge_qset
*qs
= (struct sge_qset
*)data
;
1603 struct sge_txq
*q
= &qs
->txq
[TXQ_OFLD
];
1604 const struct port_info
*pi
= netdev_priv(qs
->netdev
);
1605 struct adapter
*adap
= pi
->adapter
;
1607 spin_lock(&q
->lock
);
1608 again
:reclaim_completed_tx(adap
, q
);
1610 while ((skb
= skb_peek(&q
->sendq
)) != NULL
) {
1611 unsigned int gen
, pidx
;
1612 unsigned int ndesc
= skb
->priority
;
1614 if (unlikely(q
->size
- q
->in_use
< ndesc
)) {
1615 set_bit(TXQ_OFLD
, &qs
->txq_stopped
);
1616 smp_mb__after_clear_bit();
1618 if (should_restart_tx(q
) &&
1619 test_and_clear_bit(TXQ_OFLD
, &qs
->txq_stopped
))
1629 if (q
->pidx
>= q
->size
) {
1633 __skb_unlink(skb
, &q
->sendq
);
1634 spin_unlock(&q
->lock
);
1636 write_ofld_wr(adap
, skb
, q
, pidx
, gen
, ndesc
);
1637 spin_lock(&q
->lock
);
1639 spin_unlock(&q
->lock
);
1642 set_bit(TXQ_RUNNING
, &q
->flags
);
1643 set_bit(TXQ_LAST_PKT_DB
, &q
->flags
);
1646 t3_write_reg(adap
, A_SG_KDOORBELL
,
1647 F_SELEGRCNTX
| V_EGRCNTX(q
->cntxt_id
));
1651 * queue_set - return the queue set a packet should use
1654 * Maps a packet to the SGE queue set it should use. The desired queue
1655 * set is carried in bits 1-3 in the packet's priority.
1657 static inline int queue_set(const struct sk_buff
*skb
)
1659 return skb
->priority
>> 1;
1663 * is_ctrl_pkt - return whether an offload packet is a control packet
1666 * Determines whether an offload packet should use an OFLD or a CTRL
1667 * Tx queue. This is indicated by bit 0 in the packet's priority.
1669 static inline int is_ctrl_pkt(const struct sk_buff
*skb
)
1671 return skb
->priority
& 1;
1675 * t3_offload_tx - send an offload packet
1676 * @tdev: the offload device to send to
1679 * Sends an offload packet. We use the packet priority to select the
1680 * appropriate Tx queue as follows: bit 0 indicates whether the packet
1681 * should be sent as regular or control, bits 1-3 select the queue set.
1683 int t3_offload_tx(struct t3cdev
*tdev
, struct sk_buff
*skb
)
1685 struct adapter
*adap
= tdev2adap(tdev
);
1686 struct sge_qset
*qs
= &adap
->sge
.qs
[queue_set(skb
)];
1688 if (unlikely(is_ctrl_pkt(skb
)))
1689 return ctrl_xmit(adap
, &qs
->txq
[TXQ_CTRL
], skb
);
1691 return ofld_xmit(adap
, &qs
->txq
[TXQ_OFLD
], skb
);
1695 * offload_enqueue - add an offload packet to an SGE offload receive queue
1696 * @q: the SGE response queue
1699 * Add a new offload packet to an SGE response queue's offload packet
1700 * queue. If the packet is the first on the queue it schedules the RX
1701 * softirq to process the queue.
1703 static inline void offload_enqueue(struct sge_rspq
*q
, struct sk_buff
*skb
)
1705 skb
->next
= skb
->prev
= NULL
;
1707 q
->rx_tail
->next
= skb
;
1709 struct sge_qset
*qs
= rspq_to_qset(q
);
1711 napi_schedule(&qs
->napi
);
1718 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1719 * @tdev: the offload device that will be receiving the packets
1720 * @q: the SGE response queue that assembled the bundle
1721 * @skbs: the partial bundle
1722 * @n: the number of packets in the bundle
1724 * Delivers a (partial) bundle of Rx offload packets to an offload device.
1726 static inline void deliver_partial_bundle(struct t3cdev
*tdev
,
1728 struct sk_buff
*skbs
[], int n
)
1731 q
->offload_bundles
++;
1732 tdev
->recv(tdev
, skbs
, n
);
1737 * ofld_poll - NAPI handler for offload packets in interrupt mode
1738 * @dev: the network device doing the polling
1739 * @budget: polling budget
1741 * The NAPI handler for offload packets when a response queue is serviced
1742 * by the hard interrupt handler, i.e., when it's operating in non-polling
1743 * mode. Creates small packet batches and sends them through the offload
1744 * receive handler. Batches need to be of modest size as we do prefetches
1745 * on the packets in each.
1747 static int ofld_poll(struct napi_struct
*napi
, int budget
)
1749 struct sge_qset
*qs
= container_of(napi
, struct sge_qset
, napi
);
1750 struct sge_rspq
*q
= &qs
->rspq
;
1751 struct adapter
*adapter
= qs
->adap
;
1754 while (work_done
< budget
) {
1755 struct sk_buff
*head
, *tail
, *skbs
[RX_BUNDLE_SIZE
];
1758 spin_lock_irq(&q
->lock
);
1761 napi_complete(napi
);
1762 spin_unlock_irq(&q
->lock
);
1767 q
->rx_head
= q
->rx_tail
= NULL
;
1768 spin_unlock_irq(&q
->lock
);
1770 for (ngathered
= 0; work_done
< budget
&& head
; work_done
++) {
1771 prefetch(head
->data
);
1772 skbs
[ngathered
] = head
;
1774 skbs
[ngathered
]->next
= NULL
;
1775 if (++ngathered
== RX_BUNDLE_SIZE
) {
1776 q
->offload_bundles
++;
1777 adapter
->tdev
.recv(&adapter
->tdev
, skbs
,
1782 if (head
) { /* splice remaining packets back onto Rx queue */
1783 spin_lock_irq(&q
->lock
);
1784 tail
->next
= q
->rx_head
;
1788 spin_unlock_irq(&q
->lock
);
1790 deliver_partial_bundle(&adapter
->tdev
, q
, skbs
, ngathered
);
1797 * rx_offload - process a received offload packet
1798 * @tdev: the offload device receiving the packet
1799 * @rq: the response queue that received the packet
1801 * @rx_gather: a gather list of packets if we are building a bundle
1802 * @gather_idx: index of the next available slot in the bundle
1804 * Process an ingress offload pakcet and add it to the offload ingress
1805 * queue. Returns the index of the next available slot in the bundle.
1807 static inline int rx_offload(struct t3cdev
*tdev
, struct sge_rspq
*rq
,
1808 struct sk_buff
*skb
, struct sk_buff
*rx_gather
[],
1809 unsigned int gather_idx
)
1811 skb_reset_mac_header(skb
);
1812 skb_reset_network_header(skb
);
1813 skb_reset_transport_header(skb
);
1816 rx_gather
[gather_idx
++] = skb
;
1817 if (gather_idx
== RX_BUNDLE_SIZE
) {
1818 tdev
->recv(tdev
, rx_gather
, RX_BUNDLE_SIZE
);
1820 rq
->offload_bundles
++;
1823 offload_enqueue(rq
, skb
);
1829 * restart_tx - check whether to restart suspended Tx queues
1830 * @qs: the queue set to resume
1832 * Restarts suspended Tx queues of an SGE queue set if they have enough
1833 * free resources to resume operation.
1835 static void restart_tx(struct sge_qset
*qs
)
1837 if (test_bit(TXQ_ETH
, &qs
->txq_stopped
) &&
1838 should_restart_tx(&qs
->txq
[TXQ_ETH
]) &&
1839 test_and_clear_bit(TXQ_ETH
, &qs
->txq_stopped
)) {
1840 qs
->txq
[TXQ_ETH
].restarts
++;
1841 if (netif_running(qs
->netdev
))
1842 netif_wake_queue(qs
->netdev
);
1845 if (test_bit(TXQ_OFLD
, &qs
->txq_stopped
) &&
1846 should_restart_tx(&qs
->txq
[TXQ_OFLD
]) &&
1847 test_and_clear_bit(TXQ_OFLD
, &qs
->txq_stopped
)) {
1848 qs
->txq
[TXQ_OFLD
].restarts
++;
1849 tasklet_schedule(&qs
->txq
[TXQ_OFLD
].qresume_tsk
);
1851 if (test_bit(TXQ_CTRL
, &qs
->txq_stopped
) &&
1852 should_restart_tx(&qs
->txq
[TXQ_CTRL
]) &&
1853 test_and_clear_bit(TXQ_CTRL
, &qs
->txq_stopped
)) {
1854 qs
->txq
[TXQ_CTRL
].restarts
++;
1855 tasklet_schedule(&qs
->txq
[TXQ_CTRL
].qresume_tsk
);
1860 * rx_eth - process an ingress ethernet packet
1861 * @adap: the adapter
1862 * @rq: the response queue that received the packet
1864 * @pad: amount of padding at the start of the buffer
1866 * Process an ingress ethernet pakcet and deliver it to the stack.
1867 * The padding is 2 if the packet was delivered in an Rx buffer and 0
1868 * if it was immediate data in a response.
1870 static void rx_eth(struct adapter
*adap
, struct sge_rspq
*rq
,
1871 struct sk_buff
*skb
, int pad
)
1873 struct cpl_rx_pkt
*p
= (struct cpl_rx_pkt
*)(skb
->data
+ pad
);
1874 struct port_info
*pi
;
1876 skb_pull(skb
, sizeof(*p
) + pad
);
1877 skb
->protocol
= eth_type_trans(skb
, adap
->port
[p
->iff
]);
1878 skb
->dev
->last_rx
= jiffies
;
1879 pi
= netdev_priv(skb
->dev
);
1880 if (pi
->rx_csum_offload
&& p
->csum_valid
&& p
->csum
== htons(0xffff) &&
1882 rspq_to_qset(rq
)->port_stats
[SGE_PSTAT_RX_CSUM_GOOD
]++;
1883 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1885 skb
->ip_summed
= CHECKSUM_NONE
;
1887 if (unlikely(p
->vlan_valid
)) {
1888 struct vlan_group
*grp
= pi
->vlan_grp
;
1890 rspq_to_qset(rq
)->port_stats
[SGE_PSTAT_VLANEX
]++;
1892 __vlan_hwaccel_rx(skb
, grp
, ntohs(p
->vlan
),
1895 dev_kfree_skb_any(skb
);
1896 } else if (rq
->polling
)
1897 netif_receive_skb(skb
);
1903 * handle_rsp_cntrl_info - handles control information in a response
1904 * @qs: the queue set corresponding to the response
1905 * @flags: the response control flags
1907 * Handles the control information of an SGE response, such as GTS
1908 * indications and completion credits for the queue set's Tx queues.
1909 * HW coalesces credits, we don't do any extra SW coalescing.
1911 static inline void handle_rsp_cntrl_info(struct sge_qset
*qs
, u32 flags
)
1913 unsigned int credits
;
1916 if (flags
& F_RSPD_TXQ0_GTS
)
1917 clear_bit(TXQ_RUNNING
, &qs
->txq
[TXQ_ETH
].flags
);
1920 credits
= G_RSPD_TXQ0_CR(flags
);
1922 qs
->txq
[TXQ_ETH
].processed
+= credits
;
1924 credits
= G_RSPD_TXQ2_CR(flags
);
1926 qs
->txq
[TXQ_CTRL
].processed
+= credits
;
1929 if (flags
& F_RSPD_TXQ1_GTS
)
1930 clear_bit(TXQ_RUNNING
, &qs
->txq
[TXQ_OFLD
].flags
);
1932 credits
= G_RSPD_TXQ1_CR(flags
);
1934 qs
->txq
[TXQ_OFLD
].processed
+= credits
;
1938 * check_ring_db - check if we need to ring any doorbells
1939 * @adapter: the adapter
1940 * @qs: the queue set whose Tx queues are to be examined
1941 * @sleeping: indicates which Tx queue sent GTS
1943 * Checks if some of a queue set's Tx queues need to ring their doorbells
1944 * to resume transmission after idling while they still have unprocessed
1947 static void check_ring_db(struct adapter
*adap
, struct sge_qset
*qs
,
1948 unsigned int sleeping
)
1950 if (sleeping
& F_RSPD_TXQ0_GTS
) {
1951 struct sge_txq
*txq
= &qs
->txq
[TXQ_ETH
];
1953 if (txq
->cleaned
+ txq
->in_use
!= txq
->processed
&&
1954 !test_and_set_bit(TXQ_LAST_PKT_DB
, &txq
->flags
)) {
1955 set_bit(TXQ_RUNNING
, &txq
->flags
);
1956 t3_write_reg(adap
, A_SG_KDOORBELL
, F_SELEGRCNTX
|
1957 V_EGRCNTX(txq
->cntxt_id
));
1961 if (sleeping
& F_RSPD_TXQ1_GTS
) {
1962 struct sge_txq
*txq
= &qs
->txq
[TXQ_OFLD
];
1964 if (txq
->cleaned
+ txq
->in_use
!= txq
->processed
&&
1965 !test_and_set_bit(TXQ_LAST_PKT_DB
, &txq
->flags
)) {
1966 set_bit(TXQ_RUNNING
, &txq
->flags
);
1967 t3_write_reg(adap
, A_SG_KDOORBELL
, F_SELEGRCNTX
|
1968 V_EGRCNTX(txq
->cntxt_id
));
1974 * is_new_response - check if a response is newly written
1975 * @r: the response descriptor
1976 * @q: the response queue
1978 * Returns true if a response descriptor contains a yet unprocessed
1981 static inline int is_new_response(const struct rsp_desc
*r
,
1982 const struct sge_rspq
*q
)
1984 return (r
->intr_gen
& F_RSPD_GEN2
) == q
->gen
;
1987 static inline void clear_rspq_bufstate(struct sge_rspq
* const q
)
1990 q
->rx_recycle_buf
= 0;
1993 #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
1994 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
1995 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
1996 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
1997 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
1999 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2000 #define NOMEM_INTR_DELAY 2500
2003 * process_responses - process responses from an SGE response queue
2004 * @adap: the adapter
2005 * @qs: the queue set to which the response queue belongs
2006 * @budget: how many responses can be processed in this round
2008 * Process responses from an SGE response queue up to the supplied budget.
2009 * Responses include received packets as well as credits and other events
2010 * for the queues that belong to the response queue's queue set.
2011 * A negative budget is effectively unlimited.
2013 * Additionally choose the interrupt holdoff time for the next interrupt
2014 * on this queue. If the system is under memory shortage use a fairly
2015 * long delay to help recovery.
2017 static int process_responses(struct adapter
*adap
, struct sge_qset
*qs
,
2020 struct sge_rspq
*q
= &qs
->rspq
;
2021 struct rsp_desc
*r
= &q
->desc
[q
->cidx
];
2022 int budget_left
= budget
;
2023 unsigned int sleeping
= 0;
2024 struct sk_buff
*offload_skbs
[RX_BUNDLE_SIZE
];
2027 q
->next_holdoff
= q
->holdoff_tmr
;
2029 while (likely(budget_left
&& is_new_response(r
, q
))) {
2030 int packet_complete
, eth
, ethpad
= 2;
2031 struct sk_buff
*skb
= NULL
;
2032 u32 len
, flags
= ntohl(r
->flags
);
2033 __be32 rss_hi
= *(const __be32
*)r
,
2034 rss_lo
= r
->rss_hdr
.rss_hash_val
;
2036 eth
= r
->rss_hdr
.opcode
== CPL_RX_PKT
;
2038 if (unlikely(flags
& F_RSPD_ASYNC_NOTIF
)) {
2039 skb
= alloc_skb(AN_PKT_SIZE
, GFP_ATOMIC
);
2043 memcpy(__skb_put(skb
, AN_PKT_SIZE
), r
, AN_PKT_SIZE
);
2044 skb
->data
[0] = CPL_ASYNC_NOTIF
;
2045 rss_hi
= htonl(CPL_ASYNC_NOTIF
<< 24);
2047 } else if (flags
& F_RSPD_IMM_DATA_VALID
) {
2048 skb
= get_imm_packet(r
);
2049 if (unlikely(!skb
)) {
2051 q
->next_holdoff
= NOMEM_INTR_DELAY
;
2053 /* consume one credit since we tried */
2059 } else if ((len
= ntohl(r
->len_cq
)) != 0) {
2062 fl
= (len
& F_RSPD_FLQ
) ? &qs
->fl
[1] : &qs
->fl
[0];
2063 if (fl
->use_pages
) {
2064 void *addr
= fl
->sdesc
[fl
->cidx
].pg_chunk
.va
;
2067 #if L1_CACHE_BYTES < 128
2068 prefetch(addr
+ L1_CACHE_BYTES
);
2070 __refill_fl(adap
, fl
);
2072 skb
= get_packet_pg(adap
, fl
, q
,
2075 SGE_RX_DROP_THRES
: 0);
2078 skb
= get_packet(adap
, fl
, G_RSPD_LEN(len
),
2079 eth
? SGE_RX_DROP_THRES
: 0);
2080 if (unlikely(!skb
)) {
2084 } else if (unlikely(r
->rss_hdr
.opcode
== CPL_TRACE_PKT
))
2087 if (++fl
->cidx
== fl
->size
)
2092 if (flags
& RSPD_CTRL_MASK
) {
2093 sleeping
|= flags
& RSPD_GTS_MASK
;
2094 handle_rsp_cntrl_info(qs
, flags
);
2098 if (unlikely(++q
->cidx
== q
->size
)) {
2105 if (++q
->credits
>= (q
->size
/ 4)) {
2106 refill_rspq(adap
, q
, q
->credits
);
2110 packet_complete
= flags
&
2111 (F_RSPD_EOP
| F_RSPD_IMM_DATA_VALID
|
2112 F_RSPD_ASYNC_NOTIF
);
2114 if (skb
!= NULL
&& packet_complete
) {
2116 rx_eth(adap
, q
, skb
, ethpad
);
2119 /* Preserve the RSS info in csum & priority */
2121 skb
->priority
= rss_lo
;
2122 ngathered
= rx_offload(&adap
->tdev
, q
, skb
,
2127 if (flags
& F_RSPD_EOP
)
2128 clear_rspq_bufstate(q
);
2133 deliver_partial_bundle(&adap
->tdev
, q
, offload_skbs
, ngathered
);
2135 check_ring_db(adap
, qs
, sleeping
);
2137 smp_mb(); /* commit Tx queue .processed updates */
2138 if (unlikely(qs
->txq_stopped
!= 0))
2141 budget
-= budget_left
;
2145 static inline int is_pure_response(const struct rsp_desc
*r
)
2147 u32 n
= ntohl(r
->flags
) & (F_RSPD_ASYNC_NOTIF
| F_RSPD_IMM_DATA_VALID
);
2149 return (n
| r
->len_cq
) == 0;
2153 * napi_rx_handler - the NAPI handler for Rx processing
2154 * @napi: the napi instance
2155 * @budget: how many packets we can process in this round
2157 * Handler for new data events when using NAPI.
2159 static int napi_rx_handler(struct napi_struct
*napi
, int budget
)
2161 struct sge_qset
*qs
= container_of(napi
, struct sge_qset
, napi
);
2162 struct adapter
*adap
= qs
->adap
;
2163 int work_done
= process_responses(adap
, qs
, budget
);
2165 if (likely(work_done
< budget
)) {
2166 napi_complete(napi
);
2169 * Because we don't atomically flush the following
2170 * write it is possible that in very rare cases it can
2171 * reach the device in a way that races with a new
2172 * response being written plus an error interrupt
2173 * causing the NAPI interrupt handler below to return
2174 * unhandled status to the OS. To protect against
2175 * this would require flushing the write and doing
2176 * both the write and the flush with interrupts off.
2177 * Way too expensive and unjustifiable given the
2178 * rarity of the race.
2180 * The race cannot happen at all with MSI-X.
2182 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(qs
->rspq
.cntxt_id
) |
2183 V_NEWTIMER(qs
->rspq
.next_holdoff
) |
2184 V_NEWINDEX(qs
->rspq
.cidx
));
2190 * Returns true if the device is already scheduled for polling.
2192 static inline int napi_is_scheduled(struct napi_struct
*napi
)
2194 return test_bit(NAPI_STATE_SCHED
, &napi
->state
);
2198 * process_pure_responses - process pure responses from a response queue
2199 * @adap: the adapter
2200 * @qs: the queue set owning the response queue
2201 * @r: the first pure response to process
2203 * A simpler version of process_responses() that handles only pure (i.e.,
2204 * non data-carrying) responses. Such respones are too light-weight to
2205 * justify calling a softirq under NAPI, so we handle them specially in
2206 * the interrupt handler. The function is called with a pointer to a
2207 * response, which the caller must ensure is a valid pure response.
2209 * Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2211 static int process_pure_responses(struct adapter
*adap
, struct sge_qset
*qs
,
2214 struct sge_rspq
*q
= &qs
->rspq
;
2215 unsigned int sleeping
= 0;
2218 u32 flags
= ntohl(r
->flags
);
2221 if (unlikely(++q
->cidx
== q
->size
)) {
2228 if (flags
& RSPD_CTRL_MASK
) {
2229 sleeping
|= flags
& RSPD_GTS_MASK
;
2230 handle_rsp_cntrl_info(qs
, flags
);
2234 if (++q
->credits
>= (q
->size
/ 4)) {
2235 refill_rspq(adap
, q
, q
->credits
);
2238 } while (is_new_response(r
, q
) && is_pure_response(r
));
2241 check_ring_db(adap
, qs
, sleeping
);
2243 smp_mb(); /* commit Tx queue .processed updates */
2244 if (unlikely(qs
->txq_stopped
!= 0))
2247 return is_new_response(r
, q
);
2251 * handle_responses - decide what to do with new responses in NAPI mode
2252 * @adap: the adapter
2253 * @q: the response queue
2255 * This is used by the NAPI interrupt handlers to decide what to do with
2256 * new SGE responses. If there are no new responses it returns -1. If
2257 * there are new responses and they are pure (i.e., non-data carrying)
2258 * it handles them straight in hard interrupt context as they are very
2259 * cheap and don't deliver any packets. Finally, if there are any data
2260 * signaling responses it schedules the NAPI handler. Returns 1 if it
2261 * schedules NAPI, 0 if all new responses were pure.
2263 * The caller must ascertain NAPI is not already running.
2265 static inline int handle_responses(struct adapter
*adap
, struct sge_rspq
*q
)
2267 struct sge_qset
*qs
= rspq_to_qset(q
);
2268 struct rsp_desc
*r
= &q
->desc
[q
->cidx
];
2270 if (!is_new_response(r
, q
))
2272 if (is_pure_response(r
) && process_pure_responses(adap
, qs
, r
) == 0) {
2273 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(q
->cntxt_id
) |
2274 V_NEWTIMER(q
->holdoff_tmr
) | V_NEWINDEX(q
->cidx
));
2277 napi_schedule(&qs
->napi
);
2282 * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2283 * (i.e., response queue serviced in hard interrupt).
2285 irqreturn_t
t3_sge_intr_msix(int irq
, void *cookie
)
2287 struct sge_qset
*qs
= cookie
;
2288 struct adapter
*adap
= qs
->adap
;
2289 struct sge_rspq
*q
= &qs
->rspq
;
2291 spin_lock(&q
->lock
);
2292 if (process_responses(adap
, qs
, -1) == 0)
2293 q
->unhandled_irqs
++;
2294 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(q
->cntxt_id
) |
2295 V_NEWTIMER(q
->next_holdoff
) | V_NEWINDEX(q
->cidx
));
2296 spin_unlock(&q
->lock
);
2301 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2302 * (i.e., response queue serviced by NAPI polling).
2304 static irqreturn_t
t3_sge_intr_msix_napi(int irq
, void *cookie
)
2306 struct sge_qset
*qs
= cookie
;
2307 struct sge_rspq
*q
= &qs
->rspq
;
2309 spin_lock(&q
->lock
);
2311 if (handle_responses(qs
->adap
, q
) < 0)
2312 q
->unhandled_irqs
++;
2313 spin_unlock(&q
->lock
);
2318 * The non-NAPI MSI interrupt handler. This needs to handle data events from
2319 * SGE response queues as well as error and other async events as they all use
2320 * the same MSI vector. We use one SGE response queue per port in this mode
2321 * and protect all response queues with queue 0's lock.
2323 static irqreturn_t
t3_intr_msi(int irq
, void *cookie
)
2325 int new_packets
= 0;
2326 struct adapter
*adap
= cookie
;
2327 struct sge_rspq
*q
= &adap
->sge
.qs
[0].rspq
;
2329 spin_lock(&q
->lock
);
2331 if (process_responses(adap
, &adap
->sge
.qs
[0], -1)) {
2332 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(q
->cntxt_id
) |
2333 V_NEWTIMER(q
->next_holdoff
) | V_NEWINDEX(q
->cidx
));
2337 if (adap
->params
.nports
== 2 &&
2338 process_responses(adap
, &adap
->sge
.qs
[1], -1)) {
2339 struct sge_rspq
*q1
= &adap
->sge
.qs
[1].rspq
;
2341 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(q1
->cntxt_id
) |
2342 V_NEWTIMER(q1
->next_holdoff
) |
2343 V_NEWINDEX(q1
->cidx
));
2347 if (!new_packets
&& t3_slow_intr_handler(adap
) == 0)
2348 q
->unhandled_irqs
++;
2350 spin_unlock(&q
->lock
);
2354 static int rspq_check_napi(struct sge_qset
*qs
)
2356 struct sge_rspq
*q
= &qs
->rspq
;
2358 if (!napi_is_scheduled(&qs
->napi
) &&
2359 is_new_response(&q
->desc
[q
->cidx
], q
)) {
2360 napi_schedule(&qs
->napi
);
2367 * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2368 * by NAPI polling). Handles data events from SGE response queues as well as
2369 * error and other async events as they all use the same MSI vector. We use
2370 * one SGE response queue per port in this mode and protect all response
2371 * queues with queue 0's lock.
2373 static irqreturn_t
t3_intr_msi_napi(int irq
, void *cookie
)
2376 struct adapter
*adap
= cookie
;
2377 struct sge_rspq
*q
= &adap
->sge
.qs
[0].rspq
;
2379 spin_lock(&q
->lock
);
2381 new_packets
= rspq_check_napi(&adap
->sge
.qs
[0]);
2382 if (adap
->params
.nports
== 2)
2383 new_packets
+= rspq_check_napi(&adap
->sge
.qs
[1]);
2384 if (!new_packets
&& t3_slow_intr_handler(adap
) == 0)
2385 q
->unhandled_irqs
++;
2387 spin_unlock(&q
->lock
);
2392 * A helper function that processes responses and issues GTS.
2394 static inline int process_responses_gts(struct adapter
*adap
,
2395 struct sge_rspq
*rq
)
2399 work
= process_responses(adap
, rspq_to_qset(rq
), -1);
2400 t3_write_reg(adap
, A_SG_GTS
, V_RSPQ(rq
->cntxt_id
) |
2401 V_NEWTIMER(rq
->next_holdoff
) | V_NEWINDEX(rq
->cidx
));
2406 * The legacy INTx interrupt handler. This needs to handle data events from
2407 * SGE response queues as well as error and other async events as they all use
2408 * the same interrupt pin. We use one SGE response queue per port in this mode
2409 * and protect all response queues with queue 0's lock.
2411 static irqreturn_t
t3_intr(int irq
, void *cookie
)
2413 int work_done
, w0
, w1
;
2414 struct adapter
*adap
= cookie
;
2415 struct sge_rspq
*q0
= &adap
->sge
.qs
[0].rspq
;
2416 struct sge_rspq
*q1
= &adap
->sge
.qs
[1].rspq
;
2418 spin_lock(&q0
->lock
);
2420 w0
= is_new_response(&q0
->desc
[q0
->cidx
], q0
);
2421 w1
= adap
->params
.nports
== 2 &&
2422 is_new_response(&q1
->desc
[q1
->cidx
], q1
);
2424 if (likely(w0
| w1
)) {
2425 t3_write_reg(adap
, A_PL_CLI
, 0);
2426 t3_read_reg(adap
, A_PL_CLI
); /* flush */
2429 process_responses_gts(adap
, q0
);
2432 process_responses_gts(adap
, q1
);
2434 work_done
= w0
| w1
;
2436 work_done
= t3_slow_intr_handler(adap
);
2438 spin_unlock(&q0
->lock
);
2439 return IRQ_RETVAL(work_done
!= 0);
2443 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2444 * Handles data events from SGE response queues as well as error and other
2445 * async events as they all use the same interrupt pin. We use one SGE
2446 * response queue per port in this mode and protect all response queues with
2449 static irqreturn_t
t3b_intr(int irq
, void *cookie
)
2452 struct adapter
*adap
= cookie
;
2453 struct sge_rspq
*q0
= &adap
->sge
.qs
[0].rspq
;
2455 t3_write_reg(adap
, A_PL_CLI
, 0);
2456 map
= t3_read_reg(adap
, A_SG_DATA_INTR
);
2458 if (unlikely(!map
)) /* shared interrupt, most likely */
2461 spin_lock(&q0
->lock
);
2463 if (unlikely(map
& F_ERRINTR
))
2464 t3_slow_intr_handler(adap
);
2466 if (likely(map
& 1))
2467 process_responses_gts(adap
, q0
);
2470 process_responses_gts(adap
, &adap
->sge
.qs
[1].rspq
);
2472 spin_unlock(&q0
->lock
);
2477 * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2478 * Handles data events from SGE response queues as well as error and other
2479 * async events as they all use the same interrupt pin. We use one SGE
2480 * response queue per port in this mode and protect all response queues with
2483 static irqreturn_t
t3b_intr_napi(int irq
, void *cookie
)
2486 struct adapter
*adap
= cookie
;
2487 struct sge_qset
*qs0
= &adap
->sge
.qs
[0];
2488 struct sge_rspq
*q0
= &qs0
->rspq
;
2490 t3_write_reg(adap
, A_PL_CLI
, 0);
2491 map
= t3_read_reg(adap
, A_SG_DATA_INTR
);
2493 if (unlikely(!map
)) /* shared interrupt, most likely */
2496 spin_lock(&q0
->lock
);
2498 if (unlikely(map
& F_ERRINTR
))
2499 t3_slow_intr_handler(adap
);
2501 if (likely(map
& 1))
2502 napi_schedule(&qs0
->napi
);
2505 napi_schedule(&adap
->sge
.qs
[1].napi
);
2507 spin_unlock(&q0
->lock
);
2512 * t3_intr_handler - select the top-level interrupt handler
2513 * @adap: the adapter
2514 * @polling: whether using NAPI to service response queues
2516 * Selects the top-level interrupt handler based on the type of interrupts
2517 * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2520 irq_handler_t
t3_intr_handler(struct adapter
*adap
, int polling
)
2522 if (adap
->flags
& USING_MSIX
)
2523 return polling
? t3_sge_intr_msix_napi
: t3_sge_intr_msix
;
2524 if (adap
->flags
& USING_MSI
)
2525 return polling
? t3_intr_msi_napi
: t3_intr_msi
;
2526 if (adap
->params
.rev
> 0)
2527 return polling
? t3b_intr_napi
: t3b_intr
;
2531 #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2532 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2533 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2534 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2536 #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2537 #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2541 * t3_sge_err_intr_handler - SGE async event interrupt handler
2542 * @adapter: the adapter
2544 * Interrupt handler for SGE asynchronous (non-data) events.
2546 void t3_sge_err_intr_handler(struct adapter
*adapter
)
2548 unsigned int v
, status
= t3_read_reg(adapter
, A_SG_INT_CAUSE
);
2550 if (status
& SGE_PARERR
)
2551 CH_ALERT(adapter
, "SGE parity error (0x%x)\n",
2552 status
& SGE_PARERR
);
2553 if (status
& SGE_FRAMINGERR
)
2554 CH_ALERT(adapter
, "SGE framing error (0x%x)\n",
2555 status
& SGE_FRAMINGERR
);
2557 if (status
& F_RSPQCREDITOVERFOW
)
2558 CH_ALERT(adapter
, "SGE response queue credit overflow\n");
2560 if (status
& F_RSPQDISABLED
) {
2561 v
= t3_read_reg(adapter
, A_SG_RSPQ_FL_STATUS
);
2564 "packet delivered to disabled response queue "
2565 "(0x%x)\n", (v
>> S_RSPQ0DISABLED
) & 0xff);
2568 if (status
& (F_HIPIODRBDROPERR
| F_LOPIODRBDROPERR
))
2569 CH_ALERT(adapter
, "SGE dropped %s priority doorbell\n",
2570 status
& F_HIPIODRBDROPERR
? "high" : "lo");
2572 t3_write_reg(adapter
, A_SG_INT_CAUSE
, status
);
2573 if (status
& SGE_FATALERR
)
2574 t3_fatal_err(adapter
);
2578 * sge_timer_cb - perform periodic maintenance of an SGE qset
2579 * @data: the SGE queue set to maintain
2581 * Runs periodically from a timer to perform maintenance of an SGE queue
2582 * set. It performs two tasks:
2584 * a) Cleans up any completed Tx descriptors that may still be pending.
2585 * Normal descriptor cleanup happens when new packets are added to a Tx
2586 * queue so this timer is relatively infrequent and does any cleanup only
2587 * if the Tx queue has not seen any new packets in a while. We make a
2588 * best effort attempt to reclaim descriptors, in that we don't wait
2589 * around if we cannot get a queue's lock (which most likely is because
2590 * someone else is queueing new packets and so will also handle the clean
2591 * up). Since control queues use immediate data exclusively we don't
2592 * bother cleaning them up here.
2594 * b) Replenishes Rx queues that have run out due to memory shortage.
2595 * Normally new Rx buffers are added when existing ones are consumed but
2596 * when out of memory a queue can become empty. We try to add only a few
2597 * buffers here, the queue will be replenished fully as these new buffers
2598 * are used up if memory shortage has subsided.
2600 static void sge_timer_cb(unsigned long data
)
2603 struct sge_qset
*qs
= (struct sge_qset
*)data
;
2604 struct adapter
*adap
= qs
->adap
;
2606 if (spin_trylock(&qs
->txq
[TXQ_ETH
].lock
)) {
2607 reclaim_completed_tx(adap
, &qs
->txq
[TXQ_ETH
]);
2608 spin_unlock(&qs
->txq
[TXQ_ETH
].lock
);
2610 if (spin_trylock(&qs
->txq
[TXQ_OFLD
].lock
)) {
2611 reclaim_completed_tx(adap
, &qs
->txq
[TXQ_OFLD
]);
2612 spin_unlock(&qs
->txq
[TXQ_OFLD
].lock
);
2614 lock
= (adap
->flags
& USING_MSIX
) ? &qs
->rspq
.lock
:
2615 &adap
->sge
.qs
[0].rspq
.lock
;
2616 if (spin_trylock_irq(lock
)) {
2617 if (!napi_is_scheduled(&qs
->napi
)) {
2618 u32 status
= t3_read_reg(adap
, A_SG_RSPQ_FL_STATUS
);
2620 if (qs
->fl
[0].credits
< qs
->fl
[0].size
)
2621 __refill_fl(adap
, &qs
->fl
[0]);
2622 if (qs
->fl
[1].credits
< qs
->fl
[1].size
)
2623 __refill_fl(adap
, &qs
->fl
[1]);
2625 if (status
& (1 << qs
->rspq
.cntxt_id
)) {
2627 if (qs
->rspq
.credits
) {
2628 refill_rspq(adap
, &qs
->rspq
, 1);
2630 qs
->rspq
.restarted
++;
2631 t3_write_reg(adap
, A_SG_RSPQ_FL_STATUS
,
2632 1 << qs
->rspq
.cntxt_id
);
2636 spin_unlock_irq(lock
);
2638 mod_timer(&qs
->tx_reclaim_timer
, jiffies
+ TX_RECLAIM_PERIOD
);
2642 * t3_update_qset_coalesce - update coalescing settings for a queue set
2643 * @qs: the SGE queue set
2644 * @p: new queue set parameters
2646 * Update the coalescing settings for an SGE queue set. Nothing is done
2647 * if the queue set is not initialized yet.
2649 void t3_update_qset_coalesce(struct sge_qset
*qs
, const struct qset_params
*p
)
2651 qs
->rspq
.holdoff_tmr
= max(p
->coalesce_usecs
* 10, 1U);/* can't be 0 */
2652 qs
->rspq
.polling
= p
->polling
;
2653 qs
->napi
.poll
= p
->polling
? napi_rx_handler
: ofld_poll
;
2657 * t3_sge_alloc_qset - initialize an SGE queue set
2658 * @adapter: the adapter
2659 * @id: the queue set id
2660 * @nports: how many Ethernet ports will be using this queue set
2661 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2662 * @p: configuration parameters for this queue set
2663 * @ntxq: number of Tx queues for the queue set
2664 * @netdev: net device associated with this queue set
2666 * Allocate resources and initialize an SGE queue set. A queue set
2667 * comprises a response queue, two Rx free-buffer queues, and up to 3
2668 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2669 * queue, offload queue, and control queue.
2671 int t3_sge_alloc_qset(struct adapter
*adapter
, unsigned int id
, int nports
,
2672 int irq_vec_idx
, const struct qset_params
*p
,
2673 int ntxq
, struct net_device
*dev
)
2675 int i
, avail
, ret
= -ENOMEM
;
2676 struct sge_qset
*q
= &adapter
->sge
.qs
[id
];
2678 init_qset_cntxt(q
, id
);
2679 init_timer(&q
->tx_reclaim_timer
);
2680 q
->tx_reclaim_timer
.data
= (unsigned long)q
;
2681 q
->tx_reclaim_timer
.function
= sge_timer_cb
;
2683 q
->fl
[0].desc
= alloc_ring(adapter
->pdev
, p
->fl_size
,
2684 sizeof(struct rx_desc
),
2685 sizeof(struct rx_sw_desc
),
2686 &q
->fl
[0].phys_addr
, &q
->fl
[0].sdesc
);
2690 q
->fl
[1].desc
= alloc_ring(adapter
->pdev
, p
->jumbo_size
,
2691 sizeof(struct rx_desc
),
2692 sizeof(struct rx_sw_desc
),
2693 &q
->fl
[1].phys_addr
, &q
->fl
[1].sdesc
);
2697 q
->rspq
.desc
= alloc_ring(adapter
->pdev
, p
->rspq_size
,
2698 sizeof(struct rsp_desc
), 0,
2699 &q
->rspq
.phys_addr
, NULL
);
2703 for (i
= 0; i
< ntxq
; ++i
) {
2705 * The control queue always uses immediate data so does not
2706 * need to keep track of any sk_buffs.
2708 size_t sz
= i
== TXQ_CTRL
? 0 : sizeof(struct tx_sw_desc
);
2710 q
->txq
[i
].desc
= alloc_ring(adapter
->pdev
, p
->txq_size
[i
],
2711 sizeof(struct tx_desc
), sz
,
2712 &q
->txq
[i
].phys_addr
,
2714 if (!q
->txq
[i
].desc
)
2718 q
->txq
[i
].size
= p
->txq_size
[i
];
2719 spin_lock_init(&q
->txq
[i
].lock
);
2720 skb_queue_head_init(&q
->txq
[i
].sendq
);
2723 tasklet_init(&q
->txq
[TXQ_OFLD
].qresume_tsk
, restart_offloadq
,
2725 tasklet_init(&q
->txq
[TXQ_CTRL
].qresume_tsk
, restart_ctrlq
,
2728 q
->fl
[0].gen
= q
->fl
[1].gen
= 1;
2729 q
->fl
[0].size
= p
->fl_size
;
2730 q
->fl
[1].size
= p
->jumbo_size
;
2733 q
->rspq
.size
= p
->rspq_size
;
2734 spin_lock_init(&q
->rspq
.lock
);
2736 q
->txq
[TXQ_ETH
].stop_thres
= nports
*
2737 flits_to_desc(sgl_len(MAX_SKB_FRAGS
+ 1) + 3);
2739 #if FL0_PG_CHUNK_SIZE > 0
2740 q
->fl
[0].buf_size
= FL0_PG_CHUNK_SIZE
;
2742 q
->fl
[0].buf_size
= SGE_RX_SM_BUF_SIZE
+ sizeof(struct cpl_rx_data
);
2744 #if FL1_PG_CHUNK_SIZE > 0
2745 q
->fl
[1].buf_size
= FL1_PG_CHUNK_SIZE
;
2747 q
->fl
[1].buf_size
= is_offload(adapter
) ?
2748 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)) :
2749 MAX_FRAME_SIZE
+ 2 + sizeof(struct cpl_rx_pkt
);
2752 q
->fl
[0].use_pages
= FL0_PG_CHUNK_SIZE
> 0;
2753 q
->fl
[1].use_pages
= FL1_PG_CHUNK_SIZE
> 0;
2754 q
->fl
[0].order
= FL0_PG_ORDER
;
2755 q
->fl
[1].order
= FL1_PG_ORDER
;
2757 spin_lock_irq(&adapter
->sge
.reg_lock
);
2759 /* FL threshold comparison uses < */
2760 ret
= t3_sge_init_rspcntxt(adapter
, q
->rspq
.cntxt_id
, irq_vec_idx
,
2761 q
->rspq
.phys_addr
, q
->rspq
.size
,
2762 q
->fl
[0].buf_size
, 1, 0);
2766 for (i
= 0; i
< SGE_RXQ_PER_SET
; ++i
) {
2767 ret
= t3_sge_init_flcntxt(adapter
, q
->fl
[i
].cntxt_id
, 0,
2768 q
->fl
[i
].phys_addr
, q
->fl
[i
].size
,
2769 q
->fl
[i
].buf_size
, p
->cong_thres
, 1,
2775 ret
= t3_sge_init_ecntxt(adapter
, q
->txq
[TXQ_ETH
].cntxt_id
, USE_GTS
,
2776 SGE_CNTXT_ETH
, id
, q
->txq
[TXQ_ETH
].phys_addr
,
2777 q
->txq
[TXQ_ETH
].size
, q
->txq
[TXQ_ETH
].token
,
2783 ret
= t3_sge_init_ecntxt(adapter
, q
->txq
[TXQ_OFLD
].cntxt_id
,
2784 USE_GTS
, SGE_CNTXT_OFLD
, id
,
2785 q
->txq
[TXQ_OFLD
].phys_addr
,
2786 q
->txq
[TXQ_OFLD
].size
, 0, 1, 0);
2792 ret
= t3_sge_init_ecntxt(adapter
, q
->txq
[TXQ_CTRL
].cntxt_id
, 0,
2794 q
->txq
[TXQ_CTRL
].phys_addr
,
2795 q
->txq
[TXQ_CTRL
].size
,
2796 q
->txq
[TXQ_CTRL
].token
, 1, 0);
2801 spin_unlock_irq(&adapter
->sge
.reg_lock
);
2805 t3_update_qset_coalesce(q
, p
);
2806 avail
= refill_fl(adapter
, &q
->fl
[0], q
->fl
[0].size
,
2807 GFP_KERNEL
| __GFP_COMP
);
2809 CH_ALERT(adapter
, "free list queue 0 initialization failed\n");
2812 if (avail
< q
->fl
[0].size
)
2813 CH_WARN(adapter
, "free list queue 0 enabled with %d credits\n",
2816 avail
= refill_fl(adapter
, &q
->fl
[1], q
->fl
[1].size
,
2817 GFP_KERNEL
| __GFP_COMP
);
2818 if (avail
< q
->fl
[1].size
)
2819 CH_WARN(adapter
, "free list queue 1 enabled with %d credits\n",
2821 refill_rspq(adapter
, &q
->rspq
, q
->rspq
.size
- 1);
2823 t3_write_reg(adapter
, A_SG_GTS
, V_RSPQ(q
->rspq
.cntxt_id
) |
2824 V_NEWTIMER(q
->rspq
.holdoff_tmr
));
2826 mod_timer(&q
->tx_reclaim_timer
, jiffies
+ TX_RECLAIM_PERIOD
);
2830 spin_unlock_irq(&adapter
->sge
.reg_lock
);
2832 t3_free_qset(adapter
, q
);
2837 * t3_free_sge_resources - free SGE resources
2838 * @adap: the adapter
2840 * Frees resources used by the SGE queue sets.
2842 void t3_free_sge_resources(struct adapter
*adap
)
2846 for (i
= 0; i
< SGE_QSETS
; ++i
)
2847 t3_free_qset(adap
, &adap
->sge
.qs
[i
]);
2851 * t3_sge_start - enable SGE
2852 * @adap: the adapter
2854 * Enables the SGE for DMAs. This is the last step in starting packet
2857 void t3_sge_start(struct adapter
*adap
)
2859 t3_set_reg_field(adap
, A_SG_CONTROL
, F_GLOBALENABLE
, F_GLOBALENABLE
);
2863 * t3_sge_stop - disable SGE operation
2864 * @adap: the adapter
2866 * Disables the DMA engine. This can be called in emeregencies (e.g.,
2867 * from error interrupts) or from normal process context. In the latter
2868 * case it also disables any pending queue restart tasklets. Note that
2869 * if it is called in interrupt context it cannot disable the restart
2870 * tasklets as it cannot wait, however the tasklets will have no effect
2871 * since the doorbells are disabled and the driver will call this again
2872 * later from process context, at which time the tasklets will be stopped
2873 * if they are still running.
2875 void t3_sge_stop(struct adapter
*adap
)
2877 t3_set_reg_field(adap
, A_SG_CONTROL
, F_GLOBALENABLE
, 0);
2878 if (!in_interrupt()) {
2881 for (i
= 0; i
< SGE_QSETS
; ++i
) {
2882 struct sge_qset
*qs
= &adap
->sge
.qs
[i
];
2884 tasklet_kill(&qs
->txq
[TXQ_OFLD
].qresume_tsk
);
2885 tasklet_kill(&qs
->txq
[TXQ_CTRL
].qresume_tsk
);
2891 * t3_sge_init - initialize SGE
2892 * @adap: the adapter
2893 * @p: the SGE parameters
2895 * Performs SGE initialization needed every time after a chip reset.
2896 * We do not initialize any of the queue sets here, instead the driver
2897 * top-level must request those individually. We also do not enable DMA
2898 * here, that should be done after the queues have been set up.
2900 void t3_sge_init(struct adapter
*adap
, struct sge_params
*p
)
2902 unsigned int ctrl
, ups
= ffs(pci_resource_len(adap
->pdev
, 2) >> 12);
2904 ctrl
= F_DROPPKT
| V_PKTSHIFT(2) | F_FLMODE
| F_AVOIDCQOVFL
|
2905 F_CQCRDTCTRL
| F_CONGMODE
| F_TNLFLMODE
| F_FATLPERREN
|
2906 V_HOSTPAGESIZE(PAGE_SHIFT
- 11) | F_BIGENDIANINGRESS
|
2907 V_USERSPACESIZE(ups
? ups
- 1 : 0) | F_ISCSICOALESCING
;
2908 #if SGE_NUM_GENBITS == 1
2909 ctrl
|= F_EGRGENCTRL
;
2911 if (adap
->params
.rev
> 0) {
2912 if (!(adap
->flags
& (USING_MSIX
| USING_MSI
)))
2913 ctrl
|= F_ONEINTMULTQ
| F_OPTONEINTMULTQ
;
2915 t3_write_reg(adap
, A_SG_CONTROL
, ctrl
);
2916 t3_write_reg(adap
, A_SG_EGR_RCQ_DRB_THRSH
, V_HIRCQDRBTHRSH(512) |
2917 V_LORCQDRBTHRSH(512));
2918 t3_write_reg(adap
, A_SG_TIMER_TICK
, core_ticks_per_usec(adap
) / 10);
2919 t3_write_reg(adap
, A_SG_CMDQ_CREDIT_TH
, V_THRESHOLD(32) |
2920 V_TIMEOUT(200 * core_ticks_per_usec(adap
)));
2921 t3_write_reg(adap
, A_SG_HI_DRB_HI_THRSH
,
2922 adap
->params
.rev
< T3_REV_C
? 1000 : 500);
2923 t3_write_reg(adap
, A_SG_HI_DRB_LO_THRSH
, 256);
2924 t3_write_reg(adap
, A_SG_LO_DRB_HI_THRSH
, 1000);
2925 t3_write_reg(adap
, A_SG_LO_DRB_LO_THRSH
, 256);
2926 t3_write_reg(adap
, A_SG_OCO_BASE
, V_BASE1(0xfff));
2927 t3_write_reg(adap
, A_SG_DRB_PRI_THRESH
, 63 * 1024);
2931 * t3_sge_prep - one-time SGE initialization
2932 * @adap: the associated adapter
2933 * @p: SGE parameters
2935 * Performs one-time initialization of SGE SW state. Includes determining
2936 * defaults for the assorted SGE parameters, which admins can change until
2937 * they are used to initialize the SGE.
2939 void t3_sge_prep(struct adapter
*adap
, struct sge_params
*p
)
2943 p
->max_pkt_size
= (16 * 1024) - sizeof(struct cpl_rx_data
) -
2944 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
2946 for (i
= 0; i
< SGE_QSETS
; ++i
) {
2947 struct qset_params
*q
= p
->qset
+ i
;
2949 q
->polling
= adap
->params
.rev
> 0;
2950 q
->coalesce_usecs
= 5;
2951 q
->rspq_size
= 1024;
2953 q
->jumbo_size
= 512;
2954 q
->txq_size
[TXQ_ETH
] = 1024;
2955 q
->txq_size
[TXQ_OFLD
] = 1024;
2956 q
->txq_size
[TXQ_CTRL
] = 256;
2960 spin_lock_init(&adap
->sge
.reg_lock
);
2964 * t3_get_desc - dump an SGE descriptor for debugging purposes
2965 * @qs: the queue set
2966 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2967 * @idx: the descriptor index in the queue
2968 * @data: where to dump the descriptor contents
2970 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
2971 * size of the descriptor.
2973 int t3_get_desc(const struct sge_qset
*qs
, unsigned int qnum
, unsigned int idx
,
2974 unsigned char *data
)
2980 if (!qs
->txq
[qnum
].desc
|| idx
>= qs
->txq
[qnum
].size
)
2982 memcpy(data
, &qs
->txq
[qnum
].desc
[idx
], sizeof(struct tx_desc
));
2983 return sizeof(struct tx_desc
);
2987 if (!qs
->rspq
.desc
|| idx
>= qs
->rspq
.size
)
2989 memcpy(data
, &qs
->rspq
.desc
[idx
], sizeof(struct rsp_desc
));
2990 return sizeof(struct rsp_desc
);
2994 if (!qs
->fl
[qnum
].desc
|| idx
>= qs
->fl
[qnum
].size
)
2996 memcpy(data
, &qs
->fl
[qnum
].desc
[idx
], sizeof(struct rx_desc
));
2997 return sizeof(struct rx_desc
);