1 /* bnx2x_cmn.c: Broadcom Everest network driver.
3 * Copyright (c) 2007-2010 Broadcom Corporation
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
19 #include <linux/etherdevice.h>
22 #include <net/ip6_checksum.h>
23 #include <linux/firmware.h>
24 #include "bnx2x_cmn.h"
27 #include <linux/if_vlan.h>
30 #include "bnx2x_init.h"
32 static int bnx2x_poll(struct napi_struct
*napi
, int budget
);
34 /* free skb in the packet ring at pos idx
35 * return idx of last bd freed
37 static u16
bnx2x_free_tx_pkt(struct bnx2x
*bp
, struct bnx2x_fastpath
*fp
,
40 struct sw_tx_bd
*tx_buf
= &fp
->tx_buf_ring
[idx
];
41 struct eth_tx_start_bd
*tx_start_bd
;
42 struct eth_tx_bd
*tx_data_bd
;
43 struct sk_buff
*skb
= tx_buf
->skb
;
44 u16 bd_idx
= TX_BD(tx_buf
->first_bd
), new_cons
;
47 /* prefetch skb end pointer to speedup dev_kfree_skb() */
50 DP(BNX2X_MSG_OFF
, "pkt_idx %d buff @(%p)->skb %p\n",
54 DP(BNX2X_MSG_OFF
, "free bd_idx %d\n", bd_idx
);
55 tx_start_bd
= &fp
->tx_desc_ring
[bd_idx
].start_bd
;
56 dma_unmap_single(&bp
->pdev
->dev
, BD_UNMAP_ADDR(tx_start_bd
),
57 BD_UNMAP_LEN(tx_start_bd
), DMA_TO_DEVICE
);
59 nbd
= le16_to_cpu(tx_start_bd
->nbd
) - 1;
60 #ifdef BNX2X_STOP_ON_ERROR
61 if ((nbd
- 1) > (MAX_SKB_FRAGS
+ 2)) {
62 BNX2X_ERR("BAD nbd!\n");
66 new_cons
= nbd
+ tx_buf
->first_bd
;
69 bd_idx
= TX_BD(NEXT_TX_IDX(bd_idx
));
71 /* Skip a parse bd... */
73 bd_idx
= TX_BD(NEXT_TX_IDX(bd_idx
));
75 /* ...and the TSO split header bd since they have no mapping */
76 if (tx_buf
->flags
& BNX2X_TSO_SPLIT_BD
) {
78 bd_idx
= TX_BD(NEXT_TX_IDX(bd_idx
));
84 DP(BNX2X_MSG_OFF
, "free frag bd_idx %d\n", bd_idx
);
85 tx_data_bd
= &fp
->tx_desc_ring
[bd_idx
].reg_bd
;
86 dma_unmap_page(&bp
->pdev
->dev
, BD_UNMAP_ADDR(tx_data_bd
),
87 BD_UNMAP_LEN(tx_data_bd
), DMA_TO_DEVICE
);
89 bd_idx
= TX_BD(NEXT_TX_IDX(bd_idx
));
101 int bnx2x_tx_int(struct bnx2x_fastpath
*fp
)
103 struct bnx2x
*bp
= fp
->bp
;
104 struct netdev_queue
*txq
;
105 u16 hw_cons
, sw_cons
, bd_cons
= fp
->tx_bd_cons
;
107 #ifdef BNX2X_STOP_ON_ERROR
108 if (unlikely(bp
->panic
))
112 txq
= netdev_get_tx_queue(bp
->dev
, fp
->index
);
113 hw_cons
= le16_to_cpu(*fp
->tx_cons_sb
);
114 sw_cons
= fp
->tx_pkt_cons
;
116 while (sw_cons
!= hw_cons
) {
119 pkt_cons
= TX_BD(sw_cons
);
121 DP(NETIF_MSG_TX_DONE
, "queue[%d]: hw_cons %u sw_cons %u "
123 fp
->index
, hw_cons
, sw_cons
, pkt_cons
);
125 bd_cons
= bnx2x_free_tx_pkt(bp
, fp
, pkt_cons
);
129 fp
->tx_pkt_cons
= sw_cons
;
130 fp
->tx_bd_cons
= bd_cons
;
132 /* Need to make the tx_bd_cons update visible to start_xmit()
133 * before checking for netif_tx_queue_stopped(). Without the
134 * memory barrier, there is a small possibility that
135 * start_xmit() will miss it and cause the queue to be stopped
140 /* TBD need a thresh? */
141 if (unlikely(netif_tx_queue_stopped(txq
))) {
142 /* Taking tx_lock() is needed to prevent reenabling the queue
143 * while it's empty. This could have happen if rx_action() gets
144 * suspended in bnx2x_tx_int() after the condition before
145 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
147 * stops the queue->sees fresh tx_bd_cons->releases the queue->
148 * sends some packets consuming the whole queue again->
152 __netif_tx_lock(txq
, smp_processor_id());
154 if ((netif_tx_queue_stopped(txq
)) &&
155 (bp
->state
== BNX2X_STATE_OPEN
) &&
156 (bnx2x_tx_avail(fp
) >= MAX_SKB_FRAGS
+ 3))
157 netif_tx_wake_queue(txq
);
159 __netif_tx_unlock(txq
);
164 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath
*fp
,
167 u16 last_max
= fp
->last_max_sge
;
169 if (SUB_S16(idx
, last_max
) > 0)
170 fp
->last_max_sge
= idx
;
173 static void bnx2x_update_sge_prod(struct bnx2x_fastpath
*fp
,
174 struct eth_fast_path_rx_cqe
*fp_cqe
)
176 struct bnx2x
*bp
= fp
->bp
;
177 u16 sge_len
= SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe
->pkt_len
) -
178 le16_to_cpu(fp_cqe
->len_on_bd
)) >>
180 u16 last_max
, last_elem
, first_elem
;
187 /* First mark all used pages */
188 for (i
= 0; i
< sge_len
; i
++)
189 SGE_MASK_CLEAR_BIT(fp
,
190 RX_SGE(le16_to_cpu(fp_cqe
->sgl_or_raw_data
.sgl
[i
])));
192 DP(NETIF_MSG_RX_STATUS
, "fp_cqe->sgl[%d] = %d\n",
193 sge_len
- 1, le16_to_cpu(fp_cqe
->sgl_or_raw_data
.sgl
[sge_len
- 1]));
195 /* Here we assume that the last SGE index is the biggest */
196 prefetch((void *)(fp
->sge_mask
));
197 bnx2x_update_last_max_sge(fp
,
198 le16_to_cpu(fp_cqe
->sgl_or_raw_data
.sgl
[sge_len
- 1]));
200 last_max
= RX_SGE(fp
->last_max_sge
);
201 last_elem
= last_max
>> RX_SGE_MASK_ELEM_SHIFT
;
202 first_elem
= RX_SGE(fp
->rx_sge_prod
) >> RX_SGE_MASK_ELEM_SHIFT
;
204 /* If ring is not full */
205 if (last_elem
+ 1 != first_elem
)
208 /* Now update the prod */
209 for (i
= first_elem
; i
!= last_elem
; i
= NEXT_SGE_MASK_ELEM(i
)) {
210 if (likely(fp
->sge_mask
[i
]))
213 fp
->sge_mask
[i
] = RX_SGE_MASK_ELEM_ONE_MASK
;
214 delta
+= RX_SGE_MASK_ELEM_SZ
;
218 fp
->rx_sge_prod
+= delta
;
219 /* clear page-end entries */
220 bnx2x_clear_sge_mask_next_elems(fp
);
223 DP(NETIF_MSG_RX_STATUS
,
224 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
225 fp
->last_max_sge
, fp
->rx_sge_prod
);
228 static void bnx2x_tpa_start(struct bnx2x_fastpath
*fp
, u16 queue
,
229 struct sk_buff
*skb
, u16 cons
, u16 prod
)
231 struct bnx2x
*bp
= fp
->bp
;
232 struct sw_rx_bd
*cons_rx_buf
= &fp
->rx_buf_ring
[cons
];
233 struct sw_rx_bd
*prod_rx_buf
= &fp
->rx_buf_ring
[prod
];
234 struct eth_rx_bd
*prod_bd
= &fp
->rx_desc_ring
[prod
];
237 /* move empty skb from pool to prod and map it */
238 prod_rx_buf
->skb
= fp
->tpa_pool
[queue
].skb
;
239 mapping
= dma_map_single(&bp
->pdev
->dev
, fp
->tpa_pool
[queue
].skb
->data
,
240 bp
->rx_buf_size
, DMA_FROM_DEVICE
);
241 dma_unmap_addr_set(prod_rx_buf
, mapping
, mapping
);
243 /* move partial skb from cons to pool (don't unmap yet) */
244 fp
->tpa_pool
[queue
] = *cons_rx_buf
;
246 /* mark bin state as start - print error if current state != stop */
247 if (fp
->tpa_state
[queue
] != BNX2X_TPA_STOP
)
248 BNX2X_ERR("start of bin not in stop [%d]\n", queue
);
250 fp
->tpa_state
[queue
] = BNX2X_TPA_START
;
252 /* point prod_bd to new skb */
253 prod_bd
->addr_hi
= cpu_to_le32(U64_HI(mapping
));
254 prod_bd
->addr_lo
= cpu_to_le32(U64_LO(mapping
));
256 #ifdef BNX2X_STOP_ON_ERROR
257 fp
->tpa_queue_used
|= (1 << queue
);
258 #ifdef _ASM_GENERIC_INT_L64_H
259 DP(NETIF_MSG_RX_STATUS
, "fp->tpa_queue_used = 0x%lx\n",
261 DP(NETIF_MSG_RX_STATUS
, "fp->tpa_queue_used = 0x%llx\n",
267 static int bnx2x_fill_frag_skb(struct bnx2x
*bp
, struct bnx2x_fastpath
*fp
,
269 struct eth_fast_path_rx_cqe
*fp_cqe
,
272 struct sw_rx_page
*rx_pg
, old_rx_pg
;
273 u16 len_on_bd
= le16_to_cpu(fp_cqe
->len_on_bd
);
274 u32 i
, frag_len
, frag_size
, pages
;
278 frag_size
= le16_to_cpu(fp_cqe
->pkt_len
) - len_on_bd
;
279 pages
= SGE_PAGE_ALIGN(frag_size
) >> SGE_PAGE_SHIFT
;
281 /* This is needed in order to enable forwarding support */
283 skb_shinfo(skb
)->gso_size
= min((u32
)SGE_PAGE_SIZE
,
284 max(frag_size
, (u32
)len_on_bd
));
286 #ifdef BNX2X_STOP_ON_ERROR
287 if (pages
> min_t(u32
, 8, MAX_SKB_FRAGS
)*SGE_PAGE_SIZE
*PAGES_PER_SGE
) {
288 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
290 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n",
291 fp_cqe
->pkt_len
, len_on_bd
);
297 /* Run through the SGL and compose the fragmented skb */
298 for (i
= 0, j
= 0; i
< pages
; i
+= PAGES_PER_SGE
, j
++) {
300 RX_SGE(le16_to_cpu(fp_cqe
->sgl_or_raw_data
.sgl
[j
]));
302 /* FW gives the indices of the SGE as if the ring is an array
303 (meaning that "next" element will consume 2 indices) */
304 frag_len
= min(frag_size
, (u32
)(SGE_PAGE_SIZE
*PAGES_PER_SGE
));
305 rx_pg
= &fp
->rx_page_ring
[sge_idx
];
308 /* If we fail to allocate a substitute page, we simply stop
309 where we are and drop the whole packet */
310 err
= bnx2x_alloc_rx_sge(bp
, fp
, sge_idx
);
312 fp
->eth_q_stats
.rx_skb_alloc_failed
++;
316 /* Unmap the page as we r going to pass it to the stack */
317 dma_unmap_page(&bp
->pdev
->dev
,
318 dma_unmap_addr(&old_rx_pg
, mapping
),
319 SGE_PAGE_SIZE
*PAGES_PER_SGE
, DMA_FROM_DEVICE
);
321 /* Add one frag and update the appropriate fields in the skb */
322 skb_fill_page_desc(skb
, j
, old_rx_pg
.page
, 0, frag_len
);
324 skb
->data_len
+= frag_len
;
325 skb
->truesize
+= frag_len
;
326 skb
->len
+= frag_len
;
328 frag_size
-= frag_len
;
334 static void bnx2x_tpa_stop(struct bnx2x
*bp
, struct bnx2x_fastpath
*fp
,
335 u16 queue
, int pad
, int len
, union eth_rx_cqe
*cqe
,
338 struct sw_rx_bd
*rx_buf
= &fp
->tpa_pool
[queue
];
339 struct sk_buff
*skb
= rx_buf
->skb
;
341 struct sk_buff
*new_skb
= netdev_alloc_skb(bp
->dev
, bp
->rx_buf_size
);
343 /* Unmap skb in the pool anyway, as we are going to change
344 pool entry status to BNX2X_TPA_STOP even if new skb allocation
346 dma_unmap_single(&bp
->pdev
->dev
, dma_unmap_addr(rx_buf
, mapping
),
347 bp
->rx_buf_size
, DMA_FROM_DEVICE
);
349 if (likely(new_skb
)) {
350 /* fix ip xsum and give it to the stack */
351 /* (no need to map the new skb) */
354 (le16_to_cpu(cqe
->fast_path_cqe
.pars_flags
.flags
) &
356 int is_not_hwaccel_vlan_cqe
=
357 (is_vlan_cqe
&& (!(bp
->flags
& HW_VLAN_RX_FLAG
)));
361 prefetch(((char *)(skb
)) + 128);
363 #ifdef BNX2X_STOP_ON_ERROR
364 if (pad
+ len
> bp
->rx_buf_size
) {
365 BNX2X_ERR("skb_put is about to fail... "
366 "pad %d len %d rx_buf_size %d\n",
367 pad
, len
, bp
->rx_buf_size
);
373 skb_reserve(skb
, pad
);
376 skb
->protocol
= eth_type_trans(skb
, bp
->dev
);
377 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
382 iph
= (struct iphdr
*)skb
->data
;
384 /* If there is no Rx VLAN offloading -
385 take VLAN tag into an account */
386 if (unlikely(is_not_hwaccel_vlan_cqe
))
387 iph
= (struct iphdr
*)((u8
*)iph
+ VLAN_HLEN
);
390 iph
->check
= ip_fast_csum((u8
*)iph
, iph
->ihl
);
393 if (!bnx2x_fill_frag_skb(bp
, fp
, skb
,
394 &cqe
->fast_path_cqe
, cqe_idx
)) {
396 if ((bp
->vlgrp
!= NULL
) &&
397 (le16_to_cpu(cqe
->fast_path_cqe
.
398 pars_flags
.flags
) & PARSING_FLAGS_VLAN
))
399 vlan_gro_receive(&fp
->napi
, bp
->vlgrp
,
400 le16_to_cpu(cqe
->fast_path_cqe
.
404 napi_gro_receive(&fp
->napi
, skb
);
406 DP(NETIF_MSG_RX_STATUS
, "Failed to allocate new pages"
407 " - dropping packet!\n");
412 /* put new skb in bin */
413 fp
->tpa_pool
[queue
].skb
= new_skb
;
416 /* else drop the packet and keep the buffer in the bin */
417 DP(NETIF_MSG_RX_STATUS
,
418 "Failed to allocate new skb - dropping packet!\n");
419 fp
->eth_q_stats
.rx_skb_alloc_failed
++;
422 fp
->tpa_state
[queue
] = BNX2X_TPA_STOP
;
425 /* Set Toeplitz hash value in the skb using the value from the
426 * CQE (calculated by HW).
428 static inline void bnx2x_set_skb_rxhash(struct bnx2x
*bp
, union eth_rx_cqe
*cqe
,
431 /* Set Toeplitz hash from CQE */
432 if ((bp
->dev
->features
& NETIF_F_RXHASH
) &&
433 (cqe
->fast_path_cqe
.status_flags
&
434 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG
))
436 le32_to_cpu(cqe
->fast_path_cqe
.rss_hash_result
);
439 int bnx2x_rx_int(struct bnx2x_fastpath
*fp
, int budget
)
441 struct bnx2x
*bp
= fp
->bp
;
442 u16 bd_cons
, bd_prod
, bd_prod_fw
, comp_ring_cons
;
443 u16 hw_comp_cons
, sw_comp_cons
, sw_comp_prod
;
446 #ifdef BNX2X_STOP_ON_ERROR
447 if (unlikely(bp
->panic
))
451 /* CQ "next element" is of the size of the regular element,
452 that's why it's ok here */
453 hw_comp_cons
= le16_to_cpu(*fp
->rx_cons_sb
);
454 if ((hw_comp_cons
& MAX_RCQ_DESC_CNT
) == MAX_RCQ_DESC_CNT
)
457 bd_cons
= fp
->rx_bd_cons
;
458 bd_prod
= fp
->rx_bd_prod
;
459 bd_prod_fw
= bd_prod
;
460 sw_comp_cons
= fp
->rx_comp_cons
;
461 sw_comp_prod
= fp
->rx_comp_prod
;
463 /* Memory barrier necessary as speculative reads of the rx
464 * buffer can be ahead of the index in the status block
468 DP(NETIF_MSG_RX_STATUS
,
469 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
470 fp
->index
, hw_comp_cons
, sw_comp_cons
);
472 while (sw_comp_cons
!= hw_comp_cons
) {
473 struct sw_rx_bd
*rx_buf
= NULL
;
475 union eth_rx_cqe
*cqe
;
479 comp_ring_cons
= RCQ_BD(sw_comp_cons
);
480 bd_prod
= RX_BD(bd_prod
);
481 bd_cons
= RX_BD(bd_cons
);
483 /* Prefetch the page containing the BD descriptor
484 at producer's index. It will be needed when new skb is
486 prefetch((void *)(PAGE_ALIGN((unsigned long)
487 (&fp
->rx_desc_ring
[bd_prod
])) -
490 cqe
= &fp
->rx_comp_ring
[comp_ring_cons
];
491 cqe_fp_flags
= cqe
->fast_path_cqe
.type_error_flags
;
493 DP(NETIF_MSG_RX_STATUS
, "CQE type %x err %x status %x"
494 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags
),
495 cqe_fp_flags
, cqe
->fast_path_cqe
.status_flags
,
496 le32_to_cpu(cqe
->fast_path_cqe
.rss_hash_result
),
497 le16_to_cpu(cqe
->fast_path_cqe
.vlan_tag
),
498 le16_to_cpu(cqe
->fast_path_cqe
.pkt_len
));
500 /* is this a slowpath msg? */
501 if (unlikely(CQE_TYPE(cqe_fp_flags
))) {
502 bnx2x_sp_event(fp
, cqe
);
505 /* this is an rx packet */
507 rx_buf
= &fp
->rx_buf_ring
[bd_cons
];
510 len
= le16_to_cpu(cqe
->fast_path_cqe
.pkt_len
);
511 pad
= cqe
->fast_path_cqe
.placement_offset
;
513 /* If CQE is marked both TPA_START and TPA_END
514 it is a non-TPA CQE */
515 if ((!fp
->disable_tpa
) &&
516 (TPA_TYPE(cqe_fp_flags
) !=
517 (TPA_TYPE_START
| TPA_TYPE_END
))) {
518 u16 queue
= cqe
->fast_path_cqe
.queue_index
;
520 if (TPA_TYPE(cqe_fp_flags
) == TPA_TYPE_START
) {
521 DP(NETIF_MSG_RX_STATUS
,
522 "calling tpa_start on queue %d\n",
525 bnx2x_tpa_start(fp
, queue
, skb
,
528 /* Set Toeplitz hash for an LRO skb */
529 bnx2x_set_skb_rxhash(bp
, cqe
, skb
);
534 if (TPA_TYPE(cqe_fp_flags
) == TPA_TYPE_END
) {
535 DP(NETIF_MSG_RX_STATUS
,
536 "calling tpa_stop on queue %d\n",
539 if (!BNX2X_RX_SUM_FIX(cqe
))
540 BNX2X_ERR("STOP on none TCP "
543 /* This is a size of the linear data
545 len
= le16_to_cpu(cqe
->fast_path_cqe
.
547 bnx2x_tpa_stop(bp
, fp
, queue
, pad
,
548 len
, cqe
, comp_ring_cons
);
549 #ifdef BNX2X_STOP_ON_ERROR
554 bnx2x_update_sge_prod(fp
,
555 &cqe
->fast_path_cqe
);
560 dma_sync_single_for_device(&bp
->pdev
->dev
,
561 dma_unmap_addr(rx_buf
, mapping
),
562 pad
+ RX_COPY_THRESH
,
564 prefetch(((char *)(skb
)) + 128);
566 /* is this an error packet? */
567 if (unlikely(cqe_fp_flags
& ETH_RX_ERROR_FALGS
)) {
569 "ERROR flags %x rx packet %u\n",
570 cqe_fp_flags
, sw_comp_cons
);
571 fp
->eth_q_stats
.rx_err_discard_pkt
++;
575 /* Since we don't have a jumbo ring
576 * copy small packets if mtu > 1500
578 if ((bp
->dev
->mtu
> ETH_MAX_PACKET_SIZE
) &&
579 (len
<= RX_COPY_THRESH
)) {
580 struct sk_buff
*new_skb
;
582 new_skb
= netdev_alloc_skb(bp
->dev
,
584 if (new_skb
== NULL
) {
586 "ERROR packet dropped "
587 "because of alloc failure\n");
588 fp
->eth_q_stats
.rx_skb_alloc_failed
++;
593 skb_copy_from_linear_data_offset(skb
, pad
,
594 new_skb
->data
+ pad
, len
);
595 skb_reserve(new_skb
, pad
);
596 skb_put(new_skb
, len
);
598 bnx2x_reuse_rx_skb(fp
, bd_cons
, bd_prod
);
603 if (likely(bnx2x_alloc_rx_skb(bp
, fp
, bd_prod
) == 0)) {
604 dma_unmap_single(&bp
->pdev
->dev
,
605 dma_unmap_addr(rx_buf
, mapping
),
608 skb_reserve(skb
, pad
);
613 "ERROR packet dropped because "
614 "of alloc failure\n");
615 fp
->eth_q_stats
.rx_skb_alloc_failed
++;
617 bnx2x_reuse_rx_skb(fp
, bd_cons
, bd_prod
);
621 skb
->protocol
= eth_type_trans(skb
, bp
->dev
);
623 /* Set Toeplitz hash for a none-LRO skb */
624 bnx2x_set_skb_rxhash(bp
, cqe
, skb
);
626 skb_checksum_none_assert(skb
);
628 if (likely(BNX2X_RX_CSUM_OK(cqe
)))
629 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
631 fp
->eth_q_stats
.hw_csum_err
++;
635 skb_record_rx_queue(skb
, fp
->index
);
638 if ((bp
->vlgrp
!= NULL
) && (bp
->flags
& HW_VLAN_RX_FLAG
) &&
639 (le16_to_cpu(cqe
->fast_path_cqe
.pars_flags
.flags
) &
641 vlan_gro_receive(&fp
->napi
, bp
->vlgrp
,
642 le16_to_cpu(cqe
->fast_path_cqe
.vlan_tag
), skb
);
645 napi_gro_receive(&fp
->napi
, skb
);
651 bd_cons
= NEXT_RX_IDX(bd_cons
);
652 bd_prod
= NEXT_RX_IDX(bd_prod
);
653 bd_prod_fw
= NEXT_RX_IDX(bd_prod_fw
);
656 sw_comp_prod
= NEXT_RCQ_IDX(sw_comp_prod
);
657 sw_comp_cons
= NEXT_RCQ_IDX(sw_comp_cons
);
659 if (rx_pkt
== budget
)
663 fp
->rx_bd_cons
= bd_cons
;
664 fp
->rx_bd_prod
= bd_prod_fw
;
665 fp
->rx_comp_cons
= sw_comp_cons
;
666 fp
->rx_comp_prod
= sw_comp_prod
;
668 /* Update producers */
669 bnx2x_update_rx_prod(bp
, fp
, bd_prod_fw
, sw_comp_prod
,
672 fp
->rx_pkt
+= rx_pkt
;
678 static irqreturn_t
bnx2x_msix_fp_int(int irq
, void *fp_cookie
)
680 struct bnx2x_fastpath
*fp
= fp_cookie
;
681 struct bnx2x
*bp
= fp
->bp
;
683 /* Return here if interrupt is disabled */
684 if (unlikely(atomic_read(&bp
->intr_sem
) != 0)) {
685 DP(NETIF_MSG_INTR
, "called but intr_sem not 0, returning\n");
689 DP(BNX2X_MSG_FP
, "got an MSI-X interrupt on IDX:SB "
690 "[fp %d fw_sd %d igusb %d]\n",
691 fp
->index
, fp
->fw_sb_id
, fp
->igu_sb_id
);
692 bnx2x_ack_sb(bp
, fp
->igu_sb_id
, USTORM_ID
, 0, IGU_INT_DISABLE
, 0);
694 #ifdef BNX2X_STOP_ON_ERROR
695 if (unlikely(bp
->panic
))
699 /* Handle Rx and Tx according to MSI-X vector */
700 prefetch(fp
->rx_cons_sb
);
701 prefetch(fp
->tx_cons_sb
);
702 prefetch(&fp
->sb_running_index
[SM_RX_ID
]);
703 napi_schedule(&bnx2x_fp(bp
, fp
->index
, napi
));
709 /* HW Lock for shared dual port PHYs */
710 void bnx2x_acquire_phy_lock(struct bnx2x
*bp
)
712 mutex_lock(&bp
->port
.phy_mutex
);
714 if (bp
->port
.need_hw_lock
)
715 bnx2x_acquire_hw_lock(bp
, HW_LOCK_RESOURCE_MDIO
);
718 void bnx2x_release_phy_lock(struct bnx2x
*bp
)
720 if (bp
->port
.need_hw_lock
)
721 bnx2x_release_hw_lock(bp
, HW_LOCK_RESOURCE_MDIO
);
723 mutex_unlock(&bp
->port
.phy_mutex
);
726 void bnx2x_link_report(struct bnx2x
*bp
)
728 if (bp
->flags
& MF_FUNC_DIS
) {
729 netif_carrier_off(bp
->dev
);
730 netdev_err(bp
->dev
, "NIC Link is Down\n");
734 if (bp
->link_vars
.link_up
) {
737 if (bp
->state
== BNX2X_STATE_OPEN
)
738 netif_carrier_on(bp
->dev
);
739 netdev_info(bp
->dev
, "NIC Link is Up, ");
741 line_speed
= bp
->link_vars
.line_speed
;
746 ((bp
->mf_config
[BP_VN(bp
)] &
747 FUNC_MF_CFG_MAX_BW_MASK
) >>
748 FUNC_MF_CFG_MAX_BW_SHIFT
) * 100;
749 if (vn_max_rate
< line_speed
)
750 line_speed
= vn_max_rate
;
752 pr_cont("%d Mbps ", line_speed
);
754 if (bp
->link_vars
.duplex
== DUPLEX_FULL
)
755 pr_cont("full duplex");
757 pr_cont("half duplex");
759 if (bp
->link_vars
.flow_ctrl
!= BNX2X_FLOW_CTRL_NONE
) {
760 if (bp
->link_vars
.flow_ctrl
& BNX2X_FLOW_CTRL_RX
) {
761 pr_cont(", receive ");
762 if (bp
->link_vars
.flow_ctrl
&
764 pr_cont("& transmit ");
766 pr_cont(", transmit ");
768 pr_cont("flow control ON");
772 } else { /* link_down */
773 netif_carrier_off(bp
->dev
);
774 netdev_err(bp
->dev
, "NIC Link is Down\n");
778 /* Returns the number of actually allocated BDs */
779 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath
*fp
,
782 struct bnx2x
*bp
= fp
->bp
;
783 u16 ring_prod
, cqe_ring_prod
;
786 fp
->rx_comp_cons
= 0;
787 cqe_ring_prod
= ring_prod
= 0;
788 for (i
= 0; i
< rx_ring_size
; i
++) {
789 if (bnx2x_alloc_rx_skb(bp
, fp
, ring_prod
) < 0) {
790 BNX2X_ERR("was only able to allocate "
791 "%d rx skbs on queue[%d]\n", i
, fp
->index
);
792 fp
->eth_q_stats
.rx_skb_alloc_failed
++;
795 ring_prod
= NEXT_RX_IDX(ring_prod
);
796 cqe_ring_prod
= NEXT_RCQ_IDX(cqe_ring_prod
);
797 WARN_ON(ring_prod
<= i
);
800 fp
->rx_bd_prod
= ring_prod
;
801 /* Limit the CQE producer by the CQE ring size */
802 fp
->rx_comp_prod
= min_t(u16
, NUM_RCQ_RINGS
*RCQ_DESC_CNT
,
804 fp
->rx_pkt
= fp
->rx_calls
= 0;
809 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath
*fp
)
811 struct bnx2x
*bp
= fp
->bp
;
812 int rx_ring_size
= bp
->rx_ring_size
? bp
->rx_ring_size
:
813 MAX_RX_AVAIL
/bp
->num_queues
;
815 rx_ring_size
= max_t(int, MIN_RX_AVAIL
, rx_ring_size
);
817 bnx2x_alloc_rx_bds(fp
, rx_ring_size
);
820 * this will generate an interrupt (to the TSTORM)
821 * must only be done after chip is initialized
823 bnx2x_update_rx_prod(bp
, fp
, fp
->rx_bd_prod
, fp
->rx_comp_prod
,
827 void bnx2x_init_rx_rings(struct bnx2x
*bp
)
829 int func
= BP_FUNC(bp
);
830 int max_agg_queues
= CHIP_IS_E1(bp
) ? ETH_MAX_AGGREGATION_QUEUES_E1
:
831 ETH_MAX_AGGREGATION_QUEUES_E1H
;
835 bp
->rx_buf_size
= bp
->dev
->mtu
+ ETH_OVREHEAD
+ BNX2X_RX_ALIGN
+
836 BNX2X_FW_IP_HDR_ALIGN_PAD
;
839 "mtu %d rx_buf_size %d\n", bp
->dev
->mtu
, bp
->rx_buf_size
);
841 for_each_queue(bp
, j
) {
842 struct bnx2x_fastpath
*fp
= &bp
->fp
[j
];
844 if (!fp
->disable_tpa
) {
845 for (i
= 0; i
< max_agg_queues
; i
++) {
846 fp
->tpa_pool
[i
].skb
=
847 netdev_alloc_skb(bp
->dev
, bp
->rx_buf_size
);
848 if (!fp
->tpa_pool
[i
].skb
) {
849 BNX2X_ERR("Failed to allocate TPA "
850 "skb pool for queue[%d] - "
851 "disabling TPA on this "
853 bnx2x_free_tpa_pool(bp
, fp
, i
);
857 dma_unmap_addr_set((struct sw_rx_bd
*)
858 &bp
->fp
->tpa_pool
[i
],
860 fp
->tpa_state
[i
] = BNX2X_TPA_STOP
;
863 /* "next page" elements initialization */
864 bnx2x_set_next_page_sgl(fp
);
866 /* set SGEs bit mask */
867 bnx2x_init_sge_ring_bit_mask(fp
);
869 /* Allocate SGEs and initialize the ring elements */
870 for (i
= 0, ring_prod
= 0;
871 i
< MAX_RX_SGE_CNT
*NUM_RX_SGE_PAGES
; i
++) {
873 if (bnx2x_alloc_rx_sge(bp
, fp
, ring_prod
) < 0) {
874 BNX2X_ERR("was only able to allocate "
876 BNX2X_ERR("disabling TPA for"
878 /* Cleanup already allocated elements */
879 bnx2x_free_rx_sge_range(bp
,
881 bnx2x_free_tpa_pool(bp
,
887 ring_prod
= NEXT_SGE_IDX(ring_prod
);
890 fp
->rx_sge_prod
= ring_prod
;
894 for_each_queue(bp
, j
) {
895 struct bnx2x_fastpath
*fp
= &bp
->fp
[j
];
899 bnx2x_set_next_page_rx_bd(fp
);
902 bnx2x_set_next_page_rx_cq(fp
);
904 /* Allocate BDs and initialize BD ring */
905 bnx2x_alloc_rx_bd_ring(fp
);
910 if (!CHIP_IS_E2(bp
)) {
911 REG_WR(bp
, BAR_USTRORM_INTMEM
+
912 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func
),
913 U64_LO(fp
->rx_comp_mapping
));
914 REG_WR(bp
, BAR_USTRORM_INTMEM
+
915 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func
) + 4,
916 U64_HI(fp
->rx_comp_mapping
));
920 static void bnx2x_free_tx_skbs(struct bnx2x
*bp
)
924 for_each_queue(bp
, i
) {
925 struct bnx2x_fastpath
*fp
= &bp
->fp
[i
];
927 u16 bd_cons
= fp
->tx_bd_cons
;
928 u16 sw_prod
= fp
->tx_pkt_prod
;
929 u16 sw_cons
= fp
->tx_pkt_cons
;
931 while (sw_cons
!= sw_prod
) {
932 bd_cons
= bnx2x_free_tx_pkt(bp
, fp
, TX_BD(sw_cons
));
938 static void bnx2x_free_rx_skbs(struct bnx2x
*bp
)
942 for_each_queue(bp
, j
) {
943 struct bnx2x_fastpath
*fp
= &bp
->fp
[j
];
945 for (i
= 0; i
< NUM_RX_BD
; i
++) {
946 struct sw_rx_bd
*rx_buf
= &fp
->rx_buf_ring
[i
];
947 struct sk_buff
*skb
= rx_buf
->skb
;
952 dma_unmap_single(&bp
->pdev
->dev
,
953 dma_unmap_addr(rx_buf
, mapping
),
954 bp
->rx_buf_size
, DMA_FROM_DEVICE
);
959 if (!fp
->disable_tpa
)
960 bnx2x_free_tpa_pool(bp
, fp
, CHIP_IS_E1(bp
) ?
961 ETH_MAX_AGGREGATION_QUEUES_E1
:
962 ETH_MAX_AGGREGATION_QUEUES_E1H
);
966 void bnx2x_free_skbs(struct bnx2x
*bp
)
968 bnx2x_free_tx_skbs(bp
);
969 bnx2x_free_rx_skbs(bp
);
972 static void bnx2x_free_msix_irqs(struct bnx2x
*bp
)
976 free_irq(bp
->msix_table
[0].vector
, bp
->dev
);
977 DP(NETIF_MSG_IFDOWN
, "released sp irq (%d)\n",
978 bp
->msix_table
[0].vector
);
983 for_each_queue(bp
, i
) {
984 DP(NETIF_MSG_IFDOWN
, "about to release fp #%d->%d irq "
985 "state %x\n", i
, bp
->msix_table
[i
+ offset
].vector
,
986 bnx2x_fp(bp
, i
, state
));
988 free_irq(bp
->msix_table
[i
+ offset
].vector
, &bp
->fp
[i
]);
992 void bnx2x_free_irq(struct bnx2x
*bp
, bool disable_only
)
994 if (bp
->flags
& USING_MSIX_FLAG
) {
996 bnx2x_free_msix_irqs(bp
);
997 pci_disable_msix(bp
->pdev
);
998 bp
->flags
&= ~USING_MSIX_FLAG
;
1000 } else if (bp
->flags
& USING_MSI_FLAG
) {
1002 free_irq(bp
->pdev
->irq
, bp
->dev
);
1003 pci_disable_msi(bp
->pdev
);
1004 bp
->flags
&= ~USING_MSI_FLAG
;
1006 } else if (!disable_only
)
1007 free_irq(bp
->pdev
->irq
, bp
->dev
);
1010 static int bnx2x_enable_msix(struct bnx2x
*bp
)
1012 int i
, rc
, offset
= 1;
1015 bp
->msix_table
[0].entry
= igu_vec
;
1016 DP(NETIF_MSG_IFUP
, "msix_table[0].entry = %d (slowpath)\n", igu_vec
);
1019 igu_vec
= BP_L_ID(bp
) + offset
;
1020 bp
->msix_table
[1].entry
= igu_vec
;
1021 DP(NETIF_MSG_IFUP
, "msix_table[1].entry = %d (CNIC)\n", igu_vec
);
1024 for_each_queue(bp
, i
) {
1025 igu_vec
= BP_L_ID(bp
) + offset
+ i
;
1026 bp
->msix_table
[i
+ offset
].entry
= igu_vec
;
1027 DP(NETIF_MSG_IFUP
, "msix_table[%d].entry = %d "
1028 "(fastpath #%u)\n", i
+ offset
, igu_vec
, i
);
1031 rc
= pci_enable_msix(bp
->pdev
, &bp
->msix_table
[0],
1032 BNX2X_NUM_QUEUES(bp
) + offset
);
1035 * reconfigure number of tx/rx queues according to available
1038 if (rc
>= BNX2X_MIN_MSIX_VEC_CNT
) {
1039 /* vectors available for FP */
1040 int fp_vec
= rc
- BNX2X_MSIX_VEC_FP_START
;
1043 "Trying to use less MSI-X vectors: %d\n", rc
);
1045 rc
= pci_enable_msix(bp
->pdev
, &bp
->msix_table
[0], rc
);
1049 "MSI-X is not attainable rc %d\n", rc
);
1053 bp
->num_queues
= min(bp
->num_queues
, fp_vec
);
1055 DP(NETIF_MSG_IFUP
, "New queue configuration set: %d\n",
1058 DP(NETIF_MSG_IFUP
, "MSI-X is not attainable rc %d\n", rc
);
1062 bp
->flags
|= USING_MSIX_FLAG
;
1067 static int bnx2x_req_msix_irqs(struct bnx2x
*bp
)
1069 int i
, rc
, offset
= 1;
1071 rc
= request_irq(bp
->msix_table
[0].vector
, bnx2x_msix_sp_int
, 0,
1072 bp
->dev
->name
, bp
->dev
);
1074 BNX2X_ERR("request sp irq failed\n");
1081 for_each_queue(bp
, i
) {
1082 struct bnx2x_fastpath
*fp
= &bp
->fp
[i
];
1083 snprintf(fp
->name
, sizeof(fp
->name
), "%s-fp-%d",
1086 rc
= request_irq(bp
->msix_table
[i
+ offset
].vector
,
1087 bnx2x_msix_fp_int
, 0, fp
->name
, fp
);
1089 BNX2X_ERR("request fp #%d irq failed rc %d\n", i
, rc
);
1090 bnx2x_free_msix_irqs(bp
);
1094 fp
->state
= BNX2X_FP_STATE_IRQ
;
1097 i
= BNX2X_NUM_QUEUES(bp
);
1098 netdev_info(bp
->dev
, "using MSI-X IRQs: sp %d fp[%d] %d"
1100 bp
->msix_table
[0].vector
,
1101 0, bp
->msix_table
[offset
].vector
,
1102 i
- 1, bp
->msix_table
[offset
+ i
- 1].vector
);
1107 static int bnx2x_enable_msi(struct bnx2x
*bp
)
1111 rc
= pci_enable_msi(bp
->pdev
);
1113 DP(NETIF_MSG_IFUP
, "MSI is not attainable\n");
1116 bp
->flags
|= USING_MSI_FLAG
;
1121 static int bnx2x_req_irq(struct bnx2x
*bp
)
1123 unsigned long flags
;
1126 if (bp
->flags
& USING_MSI_FLAG
)
1129 flags
= IRQF_SHARED
;
1131 rc
= request_irq(bp
->pdev
->irq
, bnx2x_interrupt
, flags
,
1132 bp
->dev
->name
, bp
->dev
);
1134 bnx2x_fp(bp
, 0, state
) = BNX2X_FP_STATE_IRQ
;
1139 static void bnx2x_napi_enable(struct bnx2x
*bp
)
1143 for_each_queue(bp
, i
)
1144 napi_enable(&bnx2x_fp(bp
, i
, napi
));
1147 static void bnx2x_napi_disable(struct bnx2x
*bp
)
1151 for_each_queue(bp
, i
)
1152 napi_disable(&bnx2x_fp(bp
, i
, napi
));
1155 void bnx2x_netif_start(struct bnx2x
*bp
)
1159 intr_sem
= atomic_dec_and_test(&bp
->intr_sem
);
1160 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1163 if (netif_running(bp
->dev
)) {
1164 bnx2x_napi_enable(bp
);
1165 bnx2x_int_enable(bp
);
1166 if (bp
->state
== BNX2X_STATE_OPEN
)
1167 netif_tx_wake_all_queues(bp
->dev
);
1172 void bnx2x_netif_stop(struct bnx2x
*bp
, int disable_hw
)
1174 bnx2x_int_disable_sync(bp
, disable_hw
);
1175 bnx2x_napi_disable(bp
);
1176 netif_tx_disable(bp
->dev
);
1178 static int bnx2x_set_num_queues(struct bnx2x
*bp
)
1182 switch (bp
->int_mode
) {
1184 bnx2x_enable_msi(bp
);
1185 /* falling through... */
1188 DP(NETIF_MSG_IFUP
, "set number of queues to 1\n");
1191 /* Set number of queues according to bp->multi_mode value */
1192 bnx2x_set_num_queues_msix(bp
);
1194 DP(NETIF_MSG_IFUP
, "set number of queues to %d\n",
1197 /* if we can't use MSI-X we only need one fp,
1198 * so try to enable MSI-X with the requested number of fp's
1199 * and fallback to MSI or legacy INTx with one fp
1201 rc
= bnx2x_enable_msix(bp
);
1203 /* failed to enable MSI-X */
1206 /* Fall to INTx if failed to enable MSI-X due to lack of
1207 * memory (in bnx2x_set_num_queues()) */
1208 if ((rc
!= -ENOMEM
) && (bp
->int_mode
!= INT_MODE_INTx
))
1209 bnx2x_enable_msi(bp
);
1214 netif_set_real_num_tx_queues(bp
->dev
, bp
->num_queues
);
1215 return netif_set_real_num_rx_queues(bp
->dev
, bp
->num_queues
);
1218 static void bnx2x_release_firmware(struct bnx2x
*bp
)
1220 kfree(bp
->init_ops_offsets
);
1221 kfree(bp
->init_ops
);
1222 kfree(bp
->init_data
);
1223 release_firmware(bp
->firmware
);
1226 /* must be called with rtnl_lock */
1227 int bnx2x_nic_load(struct bnx2x
*bp
, int load_mode
)
1232 /* Set init arrays */
1233 rc
= bnx2x_init_firmware(bp
);
1235 BNX2X_ERR("Error loading firmware\n");
1239 #ifdef BNX2X_STOP_ON_ERROR
1240 if (unlikely(bp
->panic
))
1244 bp
->state
= BNX2X_STATE_OPENING_WAIT4_LOAD
;
1246 rc
= bnx2x_set_num_queues(bp
);
1250 /* must be called before memory allocation and HW init */
1251 bnx2x_ilt_set_info(bp
);
1253 if (bnx2x_alloc_mem(bp
)) {
1254 bnx2x_free_irq(bp
, true);
1258 for_each_queue(bp
, i
)
1259 bnx2x_fp(bp
, i
, disable_tpa
) =
1260 ((bp
->flags
& TPA_ENABLE_FLAG
) == 0);
1262 for_each_queue(bp
, i
)
1263 netif_napi_add(bp
->dev
, &bnx2x_fp(bp
, i
, napi
),
1266 bnx2x_napi_enable(bp
);
1268 if (bp
->flags
& USING_MSIX_FLAG
) {
1269 rc
= bnx2x_req_msix_irqs(bp
);
1271 bnx2x_free_irq(bp
, true);
1276 rc
= bnx2x_req_irq(bp
);
1278 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc
);
1279 bnx2x_free_irq(bp
, true);
1282 if (bp
->flags
& USING_MSI_FLAG
) {
1283 bp
->dev
->irq
= bp
->pdev
->irq
;
1284 netdev_info(bp
->dev
, "using MSI IRQ %d\n",
1289 /* Send LOAD_REQUEST command to MCP
1290 Returns the type of LOAD command:
1291 if it is the first port to be initialized
1292 common blocks should be initialized, otherwise - not
1294 if (!BP_NOMCP(bp
)) {
1295 load_code
= bnx2x_fw_command(bp
, DRV_MSG_CODE_LOAD_REQ
, 0);
1297 BNX2X_ERR("MCP response failure, aborting\n");
1301 if (load_code
== FW_MSG_CODE_DRV_LOAD_REFUSED
) {
1302 rc
= -EBUSY
; /* other port in diagnostic mode */
1307 int path
= BP_PATH(bp
);
1308 int port
= BP_PORT(bp
);
1310 DP(NETIF_MSG_IFUP
, "NO MCP - load counts[%d] %d, %d, %d\n",
1311 path
, load_count
[path
][0], load_count
[path
][1],
1312 load_count
[path
][2]);
1313 load_count
[path
][0]++;
1314 load_count
[path
][1 + port
]++;
1315 DP(NETIF_MSG_IFUP
, "NO MCP - new load counts[%d] %d, %d, %d\n",
1316 path
, load_count
[path
][0], load_count
[path
][1],
1317 load_count
[path
][2]);
1318 if (load_count
[path
][0] == 1)
1319 load_code
= FW_MSG_CODE_DRV_LOAD_COMMON
;
1320 else if (load_count
[path
][1 + port
] == 1)
1321 load_code
= FW_MSG_CODE_DRV_LOAD_PORT
;
1323 load_code
= FW_MSG_CODE_DRV_LOAD_FUNCTION
;
1326 if ((load_code
== FW_MSG_CODE_DRV_LOAD_COMMON
) ||
1327 (load_code
== FW_MSG_CODE_DRV_LOAD_COMMON_CHIP
) ||
1328 (load_code
== FW_MSG_CODE_DRV_LOAD_PORT
))
1332 DP(NETIF_MSG_LINK
, "pmf %d\n", bp
->port
.pmf
);
1335 rc
= bnx2x_init_hw(bp
, load_code
);
1337 BNX2X_ERR("HW init failed, aborting\n");
1338 bnx2x_fw_command(bp
, DRV_MSG_CODE_LOAD_DONE
, 0);
1339 bnx2x_fw_command(bp
, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP
, 0);
1340 bnx2x_fw_command(bp
, DRV_MSG_CODE_UNLOAD_DONE
, 0);
1345 bnx2x_fw_command(bp
, DRV_MSG_CODE_LOAD_DONE
, 0);
1349 /* Setup NIC internals and enable interrupts */
1350 bnx2x_nic_init(bp
, load_code
);
1352 if (((load_code
== FW_MSG_CODE_DRV_LOAD_COMMON
) ||
1353 (load_code
== FW_MSG_CODE_DRV_LOAD_COMMON_CHIP
)) &&
1354 (bp
->common
.shmem2_base
))
1355 SHMEM2_WR(bp
, dcc_support
,
1356 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV
|
1357 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV
));
1359 /* Send LOAD_DONE command to MCP */
1360 if (!BP_NOMCP(bp
)) {
1361 load_code
= bnx2x_fw_command(bp
, DRV_MSG_CODE_LOAD_DONE
, 0);
1363 BNX2X_ERR("MCP response failure, aborting\n");
1369 bp
->state
= BNX2X_STATE_OPENING_WAIT4_PORT
;
1371 rc
= bnx2x_func_start(bp
);
1373 BNX2X_ERR("Function start failed!\n");
1374 #ifndef BNX2X_STOP_ON_ERROR
1382 rc
= bnx2x_setup_client(bp
, &bp
->fp
[0], 1 /* Leading */);
1384 BNX2X_ERR("Setup leading failed!\n");
1385 #ifndef BNX2X_STOP_ON_ERROR
1393 if (!CHIP_IS_E1(bp
) &&
1394 (bp
->mf_config
[BP_VN(bp
)] & FUNC_MF_CFG_FUNC_DISABLED
)) {
1395 DP(NETIF_MSG_IFUP
, "mf_cfg function disabled\n");
1396 bp
->flags
|= MF_FUNC_DIS
;
1400 /* Enable Timer scan */
1401 REG_WR(bp
, TM_REG_EN_LINEAR0_TIMER
+ BP_PORT(bp
)*4, 1);
1403 for_each_nondefault_queue(bp
, i
) {
1404 rc
= bnx2x_setup_client(bp
, &bp
->fp
[i
], 0);
1413 /* Now when Clients are configured we are ready to work */
1414 bp
->state
= BNX2X_STATE_OPEN
;
1416 bnx2x_set_eth_mac(bp
, 1);
1419 /* Set iSCSI L2 MAC */
1420 mutex_lock(&bp
->cnic_mutex
);
1421 if (bp
->cnic_eth_dev
.drv_state
& CNIC_DRV_STATE_REGD
) {
1422 bnx2x_set_iscsi_eth_mac_addr(bp
, 1);
1423 bp
->cnic_flags
|= BNX2X_CNIC_FLAG_MAC_SET
;
1424 bnx2x_init_sb(bp
, bp
->cnic_sb_mapping
,
1425 BNX2X_VF_ID_INVALID
, false,
1426 CNIC_SB_ID(bp
), CNIC_IGU_SB_ID(bp
));
1428 mutex_unlock(&bp
->cnic_mutex
);
1432 bnx2x_initial_phy_init(bp
, load_mode
);
1434 /* Start fast path */
1435 switch (load_mode
) {
1437 /* Tx queue should be only reenabled */
1438 netif_tx_wake_all_queues(bp
->dev
);
1439 /* Initialize the receive filter. */
1440 bnx2x_set_rx_mode(bp
->dev
);
1444 netif_tx_start_all_queues(bp
->dev
);
1445 smp_mb__after_clear_bit();
1446 /* Initialize the receive filter. */
1447 bnx2x_set_rx_mode(bp
->dev
);
1451 /* Initialize the receive filter. */
1452 bnx2x_set_rx_mode(bp
->dev
);
1453 bp
->state
= BNX2X_STATE_DIAG
;
1461 bnx2x__link_status_update(bp
);
1463 /* start the timer */
1464 mod_timer(&bp
->timer
, jiffies
+ bp
->current_interval
);
1467 bnx2x_setup_cnic_irq_info(bp
);
1468 if (bp
->state
== BNX2X_STATE_OPEN
)
1469 bnx2x_cnic_notify(bp
, CNIC_CTL_START_CMD
);
1471 bnx2x_inc_load_cnt(bp
);
1473 bnx2x_release_firmware(bp
);
1479 /* Disable Timer scan */
1480 REG_WR(bp
, TM_REG_EN_LINEAR0_TIMER
+ BP_PORT(bp
)*4, 0);
1483 bnx2x_int_disable_sync(bp
, 1);
1484 if (!BP_NOMCP(bp
)) {
1485 bnx2x_fw_command(bp
, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP
, 0);
1486 bnx2x_fw_command(bp
, DRV_MSG_CODE_UNLOAD_DONE
, 0);
1489 /* Free SKBs, SGEs, TPA pool and driver internals */
1490 bnx2x_free_skbs(bp
);
1491 for_each_queue(bp
, i
)
1492 bnx2x_free_rx_sge_range(bp
, bp
->fp
+ i
, NUM_RX_SGE
);
1495 bnx2x_free_irq(bp
, false);
1497 bnx2x_napi_disable(bp
);
1498 for_each_queue(bp
, i
)
1499 netif_napi_del(&bnx2x_fp(bp
, i
, napi
));
1502 bnx2x_release_firmware(bp
);
1507 /* must be called with rtnl_lock */
1508 int bnx2x_nic_unload(struct bnx2x
*bp
, int unload_mode
)
1512 if (bp
->state
== BNX2X_STATE_CLOSED
) {
1513 /* Interface has been removed - nothing to recover */
1514 bp
->recovery_state
= BNX2X_RECOVERY_DONE
;
1516 bnx2x_release_hw_lock(bp
, HW_LOCK_RESOURCE_RESERVED_08
);
1523 bnx2x_cnic_notify(bp
, CNIC_CTL_STOP_CMD
);
1525 bp
->state
= BNX2X_STATE_CLOSING_WAIT4_HALT
;
1527 /* Set "drop all" */
1528 bp
->rx_mode
= BNX2X_RX_MODE_NONE
;
1529 bnx2x_set_storm_rx_mode(bp
);
1532 bnx2x_tx_disable(bp
);
1533 del_timer_sync(&bp
->timer
);
1534 SHMEM_WR(bp
, func_mb
[BP_FW_MB_IDX(bp
)].drv_pulse_mb
,
1535 (DRV_PULSE_ALWAYS_ALIVE
| bp
->fw_drv_pulse_wr_seq
));
1536 bnx2x_stats_handle(bp
, STATS_EVENT_STOP
);
1539 /* Cleanup the chip if needed */
1540 if (unload_mode
!= UNLOAD_RECOVERY
)
1541 bnx2x_chip_cleanup(bp
, unload_mode
);
1543 /* Disable HW interrupts, NAPI and Tx */
1544 bnx2x_netif_stop(bp
, 1);
1547 bnx2x_free_irq(bp
, false);
1552 /* Free SKBs, SGEs, TPA pool and driver internals */
1553 bnx2x_free_skbs(bp
);
1554 for_each_queue(bp
, i
)
1555 bnx2x_free_rx_sge_range(bp
, bp
->fp
+ i
, NUM_RX_SGE
);
1556 for_each_queue(bp
, i
)
1557 netif_napi_del(&bnx2x_fp(bp
, i
, napi
));
1560 bp
->state
= BNX2X_STATE_CLOSED
;
1562 /* The last driver must disable a "close the gate" if there is no
1563 * parity attention or "process kill" pending.
1565 if ((!bnx2x_dec_load_cnt(bp
)) && (!bnx2x_chk_parity_attn(bp
)) &&
1566 bnx2x_reset_is_done(bp
))
1567 bnx2x_disable_close_the_gate(bp
);
1569 /* Reset MCP mail box sequence if there is on going recovery */
1570 if (unload_mode
== UNLOAD_RECOVERY
)
1575 int bnx2x_set_power_state(struct bnx2x
*bp
, pci_power_t state
)
1579 pci_read_config_word(bp
->pdev
, bp
->pm_cap
+ PCI_PM_CTRL
, &pmcsr
);
1583 pci_write_config_word(bp
->pdev
, bp
->pm_cap
+ PCI_PM_CTRL
,
1584 ((pmcsr
& ~PCI_PM_CTRL_STATE_MASK
) |
1585 PCI_PM_CTRL_PME_STATUS
));
1587 if (pmcsr
& PCI_PM_CTRL_STATE_MASK
)
1588 /* delay required during transition out of D3hot */
1593 /* If there are other clients above don't
1594 shut down the power */
1595 if (atomic_read(&bp
->pdev
->enable_cnt
) != 1)
1597 /* Don't shut down the power for emulation and FPGA */
1598 if (CHIP_REV_IS_SLOW(bp
))
1601 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
1605 pmcsr
|= PCI_PM_CTRL_PME_ENABLE
;
1607 pci_write_config_word(bp
->pdev
, bp
->pm_cap
+ PCI_PM_CTRL
,
1610 /* No more memory access after this point until
1611 * device is brought back to D0.
1624 * net_device service functions
1627 static int bnx2x_poll(struct napi_struct
*napi
, int budget
)
1630 struct bnx2x_fastpath
*fp
= container_of(napi
, struct bnx2x_fastpath
,
1632 struct bnx2x
*bp
= fp
->bp
;
1635 #ifdef BNX2X_STOP_ON_ERROR
1636 if (unlikely(bp
->panic
)) {
1637 napi_complete(napi
);
1642 if (bnx2x_has_tx_work(fp
))
1645 if (bnx2x_has_rx_work(fp
)) {
1646 work_done
+= bnx2x_rx_int(fp
, budget
- work_done
);
1648 /* must not complete if we consumed full budget */
1649 if (work_done
>= budget
)
1653 /* Fall out from the NAPI loop if needed */
1654 if (!(bnx2x_has_rx_work(fp
) || bnx2x_has_tx_work(fp
))) {
1655 bnx2x_update_fpsb_idx(fp
);
1656 /* bnx2x_has_rx_work() reads the status block,
1657 * thus we need to ensure that status block indices
1658 * have been actually read (bnx2x_update_fpsb_idx)
1659 * prior to this check (bnx2x_has_rx_work) so that
1660 * we won't write the "newer" value of the status block
1661 * to IGU (if there was a DMA right after
1662 * bnx2x_has_rx_work and if there is no rmb, the memory
1663 * reading (bnx2x_update_fpsb_idx) may be postponed
1664 * to right before bnx2x_ack_sb). In this case there
1665 * will never be another interrupt until there is
1666 * another update of the status block, while there
1667 * is still unhandled work.
1671 if (!(bnx2x_has_rx_work(fp
) || bnx2x_has_tx_work(fp
))) {
1672 napi_complete(napi
);
1673 /* Re-enable interrupts */
1675 "Update index to %d\n", fp
->fp_hc_idx
);
1676 bnx2x_ack_sb(bp
, fp
->igu_sb_id
, USTORM_ID
,
1677 le16_to_cpu(fp
->fp_hc_idx
),
1688 /* we split the first BD into headers and data BDs
1689 * to ease the pain of our fellow microcode engineers
1690 * we use one mapping for both BDs
1691 * So far this has only been observed to happen
1692 * in Other Operating Systems(TM)
1694 static noinline u16
bnx2x_tx_split(struct bnx2x
*bp
,
1695 struct bnx2x_fastpath
*fp
,
1696 struct sw_tx_bd
*tx_buf
,
1697 struct eth_tx_start_bd
**tx_bd
, u16 hlen
,
1698 u16 bd_prod
, int nbd
)
1700 struct eth_tx_start_bd
*h_tx_bd
= *tx_bd
;
1701 struct eth_tx_bd
*d_tx_bd
;
1703 int old_len
= le16_to_cpu(h_tx_bd
->nbytes
);
1705 /* first fix first BD */
1706 h_tx_bd
->nbd
= cpu_to_le16(nbd
);
1707 h_tx_bd
->nbytes
= cpu_to_le16(hlen
);
1709 DP(NETIF_MSG_TX_QUEUED
, "TSO split header size is %d "
1710 "(%x:%x) nbd %d\n", h_tx_bd
->nbytes
, h_tx_bd
->addr_hi
,
1711 h_tx_bd
->addr_lo
, h_tx_bd
->nbd
);
1713 /* now get a new data BD
1714 * (after the pbd) and fill it */
1715 bd_prod
= TX_BD(NEXT_TX_IDX(bd_prod
));
1716 d_tx_bd
= &fp
->tx_desc_ring
[bd_prod
].reg_bd
;
1718 mapping
= HILO_U64(le32_to_cpu(h_tx_bd
->addr_hi
),
1719 le32_to_cpu(h_tx_bd
->addr_lo
)) + hlen
;
1721 d_tx_bd
->addr_hi
= cpu_to_le32(U64_HI(mapping
));
1722 d_tx_bd
->addr_lo
= cpu_to_le32(U64_LO(mapping
));
1723 d_tx_bd
->nbytes
= cpu_to_le16(old_len
- hlen
);
1725 /* this marks the BD as one that has no individual mapping */
1726 tx_buf
->flags
|= BNX2X_TSO_SPLIT_BD
;
1728 DP(NETIF_MSG_TX_QUEUED
,
1729 "TSO split data size is %d (%x:%x)\n",
1730 d_tx_bd
->nbytes
, d_tx_bd
->addr_hi
, d_tx_bd
->addr_lo
);
1733 *tx_bd
= (struct eth_tx_start_bd
*)d_tx_bd
;
1738 static inline u16
bnx2x_csum_fix(unsigned char *t_header
, u16 csum
, s8 fix
)
1741 csum
= (u16
) ~csum_fold(csum_sub(csum
,
1742 csum_partial(t_header
- fix
, fix
, 0)));
1745 csum
= (u16
) ~csum_fold(csum_add(csum
,
1746 csum_partial(t_header
, -fix
, 0)));
1748 return swab16(csum
);
1751 static inline u32
bnx2x_xmit_type(struct bnx2x
*bp
, struct sk_buff
*skb
)
1755 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1759 if (skb
->protocol
== htons(ETH_P_IPV6
)) {
1761 if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_TCP
)
1762 rc
|= XMIT_CSUM_TCP
;
1766 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
1767 rc
|= XMIT_CSUM_TCP
;
1771 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
)
1772 rc
|= (XMIT_GSO_V4
| XMIT_CSUM_V4
| XMIT_CSUM_TCP
);
1774 else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV6
)
1775 rc
|= (XMIT_GSO_V6
| XMIT_CSUM_TCP
| XMIT_CSUM_V6
);
1780 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1781 /* check if packet requires linearization (packet is too fragmented)
1782 no need to check fragmentation if page size > 8K (there will be no
1783 violation to FW restrictions) */
1784 static int bnx2x_pkt_req_lin(struct bnx2x
*bp
, struct sk_buff
*skb
,
1789 int first_bd_sz
= 0;
1791 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1792 if (skb_shinfo(skb
)->nr_frags
>= (MAX_FETCH_BD
- 3)) {
1794 if (xmit_type
& XMIT_GSO
) {
1795 unsigned short lso_mss
= skb_shinfo(skb
)->gso_size
;
1796 /* Check if LSO packet needs to be copied:
1797 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1798 int wnd_size
= MAX_FETCH_BD
- 3;
1799 /* Number of windows to check */
1800 int num_wnds
= skb_shinfo(skb
)->nr_frags
- wnd_size
;
1805 /* Headers length */
1806 hlen
= (int)(skb_transport_header(skb
) - skb
->data
) +
1809 /* Amount of data (w/o headers) on linear part of SKB*/
1810 first_bd_sz
= skb_headlen(skb
) - hlen
;
1812 wnd_sum
= first_bd_sz
;
1814 /* Calculate the first sum - it's special */
1815 for (frag_idx
= 0; frag_idx
< wnd_size
- 1; frag_idx
++)
1817 skb_shinfo(skb
)->frags
[frag_idx
].size
;
1819 /* If there was data on linear skb data - check it */
1820 if (first_bd_sz
> 0) {
1821 if (unlikely(wnd_sum
< lso_mss
)) {
1826 wnd_sum
-= first_bd_sz
;
1829 /* Others are easier: run through the frag list and
1830 check all windows */
1831 for (wnd_idx
= 0; wnd_idx
<= num_wnds
; wnd_idx
++) {
1833 skb_shinfo(skb
)->frags
[wnd_idx
+ wnd_size
- 1].size
;
1835 if (unlikely(wnd_sum
< lso_mss
)) {
1840 skb_shinfo(skb
)->frags
[wnd_idx
].size
;
1843 /* in non-LSO too fragmented packet should always
1850 if (unlikely(to_copy
))
1851 DP(NETIF_MSG_TX_QUEUED
,
1852 "Linearization IS REQUIRED for %s packet. "
1853 "num_frags %d hlen %d first_bd_sz %d\n",
1854 (xmit_type
& XMIT_GSO
) ? "LSO" : "non-LSO",
1855 skb_shinfo(skb
)->nr_frags
, hlen
, first_bd_sz
);
1861 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff
*skb
,
1862 struct eth_tx_parse_bd_e2
*pbd
,
1865 pbd
->parsing_data
|= cpu_to_le16(skb_shinfo(skb
)->gso_size
) <<
1866 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT
;
1867 if ((xmit_type
& XMIT_GSO_V6
) &&
1868 (ipv6_hdr(skb
)->nexthdr
== NEXTHDR_IPV6
))
1869 pbd
->parsing_data
|= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR
;
1873 * Update PBD in GSO case.
1876 * @param tx_start_bd
1880 static inline void bnx2x_set_pbd_gso(struct sk_buff
*skb
,
1881 struct eth_tx_parse_bd_e1x
*pbd
,
1884 pbd
->lso_mss
= cpu_to_le16(skb_shinfo(skb
)->gso_size
);
1885 pbd
->tcp_send_seq
= swab32(tcp_hdr(skb
)->seq
);
1886 pbd
->tcp_flags
= pbd_tcp_flags(skb
);
1888 if (xmit_type
& XMIT_GSO_V4
) {
1889 pbd
->ip_id
= swab16(ip_hdr(skb
)->id
);
1890 pbd
->tcp_pseudo_csum
=
1891 swab16(~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
1893 0, IPPROTO_TCP
, 0));
1896 pbd
->tcp_pseudo_csum
=
1897 swab16(~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
1898 &ipv6_hdr(skb
)->daddr
,
1899 0, IPPROTO_TCP
, 0));
1901 pbd
->global_data
|= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN
;
1906 * @param tx_start_bd
1910 * @return header len
1912 static inline u8
bnx2x_set_pbd_csum_e2(struct bnx2x
*bp
, struct sk_buff
*skb
,
1913 struct eth_tx_parse_bd_e2
*pbd
,
1916 pbd
->parsing_data
|= cpu_to_le16(tcp_hdrlen(skb
)/4) <<
1917 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT
;
1919 pbd
->parsing_data
|= cpu_to_le16(((unsigned char *)tcp_hdr(skb
) -
1921 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT
;
1923 return skb_transport_header(skb
) + tcp_hdrlen(skb
) - skb
->data
;
1929 * @param tx_start_bd
1933 * @return Header length
1935 static inline u8
bnx2x_set_pbd_csum(struct bnx2x
*bp
, struct sk_buff
*skb
,
1936 struct eth_tx_parse_bd_e1x
*pbd
,
1939 u8 hlen
= (skb_network_header(skb
) - skb
->data
) / 2;
1941 /* for now NS flag is not used in Linux */
1943 (hlen
| ((skb
->protocol
== cpu_to_be16(ETH_P_8021Q
)) <<
1944 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT
));
1946 pbd
->ip_hlen_w
= (skb_transport_header(skb
) -
1947 skb_network_header(skb
)) / 2;
1949 hlen
+= pbd
->ip_hlen_w
+ tcp_hdrlen(skb
) / 2;
1951 pbd
->total_hlen_w
= cpu_to_le16(hlen
);
1954 if (xmit_type
& XMIT_CSUM_TCP
) {
1955 pbd
->tcp_pseudo_csum
= swab16(tcp_hdr(skb
)->check
);
1958 s8 fix
= SKB_CS_OFF(skb
); /* signed! */
1960 DP(NETIF_MSG_TX_QUEUED
,
1961 "hlen %d fix %d csum before fix %x\n",
1962 le16_to_cpu(pbd
->total_hlen_w
), fix
, SKB_CS(skb
));
1964 /* HW bug: fixup the CSUM */
1965 pbd
->tcp_pseudo_csum
=
1966 bnx2x_csum_fix(skb_transport_header(skb
),
1969 DP(NETIF_MSG_TX_QUEUED
, "csum after fix %x\n",
1970 pbd
->tcp_pseudo_csum
);
1975 /* called with netif_tx_lock
1976 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1977 * netif_wake_queue()
1979 netdev_tx_t
bnx2x_start_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1981 struct bnx2x
*bp
= netdev_priv(dev
);
1982 struct bnx2x_fastpath
*fp
;
1983 struct netdev_queue
*txq
;
1984 struct sw_tx_bd
*tx_buf
;
1985 struct eth_tx_start_bd
*tx_start_bd
;
1986 struct eth_tx_bd
*tx_data_bd
, *total_pkt_bd
= NULL
;
1987 struct eth_tx_parse_bd_e1x
*pbd_e1x
= NULL
;
1988 struct eth_tx_parse_bd_e2
*pbd_e2
= NULL
;
1989 u16 pkt_prod
, bd_prod
;
1992 u32 xmit_type
= bnx2x_xmit_type(bp
, skb
);
1995 __le16 pkt_size
= 0;
1997 u8 mac_type
= UNICAST_ADDRESS
;
1999 #ifdef BNX2X_STOP_ON_ERROR
2000 if (unlikely(bp
->panic
))
2001 return NETDEV_TX_BUSY
;
2004 fp_index
= skb_get_queue_mapping(skb
);
2005 txq
= netdev_get_tx_queue(dev
, fp_index
);
2007 fp
= &bp
->fp
[fp_index
];
2009 if (unlikely(bnx2x_tx_avail(fp
) < (skb_shinfo(skb
)->nr_frags
+ 3))) {
2010 fp
->eth_q_stats
.driver_xoff
++;
2011 netif_tx_stop_queue(txq
);
2012 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2013 return NETDEV_TX_BUSY
;
2016 DP(NETIF_MSG_TX_QUEUED
, "queue[%d]: SKB: summed %x protocol %x "
2017 "protocol(%x,%x) gso type %x xmit_type %x\n",
2018 fp_index
, skb
->ip_summed
, skb
->protocol
, ipv6_hdr(skb
)->nexthdr
,
2019 ip_hdr(skb
)->protocol
, skb_shinfo(skb
)->gso_type
, xmit_type
);
2021 eth
= (struct ethhdr
*)skb
->data
;
2023 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2024 if (unlikely(is_multicast_ether_addr(eth
->h_dest
))) {
2025 if (is_broadcast_ether_addr(eth
->h_dest
))
2026 mac_type
= BROADCAST_ADDRESS
;
2028 mac_type
= MULTICAST_ADDRESS
;
2031 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2032 /* First, check if we need to linearize the skb (due to FW
2033 restrictions). No need to check fragmentation if page size > 8K
2034 (there will be no violation to FW restrictions) */
2035 if (bnx2x_pkt_req_lin(bp
, skb
, xmit_type
)) {
2036 /* Statistics of linearization */
2038 if (skb_linearize(skb
) != 0) {
2039 DP(NETIF_MSG_TX_QUEUED
, "SKB linearization failed - "
2040 "silently dropping this SKB\n");
2041 dev_kfree_skb_any(skb
);
2042 return NETDEV_TX_OK
;
2048 Please read carefully. First we use one BD which we mark as start,
2049 then we have a parsing info BD (used for TSO or xsum),
2050 and only then we have the rest of the TSO BDs.
2051 (don't forget to mark the last one as last,
2052 and to unmap only AFTER you write to the BD ...)
2053 And above all, all pdb sizes are in words - NOT DWORDS!
2056 pkt_prod
= fp
->tx_pkt_prod
++;
2057 bd_prod
= TX_BD(fp
->tx_bd_prod
);
2059 /* get a tx_buf and first BD */
2060 tx_buf
= &fp
->tx_buf_ring
[TX_BD(pkt_prod
)];
2061 tx_start_bd
= &fp
->tx_desc_ring
[bd_prod
].start_bd
;
2063 tx_start_bd
->bd_flags
.as_bitfield
= ETH_TX_BD_FLAGS_START_BD
;
2064 SET_FLAG(tx_start_bd
->general_data
,
2065 ETH_TX_START_BD_ETH_ADDR_TYPE
,
2068 SET_FLAG(tx_start_bd
->general_data
,
2069 ETH_TX_START_BD_HDR_NBDS
,
2072 /* remember the first BD of the packet */
2073 tx_buf
->first_bd
= fp
->tx_bd_prod
;
2077 DP(NETIF_MSG_TX_QUEUED
,
2078 "sending pkt %u @%p next_idx %u bd %u @%p\n",
2079 pkt_prod
, tx_buf
, fp
->tx_pkt_prod
, bd_prod
, tx_start_bd
);
2082 if ((bp
->vlgrp
!= NULL
) && vlan_tx_tag_present(skb
) &&
2083 (bp
->flags
& HW_VLAN_TX_FLAG
)) {
2084 tx_start_bd
->vlan_or_ethertype
=
2085 cpu_to_le16(vlan_tx_tag_get(skb
));
2086 tx_start_bd
->bd_flags
.as_bitfield
|=
2087 (X_ETH_OUTBAND_VLAN
<< ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT
);
2090 tx_start_bd
->vlan_or_ethertype
= cpu_to_le16(pkt_prod
);
2092 /* turn on parsing and get a BD */
2093 bd_prod
= TX_BD(NEXT_TX_IDX(bd_prod
));
2095 if (xmit_type
& XMIT_CSUM
) {
2096 tx_start_bd
->bd_flags
.as_bitfield
|= ETH_TX_BD_FLAGS_L4_CSUM
;
2098 if (xmit_type
& XMIT_CSUM_V4
)
2099 tx_start_bd
->bd_flags
.as_bitfield
|=
2100 ETH_TX_BD_FLAGS_IP_CSUM
;
2102 tx_start_bd
->bd_flags
.as_bitfield
|=
2103 ETH_TX_BD_FLAGS_IPV6
;
2105 if (!(xmit_type
& XMIT_CSUM_TCP
))
2106 tx_start_bd
->bd_flags
.as_bitfield
|=
2107 ETH_TX_BD_FLAGS_IS_UDP
;
2110 if (CHIP_IS_E2(bp
)) {
2111 pbd_e2
= &fp
->tx_desc_ring
[bd_prod
].parse_bd_e2
;
2112 memset(pbd_e2
, 0, sizeof(struct eth_tx_parse_bd_e2
));
2113 /* Set PBD in checksum offload case */
2114 if (xmit_type
& XMIT_CSUM
)
2115 hlen
= bnx2x_set_pbd_csum_e2(bp
,
2116 skb
, pbd_e2
, xmit_type
);
2118 pbd_e1x
= &fp
->tx_desc_ring
[bd_prod
].parse_bd_e1x
;
2119 memset(pbd_e1x
, 0, sizeof(struct eth_tx_parse_bd_e1x
));
2120 /* Set PBD in checksum offload case */
2121 if (xmit_type
& XMIT_CSUM
)
2122 hlen
= bnx2x_set_pbd_csum(bp
, skb
, pbd_e1x
, xmit_type
);
2126 mapping
= dma_map_single(&bp
->pdev
->dev
, skb
->data
,
2127 skb_headlen(skb
), DMA_TO_DEVICE
);
2129 tx_start_bd
->addr_hi
= cpu_to_le32(U64_HI(mapping
));
2130 tx_start_bd
->addr_lo
= cpu_to_le32(U64_LO(mapping
));
2131 nbd
= skb_shinfo(skb
)->nr_frags
+ 2; /* start_bd + pbd + frags */
2132 tx_start_bd
->nbd
= cpu_to_le16(nbd
);
2133 tx_start_bd
->nbytes
= cpu_to_le16(skb_headlen(skb
));
2134 pkt_size
= tx_start_bd
->nbytes
;
2136 DP(NETIF_MSG_TX_QUEUED
, "first bd @%p addr (%x:%x) nbd %d"
2137 " nbytes %d flags %x vlan %x\n",
2138 tx_start_bd
, tx_start_bd
->addr_hi
, tx_start_bd
->addr_lo
,
2139 le16_to_cpu(tx_start_bd
->nbd
), le16_to_cpu(tx_start_bd
->nbytes
),
2140 tx_start_bd
->bd_flags
.as_bitfield
,
2141 le16_to_cpu(tx_start_bd
->vlan_or_ethertype
));
2143 if (xmit_type
& XMIT_GSO
) {
2145 DP(NETIF_MSG_TX_QUEUED
,
2146 "TSO packet len %d hlen %d total len %d tso size %d\n",
2147 skb
->len
, hlen
, skb_headlen(skb
),
2148 skb_shinfo(skb
)->gso_size
);
2150 tx_start_bd
->bd_flags
.as_bitfield
|= ETH_TX_BD_FLAGS_SW_LSO
;
2152 if (unlikely(skb_headlen(skb
) > hlen
))
2153 bd_prod
= bnx2x_tx_split(bp
, fp
, tx_buf
, &tx_start_bd
,
2154 hlen
, bd_prod
, ++nbd
);
2156 bnx2x_set_pbd_gso_e2(skb
, pbd_e2
, xmit_type
);
2158 bnx2x_set_pbd_gso(skb
, pbd_e1x
, xmit_type
);
2160 tx_data_bd
= (struct eth_tx_bd
*)tx_start_bd
;
2162 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
2163 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
2165 bd_prod
= TX_BD(NEXT_TX_IDX(bd_prod
));
2166 tx_data_bd
= &fp
->tx_desc_ring
[bd_prod
].reg_bd
;
2167 if (total_pkt_bd
== NULL
)
2168 total_pkt_bd
= &fp
->tx_desc_ring
[bd_prod
].reg_bd
;
2170 mapping
= dma_map_page(&bp
->pdev
->dev
, frag
->page
,
2172 frag
->size
, DMA_TO_DEVICE
);
2174 tx_data_bd
->addr_hi
= cpu_to_le32(U64_HI(mapping
));
2175 tx_data_bd
->addr_lo
= cpu_to_le32(U64_LO(mapping
));
2176 tx_data_bd
->nbytes
= cpu_to_le16(frag
->size
);
2177 le16_add_cpu(&pkt_size
, frag
->size
);
2179 DP(NETIF_MSG_TX_QUEUED
,
2180 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2181 i
, tx_data_bd
, tx_data_bd
->addr_hi
, tx_data_bd
->addr_lo
,
2182 le16_to_cpu(tx_data_bd
->nbytes
));
2185 DP(NETIF_MSG_TX_QUEUED
, "last bd @%p\n", tx_data_bd
);
2187 bd_prod
= TX_BD(NEXT_TX_IDX(bd_prod
));
2189 /* now send a tx doorbell, counting the next BD
2190 * if the packet contains or ends with it
2192 if (TX_BD_POFF(bd_prod
) < nbd
)
2195 if (total_pkt_bd
!= NULL
)
2196 total_pkt_bd
->total_pkt_bytes
= pkt_size
;
2199 DP(NETIF_MSG_TX_QUEUED
,
2200 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2201 " tcp_flags %x xsum %x seq %u hlen %u\n",
2202 pbd_e1x
, pbd_e1x
->global_data
, pbd_e1x
->ip_hlen_w
,
2203 pbd_e1x
->ip_id
, pbd_e1x
->lso_mss
, pbd_e1x
->tcp_flags
,
2204 pbd_e1x
->tcp_pseudo_csum
, pbd_e1x
->tcp_send_seq
,
2205 le16_to_cpu(pbd_e1x
->total_hlen_w
));
2207 DP(NETIF_MSG_TX_QUEUED
,
2208 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
2209 pbd_e2
, pbd_e2
->dst_mac_addr_hi
, pbd_e2
->dst_mac_addr_mid
,
2210 pbd_e2
->dst_mac_addr_lo
, pbd_e2
->src_mac_addr_hi
,
2211 pbd_e2
->src_mac_addr_mid
, pbd_e2
->src_mac_addr_lo
,
2212 pbd_e2
->parsing_data
);
2213 DP(NETIF_MSG_TX_QUEUED
, "doorbell: nbd %d bd %u\n", nbd
, bd_prod
);
2216 * Make sure that the BD data is updated before updating the producer
2217 * since FW might read the BD right after the producer is updated.
2218 * This is only applicable for weak-ordered memory model archs such
2219 * as IA-64. The following barrier is also mandatory since FW will
2220 * assumes packets must have BDs.
2224 fp
->tx_db
.data
.prod
+= nbd
;
2226 DOORBELL(bp
, fp
->cid
, fp
->tx_db
.raw
);
2230 fp
->tx_bd_prod
+= nbd
;
2232 if (unlikely(bnx2x_tx_avail(fp
) < MAX_SKB_FRAGS
+ 3)) {
2233 netif_tx_stop_queue(txq
);
2235 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2236 * ordering of set_bit() in netif_tx_stop_queue() and read of
2240 fp
->eth_q_stats
.driver_xoff
++;
2241 if (bnx2x_tx_avail(fp
) >= MAX_SKB_FRAGS
+ 3)
2242 netif_tx_wake_queue(txq
);
2246 return NETDEV_TX_OK
;
2248 /* called with rtnl_lock */
2249 int bnx2x_change_mac_addr(struct net_device
*dev
, void *p
)
2251 struct sockaddr
*addr
= p
;
2252 struct bnx2x
*bp
= netdev_priv(dev
);
2254 if (!is_valid_ether_addr((u8
*)(addr
->sa_data
)))
2257 memcpy(dev
->dev_addr
, addr
->sa_data
, dev
->addr_len
);
2258 if (netif_running(dev
))
2259 bnx2x_set_eth_mac(bp
, 1);
2264 void bnx2x_free_mem_bp(struct bnx2x
*bp
)
2267 kfree(bp
->msix_table
);
2271 int __devinit
bnx2x_alloc_mem_bp(struct bnx2x
*bp
)
2273 struct bnx2x_fastpath
*fp
;
2274 struct msix_entry
*tbl
;
2275 struct bnx2x_ilt
*ilt
;
2278 fp
= kzalloc(L2_FP_COUNT(bp
->l2_cid_count
)*sizeof(*fp
), GFP_KERNEL
);
2284 tbl
= kzalloc((bp
->l2_cid_count
+ 1) * sizeof(*tbl
),
2288 bp
->msix_table
= tbl
;
2291 ilt
= kzalloc(sizeof(*ilt
), GFP_KERNEL
);
2298 bnx2x_free_mem_bp(bp
);
2303 /* called with rtnl_lock */
2304 int bnx2x_change_mtu(struct net_device
*dev
, int new_mtu
)
2306 struct bnx2x
*bp
= netdev_priv(dev
);
2309 if (bp
->recovery_state
!= BNX2X_RECOVERY_DONE
) {
2310 printk(KERN_ERR
"Handling parity error recovery. Try again later\n");
2314 if ((new_mtu
> ETH_MAX_JUMBO_PACKET_SIZE
) ||
2315 ((new_mtu
+ ETH_HLEN
) < ETH_MIN_PACKET_SIZE
))
2318 /* This does not race with packet allocation
2319 * because the actual alloc size is
2320 * only updated as part of load
2324 if (netif_running(dev
)) {
2325 bnx2x_nic_unload(bp
, UNLOAD_NORMAL
);
2326 rc
= bnx2x_nic_load(bp
, LOAD_NORMAL
);
2332 void bnx2x_tx_timeout(struct net_device
*dev
)
2334 struct bnx2x
*bp
= netdev_priv(dev
);
2336 #ifdef BNX2X_STOP_ON_ERROR
2340 /* This allows the netif to be shutdown gracefully before resetting */
2341 schedule_delayed_work(&bp
->reset_task
, 0);
2345 /* called with rtnl_lock */
2346 void bnx2x_vlan_rx_register(struct net_device
*dev
,
2347 struct vlan_group
*vlgrp
)
2349 struct bnx2x
*bp
= netdev_priv(dev
);
2355 int bnx2x_suspend(struct pci_dev
*pdev
, pm_message_t state
)
2357 struct net_device
*dev
= pci_get_drvdata(pdev
);
2361 dev_err(&pdev
->dev
, "BAD net device from bnx2x_init_one\n");
2364 bp
= netdev_priv(dev
);
2368 pci_save_state(pdev
);
2370 if (!netif_running(dev
)) {
2375 netif_device_detach(dev
);
2377 bnx2x_nic_unload(bp
, UNLOAD_CLOSE
);
2379 bnx2x_set_power_state(bp
, pci_choose_state(pdev
, state
));
2386 int bnx2x_resume(struct pci_dev
*pdev
)
2388 struct net_device
*dev
= pci_get_drvdata(pdev
);
2393 dev_err(&pdev
->dev
, "BAD net device from bnx2x_init_one\n");
2396 bp
= netdev_priv(dev
);
2398 if (bp
->recovery_state
!= BNX2X_RECOVERY_DONE
) {
2399 printk(KERN_ERR
"Handling parity error recovery. Try again later\n");
2405 pci_restore_state(pdev
);
2407 if (!netif_running(dev
)) {
2412 bnx2x_set_power_state(bp
, PCI_D0
);
2413 netif_device_attach(dev
);
2415 /* Since the chip was reset, clear the FW sequence number */
2417 rc
= bnx2x_nic_load(bp
, LOAD_OPEN
);