?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
GUI: Fix Tomato RAF theme for all builds. Compilation typo.
[tomato.git] / release / src-rt-6.x.4708 / linux / linux-2.6.36 / drivers / net / bnx2x / bnx2x_cmn.c
bloba5a22d49798666ccba19835855a285a42acc902b
1
2
3
4 #include <linux/etherdevice.h>
5 #include <linux/ip.h>
6 #include <linux/ipv6.h>
7 #include <net/ip6_checksum.h>
8 #include "bnx2x_cmn.h"
9
10 #ifdef BCM_VLAN
11 #include <linux/if_vlan.h>
12 #endif
13
14 static int bnx2x_poll(struct napi_struct *napi, int budget);
15
16 /* free skb in the packet ring at pos idx
17 * return idx of last bd freed
18 */
19 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
20 u16 idx)
21 {
22 struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
23 struct eth_tx_start_bd *tx_start_bd;
24 struct eth_tx_bd *tx_data_bd;
25 struct sk_buff *skb = tx_buf->skb;
26 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
27 int nbd;
28
29 /* prefetch skb end pointer to speedup dev_kfree_skb() */
30 prefetch(&skb->end);
31
32 DP(BNX2X_MSG_OFF, "pkt_idx %d buff @(%p)->skb %p\n",
33 idx, tx_buf, skb);
34
35 /* unmap first bd */
36 DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
37 tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
38 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
39 BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
40
41 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
42 #ifdef BNX2X_STOP_ON_ERROR
43 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
44 BNX2X_ERR("BAD nbd!\n");
45 bnx2x_panic();
46 }
47 #endif
48 new_cons = nbd + tx_buf->first_bd;
49
50 /* Get the next bd */
51 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
52
53 /* Skip a parse bd... */
54 --nbd;
55 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
56
57 /* ...and the TSO split header bd since they have no mapping */
58 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
59 --nbd;
60 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
61 }
62
63 /* now free frags */
64 while (nbd > 0) {
65
66 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
67 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
68 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
69 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
70 if (--nbd)
71 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
72 }
73
74 /* release skb */
75 WARN_ON(!skb);
76 dev_kfree_skb(skb);
77 tx_buf->first_bd = 0;
78 tx_buf->skb = NULL;
79
80 return new_cons;
81 }
82
83 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
84 {
85 struct bnx2x *bp = fp->bp;
86 struct netdev_queue *txq;
87 u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
88
89 #ifdef BNX2X_STOP_ON_ERROR
90 if (unlikely(bp->panic))
91 return -1;
92 #endif
93
94 txq = netdev_get_tx_queue(bp->dev, fp->index);
95 hw_cons = le16_to_cpu(*fp->tx_cons_sb);
96 sw_cons = fp->tx_pkt_cons;
97
98 while (sw_cons != hw_cons) {
99 u16 pkt_cons;
100
101 pkt_cons = TX_BD(sw_cons);
102
103 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
104
105 DP(NETIF_MSG_TX_DONE, "hw_cons %u sw_cons %u pkt_cons %u\n",
106 hw_cons, sw_cons, pkt_cons);
107
108 /* if (NEXT_TX_IDX(sw_cons) != hw_cons) {
109 rmb();
110 prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
111 }
112 */
113 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
114 sw_cons++;
115 }
116
117 fp->tx_pkt_cons = sw_cons;
118 fp->tx_bd_cons = bd_cons;
119
120 /* Need to make the tx_bd_cons update visible to start_xmit()
121 * before checking for netif_tx_queue_stopped(). Without the
122 * memory barrier, there is a small possibility that
123 * start_xmit() will miss it and cause the queue to be stopped
124 * forever.
125 */
126 smp_mb();
127
128 /* TBD need a thresh? */
129 if (unlikely(netif_tx_queue_stopped(txq))) {
130 /* Taking tx_lock() is needed to prevent reenabling the queue
131 * while it's empty. This could have happen if rx_action() gets
132 * suspended in bnx2x_tx_int() after the condition before
133 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
134 *
135 * stops the queue->sees fresh tx_bd_cons->releases the queue->
136 * sends some packets consuming the whole queue again->
137 * stops the queue
138 */
139
140 __netif_tx_lock(txq, smp_processor_id());
141
142 if ((netif_tx_queue_stopped(txq)) &&
143 (bp->state == BNX2X_STATE_OPEN) &&
144 (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
145 netif_tx_wake_queue(txq);
146
147 __netif_tx_unlock(txq);
148 }
149 return 0;
150 }
151
152 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
153 u16 idx)
154 {
155 u16 last_max = fp->last_max_sge;
156
157 if (SUB_S16(idx, last_max) > 0)
158 fp->last_max_sge = idx;
159 }
160
161 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
162 struct eth_fast_path_rx_cqe *fp_cqe)
163 {
164 struct bnx2x *bp = fp->bp;
165 u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
166 le16_to_cpu(fp_cqe->len_on_bd)) >>
167 SGE_PAGE_SHIFT;
168 u16 last_max, last_elem, first_elem;
169 u16 delta = 0;
170 u16 i;
171
172 if (!sge_len)
173 return;
174
175 /* First mark all used pages */
176 for (i = 0; i < sge_len; i++)
177 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
178
179 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
180 sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
181
182 /* Here we assume that the last SGE index is the biggest */
183 prefetch((void *)(fp->sge_mask));
184 bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
185
186 last_max = RX_SGE(fp->last_max_sge);
187 last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
188 first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
189
190 /* If ring is not full */
191 if (last_elem + 1 != first_elem)
192 last_elem++;
193
194 /* Now update the prod */
195 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
196 if (likely(fp->sge_mask[i]))
197 break;
198
199 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
200 delta += RX_SGE_MASK_ELEM_SZ;
201 }
202
203 if (delta > 0) {
204 fp->rx_sge_prod += delta;
205 /* clear page-end entries */
206 bnx2x_clear_sge_mask_next_elems(fp);
207 }
208
209 DP(NETIF_MSG_RX_STATUS,
210 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
211 fp->last_max_sge, fp->rx_sge_prod);
212 }
213
214 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
215 struct sk_buff *skb, u16 cons, u16 prod)
216 {
217 struct bnx2x *bp = fp->bp;
218 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
219 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
220 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
221 dma_addr_t mapping;
222
223 /* move empty skb from pool to prod and map it */
224 prod_rx_buf->skb = fp->tpa_pool[queue].skb;
225 mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
226 bp->rx_buf_size, DMA_FROM_DEVICE);
227 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
228
229 /* move partial skb from cons to pool (don't unmap yet) */
230 fp->tpa_pool[queue] = *cons_rx_buf;
231
232 /* mark bin state as start - print error if current state != stop */
233 if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
234 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
235
236 fp->tpa_state[queue] = BNX2X_TPA_START;
237
238 /* point prod_bd to new skb */
239 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
240 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
241
242 #ifdef BNX2X_STOP_ON_ERROR
243 fp->tpa_queue_used |= (1 << queue);
244 #ifdef _ASM_GENERIC_INT_L64_H
245 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
246 #else
247 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
248 #endif
249 fp->tpa_queue_used);
250 #endif
251 }
252
253 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
254 struct sk_buff *skb,
255 struct eth_fast_path_rx_cqe *fp_cqe,
256 u16 cqe_idx)
257 {
258 struct sw_rx_page *rx_pg, old_rx_pg;
259 u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
260 u32 i, frag_len, frag_size, pages;
261 int err;
262 int j;
263
264 frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
265 pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
266
267 /* This is needed in order to enable forwarding support */
268 if (frag_size)
269 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
270 max(frag_size, (u32)len_on_bd));
271
272 #ifdef BNX2X_STOP_ON_ERROR
273 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
274 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
275 pages, cqe_idx);
276 BNX2X_ERR("fp_cqe->pkt_len = %d fp_cqe->len_on_bd = %d\n",
277 fp_cqe->pkt_len, len_on_bd);
278 bnx2x_panic();
279 return -EINVAL;
280 }
281 #endif
282
283 /* Run through the SGL and compose the fragmented skb */
284 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
285 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
286
287 /* FW gives the indices of the SGE as if the ring is an array
288 (meaning that "next" element will consume 2 indices) */
289 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
290 rx_pg = &fp->rx_page_ring[sge_idx];
291 old_rx_pg = *rx_pg;
292
293 /* If we fail to allocate a substitute page, we simply stop
294 where we are and drop the whole packet */
295 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
296 if (unlikely(err)) {
297 fp->eth_q_stats.rx_skb_alloc_failed++;
298 return err;
299 }
300
301 /* Unmap the page as we r going to pass it to the stack */
302 dma_unmap_page(&bp->pdev->dev,
303 dma_unmap_addr(&old_rx_pg, mapping),
304 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
305
306 /* Add one frag and update the appropriate fields in the skb */
307 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
308
309 skb->data_len += frag_len;
310 skb->truesize += frag_len;
311 skb->len += frag_len;
312
313 frag_size -= frag_len;
314 }
315
316 return 0;
317 }
318
319 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
320 u16 queue, int pad, int len, union eth_rx_cqe *cqe,
321 u16 cqe_idx)
322 {
323 struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
324 struct sk_buff *skb = rx_buf->skb;
325 /* alloc new skb */
326 struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
327
328 /* Unmap skb in the pool anyway, as we are going to change
329 pool entry status to BNX2X_TPA_STOP even if new skb allocation
330 fails. */
331 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
332 bp->rx_buf_size, DMA_FROM_DEVICE);
333
334 if (likely(new_skb)) {
335 /* fix ip xsum and give it to the stack */
336 /* (no need to map the new skb) */
337 #ifdef BCM_VLAN
338 int is_vlan_cqe =
339 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
340 PARSING_FLAGS_VLAN);
341 int is_not_hwaccel_vlan_cqe =
342 (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
343 #endif
344
345 prefetch(skb);
346 prefetch(((char *)(skb)) + 128);
347
348 #ifdef BNX2X_STOP_ON_ERROR
349 if (pad + len > bp->rx_buf_size) {
350 BNX2X_ERR("skb_put is about to fail... "
351 "pad %d len %d rx_buf_size %d\n",
352 pad, len, bp->rx_buf_size);
353 bnx2x_panic();
354 return;
355 }
356 #endif
357
358 skb_reserve(skb, pad);
359 skb_put(skb, len);
360
361 skb->protocol = eth_type_trans(skb, bp->dev);
362 skb->ip_summed = CHECKSUM_UNNECESSARY;
363
364 {
365 struct iphdr *iph;
366
367 iph = (struct iphdr *)skb->data;
368 #ifdef BCM_VLAN
369 /* If there is no Rx VLAN offloading -
370 take VLAN tag into an account */
371 if (unlikely(is_not_hwaccel_vlan_cqe))
372 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
373 #endif
374 iph->check = 0;
375 iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
376 }
377
378 if (!bnx2x_fill_frag_skb(bp, fp, skb,
379 &cqe->fast_path_cqe, cqe_idx)) {
380 #ifdef BCM_VLAN
381 if ((bp->vlgrp != NULL) && is_vlan_cqe &&
382 (!is_not_hwaccel_vlan_cqe))
383 vlan_gro_receive(&fp->napi, bp->vlgrp,
384 le16_to_cpu(cqe->fast_path_cqe.
385 vlan_tag), skb);
386 else
387 #endif
388 napi_gro_receive(&fp->napi, skb);
389 } else {
390 DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
391 " - dropping packet!\n");
392 dev_kfree_skb(skb);
393 }
394
395
396 /* put new skb in bin */
397 fp->tpa_pool[queue].skb = new_skb;
398
399 } else {
400 /* else drop the packet and keep the buffer in the bin */
401 DP(NETIF_MSG_RX_STATUS,
402 "Failed to allocate new skb - dropping packet!\n");
403 fp->eth_q_stats.rx_skb_alloc_failed++;
404 }
405
406 fp->tpa_state[queue] = BNX2X_TPA_STOP;
407 }
408
409 /* Set Toeplitz hash value in the skb using the value from the
410 * CQE (calculated by HW).
411 */
412 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
413 struct sk_buff *skb)
414 {
415 /* Set Toeplitz hash from CQE */
416 if ((bp->dev->features & NETIF_F_RXHASH) &&
417 (cqe->fast_path_cqe.status_flags &
418 ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
419 skb->rxhash =
420 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
421 }
422
423 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
424 {
425 struct bnx2x *bp = fp->bp;
426 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
427 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
428 int rx_pkt = 0;
429
430 #ifdef BNX2X_STOP_ON_ERROR
431 if (unlikely(bp->panic))
432 return 0;
433 #endif
434
435 /* CQ "next element" is of the size of the regular element,
436 that's why it's ok here */
437 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
438 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
439 hw_comp_cons++;
440
441 bd_cons = fp->rx_bd_cons;
442 bd_prod = fp->rx_bd_prod;
443 bd_prod_fw = bd_prod;
444 sw_comp_cons = fp->rx_comp_cons;
445 sw_comp_prod = fp->rx_comp_prod;
446
447 /* Memory barrier necessary as speculative reads of the rx
448 * buffer can be ahead of the index in the status block
449 */
450 rmb();
451
452 DP(NETIF_MSG_RX_STATUS,
453 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
454 fp->index, hw_comp_cons, sw_comp_cons);
455
456 while (sw_comp_cons != hw_comp_cons) {
457 struct sw_rx_bd *rx_buf = NULL;
458 struct sk_buff *skb;
459 union eth_rx_cqe *cqe;
460 u8 cqe_fp_flags;
461 u16 len, pad;
462
463 comp_ring_cons = RCQ_BD(sw_comp_cons);
464 bd_prod = RX_BD(bd_prod);
465 bd_cons = RX_BD(bd_cons);
466
467 /* Prefetch the page containing the BD descriptor
468 at producer's index. It will be needed when new skb is
469 allocated */
470 prefetch((void *)(PAGE_ALIGN((unsigned long)
471 (&fp->rx_desc_ring[bd_prod])) -
472 PAGE_SIZE + 1));
473
474 cqe = &fp->rx_comp_ring[comp_ring_cons];
475 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
476
477 DP(NETIF_MSG_RX_STATUS, "CQE type %x err %x status %x"
478 " queue %x vlan %x len %u\n", CQE_TYPE(cqe_fp_flags),
479 cqe_fp_flags, cqe->fast_path_cqe.status_flags,
480 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
481 le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
482 le16_to_cpu(cqe->fast_path_cqe.pkt_len));
483
484 /* is this a slowpath msg? */
485 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
486 bnx2x_sp_event(fp, cqe);
487 goto next_cqe;
488
489 /* this is an rx packet */
490 } else {
491 rx_buf = &fp->rx_buf_ring[bd_cons];
492 skb = rx_buf->skb;
493 prefetch(skb);
494 len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
495 pad = cqe->fast_path_cqe.placement_offset;
496
497 /* If CQE is marked both TPA_START and TPA_END
498 it is a non-TPA CQE */
499 if ((!fp->disable_tpa) &&
500 (TPA_TYPE(cqe_fp_flags) !=
501 (TPA_TYPE_START | TPA_TYPE_END))) {
502 u16 queue = cqe->fast_path_cqe.queue_index;
503
504 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
505 DP(NETIF_MSG_RX_STATUS,
506 "calling tpa_start on queue %d\n",
507 queue);
508
509 bnx2x_tpa_start(fp, queue, skb,
510 bd_cons, bd_prod);
511
512 /* Set Toeplitz hash for an LRO skb */
513 bnx2x_set_skb_rxhash(bp, cqe, skb);
514
515 goto next_rx;
516 }
517
518 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
519 DP(NETIF_MSG_RX_STATUS,
520 "calling tpa_stop on queue %d\n",
521 queue);
522
523 if (!BNX2X_RX_SUM_FIX(cqe))
524 BNX2X_ERR("STOP on none TCP "
525 "data\n");
526
527 /* This is a size of the linear data
528 on this skb */
529 len = le16_to_cpu(cqe->fast_path_cqe.
530 len_on_bd);
531 bnx2x_tpa_stop(bp, fp, queue, pad,
532 len, cqe, comp_ring_cons);
533 #ifdef BNX2X_STOP_ON_ERROR
534 if (bp->panic)
535 return 0;
536 #endif
537
538 bnx2x_update_sge_prod(fp,
539 &cqe->fast_path_cqe);
540 goto next_cqe;
541 }
542 }
543
544 dma_sync_single_for_device(&bp->pdev->dev,
545 dma_unmap_addr(rx_buf, mapping),
546 pad + RX_COPY_THRESH,
547 DMA_FROM_DEVICE);
548 prefetch(((char *)(skb)) + 128);
549
550 /* is this an error packet? */
551 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
552 DP(NETIF_MSG_RX_ERR,
553 "ERROR flags %x rx packet %u\n",
554 cqe_fp_flags, sw_comp_cons);
555 fp->eth_q_stats.rx_err_discard_pkt++;
556 goto reuse_rx;
557 }
558
559 /* Since we don't have a jumbo ring
560 * copy small packets if mtu > 1500
561 */
562 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
563 (len <= RX_COPY_THRESH)) {
564 struct sk_buff *new_skb;
565
566 new_skb = netdev_alloc_skb(bp->dev,
567 len + pad);
568 if (new_skb == NULL) {
569 DP(NETIF_MSG_RX_ERR,
570 "ERROR packet dropped "
571 "because of alloc failure\n");
572 fp->eth_q_stats.rx_skb_alloc_failed++;
573 goto reuse_rx;
574 }
575
576 /* aligned copy */
577 skb_copy_from_linear_data_offset(skb, pad,
578 new_skb->data + pad, len);
579 skb_reserve(new_skb, pad);
580 skb_put(new_skb, len);
581
582 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
583
584 skb = new_skb;
585
586 } else
587 if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
588 dma_unmap_single(&bp->pdev->dev,
589 dma_unmap_addr(rx_buf, mapping),
590 bp->rx_buf_size,
591 DMA_FROM_DEVICE);
592 skb_reserve(skb, pad);
593 skb_put(skb, len);
594
595 } else {
596 DP(NETIF_MSG_RX_ERR,
597 "ERROR packet dropped because "
598 "of alloc failure\n");
599 fp->eth_q_stats.rx_skb_alloc_failed++;
600 reuse_rx:
601 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
602 goto next_rx;
603 }
604
605 skb->protocol = eth_type_trans(skb, bp->dev);
606
607 /* Set Toeplitz hash for a none-LRO skb */
608 bnx2x_set_skb_rxhash(bp, cqe, skb);
609
610 skb->ip_summed = CHECKSUM_NONE;
611 if (bp->rx_csum) {
612 if (likely(BNX2X_RX_CSUM_OK(cqe)))
613 skb->ip_summed = CHECKSUM_UNNECESSARY;
614 else
615 fp->eth_q_stats.hw_csum_err++;
616 }
617 }
618
619 skb_record_rx_queue(skb, fp->index);
620
621 #ifdef BCM_VLAN
622 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
623 (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
624 PARSING_FLAGS_VLAN))
625 vlan_gro_receive(&fp->napi, bp->vlgrp,
626 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
627 else
628 #endif
629 napi_gro_receive(&fp->napi, skb);
630
631
632 next_rx:
633 rx_buf->skb = NULL;
634
635 bd_cons = NEXT_RX_IDX(bd_cons);
636 bd_prod = NEXT_RX_IDX(bd_prod);
637 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
638 rx_pkt++;
639 next_cqe:
640 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
641 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
642
643 if (rx_pkt == budget)
644 break;
645 } /* while */
646
647 fp->rx_bd_cons = bd_cons;
648 fp->rx_bd_prod = bd_prod_fw;
649 fp->rx_comp_cons = sw_comp_cons;
650 fp->rx_comp_prod = sw_comp_prod;
651
652 /* Update producers */
653 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
654 fp->rx_sge_prod);
655
656 fp->rx_pkt += rx_pkt;
657 fp->rx_calls++;
658
659 return rx_pkt;
660 }
661
662 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
663 {
664 struct bnx2x_fastpath *fp = fp_cookie;
665 struct bnx2x *bp = fp->bp;
666
667 /* Return here if interrupt is disabled */
668 if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
669 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
670 return IRQ_HANDLED;
671 }
672
673 DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
674 fp->index, fp->sb_id);
675 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
676
677 #ifdef BNX2X_STOP_ON_ERROR
678 if (unlikely(bp->panic))
679 return IRQ_HANDLED;
680 #endif
681
682 /* Handle Rx and Tx according to MSI-X vector */
683 prefetch(fp->rx_cons_sb);
684 prefetch(fp->tx_cons_sb);
685 prefetch(&fp->status_blk->u_status_block.status_block_index);
686 prefetch(&fp->status_blk->c_status_block.status_block_index);
687 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
688
689 return IRQ_HANDLED;
690 }
691
692
693 /* HW Lock for shared dual port PHYs */
694 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
695 {
696 mutex_lock(&bp->port.phy_mutex);
697
698 if (bp->port.need_hw_lock)
699 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
700 }
701
702 void bnx2x_release_phy_lock(struct bnx2x *bp)
703 {
704 if (bp->port.need_hw_lock)
705 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
706
707 mutex_unlock(&bp->port.phy_mutex);
708 }
709
710 void bnx2x_link_report(struct bnx2x *bp)
711 {
712 if (bp->flags & MF_FUNC_DIS) {
713 netif_carrier_off(bp->dev);
714 netdev_err(bp->dev, "NIC Link is Down\n");
715 return;
716 }
717
718 if (bp->link_vars.link_up) {
719 u16 line_speed;
720
721 if (bp->state == BNX2X_STATE_OPEN)
722 netif_carrier_on(bp->dev);
723 netdev_info(bp->dev, "NIC Link is Up, ");
724
725 line_speed = bp->link_vars.line_speed;
726 if (IS_E1HMF(bp)) {
727 u16 vn_max_rate;
728
729 vn_max_rate =
730 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
731 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
732 if (vn_max_rate < line_speed)
733 line_speed = vn_max_rate;
734 }
735 pr_cont("%d Mbps ", line_speed);
736
737 if (bp->link_vars.duplex == DUPLEX_FULL)
738 pr_cont("full duplex");
739 else
740 pr_cont("half duplex");
741
742 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
743 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
744 pr_cont(", receive ");
745 if (bp->link_vars.flow_ctrl &
746 BNX2X_FLOW_CTRL_TX)
747 pr_cont("& transmit ");
748 } else {
749 pr_cont(", transmit ");
750 }
751 pr_cont("flow control ON");
752 }
753 pr_cont("\n");
754
755 } else { /* link_down */
756 netif_carrier_off(bp->dev);
757 netdev_err(bp->dev, "NIC Link is Down\n");
758 }
759 }
760
761 void bnx2x_init_rx_rings(struct bnx2x *bp)
762 {
763 int func = BP_FUNC(bp);
764 int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
765 ETH_MAX_AGGREGATION_QUEUES_E1H;
766 u16 ring_prod, cqe_ring_prod;
767 int i, j;
768
769 bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
770 DP(NETIF_MSG_IFUP,
771 "mtu %d rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
772
773 if (bp->flags & TPA_ENABLE_FLAG) {
774
775 for_each_queue(bp, j) {
776 struct bnx2x_fastpath *fp = &bp->fp[j];
777
778 for (i = 0; i < max_agg_queues; i++) {
779 fp->tpa_pool[i].skb =
780 netdev_alloc_skb(bp->dev, bp->rx_buf_size);
781 if (!fp->tpa_pool[i].skb) {
782 BNX2X_ERR("Failed to allocate TPA "
783 "skb pool for queue[%d] - "
784 "disabling TPA on this "
785 "queue!\n", j);
786 bnx2x_free_tpa_pool(bp, fp, i);
787 fp->disable_tpa = 1;
788 break;
789 }
790 dma_unmap_addr_set((struct sw_rx_bd *)
791 &bp->fp->tpa_pool[i],
792 mapping, 0);
793 fp->tpa_state[i] = BNX2X_TPA_STOP;
794 }
795 }
796 }
797
798 for_each_queue(bp, j) {
799 struct bnx2x_fastpath *fp = &bp->fp[j];
800
801 fp->rx_bd_cons = 0;
802 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
803 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
804
805 /* "next page" elements initialization */
806 /* SGE ring */
807 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
808 struct eth_rx_sge *sge;
809
810 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
811 sge->addr_hi =
812 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
813 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
814 sge->addr_lo =
815 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
816 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
817 }
818
819 bnx2x_init_sge_ring_bit_mask(fp);
820
821 /* RX BD ring */
822 for (i = 1; i <= NUM_RX_RINGS; i++) {
823 struct eth_rx_bd *rx_bd;
824
825 rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
826 rx_bd->addr_hi =
827 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
828 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
829 rx_bd->addr_lo =
830 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
831 BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
832 }
833
834 /* CQ ring */
835 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
836 struct eth_rx_cqe_next_page *nextpg;
837
838 nextpg = (struct eth_rx_cqe_next_page *)
839 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
840 nextpg->addr_hi =
841 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
842 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
843 nextpg->addr_lo =
844 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
845 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
846 }
847
848 /* Allocate SGEs and initialize the ring elements */
849 for (i = 0, ring_prod = 0;
850 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
851
852 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
853 BNX2X_ERR("was only able to allocate "
854 "%d rx sges\n", i);
855 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
856 /* Cleanup already allocated elements */
857 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
858 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
859 fp->disable_tpa = 1;
860 ring_prod = 0;
861 break;
862 }
863 ring_prod = NEXT_SGE_IDX(ring_prod);
864 }
865 fp->rx_sge_prod = ring_prod;
866
867 /* Allocate BDs and initialize BD ring */
868 fp->rx_comp_cons = 0;
869 cqe_ring_prod = ring_prod = 0;
870 for (i = 0; i < bp->rx_ring_size; i++) {
871 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
872 BNX2X_ERR("was only able to allocate "
873 "%d rx skbs on queue[%d]\n", i, j);
874 fp->eth_q_stats.rx_skb_alloc_failed++;
875 break;
876 }
877 ring_prod = NEXT_RX_IDX(ring_prod);
878 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
879 WARN_ON(ring_prod <= i);
880 }
881
882 fp->rx_bd_prod = ring_prod;
883 /* must not have more available CQEs than BDs */
884 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
885 cqe_ring_prod);
886 fp->rx_pkt = fp->rx_calls = 0;
887
888 /* Warning!
889 * this will generate an interrupt (to the TSTORM)
890 * must only be done after chip is initialized
891 */
892 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
893 fp->rx_sge_prod);
894 if (j != 0)
895 continue;
896
897 REG_WR(bp, BAR_USTRORM_INTMEM +
898 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
899 U64_LO(fp->rx_comp_mapping));
900 REG_WR(bp, BAR_USTRORM_INTMEM +
901 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
902 U64_HI(fp->rx_comp_mapping));
903 }
904 }
905 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
906 {
907 int i;
908
909 for_each_queue(bp, i) {
910 struct bnx2x_fastpath *fp = &bp->fp[i];
911
912 u16 bd_cons = fp->tx_bd_cons;
913 u16 sw_prod = fp->tx_pkt_prod;
914 u16 sw_cons = fp->tx_pkt_cons;
915
916 while (sw_cons != sw_prod) {
917 bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
918 sw_cons++;
919 }
920 }
921 }
922
923 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
924 {
925 int i, j;
926
927 for_each_queue(bp, j) {
928 struct bnx2x_fastpath *fp = &bp->fp[j];
929
930 for (i = 0; i < NUM_RX_BD; i++) {
931 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
932 struct sk_buff *skb = rx_buf->skb;
933
934 if (skb == NULL)
935 continue;
936
937 dma_unmap_single(&bp->pdev->dev,
938 dma_unmap_addr(rx_buf, mapping),
939 bp->rx_buf_size, DMA_FROM_DEVICE);
940
941 rx_buf->skb = NULL;
942 dev_kfree_skb(skb);
943 }
944 if (!fp->disable_tpa)
945 bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
946 ETH_MAX_AGGREGATION_QUEUES_E1 :
947 ETH_MAX_AGGREGATION_QUEUES_E1H);
948 }
949 }
950
951 void bnx2x_free_skbs(struct bnx2x *bp)
952 {
953 bnx2x_free_tx_skbs(bp);
954 bnx2x_free_rx_skbs(bp);
955 }
956
957 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
958 {
959 int i, offset = 1;
960
961 free_irq(bp->msix_table[0].vector, bp->dev);
962 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
963 bp->msix_table[0].vector);
964
965 #ifdef BCM_CNIC
966 offset++;
967 #endif
968 for_each_queue(bp, i) {
969 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq "
970 "state %x\n", i, bp->msix_table[i + offset].vector,
971 bnx2x_fp(bp, i, state));
972
973 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
974 }
975 }
976
977 void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
978 {
979 if (bp->flags & USING_MSIX_FLAG) {
980 if (!disable_only)
981 bnx2x_free_msix_irqs(bp);
982 pci_disable_msix(bp->pdev);
983 bp->flags &= ~USING_MSIX_FLAG;
984
985 } else if (bp->flags & USING_MSI_FLAG) {
986 if (!disable_only)
987 free_irq(bp->pdev->irq, bp->dev);
988 pci_disable_msi(bp->pdev);
989 bp->flags &= ~USING_MSI_FLAG;
990
991 } else if (!disable_only)
992 free_irq(bp->pdev->irq, bp->dev);
993 }
994
995 static int bnx2x_enable_msix(struct bnx2x *bp)
996 {
997 int i, rc, offset = 1;
998 int igu_vec = 0;
999
1000 bp->msix_table[0].entry = igu_vec;
1001 DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1002
1003 #ifdef BCM_CNIC
1004 igu_vec = BP_L_ID(bp) + offset;
1005 bp->msix_table[1].entry = igu_vec;
1006 DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1007 offset++;
1008 #endif
1009 for_each_queue(bp, i) {
1010 igu_vec = BP_L_ID(bp) + offset + i;
1011 bp->msix_table[i + offset].entry = igu_vec;
1012 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1013 "(fastpath #%u)\n", i + offset, igu_vec, i);
1014 }
1015
1016 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1017 BNX2X_NUM_QUEUES(bp) + offset);
1018
1019 /*
1020 * reconfigure number of tx/rx queues according to available
1021 * MSI-X vectors
1022 */
1023 if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1024 /* vectors available for FP */
1025 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1026
1027 DP(NETIF_MSG_IFUP,
1028 "Trying to use less MSI-X vectors: %d\n", rc);
1029
1030 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1031
1032 if (rc) {
1033 DP(NETIF_MSG_IFUP,
1034 "MSI-X is not attainable rc %d\n", rc);
1035 return rc;
1036 }
1037
1038 bp->num_queues = min(bp->num_queues, fp_vec);
1039
1040 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1041 bp->num_queues);
1042 } else if (rc) {
1043 DP(NETIF_MSG_IFUP, "MSI-X is not attainable rc %d\n", rc);
1044 return rc;
1045 }
1046
1047 bp->flags |= USING_MSIX_FLAG;
1048
1049 return 0;
1050 }
1051
1052 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1053 {
1054 int i, rc, offset = 1;
1055
1056 rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1057 bp->dev->name, bp->dev);
1058 if (rc) {
1059 BNX2X_ERR("request sp irq failed\n");
1060 return -EBUSY;
1061 }
1062
1063 #ifdef BCM_CNIC
1064 offset++;
1065 #endif
1066 for_each_queue(bp, i) {
1067 struct bnx2x_fastpath *fp = &bp->fp[i];
1068 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1069 bp->dev->name, i);
1070
1071 rc = request_irq(bp->msix_table[i + offset].vector,
1072 bnx2x_msix_fp_int, 0, fp->name, fp);
1073 if (rc) {
1074 BNX2X_ERR("request fp #%d irq failed rc %d\n", i, rc);
1075 bnx2x_free_msix_irqs(bp);
1076 return -EBUSY;
1077 }
1078
1079 fp->state = BNX2X_FP_STATE_IRQ;
1080 }
1081
1082 i = BNX2X_NUM_QUEUES(bp);
1083 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d"
1084 " ... fp[%d] %d\n",
1085 bp->msix_table[0].vector,
1086 0, bp->msix_table[offset].vector,
1087 i - 1, bp->msix_table[offset + i - 1].vector);
1088
1089 return 0;
1090 }
1091
1092 static int bnx2x_enable_msi(struct bnx2x *bp)
1093 {
1094 int rc;
1095
1096 rc = pci_enable_msi(bp->pdev);
1097 if (rc) {
1098 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1099 return -1;
1100 }
1101 bp->flags |= USING_MSI_FLAG;
1102
1103 return 0;
1104 }
1105
1106 static int bnx2x_req_irq(struct bnx2x *bp)
1107 {
1108 unsigned long flags;
1109 int rc;
1110
1111 if (bp->flags & USING_MSI_FLAG)
1112 flags = 0;
1113 else
1114 flags = IRQF_SHARED;
1115
1116 rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1117 bp->dev->name, bp->dev);
1118 if (!rc)
1119 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1120
1121 return rc;
1122 }
1123
1124 static void bnx2x_napi_enable(struct bnx2x *bp)
1125 {
1126 int i;
1127
1128 for_each_queue(bp, i)
1129 napi_enable(&bnx2x_fp(bp, i, napi));
1130 }
1131
1132 static void bnx2x_napi_disable(struct bnx2x *bp)
1133 {
1134 int i;
1135
1136 for_each_queue(bp, i)
1137 napi_disable(&bnx2x_fp(bp, i, napi));
1138 }
1139
1140 void bnx2x_netif_start(struct bnx2x *bp)
1141 {
1142 int intr_sem;
1143
1144 intr_sem = atomic_dec_and_test(&bp->intr_sem);
1145 smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1146
1147 if (intr_sem) {
1148 if (netif_running(bp->dev)) {
1149 bnx2x_napi_enable(bp);
1150 bnx2x_int_enable(bp);
1151 if (bp->state == BNX2X_STATE_OPEN)
1152 netif_tx_wake_all_queues(bp->dev);
1153 }
1154 }
1155 }
1156
1157 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1158 {
1159 bnx2x_int_disable_sync(bp, disable_hw);
1160 bnx2x_napi_disable(bp);
1161 netif_tx_disable(bp->dev);
1162 }
1163 static int bnx2x_set_num_queues(struct bnx2x *bp)
1164 {
1165 int rc = 0;
1166
1167 switch (bp->int_mode) {
1168 case INT_MODE_INTx:
1169 case INT_MODE_MSI:
1170 bp->num_queues = 1;
1171 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1172 break;
1173 default:
1174 /* Set number of queues according to bp->multi_mode value */
1175 bnx2x_set_num_queues_msix(bp);
1176
1177 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1178 bp->num_queues);
1179
1180 /* if we can't use MSI-X we only need one fp,
1181 * so try to enable MSI-X with the requested number of fp's
1182 * and fallback to MSI or legacy INTx with one fp
1183 */
1184 rc = bnx2x_enable_msix(bp);
1185 if (rc)
1186 /* failed to enable MSI-X */
1187 bp->num_queues = 1;
1188 break;
1189 }
1190 bp->dev->real_num_tx_queues = bp->num_queues;
1191 return rc;
1192 }
1193
1194 /* must be called with rtnl_lock */
1195 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1196 {
1197 u32 load_code;
1198 int i, rc;
1199
1200 #ifdef BNX2X_STOP_ON_ERROR
1201 if (unlikely(bp->panic))
1202 return -EPERM;
1203 #endif
1204
1205 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1206
1207 rc = bnx2x_set_num_queues(bp);
1208
1209 if (bnx2x_alloc_mem(bp)) {
1210 bnx2x_free_irq(bp, true);
1211 return -ENOMEM;
1212 }
1213
1214 for_each_queue(bp, i)
1215 bnx2x_fp(bp, i, disable_tpa) =
1216 ((bp->flags & TPA_ENABLE_FLAG) == 0);
1217
1218 for_each_queue(bp, i)
1219 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1220 bnx2x_poll, 128);
1221
1222 bnx2x_napi_enable(bp);
1223
1224 if (bp->flags & USING_MSIX_FLAG) {
1225 rc = bnx2x_req_msix_irqs(bp);
1226 if (rc) {
1227 bnx2x_free_irq(bp, true);
1228 goto load_error1;
1229 }
1230 } else {
1231 /* Fall to INTx if failed to enable MSI-X due to lack of
1232 memory (in bnx2x_set_num_queues()) */
1233 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1234 bnx2x_enable_msi(bp);
1235 bnx2x_ack_int(bp);
1236 rc = bnx2x_req_irq(bp);
1237 if (rc) {
1238 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1239 bnx2x_free_irq(bp, true);
1240 goto load_error1;
1241 }
1242 if (bp->flags & USING_MSI_FLAG) {
1243 bp->dev->irq = bp->pdev->irq;
1244 netdev_info(bp->dev, "using MSI IRQ %d\n",
1245 bp->pdev->irq);
1246 }
1247 }
1248
1249 /* Send LOAD_REQUEST command to MCP
1250 Returns the type of LOAD command:
1251 if it is the first port to be initialized
1252 common blocks should be initialized, otherwise - not
1253 */
1254 if (!BP_NOMCP(bp)) {
1255 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ);
1256 if (!load_code) {
1257 BNX2X_ERR("MCP response failure, aborting\n");
1258 rc = -EBUSY;
1259 goto load_error2;
1260 }
1261 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1262 rc = -EBUSY; /* other port in diagnostic mode */
1263 goto load_error2;
1264 }
1265
1266 } else {
1267 int port = BP_PORT(bp);
1268
1269 DP(NETIF_MSG_IFUP, "NO MCP - load counts %d, %d, %d\n",
1270 load_count[0], load_count[1], load_count[2]);
1271 load_count[0]++;
1272 load_count[1 + port]++;
1273 DP(NETIF_MSG_IFUP, "NO MCP - new load counts %d, %d, %d\n",
1274 load_count[0], load_count[1], load_count[2]);
1275 if (load_count[0] == 1)
1276 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1277 else if (load_count[1 + port] == 1)
1278 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1279 else
1280 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1281 }
1282
1283 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1284 (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1285 bp->port.pmf = 1;
1286 else
1287 bp->port.pmf = 0;
1288 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1289
1290 /* Initialize HW */
1291 rc = bnx2x_init_hw(bp, load_code);
1292 if (rc) {
1293 BNX2X_ERR("HW init failed, aborting\n");
1294 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1295 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1296 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1297 goto load_error2;
1298 }
1299
1300 /* Setup NIC internals and enable interrupts */
1301 bnx2x_nic_init(bp, load_code);
1302
1303 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1304 (bp->common.shmem2_base))
1305 SHMEM2_WR(bp, dcc_support,
1306 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1307 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1308
1309 /* Send LOAD_DONE command to MCP */
1310 if (!BP_NOMCP(bp)) {
1311 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE);
1312 if (!load_code) {
1313 BNX2X_ERR("MCP response failure, aborting\n");
1314 rc = -EBUSY;
1315 goto load_error3;
1316 }
1317 }
1318
1319 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1320
1321 rc = bnx2x_setup_leading(bp);
1322 if (rc) {
1323 BNX2X_ERR("Setup leading failed!\n");
1324 #ifndef BNX2X_STOP_ON_ERROR
1325 goto load_error3;
1326 #else
1327 bp->panic = 1;
1328 return -EBUSY;
1329 #endif
1330 }
1331
1332 if (CHIP_IS_E1H(bp))
1333 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1334 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1335 bp->flags |= MF_FUNC_DIS;
1336 }
1337
1338 if (bp->state == BNX2X_STATE_OPEN) {
1339 #ifdef BCM_CNIC
1340 /* Enable Timer scan */
1341 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1342 #endif
1343 for_each_nondefault_queue(bp, i) {
1344 rc = bnx2x_setup_multi(bp, i);
1345 if (rc)
1346 #ifdef BCM_CNIC
1347 goto load_error4;
1348 #else
1349 goto load_error3;
1350 #endif
1351 }
1352
1353 if (CHIP_IS_E1(bp))
1354 bnx2x_set_eth_mac_addr_e1(bp, 1);
1355 else
1356 bnx2x_set_eth_mac_addr_e1h(bp, 1);
1357 #ifdef BCM_CNIC
1358 /* Set iSCSI L2 MAC */
1359 mutex_lock(&bp->cnic_mutex);
1360 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1361 bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1362 bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1363 bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1364 CNIC_SB_ID(bp));
1365 }
1366 mutex_unlock(&bp->cnic_mutex);
1367 #endif
1368 }
1369
1370 if (bp->port.pmf)
1371 bnx2x_initial_phy_init(bp, load_mode);
1372
1373 /* Start fast path */
1374 switch (load_mode) {
1375 case LOAD_NORMAL:
1376 if (bp->state == BNX2X_STATE_OPEN) {
1377 /* Tx queue should be only reenabled */
1378 netif_tx_wake_all_queues(bp->dev);
1379 }
1380 /* Initialize the receive filter. */
1381 bnx2x_set_rx_mode(bp->dev);
1382 break;
1383
1384 case LOAD_OPEN:
1385 netif_tx_start_all_queues(bp->dev);
1386 if (bp->state != BNX2X_STATE_OPEN)
1387 netif_tx_disable(bp->dev);
1388 /* Initialize the receive filter. */
1389 bnx2x_set_rx_mode(bp->dev);
1390 break;
1391
1392 case LOAD_DIAG:
1393 /* Initialize the receive filter. */
1394 bnx2x_set_rx_mode(bp->dev);
1395 bp->state = BNX2X_STATE_DIAG;
1396 break;
1397
1398 default:
1399 break;
1400 }
1401
1402 if (!bp->port.pmf)
1403 bnx2x__link_status_update(bp);
1404
1405 /* start the timer */
1406 mod_timer(&bp->timer, jiffies + bp->current_interval);
1407
1408 #ifdef BCM_CNIC
1409 bnx2x_setup_cnic_irq_info(bp);
1410 if (bp->state == BNX2X_STATE_OPEN)
1411 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1412 #endif
1413 bnx2x_inc_load_cnt(bp);
1414
1415 return 0;
1416
1417 #ifdef BCM_CNIC
1418 load_error4:
1419 /* Disable Timer scan */
1420 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1421 #endif
1422 load_error3:
1423 bnx2x_int_disable_sync(bp, 1);
1424 if (!BP_NOMCP(bp)) {
1425 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP);
1426 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE);
1427 }
1428 bp->port.pmf = 0;
1429 /* Free SKBs, SGEs, TPA pool and driver internals */
1430 bnx2x_free_skbs(bp);
1431 for_each_queue(bp, i)
1432 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1433 load_error2:
1434 /* Release IRQs */
1435 bnx2x_free_irq(bp, false);
1436 load_error1:
1437 bnx2x_napi_disable(bp);
1438 for_each_queue(bp, i)
1439 netif_napi_del(&bnx2x_fp(bp, i, napi));
1440 bnx2x_free_mem(bp);
1441
1442 return rc;
1443 }
1444
1445 /* must be called with rtnl_lock */
1446 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1447 {
1448 int i;
1449
1450 if (bp->state == BNX2X_STATE_CLOSED) {
1451 /* Interface has been removed - nothing to recover */
1452 bp->recovery_state = BNX2X_RECOVERY_DONE;
1453 bp->is_leader = 0;
1454 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1455 smp_wmb();
1456
1457 return -EINVAL;
1458 }
1459
1460 #ifdef BCM_CNIC
1461 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1462 #endif
1463 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1464
1465 /* Set "drop all" */
1466 bp->rx_mode = BNX2X_RX_MODE_NONE;
1467 bnx2x_set_storm_rx_mode(bp);
1468
1469 /* Disable HW interrupts, NAPI and Tx */
1470 bnx2x_netif_stop(bp, 1);
1471 netif_carrier_off(bp->dev);
1472
1473 del_timer_sync(&bp->timer);
1474 SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1475 (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1476 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1477
1478 /* Release IRQs */
1479 bnx2x_free_irq(bp, false);
1480
1481 /* Cleanup the chip if needed */
1482 if (unload_mode != UNLOAD_RECOVERY)
1483 bnx2x_chip_cleanup(bp, unload_mode);
1484
1485 bp->port.pmf = 0;
1486
1487 /* Free SKBs, SGEs, TPA pool and driver internals */
1488 bnx2x_free_skbs(bp);
1489 for_each_queue(bp, i)
1490 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1491 for_each_queue(bp, i)
1492 netif_napi_del(&bnx2x_fp(bp, i, napi));
1493 bnx2x_free_mem(bp);
1494
1495 bp->state = BNX2X_STATE_CLOSED;
1496
1497 /* The last driver must disable a "close the gate" if there is no
1498 * parity attention or "process kill" pending.
1499 */
1500 if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1501 bnx2x_reset_is_done(bp))
1502 bnx2x_disable_close_the_gate(bp);
1503
1504 /* Reset MCP mail box sequence if there is on going recovery */
1505 if (unload_mode == UNLOAD_RECOVERY)
1506 bp->fw_seq = 0;
1507
1508 return 0;
1509 }
1510 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1511 {
1512 u16 pmcsr;
1513
1514 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1515
1516 switch (state) {
1517 case PCI_D0:
1518 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1519 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1520 PCI_PM_CTRL_PME_STATUS));
1521
1522 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1523 /* delay required during transition out of D3hot */
1524 msleep(20);
1525 break;
1526
1527 case PCI_D3hot:
1528 /* If there are other clients above don't
1529 shut down the power */
1530 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1531 return 0;
1532 /* Don't shut down the power for emulation and FPGA */
1533 if (CHIP_REV_IS_SLOW(bp))
1534 return 0;
1535
1536 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1537 pmcsr |= 3;
1538
1539 if (bp->wol)
1540 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1541
1542 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1543 pmcsr);
1544
1545 /* No more memory access after this point until
1546 * device is brought back to D0.
1547 */
1548 break;
1549
1550 default:
1551 return -EINVAL;
1552 }
1553 return 0;
1554 }
1555
1556
1557
1558 /*
1559 * net_device service functions
1560 */
1561
1562 static int bnx2x_poll(struct napi_struct *napi, int budget)
1563 {
1564 int work_done = 0;
1565 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1566 napi);
1567 struct bnx2x *bp = fp->bp;
1568
1569 while (1) {
1570 #ifdef BNX2X_STOP_ON_ERROR
1571 if (unlikely(bp->panic)) {
1572 napi_complete(napi);
1573 return 0;
1574 }
1575 #endif
1576
1577 if (bnx2x_has_tx_work(fp))
1578 bnx2x_tx_int(fp);
1579
1580 if (bnx2x_has_rx_work(fp)) {
1581 work_done += bnx2x_rx_int(fp, budget - work_done);
1582
1583 /* must not complete if we consumed full budget */
1584 if (work_done >= budget)
1585 break;
1586 }
1587
1588 /* Fall out from the NAPI loop if needed */
1589 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1590 bnx2x_update_fpsb_idx(fp);
1591 /* bnx2x_has_rx_work() reads the status block, thus we need
1592 * to ensure that status block indices have been actually read
1593 * (bnx2x_update_fpsb_idx) prior to this check
1594 * (bnx2x_has_rx_work) so that we won't write the "newer"
1595 * value of the status block to IGU (if there was a DMA right
1596 * after bnx2x_has_rx_work and if there is no rmb, the memory
1597 * reading (bnx2x_update_fpsb_idx) may be postponed to right
1598 * before bnx2x_ack_sb). In this case there will never be
1599 * another interrupt until there is another update of the
1600 * status block, while there is still unhandled work.
1601 */
1602 rmb();
1603
1604 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1605 napi_complete(napi);
1606 /* Re-enable interrupts */
1607 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1608 le16_to_cpu(fp->fp_c_idx),
1609 IGU_INT_NOP, 1);
1610 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1611 le16_to_cpu(fp->fp_u_idx),
1612 IGU_INT_ENABLE, 1);
1613 break;
1614 }
1615 }
1616 }
1617
1618 return work_done;
1619 }
1620
1621
1622 /* we split the first BD into headers and data BDs
1623 * to ease the pain of our fellow microcode engineers
1624 * we use one mapping for both BDs
1625 * So far this has only been observed to happen
1626 * in Other Operating Systems(TM)
1627 */
1628 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1629 struct bnx2x_fastpath *fp,
1630 struct sw_tx_bd *tx_buf,
1631 struct eth_tx_start_bd **tx_bd, u16 hlen,
1632 u16 bd_prod, int nbd)
1633 {
1634 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1635 struct eth_tx_bd *d_tx_bd;
1636 dma_addr_t mapping;
1637 int old_len = le16_to_cpu(h_tx_bd->nbytes);
1638
1639 /* first fix first BD */
1640 h_tx_bd->nbd = cpu_to_le16(nbd);
1641 h_tx_bd->nbytes = cpu_to_le16(hlen);
1642
1643 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1644 "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1645 h_tx_bd->addr_lo, h_tx_bd->nbd);
1646
1647 /* now get a new data BD
1648 * (after the pbd) and fill it */
1649 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1650 d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1651
1652 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1653 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1654
1655 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1656 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1657 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1658
1659 /* this marks the BD as one that has no individual mapping */
1660 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1661
1662 DP(NETIF_MSG_TX_QUEUED,
1663 "TSO split data size is %d (%x:%x)\n",
1664 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1665
1666 /* update tx_bd */
1667 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1668
1669 return bd_prod;
1670 }
1671
1672 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1673 {
1674 if (fix > 0)
1675 csum = (u16) ~csum_fold(csum_sub(csum,
1676 csum_partial(t_header - fix, fix, 0)));
1677
1678 else if (fix < 0)
1679 csum = (u16) ~csum_fold(csum_add(csum,
1680 csum_partial(t_header, -fix, 0)));
1681
1682 return swab16(csum);
1683 }
1684
1685 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1686 {
1687 u32 rc;
1688
1689 if (skb->ip_summed != CHECKSUM_PARTIAL)
1690 rc = XMIT_PLAIN;
1691
1692 else {
1693 if (skb->protocol == htons(ETH_P_IPV6)) {
1694 rc = XMIT_CSUM_V6;
1695 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1696 rc |= XMIT_CSUM_TCP;
1697
1698 } else {
1699 rc = XMIT_CSUM_V4;
1700 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1701 rc |= XMIT_CSUM_TCP;
1702 }
1703 }
1704
1705 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1706 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1707
1708 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1709 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1710
1711 return rc;
1712 }
1713
1714 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1715 /* check if packet requires linearization (packet is too fragmented)
1716 no need to check fragmentation if page size > 8K (there will be no
1717 violation to FW restrictions) */
1718 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1719 u32 xmit_type)
1720 {
1721 int to_copy = 0;
1722 int hlen = 0;
1723 int first_bd_sz = 0;
1724
1725 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1726 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1727
1728 if (xmit_type & XMIT_GSO) {
1729 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1730 /* Check if LSO packet needs to be copied:
1731 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1732 int wnd_size = MAX_FETCH_BD - 3;
1733 /* Number of windows to check */
1734 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1735 int wnd_idx = 0;
1736 int frag_idx = 0;
1737 u32 wnd_sum = 0;
1738
1739 /* Headers length */
1740 hlen = (int)(skb_transport_header(skb) - skb->data) +
1741 tcp_hdrlen(skb);
1742
1743 /* Amount of data (w/o headers) on linear part of SKB*/
1744 first_bd_sz = skb_headlen(skb) - hlen;
1745
1746 wnd_sum = first_bd_sz;
1747
1748 /* Calculate the first sum - it's special */
1749 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1750 wnd_sum +=
1751 skb_shinfo(skb)->frags[frag_idx].size;
1752
1753 /* If there was data on linear skb data - check it */
1754 if (first_bd_sz > 0) {
1755 if (unlikely(wnd_sum < lso_mss)) {
1756 to_copy = 1;
1757 goto exit_lbl;
1758 }
1759
1760 wnd_sum -= first_bd_sz;
1761 }
1762
1763 /* Others are easier: run through the frag list and
1764 check all windows */
1765 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1766 wnd_sum +=
1767 skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1768
1769 if (unlikely(wnd_sum < lso_mss)) {
1770 to_copy = 1;
1771 break;
1772 }
1773 wnd_sum -=
1774 skb_shinfo(skb)->frags[wnd_idx].size;
1775 }
1776 } else {
1777 /* in non-LSO too fragmented packet should always
1778 be linearized */
1779 to_copy = 1;
1780 }
1781 }
1782
1783 exit_lbl:
1784 if (unlikely(to_copy))
1785 DP(NETIF_MSG_TX_QUEUED,
1786 "Linearization IS REQUIRED for %s packet. "
1787 "num_frags %d hlen %d first_bd_sz %d\n",
1788 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1789 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1790
1791 return to_copy;
1792 }
1793 #endif
1794
1795 /* called with netif_tx_lock
1796 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1797 * netif_wake_queue()
1798 */
1799 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1800 {
1801 struct bnx2x *bp = netdev_priv(dev);
1802 struct bnx2x_fastpath *fp;
1803 struct netdev_queue *txq;
1804 struct sw_tx_bd *tx_buf;
1805 struct eth_tx_start_bd *tx_start_bd;
1806 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1807 struct eth_tx_parse_bd *pbd = NULL;
1808 u16 pkt_prod, bd_prod;
1809 int nbd, fp_index;
1810 dma_addr_t mapping;
1811 u32 xmit_type = bnx2x_xmit_type(bp, skb);
1812 int i;
1813 u8 hlen = 0;
1814 __le16 pkt_size = 0;
1815 struct ethhdr *eth;
1816 u8 mac_type = UNICAST_ADDRESS;
1817
1818 #ifdef BNX2X_STOP_ON_ERROR
1819 if (unlikely(bp->panic))
1820 return NETDEV_TX_BUSY;
1821 #endif
1822
1823 fp_index = skb_get_queue_mapping(skb);
1824 txq = netdev_get_tx_queue(dev, fp_index);
1825
1826 fp = &bp->fp[fp_index];
1827
1828 if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1829 fp->eth_q_stats.driver_xoff++;
1830 netif_tx_stop_queue(txq);
1831 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1832 return NETDEV_TX_BUSY;
1833 }
1834
1835 DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x protocol %x protocol(%x,%x)"
1836 " gso type %x xmit_type %x\n",
1837 skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1838 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1839
1840 eth = (struct ethhdr *)skb->data;
1841
1842 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1843 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1844 if (is_broadcast_ether_addr(eth->h_dest))
1845 mac_type = BROADCAST_ADDRESS;
1846 else
1847 mac_type = MULTICAST_ADDRESS;
1848 }
1849
1850 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1851 /* First, check if we need to linearize the skb (due to FW
1852 restrictions). No need to check fragmentation if page size > 8K
1853 (there will be no violation to FW restrictions) */
1854 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1855 /* Statistics of linearization */
1856 bp->lin_cnt++;
1857 if (skb_linearize(skb) != 0) {
1858 DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1859 "silently dropping this SKB\n");
1860 dev_kfree_skb_any(skb);
1861 return NETDEV_TX_OK;
1862 }
1863 }
1864 #endif
1865
1866 /*
1867 Please read carefully. First we use one BD which we mark as start,
1868 then we have a parsing info BD (used for TSO or xsum),
1869 and only then we have the rest of the TSO BDs.
1870 (don't forget to mark the last one as last,
1871 and to unmap only AFTER you write to the BD ...)
1872 And above all, all pdb sizes are in words - NOT DWORDS!
1873 */
1874
1875 pkt_prod = fp->tx_pkt_prod++;
1876 bd_prod = TX_BD(fp->tx_bd_prod);
1877
1878 /* get a tx_buf and first BD */
1879 tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1880 tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1881
1882 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1883 tx_start_bd->general_data = (mac_type <<
1884 ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1885 /* header nbd */
1886 tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1887
1888 /* remember the first BD of the packet */
1889 tx_buf->first_bd = fp->tx_bd_prod;
1890 tx_buf->skb = skb;
1891 tx_buf->flags = 0;
1892
1893 DP(NETIF_MSG_TX_QUEUED,
1894 "sending pkt %u @%p next_idx %u bd %u @%p\n",
1895 pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1896
1897 #ifdef BCM_VLAN
1898 if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1899 (bp->flags & HW_VLAN_TX_FLAG)) {
1900 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1901 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1902 } else
1903 #endif
1904 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1905
1906 /* turn on parsing and get a BD */
1907 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1908 pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1909
1910 memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1911
1912 if (xmit_type & XMIT_CSUM) {
1913 hlen = (skb_network_header(skb) - skb->data) / 2;
1914
1915 /* for now NS flag is not used in Linux */
1916 pbd->global_data =
1917 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1918 ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1919
1920 pbd->ip_hlen = (skb_transport_header(skb) -
1921 skb_network_header(skb)) / 2;
1922
1923 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1924
1925 pbd->total_hlen = cpu_to_le16(hlen);
1926 hlen = hlen*2;
1927
1928 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1929
1930 if (xmit_type & XMIT_CSUM_V4)
1931 tx_start_bd->bd_flags.as_bitfield |=
1932 ETH_TX_BD_FLAGS_IP_CSUM;
1933 else
1934 tx_start_bd->bd_flags.as_bitfield |=
1935 ETH_TX_BD_FLAGS_IPV6;
1936
1937 if (xmit_type & XMIT_CSUM_TCP) {
1938 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1939
1940 } else {
1941 s8 fix = SKB_CS_OFF(skb); /* signed! */
1942
1943 pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1944
1945 DP(NETIF_MSG_TX_QUEUED,
1946 "hlen %d fix %d csum before fix %x\n",
1947 le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1948
1949 /* HW bug: fixup the CSUM */
1950 pbd->tcp_pseudo_csum =
1951 bnx2x_csum_fix(skb_transport_header(skb),
1952 SKB_CS(skb), fix);
1953
1954 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1955 pbd->tcp_pseudo_csum);
1956 }
1957 }
1958
1959 mapping = dma_map_single(&bp->pdev->dev, skb->data,
1960 skb_headlen(skb), DMA_TO_DEVICE);
1961
1962 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1963 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1964 nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
1965 tx_start_bd->nbd = cpu_to_le16(nbd);
1966 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
1967 pkt_size = tx_start_bd->nbytes;
1968
1969 DP(NETIF_MSG_TX_QUEUED, "first bd @%p addr (%x:%x) nbd %d"
1970 " nbytes %d flags %x vlan %x\n",
1971 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
1972 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
1973 tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
1974
1975 if (xmit_type & XMIT_GSO) {
1976
1977 DP(NETIF_MSG_TX_QUEUED,
1978 "TSO packet len %d hlen %d total len %d tso size %d\n",
1979 skb->len, hlen, skb_headlen(skb),
1980 skb_shinfo(skb)->gso_size);
1981
1982 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
1983
1984 if (unlikely(skb_headlen(skb) > hlen))
1985 bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
1986 hlen, bd_prod, ++nbd);
1987
1988 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1989 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1990 pbd->tcp_flags = pbd_tcp_flags(skb);
1991
1992 if (xmit_type & XMIT_GSO_V4) {
1993 pbd->ip_id = swab16(ip_hdr(skb)->id);
1994 pbd->tcp_pseudo_csum =
1995 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1996 ip_hdr(skb)->daddr,
1997 0, IPPROTO_TCP, 0));
1998
1999 } else
2000 pbd->tcp_pseudo_csum =
2001 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2002 &ipv6_hdr(skb)->daddr,
2003 0, IPPROTO_TCP, 0));
2004
2005 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2006 }
2007 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2008
2009 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2010 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2011
2012 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2013 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2014 if (total_pkt_bd == NULL)
2015 total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2016
2017 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2018 frag->page_offset,
2019 frag->size, DMA_TO_DEVICE);
2020
2021 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2022 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2023 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2024 le16_add_cpu(&pkt_size, frag->size);
2025
2026 DP(NETIF_MSG_TX_QUEUED,
2027 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
2028 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2029 le16_to_cpu(tx_data_bd->nbytes));
2030 }
2031
2032 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2033
2034 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2035
2036 /* now send a tx doorbell, counting the next BD
2037 * if the packet contains or ends with it
2038 */
2039 if (TX_BD_POFF(bd_prod) < nbd)
2040 nbd++;
2041
2042 if (total_pkt_bd != NULL)
2043 total_pkt_bd->total_pkt_bytes = pkt_size;
2044
2045 if (pbd)
2046 DP(NETIF_MSG_TX_QUEUED,
2047 "PBD @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u"
2048 " tcp_flags %x xsum %x seq %u hlen %u\n",
2049 pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2050 pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2051 pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2052
2053 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
2054
2055 /*
2056 * Make sure that the BD data is updated before updating the producer
2057 * since FW might read the BD right after the producer is updated.
2058 * This is only applicable for weak-ordered memory model archs such
2059 * as IA-64. The following barrier is also mandatory since FW will
2060 * assumes packets must have BDs.
2061 */
2062 wmb();
2063
2064 fp->tx_db.data.prod += nbd;
2065 barrier();
2066 DOORBELL(bp, fp->index, fp->tx_db.raw);
2067
2068 mmiowb();
2069
2070 fp->tx_bd_prod += nbd;
2071
2072 if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2073 netif_tx_stop_queue(txq);
2074
2075 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2076 * ordering of set_bit() in netif_tx_stop_queue() and read of
2077 * fp->bd_tx_cons */
2078 smp_mb();
2079
2080 fp->eth_q_stats.driver_xoff++;
2081 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2082 netif_tx_wake_queue(txq);
2083 }
2084 fp->tx_pkt++;
2085
2086 return NETDEV_TX_OK;
2087 }
2088 /* called with rtnl_lock */
2089 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2090 {
2091 struct sockaddr *addr = p;
2092 struct bnx2x *bp = netdev_priv(dev);
2093
2094 if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2095 return -EINVAL;
2096
2097 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2098 if (netif_running(dev)) {
2099 if (CHIP_IS_E1(bp))
2100 bnx2x_set_eth_mac_addr_e1(bp, 1);
2101 else
2102 bnx2x_set_eth_mac_addr_e1h(bp, 1);
2103 }
2104
2105 return 0;
2106 }
2107
2108 /* called with rtnl_lock */
2109 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2110 {
2111 struct bnx2x *bp = netdev_priv(dev);
2112 int rc = 0;
2113
2114 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2115 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2116 return -EAGAIN;
2117 }
2118
2119 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2120 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2121 return -EINVAL;
2122
2123 /* This does not race with packet allocation
2124 * because the actual alloc size is
2125 * only updated as part of load
2126 */
2127 dev->mtu = new_mtu;
2128
2129 if (netif_running(dev)) {
2130 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2131 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2132 }
2133
2134 return rc;
2135 }
2136
2137 void bnx2x_tx_timeout(struct net_device *dev)
2138 {
2139 struct bnx2x *bp = netdev_priv(dev);
2140
2141 #ifdef BNX2X_STOP_ON_ERROR
2142 if (!bp->panic)
2143 bnx2x_panic();
2144 #endif
2145 /* This allows the netif to be shutdown gracefully before resetting */
2146 schedule_delayed_work(&bp->reset_task, 0);
2147 }
2148
2149 #ifdef BCM_VLAN
2150 /* called with rtnl_lock */
2151 void bnx2x_vlan_rx_register(struct net_device *dev,
2152 struct vlan_group *vlgrp)
2153 {
2154 struct bnx2x *bp = netdev_priv(dev);
2155
2156 bp->vlgrp = vlgrp;
2157
2158 /* Set flags according to the required capabilities */
2159 bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2160
2161 if (dev->features & NETIF_F_HW_VLAN_TX)
2162 bp->flags |= HW_VLAN_TX_FLAG;
2163
2164 if (dev->features & NETIF_F_HW_VLAN_RX)
2165 bp->flags |= HW_VLAN_RX_FLAG;
2166
2167 if (netif_running(dev))
2168 bnx2x_set_client_config(bp);
2169 }
2170
2171 #endif
2172 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2173 {
2174 struct net_device *dev = pci_get_drvdata(pdev);
2175 struct bnx2x *bp;
2176
2177 if (!dev) {
2178 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2179 return -ENODEV;
2180 }
2181 bp = netdev_priv(dev);
2182
2183 rtnl_lock();
2184
2185 pci_save_state(pdev);
2186
2187 if (!netif_running(dev)) {
2188 rtnl_unlock();
2189 return 0;
2190 }
2191
2192 netif_device_detach(dev);
2193
2194 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2195
2196 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2197
2198 rtnl_unlock();
2199
2200 return 0;
2201 }
2202
2203 int bnx2x_resume(struct pci_dev *pdev)
2204 {
2205 struct net_device *dev = pci_get_drvdata(pdev);
2206 struct bnx2x *bp;
2207 int rc;
2208
2209 if (!dev) {
2210 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2211 return -ENODEV;
2212 }
2213 bp = netdev_priv(dev);
2214
2215 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2216 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2217 return -EAGAIN;
2218 }
2219
2220 rtnl_lock();
2221
2222 pci_restore_state(pdev);
2223
2224 if (!netif_running(dev)) {
2225 rtnl_unlock();
2226 return 0;
2227 }
2228
2229 bnx2x_set_power_state(bp, PCI_D0);
2230 netif_device_attach(dev);
2231
2232 rc = bnx2x_nic_load(bp, LOAD_OPEN);
2233
2234 rtnl_unlock();
2235
2236 return rc;
2237 }
Tomato firmware and modifications.