2 * Copyright 2008 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 * This program is free software; you may redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; version 2 of the License.
9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 #include <linux/module.h>
21 #include <linux/kernel.h>
22 #include <linux/string.h>
23 #include <linux/errno.h>
24 #include <linux/types.h>
25 #include <linux/init.h>
26 #include <linux/workqueue.h>
27 #include <linux/pci.h>
28 #include <linux/netdevice.h>
29 #include <linux/etherdevice.h>
30 #include <linux/if_ether.h>
31 #include <linux/if_vlan.h>
32 #include <linux/ethtool.h>
35 #include <linux/ipv6.h>
36 #include <linux/tcp.h>
37 #include <net/ip6_checksum.h>
39 #include "cq_enet_desc.h"
41 #include "vnic_intr.h"
42 #include "vnic_stats.h"
46 #define ENIC_NOTIFY_TIMER_PERIOD (2 * HZ)
47 #define WQ_ENET_MAX_DESC_LEN (1 << WQ_ENET_LEN_BITS)
48 #define MAX_TSO (1 << 16)
49 #define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1)
51 #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */
53 /* Supported devices */
54 static struct pci_device_id enic_id_table
[] = {
55 { PCI_VDEVICE(CISCO
, PCI_DEVICE_ID_CISCO_VIC_ENET
) },
56 { 0, } /* end of table */
59 MODULE_DESCRIPTION(DRV_DESCRIPTION
);
60 MODULE_AUTHOR("Scott Feldman <scofeldm@cisco.com>");
61 MODULE_LICENSE("GPL");
62 MODULE_VERSION(DRV_VERSION
);
63 MODULE_DEVICE_TABLE(pci
, enic_id_table
);
66 char name
[ETH_GSTRING_LEN
];
70 #define ENIC_TX_STAT(stat) \
71 { .name = #stat, .offset = offsetof(struct vnic_tx_stats, stat) / 8 }
72 #define ENIC_RX_STAT(stat) \
73 { .name = #stat, .offset = offsetof(struct vnic_rx_stats, stat) / 8 }
75 static const struct enic_stat enic_tx_stats
[] = {
76 ENIC_TX_STAT(tx_frames_ok
),
77 ENIC_TX_STAT(tx_unicast_frames_ok
),
78 ENIC_TX_STAT(tx_multicast_frames_ok
),
79 ENIC_TX_STAT(tx_broadcast_frames_ok
),
80 ENIC_TX_STAT(tx_bytes_ok
),
81 ENIC_TX_STAT(tx_unicast_bytes_ok
),
82 ENIC_TX_STAT(tx_multicast_bytes_ok
),
83 ENIC_TX_STAT(tx_broadcast_bytes_ok
),
84 ENIC_TX_STAT(tx_drops
),
85 ENIC_TX_STAT(tx_errors
),
89 static const struct enic_stat enic_rx_stats
[] = {
90 ENIC_RX_STAT(rx_frames_ok
),
91 ENIC_RX_STAT(rx_frames_total
),
92 ENIC_RX_STAT(rx_unicast_frames_ok
),
93 ENIC_RX_STAT(rx_multicast_frames_ok
),
94 ENIC_RX_STAT(rx_broadcast_frames_ok
),
95 ENIC_RX_STAT(rx_bytes_ok
),
96 ENIC_RX_STAT(rx_unicast_bytes_ok
),
97 ENIC_RX_STAT(rx_multicast_bytes_ok
),
98 ENIC_RX_STAT(rx_broadcast_bytes_ok
),
99 ENIC_RX_STAT(rx_drop
),
100 ENIC_RX_STAT(rx_no_bufs
),
101 ENIC_RX_STAT(rx_errors
),
102 ENIC_RX_STAT(rx_rss
),
103 ENIC_RX_STAT(rx_crc_errors
),
104 ENIC_RX_STAT(rx_frames_64
),
105 ENIC_RX_STAT(rx_frames_127
),
106 ENIC_RX_STAT(rx_frames_255
),
107 ENIC_RX_STAT(rx_frames_511
),
108 ENIC_RX_STAT(rx_frames_1023
),
109 ENIC_RX_STAT(rx_frames_1518
),
110 ENIC_RX_STAT(rx_frames_to_max
),
113 static const unsigned int enic_n_tx_stats
= ARRAY_SIZE(enic_tx_stats
);
114 static const unsigned int enic_n_rx_stats
= ARRAY_SIZE(enic_rx_stats
);
116 static int enic_get_settings(struct net_device
*netdev
,
117 struct ethtool_cmd
*ecmd
)
119 struct enic
*enic
= netdev_priv(netdev
);
121 ecmd
->supported
= (SUPPORTED_10000baseT_Full
| SUPPORTED_FIBRE
);
122 ecmd
->advertising
= (ADVERTISED_10000baseT_Full
| ADVERTISED_FIBRE
);
123 ecmd
->port
= PORT_FIBRE
;
124 ecmd
->transceiver
= XCVR_EXTERNAL
;
126 if (netif_carrier_ok(netdev
)) {
127 ecmd
->speed
= vnic_dev_port_speed(enic
->vdev
);
128 ecmd
->duplex
= DUPLEX_FULL
;
134 ecmd
->autoneg
= AUTONEG_DISABLE
;
139 static void enic_get_drvinfo(struct net_device
*netdev
,
140 struct ethtool_drvinfo
*drvinfo
)
142 struct enic
*enic
= netdev_priv(netdev
);
143 struct vnic_devcmd_fw_info
*fw_info
;
145 spin_lock(&enic
->devcmd_lock
);
146 vnic_dev_fw_info(enic
->vdev
, &fw_info
);
147 spin_unlock(&enic
->devcmd_lock
);
149 strncpy(drvinfo
->driver
, DRV_NAME
, sizeof(drvinfo
->driver
));
150 strncpy(drvinfo
->version
, DRV_VERSION
, sizeof(drvinfo
->version
));
151 strncpy(drvinfo
->fw_version
, fw_info
->fw_version
,
152 sizeof(drvinfo
->fw_version
));
153 strncpy(drvinfo
->bus_info
, pci_name(enic
->pdev
),
154 sizeof(drvinfo
->bus_info
));
157 static void enic_get_strings(struct net_device
*netdev
, u32 stringset
, u8
*data
)
163 for (i
= 0; i
< enic_n_tx_stats
; i
++) {
164 memcpy(data
, enic_tx_stats
[i
].name
, ETH_GSTRING_LEN
);
165 data
+= ETH_GSTRING_LEN
;
167 for (i
= 0; i
< enic_n_rx_stats
; i
++) {
168 memcpy(data
, enic_rx_stats
[i
].name
, ETH_GSTRING_LEN
);
169 data
+= ETH_GSTRING_LEN
;
175 static int enic_get_sset_count(struct net_device
*netdev
, int sset
)
179 return enic_n_tx_stats
+ enic_n_rx_stats
;
185 static void enic_get_ethtool_stats(struct net_device
*netdev
,
186 struct ethtool_stats
*stats
, u64
*data
)
188 struct enic
*enic
= netdev_priv(netdev
);
189 struct vnic_stats
*vstats
;
192 spin_lock(&enic
->devcmd_lock
);
193 vnic_dev_stats_dump(enic
->vdev
, &vstats
);
194 spin_unlock(&enic
->devcmd_lock
);
196 for (i
= 0; i
< enic_n_tx_stats
; i
++)
197 *(data
++) = ((u64
*)&vstats
->tx
)[enic_tx_stats
[i
].offset
];
198 for (i
= 0; i
< enic_n_rx_stats
; i
++)
199 *(data
++) = ((u64
*)&vstats
->rx
)[enic_rx_stats
[i
].offset
];
202 static u32
enic_get_rx_csum(struct net_device
*netdev
)
204 struct enic
*enic
= netdev_priv(netdev
);
205 return enic
->csum_rx_enabled
;
208 static int enic_set_rx_csum(struct net_device
*netdev
, u32 data
)
210 struct enic
*enic
= netdev_priv(netdev
);
212 if (data
&& !ENIC_SETTING(enic
, RXCSUM
))
215 enic
->csum_rx_enabled
= !!data
;
220 static int enic_set_tx_csum(struct net_device
*netdev
, u32 data
)
222 struct enic
*enic
= netdev_priv(netdev
);
224 if (data
&& !ENIC_SETTING(enic
, TXCSUM
))
228 netdev
->features
|= NETIF_F_HW_CSUM
;
230 netdev
->features
&= ~NETIF_F_HW_CSUM
;
235 static int enic_set_tso(struct net_device
*netdev
, u32 data
)
237 struct enic
*enic
= netdev_priv(netdev
);
239 if (data
&& !ENIC_SETTING(enic
, TSO
))
244 NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_TSO_ECN
;
247 ~(NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_TSO_ECN
);
252 static u32
enic_get_msglevel(struct net_device
*netdev
)
254 struct enic
*enic
= netdev_priv(netdev
);
255 return enic
->msg_enable
;
258 static void enic_set_msglevel(struct net_device
*netdev
, u32 value
)
260 struct enic
*enic
= netdev_priv(netdev
);
261 enic
->msg_enable
= value
;
264 static const struct ethtool_ops enic_ethtool_ops
= {
265 .get_settings
= enic_get_settings
,
266 .get_drvinfo
= enic_get_drvinfo
,
267 .get_msglevel
= enic_get_msglevel
,
268 .set_msglevel
= enic_set_msglevel
,
269 .get_link
= ethtool_op_get_link
,
270 .get_strings
= enic_get_strings
,
271 .get_sset_count
= enic_get_sset_count
,
272 .get_ethtool_stats
= enic_get_ethtool_stats
,
273 .get_rx_csum
= enic_get_rx_csum
,
274 .set_rx_csum
= enic_set_rx_csum
,
275 .get_tx_csum
= ethtool_op_get_tx_csum
,
276 .set_tx_csum
= enic_set_tx_csum
,
277 .get_sg
= ethtool_op_get_sg
,
278 .set_sg
= ethtool_op_set_sg
,
279 .get_tso
= ethtool_op_get_tso
,
280 .set_tso
= enic_set_tso
,
281 .get_flags
= ethtool_op_get_flags
,
282 .set_flags
= ethtool_op_set_flags
,
285 static void enic_free_wq_buf(struct vnic_wq
*wq
, struct vnic_wq_buf
*buf
)
287 struct enic
*enic
= vnic_dev_priv(wq
->vdev
);
290 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
291 buf
->len
, PCI_DMA_TODEVICE
);
293 pci_unmap_page(enic
->pdev
, buf
->dma_addr
,
294 buf
->len
, PCI_DMA_TODEVICE
);
297 dev_kfree_skb_any(buf
->os_buf
);
300 static void enic_wq_free_buf(struct vnic_wq
*wq
,
301 struct cq_desc
*cq_desc
, struct vnic_wq_buf
*buf
, void *opaque
)
303 enic_free_wq_buf(wq
, buf
);
306 static int enic_wq_service(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
307 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
309 struct enic
*enic
= vnic_dev_priv(vdev
);
311 spin_lock(&enic
->wq_lock
[q_number
]);
313 vnic_wq_service(&enic
->wq
[q_number
], cq_desc
,
314 completed_index
, enic_wq_free_buf
,
317 if (netif_queue_stopped(enic
->netdev
) &&
318 vnic_wq_desc_avail(&enic
->wq
[q_number
]) >=
319 (MAX_SKB_FRAGS
+ ENIC_DESC_MAX_SPLITS
))
320 netif_wake_queue(enic
->netdev
);
322 spin_unlock(&enic
->wq_lock
[q_number
]);
327 static void enic_log_q_error(struct enic
*enic
)
332 for (i
= 0; i
< enic
->wq_count
; i
++) {
333 error_status
= vnic_wq_error_status(&enic
->wq
[i
]);
335 printk(KERN_ERR PFX
"%s: WQ[%d] error_status %d\n",
336 enic
->netdev
->name
, i
, error_status
);
339 for (i
= 0; i
< enic
->rq_count
; i
++) {
340 error_status
= vnic_rq_error_status(&enic
->rq
[i
]);
342 printk(KERN_ERR PFX
"%s: RQ[%d] error_status %d\n",
343 enic
->netdev
->name
, i
, error_status
);
347 static void enic_link_check(struct enic
*enic
)
349 int link_status
= vnic_dev_link_status(enic
->vdev
);
350 int carrier_ok
= netif_carrier_ok(enic
->netdev
);
352 if (link_status
&& !carrier_ok
) {
353 printk(KERN_INFO PFX
"%s: Link UP\n", enic
->netdev
->name
);
354 netif_carrier_on(enic
->netdev
);
355 } else if (!link_status
&& carrier_ok
) {
356 printk(KERN_INFO PFX
"%s: Link DOWN\n", enic
->netdev
->name
);
357 netif_carrier_off(enic
->netdev
);
361 static void enic_mtu_check(struct enic
*enic
)
363 u32 mtu
= vnic_dev_mtu(enic
->vdev
);
365 if (mtu
!= enic
->port_mtu
) {
366 if (mtu
< enic
->netdev
->mtu
)
367 printk(KERN_WARNING PFX
368 "%s: interface MTU (%d) set higher "
369 "than switch port MTU (%d)\n",
370 enic
->netdev
->name
, enic
->netdev
->mtu
, mtu
);
371 enic
->port_mtu
= mtu
;
375 static void enic_msglvl_check(struct enic
*enic
)
377 u32 msg_enable
= vnic_dev_msg_lvl(enic
->vdev
);
379 if (msg_enable
!= enic
->msg_enable
) {
380 printk(KERN_INFO PFX
"%s: msg lvl changed from 0x%x to 0x%x\n",
381 enic
->netdev
->name
, enic
->msg_enable
, msg_enable
);
382 enic
->msg_enable
= msg_enable
;
386 static void enic_notify_check(struct enic
*enic
)
388 enic_msglvl_check(enic
);
389 enic_mtu_check(enic
);
390 enic_link_check(enic
);
393 #define ENIC_TEST_INTR(pba, i) (pba & (1 << i))
395 static irqreturn_t
enic_isr_legacy(int irq
, void *data
)
397 struct net_device
*netdev
= data
;
398 struct enic
*enic
= netdev_priv(netdev
);
401 vnic_intr_mask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
403 pba
= vnic_intr_legacy_pba(enic
->legacy_pba
);
405 vnic_intr_unmask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
406 return IRQ_NONE
; /* not our interrupt */
409 if (ENIC_TEST_INTR(pba
, ENIC_INTX_NOTIFY
)) {
410 vnic_intr_return_all_credits(&enic
->intr
[ENIC_INTX_NOTIFY
]);
411 enic_notify_check(enic
);
414 if (ENIC_TEST_INTR(pba
, ENIC_INTX_ERR
)) {
415 vnic_intr_return_all_credits(&enic
->intr
[ENIC_INTX_ERR
]);
416 enic_log_q_error(enic
);
417 /* schedule recovery from WQ/RQ error */
418 schedule_work(&enic
->reset
);
422 if (ENIC_TEST_INTR(pba
, ENIC_INTX_WQ_RQ
)) {
423 if (napi_schedule_prep(&enic
->napi
))
424 __napi_schedule(&enic
->napi
);
426 vnic_intr_unmask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
432 static irqreturn_t
enic_isr_msi(int irq
, void *data
)
434 struct enic
*enic
= data
;
436 /* With MSI, there is no sharing of interrupts, so this is
437 * our interrupt and there is no need to ack it. The device
438 * is not providing per-vector masking, so the OS will not
439 * write to PCI config space to mask/unmask the interrupt.
440 * We're using mask_on_assertion for MSI, so the device
441 * automatically masks the interrupt when the interrupt is
442 * generated. Later, when exiting polling, the interrupt
443 * will be unmasked (see enic_poll).
445 * Also, the device uses the same PCIe Traffic Class (TC)
446 * for Memory Write data and MSI, so there are no ordering
447 * issues; the MSI will always arrive at the Root Complex
448 * _after_ corresponding Memory Writes (i.e. descriptor
452 napi_schedule(&enic
->napi
);
457 static irqreturn_t
enic_isr_msix_rq(int irq
, void *data
)
459 struct enic
*enic
= data
;
461 /* schedule NAPI polling for RQ cleanup */
462 napi_schedule(&enic
->napi
);
467 static irqreturn_t
enic_isr_msix_wq(int irq
, void *data
)
469 struct enic
*enic
= data
;
470 unsigned int wq_work_to_do
= -1; /* no limit */
471 unsigned int wq_work_done
;
473 wq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
474 wq_work_to_do
, enic_wq_service
, NULL
);
476 vnic_intr_return_credits(&enic
->intr
[ENIC_MSIX_WQ
],
479 1 /* reset intr timer */);
484 static irqreturn_t
enic_isr_msix_err(int irq
, void *data
)
486 struct enic
*enic
= data
;
488 vnic_intr_return_all_credits(&enic
->intr
[ENIC_MSIX_ERR
]);
490 enic_log_q_error(enic
);
492 /* schedule recovery from WQ/RQ error */
493 schedule_work(&enic
->reset
);
498 static irqreturn_t
enic_isr_msix_notify(int irq
, void *data
)
500 struct enic
*enic
= data
;
502 vnic_intr_return_all_credits(&enic
->intr
[ENIC_MSIX_NOTIFY
]);
503 enic_notify_check(enic
);
508 static inline void enic_queue_wq_skb_cont(struct enic
*enic
,
509 struct vnic_wq
*wq
, struct sk_buff
*skb
,
510 unsigned int len_left
)
514 /* Queue additional data fragments */
515 for (frag
= skb_shinfo(skb
)->frags
; len_left
; frag
++) {
516 len_left
-= frag
->size
;
517 enic_queue_wq_desc_cont(wq
, skb
,
518 pci_map_page(enic
->pdev
, frag
->page
,
519 frag
->page_offset
, frag
->size
,
522 (len_left
== 0)); /* EOP? */
526 static inline void enic_queue_wq_skb_vlan(struct enic
*enic
,
527 struct vnic_wq
*wq
, struct sk_buff
*skb
,
528 int vlan_tag_insert
, unsigned int vlan_tag
)
530 unsigned int head_len
= skb_headlen(skb
);
531 unsigned int len_left
= skb
->len
- head_len
;
532 int eop
= (len_left
== 0);
534 /* Queue the main skb fragment. The fragments are no larger
535 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less
536 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor
537 * per fragment is queued.
539 enic_queue_wq_desc(wq
, skb
,
540 pci_map_single(enic
->pdev
, skb
->data
,
541 head_len
, PCI_DMA_TODEVICE
),
543 vlan_tag_insert
, vlan_tag
,
547 enic_queue_wq_skb_cont(enic
, wq
, skb
, len_left
);
550 static inline void enic_queue_wq_skb_csum_l4(struct enic
*enic
,
551 struct vnic_wq
*wq
, struct sk_buff
*skb
,
552 int vlan_tag_insert
, unsigned int vlan_tag
)
554 unsigned int head_len
= skb_headlen(skb
);
555 unsigned int len_left
= skb
->len
- head_len
;
556 unsigned int hdr_len
= skb_transport_offset(skb
);
557 unsigned int csum_offset
= hdr_len
+ skb
->csum_offset
;
558 int eop
= (len_left
== 0);
560 /* Queue the main skb fragment. The fragments are no larger
561 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less
562 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor
563 * per fragment is queued.
565 enic_queue_wq_desc_csum_l4(wq
, skb
,
566 pci_map_single(enic
->pdev
, skb
->data
,
567 head_len
, PCI_DMA_TODEVICE
),
571 vlan_tag_insert
, vlan_tag
,
575 enic_queue_wq_skb_cont(enic
, wq
, skb
, len_left
);
578 static inline void enic_queue_wq_skb_tso(struct enic
*enic
,
579 struct vnic_wq
*wq
, struct sk_buff
*skb
, unsigned int mss
,
580 int vlan_tag_insert
, unsigned int vlan_tag
)
582 unsigned int frag_len_left
= skb_headlen(skb
);
583 unsigned int len_left
= skb
->len
- frag_len_left
;
584 unsigned int hdr_len
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
585 int eop
= (len_left
== 0);
588 unsigned int offset
= 0;
591 /* Preload TCP csum field with IP pseudo hdr calculated
592 * with IP length set to zero. HW will later add in length
593 * to each TCP segment resulting from the TSO.
596 if (skb
->protocol
== cpu_to_be16(ETH_P_IP
)) {
597 ip_hdr(skb
)->check
= 0;
598 tcp_hdr(skb
)->check
= ~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
599 ip_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
600 } else if (skb
->protocol
== cpu_to_be16(ETH_P_IPV6
)) {
601 tcp_hdr(skb
)->check
= ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
602 &ipv6_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
605 /* Queue WQ_ENET_MAX_DESC_LEN length descriptors
606 * for the main skb fragment
608 while (frag_len_left
) {
609 len
= min(frag_len_left
, (unsigned int)WQ_ENET_MAX_DESC_LEN
);
610 dma_addr
= pci_map_single(enic
->pdev
, skb
->data
+ offset
,
611 len
, PCI_DMA_TODEVICE
);
612 enic_queue_wq_desc_tso(wq
, skb
,
616 vlan_tag_insert
, vlan_tag
,
617 eop
&& (len
== frag_len_left
));
618 frag_len_left
-= len
;
625 /* Queue WQ_ENET_MAX_DESC_LEN length descriptors
626 * for additional data fragments
628 for (frag
= skb_shinfo(skb
)->frags
; len_left
; frag
++) {
629 len_left
-= frag
->size
;
630 frag_len_left
= frag
->size
;
631 offset
= frag
->page_offset
;
633 while (frag_len_left
) {
634 len
= min(frag_len_left
,
635 (unsigned int)WQ_ENET_MAX_DESC_LEN
);
636 dma_addr
= pci_map_page(enic
->pdev
, frag
->page
,
639 enic_queue_wq_desc_cont(wq
, skb
,
643 (len
== frag_len_left
)); /* EOP? */
644 frag_len_left
-= len
;
650 static inline void enic_queue_wq_skb(struct enic
*enic
,
651 struct vnic_wq
*wq
, struct sk_buff
*skb
)
653 unsigned int mss
= skb_shinfo(skb
)->gso_size
;
654 unsigned int vlan_tag
= 0;
655 int vlan_tag_insert
= 0;
657 if (enic
->vlan_group
&& vlan_tx_tag_present(skb
)) {
658 /* VLAN tag from trunking driver */
660 vlan_tag
= vlan_tx_tag_get(skb
);
664 enic_queue_wq_skb_tso(enic
, wq
, skb
, mss
,
665 vlan_tag_insert
, vlan_tag
);
666 else if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
667 enic_queue_wq_skb_csum_l4(enic
, wq
, skb
,
668 vlan_tag_insert
, vlan_tag
);
670 enic_queue_wq_skb_vlan(enic
, wq
, skb
,
671 vlan_tag_insert
, vlan_tag
);
674 /* netif_tx_lock held, process context with BHs disabled, or BH */
675 static netdev_tx_t
enic_hard_start_xmit(struct sk_buff
*skb
,
676 struct net_device
*netdev
)
678 struct enic
*enic
= netdev_priv(netdev
);
679 struct vnic_wq
*wq
= &enic
->wq
[0];
687 /* Non-TSO sends must fit within ENIC_NON_TSO_MAX_DESC descs,
688 * which is very likely. In the off chance it's going to take
689 * more than * ENIC_NON_TSO_MAX_DESC, linearize the skb.
692 if (skb_shinfo(skb
)->gso_size
== 0 &&
693 skb_shinfo(skb
)->nr_frags
+ 1 > ENIC_NON_TSO_MAX_DESC
&&
694 skb_linearize(skb
)) {
699 spin_lock_irqsave(&enic
->wq_lock
[0], flags
);
701 if (vnic_wq_desc_avail(wq
) <
702 skb_shinfo(skb
)->nr_frags
+ ENIC_DESC_MAX_SPLITS
) {
703 netif_stop_queue(netdev
);
704 /* This is a hard error, log it */
705 printk(KERN_ERR PFX
"%s: BUG! Tx ring full when "
706 "queue awake!\n", netdev
->name
);
707 spin_unlock_irqrestore(&enic
->wq_lock
[0], flags
);
708 return NETDEV_TX_BUSY
;
711 enic_queue_wq_skb(enic
, wq
, skb
);
713 if (vnic_wq_desc_avail(wq
) < MAX_SKB_FRAGS
+ ENIC_DESC_MAX_SPLITS
)
714 netif_stop_queue(netdev
);
716 spin_unlock_irqrestore(&enic
->wq_lock
[0], flags
);
721 /* dev_base_lock rwlock held, nominally process context */
722 static struct net_device_stats
*enic_get_stats(struct net_device
*netdev
)
724 struct enic
*enic
= netdev_priv(netdev
);
725 struct net_device_stats
*net_stats
= &netdev
->stats
;
726 struct vnic_stats
*stats
;
728 spin_lock(&enic
->devcmd_lock
);
729 vnic_dev_stats_dump(enic
->vdev
, &stats
);
730 spin_unlock(&enic
->devcmd_lock
);
732 net_stats
->tx_packets
= stats
->tx
.tx_frames_ok
;
733 net_stats
->tx_bytes
= stats
->tx
.tx_bytes_ok
;
734 net_stats
->tx_errors
= stats
->tx
.tx_errors
;
735 net_stats
->tx_dropped
= stats
->tx
.tx_drops
;
737 net_stats
->rx_packets
= stats
->rx
.rx_frames_ok
;
738 net_stats
->rx_bytes
= stats
->rx
.rx_bytes_ok
;
739 net_stats
->rx_errors
= stats
->rx
.rx_errors
;
740 net_stats
->multicast
= stats
->rx
.rx_multicast_frames_ok
;
741 net_stats
->rx_crc_errors
= enic
->rq_bad_fcs
;
742 net_stats
->rx_dropped
= stats
->rx
.rx_no_bufs
;
747 static void enic_reset_mcaddrs(struct enic
*enic
)
752 static int enic_set_mac_addr(struct net_device
*netdev
, char *addr
)
754 if (!is_valid_ether_addr(addr
))
755 return -EADDRNOTAVAIL
;
757 memcpy(netdev
->dev_addr
, addr
, netdev
->addr_len
);
762 /* netif_tx_lock held, BHs disabled */
763 static void enic_set_multicast_list(struct net_device
*netdev
)
765 struct enic
*enic
= netdev_priv(netdev
);
766 struct dev_mc_list
*list
= netdev
->mc_list
;
768 int multicast
= (netdev
->flags
& IFF_MULTICAST
) ? 1 : 0;
769 int broadcast
= (netdev
->flags
& IFF_BROADCAST
) ? 1 : 0;
770 int promisc
= (netdev
->flags
& IFF_PROMISC
) ? 1 : 0;
771 int allmulti
= (netdev
->flags
& IFF_ALLMULTI
) ||
772 (netdev
->mc_count
> ENIC_MULTICAST_PERFECT_FILTERS
);
773 u8 mc_addr
[ENIC_MULTICAST_PERFECT_FILTERS
][ETH_ALEN
];
774 unsigned int mc_count
= netdev
->mc_count
;
777 if (mc_count
> ENIC_MULTICAST_PERFECT_FILTERS
)
778 mc_count
= ENIC_MULTICAST_PERFECT_FILTERS
;
780 spin_lock(&enic
->devcmd_lock
);
782 vnic_dev_packet_filter(enic
->vdev
, directed
,
783 multicast
, broadcast
, promisc
, allmulti
);
785 /* Is there an easier way? Trying to minimize to
786 * calls to add/del multicast addrs. We keep the
787 * addrs from the last call in enic->mc_addr and
788 * look for changes to add/del.
791 for (i
= 0; list
&& i
< mc_count
; i
++) {
792 memcpy(mc_addr
[i
], list
->dmi_addr
, ETH_ALEN
);
796 for (i
= 0; i
< enic
->mc_count
; i
++) {
797 for (j
= 0; j
< mc_count
; j
++)
798 if (compare_ether_addr(enic
->mc_addr
[i
],
802 enic_del_multicast_addr(enic
, enic
->mc_addr
[i
]);
805 for (i
= 0; i
< mc_count
; i
++) {
806 for (j
= 0; j
< enic
->mc_count
; j
++)
807 if (compare_ether_addr(mc_addr
[i
],
808 enic
->mc_addr
[j
]) == 0)
810 if (j
== enic
->mc_count
)
811 enic_add_multicast_addr(enic
, mc_addr
[i
]);
814 /* Save the list to compare against next time
817 for (i
= 0; i
< mc_count
; i
++)
818 memcpy(enic
->mc_addr
[i
], mc_addr
[i
], ETH_ALEN
);
820 enic
->mc_count
= mc_count
;
822 spin_unlock(&enic
->devcmd_lock
);
825 /* rtnl lock is held */
826 static void enic_vlan_rx_register(struct net_device
*netdev
,
827 struct vlan_group
*vlan_group
)
829 struct enic
*enic
= netdev_priv(netdev
);
830 enic
->vlan_group
= vlan_group
;
833 /* rtnl lock is held */
834 static void enic_vlan_rx_add_vid(struct net_device
*netdev
, u16 vid
)
836 struct enic
*enic
= netdev_priv(netdev
);
838 spin_lock(&enic
->devcmd_lock
);
839 enic_add_vlan(enic
, vid
);
840 spin_unlock(&enic
->devcmd_lock
);
843 /* rtnl lock is held */
844 static void enic_vlan_rx_kill_vid(struct net_device
*netdev
, u16 vid
)
846 struct enic
*enic
= netdev_priv(netdev
);
848 spin_lock(&enic
->devcmd_lock
);
849 enic_del_vlan(enic
, vid
);
850 spin_unlock(&enic
->devcmd_lock
);
853 /* netif_tx_lock held, BHs disabled */
854 static void enic_tx_timeout(struct net_device
*netdev
)
856 struct enic
*enic
= netdev_priv(netdev
);
857 schedule_work(&enic
->reset
);
860 static void enic_free_rq_buf(struct vnic_rq
*rq
, struct vnic_rq_buf
*buf
)
862 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
867 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
868 buf
->len
, PCI_DMA_FROMDEVICE
);
869 dev_kfree_skb_any(buf
->os_buf
);
872 static inline struct sk_buff
*enic_rq_alloc_skb(struct net_device
*netdev
,
877 skb
= netdev_alloc_skb(netdev
, size
+ NET_IP_ALIGN
);
880 skb_reserve(skb
, NET_IP_ALIGN
);
885 static int enic_rq_alloc_buf(struct vnic_rq
*rq
)
887 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
888 struct net_device
*netdev
= enic
->netdev
;
890 unsigned int len
= netdev
->mtu
+ ETH_HLEN
;
891 unsigned int os_buf_index
= 0;
894 skb
= enic_rq_alloc_skb(netdev
, len
);
898 dma_addr
= pci_map_single(enic
->pdev
, skb
->data
,
899 len
, PCI_DMA_FROMDEVICE
);
901 enic_queue_rq_desc(rq
, skb
, os_buf_index
,
907 static int enic_rq_alloc_buf_a1(struct vnic_rq
*rq
)
909 struct rq_enet_desc
*desc
= vnic_rq_next_desc(rq
);
911 if (vnic_rq_posting_soon(rq
)) {
913 /* SW workaround for A0 HW erratum: if we're just about
914 * to write posted_index, insert a dummy desc
918 rq_enet_desc_enc(desc
, 0, RQ_ENET_TYPE_RESV2
, 0);
919 vnic_rq_post(rq
, 0, 0, 0, 0);
921 return enic_rq_alloc_buf(rq
);
927 static int enic_set_rq_alloc_buf(struct enic
*enic
)
929 enum vnic_dev_hw_version hw_ver
;
932 err
= vnic_dev_hw_version(enic
->vdev
, &hw_ver
);
937 case VNIC_DEV_HW_VER_A1
:
938 enic
->rq_alloc_buf
= enic_rq_alloc_buf_a1
;
940 case VNIC_DEV_HW_VER_A2
:
941 case VNIC_DEV_HW_VER_UNKNOWN
:
942 enic
->rq_alloc_buf
= enic_rq_alloc_buf
;
951 static int enic_get_skb_header(struct sk_buff
*skb
, void **iphdr
,
952 void **tcph
, u64
*hdr_flags
, void *priv
)
954 struct cq_enet_rq_desc
*cq_desc
= priv
;
958 u8 type
, color
, eop
, sop
, ingress_port
, vlan_stripped
;
959 u8 fcoe
, fcoe_sof
, fcoe_fc_crc_ok
, fcoe_enc_error
, fcoe_eof
;
960 u8 tcp_udp_csum_ok
, udp
, tcp
, ipv4_csum_ok
;
961 u8 ipv6
, ipv4
, ipv4_fragment
, fcs_ok
, rss_type
, csum_not_calc
;
963 u16 q_number
, completed_index
, bytes_written
, vlan
, checksum
;
966 cq_enet_rq_desc_dec(cq_desc
,
967 &type
, &color
, &q_number
, &completed_index
,
968 &ingress_port
, &fcoe
, &eop
, &sop
, &rss_type
,
969 &csum_not_calc
, &rss_hash
, &bytes_written
,
970 &packet_error
, &vlan_stripped
, &vlan
, &checksum
,
971 &fcoe_sof
, &fcoe_fc_crc_ok
, &fcoe_enc_error
,
972 &fcoe_eof
, &tcp_udp_csum_ok
, &udp
, &tcp
,
973 &ipv4_csum_ok
, &ipv6
, &ipv4
, &ipv4_fragment
,
976 if (!(ipv4
&& tcp
&& !ipv4_fragment
))
979 skb_reset_network_header(skb
);
982 ip_len
= ip_hdrlen(skb
);
983 skb_set_transport_header(skb
, ip_len
);
985 /* check if ip header and tcp header are complete */
986 if (ntohs(iph
->tot_len
) < ip_len
+ tcp_hdrlen(skb
))
989 *hdr_flags
= LRO_IPV4
| LRO_TCP
;
990 *tcph
= tcp_hdr(skb
);
996 static void enic_rq_indicate_buf(struct vnic_rq
*rq
,
997 struct cq_desc
*cq_desc
, struct vnic_rq_buf
*buf
,
998 int skipped
, void *opaque
)
1000 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
1001 struct net_device
*netdev
= enic
->netdev
;
1002 struct sk_buff
*skb
;
1004 u8 type
, color
, eop
, sop
, ingress_port
, vlan_stripped
;
1005 u8 fcoe
, fcoe_sof
, fcoe_fc_crc_ok
, fcoe_enc_error
, fcoe_eof
;
1006 u8 tcp_udp_csum_ok
, udp
, tcp
, ipv4_csum_ok
;
1007 u8 ipv6
, ipv4
, ipv4_fragment
, fcs_ok
, rss_type
, csum_not_calc
;
1009 u16 q_number
, completed_index
, bytes_written
, vlan
, checksum
;
1016 prefetch(skb
->data
- NET_IP_ALIGN
);
1017 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
1018 buf
->len
, PCI_DMA_FROMDEVICE
);
1020 cq_enet_rq_desc_dec((struct cq_enet_rq_desc
*)cq_desc
,
1021 &type
, &color
, &q_number
, &completed_index
,
1022 &ingress_port
, &fcoe
, &eop
, &sop
, &rss_type
,
1023 &csum_not_calc
, &rss_hash
, &bytes_written
,
1024 &packet_error
, &vlan_stripped
, &vlan
, &checksum
,
1025 &fcoe_sof
, &fcoe_fc_crc_ok
, &fcoe_enc_error
,
1026 &fcoe_eof
, &tcp_udp_csum_ok
, &udp
, &tcp
,
1027 &ipv4_csum_ok
, &ipv6
, &ipv4
, &ipv4_fragment
,
1032 if (bytes_written
> 0 && !fcs_ok
)
1035 dev_kfree_skb_any(skb
);
1040 if (eop
&& bytes_written
> 0) {
1045 skb_put(skb
, bytes_written
);
1046 skb
->protocol
= eth_type_trans(skb
, netdev
);
1048 if (enic
->csum_rx_enabled
&& !csum_not_calc
) {
1049 skb
->csum
= htons(checksum
);
1050 skb
->ip_summed
= CHECKSUM_COMPLETE
;
1055 if (enic
->vlan_group
&& vlan_stripped
) {
1057 if ((netdev
->features
& NETIF_F_LRO
) && ipv4
)
1058 lro_vlan_hwaccel_receive_skb(&enic
->lro_mgr
,
1059 skb
, enic
->vlan_group
,
1062 vlan_hwaccel_receive_skb(skb
,
1063 enic
->vlan_group
, vlan
);
1067 if ((netdev
->features
& NETIF_F_LRO
) && ipv4
)
1068 lro_receive_skb(&enic
->lro_mgr
, skb
, cq_desc
);
1070 netif_receive_skb(skb
);
1079 dev_kfree_skb_any(skb
);
1083 static int enic_rq_service(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
1084 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
1086 struct enic
*enic
= vnic_dev_priv(vdev
);
1088 vnic_rq_service(&enic
->rq
[q_number
], cq_desc
,
1089 completed_index
, VNIC_RQ_RETURN_DESC
,
1090 enic_rq_indicate_buf
, opaque
);
1095 static void enic_rq_drop_buf(struct vnic_rq
*rq
,
1096 struct cq_desc
*cq_desc
, struct vnic_rq_buf
*buf
,
1097 int skipped
, void *opaque
)
1099 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
1100 struct sk_buff
*skb
= buf
->os_buf
;
1105 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
1106 buf
->len
, PCI_DMA_FROMDEVICE
);
1108 dev_kfree_skb_any(skb
);
1111 static int enic_rq_service_drop(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
1112 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
1114 struct enic
*enic
= vnic_dev_priv(vdev
);
1116 vnic_rq_service(&enic
->rq
[q_number
], cq_desc
,
1117 completed_index
, VNIC_RQ_RETURN_DESC
,
1118 enic_rq_drop_buf
, opaque
);
1123 static int enic_poll(struct napi_struct
*napi
, int budget
)
1125 struct enic
*enic
= container_of(napi
, struct enic
, napi
);
1126 struct net_device
*netdev
= enic
->netdev
;
1127 unsigned int rq_work_to_do
= budget
;
1128 unsigned int wq_work_to_do
= -1; /* no limit */
1129 unsigned int work_done
, rq_work_done
, wq_work_done
;
1131 /* Service RQ (first) and WQ
1134 rq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1135 rq_work_to_do
, enic_rq_service
, NULL
);
1137 wq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
1138 wq_work_to_do
, enic_wq_service
, NULL
);
1140 /* Accumulate intr event credits for this polling
1141 * cycle. An intr event is the completion of a
1142 * a WQ or RQ packet.
1145 work_done
= rq_work_done
+ wq_work_done
;
1148 vnic_intr_return_credits(&enic
->intr
[ENIC_INTX_WQ_RQ
],
1150 0 /* don't unmask intr */,
1151 0 /* don't reset intr timer */);
1153 if (rq_work_done
> 0) {
1158 vnic_rq_fill(&enic
->rq
[0], enic
->rq_alloc_buf
);
1162 /* If no work done, flush all LROs and exit polling
1165 if (netdev
->features
& NETIF_F_LRO
)
1166 lro_flush_all(&enic
->lro_mgr
);
1168 napi_complete(napi
);
1169 vnic_intr_unmask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
1172 return rq_work_done
;
1175 static int enic_poll_msix(struct napi_struct
*napi
, int budget
)
1177 struct enic
*enic
= container_of(napi
, struct enic
, napi
);
1178 struct net_device
*netdev
= enic
->netdev
;
1179 unsigned int work_to_do
= budget
;
1180 unsigned int work_done
;
1185 work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1186 work_to_do
, enic_rq_service
, NULL
);
1188 if (work_done
> 0) {
1193 vnic_rq_fill(&enic
->rq
[0], enic
->rq_alloc_buf
);
1195 /* Return intr event credits for this polling
1196 * cycle. An intr event is the completion of a
1200 vnic_intr_return_credits(&enic
->intr
[ENIC_MSIX_RQ
],
1202 0 /* don't unmask intr */,
1203 0 /* don't reset intr timer */);
1206 /* If no work done, flush all LROs and exit polling
1209 if (netdev
->features
& NETIF_F_LRO
)
1210 lro_flush_all(&enic
->lro_mgr
);
1212 napi_complete(napi
);
1213 vnic_intr_unmask(&enic
->intr
[ENIC_MSIX_RQ
]);
1219 static void enic_notify_timer(unsigned long data
)
1221 struct enic
*enic
= (struct enic
*)data
;
1223 enic_notify_check(enic
);
1225 mod_timer(&enic
->notify_timer
,
1226 round_jiffies(jiffies
+ ENIC_NOTIFY_TIMER_PERIOD
));
1229 static void enic_free_intr(struct enic
*enic
)
1231 struct net_device
*netdev
= enic
->netdev
;
1234 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1235 case VNIC_DEV_INTR_MODE_INTX
:
1236 free_irq(enic
->pdev
->irq
, netdev
);
1238 case VNIC_DEV_INTR_MODE_MSI
:
1239 free_irq(enic
->pdev
->irq
, enic
);
1241 case VNIC_DEV_INTR_MODE_MSIX
:
1242 for (i
= 0; i
< ARRAY_SIZE(enic
->msix
); i
++)
1243 if (enic
->msix
[i
].requested
)
1244 free_irq(enic
->msix_entry
[i
].vector
,
1245 enic
->msix
[i
].devid
);
1252 static int enic_request_intr(struct enic
*enic
)
1254 struct net_device
*netdev
= enic
->netdev
;
1258 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1260 case VNIC_DEV_INTR_MODE_INTX
:
1262 err
= request_irq(enic
->pdev
->irq
, enic_isr_legacy
,
1263 IRQF_SHARED
, netdev
->name
, netdev
);
1266 case VNIC_DEV_INTR_MODE_MSI
:
1268 err
= request_irq(enic
->pdev
->irq
, enic_isr_msi
,
1269 0, netdev
->name
, enic
);
1272 case VNIC_DEV_INTR_MODE_MSIX
:
1274 sprintf(enic
->msix
[ENIC_MSIX_RQ
].devname
,
1275 "%.11s-rx-0", netdev
->name
);
1276 enic
->msix
[ENIC_MSIX_RQ
].isr
= enic_isr_msix_rq
;
1277 enic
->msix
[ENIC_MSIX_RQ
].devid
= enic
;
1279 sprintf(enic
->msix
[ENIC_MSIX_WQ
].devname
,
1280 "%.11s-tx-0", netdev
->name
);
1281 enic
->msix
[ENIC_MSIX_WQ
].isr
= enic_isr_msix_wq
;
1282 enic
->msix
[ENIC_MSIX_WQ
].devid
= enic
;
1284 sprintf(enic
->msix
[ENIC_MSIX_ERR
].devname
,
1285 "%.11s-err", netdev
->name
);
1286 enic
->msix
[ENIC_MSIX_ERR
].isr
= enic_isr_msix_err
;
1287 enic
->msix
[ENIC_MSIX_ERR
].devid
= enic
;
1289 sprintf(enic
->msix
[ENIC_MSIX_NOTIFY
].devname
,
1290 "%.11s-notify", netdev
->name
);
1291 enic
->msix
[ENIC_MSIX_NOTIFY
].isr
= enic_isr_msix_notify
;
1292 enic
->msix
[ENIC_MSIX_NOTIFY
].devid
= enic
;
1294 for (i
= 0; i
< ARRAY_SIZE(enic
->msix
); i
++) {
1295 err
= request_irq(enic
->msix_entry
[i
].vector
,
1296 enic
->msix
[i
].isr
, 0,
1297 enic
->msix
[i
].devname
,
1298 enic
->msix
[i
].devid
);
1300 enic_free_intr(enic
);
1303 enic
->msix
[i
].requested
= 1;
1315 static int enic_notify_set(struct enic
*enic
)
1319 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1320 case VNIC_DEV_INTR_MODE_INTX
:
1321 err
= vnic_dev_notify_set(enic
->vdev
, ENIC_INTX_NOTIFY
);
1323 case VNIC_DEV_INTR_MODE_MSIX
:
1324 err
= vnic_dev_notify_set(enic
->vdev
, ENIC_MSIX_NOTIFY
);
1327 err
= vnic_dev_notify_set(enic
->vdev
, -1 /* no intr */);
1334 static void enic_notify_timer_start(struct enic
*enic
)
1336 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1337 case VNIC_DEV_INTR_MODE_MSI
:
1338 mod_timer(&enic
->notify_timer
, jiffies
);
1341 /* Using intr for notification for INTx/MSI-X */
1346 /* rtnl lock is held, process context */
1347 static int enic_open(struct net_device
*netdev
)
1349 struct enic
*enic
= netdev_priv(netdev
);
1353 err
= enic_request_intr(enic
);
1355 printk(KERN_ERR PFX
"%s: Unable to request irq.\n",
1360 err
= enic_notify_set(enic
);
1363 "%s: Failed to alloc notify buffer, aborting.\n",
1365 goto err_out_free_intr
;
1368 for (i
= 0; i
< enic
->rq_count
; i
++) {
1369 err
= vnic_rq_fill(&enic
->rq
[i
], enic
->rq_alloc_buf
);
1372 "%s: Unable to alloc receive buffers.\n",
1374 goto err_out_notify_unset
;
1378 for (i
= 0; i
< enic
->wq_count
; i
++)
1379 vnic_wq_enable(&enic
->wq
[i
]);
1380 for (i
= 0; i
< enic
->rq_count
; i
++)
1381 vnic_rq_enable(&enic
->rq
[i
]);
1383 enic_add_station_addr(enic
);
1384 enic_set_multicast_list(netdev
);
1386 netif_wake_queue(netdev
);
1387 napi_enable(&enic
->napi
);
1388 vnic_dev_enable(enic
->vdev
);
1390 for (i
= 0; i
< enic
->intr_count
; i
++)
1391 vnic_intr_unmask(&enic
->intr
[i
]);
1393 enic_notify_timer_start(enic
);
1397 err_out_notify_unset
:
1398 vnic_dev_notify_unset(enic
->vdev
);
1400 enic_free_intr(enic
);
1405 /* rtnl lock is held, process context */
1406 static int enic_stop(struct net_device
*netdev
)
1408 struct enic
*enic
= netdev_priv(netdev
);
1412 del_timer_sync(&enic
->notify_timer
);
1414 vnic_dev_disable(enic
->vdev
);
1415 napi_disable(&enic
->napi
);
1416 netif_stop_queue(netdev
);
1418 for (i
= 0; i
< enic
->intr_count
; i
++)
1419 vnic_intr_mask(&enic
->intr
[i
]);
1421 for (i
= 0; i
< enic
->wq_count
; i
++) {
1422 err
= vnic_wq_disable(&enic
->wq
[i
]);
1426 for (i
= 0; i
< enic
->rq_count
; i
++) {
1427 err
= vnic_rq_disable(&enic
->rq
[i
]);
1432 vnic_dev_notify_unset(enic
->vdev
);
1433 enic_free_intr(enic
);
1435 (void)vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1436 -1, enic_rq_service_drop
, NULL
);
1437 (void)vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
1438 -1, enic_wq_service
, NULL
);
1440 for (i
= 0; i
< enic
->wq_count
; i
++)
1441 vnic_wq_clean(&enic
->wq
[i
], enic_free_wq_buf
);
1442 for (i
= 0; i
< enic
->rq_count
; i
++)
1443 vnic_rq_clean(&enic
->rq
[i
], enic_free_rq_buf
);
1444 for (i
= 0; i
< enic
->cq_count
; i
++)
1445 vnic_cq_clean(&enic
->cq
[i
]);
1446 for (i
= 0; i
< enic
->intr_count
; i
++)
1447 vnic_intr_clean(&enic
->intr
[i
]);
1452 static int enic_change_mtu(struct net_device
*netdev
, int new_mtu
)
1454 struct enic
*enic
= netdev_priv(netdev
);
1455 int running
= netif_running(netdev
);
1457 if (new_mtu
< ENIC_MIN_MTU
|| new_mtu
> ENIC_MAX_MTU
)
1463 netdev
->mtu
= new_mtu
;
1465 if (netdev
->mtu
> enic
->port_mtu
)
1466 printk(KERN_WARNING PFX
1467 "%s: interface MTU (%d) set higher "
1468 "than port MTU (%d)\n",
1469 netdev
->name
, netdev
->mtu
, enic
->port_mtu
);
1477 #ifdef CONFIG_NET_POLL_CONTROLLER
1478 static void enic_poll_controller(struct net_device
*netdev
)
1480 struct enic
*enic
= netdev_priv(netdev
);
1481 struct vnic_dev
*vdev
= enic
->vdev
;
1483 switch (vnic_dev_get_intr_mode(vdev
)) {
1484 case VNIC_DEV_INTR_MODE_MSIX
:
1485 enic_isr_msix_rq(enic
->pdev
->irq
, enic
);
1486 enic_isr_msix_wq(enic
->pdev
->irq
, enic
);
1488 case VNIC_DEV_INTR_MODE_MSI
:
1489 enic_isr_msi(enic
->pdev
->irq
, enic
);
1491 case VNIC_DEV_INTR_MODE_INTX
:
1492 enic_isr_legacy(enic
->pdev
->irq
, netdev
);
1500 static int enic_dev_wait(struct vnic_dev
*vdev
,
1501 int (*start
)(struct vnic_dev
*, int),
1502 int (*finished
)(struct vnic_dev
*, int *),
1509 BUG_ON(in_interrupt());
1511 err
= start(vdev
, arg
);
1515 /* Wait for func to complete...2 seconds max
1518 time
= jiffies
+ (HZ
* 2);
1521 err
= finished(vdev
, &done
);
1528 schedule_timeout_uninterruptible(HZ
/ 10);
1530 } while (time_after(time
, jiffies
));
1535 static int enic_dev_open(struct enic
*enic
)
1539 err
= enic_dev_wait(enic
->vdev
, vnic_dev_open
,
1540 vnic_dev_open_done
, 0);
1543 "vNIC device open failed, err %d.\n", err
);
1548 static int enic_dev_soft_reset(struct enic
*enic
)
1552 err
= enic_dev_wait(enic
->vdev
, vnic_dev_soft_reset
,
1553 vnic_dev_soft_reset_done
, 0);
1556 "vNIC soft reset failed, err %d.\n", err
);
1561 static int enic_set_niccfg(struct enic
*enic
)
1563 const u8 rss_default_cpu
= 0;
1564 const u8 rss_hash_type
= 0;
1565 const u8 rss_hash_bits
= 0;
1566 const u8 rss_base_cpu
= 0;
1567 const u8 rss_enable
= 0;
1568 const u8 tso_ipid_split_en
= 0;
1569 const u8 ig_vlan_strip_en
= 1;
1571 /* Enable VLAN tag stripping. RSS not enabled (yet).
1574 return enic_set_nic_cfg(enic
,
1575 rss_default_cpu
, rss_hash_type
,
1576 rss_hash_bits
, rss_base_cpu
,
1577 rss_enable
, tso_ipid_split_en
,
1581 static void enic_reset(struct work_struct
*work
)
1583 struct enic
*enic
= container_of(work
, struct enic
, reset
);
1585 if (!netif_running(enic
->netdev
))
1590 spin_lock(&enic
->devcmd_lock
);
1591 vnic_dev_hang_notify(enic
->vdev
);
1592 spin_unlock(&enic
->devcmd_lock
);
1594 enic_stop(enic
->netdev
);
1595 enic_dev_soft_reset(enic
);
1596 vnic_dev_init(enic
->vdev
, 0);
1597 enic_reset_mcaddrs(enic
);
1598 enic_init_vnic_resources(enic
);
1599 enic_set_niccfg(enic
);
1600 enic_open(enic
->netdev
);
1605 static int enic_set_intr_mode(struct enic
*enic
)
1607 unsigned int n
= ARRAY_SIZE(enic
->rq
);
1608 unsigned int m
= ARRAY_SIZE(enic
->wq
);
1611 /* Set interrupt mode (INTx, MSI, MSI-X) depending
1612 * system capabilities.
1616 * We need n RQs, m WQs, n+m CQs, and n+m+2 INTRs
1617 * (the second to last INTR is used for WQ/RQ errors)
1618 * (the last INTR is used for notifications)
1621 BUG_ON(ARRAY_SIZE(enic
->msix_entry
) < n
+ m
+ 2);
1622 for (i
= 0; i
< n
+ m
+ 2; i
++)
1623 enic
->msix_entry
[i
].entry
= i
;
1625 if (enic
->config
.intr_mode
< 1 &&
1626 enic
->rq_count
>= n
&&
1627 enic
->wq_count
>= m
&&
1628 enic
->cq_count
>= n
+ m
&&
1629 enic
->intr_count
>= n
+ m
+ 2 &&
1630 !pci_enable_msix(enic
->pdev
, enic
->msix_entry
, n
+ m
+ 2)) {
1634 enic
->cq_count
= n
+ m
;
1635 enic
->intr_count
= n
+ m
+ 2;
1637 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_MSIX
);
1644 * We need 1 RQ, 1 WQ, 2 CQs, and 1 INTR
1647 if (enic
->config
.intr_mode
< 2 &&
1648 enic
->rq_count
>= 1 &&
1649 enic
->wq_count
>= 1 &&
1650 enic
->cq_count
>= 2 &&
1651 enic
->intr_count
>= 1 &&
1652 !pci_enable_msi(enic
->pdev
)) {
1657 enic
->intr_count
= 1;
1659 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_MSI
);
1666 * We need 1 RQ, 1 WQ, 2 CQs, and 3 INTRs
1667 * (the first INTR is used for WQ/RQ)
1668 * (the second INTR is used for WQ/RQ errors)
1669 * (the last INTR is used for notifications)
1672 if (enic
->config
.intr_mode
< 3 &&
1673 enic
->rq_count
>= 1 &&
1674 enic
->wq_count
>= 1 &&
1675 enic
->cq_count
>= 2 &&
1676 enic
->intr_count
>= 3) {
1681 enic
->intr_count
= 3;
1683 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_INTX
);
1688 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_UNKNOWN
);
1693 static void enic_clear_intr_mode(struct enic
*enic
)
1695 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1696 case VNIC_DEV_INTR_MODE_MSIX
:
1697 pci_disable_msix(enic
->pdev
);
1699 case VNIC_DEV_INTR_MODE_MSI
:
1700 pci_disable_msi(enic
->pdev
);
1706 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_UNKNOWN
);
1709 static const struct net_device_ops enic_netdev_ops
= {
1710 .ndo_open
= enic_open
,
1711 .ndo_stop
= enic_stop
,
1712 .ndo_start_xmit
= enic_hard_start_xmit
,
1713 .ndo_get_stats
= enic_get_stats
,
1714 .ndo_validate_addr
= eth_validate_addr
,
1715 .ndo_set_mac_address
= eth_mac_addr
,
1716 .ndo_set_multicast_list
= enic_set_multicast_list
,
1717 .ndo_change_mtu
= enic_change_mtu
,
1718 .ndo_vlan_rx_register
= enic_vlan_rx_register
,
1719 .ndo_vlan_rx_add_vid
= enic_vlan_rx_add_vid
,
1720 .ndo_vlan_rx_kill_vid
= enic_vlan_rx_kill_vid
,
1721 .ndo_tx_timeout
= enic_tx_timeout
,
1722 #ifdef CONFIG_NET_POLL_CONTROLLER
1723 .ndo_poll_controller
= enic_poll_controller
,
1727 static void enic_iounmap(struct enic
*enic
)
1731 for (i
= 0; i
< ARRAY_SIZE(enic
->bar
); i
++)
1732 if (enic
->bar
[i
].vaddr
)
1733 iounmap(enic
->bar
[i
].vaddr
);
1736 static int __devinit
enic_probe(struct pci_dev
*pdev
,
1737 const struct pci_device_id
*ent
)
1739 struct net_device
*netdev
;
1745 /* Allocate net device structure and initialize. Private
1746 * instance data is initialized to zero.
1749 netdev
= alloc_etherdev(sizeof(struct enic
));
1751 printk(KERN_ERR PFX
"Etherdev alloc failed, aborting.\n");
1755 pci_set_drvdata(pdev
, netdev
);
1757 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
1759 enic
= netdev_priv(netdev
);
1760 enic
->netdev
= netdev
;
1763 /* Setup PCI resources
1766 err
= pci_enable_device(pdev
);
1769 "Cannot enable PCI device, aborting.\n");
1770 goto err_out_free_netdev
;
1773 err
= pci_request_regions(pdev
, DRV_NAME
);
1776 "Cannot request PCI regions, aborting.\n");
1777 goto err_out_disable_device
;
1780 pci_set_master(pdev
);
1782 /* Query PCI controller on system for DMA addressing
1783 * limitation for the device. Try 40-bit first, and
1787 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(40));
1789 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
1792 "No usable DMA configuration, aborting.\n");
1793 goto err_out_release_regions
;
1795 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(32));
1798 "Unable to obtain 32-bit DMA "
1799 "for consistent allocations, aborting.\n");
1800 goto err_out_release_regions
;
1803 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(40));
1806 "Unable to obtain 40-bit DMA "
1807 "for consistent allocations, aborting.\n");
1808 goto err_out_release_regions
;
1813 /* Map vNIC resources from BAR0-5
1816 for (i
= 0; i
< ARRAY_SIZE(enic
->bar
); i
++) {
1817 if (!(pci_resource_flags(pdev
, i
) & IORESOURCE_MEM
))
1819 enic
->bar
[i
].len
= pci_resource_len(pdev
, i
);
1820 enic
->bar
[i
].vaddr
= pci_iomap(pdev
, i
, enic
->bar
[i
].len
);
1821 if (!enic
->bar
[i
].vaddr
) {
1823 "Cannot memory-map BAR %d, aborting.\n", i
);
1825 goto err_out_iounmap
;
1827 enic
->bar
[i
].bus_addr
= pci_resource_start(pdev
, i
);
1830 /* Register vNIC device
1833 enic
->vdev
= vnic_dev_register(NULL
, enic
, pdev
, enic
->bar
,
1834 ARRAY_SIZE(enic
->bar
));
1837 "vNIC registration failed, aborting.\n");
1839 goto err_out_iounmap
;
1842 /* Issue device open to get device in known state
1845 err
= enic_dev_open(enic
);
1848 "vNIC dev open failed, aborting.\n");
1849 goto err_out_vnic_unregister
;
1852 /* Issue device init to initialize the vnic-to-switch link.
1853 * We'll start with carrier off and wait for link UP
1854 * notification later to turn on carrier. We don't need
1855 * to wait here for the vnic-to-switch link initialization
1856 * to complete; link UP notification is the indication that
1857 * the process is complete.
1860 netif_carrier_off(netdev
);
1862 err
= vnic_dev_init(enic
->vdev
, 0);
1865 "vNIC dev init failed, aborting.\n");
1866 goto err_out_dev_close
;
1869 /* Get vNIC configuration
1872 err
= enic_get_vnic_config(enic
);
1875 "Get vNIC configuration failed, aborting.\n");
1876 goto err_out_dev_close
;
1879 /* Get available resource counts
1882 enic_get_res_counts(enic
);
1884 /* Set interrupt mode based on resource counts and system
1888 err
= enic_set_intr_mode(enic
);
1891 "Failed to set intr mode, aborting.\n");
1892 goto err_out_dev_close
;
1895 /* Allocate and configure vNIC resources
1898 err
= enic_alloc_vnic_resources(enic
);
1901 "Failed to alloc vNIC resources, aborting.\n");
1902 goto err_out_free_vnic_resources
;
1905 enic_init_vnic_resources(enic
);
1907 err
= enic_set_niccfg(enic
);
1910 "Failed to config nic, aborting.\n");
1911 goto err_out_free_vnic_resources
;
1914 /* Setup notification timer, HW reset task, and locks
1917 init_timer(&enic
->notify_timer
);
1918 enic
->notify_timer
.function
= enic_notify_timer
;
1919 enic
->notify_timer
.data
= (unsigned long)enic
;
1921 INIT_WORK(&enic
->reset
, enic_reset
);
1923 for (i
= 0; i
< enic
->wq_count
; i
++)
1924 spin_lock_init(&enic
->wq_lock
[i
]);
1926 spin_lock_init(&enic
->devcmd_lock
);
1928 /* Register net device
1931 enic
->port_mtu
= enic
->config
.mtu
;
1932 (void)enic_change_mtu(netdev
, enic
->port_mtu
);
1934 err
= enic_set_mac_addr(netdev
, enic
->mac_addr
);
1937 "Invalid MAC address, aborting.\n");
1938 goto err_out_free_vnic_resources
;
1941 netdev
->netdev_ops
= &enic_netdev_ops
;
1942 netdev
->watchdog_timeo
= 2 * HZ
;
1943 netdev
->ethtool_ops
= &enic_ethtool_ops
;
1945 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1947 netif_napi_add(netdev
, &enic
->napi
, enic_poll
, 64);
1949 case VNIC_DEV_INTR_MODE_MSIX
:
1950 netif_napi_add(netdev
, &enic
->napi
, enic_poll_msix
, 64);
1954 netdev
->features
|= NETIF_F_HW_VLAN_TX
| NETIF_F_HW_VLAN_RX
;
1955 if (ENIC_SETTING(enic
, TXCSUM
))
1956 netdev
->features
|= NETIF_F_SG
| NETIF_F_HW_CSUM
;
1957 if (ENIC_SETTING(enic
, TSO
))
1958 netdev
->features
|= NETIF_F_TSO
|
1959 NETIF_F_TSO6
| NETIF_F_TSO_ECN
;
1960 if (ENIC_SETTING(enic
, LRO
))
1961 netdev
->features
|= NETIF_F_LRO
;
1963 netdev
->features
|= NETIF_F_HIGHDMA
;
1965 enic
->csum_rx_enabled
= ENIC_SETTING(enic
, RXCSUM
);
1967 enic
->lro_mgr
.max_aggr
= ENIC_LRO_MAX_AGGR
;
1968 enic
->lro_mgr
.max_desc
= ENIC_LRO_MAX_DESC
;
1969 enic
->lro_mgr
.lro_arr
= enic
->lro_desc
;
1970 enic
->lro_mgr
.get_skb_header
= enic_get_skb_header
;
1971 enic
->lro_mgr
.features
= LRO_F_NAPI
| LRO_F_EXTRACT_VLAN_ID
;
1972 enic
->lro_mgr
.dev
= netdev
;
1973 enic
->lro_mgr
.ip_summed
= CHECKSUM_COMPLETE
;
1974 enic
->lro_mgr
.ip_summed_aggr
= CHECKSUM_UNNECESSARY
;
1976 err
= register_netdev(netdev
);
1979 "Cannot register net device, aborting.\n");
1980 goto err_out_free_vnic_resources
;
1985 err_out_free_vnic_resources
:
1986 enic_free_vnic_resources(enic
);
1988 vnic_dev_close(enic
->vdev
);
1989 err_out_vnic_unregister
:
1990 enic_clear_intr_mode(enic
);
1991 vnic_dev_unregister(enic
->vdev
);
1994 err_out_release_regions
:
1995 pci_release_regions(pdev
);
1996 err_out_disable_device
:
1997 pci_disable_device(pdev
);
1998 err_out_free_netdev
:
1999 pci_set_drvdata(pdev
, NULL
);
2000 free_netdev(netdev
);
2005 static void __devexit
enic_remove(struct pci_dev
*pdev
)
2007 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2010 struct enic
*enic
= netdev_priv(netdev
);
2012 flush_scheduled_work();
2013 unregister_netdev(netdev
);
2014 enic_free_vnic_resources(enic
);
2015 vnic_dev_close(enic
->vdev
);
2016 enic_clear_intr_mode(enic
);
2017 vnic_dev_unregister(enic
->vdev
);
2019 pci_release_regions(pdev
);
2020 pci_disable_device(pdev
);
2021 pci_set_drvdata(pdev
, NULL
);
2022 free_netdev(netdev
);
2026 static struct pci_driver enic_driver
= {
2028 .id_table
= enic_id_table
,
2029 .probe
= enic_probe
,
2030 .remove
= __devexit_p(enic_remove
),
2033 static int __init
enic_init_module(void)
2035 printk(KERN_INFO PFX
"%s, ver %s\n", DRV_DESCRIPTION
, DRV_VERSION
);
2037 return pci_register_driver(&enic_driver
);
2040 static void __exit
enic_cleanup_module(void)
2042 pci_unregister_driver(&enic_driver
);
2045 module_init(enic_init_module
);
2046 module_exit(enic_cleanup_module
);