2 * Copyright 2008 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 * This program is free software; you may redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; version 2 of the License.
9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 #include <linux/module.h>
21 #include <linux/kernel.h>
22 #include <linux/string.h>
23 #include <linux/errno.h>
24 #include <linux/types.h>
25 #include <linux/init.h>
26 #include <linux/workqueue.h>
27 #include <linux/pci.h>
28 #include <linux/netdevice.h>
29 #include <linux/etherdevice.h>
30 #include <linux/if_ether.h>
31 #include <linux/if_vlan.h>
32 #include <linux/ethtool.h>
35 #include <linux/ipv6.h>
36 #include <linux/tcp.h>
37 #include <net/ip6_checksum.h>
39 #include "cq_enet_desc.h"
41 #include "vnic_intr.h"
42 #include "vnic_stats.h"
46 #define ENIC_NOTIFY_TIMER_PERIOD (2 * HZ)
48 /* Supported devices */
49 static struct pci_device_id enic_id_table
[] = {
50 { PCI_VDEVICE(CISCO
, 0x0043) },
51 { 0, } /* end of table */
54 MODULE_DESCRIPTION(DRV_DESCRIPTION
);
55 MODULE_AUTHOR("Scott Feldman <scofeldm@cisco.com>");
56 MODULE_LICENSE("GPL");
57 MODULE_VERSION(DRV_VERSION
);
58 MODULE_DEVICE_TABLE(pci
, enic_id_table
);
61 char name
[ETH_GSTRING_LEN
];
65 #define ENIC_TX_STAT(stat) \
66 { .name = #stat, .offset = offsetof(struct vnic_tx_stats, stat) / 8 }
67 #define ENIC_RX_STAT(stat) \
68 { .name = #stat, .offset = offsetof(struct vnic_rx_stats, stat) / 8 }
70 static const struct enic_stat enic_tx_stats
[] = {
71 ENIC_TX_STAT(tx_frames_ok
),
72 ENIC_TX_STAT(tx_unicast_frames_ok
),
73 ENIC_TX_STAT(tx_multicast_frames_ok
),
74 ENIC_TX_STAT(tx_broadcast_frames_ok
),
75 ENIC_TX_STAT(tx_bytes_ok
),
76 ENIC_TX_STAT(tx_unicast_bytes_ok
),
77 ENIC_TX_STAT(tx_multicast_bytes_ok
),
78 ENIC_TX_STAT(tx_broadcast_bytes_ok
),
79 ENIC_TX_STAT(tx_drops
),
80 ENIC_TX_STAT(tx_errors
),
84 static const struct enic_stat enic_rx_stats
[] = {
85 ENIC_RX_STAT(rx_frames_ok
),
86 ENIC_RX_STAT(rx_frames_total
),
87 ENIC_RX_STAT(rx_unicast_frames_ok
),
88 ENIC_RX_STAT(rx_multicast_frames_ok
),
89 ENIC_RX_STAT(rx_broadcast_frames_ok
),
90 ENIC_RX_STAT(rx_bytes_ok
),
91 ENIC_RX_STAT(rx_unicast_bytes_ok
),
92 ENIC_RX_STAT(rx_multicast_bytes_ok
),
93 ENIC_RX_STAT(rx_broadcast_bytes_ok
),
94 ENIC_RX_STAT(rx_drop
),
95 ENIC_RX_STAT(rx_no_bufs
),
96 ENIC_RX_STAT(rx_errors
),
98 ENIC_RX_STAT(rx_crc_errors
),
99 ENIC_RX_STAT(rx_frames_64
),
100 ENIC_RX_STAT(rx_frames_127
),
101 ENIC_RX_STAT(rx_frames_255
),
102 ENIC_RX_STAT(rx_frames_511
),
103 ENIC_RX_STAT(rx_frames_1023
),
104 ENIC_RX_STAT(rx_frames_1518
),
105 ENIC_RX_STAT(rx_frames_to_max
),
108 static const unsigned int enic_n_tx_stats
= ARRAY_SIZE(enic_tx_stats
);
109 static const unsigned int enic_n_rx_stats
= ARRAY_SIZE(enic_rx_stats
);
111 static int enic_get_settings(struct net_device
*netdev
,
112 struct ethtool_cmd
*ecmd
)
114 struct enic
*enic
= netdev_priv(netdev
);
116 ecmd
->supported
= (SUPPORTED_10000baseT_Full
| SUPPORTED_FIBRE
);
117 ecmd
->advertising
= (ADVERTISED_10000baseT_Full
| ADVERTISED_FIBRE
);
118 ecmd
->port
= PORT_FIBRE
;
119 ecmd
->transceiver
= XCVR_EXTERNAL
;
121 if (netif_carrier_ok(netdev
)) {
122 ecmd
->speed
= vnic_dev_port_speed(enic
->vdev
);
123 ecmd
->duplex
= DUPLEX_FULL
;
129 ecmd
->autoneg
= AUTONEG_DISABLE
;
134 static void enic_get_drvinfo(struct net_device
*netdev
,
135 struct ethtool_drvinfo
*drvinfo
)
137 struct enic
*enic
= netdev_priv(netdev
);
138 struct vnic_devcmd_fw_info
*fw_info
;
140 spin_lock(&enic
->devcmd_lock
);
141 vnic_dev_fw_info(enic
->vdev
, &fw_info
);
142 spin_unlock(&enic
->devcmd_lock
);
144 strncpy(drvinfo
->driver
, DRV_NAME
, sizeof(drvinfo
->driver
));
145 strncpy(drvinfo
->version
, DRV_VERSION
, sizeof(drvinfo
->version
));
146 strncpy(drvinfo
->fw_version
, fw_info
->fw_version
,
147 sizeof(drvinfo
->fw_version
));
148 strncpy(drvinfo
->bus_info
, pci_name(enic
->pdev
),
149 sizeof(drvinfo
->bus_info
));
152 static void enic_get_strings(struct net_device
*netdev
, u32 stringset
, u8
*data
)
158 for (i
= 0; i
< enic_n_tx_stats
; i
++) {
159 memcpy(data
, enic_tx_stats
[i
].name
, ETH_GSTRING_LEN
);
160 data
+= ETH_GSTRING_LEN
;
162 for (i
= 0; i
< enic_n_rx_stats
; i
++) {
163 memcpy(data
, enic_rx_stats
[i
].name
, ETH_GSTRING_LEN
);
164 data
+= ETH_GSTRING_LEN
;
170 static int enic_get_sset_count(struct net_device
*netdev
, int sset
)
174 return enic_n_tx_stats
+ enic_n_rx_stats
;
180 static void enic_get_ethtool_stats(struct net_device
*netdev
,
181 struct ethtool_stats
*stats
, u64
*data
)
183 struct enic
*enic
= netdev_priv(netdev
);
184 struct vnic_stats
*vstats
;
187 spin_lock(&enic
->devcmd_lock
);
188 vnic_dev_stats_dump(enic
->vdev
, &vstats
);
189 spin_unlock(&enic
->devcmd_lock
);
191 for (i
= 0; i
< enic_n_tx_stats
; i
++)
192 *(data
++) = ((u64
*)&vstats
->tx
)[enic_tx_stats
[i
].offset
];
193 for (i
= 0; i
< enic_n_rx_stats
; i
++)
194 *(data
++) = ((u64
*)&vstats
->rx
)[enic_rx_stats
[i
].offset
];
197 static u32
enic_get_rx_csum(struct net_device
*netdev
)
199 struct enic
*enic
= netdev_priv(netdev
);
200 return enic
->csum_rx_enabled
;
203 static int enic_set_rx_csum(struct net_device
*netdev
, u32 data
)
205 struct enic
*enic
= netdev_priv(netdev
);
207 if (data
&& !ENIC_SETTING(enic
, RXCSUM
))
210 enic
->csum_rx_enabled
= !!data
;
215 static int enic_set_tx_csum(struct net_device
*netdev
, u32 data
)
217 struct enic
*enic
= netdev_priv(netdev
);
219 if (data
&& !ENIC_SETTING(enic
, TXCSUM
))
223 netdev
->features
|= NETIF_F_HW_CSUM
;
225 netdev
->features
&= ~NETIF_F_HW_CSUM
;
230 static int enic_set_tso(struct net_device
*netdev
, u32 data
)
232 struct enic
*enic
= netdev_priv(netdev
);
234 if (data
&& !ENIC_SETTING(enic
, TSO
))
239 NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_TSO_ECN
;
242 ~(NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_TSO_ECN
);
247 static u32
enic_get_msglevel(struct net_device
*netdev
)
249 struct enic
*enic
= netdev_priv(netdev
);
250 return enic
->msg_enable
;
253 static void enic_set_msglevel(struct net_device
*netdev
, u32 value
)
255 struct enic
*enic
= netdev_priv(netdev
);
256 enic
->msg_enable
= value
;
259 static struct ethtool_ops enic_ethtool_ops
= {
260 .get_settings
= enic_get_settings
,
261 .get_drvinfo
= enic_get_drvinfo
,
262 .get_msglevel
= enic_get_msglevel
,
263 .set_msglevel
= enic_set_msglevel
,
264 .get_link
= ethtool_op_get_link
,
265 .get_strings
= enic_get_strings
,
266 .get_sset_count
= enic_get_sset_count
,
267 .get_ethtool_stats
= enic_get_ethtool_stats
,
268 .get_rx_csum
= enic_get_rx_csum
,
269 .set_rx_csum
= enic_set_rx_csum
,
270 .get_tx_csum
= ethtool_op_get_tx_csum
,
271 .set_tx_csum
= enic_set_tx_csum
,
272 .get_sg
= ethtool_op_get_sg
,
273 .set_sg
= ethtool_op_set_sg
,
274 .get_tso
= ethtool_op_get_tso
,
275 .set_tso
= enic_set_tso
,
278 static void enic_free_wq_buf(struct vnic_wq
*wq
, struct vnic_wq_buf
*buf
)
280 struct enic
*enic
= vnic_dev_priv(wq
->vdev
);
283 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
284 buf
->len
, PCI_DMA_TODEVICE
);
286 pci_unmap_page(enic
->pdev
, buf
->dma_addr
,
287 buf
->len
, PCI_DMA_TODEVICE
);
290 dev_kfree_skb_any(buf
->os_buf
);
293 static void enic_wq_free_buf(struct vnic_wq
*wq
,
294 struct cq_desc
*cq_desc
, struct vnic_wq_buf
*buf
, void *opaque
)
296 enic_free_wq_buf(wq
, buf
);
299 static int enic_wq_service(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
300 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
302 struct enic
*enic
= vnic_dev_priv(vdev
);
304 spin_lock(&enic
->wq_lock
[q_number
]);
306 vnic_wq_service(&enic
->wq
[q_number
], cq_desc
,
307 completed_index
, enic_wq_free_buf
,
310 if (netif_queue_stopped(enic
->netdev
) &&
311 vnic_wq_desc_avail(&enic
->wq
[q_number
]) >= MAX_SKB_FRAGS
+ 1)
312 netif_wake_queue(enic
->netdev
);
314 spin_unlock(&enic
->wq_lock
[q_number
]);
319 static void enic_log_q_error(struct enic
*enic
)
324 for (i
= 0; i
< enic
->wq_count
; i
++) {
325 error_status
= vnic_wq_error_status(&enic
->wq
[i
]);
327 printk(KERN_ERR PFX
"%s: WQ[%d] error_status %d\n",
328 enic
->netdev
->name
, i
, error_status
);
331 for (i
= 0; i
< enic
->rq_count
; i
++) {
332 error_status
= vnic_rq_error_status(&enic
->rq
[i
]);
334 printk(KERN_ERR PFX
"%s: RQ[%d] error_status %d\n",
335 enic
->netdev
->name
, i
, error_status
);
339 static void enic_link_check(struct enic
*enic
)
341 int link_status
= vnic_dev_link_status(enic
->vdev
);
342 int carrier_ok
= netif_carrier_ok(enic
->netdev
);
344 if (link_status
&& !carrier_ok
) {
345 printk(KERN_INFO PFX
"%s: Link UP\n", enic
->netdev
->name
);
346 netif_carrier_on(enic
->netdev
);
347 } else if (!link_status
&& carrier_ok
) {
348 printk(KERN_INFO PFX
"%s: Link DOWN\n", enic
->netdev
->name
);
349 netif_carrier_off(enic
->netdev
);
353 static void enic_mtu_check(struct enic
*enic
)
355 u32 mtu
= vnic_dev_mtu(enic
->vdev
);
357 if (mtu
!= enic
->port_mtu
) {
358 if (mtu
< enic
->netdev
->mtu
)
359 printk(KERN_WARNING PFX
360 "%s: interface MTU (%d) set higher "
361 "than switch port MTU (%d)\n",
362 enic
->netdev
->name
, enic
->netdev
->mtu
, mtu
);
363 enic
->port_mtu
= mtu
;
367 static void enic_msglvl_check(struct enic
*enic
)
369 u32 msg_enable
= vnic_dev_msg_lvl(enic
->vdev
);
371 if (msg_enable
!= enic
->msg_enable
) {
372 printk(KERN_INFO PFX
"%s: msg lvl changed from 0x%x to 0x%x\n",
373 enic
->netdev
->name
, enic
->msg_enable
, msg_enable
);
374 enic
->msg_enable
= msg_enable
;
378 static void enic_notify_check(struct enic
*enic
)
380 enic_msglvl_check(enic
);
381 enic_mtu_check(enic
);
382 enic_link_check(enic
);
385 #define ENIC_TEST_INTR(pba, i) (pba & (1 << i))
387 static irqreturn_t
enic_isr_legacy(int irq
, void *data
)
389 struct net_device
*netdev
= data
;
390 struct enic
*enic
= netdev_priv(netdev
);
393 vnic_intr_mask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
395 pba
= vnic_intr_legacy_pba(enic
->legacy_pba
);
397 vnic_intr_unmask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
398 return IRQ_NONE
; /* not our interrupt */
401 if (ENIC_TEST_INTR(pba
, ENIC_INTX_NOTIFY
))
402 enic_notify_check(enic
);
404 if (ENIC_TEST_INTR(pba
, ENIC_INTX_ERR
)) {
405 enic_log_q_error(enic
);
406 /* schedule recovery from WQ/RQ error */
407 schedule_work(&enic
->reset
);
411 if (ENIC_TEST_INTR(pba
, ENIC_INTX_WQ_RQ
)) {
412 if (netif_rx_schedule_prep(netdev
, &enic
->napi
))
413 __netif_rx_schedule(netdev
, &enic
->napi
);
415 vnic_intr_unmask(&enic
->intr
[ENIC_INTX_WQ_RQ
]);
421 static irqreturn_t
enic_isr_msi(int irq
, void *data
)
423 struct enic
*enic
= data
;
425 /* With MSI, there is no sharing of interrupts, so this is
426 * our interrupt and there is no need to ack it. The device
427 * is not providing per-vector masking, so the OS will not
428 * write to PCI config space to mask/unmask the interrupt.
429 * We're using mask_on_assertion for MSI, so the device
430 * automatically masks the interrupt when the interrupt is
431 * generated. Later, when exiting polling, the interrupt
432 * will be unmasked (see enic_poll).
434 * Also, the device uses the same PCIe Traffic Class (TC)
435 * for Memory Write data and MSI, so there are no ordering
436 * issues; the MSI will always arrive at the Root Complex
437 * _after_ corresponding Memory Writes (i.e. descriptor
441 netif_rx_schedule(enic
->netdev
, &enic
->napi
);
446 static irqreturn_t
enic_isr_msix_rq(int irq
, void *data
)
448 struct enic
*enic
= data
;
450 /* schedule NAPI polling for RQ cleanup */
451 netif_rx_schedule(enic
->netdev
, &enic
->napi
);
456 static irqreturn_t
enic_isr_msix_wq(int irq
, void *data
)
458 struct enic
*enic
= data
;
459 unsigned int wq_work_to_do
= -1; /* no limit */
460 unsigned int wq_work_done
;
462 wq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
463 wq_work_to_do
, enic_wq_service
, NULL
);
465 vnic_intr_return_credits(&enic
->intr
[ENIC_MSIX_WQ
],
468 1 /* reset intr timer */);
473 static irqreturn_t
enic_isr_msix_err(int irq
, void *data
)
475 struct enic
*enic
= data
;
477 enic_log_q_error(enic
);
479 /* schedule recovery from WQ/RQ error */
480 schedule_work(&enic
->reset
);
485 static irqreturn_t
enic_isr_msix_notify(int irq
, void *data
)
487 struct enic
*enic
= data
;
489 enic_notify_check(enic
);
490 vnic_intr_unmask(&enic
->intr
[ENIC_MSIX_NOTIFY
]);
495 static inline void enic_queue_wq_skb_cont(struct enic
*enic
,
496 struct vnic_wq
*wq
, struct sk_buff
*skb
,
497 unsigned int len_left
)
501 /* Queue additional data fragments */
502 for (frag
= skb_shinfo(skb
)->frags
; len_left
; frag
++) {
503 len_left
-= frag
->size
;
504 enic_queue_wq_desc_cont(wq
, skb
,
505 pci_map_page(enic
->pdev
, frag
->page
,
506 frag
->page_offset
, frag
->size
,
509 (len_left
== 0)); /* EOP? */
513 static inline void enic_queue_wq_skb_vlan(struct enic
*enic
,
514 struct vnic_wq
*wq
, struct sk_buff
*skb
,
515 int vlan_tag_insert
, unsigned int vlan_tag
)
517 unsigned int head_len
= skb_headlen(skb
);
518 unsigned int len_left
= skb
->len
- head_len
;
519 int eop
= (len_left
== 0);
521 /* Queue the main skb fragment */
522 enic_queue_wq_desc(wq
, skb
,
523 pci_map_single(enic
->pdev
, skb
->data
,
524 head_len
, PCI_DMA_TODEVICE
),
526 vlan_tag_insert
, vlan_tag
,
530 enic_queue_wq_skb_cont(enic
, wq
, skb
, len_left
);
533 static inline void enic_queue_wq_skb_csum_l4(struct enic
*enic
,
534 struct vnic_wq
*wq
, struct sk_buff
*skb
,
535 int vlan_tag_insert
, unsigned int vlan_tag
)
537 unsigned int head_len
= skb_headlen(skb
);
538 unsigned int len_left
= skb
->len
- head_len
;
539 unsigned int hdr_len
= skb_transport_offset(skb
);
540 unsigned int csum_offset
= hdr_len
+ skb
->csum_offset
;
541 int eop
= (len_left
== 0);
543 /* Queue the main skb fragment */
544 enic_queue_wq_desc_csum_l4(wq
, skb
,
545 pci_map_single(enic
->pdev
, skb
->data
,
546 head_len
, PCI_DMA_TODEVICE
),
550 vlan_tag_insert
, vlan_tag
,
554 enic_queue_wq_skb_cont(enic
, wq
, skb
, len_left
);
557 static inline void enic_queue_wq_skb_tso(struct enic
*enic
,
558 struct vnic_wq
*wq
, struct sk_buff
*skb
, unsigned int mss
,
559 int vlan_tag_insert
, unsigned int vlan_tag
)
561 unsigned int head_len
= skb_headlen(skb
);
562 unsigned int len_left
= skb
->len
- head_len
;
563 unsigned int hdr_len
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
564 int eop
= (len_left
== 0);
566 /* Preload TCP csum field with IP pseudo hdr calculated
567 * with IP length set to zero. HW will later add in length
568 * to each TCP segment resulting from the TSO.
571 if (skb
->protocol
== __constant_htons(ETH_P_IP
)) {
572 ip_hdr(skb
)->check
= 0;
573 tcp_hdr(skb
)->check
= ~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
574 ip_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
575 } else if (skb
->protocol
== __constant_htons(ETH_P_IPV6
)) {
576 tcp_hdr(skb
)->check
= ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
577 &ipv6_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
580 /* Queue the main skb fragment */
581 enic_queue_wq_desc_tso(wq
, skb
,
582 pci_map_single(enic
->pdev
, skb
->data
,
583 head_len
, PCI_DMA_TODEVICE
),
586 vlan_tag_insert
, vlan_tag
,
590 enic_queue_wq_skb_cont(enic
, wq
, skb
, len_left
);
593 static inline void enic_queue_wq_skb(struct enic
*enic
,
594 struct vnic_wq
*wq
, struct sk_buff
*skb
)
596 unsigned int mss
= skb_shinfo(skb
)->gso_size
;
597 unsigned int vlan_tag
= 0;
598 int vlan_tag_insert
= 0;
600 if (enic
->vlan_group
&& vlan_tx_tag_present(skb
)) {
601 /* VLAN tag from trunking driver */
603 vlan_tag
= vlan_tx_tag_get(skb
);
607 enic_queue_wq_skb_tso(enic
, wq
, skb
, mss
,
608 vlan_tag_insert
, vlan_tag
);
609 else if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
610 enic_queue_wq_skb_csum_l4(enic
, wq
, skb
,
611 vlan_tag_insert
, vlan_tag
);
613 enic_queue_wq_skb_vlan(enic
, wq
, skb
,
614 vlan_tag_insert
, vlan_tag
);
617 /* netif_tx_lock held, process context with BHs disabled */
618 static int enic_hard_start_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
620 struct enic
*enic
= netdev_priv(netdev
);
621 struct vnic_wq
*wq
= &enic
->wq
[0];
629 /* Non-TSO sends must fit within ENIC_NON_TSO_MAX_DESC descs,
630 * which is very likely. In the off chance it's going to take
631 * more than * ENIC_NON_TSO_MAX_DESC, linearize the skb.
634 if (skb_shinfo(skb
)->gso_size
== 0 &&
635 skb_shinfo(skb
)->nr_frags
+ 1 > ENIC_NON_TSO_MAX_DESC
&&
636 skb_linearize(skb
)) {
641 spin_lock_irqsave(&enic
->wq_lock
[0], flags
);
643 if (vnic_wq_desc_avail(wq
) < skb_shinfo(skb
)->nr_frags
+ 1) {
644 netif_stop_queue(netdev
);
645 /* This is a hard error, log it */
646 printk(KERN_ERR PFX
"%s: BUG! Tx ring full when "
647 "queue awake!\n", netdev
->name
);
648 spin_unlock_irqrestore(&enic
->wq_lock
[0], flags
);
649 return NETDEV_TX_BUSY
;
652 enic_queue_wq_skb(enic
, wq
, skb
);
654 if (vnic_wq_desc_avail(wq
) < MAX_SKB_FRAGS
+ 1)
655 netif_stop_queue(netdev
);
657 netdev
->trans_start
= jiffies
;
659 spin_unlock_irqrestore(&enic
->wq_lock
[0], flags
);
664 /* dev_base_lock rwlock held, nominally process context */
665 static struct net_device_stats
*enic_get_stats(struct net_device
*netdev
)
667 struct enic
*enic
= netdev_priv(netdev
);
668 struct net_device_stats
*net_stats
= &netdev
->stats
;
669 struct vnic_stats
*stats
;
671 spin_lock(&enic
->devcmd_lock
);
672 vnic_dev_stats_dump(enic
->vdev
, &stats
);
673 spin_unlock(&enic
->devcmd_lock
);
675 net_stats
->tx_packets
= stats
->tx
.tx_frames_ok
;
676 net_stats
->tx_bytes
= stats
->tx
.tx_bytes_ok
;
677 net_stats
->tx_errors
= stats
->tx
.tx_errors
;
678 net_stats
->tx_dropped
= stats
->tx
.tx_drops
;
680 net_stats
->rx_packets
= stats
->rx
.rx_frames_ok
;
681 net_stats
->rx_bytes
= stats
->rx
.rx_bytes_ok
;
682 net_stats
->rx_errors
= stats
->rx
.rx_errors
;
683 net_stats
->multicast
= stats
->rx
.rx_multicast_frames_ok
;
684 net_stats
->rx_crc_errors
= stats
->rx
.rx_crc_errors
;
685 net_stats
->rx_dropped
= stats
->rx
.rx_no_bufs
;
690 static void enic_reset_mcaddrs(struct enic
*enic
)
695 static int enic_set_mac_addr(struct net_device
*netdev
, char *addr
)
697 if (!is_valid_ether_addr(addr
))
698 return -EADDRNOTAVAIL
;
700 memcpy(netdev
->dev_addr
, addr
, netdev
->addr_len
);
705 /* netif_tx_lock held, BHs disabled */
706 static void enic_set_multicast_list(struct net_device
*netdev
)
708 struct enic
*enic
= netdev_priv(netdev
);
709 struct dev_mc_list
*list
= netdev
->mc_list
;
711 int multicast
= (netdev
->flags
& IFF_MULTICAST
) ? 1 : 0;
712 int broadcast
= (netdev
->flags
& IFF_BROADCAST
) ? 1 : 0;
713 int promisc
= (netdev
->flags
& IFF_PROMISC
) ? 1 : 0;
714 int allmulti
= (netdev
->flags
& IFF_ALLMULTI
) ||
715 (netdev
->mc_count
> ENIC_MULTICAST_PERFECT_FILTERS
);
716 u8 mc_addr
[ENIC_MULTICAST_PERFECT_FILTERS
][ETH_ALEN
];
717 unsigned int mc_count
= netdev
->mc_count
;
720 if (mc_count
> ENIC_MULTICAST_PERFECT_FILTERS
)
721 mc_count
= ENIC_MULTICAST_PERFECT_FILTERS
;
723 spin_lock(&enic
->devcmd_lock
);
725 vnic_dev_packet_filter(enic
->vdev
, directed
,
726 multicast
, broadcast
, promisc
, allmulti
);
728 /* Is there an easier way? Trying to minimize to
729 * calls to add/del multicast addrs. We keep the
730 * addrs from the last call in enic->mc_addr and
731 * look for changes to add/del.
734 for (i
= 0; list
&& i
< mc_count
; i
++) {
735 memcpy(mc_addr
[i
], list
->dmi_addr
, ETH_ALEN
);
739 for (i
= 0; i
< enic
->mc_count
; i
++) {
740 for (j
= 0; j
< mc_count
; j
++)
741 if (compare_ether_addr(enic
->mc_addr
[i
],
745 enic_del_multicast_addr(enic
, enic
->mc_addr
[i
]);
748 for (i
= 0; i
< mc_count
; i
++) {
749 for (j
= 0; j
< enic
->mc_count
; j
++)
750 if (compare_ether_addr(mc_addr
[i
],
751 enic
->mc_addr
[j
]) == 0)
753 if (j
== enic
->mc_count
)
754 enic_add_multicast_addr(enic
, mc_addr
[i
]);
757 /* Save the list to compare against next time
760 for (i
= 0; i
< mc_count
; i
++)
761 memcpy(enic
->mc_addr
[i
], mc_addr
[i
], ETH_ALEN
);
763 enic
->mc_count
= mc_count
;
765 spin_unlock(&enic
->devcmd_lock
);
768 /* rtnl lock is held */
769 static void enic_vlan_rx_register(struct net_device
*netdev
,
770 struct vlan_group
*vlan_group
)
772 struct enic
*enic
= netdev_priv(netdev
);
773 enic
->vlan_group
= vlan_group
;
776 /* rtnl lock is held */
777 static void enic_vlan_rx_add_vid(struct net_device
*netdev
, u16 vid
)
779 struct enic
*enic
= netdev_priv(netdev
);
781 spin_lock(&enic
->devcmd_lock
);
782 enic_add_vlan(enic
, vid
);
783 spin_unlock(&enic
->devcmd_lock
);
786 /* rtnl lock is held */
787 static void enic_vlan_rx_kill_vid(struct net_device
*netdev
, u16 vid
)
789 struct enic
*enic
= netdev_priv(netdev
);
791 spin_lock(&enic
->devcmd_lock
);
792 enic_del_vlan(enic
, vid
);
793 spin_unlock(&enic
->devcmd_lock
);
796 /* netif_tx_lock held, BHs disabled */
797 static void enic_tx_timeout(struct net_device
*netdev
)
799 struct enic
*enic
= netdev_priv(netdev
);
800 schedule_work(&enic
->reset
);
803 static void enic_free_rq_buf(struct vnic_rq
*rq
, struct vnic_rq_buf
*buf
)
805 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
810 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
811 buf
->len
, PCI_DMA_FROMDEVICE
);
812 dev_kfree_skb_any(buf
->os_buf
);
815 static inline struct sk_buff
*enic_rq_alloc_skb(unsigned int size
)
819 skb
= dev_alloc_skb(size
+ NET_IP_ALIGN
);
822 skb_reserve(skb
, NET_IP_ALIGN
);
827 static int enic_rq_alloc_buf(struct vnic_rq
*rq
)
829 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
831 unsigned int len
= enic
->netdev
->mtu
+ ETH_HLEN
;
832 unsigned int os_buf_index
= 0;
835 skb
= enic_rq_alloc_skb(len
);
839 dma_addr
= pci_map_single(enic
->pdev
, skb
->data
,
840 len
, PCI_DMA_FROMDEVICE
);
842 enic_queue_rq_desc(rq
, skb
, os_buf_index
,
848 static int enic_get_skb_header(struct sk_buff
*skb
, void **iphdr
,
849 void **tcph
, u64
*hdr_flags
, void *priv
)
851 struct cq_enet_rq_desc
*cq_desc
= priv
;
855 u8 type
, color
, eop
, sop
, ingress_port
, vlan_stripped
;
856 u8 fcoe
, fcoe_sof
, fcoe_fc_crc_ok
, fcoe_enc_error
, fcoe_eof
;
857 u8 tcp_udp_csum_ok
, udp
, tcp
, ipv4_csum_ok
;
858 u8 ipv6
, ipv4
, ipv4_fragment
, fcs_ok
, rss_type
, csum_not_calc
;
860 u16 q_number
, completed_index
, bytes_written
, vlan
, checksum
;
863 cq_enet_rq_desc_dec(cq_desc
,
864 &type
, &color
, &q_number
, &completed_index
,
865 &ingress_port
, &fcoe
, &eop
, &sop
, &rss_type
,
866 &csum_not_calc
, &rss_hash
, &bytes_written
,
867 &packet_error
, &vlan_stripped
, &vlan
, &checksum
,
868 &fcoe_sof
, &fcoe_fc_crc_ok
, &fcoe_enc_error
,
869 &fcoe_eof
, &tcp_udp_csum_ok
, &udp
, &tcp
,
870 &ipv4_csum_ok
, &ipv6
, &ipv4
, &ipv4_fragment
,
873 if (!(ipv4
&& tcp
&& !ipv4_fragment
))
876 skb_reset_network_header(skb
);
879 ip_len
= ip_hdrlen(skb
);
880 skb_set_transport_header(skb
, ip_len
);
882 /* check if ip header and tcp header are complete */
883 if (ntohs(iph
->tot_len
) < ip_len
+ tcp_hdrlen(skb
))
886 *hdr_flags
= LRO_IPV4
| LRO_TCP
;
887 *tcph
= tcp_hdr(skb
);
893 static void enic_rq_indicate_buf(struct vnic_rq
*rq
,
894 struct cq_desc
*cq_desc
, struct vnic_rq_buf
*buf
,
895 int skipped
, void *opaque
)
897 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
900 u8 type
, color
, eop
, sop
, ingress_port
, vlan_stripped
;
901 u8 fcoe
, fcoe_sof
, fcoe_fc_crc_ok
, fcoe_enc_error
, fcoe_eof
;
902 u8 tcp_udp_csum_ok
, udp
, tcp
, ipv4_csum_ok
;
903 u8 ipv6
, ipv4
, ipv4_fragment
, fcs_ok
, rss_type
, csum_not_calc
;
905 u16 q_number
, completed_index
, bytes_written
, vlan
, checksum
;
912 prefetch(skb
->data
- NET_IP_ALIGN
);
913 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
914 buf
->len
, PCI_DMA_FROMDEVICE
);
916 cq_enet_rq_desc_dec((struct cq_enet_rq_desc
*)cq_desc
,
917 &type
, &color
, &q_number
, &completed_index
,
918 &ingress_port
, &fcoe
, &eop
, &sop
, &rss_type
,
919 &csum_not_calc
, &rss_hash
, &bytes_written
,
920 &packet_error
, &vlan_stripped
, &vlan
, &checksum
,
921 &fcoe_sof
, &fcoe_fc_crc_ok
, &fcoe_enc_error
,
922 &fcoe_eof
, &tcp_udp_csum_ok
, &udp
, &tcp
,
923 &ipv4_csum_ok
, &ipv6
, &ipv4
, &ipv4_fragment
,
928 if (bytes_written
> 0 && !fcs_ok
) {
931 "%s: packet error: bad FCS\n",
935 dev_kfree_skb_any(skb
);
940 if (eop
&& bytes_written
> 0) {
945 skb_put(skb
, bytes_written
);
946 skb
->protocol
= eth_type_trans(skb
, enic
->netdev
);
948 if (enic
->csum_rx_enabled
&& !csum_not_calc
) {
949 skb
->csum
= htons(checksum
);
950 skb
->ip_summed
= CHECKSUM_COMPLETE
;
953 skb
->dev
= enic
->netdev
;
955 if (enic
->vlan_group
&& vlan_stripped
) {
957 if (ENIC_SETTING(enic
, LRO
) && ipv4
)
958 lro_vlan_hwaccel_receive_skb(&enic
->lro_mgr
,
959 skb
, enic
->vlan_group
,
962 vlan_hwaccel_receive_skb(skb
,
963 enic
->vlan_group
, vlan
);
967 if (ENIC_SETTING(enic
, LRO
) && ipv4
)
968 lro_receive_skb(&enic
->lro_mgr
, skb
, cq_desc
);
970 netif_receive_skb(skb
);
979 dev_kfree_skb_any(skb
);
983 static int enic_rq_service(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
984 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
986 struct enic
*enic
= vnic_dev_priv(vdev
);
988 vnic_rq_service(&enic
->rq
[q_number
], cq_desc
,
989 completed_index
, VNIC_RQ_RETURN_DESC
,
990 enic_rq_indicate_buf
, opaque
);
995 static void enic_rq_drop_buf(struct vnic_rq
*rq
,
996 struct cq_desc
*cq_desc
, struct vnic_rq_buf
*buf
,
997 int skipped
, void *opaque
)
999 struct enic
*enic
= vnic_dev_priv(rq
->vdev
);
1000 struct sk_buff
*skb
= buf
->os_buf
;
1005 pci_unmap_single(enic
->pdev
, buf
->dma_addr
,
1006 buf
->len
, PCI_DMA_FROMDEVICE
);
1008 dev_kfree_skb_any(skb
);
1011 static int enic_rq_service_drop(struct vnic_dev
*vdev
, struct cq_desc
*cq_desc
,
1012 u8 type
, u16 q_number
, u16 completed_index
, void *opaque
)
1014 struct enic
*enic
= vnic_dev_priv(vdev
);
1016 vnic_rq_service(&enic
->rq
[q_number
], cq_desc
,
1017 completed_index
, VNIC_RQ_RETURN_DESC
,
1018 enic_rq_drop_buf
, opaque
);
1023 static int enic_poll(struct napi_struct
*napi
, int budget
)
1025 struct enic
*enic
= container_of(napi
, struct enic
, napi
);
1026 struct net_device
*netdev
= enic
->netdev
;
1027 unsigned int rq_work_to_do
= budget
;
1028 unsigned int wq_work_to_do
= -1; /* no limit */
1029 unsigned int work_done
, rq_work_done
, wq_work_done
;
1031 /* Service RQ (first) and WQ
1034 rq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1035 rq_work_to_do
, enic_rq_service
, NULL
);
1037 wq_work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
1038 wq_work_to_do
, enic_wq_service
, NULL
);
1040 /* Accumulate intr event credits for this polling
1041 * cycle. An intr event is the completion of a
1042 * a WQ or RQ packet.
1045 work_done
= rq_work_done
+ wq_work_done
;
1048 vnic_intr_return_credits(&enic
->intr
[ENIC_INTX_WQ_RQ
],
1050 0 /* don't unmask intr */,
1051 0 /* don't reset intr timer */);
1053 if (rq_work_done
> 0) {
1058 vnic_rq_fill(&enic
->rq
[0], enic_rq_alloc_buf
);
1062 /* If no work done, flush all LROs and exit polling
1065 if (ENIC_SETTING(enic
, LRO
))
1066 lro_flush_all(&enic
->lro_mgr
);
1068 netif_rx_complete(netdev
, napi
);
1069 vnic_intr_unmask(&enic
->intr
[ENIC_MSIX_RQ
]);
1072 return rq_work_done
;
1075 static int enic_poll_msix(struct napi_struct
*napi
, int budget
)
1077 struct enic
*enic
= container_of(napi
, struct enic
, napi
);
1078 struct net_device
*netdev
= enic
->netdev
;
1079 unsigned int work_to_do
= budget
;
1080 unsigned int work_done
;
1085 work_done
= vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1086 work_to_do
, enic_rq_service
, NULL
);
1088 if (work_done
> 0) {
1093 vnic_rq_fill(&enic
->rq
[0], enic_rq_alloc_buf
);
1095 /* Accumulate intr event credits for this polling
1096 * cycle. An intr event is the completion of a
1097 * a WQ or RQ packet.
1100 vnic_intr_return_credits(&enic
->intr
[ENIC_MSIX_RQ
],
1102 0 /* don't unmask intr */,
1103 0 /* don't reset intr timer */);
1106 /* If no work done, flush all LROs and exit polling
1109 if (ENIC_SETTING(enic
, LRO
))
1110 lro_flush_all(&enic
->lro_mgr
);
1112 netif_rx_complete(netdev
, napi
);
1113 vnic_intr_unmask(&enic
->intr
[ENIC_MSIX_RQ
]);
1119 static void enic_notify_timer(unsigned long data
)
1121 struct enic
*enic
= (struct enic
*)data
;
1123 enic_notify_check(enic
);
1125 mod_timer(&enic
->notify_timer
,
1126 round_jiffies(jiffies
+ ENIC_NOTIFY_TIMER_PERIOD
));
1129 static void enic_free_intr(struct enic
*enic
)
1131 struct net_device
*netdev
= enic
->netdev
;
1134 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1135 case VNIC_DEV_INTR_MODE_INTX
:
1136 free_irq(enic
->pdev
->irq
, netdev
);
1138 case VNIC_DEV_INTR_MODE_MSI
:
1139 free_irq(enic
->pdev
->irq
, enic
);
1141 case VNIC_DEV_INTR_MODE_MSIX
:
1142 for (i
= 0; i
< ARRAY_SIZE(enic
->msix
); i
++)
1143 if (enic
->msix
[i
].requested
)
1144 free_irq(enic
->msix_entry
[i
].vector
,
1145 enic
->msix
[i
].devid
);
1152 static int enic_request_intr(struct enic
*enic
)
1154 struct net_device
*netdev
= enic
->netdev
;
1158 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1160 case VNIC_DEV_INTR_MODE_INTX
:
1162 err
= request_irq(enic
->pdev
->irq
, enic_isr_legacy
,
1163 IRQF_SHARED
, netdev
->name
, netdev
);
1166 case VNIC_DEV_INTR_MODE_MSI
:
1168 err
= request_irq(enic
->pdev
->irq
, enic_isr_msi
,
1169 0, netdev
->name
, enic
);
1172 case VNIC_DEV_INTR_MODE_MSIX
:
1174 sprintf(enic
->msix
[ENIC_MSIX_RQ
].devname
,
1175 "%.11s-rx-0", netdev
->name
);
1176 enic
->msix
[ENIC_MSIX_RQ
].isr
= enic_isr_msix_rq
;
1177 enic
->msix
[ENIC_MSIX_RQ
].devid
= enic
;
1179 sprintf(enic
->msix
[ENIC_MSIX_WQ
].devname
,
1180 "%.11s-tx-0", netdev
->name
);
1181 enic
->msix
[ENIC_MSIX_WQ
].isr
= enic_isr_msix_wq
;
1182 enic
->msix
[ENIC_MSIX_WQ
].devid
= enic
;
1184 sprintf(enic
->msix
[ENIC_MSIX_ERR
].devname
,
1185 "%.11s-err", netdev
->name
);
1186 enic
->msix
[ENIC_MSIX_ERR
].isr
= enic_isr_msix_err
;
1187 enic
->msix
[ENIC_MSIX_ERR
].devid
= enic
;
1189 sprintf(enic
->msix
[ENIC_MSIX_NOTIFY
].devname
,
1190 "%.11s-notify", netdev
->name
);
1191 enic
->msix
[ENIC_MSIX_NOTIFY
].isr
= enic_isr_msix_notify
;
1192 enic
->msix
[ENIC_MSIX_NOTIFY
].devid
= enic
;
1194 for (i
= 0; i
< ARRAY_SIZE(enic
->msix
); i
++) {
1195 err
= request_irq(enic
->msix_entry
[i
].vector
,
1196 enic
->msix
[i
].isr
, 0,
1197 enic
->msix
[i
].devname
,
1198 enic
->msix
[i
].devid
);
1200 enic_free_intr(enic
);
1203 enic
->msix
[i
].requested
= 1;
1215 static int enic_notify_set(struct enic
*enic
)
1219 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1220 case VNIC_DEV_INTR_MODE_INTX
:
1221 err
= vnic_dev_notify_set(enic
->vdev
, ENIC_INTX_NOTIFY
);
1223 case VNIC_DEV_INTR_MODE_MSIX
:
1224 err
= vnic_dev_notify_set(enic
->vdev
, ENIC_MSIX_NOTIFY
);
1227 err
= vnic_dev_notify_set(enic
->vdev
, -1 /* no intr */);
1234 static void enic_notify_timer_start(struct enic
*enic
)
1236 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1237 case VNIC_DEV_INTR_MODE_MSI
:
1238 mod_timer(&enic
->notify_timer
, jiffies
);
1241 /* Using intr for notification for INTx/MSI-X */
1246 /* rtnl lock is held, process context */
1247 static int enic_open(struct net_device
*netdev
)
1249 struct enic
*enic
= netdev_priv(netdev
);
1253 err
= enic_request_intr(enic
);
1255 printk(KERN_ERR PFX
"%s: Unable to request irq.\n",
1260 err
= enic_notify_set(enic
);
1263 "%s: Failed to alloc notify buffer, aborting.\n",
1265 goto err_out_free_intr
;
1268 for (i
= 0; i
< enic
->rq_count
; i
++) {
1269 err
= vnic_rq_fill(&enic
->rq
[i
], enic_rq_alloc_buf
);
1272 "%s: Unable to alloc receive buffers.\n",
1274 goto err_out_notify_unset
;
1278 for (i
= 0; i
< enic
->wq_count
; i
++)
1279 vnic_wq_enable(&enic
->wq
[i
]);
1280 for (i
= 0; i
< enic
->rq_count
; i
++)
1281 vnic_rq_enable(&enic
->rq
[i
]);
1283 enic_add_station_addr(enic
);
1284 enic_set_multicast_list(netdev
);
1286 netif_wake_queue(netdev
);
1287 napi_enable(&enic
->napi
);
1288 vnic_dev_enable(enic
->vdev
);
1290 for (i
= 0; i
< enic
->intr_count
; i
++)
1291 vnic_intr_unmask(&enic
->intr
[i
]);
1293 enic_notify_timer_start(enic
);
1297 err_out_notify_unset
:
1298 vnic_dev_notify_unset(enic
->vdev
);
1300 enic_free_intr(enic
);
1305 /* rtnl lock is held, process context */
1306 static int enic_stop(struct net_device
*netdev
)
1308 struct enic
*enic
= netdev_priv(netdev
);
1312 del_timer_sync(&enic
->notify_timer
);
1314 vnic_dev_disable(enic
->vdev
);
1315 napi_disable(&enic
->napi
);
1316 netif_stop_queue(netdev
);
1318 for (i
= 0; i
< enic
->intr_count
; i
++)
1319 vnic_intr_mask(&enic
->intr
[i
]);
1321 for (i
= 0; i
< enic
->wq_count
; i
++) {
1322 err
= vnic_wq_disable(&enic
->wq
[i
]);
1326 for (i
= 0; i
< enic
->rq_count
; i
++) {
1327 err
= vnic_rq_disable(&enic
->rq
[i
]);
1332 vnic_dev_notify_unset(enic
->vdev
);
1333 enic_free_intr(enic
);
1335 (void)vnic_cq_service(&enic
->cq
[ENIC_CQ_RQ
],
1336 -1, enic_rq_service_drop
, NULL
);
1337 (void)vnic_cq_service(&enic
->cq
[ENIC_CQ_WQ
],
1338 -1, enic_wq_service
, NULL
);
1340 for (i
= 0; i
< enic
->wq_count
; i
++)
1341 vnic_wq_clean(&enic
->wq
[i
], enic_free_wq_buf
);
1342 for (i
= 0; i
< enic
->rq_count
; i
++)
1343 vnic_rq_clean(&enic
->rq
[i
], enic_free_rq_buf
);
1344 for (i
= 0; i
< enic
->cq_count
; i
++)
1345 vnic_cq_clean(&enic
->cq
[i
]);
1346 for (i
= 0; i
< enic
->intr_count
; i
++)
1347 vnic_intr_clean(&enic
->intr
[i
]);
1352 static int enic_change_mtu(struct net_device
*netdev
, int new_mtu
)
1354 struct enic
*enic
= netdev_priv(netdev
);
1355 int running
= netif_running(netdev
);
1357 if (new_mtu
< ENIC_MIN_MTU
|| new_mtu
> ENIC_MAX_MTU
)
1363 netdev
->mtu
= new_mtu
;
1365 if (netdev
->mtu
> enic
->port_mtu
)
1366 printk(KERN_WARNING PFX
1367 "%s: interface MTU (%d) set higher "
1368 "than port MTU (%d)\n",
1369 netdev
->name
, netdev
->mtu
, enic
->port_mtu
);
1377 #ifdef CONFIG_NET_POLL_CONTROLLER
1378 static void enic_poll_controller(struct net_device
*netdev
)
1380 struct enic
*enic
= netdev_priv(netdev
);
1381 struct vnic_dev
*vdev
= enic
->vdev
;
1383 switch (vnic_dev_get_intr_mode(vdev
)) {
1384 case VNIC_DEV_INTR_MODE_MSIX
:
1385 enic_isr_msix_rq(enic
->pdev
->irq
, enic
);
1386 enic_isr_msix_wq(enic
->pdev
->irq
, enic
);
1388 case VNIC_DEV_INTR_MODE_MSI
:
1389 enic_isr_msi(enic
->pdev
->irq
, enic
);
1391 case VNIC_DEV_INTR_MODE_INTX
:
1392 enic_isr_legacy(enic
->pdev
->irq
, netdev
);
1400 static int enic_dev_wait(struct vnic_dev
*vdev
,
1401 int (*start
)(struct vnic_dev
*, int),
1402 int (*finished
)(struct vnic_dev
*, int *),
1409 BUG_ON(in_interrupt());
1411 err
= start(vdev
, arg
);
1415 /* Wait for func to complete...2 seconds max
1418 time
= jiffies
+ (HZ
* 2);
1421 err
= finished(vdev
, &done
);
1428 schedule_timeout_uninterruptible(HZ
/ 10);
1430 } while (time_after(time
, jiffies
));
1435 static int enic_dev_open(struct enic
*enic
)
1439 err
= enic_dev_wait(enic
->vdev
, vnic_dev_open
,
1440 vnic_dev_open_done
, 0);
1443 "vNIC device open failed, err %d.\n", err
);
1448 static int enic_dev_soft_reset(struct enic
*enic
)
1452 err
= enic_dev_wait(enic
->vdev
, vnic_dev_soft_reset
,
1453 vnic_dev_soft_reset_done
, 0);
1456 "vNIC soft reset failed, err %d.\n", err
);
1461 static void enic_reset(struct work_struct
*work
)
1463 struct enic
*enic
= container_of(work
, struct enic
, reset
);
1465 if (!netif_running(enic
->netdev
))
1470 spin_lock(&enic
->devcmd_lock
);
1471 vnic_dev_hang_notify(enic
->vdev
);
1472 spin_unlock(&enic
->devcmd_lock
);
1474 enic_stop(enic
->netdev
);
1475 enic_dev_soft_reset(enic
);
1476 enic_reset_mcaddrs(enic
);
1477 enic_init_vnic_resources(enic
);
1478 enic_open(enic
->netdev
);
1483 static int enic_set_intr_mode(struct enic
*enic
)
1485 unsigned int n
= ARRAY_SIZE(enic
->rq
);
1486 unsigned int m
= ARRAY_SIZE(enic
->wq
);
1489 /* Set interrupt mode (INTx, MSI, MSI-X) depending
1490 * system capabilities.
1494 * We need n RQs, m WQs, n+m CQs, and n+m+2 INTRs
1495 * (the second to last INTR is used for WQ/RQ errors)
1496 * (the last INTR is used for notifications)
1499 BUG_ON(ARRAY_SIZE(enic
->msix_entry
) < n
+ m
+ 2);
1500 for (i
= 0; i
< n
+ m
+ 2; i
++)
1501 enic
->msix_entry
[i
].entry
= i
;
1503 if (enic
->config
.intr_mode
< 1 &&
1504 enic
->rq_count
>= n
&&
1505 enic
->wq_count
>= m
&&
1506 enic
->cq_count
>= n
+ m
&&
1507 enic
->intr_count
>= n
+ m
+ 2 &&
1508 !pci_enable_msix(enic
->pdev
, enic
->msix_entry
, n
+ m
+ 2)) {
1512 enic
->cq_count
= n
+ m
;
1513 enic
->intr_count
= n
+ m
+ 2;
1515 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_MSIX
);
1522 * We need 1 RQ, 1 WQ, 2 CQs, and 1 INTR
1525 if (enic
->config
.intr_mode
< 2 &&
1526 enic
->rq_count
>= 1 &&
1527 enic
->wq_count
>= 1 &&
1528 enic
->cq_count
>= 2 &&
1529 enic
->intr_count
>= 1 &&
1530 !pci_enable_msi(enic
->pdev
)) {
1535 enic
->intr_count
= 1;
1537 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_MSI
);
1544 * We need 1 RQ, 1 WQ, 2 CQs, and 3 INTRs
1545 * (the first INTR is used for WQ/RQ)
1546 * (the second INTR is used for WQ/RQ errors)
1547 * (the last INTR is used for notifications)
1550 if (enic
->config
.intr_mode
< 3 &&
1551 enic
->rq_count
>= 1 &&
1552 enic
->wq_count
>= 1 &&
1553 enic
->cq_count
>= 2 &&
1554 enic
->intr_count
>= 3) {
1559 enic
->intr_count
= 3;
1561 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_INTX
);
1566 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_UNKNOWN
);
1571 static void enic_clear_intr_mode(struct enic
*enic
)
1573 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1574 case VNIC_DEV_INTR_MODE_MSIX
:
1575 pci_disable_msix(enic
->pdev
);
1577 case VNIC_DEV_INTR_MODE_MSI
:
1578 pci_disable_msi(enic
->pdev
);
1584 vnic_dev_set_intr_mode(enic
->vdev
, VNIC_DEV_INTR_MODE_UNKNOWN
);
1587 static void enic_iounmap(struct enic
*enic
)
1589 if (enic
->bar0
.vaddr
)
1590 iounmap(enic
->bar0
.vaddr
);
1593 static const struct net_device_ops enic_netdev_ops
= {
1594 .ndo_open
= enic_open
,
1595 .ndo_stop
= enic_stop
,
1596 .ndo_get_stats
= enic_get_stats
,
1597 .ndo_validate_addr
= eth_validate_addr
,
1598 .ndo_set_multicast_list
= enic_set_multicast_list
,
1599 .ndo_change_mtu
= enic_change_mtu
,
1600 .ndo_vlan_rx_register
= enic_vlan_rx_register
,
1601 .ndo_vlan_rx_add_vid
= enic_vlan_rx_add_vid
,
1602 .ndo_vlan_rx_kill_vid
= enic_vlan_rx_kill_vid
,
1603 .ndo_tx_timeout
= enic_tx_timeout
,
1604 #ifdef CONFIG_NET_POLL_CONTROLLER
1605 .ndo_poll_controller
= enic_poll_controller
,
1609 static int __devinit
enic_probe(struct pci_dev
*pdev
,
1610 const struct pci_device_id
*ent
)
1612 struct net_device
*netdev
;
1618 const u8 rss_default_cpu
= 0;
1619 const u8 rss_hash_type
= 0;
1620 const u8 rss_hash_bits
= 0;
1621 const u8 rss_base_cpu
= 0;
1622 const u8 rss_enable
= 0;
1623 const u8 tso_ipid_split_en
= 0;
1624 const u8 ig_vlan_strip_en
= 1;
1626 /* Allocate net device structure and initialize. Private
1627 * instance data is initialized to zero.
1630 netdev
= alloc_etherdev(sizeof(struct enic
));
1632 printk(KERN_ERR PFX
"Etherdev alloc failed, aborting.\n");
1636 pci_set_drvdata(pdev
, netdev
);
1638 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
1640 enic
= netdev_priv(netdev
);
1641 enic
->netdev
= netdev
;
1644 /* Setup PCI resources
1647 err
= pci_enable_device(pdev
);
1650 "Cannot enable PCI device, aborting.\n");
1651 goto err_out_free_netdev
;
1654 err
= pci_request_regions(pdev
, DRV_NAME
);
1657 "Cannot request PCI regions, aborting.\n");
1658 goto err_out_disable_device
;
1661 pci_set_master(pdev
);
1663 /* Query PCI controller on system for DMA addressing
1664 * limitation for the device. Try 40-bit first, and
1668 err
= pci_set_dma_mask(pdev
, DMA_40BIT_MASK
);
1670 err
= pci_set_dma_mask(pdev
, DMA_32BIT_MASK
);
1673 "No usable DMA configuration, aborting.\n");
1674 goto err_out_release_regions
;
1676 err
= pci_set_consistent_dma_mask(pdev
, DMA_32BIT_MASK
);
1679 "Unable to obtain 32-bit DMA "
1680 "for consistent allocations, aborting.\n");
1681 goto err_out_release_regions
;
1684 err
= pci_set_consistent_dma_mask(pdev
, DMA_40BIT_MASK
);
1687 "Unable to obtain 40-bit DMA "
1688 "for consistent allocations, aborting.\n");
1689 goto err_out_release_regions
;
1694 /* Map vNIC resources from BAR0
1697 if (!(pci_resource_flags(pdev
, 0) & IORESOURCE_MEM
)) {
1699 "BAR0 not memory-map'able, aborting.\n");
1701 goto err_out_release_regions
;
1704 enic
->bar0
.vaddr
= pci_iomap(pdev
, 0, enic
->bar0
.len
);
1705 enic
->bar0
.bus_addr
= pci_resource_start(pdev
, 0);
1706 enic
->bar0
.len
= pci_resource_len(pdev
, 0);
1708 if (!enic
->bar0
.vaddr
) {
1710 "Cannot memory-map BAR0 res hdr, aborting.\n");
1712 goto err_out_release_regions
;
1715 /* Register vNIC device
1718 enic
->vdev
= vnic_dev_register(NULL
, enic
, pdev
, &enic
->bar0
);
1721 "vNIC registration failed, aborting.\n");
1723 goto err_out_iounmap
;
1726 /* Issue device open to get device in known state
1729 err
= enic_dev_open(enic
);
1732 "vNIC dev open failed, aborting.\n");
1733 goto err_out_vnic_unregister
;
1736 /* Issue device init to initialize the vnic-to-switch link.
1737 * We'll start with carrier off and wait for link UP
1738 * notification later to turn on carrier. We don't need
1739 * to wait here for the vnic-to-switch link initialization
1740 * to complete; link UP notification is the indication that
1741 * the process is complete.
1744 netif_carrier_off(netdev
);
1746 err
= vnic_dev_init(enic
->vdev
, 0);
1749 "vNIC dev init failed, aborting.\n");
1750 goto err_out_dev_close
;
1753 /* Get vNIC configuration
1756 err
= enic_get_vnic_config(enic
);
1759 "Get vNIC configuration failed, aborting.\n");
1760 goto err_out_dev_close
;
1763 /* Get available resource counts
1766 enic_get_res_counts(enic
);
1768 /* Set interrupt mode based on resource counts and system
1772 err
= enic_set_intr_mode(enic
);
1775 "Failed to set intr mode, aborting.\n");
1776 goto err_out_dev_close
;
1779 /* Allocate and configure vNIC resources
1782 err
= enic_alloc_vnic_resources(enic
);
1785 "Failed to alloc vNIC resources, aborting.\n");
1786 goto err_out_free_vnic_resources
;
1789 enic_init_vnic_resources(enic
);
1791 /* Enable VLAN tag stripping. RSS not enabled (yet).
1794 err
= enic_set_nic_cfg(enic
,
1795 rss_default_cpu
, rss_hash_type
,
1796 rss_hash_bits
, rss_base_cpu
,
1797 rss_enable
, tso_ipid_split_en
,
1801 "Failed to config nic, aborting.\n");
1802 goto err_out_free_vnic_resources
;
1805 /* Setup notification timer, HW reset task, and locks
1808 init_timer(&enic
->notify_timer
);
1809 enic
->notify_timer
.function
= enic_notify_timer
;
1810 enic
->notify_timer
.data
= (unsigned long)enic
;
1812 INIT_WORK(&enic
->reset
, enic_reset
);
1814 for (i
= 0; i
< enic
->wq_count
; i
++)
1815 spin_lock_init(&enic
->wq_lock
[i
]);
1817 spin_lock_init(&enic
->devcmd_lock
);
1819 /* Register net device
1822 enic
->port_mtu
= enic
->config
.mtu
;
1823 (void)enic_change_mtu(netdev
, enic
->port_mtu
);
1825 err
= enic_set_mac_addr(netdev
, enic
->mac_addr
);
1828 "Invalid MAC address, aborting.\n");
1829 goto err_out_free_vnic_resources
;
1832 netdev
->netdev_ops
= &enic_netdev_ops
;
1833 netdev
->hard_start_xmit
= enic_hard_start_xmit
;
1834 netdev
->watchdog_timeo
= 2 * HZ
;
1835 netdev
->ethtool_ops
= &enic_ethtool_ops
;
1837 switch (vnic_dev_get_intr_mode(enic
->vdev
)) {
1839 netif_napi_add(netdev
, &enic
->napi
, enic_poll
, 64);
1841 case VNIC_DEV_INTR_MODE_MSIX
:
1842 netif_napi_add(netdev
, &enic
->napi
, enic_poll_msix
, 64);
1846 netdev
->features
|= NETIF_F_HW_VLAN_TX
| NETIF_F_HW_VLAN_RX
;
1847 if (ENIC_SETTING(enic
, TXCSUM
))
1848 netdev
->features
|= NETIF_F_SG
| NETIF_F_HW_CSUM
;
1849 if (ENIC_SETTING(enic
, TSO
))
1850 netdev
->features
|= NETIF_F_TSO
|
1851 NETIF_F_TSO6
| NETIF_F_TSO_ECN
;
1853 netdev
->features
|= NETIF_F_HIGHDMA
;
1856 enic
->csum_rx_enabled
= ENIC_SETTING(enic
, RXCSUM
);
1858 if (ENIC_SETTING(enic
, LRO
)) {
1859 enic
->lro_mgr
.max_aggr
= ENIC_LRO_MAX_AGGR
;
1860 enic
->lro_mgr
.max_desc
= ENIC_LRO_MAX_DESC
;
1861 enic
->lro_mgr
.lro_arr
= enic
->lro_desc
;
1862 enic
->lro_mgr
.get_skb_header
= enic_get_skb_header
;
1863 enic
->lro_mgr
.features
= LRO_F_NAPI
| LRO_F_EXTRACT_VLAN_ID
;
1864 enic
->lro_mgr
.dev
= netdev
;
1865 enic
->lro_mgr
.ip_summed
= CHECKSUM_COMPLETE
;
1866 enic
->lro_mgr
.ip_summed_aggr
= CHECKSUM_UNNECESSARY
;
1869 err
= register_netdev(netdev
);
1872 "Cannot register net device, aborting.\n");
1873 goto err_out_free_vnic_resources
;
1878 err_out_free_vnic_resources
:
1879 enic_free_vnic_resources(enic
);
1881 vnic_dev_close(enic
->vdev
);
1882 err_out_vnic_unregister
:
1883 enic_clear_intr_mode(enic
);
1884 vnic_dev_unregister(enic
->vdev
);
1887 err_out_release_regions
:
1888 pci_release_regions(pdev
);
1889 err_out_disable_device
:
1890 pci_disable_device(pdev
);
1891 err_out_free_netdev
:
1892 pci_set_drvdata(pdev
, NULL
);
1893 free_netdev(netdev
);
1898 static void __devexit
enic_remove(struct pci_dev
*pdev
)
1900 struct net_device
*netdev
= pci_get_drvdata(pdev
);
1903 struct enic
*enic
= netdev_priv(netdev
);
1905 flush_scheduled_work();
1906 unregister_netdev(netdev
);
1907 enic_free_vnic_resources(enic
);
1908 vnic_dev_close(enic
->vdev
);
1909 enic_clear_intr_mode(enic
);
1910 vnic_dev_unregister(enic
->vdev
);
1912 pci_release_regions(pdev
);
1913 pci_disable_device(pdev
);
1914 pci_set_drvdata(pdev
, NULL
);
1915 free_netdev(netdev
);
1919 static struct pci_driver enic_driver
= {
1921 .id_table
= enic_id_table
,
1922 .probe
= enic_probe
,
1923 .remove
= __devexit_p(enic_remove
),
1926 static int __init
enic_init_module(void)
1928 printk(KERN_INFO PFX
"%s, ver %s\n", DRV_DESCRIPTION
, DRV_VERSION
);
1930 return pci_register_driver(&enic_driver
);
1933 static void __exit
enic_cleanup_module(void)
1935 pci_unregister_driver(&enic_driver
);
1938 module_init(enic_init_module
);
1939 module_exit(enic_cleanup_module
);