2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <asm/uaccess.h>
71 #define DRV_VERSION "1.3.0-ko"
72 #define DRV_DESC "Chelsio T4 Network Driver"
75 * Max interrupt hold-off timer value in us. Queues fall back to this value
76 * under extreme memory pressure so it's largish to give the system time to
79 #define MAX_SGE_TIMERVAL 200U
83 * Physical Function provisioning constants.
85 PFRES_NVI
= 4, /* # of Virtual Interfaces */
86 PFRES_NETHCTRL
= 128, /* # of EQs used for ETH or CTRL Qs */
87 PFRES_NIQFLINT
= 128, /* # of ingress Qs/w Free List(s)/intr
89 PFRES_NEQ
= 256, /* # of egress queues */
90 PFRES_NIQ
= 0, /* # of ingress queues */
91 PFRES_TC
= 0, /* PCI-E traffic class */
92 PFRES_NEXACTF
= 128, /* # of exact MPS filters */
94 PFRES_R_CAPS
= FW_CMD_CAP_PF
,
95 PFRES_WX_CAPS
= FW_CMD_CAP_PF
,
99 * Virtual Function provisioning constants. We need two extra Ingress
100 * Queues with Interrupt capability to serve as the VF's Firmware
101 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
102 * neither will have Free Lists associated with them). For each
103 * Ethernet/Control Egress Queue and for each Free List, we need an
106 VFRES_NPORTS
= 1, /* # of "ports" per VF */
107 VFRES_NQSETS
= 2, /* # of "Queue Sets" per VF */
109 VFRES_NVI
= VFRES_NPORTS
, /* # of Virtual Interfaces */
110 VFRES_NETHCTRL
= VFRES_NQSETS
, /* # of EQs used for ETH or CTRL Qs */
111 VFRES_NIQFLINT
= VFRES_NQSETS
+2,/* # of ingress Qs/w Free List(s)/intr */
112 VFRES_NEQ
= VFRES_NQSETS
*2, /* # of egress queues */
113 VFRES_NIQ
= 0, /* # of non-fl/int ingress queues */
114 VFRES_TC
= 0, /* PCI-E traffic class */
115 VFRES_NEXACTF
= 16, /* # of exact MPS filters */
117 VFRES_R_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
|FW_CMD_CAP_PORT
,
118 VFRES_WX_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
,
123 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
124 * static and likely not to be useful in the long run. We really need to
125 * implement some form of persistent configuration which the firmware
128 static unsigned int pfvfres_pmask(struct adapter
*adapter
,
129 unsigned int pf
, unsigned int vf
)
131 unsigned int portn
, portvec
;
134 * Give PF's access to all of the ports.
137 return FW_PFVF_CMD_PMASK_MASK
;
140 * For VFs, we'll assign them access to the ports based purely on the
141 * PF. We assign active ports in order, wrapping around if there are
142 * fewer active ports than PFs: e.g. active port[pf % nports].
143 * Unfortunately the adapter's port_info structs haven't been
144 * initialized yet so we have to compute this.
146 if (adapter
->params
.nports
== 0)
149 portn
= pf
% adapter
->params
.nports
;
150 portvec
= adapter
->params
.portvec
;
153 * Isolate the lowest set bit in the port vector. If we're at
154 * the port number that we want, return that as the pmask.
155 * otherwise mask that bit out of the port vector and
156 * decrement our port number ...
158 unsigned int pmask
= portvec
^ (portvec
& (portvec
-1));
168 MAX_TXQ_ENTRIES
= 16384,
169 MAX_CTRL_TXQ_ENTRIES
= 1024,
170 MAX_RSPQ_ENTRIES
= 16384,
171 MAX_RX_BUFFERS
= 16384,
172 MIN_TXQ_ENTRIES
= 32,
173 MIN_CTRL_TXQ_ENTRIES
= 32,
174 MIN_RSPQ_ENTRIES
= 128,
178 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
179 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
180 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
182 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
184 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl
) = {
185 CH_DEVICE(0xa000, 0), /* PE10K */
186 CH_DEVICE(0x4001, -1),
187 CH_DEVICE(0x4002, -1),
188 CH_DEVICE(0x4003, -1),
189 CH_DEVICE(0x4004, -1),
190 CH_DEVICE(0x4005, -1),
191 CH_DEVICE(0x4006, -1),
192 CH_DEVICE(0x4007, -1),
193 CH_DEVICE(0x4008, -1),
194 CH_DEVICE(0x4009, -1),
195 CH_DEVICE(0x400a, -1),
196 CH_DEVICE(0x4401, 4),
197 CH_DEVICE(0x4402, 4),
198 CH_DEVICE(0x4403, 4),
199 CH_DEVICE(0x4404, 4),
200 CH_DEVICE(0x4405, 4),
201 CH_DEVICE(0x4406, 4),
202 CH_DEVICE(0x4407, 4),
203 CH_DEVICE(0x4408, 4),
204 CH_DEVICE(0x4409, 4),
205 CH_DEVICE(0x440a, 4),
206 CH_DEVICE(0x440d, 4),
207 CH_DEVICE(0x440e, 4),
211 #define FW_FNAME "cxgb4/t4fw.bin"
212 #define FW_CFNAME "cxgb4/t4-config.txt"
214 MODULE_DESCRIPTION(DRV_DESC
);
215 MODULE_AUTHOR("Chelsio Communications");
216 MODULE_LICENSE("Dual BSD/GPL");
217 MODULE_VERSION(DRV_VERSION
);
218 MODULE_DEVICE_TABLE(pci
, cxgb4_pci_tbl
);
219 MODULE_FIRMWARE(FW_FNAME
);
222 * Normally we're willing to become the firmware's Master PF but will be happy
223 * if another PF has already become the Master and initialized the adapter.
224 * Setting "force_init" will cause this driver to forcibly establish itself as
225 * the Master PF and initialize the adapter.
227 static uint force_init
;
229 module_param(force_init
, uint
, 0644);
230 MODULE_PARM_DESC(force_init
, "Forcibly become Master PF and initialize adapter");
233 * Normally if the firmware we connect to has Configuration File support, we
234 * use that and only fall back to the old Driver-based initialization if the
235 * Configuration File fails for some reason. If force_old_init is set, then
236 * we'll always use the old Driver-based initialization sequence.
238 static uint force_old_init
;
240 module_param(force_old_init
, uint
, 0644);
241 MODULE_PARM_DESC(force_old_init
, "Force old initialization sequence");
243 static int dflt_msg_enable
= DFLT_MSG_ENABLE
;
245 module_param(dflt_msg_enable
, int, 0644);
246 MODULE_PARM_DESC(dflt_msg_enable
, "Chelsio T4 default message enable bitmap");
249 * The driver uses the best interrupt scheme available on a platform in the
250 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
251 * of these schemes the driver may consider as follows:
253 * msi = 2: choose from among all three options
254 * msi = 1: only consider MSI and INTx interrupts
255 * msi = 0: force INTx interrupts
259 module_param(msi
, int, 0644);
260 MODULE_PARM_DESC(msi
, "whether to use INTx (0), MSI (1) or MSI-X (2)");
263 * Queue interrupt hold-off timer values. Queues default to the first of these
266 static unsigned int intr_holdoff
[SGE_NTIMERS
- 1] = { 5, 10, 20, 50, 100 };
268 module_param_array(intr_holdoff
, uint
, NULL
, 0644);
269 MODULE_PARM_DESC(intr_holdoff
, "values for queue interrupt hold-off timers "
270 "0..4 in microseconds");
272 static unsigned int intr_cnt
[SGE_NCOUNTERS
- 1] = { 4, 8, 16 };
274 module_param_array(intr_cnt
, uint
, NULL
, 0644);
275 MODULE_PARM_DESC(intr_cnt
,
276 "thresholds 1..3 for queue interrupt packet counters");
279 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
280 * offset by 2 bytes in order to have the IP headers line up on 4-byte
281 * boundaries. This is a requirement for many architectures which will throw
282 * a machine check fault if an attempt is made to access one of the 4-byte IP
283 * header fields on a non-4-byte boundary. And it's a major performance issue
284 * even on some architectures which allow it like some implementations of the
285 * x86 ISA. However, some architectures don't mind this and for some very
286 * edge-case performance sensitive applications (like forwarding large volumes
287 * of small packets), setting this DMA offset to 0 will decrease the number of
288 * PCI-E Bus transfers enough to measurably affect performance.
290 static int rx_dma_offset
= 2;
294 #ifdef CONFIG_PCI_IOV
295 module_param(vf_acls
, bool, 0644);
296 MODULE_PARM_DESC(vf_acls
, "if set enable virtualization L2 ACL enforcement");
298 static unsigned int num_vf
[4];
300 module_param_array(num_vf
, uint
, NULL
, 0644);
301 MODULE_PARM_DESC(num_vf
, "number of VFs for each of PFs 0-3");
305 * The filter TCAM has a fixed portion and a variable portion. The fixed
306 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
307 * ports. The variable portion is 36 bits which can include things like Exact
308 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
309 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
310 * far exceed the 36-bit budget for this "compressed" header portion of the
311 * filter. Thus, we have a scarce resource which must be carefully managed.
313 * By default we set this up to mostly match the set of filter matching
314 * capabilities of T3 but with accommodations for some of T4's more
315 * interesting features:
317 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
318 * [Inner] VLAN (17), Port (3), FCoE (1) }
321 TP_VLAN_PRI_MAP_DEFAULT
= HW_TPL_FR_MT_PR_IV_P_FC
,
322 TP_VLAN_PRI_MAP_FIRST
= FCOE_SHIFT
,
323 TP_VLAN_PRI_MAP_LAST
= FRAGMENTATION_SHIFT
,
326 static unsigned int tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
328 static struct dentry
*cxgb4_debugfs_root
;
330 static LIST_HEAD(adapter_list
);
331 static DEFINE_MUTEX(uld_mutex
);
332 static struct cxgb4_uld_info ulds
[CXGB4_ULD_MAX
];
333 static const char *uld_str
[] = { "RDMA", "iSCSI" };
335 static void link_report(struct net_device
*dev
)
337 if (!netif_carrier_ok(dev
))
338 netdev_info(dev
, "link down\n");
340 static const char *fc
[] = { "no", "Rx", "Tx", "Tx/Rx" };
342 const char *s
= "10Mbps";
343 const struct port_info
*p
= netdev_priv(dev
);
345 switch (p
->link_cfg
.speed
) {
357 netdev_info(dev
, "link up, %s, full-duplex, %s PAUSE\n", s
,
362 void t4_os_link_changed(struct adapter
*adapter
, int port_id
, int link_stat
)
364 struct net_device
*dev
= adapter
->port
[port_id
];
366 /* Skip changes from disabled ports. */
367 if (netif_running(dev
) && link_stat
!= netif_carrier_ok(dev
)) {
369 netif_carrier_on(dev
);
371 netif_carrier_off(dev
);
377 void t4_os_portmod_changed(const struct adapter
*adap
, int port_id
)
379 static const char *mod_str
[] = {
380 NULL
, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
383 const struct net_device
*dev
= adap
->port
[port_id
];
384 const struct port_info
*pi
= netdev_priv(dev
);
386 if (pi
->mod_type
== FW_PORT_MOD_TYPE_NONE
)
387 netdev_info(dev
, "port module unplugged\n");
388 else if (pi
->mod_type
< ARRAY_SIZE(mod_str
))
389 netdev_info(dev
, "%s module inserted\n", mod_str
[pi
->mod_type
]);
393 * Configure the exact and hash address filters to handle a port's multicast
394 * and secondary unicast MAC addresses.
396 static int set_addr_filters(const struct net_device
*dev
, bool sleep
)
404 const struct netdev_hw_addr
*ha
;
405 int uc_cnt
= netdev_uc_count(dev
);
406 int mc_cnt
= netdev_mc_count(dev
);
407 const struct port_info
*pi
= netdev_priv(dev
);
408 unsigned int mb
= pi
->adapter
->fn
;
410 /* first do the secondary unicast addresses */
411 netdev_for_each_uc_addr(ha
, dev
) {
412 addr
[naddr
++] = ha
->addr
;
413 if (--uc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
414 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
415 naddr
, addr
, filt_idx
, &uhash
, sleep
);
424 /* next set up the multicast addresses */
425 netdev_for_each_mc_addr(ha
, dev
) {
426 addr
[naddr
++] = ha
->addr
;
427 if (--mc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
428 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
429 naddr
, addr
, filt_idx
, &mhash
, sleep
);
438 return t4_set_addr_hash(pi
->adapter
, mb
, pi
->viid
, uhash
!= 0,
439 uhash
| mhash
, sleep
);
442 int dbfifo_int_thresh
= 10; /* 10 == 640 entry threshold */
443 module_param(dbfifo_int_thresh
, int, 0644);
444 MODULE_PARM_DESC(dbfifo_int_thresh
, "doorbell fifo interrupt threshold");
447 * usecs to sleep while draining the dbfifo
449 static int dbfifo_drain_delay
= 1000;
450 module_param(dbfifo_drain_delay
, int, 0644);
451 MODULE_PARM_DESC(dbfifo_drain_delay
,
452 "usecs to sleep while draining the dbfifo");
455 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
456 * If @mtu is -1 it is left unchanged.
458 static int set_rxmode(struct net_device
*dev
, int mtu
, bool sleep_ok
)
461 struct port_info
*pi
= netdev_priv(dev
);
463 ret
= set_addr_filters(dev
, sleep_ok
);
465 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, mtu
,
466 (dev
->flags
& IFF_PROMISC
) ? 1 : 0,
467 (dev
->flags
& IFF_ALLMULTI
) ? 1 : 0, 1, -1,
472 static struct workqueue_struct
*workq
;
475 * link_start - enable a port
476 * @dev: the port to enable
478 * Performs the MAC and PHY actions needed to enable a port.
480 static int link_start(struct net_device
*dev
)
483 struct port_info
*pi
= netdev_priv(dev
);
484 unsigned int mb
= pi
->adapter
->fn
;
487 * We do not set address filters and promiscuity here, the stack does
488 * that step explicitly.
490 ret
= t4_set_rxmode(pi
->adapter
, mb
, pi
->viid
, dev
->mtu
, -1, -1, -1,
491 !!(dev
->features
& NETIF_F_HW_VLAN_RX
), true);
493 ret
= t4_change_mac(pi
->adapter
, mb
, pi
->viid
,
494 pi
->xact_addr_filt
, dev
->dev_addr
, true,
497 pi
->xact_addr_filt
= ret
;
502 ret
= t4_link_start(pi
->adapter
, mb
, pi
->tx_chan
,
505 ret
= t4_enable_vi(pi
->adapter
, mb
, pi
->viid
, true, true);
510 * Response queue handler for the FW event queue.
512 static int fwevtq_handler(struct sge_rspq
*q
, const __be64
*rsp
,
513 const struct pkt_gl
*gl
)
515 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
517 rsp
++; /* skip RSS header */
518 if (likely(opcode
== CPL_SGE_EGR_UPDATE
)) {
519 const struct cpl_sge_egr_update
*p
= (void *)rsp
;
520 unsigned int qid
= EGR_QID(ntohl(p
->opcode_qid
));
523 txq
= q
->adap
->sge
.egr_map
[qid
- q
->adap
->sge
.egr_start
];
525 if ((u8
*)txq
< (u8
*)q
->adap
->sge
.ofldtxq
) {
526 struct sge_eth_txq
*eq
;
528 eq
= container_of(txq
, struct sge_eth_txq
, q
);
529 netif_tx_wake_queue(eq
->txq
);
531 struct sge_ofld_txq
*oq
;
533 oq
= container_of(txq
, struct sge_ofld_txq
, q
);
534 tasklet_schedule(&oq
->qresume_tsk
);
536 } else if (opcode
== CPL_FW6_MSG
|| opcode
== CPL_FW4_MSG
) {
537 const struct cpl_fw6_msg
*p
= (void *)rsp
;
540 t4_handle_fw_rpl(q
->adap
, p
->data
);
541 } else if (opcode
== CPL_L2T_WRITE_RPL
) {
542 const struct cpl_l2t_write_rpl
*p
= (void *)rsp
;
544 do_l2t_write_rpl(q
->adap
, p
);
546 dev_err(q
->adap
->pdev_dev
,
547 "unexpected CPL %#x on FW event queue\n", opcode
);
552 * uldrx_handler - response queue handler for ULD queues
553 * @q: the response queue that received the packet
554 * @rsp: the response queue descriptor holding the offload message
555 * @gl: the gather list of packet fragments
557 * Deliver an ingress offload packet to a ULD. All processing is done by
558 * the ULD, we just maintain statistics.
560 static int uldrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
561 const struct pkt_gl
*gl
)
563 struct sge_ofld_rxq
*rxq
= container_of(q
, struct sge_ofld_rxq
, rspq
);
565 if (ulds
[q
->uld
].rx_handler(q
->adap
->uld_handle
[q
->uld
], rsp
, gl
)) {
571 else if (gl
== CXGB4_MSG_AN
)
578 static void disable_msi(struct adapter
*adapter
)
580 if (adapter
->flags
& USING_MSIX
) {
581 pci_disable_msix(adapter
->pdev
);
582 adapter
->flags
&= ~USING_MSIX
;
583 } else if (adapter
->flags
& USING_MSI
) {
584 pci_disable_msi(adapter
->pdev
);
585 adapter
->flags
&= ~USING_MSI
;
590 * Interrupt handler for non-data events used with MSI-X.
592 static irqreturn_t
t4_nondata_intr(int irq
, void *cookie
)
594 struct adapter
*adap
= cookie
;
596 u32 v
= t4_read_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
));
599 t4_write_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
), v
);
601 t4_slow_intr_handler(adap
);
606 * Name the MSI-X interrupts.
608 static void name_msix_vecs(struct adapter
*adap
)
610 int i
, j
, msi_idx
= 2, n
= sizeof(adap
->msix_info
[0].desc
);
612 /* non-data interrupts */
613 snprintf(adap
->msix_info
[0].desc
, n
, "%s", adap
->port
[0]->name
);
616 snprintf(adap
->msix_info
[1].desc
, n
, "%s-FWeventq",
617 adap
->port
[0]->name
);
619 /* Ethernet queues */
620 for_each_port(adap
, j
) {
621 struct net_device
*d
= adap
->port
[j
];
622 const struct port_info
*pi
= netdev_priv(d
);
624 for (i
= 0; i
< pi
->nqsets
; i
++, msi_idx
++)
625 snprintf(adap
->msix_info
[msi_idx
].desc
, n
, "%s-Rx%d",
630 for_each_ofldrxq(&adap
->sge
, i
)
631 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-ofld%d",
632 adap
->port
[0]->name
, i
);
634 for_each_rdmarxq(&adap
->sge
, i
)
635 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-rdma%d",
636 adap
->port
[0]->name
, i
);
639 static int request_msix_queue_irqs(struct adapter
*adap
)
641 struct sge
*s
= &adap
->sge
;
642 int err
, ethqidx
, ofldqidx
= 0, rdmaqidx
= 0, msi_index
= 2;
644 err
= request_irq(adap
->msix_info
[1].vec
, t4_sge_intr_msix
, 0,
645 adap
->msix_info
[1].desc
, &s
->fw_evtq
);
649 for_each_ethrxq(s
, ethqidx
) {
650 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
652 adap
->msix_info
[msi_index
].desc
,
653 &s
->ethrxq
[ethqidx
].rspq
);
658 for_each_ofldrxq(s
, ofldqidx
) {
659 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
661 adap
->msix_info
[msi_index
].desc
,
662 &s
->ofldrxq
[ofldqidx
].rspq
);
667 for_each_rdmarxq(s
, rdmaqidx
) {
668 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
670 adap
->msix_info
[msi_index
].desc
,
671 &s
->rdmarxq
[rdmaqidx
].rspq
);
679 while (--rdmaqidx
>= 0)
680 free_irq(adap
->msix_info
[--msi_index
].vec
,
681 &s
->rdmarxq
[rdmaqidx
].rspq
);
682 while (--ofldqidx
>= 0)
683 free_irq(adap
->msix_info
[--msi_index
].vec
,
684 &s
->ofldrxq
[ofldqidx
].rspq
);
685 while (--ethqidx
>= 0)
686 free_irq(adap
->msix_info
[--msi_index
].vec
,
687 &s
->ethrxq
[ethqidx
].rspq
);
688 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
692 static void free_msix_queue_irqs(struct adapter
*adap
)
694 int i
, msi_index
= 2;
695 struct sge
*s
= &adap
->sge
;
697 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
698 for_each_ethrxq(s
, i
)
699 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ethrxq
[i
].rspq
);
700 for_each_ofldrxq(s
, i
)
701 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ofldrxq
[i
].rspq
);
702 for_each_rdmarxq(s
, i
)
703 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->rdmarxq
[i
].rspq
);
707 * write_rss - write the RSS table for a given port
709 * @queues: array of queue indices for RSS
711 * Sets up the portion of the HW RSS table for the port's VI to distribute
712 * packets to the Rx queues in @queues.
714 static int write_rss(const struct port_info
*pi
, const u16
*queues
)
718 const struct sge_eth_rxq
*q
= &pi
->adapter
->sge
.ethrxq
[pi
->first_qset
];
720 rss
= kmalloc(pi
->rss_size
* sizeof(u16
), GFP_KERNEL
);
724 /* map the queue indices to queue ids */
725 for (i
= 0; i
< pi
->rss_size
; i
++, queues
++)
726 rss
[i
] = q
[*queues
].rspq
.abs_id
;
728 err
= t4_config_rss_range(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, 0,
729 pi
->rss_size
, rss
, pi
->rss_size
);
735 * setup_rss - configure RSS
738 * Sets up RSS for each port.
740 static int setup_rss(struct adapter
*adap
)
744 for_each_port(adap
, i
) {
745 const struct port_info
*pi
= adap2pinfo(adap
, i
);
747 err
= write_rss(pi
, pi
->rss
);
755 * Return the channel of the ingress queue with the given qid.
757 static unsigned int rxq_to_chan(const struct sge
*p
, unsigned int qid
)
759 qid
-= p
->ingr_start
;
760 return netdev2pinfo(p
->ingr_map
[qid
]->netdev
)->tx_chan
;
764 * Wait until all NAPI handlers are descheduled.
766 static void quiesce_rx(struct adapter
*adap
)
770 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
771 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
774 napi_disable(&q
->napi
);
779 * Enable NAPI scheduling and interrupt generation for all Rx queues.
781 static void enable_rx(struct adapter
*adap
)
785 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
786 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
791 napi_enable(&q
->napi
);
792 /* 0-increment GTS to start the timer and enable interrupts */
793 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS
),
794 SEINTARM(q
->intr_params
) |
795 INGRESSQID(q
->cntxt_id
));
800 * setup_sge_queues - configure SGE Tx/Rx/response queues
803 * Determines how many sets of SGE queues to use and initializes them.
804 * We support multiple queue sets per port if we have MSI-X, otherwise
805 * just one queue set per port.
807 static int setup_sge_queues(struct adapter
*adap
)
809 int err
, msi_idx
, i
, j
;
810 struct sge
*s
= &adap
->sge
;
812 bitmap_zero(s
->starving_fl
, MAX_EGRQ
);
813 bitmap_zero(s
->txq_maperr
, MAX_EGRQ
);
815 if (adap
->flags
& USING_MSIX
)
816 msi_idx
= 1; /* vector 0 is for non-queue interrupts */
818 err
= t4_sge_alloc_rxq(adap
, &s
->intrq
, false, adap
->port
[0], 0,
822 msi_idx
= -((int)s
->intrq
.abs_id
+ 1);
825 err
= t4_sge_alloc_rxq(adap
, &s
->fw_evtq
, true, adap
->port
[0],
826 msi_idx
, NULL
, fwevtq_handler
);
828 freeout
: t4_free_sge_resources(adap
);
832 for_each_port(adap
, i
) {
833 struct net_device
*dev
= adap
->port
[i
];
834 struct port_info
*pi
= netdev_priv(dev
);
835 struct sge_eth_rxq
*q
= &s
->ethrxq
[pi
->first_qset
];
836 struct sge_eth_txq
*t
= &s
->ethtxq
[pi
->first_qset
];
838 for (j
= 0; j
< pi
->nqsets
; j
++, q
++) {
841 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
,
847 memset(&q
->stats
, 0, sizeof(q
->stats
));
849 for (j
= 0; j
< pi
->nqsets
; j
++, t
++) {
850 err
= t4_sge_alloc_eth_txq(adap
, t
, dev
,
851 netdev_get_tx_queue(dev
, j
),
852 s
->fw_evtq
.cntxt_id
);
858 j
= s
->ofldqsets
/ adap
->params
.nports
; /* ofld queues per channel */
859 for_each_ofldrxq(s
, i
) {
860 struct sge_ofld_rxq
*q
= &s
->ofldrxq
[i
];
861 struct net_device
*dev
= adap
->port
[i
/ j
];
865 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
, msi_idx
,
866 &q
->fl
, uldrx_handler
);
869 memset(&q
->stats
, 0, sizeof(q
->stats
));
870 s
->ofld_rxq
[i
] = q
->rspq
.abs_id
;
871 err
= t4_sge_alloc_ofld_txq(adap
, &s
->ofldtxq
[i
], dev
,
872 s
->fw_evtq
.cntxt_id
);
877 for_each_rdmarxq(s
, i
) {
878 struct sge_ofld_rxq
*q
= &s
->rdmarxq
[i
];
882 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, adap
->port
[i
],
883 msi_idx
, &q
->fl
, uldrx_handler
);
886 memset(&q
->stats
, 0, sizeof(q
->stats
));
887 s
->rdma_rxq
[i
] = q
->rspq
.abs_id
;
890 for_each_port(adap
, i
) {
892 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
893 * have RDMA queues, and that's the right value.
895 err
= t4_sge_alloc_ctrl_txq(adap
, &s
->ctrlq
[i
], adap
->port
[i
],
897 s
->rdmarxq
[i
].rspq
.cntxt_id
);
902 t4_write_reg(adap
, MPS_TRC_RSS_CONTROL
,
903 RSSCONTROL(netdev2pinfo(adap
->port
[0])->tx_chan
) |
904 QUEUENUMBER(s
->ethrxq
[0].rspq
.abs_id
));
909 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
910 * started but failed, and a negative errno if flash load couldn't start.
912 static int upgrade_fw(struct adapter
*adap
)
916 const struct fw_hdr
*hdr
;
917 const struct firmware
*fw
;
918 struct device
*dev
= adap
->pdev_dev
;
920 ret
= request_firmware(&fw
, FW_FNAME
, dev
);
922 dev_err(dev
, "unable to load firmware image " FW_FNAME
923 ", error %d\n", ret
);
927 hdr
= (const struct fw_hdr
*)fw
->data
;
928 vers
= ntohl(hdr
->fw_ver
);
929 if (FW_HDR_FW_VER_MAJOR_GET(vers
) != FW_VERSION_MAJOR
) {
930 ret
= -EINVAL
; /* wrong major version, won't do */
935 * If the flash FW is unusable or we found something newer, load it.
937 if (FW_HDR_FW_VER_MAJOR_GET(adap
->params
.fw_vers
) != FW_VERSION_MAJOR
||
938 vers
> adap
->params
.fw_vers
) {
939 dev_info(dev
, "upgrading firmware ...\n");
940 ret
= t4_fw_upgrade(adap
, adap
->mbox
, fw
->data
, fw
->size
,
943 dev_info(dev
, "firmware successfully upgraded to "
944 FW_FNAME
" (%d.%d.%d.%d)\n",
945 FW_HDR_FW_VER_MAJOR_GET(vers
),
946 FW_HDR_FW_VER_MINOR_GET(vers
),
947 FW_HDR_FW_VER_MICRO_GET(vers
),
948 FW_HDR_FW_VER_BUILD_GET(vers
));
950 dev_err(dev
, "firmware upgrade failed! err=%d\n", -ret
);
953 * Tell our caller that we didn't upgrade the firmware.
958 out
: release_firmware(fw
);
963 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
964 * The allocated memory is cleared.
966 void *t4_alloc_mem(size_t size
)
968 void *p
= kzalloc(size
, GFP_KERNEL
);
976 * Free memory allocated through alloc_mem().
978 static void t4_free_mem(void *addr
)
980 if (is_vmalloc_addr(addr
))
986 static inline int is_offload(const struct adapter
*adap
)
988 return adap
->params
.offload
;
992 * Implementation of ethtool operations.
995 static u32
get_msglevel(struct net_device
*dev
)
997 return netdev2adap(dev
)->msg_enable
;
1000 static void set_msglevel(struct net_device
*dev
, u32 val
)
1002 netdev2adap(dev
)->msg_enable
= val
;
1005 static char stats_strings
[][ETH_GSTRING_LEN
] = {
1008 "TxBroadcastFrames ",
1009 "TxMulticastFrames ",
1015 "TxFrames128To255 ",
1016 "TxFrames256To511 ",
1017 "TxFrames512To1023 ",
1018 "TxFrames1024To1518 ",
1019 "TxFrames1519ToMax ",
1034 "RxBroadcastFrames ",
1035 "RxMulticastFrames ",
1047 "RxFrames128To255 ",
1048 "RxFrames256To511 ",
1049 "RxFrames512To1023 ",
1050 "RxFrames1024To1518 ",
1051 "RxFrames1519ToMax ",
1063 "RxBG0FramesDropped ",
1064 "RxBG1FramesDropped ",
1065 "RxBG2FramesDropped ",
1066 "RxBG3FramesDropped ",
1067 "RxBG0FramesTrunc ",
1068 "RxBG1FramesTrunc ",
1069 "RxBG2FramesTrunc ",
1070 "RxBG3FramesTrunc ",
1081 static int get_sset_count(struct net_device
*dev
, int sset
)
1085 return ARRAY_SIZE(stats_strings
);
1091 #define T4_REGMAP_SIZE (160 * 1024)
1093 static int get_regs_len(struct net_device
*dev
)
1095 return T4_REGMAP_SIZE
;
1098 static int get_eeprom_len(struct net_device
*dev
)
1103 static void get_drvinfo(struct net_device
*dev
, struct ethtool_drvinfo
*info
)
1105 struct adapter
*adapter
= netdev2adap(dev
);
1107 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
1108 strlcpy(info
->version
, DRV_VERSION
, sizeof(info
->version
));
1109 strlcpy(info
->bus_info
, pci_name(adapter
->pdev
),
1110 sizeof(info
->bus_info
));
1112 if (adapter
->params
.fw_vers
)
1113 snprintf(info
->fw_version
, sizeof(info
->fw_version
),
1114 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1115 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.fw_vers
),
1116 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.fw_vers
),
1117 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.fw_vers
),
1118 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.fw_vers
),
1119 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.tp_vers
),
1120 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.tp_vers
),
1121 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.tp_vers
),
1122 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.tp_vers
));
1125 static void get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1127 if (stringset
== ETH_SS_STATS
)
1128 memcpy(data
, stats_strings
, sizeof(stats_strings
));
1132 * port stats maintained per queue of the port. They should be in the same
1133 * order as in stats_strings above.
1135 struct queue_port_stats
{
1145 static void collect_sge_port_stats(const struct adapter
*adap
,
1146 const struct port_info
*p
, struct queue_port_stats
*s
)
1149 const struct sge_eth_txq
*tx
= &adap
->sge
.ethtxq
[p
->first_qset
];
1150 const struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[p
->first_qset
];
1152 memset(s
, 0, sizeof(*s
));
1153 for (i
= 0; i
< p
->nqsets
; i
++, rx
++, tx
++) {
1155 s
->tx_csum
+= tx
->tx_cso
;
1156 s
->rx_csum
+= rx
->stats
.rx_cso
;
1157 s
->vlan_ex
+= rx
->stats
.vlan_ex
;
1158 s
->vlan_ins
+= tx
->vlan_ins
;
1159 s
->gro_pkts
+= rx
->stats
.lro_pkts
;
1160 s
->gro_merged
+= rx
->stats
.lro_merged
;
1164 static void get_stats(struct net_device
*dev
, struct ethtool_stats
*stats
,
1167 struct port_info
*pi
= netdev_priv(dev
);
1168 struct adapter
*adapter
= pi
->adapter
;
1170 t4_get_port_stats(adapter
, pi
->tx_chan
, (struct port_stats
*)data
);
1172 data
+= sizeof(struct port_stats
) / sizeof(u64
);
1173 collect_sge_port_stats(adapter
, pi
, (struct queue_port_stats
*)data
);
1177 * Return a version number to identify the type of adapter. The scheme is:
1178 * - bits 0..9: chip version
1179 * - bits 10..15: chip revision
1180 * - bits 16..23: register dump version
1182 static inline unsigned int mk_adap_vers(const struct adapter
*ap
)
1184 return 4 | (ap
->params
.rev
<< 10) | (1 << 16);
1187 static void reg_block_dump(struct adapter
*ap
, void *buf
, unsigned int start
,
1190 u32
*p
= buf
+ start
;
1192 for ( ; start
<= end
; start
+= sizeof(u32
))
1193 *p
++ = t4_read_reg(ap
, start
);
1196 static void get_regs(struct net_device
*dev
, struct ethtool_regs
*regs
,
1199 static const unsigned int reg_ranges
[] = {
1420 struct adapter
*ap
= netdev2adap(dev
);
1422 regs
->version
= mk_adap_vers(ap
);
1424 memset(buf
, 0, T4_REGMAP_SIZE
);
1425 for (i
= 0; i
< ARRAY_SIZE(reg_ranges
); i
+= 2)
1426 reg_block_dump(ap
, buf
, reg_ranges
[i
], reg_ranges
[i
+ 1]);
1429 static int restart_autoneg(struct net_device
*dev
)
1431 struct port_info
*p
= netdev_priv(dev
);
1433 if (!netif_running(dev
))
1435 if (p
->link_cfg
.autoneg
!= AUTONEG_ENABLE
)
1437 t4_restart_aneg(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
);
1441 static int identify_port(struct net_device
*dev
,
1442 enum ethtool_phys_id_state state
)
1445 struct adapter
*adap
= netdev2adap(dev
);
1447 if (state
== ETHTOOL_ID_ACTIVE
)
1449 else if (state
== ETHTOOL_ID_INACTIVE
)
1454 return t4_identify_port(adap
, adap
->fn
, netdev2pinfo(dev
)->viid
, val
);
1457 static unsigned int from_fw_linkcaps(unsigned int type
, unsigned int caps
)
1461 if (type
== FW_PORT_TYPE_BT_SGMII
|| type
== FW_PORT_TYPE_BT_XFI
||
1462 type
== FW_PORT_TYPE_BT_XAUI
) {
1464 if (caps
& FW_PORT_CAP_SPEED_100M
)
1465 v
|= SUPPORTED_100baseT_Full
;
1466 if (caps
& FW_PORT_CAP_SPEED_1G
)
1467 v
|= SUPPORTED_1000baseT_Full
;
1468 if (caps
& FW_PORT_CAP_SPEED_10G
)
1469 v
|= SUPPORTED_10000baseT_Full
;
1470 } else if (type
== FW_PORT_TYPE_KX4
|| type
== FW_PORT_TYPE_KX
) {
1471 v
|= SUPPORTED_Backplane
;
1472 if (caps
& FW_PORT_CAP_SPEED_1G
)
1473 v
|= SUPPORTED_1000baseKX_Full
;
1474 if (caps
& FW_PORT_CAP_SPEED_10G
)
1475 v
|= SUPPORTED_10000baseKX4_Full
;
1476 } else if (type
== FW_PORT_TYPE_KR
)
1477 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseKR_Full
;
1478 else if (type
== FW_PORT_TYPE_BP_AP
)
1479 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
1480 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
;
1481 else if (type
== FW_PORT_TYPE_BP4_AP
)
1482 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
1483 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
|
1484 SUPPORTED_10000baseKX4_Full
;
1485 else if (type
== FW_PORT_TYPE_FIBER_XFI
||
1486 type
== FW_PORT_TYPE_FIBER_XAUI
|| type
== FW_PORT_TYPE_SFP
)
1487 v
|= SUPPORTED_FIBRE
;
1489 if (caps
& FW_PORT_CAP_ANEG
)
1490 v
|= SUPPORTED_Autoneg
;
1494 static unsigned int to_fw_linkcaps(unsigned int caps
)
1498 if (caps
& ADVERTISED_100baseT_Full
)
1499 v
|= FW_PORT_CAP_SPEED_100M
;
1500 if (caps
& ADVERTISED_1000baseT_Full
)
1501 v
|= FW_PORT_CAP_SPEED_1G
;
1502 if (caps
& ADVERTISED_10000baseT_Full
)
1503 v
|= FW_PORT_CAP_SPEED_10G
;
1507 static int get_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
1509 const struct port_info
*p
= netdev_priv(dev
);
1511 if (p
->port_type
== FW_PORT_TYPE_BT_SGMII
||
1512 p
->port_type
== FW_PORT_TYPE_BT_XFI
||
1513 p
->port_type
== FW_PORT_TYPE_BT_XAUI
)
1514 cmd
->port
= PORT_TP
;
1515 else if (p
->port_type
== FW_PORT_TYPE_FIBER_XFI
||
1516 p
->port_type
== FW_PORT_TYPE_FIBER_XAUI
)
1517 cmd
->port
= PORT_FIBRE
;
1518 else if (p
->port_type
== FW_PORT_TYPE_SFP
) {
1519 if (p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_PASSIVE
||
1520 p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_ACTIVE
)
1521 cmd
->port
= PORT_DA
;
1523 cmd
->port
= PORT_FIBRE
;
1525 cmd
->port
= PORT_OTHER
;
1527 if (p
->mdio_addr
>= 0) {
1528 cmd
->phy_address
= p
->mdio_addr
;
1529 cmd
->transceiver
= XCVR_EXTERNAL
;
1530 cmd
->mdio_support
= p
->port_type
== FW_PORT_TYPE_BT_SGMII
?
1531 MDIO_SUPPORTS_C22
: MDIO_SUPPORTS_C45
;
1533 cmd
->phy_address
= 0; /* not really, but no better option */
1534 cmd
->transceiver
= XCVR_INTERNAL
;
1535 cmd
->mdio_support
= 0;
1538 cmd
->supported
= from_fw_linkcaps(p
->port_type
, p
->link_cfg
.supported
);
1539 cmd
->advertising
= from_fw_linkcaps(p
->port_type
,
1540 p
->link_cfg
.advertising
);
1541 ethtool_cmd_speed_set(cmd
,
1542 netif_carrier_ok(dev
) ? p
->link_cfg
.speed
: 0);
1543 cmd
->duplex
= DUPLEX_FULL
;
1544 cmd
->autoneg
= p
->link_cfg
.autoneg
;
1550 static unsigned int speed_to_caps(int speed
)
1552 if (speed
== SPEED_100
)
1553 return FW_PORT_CAP_SPEED_100M
;
1554 if (speed
== SPEED_1000
)
1555 return FW_PORT_CAP_SPEED_1G
;
1556 if (speed
== SPEED_10000
)
1557 return FW_PORT_CAP_SPEED_10G
;
1561 static int set_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
1564 struct port_info
*p
= netdev_priv(dev
);
1565 struct link_config
*lc
= &p
->link_cfg
;
1566 u32 speed
= ethtool_cmd_speed(cmd
);
1568 if (cmd
->duplex
!= DUPLEX_FULL
) /* only full-duplex supported */
1571 if (!(lc
->supported
& FW_PORT_CAP_ANEG
)) {
1573 * PHY offers a single speed. See if that's what's
1576 if (cmd
->autoneg
== AUTONEG_DISABLE
&&
1577 (lc
->supported
& speed_to_caps(speed
)))
1582 if (cmd
->autoneg
== AUTONEG_DISABLE
) {
1583 cap
= speed_to_caps(speed
);
1585 if (!(lc
->supported
& cap
) || (speed
== SPEED_1000
) ||
1586 (speed
== SPEED_10000
))
1588 lc
->requested_speed
= cap
;
1589 lc
->advertising
= 0;
1591 cap
= to_fw_linkcaps(cmd
->advertising
);
1592 if (!(lc
->supported
& cap
))
1594 lc
->requested_speed
= 0;
1595 lc
->advertising
= cap
| FW_PORT_CAP_ANEG
;
1597 lc
->autoneg
= cmd
->autoneg
;
1599 if (netif_running(dev
))
1600 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
1605 static void get_pauseparam(struct net_device
*dev
,
1606 struct ethtool_pauseparam
*epause
)
1608 struct port_info
*p
= netdev_priv(dev
);
1610 epause
->autoneg
= (p
->link_cfg
.requested_fc
& PAUSE_AUTONEG
) != 0;
1611 epause
->rx_pause
= (p
->link_cfg
.fc
& PAUSE_RX
) != 0;
1612 epause
->tx_pause
= (p
->link_cfg
.fc
& PAUSE_TX
) != 0;
1615 static int set_pauseparam(struct net_device
*dev
,
1616 struct ethtool_pauseparam
*epause
)
1618 struct port_info
*p
= netdev_priv(dev
);
1619 struct link_config
*lc
= &p
->link_cfg
;
1621 if (epause
->autoneg
== AUTONEG_DISABLE
)
1622 lc
->requested_fc
= 0;
1623 else if (lc
->supported
& FW_PORT_CAP_ANEG
)
1624 lc
->requested_fc
= PAUSE_AUTONEG
;
1628 if (epause
->rx_pause
)
1629 lc
->requested_fc
|= PAUSE_RX
;
1630 if (epause
->tx_pause
)
1631 lc
->requested_fc
|= PAUSE_TX
;
1632 if (netif_running(dev
))
1633 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
1638 static void get_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
1640 const struct port_info
*pi
= netdev_priv(dev
);
1641 const struct sge
*s
= &pi
->adapter
->sge
;
1643 e
->rx_max_pending
= MAX_RX_BUFFERS
;
1644 e
->rx_mini_max_pending
= MAX_RSPQ_ENTRIES
;
1645 e
->rx_jumbo_max_pending
= 0;
1646 e
->tx_max_pending
= MAX_TXQ_ENTRIES
;
1648 e
->rx_pending
= s
->ethrxq
[pi
->first_qset
].fl
.size
- 8;
1649 e
->rx_mini_pending
= s
->ethrxq
[pi
->first_qset
].rspq
.size
;
1650 e
->rx_jumbo_pending
= 0;
1651 e
->tx_pending
= s
->ethtxq
[pi
->first_qset
].q
.size
;
1654 static int set_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
1657 const struct port_info
*pi
= netdev_priv(dev
);
1658 struct adapter
*adapter
= pi
->adapter
;
1659 struct sge
*s
= &adapter
->sge
;
1661 if (e
->rx_pending
> MAX_RX_BUFFERS
|| e
->rx_jumbo_pending
||
1662 e
->tx_pending
> MAX_TXQ_ENTRIES
||
1663 e
->rx_mini_pending
> MAX_RSPQ_ENTRIES
||
1664 e
->rx_mini_pending
< MIN_RSPQ_ENTRIES
||
1665 e
->rx_pending
< MIN_FL_ENTRIES
|| e
->tx_pending
< MIN_TXQ_ENTRIES
)
1668 if (adapter
->flags
& FULL_INIT_DONE
)
1671 for (i
= 0; i
< pi
->nqsets
; ++i
) {
1672 s
->ethtxq
[pi
->first_qset
+ i
].q
.size
= e
->tx_pending
;
1673 s
->ethrxq
[pi
->first_qset
+ i
].fl
.size
= e
->rx_pending
+ 8;
1674 s
->ethrxq
[pi
->first_qset
+ i
].rspq
.size
= e
->rx_mini_pending
;
1679 static int closest_timer(const struct sge
*s
, int time
)
1681 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
1683 for (i
= 0; i
< ARRAY_SIZE(s
->timer_val
); i
++) {
1684 delta
= time
- s
->timer_val
[i
];
1687 if (delta
< min_delta
) {
1695 static int closest_thres(const struct sge
*s
, int thres
)
1697 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
1699 for (i
= 0; i
< ARRAY_SIZE(s
->counter_val
); i
++) {
1700 delta
= thres
- s
->counter_val
[i
];
1703 if (delta
< min_delta
) {
1712 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1714 static unsigned int qtimer_val(const struct adapter
*adap
,
1715 const struct sge_rspq
*q
)
1717 unsigned int idx
= q
->intr_params
>> 1;
1719 return idx
< SGE_NTIMERS
? adap
->sge
.timer_val
[idx
] : 0;
1723 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1724 * @adap: the adapter
1726 * @us: the hold-off time in us, or 0 to disable timer
1727 * @cnt: the hold-off packet count, or 0 to disable counter
1729 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1730 * one of the two needs to be enabled for the queue to generate interrupts.
1732 static int set_rxq_intr_params(struct adapter
*adap
, struct sge_rspq
*q
,
1733 unsigned int us
, unsigned int cnt
)
1735 if ((us
| cnt
) == 0)
1742 new_idx
= closest_thres(&adap
->sge
, cnt
);
1743 if (q
->desc
&& q
->pktcnt_idx
!= new_idx
) {
1744 /* the queue has already been created, update it */
1745 v
= FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ
) |
1746 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH
) |
1747 FW_PARAMS_PARAM_YZ(q
->cntxt_id
);
1748 err
= t4_set_params(adap
, adap
->fn
, adap
->fn
, 0, 1, &v
,
1753 q
->pktcnt_idx
= new_idx
;
1756 us
= us
== 0 ? 6 : closest_timer(&adap
->sge
, us
);
1757 q
->intr_params
= QINTR_TIMER_IDX(us
) | (cnt
> 0 ? QINTR_CNT_EN
: 0);
1761 static int set_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
1763 const struct port_info
*pi
= netdev_priv(dev
);
1764 struct adapter
*adap
= pi
->adapter
;
1766 return set_rxq_intr_params(adap
, &adap
->sge
.ethrxq
[pi
->first_qset
].rspq
,
1767 c
->rx_coalesce_usecs
, c
->rx_max_coalesced_frames
);
1770 static int get_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
1772 const struct port_info
*pi
= netdev_priv(dev
);
1773 const struct adapter
*adap
= pi
->adapter
;
1774 const struct sge_rspq
*rq
= &adap
->sge
.ethrxq
[pi
->first_qset
].rspq
;
1776 c
->rx_coalesce_usecs
= qtimer_val(adap
, rq
);
1777 c
->rx_max_coalesced_frames
= (rq
->intr_params
& QINTR_CNT_EN
) ?
1778 adap
->sge
.counter_val
[rq
->pktcnt_idx
] : 0;
1783 * eeprom_ptov - translate a physical EEPROM address to virtual
1784 * @phys_addr: the physical EEPROM address
1785 * @fn: the PCI function number
1786 * @sz: size of function-specific area
1788 * Translate a physical EEPROM address to virtual. The first 1K is
1789 * accessed through virtual addresses starting at 31K, the rest is
1790 * accessed through virtual addresses starting at 0.
1792 * The mapping is as follows:
1793 * [0..1K) -> [31K..32K)
1794 * [1K..1K+A) -> [31K-A..31K)
1795 * [1K+A..ES) -> [0..ES-A-1K)
1797 * where A = @fn * @sz, and ES = EEPROM size.
1799 static int eeprom_ptov(unsigned int phys_addr
, unsigned int fn
, unsigned int sz
)
1802 if (phys_addr
< 1024)
1803 return phys_addr
+ (31 << 10);
1804 if (phys_addr
< 1024 + fn
)
1805 return 31744 - fn
+ phys_addr
- 1024;
1806 if (phys_addr
< EEPROMSIZE
)
1807 return phys_addr
- 1024 - fn
;
1812 * The next two routines implement eeprom read/write from physical addresses.
1814 static int eeprom_rd_phys(struct adapter
*adap
, unsigned int phys_addr
, u32
*v
)
1816 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
1819 vaddr
= pci_read_vpd(adap
->pdev
, vaddr
, sizeof(u32
), v
);
1820 return vaddr
< 0 ? vaddr
: 0;
1823 static int eeprom_wr_phys(struct adapter
*adap
, unsigned int phys_addr
, u32 v
)
1825 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
1828 vaddr
= pci_write_vpd(adap
->pdev
, vaddr
, sizeof(u32
), &v
);
1829 return vaddr
< 0 ? vaddr
: 0;
1832 #define EEPROM_MAGIC 0x38E2F10C
1834 static int get_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*e
,
1838 struct adapter
*adapter
= netdev2adap(dev
);
1840 u8
*buf
= kmalloc(EEPROMSIZE
, GFP_KERNEL
);
1844 e
->magic
= EEPROM_MAGIC
;
1845 for (i
= e
->offset
& ~3; !err
&& i
< e
->offset
+ e
->len
; i
+= 4)
1846 err
= eeprom_rd_phys(adapter
, i
, (u32
*)&buf
[i
]);
1849 memcpy(data
, buf
+ e
->offset
, e
->len
);
1854 static int set_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*eeprom
,
1859 u32 aligned_offset
, aligned_len
, *p
;
1860 struct adapter
*adapter
= netdev2adap(dev
);
1862 if (eeprom
->magic
!= EEPROM_MAGIC
)
1865 aligned_offset
= eeprom
->offset
& ~3;
1866 aligned_len
= (eeprom
->len
+ (eeprom
->offset
& 3) + 3) & ~3;
1868 if (adapter
->fn
> 0) {
1869 u32 start
= 1024 + adapter
->fn
* EEPROMPFSIZE
;
1871 if (aligned_offset
< start
||
1872 aligned_offset
+ aligned_len
> start
+ EEPROMPFSIZE
)
1876 if (aligned_offset
!= eeprom
->offset
|| aligned_len
!= eeprom
->len
) {
1878 * RMW possibly needed for first or last words.
1880 buf
= kmalloc(aligned_len
, GFP_KERNEL
);
1883 err
= eeprom_rd_phys(adapter
, aligned_offset
, (u32
*)buf
);
1884 if (!err
&& aligned_len
> 4)
1885 err
= eeprom_rd_phys(adapter
,
1886 aligned_offset
+ aligned_len
- 4,
1887 (u32
*)&buf
[aligned_len
- 4]);
1890 memcpy(buf
+ (eeprom
->offset
& 3), data
, eeprom
->len
);
1894 err
= t4_seeprom_wp(adapter
, false);
1898 for (p
= (u32
*)buf
; !err
&& aligned_len
; aligned_len
-= 4, p
++) {
1899 err
= eeprom_wr_phys(adapter
, aligned_offset
, *p
);
1900 aligned_offset
+= 4;
1904 err
= t4_seeprom_wp(adapter
, true);
1911 static int set_flash(struct net_device
*netdev
, struct ethtool_flash
*ef
)
1914 const struct firmware
*fw
;
1915 struct adapter
*adap
= netdev2adap(netdev
);
1917 ef
->data
[sizeof(ef
->data
) - 1] = '\0';
1918 ret
= request_firmware(&fw
, ef
->data
, adap
->pdev_dev
);
1922 ret
= t4_load_fw(adap
, fw
->data
, fw
->size
);
1923 release_firmware(fw
);
1925 dev_info(adap
->pdev_dev
, "loaded firmware %s\n", ef
->data
);
1929 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1930 #define BCAST_CRC 0xa0ccc1a6
1932 static void get_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
1934 wol
->supported
= WAKE_BCAST
| WAKE_MAGIC
;
1935 wol
->wolopts
= netdev2adap(dev
)->wol
;
1936 memset(&wol
->sopass
, 0, sizeof(wol
->sopass
));
1939 static int set_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
1942 struct port_info
*pi
= netdev_priv(dev
);
1944 if (wol
->wolopts
& ~WOL_SUPPORTED
)
1946 t4_wol_magic_enable(pi
->adapter
, pi
->tx_chan
,
1947 (wol
->wolopts
& WAKE_MAGIC
) ? dev
->dev_addr
: NULL
);
1948 if (wol
->wolopts
& WAKE_BCAST
) {
1949 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0xfe, ~0ULL,
1952 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 1,
1953 ~6ULL, ~0ULL, BCAST_CRC
, true);
1955 t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0, 0, 0, 0, false);
1959 static int cxgb_set_features(struct net_device
*dev
, netdev_features_t features
)
1961 const struct port_info
*pi
= netdev_priv(dev
);
1962 netdev_features_t changed
= dev
->features
^ features
;
1965 if (!(changed
& NETIF_F_HW_VLAN_RX
))
1968 err
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, -1,
1970 !!(features
& NETIF_F_HW_VLAN_RX
), true);
1972 dev
->features
= features
^ NETIF_F_HW_VLAN_RX
;
1976 static u32
get_rss_table_size(struct net_device
*dev
)
1978 const struct port_info
*pi
= netdev_priv(dev
);
1980 return pi
->rss_size
;
1983 static int get_rss_table(struct net_device
*dev
, u32
*p
)
1985 const struct port_info
*pi
= netdev_priv(dev
);
1986 unsigned int n
= pi
->rss_size
;
1993 static int set_rss_table(struct net_device
*dev
, const u32
*p
)
1996 struct port_info
*pi
= netdev_priv(dev
);
1998 for (i
= 0; i
< pi
->rss_size
; i
++)
2000 if (pi
->adapter
->flags
& FULL_INIT_DONE
)
2001 return write_rss(pi
, pi
->rss
);
2005 static int get_rxnfc(struct net_device
*dev
, struct ethtool_rxnfc
*info
,
2008 const struct port_info
*pi
= netdev_priv(dev
);
2010 switch (info
->cmd
) {
2011 case ETHTOOL_GRXFH
: {
2012 unsigned int v
= pi
->rss_mode
;
2015 switch (info
->flow_type
) {
2017 if (v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
)
2018 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2019 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2020 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2021 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2024 if ((v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
) &&
2025 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
2026 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2027 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2028 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2029 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2032 case AH_ESP_V4_FLOW
:
2034 if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2035 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2038 if (v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
)
2039 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2040 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2041 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2042 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2045 if ((v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
) &&
2046 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
2047 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2048 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2049 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2050 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2053 case AH_ESP_V6_FLOW
:
2055 if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2056 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2061 case ETHTOOL_GRXRINGS
:
2062 info
->data
= pi
->nqsets
;
2068 static const struct ethtool_ops cxgb_ethtool_ops
= {
2069 .get_settings
= get_settings
,
2070 .set_settings
= set_settings
,
2071 .get_drvinfo
= get_drvinfo
,
2072 .get_msglevel
= get_msglevel
,
2073 .set_msglevel
= set_msglevel
,
2074 .get_ringparam
= get_sge_param
,
2075 .set_ringparam
= set_sge_param
,
2076 .get_coalesce
= get_coalesce
,
2077 .set_coalesce
= set_coalesce
,
2078 .get_eeprom_len
= get_eeprom_len
,
2079 .get_eeprom
= get_eeprom
,
2080 .set_eeprom
= set_eeprom
,
2081 .get_pauseparam
= get_pauseparam
,
2082 .set_pauseparam
= set_pauseparam
,
2083 .get_link
= ethtool_op_get_link
,
2084 .get_strings
= get_strings
,
2085 .set_phys_id
= identify_port
,
2086 .nway_reset
= restart_autoneg
,
2087 .get_sset_count
= get_sset_count
,
2088 .get_ethtool_stats
= get_stats
,
2089 .get_regs_len
= get_regs_len
,
2090 .get_regs
= get_regs
,
2093 .get_rxnfc
= get_rxnfc
,
2094 .get_rxfh_indir_size
= get_rss_table_size
,
2095 .get_rxfh_indir
= get_rss_table
,
2096 .set_rxfh_indir
= set_rss_table
,
2097 .flash_device
= set_flash
,
2103 static ssize_t
mem_read(struct file
*file
, char __user
*buf
, size_t count
,
2107 loff_t avail
= file
->f_path
.dentry
->d_inode
->i_size
;
2108 unsigned int mem
= (uintptr_t)file
->private_data
& 3;
2109 struct adapter
*adap
= file
->private_data
- mem
;
2115 if (count
> avail
- pos
)
2116 count
= avail
- pos
;
2124 ret
= t4_mc_read(adap
, pos
, data
, NULL
);
2126 ret
= t4_edc_read(adap
, mem
, pos
, data
, NULL
);
2130 ofst
= pos
% sizeof(data
);
2131 len
= min(count
, sizeof(data
) - ofst
);
2132 if (copy_to_user(buf
, (u8
*)data
+ ofst
, len
))
2139 count
= pos
- *ppos
;
2144 static const struct file_operations mem_debugfs_fops
= {
2145 .owner
= THIS_MODULE
,
2146 .open
= simple_open
,
2148 .llseek
= default_llseek
,
2151 static void __devinit
add_debugfs_mem(struct adapter
*adap
, const char *name
,
2152 unsigned int idx
, unsigned int size_mb
)
2156 de
= debugfs_create_file(name
, S_IRUSR
, adap
->debugfs_root
,
2157 (void *)adap
+ idx
, &mem_debugfs_fops
);
2158 if (de
&& de
->d_inode
)
2159 de
->d_inode
->i_size
= size_mb
<< 20;
2162 static int __devinit
setup_debugfs(struct adapter
*adap
)
2166 if (IS_ERR_OR_NULL(adap
->debugfs_root
))
2169 i
= t4_read_reg(adap
, MA_TARGET_MEM_ENABLE
);
2170 if (i
& EDRAM0_ENABLE
)
2171 add_debugfs_mem(adap
, "edc0", MEM_EDC0
, 5);
2172 if (i
& EDRAM1_ENABLE
)
2173 add_debugfs_mem(adap
, "edc1", MEM_EDC1
, 5);
2174 if (i
& EXT_MEM_ENABLE
)
2175 add_debugfs_mem(adap
, "mc", MEM_MC
,
2176 EXT_MEM_SIZE_GET(t4_read_reg(adap
, MA_EXT_MEMORY_BAR
)));
2178 debugfs_create_file("l2t", S_IRUSR
, adap
->debugfs_root
, adap
,
2184 * upper-layer driver support
2188 * Allocate an active-open TID and set it to the supplied value.
2190 int cxgb4_alloc_atid(struct tid_info
*t
, void *data
)
2194 spin_lock_bh(&t
->atid_lock
);
2196 union aopen_entry
*p
= t
->afree
;
2198 atid
= p
- t
->atid_tab
;
2203 spin_unlock_bh(&t
->atid_lock
);
2206 EXPORT_SYMBOL(cxgb4_alloc_atid
);
2209 * Release an active-open TID.
2211 void cxgb4_free_atid(struct tid_info
*t
, unsigned int atid
)
2213 union aopen_entry
*p
= &t
->atid_tab
[atid
];
2215 spin_lock_bh(&t
->atid_lock
);
2219 spin_unlock_bh(&t
->atid_lock
);
2221 EXPORT_SYMBOL(cxgb4_free_atid
);
2224 * Allocate a server TID and set it to the supplied value.
2226 int cxgb4_alloc_stid(struct tid_info
*t
, int family
, void *data
)
2230 spin_lock_bh(&t
->stid_lock
);
2231 if (family
== PF_INET
) {
2232 stid
= find_first_zero_bit(t
->stid_bmap
, t
->nstids
);
2233 if (stid
< t
->nstids
)
2234 __set_bit(stid
, t
->stid_bmap
);
2238 stid
= bitmap_find_free_region(t
->stid_bmap
, t
->nstids
, 2);
2243 t
->stid_tab
[stid
].data
= data
;
2244 stid
+= t
->stid_base
;
2247 spin_unlock_bh(&t
->stid_lock
);
2250 EXPORT_SYMBOL(cxgb4_alloc_stid
);
2253 * Release a server TID.
2255 void cxgb4_free_stid(struct tid_info
*t
, unsigned int stid
, int family
)
2257 stid
-= t
->stid_base
;
2258 spin_lock_bh(&t
->stid_lock
);
2259 if (family
== PF_INET
)
2260 __clear_bit(stid
, t
->stid_bmap
);
2262 bitmap_release_region(t
->stid_bmap
, stid
, 2);
2263 t
->stid_tab
[stid
].data
= NULL
;
2265 spin_unlock_bh(&t
->stid_lock
);
2267 EXPORT_SYMBOL(cxgb4_free_stid
);
2270 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2272 static void mk_tid_release(struct sk_buff
*skb
, unsigned int chan
,
2275 struct cpl_tid_release
*req
;
2277 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, chan
);
2278 req
= (struct cpl_tid_release
*)__skb_put(skb
, sizeof(*req
));
2279 INIT_TP_WR(req
, tid
);
2280 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE
, tid
));
2284 * Queue a TID release request and if necessary schedule a work queue to
2287 static void cxgb4_queue_tid_release(struct tid_info
*t
, unsigned int chan
,
2290 void **p
= &t
->tid_tab
[tid
];
2291 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
2293 spin_lock_bh(&adap
->tid_release_lock
);
2294 *p
= adap
->tid_release_head
;
2295 /* Low 2 bits encode the Tx channel number */
2296 adap
->tid_release_head
= (void **)((uintptr_t)p
| chan
);
2297 if (!adap
->tid_release_task_busy
) {
2298 adap
->tid_release_task_busy
= true;
2299 queue_work(workq
, &adap
->tid_release_task
);
2301 spin_unlock_bh(&adap
->tid_release_lock
);
2305 * Process the list of pending TID release requests.
2307 static void process_tid_release_list(struct work_struct
*work
)
2309 struct sk_buff
*skb
;
2310 struct adapter
*adap
;
2312 adap
= container_of(work
, struct adapter
, tid_release_task
);
2314 spin_lock_bh(&adap
->tid_release_lock
);
2315 while (adap
->tid_release_head
) {
2316 void **p
= adap
->tid_release_head
;
2317 unsigned int chan
= (uintptr_t)p
& 3;
2318 p
= (void *)p
- chan
;
2320 adap
->tid_release_head
= *p
;
2322 spin_unlock_bh(&adap
->tid_release_lock
);
2324 while (!(skb
= alloc_skb(sizeof(struct cpl_tid_release
),
2326 schedule_timeout_uninterruptible(1);
2328 mk_tid_release(skb
, chan
, p
- adap
->tids
.tid_tab
);
2329 t4_ofld_send(adap
, skb
);
2330 spin_lock_bh(&adap
->tid_release_lock
);
2332 adap
->tid_release_task_busy
= false;
2333 spin_unlock_bh(&adap
->tid_release_lock
);
2337 * Release a TID and inform HW. If we are unable to allocate the release
2338 * message we defer to a work queue.
2340 void cxgb4_remove_tid(struct tid_info
*t
, unsigned int chan
, unsigned int tid
)
2343 struct sk_buff
*skb
;
2344 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
2346 old
= t
->tid_tab
[tid
];
2347 skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_ATOMIC
);
2349 t
->tid_tab
[tid
] = NULL
;
2350 mk_tid_release(skb
, chan
, tid
);
2351 t4_ofld_send(adap
, skb
);
2353 cxgb4_queue_tid_release(t
, chan
, tid
);
2355 atomic_dec(&t
->tids_in_use
);
2357 EXPORT_SYMBOL(cxgb4_remove_tid
);
2360 * Allocate and initialize the TID tables. Returns 0 on success.
2362 static int tid_init(struct tid_info
*t
)
2365 unsigned int natids
= t
->natids
;
2367 size
= t
->ntids
* sizeof(*t
->tid_tab
) + natids
* sizeof(*t
->atid_tab
) +
2368 t
->nstids
* sizeof(*t
->stid_tab
) +
2369 BITS_TO_LONGS(t
->nstids
) * sizeof(long);
2370 t
->tid_tab
= t4_alloc_mem(size
);
2374 t
->atid_tab
= (union aopen_entry
*)&t
->tid_tab
[t
->ntids
];
2375 t
->stid_tab
= (struct serv_entry
*)&t
->atid_tab
[natids
];
2376 t
->stid_bmap
= (unsigned long *)&t
->stid_tab
[t
->nstids
];
2377 spin_lock_init(&t
->stid_lock
);
2378 spin_lock_init(&t
->atid_lock
);
2380 t
->stids_in_use
= 0;
2382 t
->atids_in_use
= 0;
2383 atomic_set(&t
->tids_in_use
, 0);
2385 /* Setup the free list for atid_tab and clear the stid bitmap. */
2388 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
2389 t
->afree
= t
->atid_tab
;
2391 bitmap_zero(t
->stid_bmap
, t
->nstids
);
2396 * cxgb4_create_server - create an IP server
2398 * @stid: the server TID
2399 * @sip: local IP address to bind server to
2400 * @sport: the server's TCP port
2401 * @queue: queue to direct messages from this server to
2403 * Create an IP server for the given port and address.
2404 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2406 int cxgb4_create_server(const struct net_device
*dev
, unsigned int stid
,
2407 __be32 sip
, __be16 sport
, unsigned int queue
)
2410 struct sk_buff
*skb
;
2411 struct adapter
*adap
;
2412 struct cpl_pass_open_req
*req
;
2414 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
2418 adap
= netdev2adap(dev
);
2419 req
= (struct cpl_pass_open_req
*)__skb_put(skb
, sizeof(*req
));
2421 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ
, stid
));
2422 req
->local_port
= sport
;
2423 req
->peer_port
= htons(0);
2424 req
->local_ip
= sip
;
2425 req
->peer_ip
= htonl(0);
2426 chan
= rxq_to_chan(&adap
->sge
, queue
);
2427 req
->opt0
= cpu_to_be64(TX_CHAN(chan
));
2428 req
->opt1
= cpu_to_be64(CONN_POLICY_ASK
|
2429 SYN_RSS_ENABLE
| SYN_RSS_QUEUE(queue
));
2430 return t4_mgmt_tx(adap
, skb
);
2432 EXPORT_SYMBOL(cxgb4_create_server
);
2435 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2436 * @mtus: the HW MTU table
2437 * @mtu: the target MTU
2438 * @idx: index of selected entry in the MTU table
2440 * Returns the index and the value in the HW MTU table that is closest to
2441 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2442 * table, in which case that smallest available value is selected.
2444 unsigned int cxgb4_best_mtu(const unsigned short *mtus
, unsigned short mtu
,
2449 while (i
< NMTUS
- 1 && mtus
[i
+ 1] <= mtu
)
2455 EXPORT_SYMBOL(cxgb4_best_mtu
);
2458 * cxgb4_port_chan - get the HW channel of a port
2459 * @dev: the net device for the port
2461 * Return the HW Tx channel of the given port.
2463 unsigned int cxgb4_port_chan(const struct net_device
*dev
)
2465 return netdev2pinfo(dev
)->tx_chan
;
2467 EXPORT_SYMBOL(cxgb4_port_chan
);
2469 unsigned int cxgb4_dbfifo_count(const struct net_device
*dev
, int lpfifo
)
2471 struct adapter
*adap
= netdev2adap(dev
);
2474 v
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
2475 return lpfifo
? G_LP_COUNT(v
) : G_HP_COUNT(v
);
2477 EXPORT_SYMBOL(cxgb4_dbfifo_count
);
2480 * cxgb4_port_viid - get the VI id of a port
2481 * @dev: the net device for the port
2483 * Return the VI id of the given port.
2485 unsigned int cxgb4_port_viid(const struct net_device
*dev
)
2487 return netdev2pinfo(dev
)->viid
;
2489 EXPORT_SYMBOL(cxgb4_port_viid
);
2492 * cxgb4_port_idx - get the index of a port
2493 * @dev: the net device for the port
2495 * Return the index of the given port.
2497 unsigned int cxgb4_port_idx(const struct net_device
*dev
)
2499 return netdev2pinfo(dev
)->port_id
;
2501 EXPORT_SYMBOL(cxgb4_port_idx
);
2503 void cxgb4_get_tcp_stats(struct pci_dev
*pdev
, struct tp_tcp_stats
*v4
,
2504 struct tp_tcp_stats
*v6
)
2506 struct adapter
*adap
= pci_get_drvdata(pdev
);
2508 spin_lock(&adap
->stats_lock
);
2509 t4_tp_get_tcp_stats(adap
, v4
, v6
);
2510 spin_unlock(&adap
->stats_lock
);
2512 EXPORT_SYMBOL(cxgb4_get_tcp_stats
);
2514 void cxgb4_iscsi_init(struct net_device
*dev
, unsigned int tag_mask
,
2515 const unsigned int *pgsz_order
)
2517 struct adapter
*adap
= netdev2adap(dev
);
2519 t4_write_reg(adap
, ULP_RX_ISCSI_TAGMASK
, tag_mask
);
2520 t4_write_reg(adap
, ULP_RX_ISCSI_PSZ
, HPZ0(pgsz_order
[0]) |
2521 HPZ1(pgsz_order
[1]) | HPZ2(pgsz_order
[2]) |
2522 HPZ3(pgsz_order
[3]));
2524 EXPORT_SYMBOL(cxgb4_iscsi_init
);
2526 int cxgb4_flush_eq_cache(struct net_device
*dev
)
2528 struct adapter
*adap
= netdev2adap(dev
);
2531 ret
= t4_fwaddrspace_write(adap
, adap
->mbox
,
2532 0xe1000000 + A_SGE_CTXT_CMD
, 0x20000000);
2535 EXPORT_SYMBOL(cxgb4_flush_eq_cache
);
2537 static int read_eq_indices(struct adapter
*adap
, u16 qid
, u16
*pidx
, u16
*cidx
)
2539 u32 addr
= t4_read_reg(adap
, A_SGE_DBQ_CTXT_BADDR
) + 24 * qid
+ 8;
2543 ret
= t4_mem_win_read_len(adap
, addr
, (__be32
*)&indices
, 8);
2545 *cidx
= (be64_to_cpu(indices
) >> 25) & 0xffff;
2546 *pidx
= (be64_to_cpu(indices
) >> 9) & 0xffff;
2551 int cxgb4_sync_txq_pidx(struct net_device
*dev
, u16 qid
, u16 pidx
,
2554 struct adapter
*adap
= netdev2adap(dev
);
2555 u16 hw_pidx
, hw_cidx
;
2558 ret
= read_eq_indices(adap
, qid
, &hw_pidx
, &hw_cidx
);
2562 if (pidx
!= hw_pidx
) {
2565 if (pidx
>= hw_pidx
)
2566 delta
= pidx
- hw_pidx
;
2568 delta
= size
- hw_pidx
+ pidx
;
2570 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
2571 QID(qid
) | PIDX(delta
));
2576 EXPORT_SYMBOL(cxgb4_sync_txq_pidx
);
2578 static struct pci_driver cxgb4_driver
;
2580 static void check_neigh_update(struct neighbour
*neigh
)
2582 const struct device
*parent
;
2583 const struct net_device
*netdev
= neigh
->dev
;
2585 if (netdev
->priv_flags
& IFF_802_1Q_VLAN
)
2586 netdev
= vlan_dev_real_dev(netdev
);
2587 parent
= netdev
->dev
.parent
;
2588 if (parent
&& parent
->driver
== &cxgb4_driver
.driver
)
2589 t4_l2t_update(dev_get_drvdata(parent
), neigh
);
2592 static int netevent_cb(struct notifier_block
*nb
, unsigned long event
,
2596 case NETEVENT_NEIGH_UPDATE
:
2597 check_neigh_update(data
);
2599 case NETEVENT_REDIRECT
:
2606 static bool netevent_registered
;
2607 static struct notifier_block cxgb4_netevent_nb
= {
2608 .notifier_call
= netevent_cb
2611 static void drain_db_fifo(struct adapter
*adap
, int usecs
)
2616 set_current_state(TASK_UNINTERRUPTIBLE
);
2617 schedule_timeout(usecs_to_jiffies(usecs
));
2618 v
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
2619 if (G_LP_COUNT(v
) == 0 && G_HP_COUNT(v
) == 0)
2624 static void disable_txq_db(struct sge_txq
*q
)
2626 spin_lock_irq(&q
->db_lock
);
2628 spin_unlock_irq(&q
->db_lock
);
2631 static void enable_txq_db(struct sge_txq
*q
)
2633 spin_lock_irq(&q
->db_lock
);
2635 spin_unlock_irq(&q
->db_lock
);
2638 static void disable_dbs(struct adapter
*adap
)
2642 for_each_ethrxq(&adap
->sge
, i
)
2643 disable_txq_db(&adap
->sge
.ethtxq
[i
].q
);
2644 for_each_ofldrxq(&adap
->sge
, i
)
2645 disable_txq_db(&adap
->sge
.ofldtxq
[i
].q
);
2646 for_each_port(adap
, i
)
2647 disable_txq_db(&adap
->sge
.ctrlq
[i
].q
);
2650 static void enable_dbs(struct adapter
*adap
)
2654 for_each_ethrxq(&adap
->sge
, i
)
2655 enable_txq_db(&adap
->sge
.ethtxq
[i
].q
);
2656 for_each_ofldrxq(&adap
->sge
, i
)
2657 enable_txq_db(&adap
->sge
.ofldtxq
[i
].q
);
2658 for_each_port(adap
, i
)
2659 enable_txq_db(&adap
->sge
.ctrlq
[i
].q
);
2662 static void sync_txq_pidx(struct adapter
*adap
, struct sge_txq
*q
)
2664 u16 hw_pidx
, hw_cidx
;
2667 spin_lock_bh(&q
->db_lock
);
2668 ret
= read_eq_indices(adap
, (u16
)q
->cntxt_id
, &hw_pidx
, &hw_cidx
);
2671 if (q
->db_pidx
!= hw_pidx
) {
2674 if (q
->db_pidx
>= hw_pidx
)
2675 delta
= q
->db_pidx
- hw_pidx
;
2677 delta
= q
->size
- hw_pidx
+ q
->db_pidx
;
2679 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
2680 QID(q
->cntxt_id
) | PIDX(delta
));
2684 spin_unlock_bh(&q
->db_lock
);
2686 CH_WARN(adap
, "DB drop recovery failed.\n");
2688 static void recover_all_queues(struct adapter
*adap
)
2692 for_each_ethrxq(&adap
->sge
, i
)
2693 sync_txq_pidx(adap
, &adap
->sge
.ethtxq
[i
].q
);
2694 for_each_ofldrxq(&adap
->sge
, i
)
2695 sync_txq_pidx(adap
, &adap
->sge
.ofldtxq
[i
].q
);
2696 for_each_port(adap
, i
)
2697 sync_txq_pidx(adap
, &adap
->sge
.ctrlq
[i
].q
);
2700 static void notify_rdma_uld(struct adapter
*adap
, enum cxgb4_control cmd
)
2702 mutex_lock(&uld_mutex
);
2703 if (adap
->uld_handle
[CXGB4_ULD_RDMA
])
2704 ulds
[CXGB4_ULD_RDMA
].control(adap
->uld_handle
[CXGB4_ULD_RDMA
],
2706 mutex_unlock(&uld_mutex
);
2709 static void process_db_full(struct work_struct
*work
)
2711 struct adapter
*adap
;
2713 adap
= container_of(work
, struct adapter
, db_full_task
);
2715 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_FULL
);
2716 drain_db_fifo(adap
, dbfifo_drain_delay
);
2717 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
2718 DBFIFO_HP_INT
| DBFIFO_LP_INT
,
2719 DBFIFO_HP_INT
| DBFIFO_LP_INT
);
2720 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_EMPTY
);
2723 static void process_db_drop(struct work_struct
*work
)
2725 struct adapter
*adap
;
2727 adap
= container_of(work
, struct adapter
, db_drop_task
);
2729 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_DROPPED_DB
, 0);
2731 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_DROP
);
2732 drain_db_fifo(adap
, 1);
2733 recover_all_queues(adap
);
2737 void t4_db_full(struct adapter
*adap
)
2739 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
2740 DBFIFO_HP_INT
| DBFIFO_LP_INT
, 0);
2741 queue_work(workq
, &adap
->db_full_task
);
2744 void t4_db_dropped(struct adapter
*adap
)
2746 queue_work(workq
, &adap
->db_drop_task
);
2749 static void uld_attach(struct adapter
*adap
, unsigned int uld
)
2752 struct cxgb4_lld_info lli
;
2754 lli
.pdev
= adap
->pdev
;
2755 lli
.l2t
= adap
->l2t
;
2756 lli
.tids
= &adap
->tids
;
2757 lli
.ports
= adap
->port
;
2758 lli
.vr
= &adap
->vres
;
2759 lli
.mtus
= adap
->params
.mtus
;
2760 if (uld
== CXGB4_ULD_RDMA
) {
2761 lli
.rxq_ids
= adap
->sge
.rdma_rxq
;
2762 lli
.nrxq
= adap
->sge
.rdmaqs
;
2763 } else if (uld
== CXGB4_ULD_ISCSI
) {
2764 lli
.rxq_ids
= adap
->sge
.ofld_rxq
;
2765 lli
.nrxq
= adap
->sge
.ofldqsets
;
2767 lli
.ntxq
= adap
->sge
.ofldqsets
;
2768 lli
.nchan
= adap
->params
.nports
;
2769 lli
.nports
= adap
->params
.nports
;
2770 lli
.wr_cred
= adap
->params
.ofldq_wr_cred
;
2771 lli
.adapter_type
= adap
->params
.rev
;
2772 lli
.iscsi_iolen
= MAXRXDATA_GET(t4_read_reg(adap
, TP_PARA_REG2
));
2773 lli
.udb_density
= 1 << QUEUESPERPAGEPF0_GET(
2774 t4_read_reg(adap
, SGE_EGRESS_QUEUES_PER_PAGE_PF
) >>
2776 lli
.ucq_density
= 1 << QUEUESPERPAGEPF0_GET(
2777 t4_read_reg(adap
, SGE_INGRESS_QUEUES_PER_PAGE_PF
) >>
2779 lli
.gts_reg
= adap
->regs
+ MYPF_REG(SGE_PF_GTS
);
2780 lli
.db_reg
= adap
->regs
+ MYPF_REG(SGE_PF_KDOORBELL
);
2781 lli
.fw_vers
= adap
->params
.fw_vers
;
2782 lli
.dbfifo_int_thresh
= dbfifo_int_thresh
;
2784 handle
= ulds
[uld
].add(&lli
);
2785 if (IS_ERR(handle
)) {
2786 dev_warn(adap
->pdev_dev
,
2787 "could not attach to the %s driver, error %ld\n",
2788 uld_str
[uld
], PTR_ERR(handle
));
2792 adap
->uld_handle
[uld
] = handle
;
2794 if (!netevent_registered
) {
2795 register_netevent_notifier(&cxgb4_netevent_nb
);
2796 netevent_registered
= true;
2799 if (adap
->flags
& FULL_INIT_DONE
)
2800 ulds
[uld
].state_change(handle
, CXGB4_STATE_UP
);
2803 static void attach_ulds(struct adapter
*adap
)
2807 mutex_lock(&uld_mutex
);
2808 list_add_tail(&adap
->list_node
, &adapter_list
);
2809 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
2811 uld_attach(adap
, i
);
2812 mutex_unlock(&uld_mutex
);
2815 static void detach_ulds(struct adapter
*adap
)
2819 mutex_lock(&uld_mutex
);
2820 list_del(&adap
->list_node
);
2821 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
2822 if (adap
->uld_handle
[i
]) {
2823 ulds
[i
].state_change(adap
->uld_handle
[i
],
2824 CXGB4_STATE_DETACH
);
2825 adap
->uld_handle
[i
] = NULL
;
2827 if (netevent_registered
&& list_empty(&adapter_list
)) {
2828 unregister_netevent_notifier(&cxgb4_netevent_nb
);
2829 netevent_registered
= false;
2831 mutex_unlock(&uld_mutex
);
2834 static void notify_ulds(struct adapter
*adap
, enum cxgb4_state new_state
)
2838 mutex_lock(&uld_mutex
);
2839 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
2840 if (adap
->uld_handle
[i
])
2841 ulds
[i
].state_change(adap
->uld_handle
[i
], new_state
);
2842 mutex_unlock(&uld_mutex
);
2846 * cxgb4_register_uld - register an upper-layer driver
2847 * @type: the ULD type
2848 * @p: the ULD methods
2850 * Registers an upper-layer driver with this driver and notifies the ULD
2851 * about any presently available devices that support its type. Returns
2852 * %-EBUSY if a ULD of the same type is already registered.
2854 int cxgb4_register_uld(enum cxgb4_uld type
, const struct cxgb4_uld_info
*p
)
2857 struct adapter
*adap
;
2859 if (type
>= CXGB4_ULD_MAX
)
2861 mutex_lock(&uld_mutex
);
2862 if (ulds
[type
].add
) {
2867 list_for_each_entry(adap
, &adapter_list
, list_node
)
2868 uld_attach(adap
, type
);
2869 out
: mutex_unlock(&uld_mutex
);
2872 EXPORT_SYMBOL(cxgb4_register_uld
);
2875 * cxgb4_unregister_uld - unregister an upper-layer driver
2876 * @type: the ULD type
2878 * Unregisters an existing upper-layer driver.
2880 int cxgb4_unregister_uld(enum cxgb4_uld type
)
2882 struct adapter
*adap
;
2884 if (type
>= CXGB4_ULD_MAX
)
2886 mutex_lock(&uld_mutex
);
2887 list_for_each_entry(adap
, &adapter_list
, list_node
)
2888 adap
->uld_handle
[type
] = NULL
;
2889 ulds
[type
].add
= NULL
;
2890 mutex_unlock(&uld_mutex
);
2893 EXPORT_SYMBOL(cxgb4_unregister_uld
);
2896 * cxgb_up - enable the adapter
2897 * @adap: adapter being enabled
2899 * Called when the first port is enabled, this function performs the
2900 * actions necessary to make an adapter operational, such as completing
2901 * the initialization of HW modules, and enabling interrupts.
2903 * Must be called with the rtnl lock held.
2905 static int cxgb_up(struct adapter
*adap
)
2909 err
= setup_sge_queues(adap
);
2912 err
= setup_rss(adap
);
2916 if (adap
->flags
& USING_MSIX
) {
2917 name_msix_vecs(adap
);
2918 err
= request_irq(adap
->msix_info
[0].vec
, t4_nondata_intr
, 0,
2919 adap
->msix_info
[0].desc
, adap
);
2923 err
= request_msix_queue_irqs(adap
);
2925 free_irq(adap
->msix_info
[0].vec
, adap
);
2929 err
= request_irq(adap
->pdev
->irq
, t4_intr_handler(adap
),
2930 (adap
->flags
& USING_MSI
) ? 0 : IRQF_SHARED
,
2931 adap
->port
[0]->name
, adap
);
2937 t4_intr_enable(adap
);
2938 adap
->flags
|= FULL_INIT_DONE
;
2939 notify_ulds(adap
, CXGB4_STATE_UP
);
2943 dev_err(adap
->pdev_dev
, "request_irq failed, err %d\n", err
);
2945 t4_free_sge_resources(adap
);
2949 static void cxgb_down(struct adapter
*adapter
)
2951 t4_intr_disable(adapter
);
2952 cancel_work_sync(&adapter
->tid_release_task
);
2953 cancel_work_sync(&adapter
->db_full_task
);
2954 cancel_work_sync(&adapter
->db_drop_task
);
2955 adapter
->tid_release_task_busy
= false;
2956 adapter
->tid_release_head
= NULL
;
2958 if (adapter
->flags
& USING_MSIX
) {
2959 free_msix_queue_irqs(adapter
);
2960 free_irq(adapter
->msix_info
[0].vec
, adapter
);
2962 free_irq(adapter
->pdev
->irq
, adapter
);
2963 quiesce_rx(adapter
);
2964 t4_sge_stop(adapter
);
2965 t4_free_sge_resources(adapter
);
2966 adapter
->flags
&= ~FULL_INIT_DONE
;
2970 * net_device operations
2972 static int cxgb_open(struct net_device
*dev
)
2975 struct port_info
*pi
= netdev_priv(dev
);
2976 struct adapter
*adapter
= pi
->adapter
;
2978 netif_carrier_off(dev
);
2980 if (!(adapter
->flags
& FULL_INIT_DONE
)) {
2981 err
= cxgb_up(adapter
);
2986 err
= link_start(dev
);
2988 netif_tx_start_all_queues(dev
);
2992 static int cxgb_close(struct net_device
*dev
)
2994 struct port_info
*pi
= netdev_priv(dev
);
2995 struct adapter
*adapter
= pi
->adapter
;
2997 netif_tx_stop_all_queues(dev
);
2998 netif_carrier_off(dev
);
2999 return t4_enable_vi(adapter
, adapter
->fn
, pi
->viid
, false, false);
3002 static struct rtnl_link_stats64
*cxgb_get_stats(struct net_device
*dev
,
3003 struct rtnl_link_stats64
*ns
)
3005 struct port_stats stats
;
3006 struct port_info
*p
= netdev_priv(dev
);
3007 struct adapter
*adapter
= p
->adapter
;
3009 spin_lock(&adapter
->stats_lock
);
3010 t4_get_port_stats(adapter
, p
->tx_chan
, &stats
);
3011 spin_unlock(&adapter
->stats_lock
);
3013 ns
->tx_bytes
= stats
.tx_octets
;
3014 ns
->tx_packets
= stats
.tx_frames
;
3015 ns
->rx_bytes
= stats
.rx_octets
;
3016 ns
->rx_packets
= stats
.rx_frames
;
3017 ns
->multicast
= stats
.rx_mcast_frames
;
3019 /* detailed rx_errors */
3020 ns
->rx_length_errors
= stats
.rx_jabber
+ stats
.rx_too_long
+
3022 ns
->rx_over_errors
= 0;
3023 ns
->rx_crc_errors
= stats
.rx_fcs_err
;
3024 ns
->rx_frame_errors
= stats
.rx_symbol_err
;
3025 ns
->rx_fifo_errors
= stats
.rx_ovflow0
+ stats
.rx_ovflow1
+
3026 stats
.rx_ovflow2
+ stats
.rx_ovflow3
+
3027 stats
.rx_trunc0
+ stats
.rx_trunc1
+
3028 stats
.rx_trunc2
+ stats
.rx_trunc3
;
3029 ns
->rx_missed_errors
= 0;
3031 /* detailed tx_errors */
3032 ns
->tx_aborted_errors
= 0;
3033 ns
->tx_carrier_errors
= 0;
3034 ns
->tx_fifo_errors
= 0;
3035 ns
->tx_heartbeat_errors
= 0;
3036 ns
->tx_window_errors
= 0;
3038 ns
->tx_errors
= stats
.tx_error_frames
;
3039 ns
->rx_errors
= stats
.rx_symbol_err
+ stats
.rx_fcs_err
+
3040 ns
->rx_length_errors
+ stats
.rx_len_err
+ ns
->rx_fifo_errors
;
3044 static int cxgb_ioctl(struct net_device
*dev
, struct ifreq
*req
, int cmd
)
3047 int ret
= 0, prtad
, devad
;
3048 struct port_info
*pi
= netdev_priv(dev
);
3049 struct mii_ioctl_data
*data
= (struct mii_ioctl_data
*)&req
->ifr_data
;
3053 if (pi
->mdio_addr
< 0)
3055 data
->phy_id
= pi
->mdio_addr
;
3059 if (mdio_phy_id_is_c45(data
->phy_id
)) {
3060 prtad
= mdio_phy_id_prtad(data
->phy_id
);
3061 devad
= mdio_phy_id_devad(data
->phy_id
);
3062 } else if (data
->phy_id
< 32) {
3063 prtad
= data
->phy_id
;
3065 data
->reg_num
&= 0x1f;
3069 mbox
= pi
->adapter
->fn
;
3070 if (cmd
== SIOCGMIIREG
)
3071 ret
= t4_mdio_rd(pi
->adapter
, mbox
, prtad
, devad
,
3072 data
->reg_num
, &data
->val_out
);
3074 ret
= t4_mdio_wr(pi
->adapter
, mbox
, prtad
, devad
,
3075 data
->reg_num
, data
->val_in
);
3083 static void cxgb_set_rxmode(struct net_device
*dev
)
3085 /* unfortunately we can't return errors to the stack */
3086 set_rxmode(dev
, -1, false);
3089 static int cxgb_change_mtu(struct net_device
*dev
, int new_mtu
)
3092 struct port_info
*pi
= netdev_priv(dev
);
3094 if (new_mtu
< 81 || new_mtu
> MAX_MTU
) /* accommodate SACK */
3096 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, new_mtu
, -1,
3103 static int cxgb_set_mac_addr(struct net_device
*dev
, void *p
)
3106 struct sockaddr
*addr
= p
;
3107 struct port_info
*pi
= netdev_priv(dev
);
3109 if (!is_valid_ether_addr(addr
->sa_data
))
3110 return -EADDRNOTAVAIL
;
3112 ret
= t4_change_mac(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
,
3113 pi
->xact_addr_filt
, addr
->sa_data
, true, true);
3117 memcpy(dev
->dev_addr
, addr
->sa_data
, dev
->addr_len
);
3118 pi
->xact_addr_filt
= ret
;
3122 #ifdef CONFIG_NET_POLL_CONTROLLER
3123 static void cxgb_netpoll(struct net_device
*dev
)
3125 struct port_info
*pi
= netdev_priv(dev
);
3126 struct adapter
*adap
= pi
->adapter
;
3128 if (adap
->flags
& USING_MSIX
) {
3130 struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[pi
->first_qset
];
3132 for (i
= pi
->nqsets
; i
; i
--, rx
++)
3133 t4_sge_intr_msix(0, &rx
->rspq
);
3135 t4_intr_handler(adap
)(0, adap
);
3139 static const struct net_device_ops cxgb4_netdev_ops
= {
3140 .ndo_open
= cxgb_open
,
3141 .ndo_stop
= cxgb_close
,
3142 .ndo_start_xmit
= t4_eth_xmit
,
3143 .ndo_get_stats64
= cxgb_get_stats
,
3144 .ndo_set_rx_mode
= cxgb_set_rxmode
,
3145 .ndo_set_mac_address
= cxgb_set_mac_addr
,
3146 .ndo_set_features
= cxgb_set_features
,
3147 .ndo_validate_addr
= eth_validate_addr
,
3148 .ndo_do_ioctl
= cxgb_ioctl
,
3149 .ndo_change_mtu
= cxgb_change_mtu
,
3150 #ifdef CONFIG_NET_POLL_CONTROLLER
3151 .ndo_poll_controller
= cxgb_netpoll
,
3155 void t4_fatal_err(struct adapter
*adap
)
3157 t4_set_reg_field(adap
, SGE_CONTROL
, GLOBALENABLE
, 0);
3158 t4_intr_disable(adap
);
3159 dev_alert(adap
->pdev_dev
, "encountered fatal error, adapter stopped\n");
3162 static void setup_memwin(struct adapter
*adap
)
3166 bar0
= pci_resource_start(adap
->pdev
, 0); /* truncation intentional */
3167 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 0),
3168 (bar0
+ MEMWIN0_BASE
) | BIR(0) |
3169 WINDOW(ilog2(MEMWIN0_APERTURE
) - 10));
3170 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 1),
3171 (bar0
+ MEMWIN1_BASE
) | BIR(0) |
3172 WINDOW(ilog2(MEMWIN1_APERTURE
) - 10));
3173 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 2),
3174 (bar0
+ MEMWIN2_BASE
) | BIR(0) |
3175 WINDOW(ilog2(MEMWIN2_APERTURE
) - 10));
3178 static void setup_memwin_rdma(struct adapter
*adap
)
3180 if (adap
->vres
.ocq
.size
) {
3181 unsigned int start
, sz_kb
;
3183 start
= pci_resource_start(adap
->pdev
, 2) +
3184 OCQ_WIN_OFFSET(adap
->pdev
, &adap
->vres
);
3185 sz_kb
= roundup_pow_of_two(adap
->vres
.ocq
.size
) >> 10;
3187 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 3),
3188 start
| BIR(1) | WINDOW(ilog2(sz_kb
)));
3190 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3),
3191 adap
->vres
.ocq
.start
);
3193 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3));
3197 static int adap_init1(struct adapter
*adap
, struct fw_caps_config_cmd
*c
)
3202 /* get device capabilities */
3203 memset(c
, 0, sizeof(*c
));
3204 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3205 FW_CMD_REQUEST
| FW_CMD_READ
);
3206 c
->retval_len16
= htonl(FW_LEN16(*c
));
3207 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), c
);
3211 /* select capabilities we'll be using */
3212 if (c
->niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
3214 c
->niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
3216 c
->niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
3217 } else if (vf_acls
) {
3218 dev_err(adap
->pdev_dev
, "virtualization ACLs not supported");
3221 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3222 FW_CMD_REQUEST
| FW_CMD_WRITE
);
3223 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), NULL
);
3227 ret
= t4_config_glbl_rss(adap
, adap
->fn
,
3228 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
3229 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
3230 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
);
3234 ret
= t4_cfg_pfvf(adap
, adap
->fn
, adap
->fn
, 0, MAX_EGRQ
, 64, MAX_INGQ
,
3235 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF
, FW_CMD_CAP_PF
);
3241 /* tweak some settings */
3242 t4_write_reg(adap
, TP_SHIFT_CNT
, 0x64f8849);
3243 t4_write_reg(adap
, ULP_RX_TDDP_PSZ
, HPZ0(PAGE_SHIFT
- 12));
3244 t4_write_reg(adap
, TP_PIO_ADDR
, TP_INGRESS_CONFIG
);
3245 v
= t4_read_reg(adap
, TP_PIO_DATA
);
3246 t4_write_reg(adap
, TP_PIO_DATA
, v
& ~CSUM_HAS_PSEUDO_HDR
);
3248 /* get basic stuff going */
3249 return t4_early_init(adap
, adap
->fn
);
3253 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
3255 #define MAX_ATIDS 8192U
3258 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3260 * If the firmware we're dealing with has Configuration File support, then
3261 * we use that to perform all configuration
3265 * Tweak configuration based on module parameters, etc. Most of these have
3266 * defaults assigned to them by Firmware Configuration Files (if we're using
3267 * them) but need to be explicitly set if we're using hard-coded
3268 * initialization. But even in the case of using Firmware Configuration
3269 * Files, we'd like to expose the ability to change these via module
3270 * parameters so these are essentially common tweaks/settings for
3271 * Configuration Files and hard-coded initialization ...
3273 static int adap_init0_tweaks(struct adapter
*adapter
)
3276 * Fix up various Host-Dependent Parameters like Page Size, Cache
3277 * Line Size, etc. The firmware default is for a 4KB Page Size and
3278 * 64B Cache Line Size ...
3280 t4_fixup_host_params(adapter
, PAGE_SIZE
, L1_CACHE_BYTES
);
3283 * Process module parameters which affect early initialization.
3285 if (rx_dma_offset
!= 2 && rx_dma_offset
!= 0) {
3286 dev_err(&adapter
->pdev
->dev
,
3287 "Ignoring illegal rx_dma_offset=%d, using 2\n",
3291 t4_set_reg_field(adapter
, SGE_CONTROL
,
3293 PKTSHIFT(rx_dma_offset
));
3296 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
3297 * adds the pseudo header itself.
3299 t4_tp_wr_bits_indirect(adapter
, TP_INGRESS_CONFIG
,
3300 CSUM_HAS_PSEUDO_HDR
, 0);
3306 * Attempt to initialize the adapter via a Firmware Configuration File.
3308 static int adap_init0_config(struct adapter
*adapter
, int reset
)
3310 struct fw_caps_config_cmd caps_cmd
;
3311 const struct firmware
*cf
;
3312 unsigned long mtype
= 0, maddr
= 0;
3313 u32 finiver
, finicsum
, cfcsum
;
3314 int ret
, using_flash
;
3317 * Reset device if necessary.
3320 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
3321 PIORSTMODE
| PIORST
);
3327 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
3328 * then use that. Otherwise, use the configuration file stored
3329 * in the adapter flash ...
3331 ret
= request_firmware(&cf
, FW_CFNAME
, adapter
->pdev_dev
);
3334 mtype
= FW_MEMTYPE_CF_FLASH
;
3335 maddr
= t4_flash_cfg_addr(adapter
);
3337 u32 params
[7], val
[7];
3340 if (cf
->size
>= FLASH_CFG_MAX_SIZE
)
3343 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
3344 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
3345 ret
= t4_query_params(adapter
, adapter
->mbox
,
3346 adapter
->fn
, 0, 1, params
, val
);
3349 * For t4_memory_write() below addresses and
3350 * sizes have to be in terms of multiples of 4
3351 * bytes. So, if the Configuration File isn't
3352 * a multiple of 4 bytes in length we'll have
3353 * to write that out separately since we can't
3354 * guarantee that the bytes following the
3355 * residual byte in the buffer returned by
3356 * request_firmware() are zeroed out ...
3358 size_t resid
= cf
->size
& 0x3;
3359 size_t size
= cf
->size
& ~0x3;
3360 __be32
*data
= (__be32
*)cf
->data
;
3362 mtype
= FW_PARAMS_PARAM_Y_GET(val
[0]);
3363 maddr
= FW_PARAMS_PARAM_Z_GET(val
[0]) << 16;
3365 ret
= t4_memory_write(adapter
, mtype
, maddr
,
3367 if (ret
== 0 && resid
!= 0) {
3374 last
.word
= data
[size
>> 2];
3375 for (i
= resid
; i
< 4; i
++)
3377 ret
= t4_memory_write(adapter
, mtype
,
3384 release_firmware(cf
);
3390 * Issue a Capability Configuration command to the firmware to get it
3391 * to parse the Configuration File. We don't use t4_fw_config_file()
3392 * because we want the ability to modify various features after we've
3393 * processed the configuration file ...
3395 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
3396 caps_cmd
.op_to_write
=
3397 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3400 caps_cmd
.retval_len16
=
3401 htonl(FW_CAPS_CONFIG_CMD_CFVALID
|
3402 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype
) |
3403 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr
>> 16) |
3404 FW_LEN16(caps_cmd
));
3405 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
3410 finiver
= ntohl(caps_cmd
.finiver
);
3411 finicsum
= ntohl(caps_cmd
.finicsum
);
3412 cfcsum
= ntohl(caps_cmd
.cfcsum
);
3413 if (finicsum
!= cfcsum
)
3414 dev_warn(adapter
->pdev_dev
, "Configuration File checksum "\
3415 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
3419 * And now tell the firmware to use the configuration we just loaded.
3421 caps_cmd
.op_to_write
=
3422 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3425 caps_cmd
.retval_len16
= htonl(FW_LEN16(caps_cmd
));
3426 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
3432 * Tweak configuration based on system architecture, module
3435 ret
= adap_init0_tweaks(adapter
);
3440 * And finally tell the firmware to initialize itself using the
3441 * parameters from the Configuration File.
3443 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
3448 * Return successfully and note that we're operating with parameters
3449 * not supplied by the driver, rather than from hard-wired
3450 * initialization constants burried in the driver.
3452 adapter
->flags
|= USING_SOFT_PARAMS
;
3453 dev_info(adapter
->pdev_dev
, "Successfully configured using Firmware "\
3454 "Configuration File %s, version %#x, computed checksum %#x\n",
3457 : "/lib/firmware/" FW_CFNAME
),
3462 * Something bad happened. Return the error ... (If the "error"
3463 * is that there's no Configuration File on the adapter we don't
3464 * want to issue a warning since this is fairly common.)
3468 dev_warn(adapter
->pdev_dev
, "Configuration file error %d\n",
3474 * Attempt to initialize the adapter via hard-coded, driver supplied
3477 static int adap_init0_no_config(struct adapter
*adapter
, int reset
)
3479 struct sge
*s
= &adapter
->sge
;
3480 struct fw_caps_config_cmd caps_cmd
;
3485 * Reset device if necessary
3488 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
3489 PIORSTMODE
| PIORST
);
3495 * Get device capabilities and select which we'll be using.
3497 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
3498 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3499 FW_CMD_REQUEST
| FW_CMD_READ
);
3500 caps_cmd
.retval_len16
= htonl(FW_LEN16(caps_cmd
));
3501 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
3506 if (caps_cmd
.niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
3508 caps_cmd
.niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
3510 caps_cmd
.niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
3511 } else if (vf_acls
) {
3512 dev_err(adapter
->pdev_dev
, "virtualization ACLs not supported");
3515 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3516 FW_CMD_REQUEST
| FW_CMD_WRITE
);
3517 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
3523 * Tweak configuration based on system architecture, module
3526 ret
= adap_init0_tweaks(adapter
);
3531 * Select RSS Global Mode we want to use. We use "Basic Virtual"
3532 * mode which maps each Virtual Interface to its own section of
3533 * the RSS Table and we turn on all map and hash enables ...
3535 adapter
->flags
|= RSS_TNLALLLOOKUP
;
3536 ret
= t4_config_glbl_rss(adapter
, adapter
->mbox
,
3537 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
3538 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
3539 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ
|
3540 ((adapter
->flags
& RSS_TNLALLLOOKUP
) ?
3541 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
: 0));
3546 * Set up our own fundamental resource provisioning ...
3548 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
, adapter
->fn
, 0,
3549 PFRES_NEQ
, PFRES_NETHCTRL
,
3550 PFRES_NIQFLINT
, PFRES_NIQ
,
3551 PFRES_TC
, PFRES_NVI
,
3552 FW_PFVF_CMD_CMASK_MASK
,
3553 pfvfres_pmask(adapter
, adapter
->fn
, 0),
3555 PFRES_R_CAPS
, PFRES_WX_CAPS
);
3560 * Perform low level SGE initialization. We need to do this before we
3561 * send the firmware the INITIALIZE command because that will cause
3562 * any other PF Drivers which are waiting for the Master
3563 * Initialization to proceed forward.
3565 for (i
= 0; i
< SGE_NTIMERS
- 1; i
++)
3566 s
->timer_val
[i
] = min(intr_holdoff
[i
], MAX_SGE_TIMERVAL
);
3567 s
->timer_val
[SGE_NTIMERS
- 1] = MAX_SGE_TIMERVAL
;
3568 s
->counter_val
[0] = 1;
3569 for (i
= 1; i
< SGE_NCOUNTERS
; i
++)
3570 s
->counter_val
[i
] = min(intr_cnt
[i
- 1],
3571 THRESHOLD_0_GET(THRESHOLD_0_MASK
));
3572 t4_sge_init(adapter
);
3574 #ifdef CONFIG_PCI_IOV
3576 * Provision resource limits for Virtual Functions. We currently
3577 * grant them all the same static resource limits except for the Port
3578 * Access Rights Mask which we're assigning based on the PF. All of
3579 * the static provisioning stuff for both the PF and VF really needs
3580 * to be managed in a persistent manner for each device which the
3581 * firmware controls.
3586 for (pf
= 0; pf
< ARRAY_SIZE(num_vf
); pf
++) {
3587 if (num_vf
[pf
] <= 0)
3590 /* VF numbering starts at 1! */
3591 for (vf
= 1; vf
<= num_vf
[pf
]; vf
++) {
3592 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
,
3594 VFRES_NEQ
, VFRES_NETHCTRL
,
3595 VFRES_NIQFLINT
, VFRES_NIQ
,
3596 VFRES_TC
, VFRES_NVI
,
3597 FW_PFVF_CMD_CMASK_GET(
3598 FW_PFVF_CMD_CMASK_MASK
),
3602 VFRES_R_CAPS
, VFRES_WX_CAPS
);
3604 dev_warn(adapter
->pdev_dev
,
3606 "provision pf/vf=%d/%d; "
3607 "err=%d\n", pf
, vf
, ret
);
3614 * Set up the default filter mode. Later we'll want to implement this
3615 * via a firmware command, etc. ... This needs to be done before the
3616 * firmare initialization command ... If the selected set of fields
3617 * isn't equal to the default value, we'll need to make sure that the
3618 * field selections will fit in the 36-bit budget.
3620 if (tp_vlan_pri_map
!= TP_VLAN_PRI_MAP_DEFAULT
) {
3623 for (j
= TP_VLAN_PRI_MAP_FIRST
; j
<= TP_VLAN_PRI_MAP_LAST
; j
++)
3624 switch (tp_vlan_pri_map
& (1 << j
)) {
3626 /* compressed filter field not enabled */
3646 case ETHERTYPE_MASK
:
3652 case MPSHITTYPE_MASK
:
3655 case FRAGMENTATION_MASK
:
3661 dev_err(adapter
->pdev_dev
,
3662 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
3663 " using %#x\n", tp_vlan_pri_map
, bits
,
3664 TP_VLAN_PRI_MAP_DEFAULT
);
3665 tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
3668 v
= tp_vlan_pri_map
;
3669 t4_write_indirect(adapter
, TP_PIO_ADDR
, TP_PIO_DATA
,
3670 &v
, 1, TP_VLAN_PRI_MAP
);
3673 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
3674 * to support any of the compressed filter fields above. Newer
3675 * versions of the firmware do this automatically but it doesn't hurt
3676 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
3677 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
3678 * since the firmware automatically turns this on and off when we have
3679 * a non-zero number of filters active (since it does have a
3680 * performance impact).
3682 if (tp_vlan_pri_map
)
3683 t4_set_reg_field(adapter
, TP_GLOBAL_CONFIG
,
3684 FIVETUPLELOOKUP_MASK
,
3685 FIVETUPLELOOKUP_MASK
);
3688 * Tweak some settings.
3690 t4_write_reg(adapter
, TP_SHIFT_CNT
, SYNSHIFTMAX(6) |
3691 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
3692 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
3693 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
3696 * Get basic stuff going by issuing the Firmware Initialize command.
3697 * Note that this _must_ be after all PFVF commands ...
3699 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
3704 * Return successfully!
3706 dev_info(adapter
->pdev_dev
, "Successfully configured using built-in "\
3707 "driver parameters\n");
3711 * Something bad happened. Return the error ...
3718 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3720 static int adap_init0(struct adapter
*adap
)
3724 enum dev_state state
;
3725 u32 params
[7], val
[7];
3726 struct fw_caps_config_cmd caps_cmd
;
3730 * Contact FW, advertising Master capability (and potentially forcing
3731 * ourselves as the Master PF if our module parameter force_init is
3734 ret
= t4_fw_hello(adap
, adap
->mbox
, adap
->fn
,
3735 force_init
? MASTER_MUST
: MASTER_MAY
,
3738 dev_err(adap
->pdev_dev
, "could not connect to FW, error %d\n",
3742 if (ret
== adap
->mbox
)
3743 adap
->flags
|= MASTER_PF
;
3744 if (force_init
&& state
== DEV_STATE_INIT
)
3745 state
= DEV_STATE_UNINIT
;
3748 * If we're the Master PF Driver and the device is uninitialized,
3749 * then let's consider upgrading the firmware ... (We always want
3750 * to check the firmware version number in order to A. get it for
3751 * later reporting and B. to warn if the currently loaded firmware
3752 * is excessively mismatched relative to the driver.)
3754 ret
= t4_check_fw_version(adap
);
3755 if ((adap
->flags
& MASTER_PF
) && state
!= DEV_STATE_INIT
) {
3756 if (ret
== -EINVAL
|| ret
> 0) {
3757 if (upgrade_fw(adap
) >= 0) {
3759 * Note that the chip was reset as part of the
3760 * firmware upgrade so we don't reset it again
3761 * below and grab the new firmware version.
3764 ret
= t4_check_fw_version(adap
);
3772 * Grab VPD parameters. This should be done after we establish a
3773 * connection to the firmware since some of the VPD parameters
3774 * (notably the Core Clock frequency) are retrieved via requests to
3775 * the firmware. On the other hand, we need these fairly early on
3776 * so we do this right after getting ahold of the firmware.
3778 ret
= get_vpd_params(adap
, &adap
->params
.vpd
);
3783 * Find out what ports are available to us. Note that we need to do
3784 * this before calling adap_init0_no_config() since it needs nports
3788 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
3789 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC
);
3790 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1, &v
, &port_vec
);
3794 adap
->params
.nports
= hweight32(port_vec
);
3795 adap
->params
.portvec
= port_vec
;
3798 * If the firmware is initialized already (and we're not forcing a
3799 * master initialization), note that we're living with existing
3800 * adapter parameters. Otherwise, it's time to try initializing the
3803 if (state
== DEV_STATE_INIT
) {
3804 dev_info(adap
->pdev_dev
, "Coming up as %s: "\
3805 "Adapter already initialized\n",
3806 adap
->flags
& MASTER_PF
? "MASTER" : "SLAVE");
3807 adap
->flags
|= USING_SOFT_PARAMS
;
3809 dev_info(adap
->pdev_dev
, "Coming up as MASTER: "\
3810 "Initializing adapter\n");
3813 * If the firmware doesn't support Configuration
3814 * Files warn user and exit,
3817 dev_warn(adap
->pdev_dev
, "Firmware doesn't support "
3818 "configuration file.\n");
3820 ret
= adap_init0_no_config(adap
, reset
);
3823 * Find out whether we're dealing with a version of
3824 * the firmware which has configuration file support.
3826 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
3827 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
3828 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1,
3832 * If the firmware doesn't support Configuration
3833 * Files, use the old Driver-based, hard-wired
3834 * initialization. Otherwise, try using the
3835 * Configuration File support and fall back to the
3836 * Driver-based initialization if there's no
3837 * Configuration File found.
3840 ret
= adap_init0_no_config(adap
, reset
);
3843 * The firmware provides us with a memory
3844 * buffer where we can load a Configuration
3845 * File from the host if we want to override
3846 * the Configuration File in flash.
3849 ret
= adap_init0_config(adap
, reset
);
3850 if (ret
== -ENOENT
) {
3851 dev_info(adap
->pdev_dev
,
3852 "No Configuration File present "
3853 "on adapter. Using hard-wired "
3854 "configuration parameters.\n");
3855 ret
= adap_init0_no_config(adap
, reset
);
3860 dev_err(adap
->pdev_dev
,
3861 "could not initialize adapter, error %d\n",
3868 * If we're living with non-hard-coded parameters (either from a
3869 * Firmware Configuration File or values programmed by a different PF
3870 * Driver), give the SGE code a chance to pull in anything that it
3871 * needs ... Note that this must be called after we retrieve our VPD
3872 * parameters in order to know how to convert core ticks to seconds.
3874 if (adap
->flags
& USING_SOFT_PARAMS
) {
3875 ret
= t4_sge_init(adap
);
3880 if (is_bypass_device(adap
->pdev
->device
))
3881 adap
->params
.bypass
= 1;
3884 * Grab some of our basic fundamental operating parameters.
3886 #define FW_PARAM_DEV(param) \
3887 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3888 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3890 #define FW_PARAM_PFVF(param) \
3891 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3892 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
3893 FW_PARAMS_PARAM_Y(0) | \
3894 FW_PARAMS_PARAM_Z(0)
3896 params
[0] = FW_PARAM_PFVF(EQ_START
);
3897 params
[1] = FW_PARAM_PFVF(L2T_START
);
3898 params
[2] = FW_PARAM_PFVF(L2T_END
);
3899 params
[3] = FW_PARAM_PFVF(FILTER_START
);
3900 params
[4] = FW_PARAM_PFVF(FILTER_END
);
3901 params
[5] = FW_PARAM_PFVF(IQFLINT_START
);
3902 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6, params
, val
);
3905 adap
->sge
.egr_start
= val
[0];
3906 adap
->l2t_start
= val
[1];
3907 adap
->l2t_end
= val
[2];
3908 adap
->tids
.ftid_base
= val
[3];
3909 adap
->tids
.nftids
= val
[4] - val
[3] + 1;
3910 adap
->sge
.ingr_start
= val
[5];
3912 /* query params related to active filter region */
3913 params
[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START
);
3914 params
[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END
);
3915 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2, params
, val
);
3916 /* If Active filter size is set we enable establishing
3917 * offload connection through firmware work request
3919 if ((val
[0] != val
[1]) && (ret
>= 0)) {
3920 adap
->flags
|= FW_OFLD_CONN
;
3921 adap
->tids
.aftid_base
= val
[0];
3922 adap
->tids
.aftid_end
= val
[1];
3926 * Get device capabilities so we can determine what resources we need
3929 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
3930 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
3931 FW_CMD_REQUEST
| FW_CMD_READ
);
3932 caps_cmd
.retval_len16
= htonl(FW_LEN16(caps_cmd
));
3933 ret
= t4_wr_mbox(adap
, adap
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
3938 if (caps_cmd
.ofldcaps
) {
3939 /* query offload-related parameters */
3940 params
[0] = FW_PARAM_DEV(NTID
);
3941 params
[1] = FW_PARAM_PFVF(SERVER_START
);
3942 params
[2] = FW_PARAM_PFVF(SERVER_END
);
3943 params
[3] = FW_PARAM_PFVF(TDDP_START
);
3944 params
[4] = FW_PARAM_PFVF(TDDP_END
);
3945 params
[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ
);
3946 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
3950 adap
->tids
.ntids
= val
[0];
3951 adap
->tids
.natids
= min(adap
->tids
.ntids
/ 2, MAX_ATIDS
);
3952 adap
->tids
.stid_base
= val
[1];
3953 adap
->tids
.nstids
= val
[2] - val
[1] + 1;
3955 * Setup server filter region. Divide the availble filter
3956 * region into two parts. Regular filters get 1/3rd and server
3957 * filters get 2/3rd part. This is only enabled if workarond
3959 * 1. For regular filters.
3960 * 2. Server filter: This are special filters which are used
3961 * to redirect SYN packets to offload queue.
3963 if (adap
->flags
& FW_OFLD_CONN
&& !is_bypass(adap
)) {
3964 adap
->tids
.sftid_base
= adap
->tids
.ftid_base
+
3965 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
3966 adap
->tids
.nsftids
= adap
->tids
.nftids
-
3967 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
3968 adap
->tids
.nftids
= adap
->tids
.sftid_base
-
3969 adap
->tids
.ftid_base
;
3971 adap
->vres
.ddp
.start
= val
[3];
3972 adap
->vres
.ddp
.size
= val
[4] - val
[3] + 1;
3973 adap
->params
.ofldq_wr_cred
= val
[5];
3975 adap
->params
.offload
= 1;
3977 if (caps_cmd
.rdmacaps
) {
3978 params
[0] = FW_PARAM_PFVF(STAG_START
);
3979 params
[1] = FW_PARAM_PFVF(STAG_END
);
3980 params
[2] = FW_PARAM_PFVF(RQ_START
);
3981 params
[3] = FW_PARAM_PFVF(RQ_END
);
3982 params
[4] = FW_PARAM_PFVF(PBL_START
);
3983 params
[5] = FW_PARAM_PFVF(PBL_END
);
3984 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
3988 adap
->vres
.stag
.start
= val
[0];
3989 adap
->vres
.stag
.size
= val
[1] - val
[0] + 1;
3990 adap
->vres
.rq
.start
= val
[2];
3991 adap
->vres
.rq
.size
= val
[3] - val
[2] + 1;
3992 adap
->vres
.pbl
.start
= val
[4];
3993 adap
->vres
.pbl
.size
= val
[5] - val
[4] + 1;
3995 params
[0] = FW_PARAM_PFVF(SQRQ_START
);
3996 params
[1] = FW_PARAM_PFVF(SQRQ_END
);
3997 params
[2] = FW_PARAM_PFVF(CQ_START
);
3998 params
[3] = FW_PARAM_PFVF(CQ_END
);
3999 params
[4] = FW_PARAM_PFVF(OCQ_START
);
4000 params
[5] = FW_PARAM_PFVF(OCQ_END
);
4001 ret
= t4_query_params(adap
, 0, 0, 0, 6, params
, val
);
4004 adap
->vres
.qp
.start
= val
[0];
4005 adap
->vres
.qp
.size
= val
[1] - val
[0] + 1;
4006 adap
->vres
.cq
.start
= val
[2];
4007 adap
->vres
.cq
.size
= val
[3] - val
[2] + 1;
4008 adap
->vres
.ocq
.start
= val
[4];
4009 adap
->vres
.ocq
.size
= val
[5] - val
[4] + 1;
4011 if (caps_cmd
.iscsicaps
) {
4012 params
[0] = FW_PARAM_PFVF(ISCSI_START
);
4013 params
[1] = FW_PARAM_PFVF(ISCSI_END
);
4014 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2,
4018 adap
->vres
.iscsi
.start
= val
[0];
4019 adap
->vres
.iscsi
.size
= val
[1] - val
[0] + 1;
4021 #undef FW_PARAM_PFVF
4025 * These are finalized by FW initialization, load their values now.
4027 v
= t4_read_reg(adap
, TP_TIMER_RESOLUTION
);
4028 adap
->params
.tp
.tre
= TIMERRESOLUTION_GET(v
);
4029 adap
->params
.tp
.dack_re
= DELAYEDACKRESOLUTION_GET(v
);
4030 t4_read_mtu_tbl(adap
, adap
->params
.mtus
, NULL
);
4031 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
4032 adap
->params
.b_wnd
);
4034 /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
4035 for (j
= 0; j
< NCHAN
; j
++)
4036 adap
->params
.tp
.tx_modq
[j
] = j
;
4038 adap
->flags
|= FW_OK
;
4042 * Something bad happened. If a command timed out or failed with EIO
4043 * FW does not operate within its spec or something catastrophic
4044 * happened to HW/FW, stop issuing commands.
4047 if (ret
!= -ETIMEDOUT
&& ret
!= -EIO
)
4048 t4_fw_bye(adap
, adap
->mbox
);
4054 static pci_ers_result_t
eeh_err_detected(struct pci_dev
*pdev
,
4055 pci_channel_state_t state
)
4058 struct adapter
*adap
= pci_get_drvdata(pdev
);
4064 adap
->flags
&= ~FW_OK
;
4065 notify_ulds(adap
, CXGB4_STATE_START_RECOVERY
);
4066 for_each_port(adap
, i
) {
4067 struct net_device
*dev
= adap
->port
[i
];
4069 netif_device_detach(dev
);
4070 netif_carrier_off(dev
);
4072 if (adap
->flags
& FULL_INIT_DONE
)
4075 pci_disable_device(pdev
);
4076 out
: return state
== pci_channel_io_perm_failure
?
4077 PCI_ERS_RESULT_DISCONNECT
: PCI_ERS_RESULT_NEED_RESET
;
4080 static pci_ers_result_t
eeh_slot_reset(struct pci_dev
*pdev
)
4083 struct fw_caps_config_cmd c
;
4084 struct adapter
*adap
= pci_get_drvdata(pdev
);
4087 pci_restore_state(pdev
);
4088 pci_save_state(pdev
);
4089 return PCI_ERS_RESULT_RECOVERED
;
4092 if (pci_enable_device(pdev
)) {
4093 dev_err(&pdev
->dev
, "cannot reenable PCI device after reset\n");
4094 return PCI_ERS_RESULT_DISCONNECT
;
4097 pci_set_master(pdev
);
4098 pci_restore_state(pdev
);
4099 pci_save_state(pdev
);
4100 pci_cleanup_aer_uncorrect_error_status(pdev
);
4102 if (t4_wait_dev_ready(adap
) < 0)
4103 return PCI_ERS_RESULT_DISCONNECT
;
4104 if (t4_fw_hello(adap
, adap
->fn
, adap
->fn
, MASTER_MUST
, NULL
))
4105 return PCI_ERS_RESULT_DISCONNECT
;
4106 adap
->flags
|= FW_OK
;
4107 if (adap_init1(adap
, &c
))
4108 return PCI_ERS_RESULT_DISCONNECT
;
4110 for_each_port(adap
, i
) {
4111 struct port_info
*p
= adap2pinfo(adap
, i
);
4113 ret
= t4_alloc_vi(adap
, adap
->fn
, p
->tx_chan
, adap
->fn
, 0, 1,
4116 return PCI_ERS_RESULT_DISCONNECT
;
4118 p
->xact_addr_filt
= -1;
4121 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
4122 adap
->params
.b_wnd
);
4125 return PCI_ERS_RESULT_DISCONNECT
;
4126 return PCI_ERS_RESULT_RECOVERED
;
4129 static void eeh_resume(struct pci_dev
*pdev
)
4132 struct adapter
*adap
= pci_get_drvdata(pdev
);
4138 for_each_port(adap
, i
) {
4139 struct net_device
*dev
= adap
->port
[i
];
4141 if (netif_running(dev
)) {
4143 cxgb_set_rxmode(dev
);
4145 netif_device_attach(dev
);
4150 static const struct pci_error_handlers cxgb4_eeh
= {
4151 .error_detected
= eeh_err_detected
,
4152 .slot_reset
= eeh_slot_reset
,
4153 .resume
= eeh_resume
,
4156 static inline bool is_10g_port(const struct link_config
*lc
)
4158 return (lc
->supported
& FW_PORT_CAP_SPEED_10G
) != 0;
4161 static inline void init_rspq(struct sge_rspq
*q
, u8 timer_idx
, u8 pkt_cnt_idx
,
4162 unsigned int size
, unsigned int iqe_size
)
4164 q
->intr_params
= QINTR_TIMER_IDX(timer_idx
) |
4165 (pkt_cnt_idx
< SGE_NCOUNTERS
? QINTR_CNT_EN
: 0);
4166 q
->pktcnt_idx
= pkt_cnt_idx
< SGE_NCOUNTERS
? pkt_cnt_idx
: 0;
4167 q
->iqe_len
= iqe_size
;
4172 * Perform default configuration of DMA queues depending on the number and type
4173 * of ports we found and the number of available CPUs. Most settings can be
4174 * modified by the admin prior to actual use.
4176 static void __devinit
cfg_queues(struct adapter
*adap
)
4178 struct sge
*s
= &adap
->sge
;
4179 int i
, q10g
= 0, n10g
= 0, qidx
= 0;
4181 for_each_port(adap
, i
)
4182 n10g
+= is_10g_port(&adap2pinfo(adap
, i
)->link_cfg
);
4185 * We default to 1 queue per non-10G port and up to # of cores queues
4189 q10g
= (MAX_ETH_QSETS
- (adap
->params
.nports
- n10g
)) / n10g
;
4190 if (q10g
> netif_get_num_default_rss_queues())
4191 q10g
= netif_get_num_default_rss_queues();
4193 for_each_port(adap
, i
) {
4194 struct port_info
*pi
= adap2pinfo(adap
, i
);
4196 pi
->first_qset
= qidx
;
4197 pi
->nqsets
= is_10g_port(&pi
->link_cfg
) ? q10g
: 1;
4202 s
->max_ethqsets
= qidx
; /* MSI-X may lower it later */
4204 if (is_offload(adap
)) {
4206 * For offload we use 1 queue/channel if all ports are up to 1G,
4207 * otherwise we divide all available queues amongst the channels
4208 * capped by the number of available cores.
4211 i
= min_t(int, ARRAY_SIZE(s
->ofldrxq
),
4213 s
->ofldqsets
= roundup(i
, adap
->params
.nports
);
4215 s
->ofldqsets
= adap
->params
.nports
;
4216 /* For RDMA one Rx queue per channel suffices */
4217 s
->rdmaqs
= adap
->params
.nports
;
4220 for (i
= 0; i
< ARRAY_SIZE(s
->ethrxq
); i
++) {
4221 struct sge_eth_rxq
*r
= &s
->ethrxq
[i
];
4223 init_rspq(&r
->rspq
, 0, 0, 1024, 64);
4227 for (i
= 0; i
< ARRAY_SIZE(s
->ethtxq
); i
++)
4228 s
->ethtxq
[i
].q
.size
= 1024;
4230 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++)
4231 s
->ctrlq
[i
].q
.size
= 512;
4233 for (i
= 0; i
< ARRAY_SIZE(s
->ofldtxq
); i
++)
4234 s
->ofldtxq
[i
].q
.size
= 1024;
4236 for (i
= 0; i
< ARRAY_SIZE(s
->ofldrxq
); i
++) {
4237 struct sge_ofld_rxq
*r
= &s
->ofldrxq
[i
];
4239 init_rspq(&r
->rspq
, 0, 0, 1024, 64);
4240 r
->rspq
.uld
= CXGB4_ULD_ISCSI
;
4244 for (i
= 0; i
< ARRAY_SIZE(s
->rdmarxq
); i
++) {
4245 struct sge_ofld_rxq
*r
= &s
->rdmarxq
[i
];
4247 init_rspq(&r
->rspq
, 0, 0, 511, 64);
4248 r
->rspq
.uld
= CXGB4_ULD_RDMA
;
4252 init_rspq(&s
->fw_evtq
, 6, 0, 512, 64);
4253 init_rspq(&s
->intrq
, 6, 0, 2 * MAX_INGQ
, 64);
4257 * Reduce the number of Ethernet queues across all ports to at most n.
4258 * n provides at least one queue per port.
4260 static void __devinit
reduce_ethqs(struct adapter
*adap
, int n
)
4263 struct port_info
*pi
;
4265 while (n
< adap
->sge
.ethqsets
)
4266 for_each_port(adap
, i
) {
4267 pi
= adap2pinfo(adap
, i
);
4268 if (pi
->nqsets
> 1) {
4270 adap
->sge
.ethqsets
--;
4271 if (adap
->sge
.ethqsets
<= n
)
4277 for_each_port(adap
, i
) {
4278 pi
= adap2pinfo(adap
, i
);
4284 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
4285 #define EXTRA_VECS 2
4287 static int __devinit
enable_msix(struct adapter
*adap
)
4290 int i
, err
, want
, need
;
4291 struct sge
*s
= &adap
->sge
;
4292 unsigned int nchan
= adap
->params
.nports
;
4293 struct msix_entry entries
[MAX_INGQ
+ 1];
4295 for (i
= 0; i
< ARRAY_SIZE(entries
); ++i
)
4296 entries
[i
].entry
= i
;
4298 want
= s
->max_ethqsets
+ EXTRA_VECS
;
4299 if (is_offload(adap
)) {
4300 want
+= s
->rdmaqs
+ s
->ofldqsets
;
4301 /* need nchan for each possible ULD */
4302 ofld_need
= 2 * nchan
;
4304 need
= adap
->params
.nports
+ EXTRA_VECS
+ ofld_need
;
4306 while ((err
= pci_enable_msix(adap
->pdev
, entries
, want
)) >= need
)
4311 * Distribute available vectors to the various queue groups.
4312 * Every group gets its minimum requirement and NIC gets top
4313 * priority for leftovers.
4315 i
= want
- EXTRA_VECS
- ofld_need
;
4316 if (i
< s
->max_ethqsets
) {
4317 s
->max_ethqsets
= i
;
4318 if (i
< s
->ethqsets
)
4319 reduce_ethqs(adap
, i
);
4321 if (is_offload(adap
)) {
4322 i
= want
- EXTRA_VECS
- s
->max_ethqsets
;
4323 i
-= ofld_need
- nchan
;
4324 s
->ofldqsets
= (i
/ nchan
) * nchan
; /* round down */
4326 for (i
= 0; i
< want
; ++i
)
4327 adap
->msix_info
[i
].vec
= entries
[i
].vector
;
4329 dev_info(adap
->pdev_dev
,
4330 "only %d MSI-X vectors left, not using MSI-X\n", err
);
4336 static int __devinit
init_rss(struct adapter
*adap
)
4340 for_each_port(adap
, i
) {
4341 struct port_info
*pi
= adap2pinfo(adap
, i
);
4343 pi
->rss
= kcalloc(pi
->rss_size
, sizeof(u16
), GFP_KERNEL
);
4346 for (j
= 0; j
< pi
->rss_size
; j
++)
4347 pi
->rss
[j
] = ethtool_rxfh_indir_default(j
, pi
->nqsets
);
4352 static void __devinit
print_port_info(const struct net_device
*dev
)
4354 static const char *base
[] = {
4355 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
4356 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
4361 const char *spd
= "";
4362 const struct port_info
*pi
= netdev_priv(dev
);
4363 const struct adapter
*adap
= pi
->adapter
;
4365 if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_2_5GB
)
4367 else if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_5_0GB
)
4370 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_100M
)
4371 bufp
+= sprintf(bufp
, "100/");
4372 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_1G
)
4373 bufp
+= sprintf(bufp
, "1000/");
4374 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_10G
)
4375 bufp
+= sprintf(bufp
, "10G/");
4378 sprintf(bufp
, "BASE-%s", base
[pi
->port_type
]);
4380 netdev_info(dev
, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
4381 adap
->params
.vpd
.id
, adap
->params
.rev
, buf
,
4382 is_offload(adap
) ? "R" : "", adap
->params
.pci
.width
, spd
,
4383 (adap
->flags
& USING_MSIX
) ? " MSI-X" :
4384 (adap
->flags
& USING_MSI
) ? " MSI" : "");
4385 netdev_info(dev
, "S/N: %s, E/C: %s\n",
4386 adap
->params
.vpd
.sn
, adap
->params
.vpd
.ec
);
4389 static void __devinit
enable_pcie_relaxed_ordering(struct pci_dev
*dev
)
4391 pcie_capability_set_word(dev
, PCI_EXP_DEVCTL
, PCI_EXP_DEVCTL_RELAX_EN
);
4395 * Free the following resources:
4396 * - memory used for tables
4399 * - resources FW is holding for us
4401 static void free_some_resources(struct adapter
*adapter
)
4405 t4_free_mem(adapter
->l2t
);
4406 t4_free_mem(adapter
->tids
.tid_tab
);
4407 disable_msi(adapter
);
4409 for_each_port(adapter
, i
)
4410 if (adapter
->port
[i
]) {
4411 kfree(adap2pinfo(adapter
, i
)->rss
);
4412 free_netdev(adapter
->port
[i
]);
4414 if (adapter
->flags
& FW_OK
)
4415 t4_fw_bye(adapter
, adapter
->fn
);
4418 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
4419 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
4420 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
4422 static int __devinit
init_one(struct pci_dev
*pdev
,
4423 const struct pci_device_id
*ent
)
4426 struct port_info
*pi
;
4427 bool highdma
= false;
4428 struct adapter
*adapter
= NULL
;
4430 printk_once(KERN_INFO
"%s - version %s\n", DRV_DESC
, DRV_VERSION
);
4432 err
= pci_request_regions(pdev
, KBUILD_MODNAME
);
4434 /* Just info, some other driver may have claimed the device. */
4435 dev_info(&pdev
->dev
, "cannot obtain PCI resources\n");
4439 /* We control everything through one PF */
4440 func
= PCI_FUNC(pdev
->devfn
);
4441 if (func
!= ent
->driver_data
) {
4442 pci_save_state(pdev
); /* to restore SR-IOV later */
4446 err
= pci_enable_device(pdev
);
4448 dev_err(&pdev
->dev
, "cannot enable PCI device\n");
4449 goto out_release_regions
;
4452 if (!pci_set_dma_mask(pdev
, DMA_BIT_MASK(64))) {
4454 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
4456 dev_err(&pdev
->dev
, "unable to obtain 64-bit DMA for "
4457 "coherent allocations\n");
4458 goto out_disable_device
;
4461 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
4463 dev_err(&pdev
->dev
, "no usable DMA configuration\n");
4464 goto out_disable_device
;
4468 pci_enable_pcie_error_reporting(pdev
);
4469 enable_pcie_relaxed_ordering(pdev
);
4470 pci_set_master(pdev
);
4471 pci_save_state(pdev
);
4473 adapter
= kzalloc(sizeof(*adapter
), GFP_KERNEL
);
4476 goto out_disable_device
;
4479 adapter
->regs
= pci_ioremap_bar(pdev
, 0);
4480 if (!adapter
->regs
) {
4481 dev_err(&pdev
->dev
, "cannot map device registers\n");
4483 goto out_free_adapter
;
4486 adapter
->pdev
= pdev
;
4487 adapter
->pdev_dev
= &pdev
->dev
;
4488 adapter
->mbox
= func
;
4490 adapter
->msg_enable
= dflt_msg_enable
;
4491 memset(adapter
->chan_map
, 0xff, sizeof(adapter
->chan_map
));
4493 spin_lock_init(&adapter
->stats_lock
);
4494 spin_lock_init(&adapter
->tid_release_lock
);
4496 INIT_WORK(&adapter
->tid_release_task
, process_tid_release_list
);
4497 INIT_WORK(&adapter
->db_full_task
, process_db_full
);
4498 INIT_WORK(&adapter
->db_drop_task
, process_db_drop
);
4500 err
= t4_prep_adapter(adapter
);
4503 setup_memwin(adapter
);
4504 err
= adap_init0(adapter
);
4505 setup_memwin_rdma(adapter
);
4509 for_each_port(adapter
, i
) {
4510 struct net_device
*netdev
;
4512 netdev
= alloc_etherdev_mq(sizeof(struct port_info
),
4519 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
4521 adapter
->port
[i
] = netdev
;
4522 pi
= netdev_priv(netdev
);
4523 pi
->adapter
= adapter
;
4524 pi
->xact_addr_filt
= -1;
4526 netdev
->irq
= pdev
->irq
;
4528 netdev
->hw_features
= NETIF_F_SG
| TSO_FLAGS
|
4529 NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
4530 NETIF_F_RXCSUM
| NETIF_F_RXHASH
|
4531 NETIF_F_HW_VLAN_TX
| NETIF_F_HW_VLAN_RX
;
4533 netdev
->hw_features
|= NETIF_F_HIGHDMA
;
4534 netdev
->features
|= netdev
->hw_features
;
4535 netdev
->vlan_features
= netdev
->features
& VLAN_FEAT
;
4537 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
4539 netdev
->netdev_ops
= &cxgb4_netdev_ops
;
4540 SET_ETHTOOL_OPS(netdev
, &cxgb_ethtool_ops
);
4543 pci_set_drvdata(pdev
, adapter
);
4545 if (adapter
->flags
& FW_OK
) {
4546 err
= t4_port_init(adapter
, func
, func
, 0);
4552 * Configure queues and allocate tables now, they can be needed as
4553 * soon as the first register_netdev completes.
4555 cfg_queues(adapter
);
4557 adapter
->l2t
= t4_init_l2t();
4558 if (!adapter
->l2t
) {
4559 /* We tolerate a lack of L2T, giving up some functionality */
4560 dev_warn(&pdev
->dev
, "could not allocate L2T, continuing\n");
4561 adapter
->params
.offload
= 0;
4564 if (is_offload(adapter
) && tid_init(&adapter
->tids
) < 0) {
4565 dev_warn(&pdev
->dev
, "could not allocate TID table, "
4567 adapter
->params
.offload
= 0;
4570 /* See what interrupts we'll be using */
4571 if (msi
> 1 && enable_msix(adapter
) == 0)
4572 adapter
->flags
|= USING_MSIX
;
4573 else if (msi
> 0 && pci_enable_msi(pdev
) == 0)
4574 adapter
->flags
|= USING_MSI
;
4576 err
= init_rss(adapter
);
4581 * The card is now ready to go. If any errors occur during device
4582 * registration we do not fail the whole card but rather proceed only
4583 * with the ports we manage to register successfully. However we must
4584 * register at least one net device.
4586 for_each_port(adapter
, i
) {
4587 pi
= adap2pinfo(adapter
, i
);
4588 netif_set_real_num_tx_queues(adapter
->port
[i
], pi
->nqsets
);
4589 netif_set_real_num_rx_queues(adapter
->port
[i
], pi
->nqsets
);
4591 err
= register_netdev(adapter
->port
[i
]);
4594 adapter
->chan_map
[pi
->tx_chan
] = i
;
4595 print_port_info(adapter
->port
[i
]);
4598 dev_err(&pdev
->dev
, "could not register any net devices\n");
4602 dev_warn(&pdev
->dev
, "only %d net devices registered\n", i
);
4606 if (cxgb4_debugfs_root
) {
4607 adapter
->debugfs_root
= debugfs_create_dir(pci_name(pdev
),
4608 cxgb4_debugfs_root
);
4609 setup_debugfs(adapter
);
4612 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
4613 pdev
->needs_freset
= 1;
4615 if (is_offload(adapter
))
4616 attach_ulds(adapter
);
4619 #ifdef CONFIG_PCI_IOV
4620 if (func
< ARRAY_SIZE(num_vf
) && num_vf
[func
] > 0)
4621 if (pci_enable_sriov(pdev
, num_vf
[func
]) == 0)
4622 dev_info(&pdev
->dev
,
4623 "instantiated %u virtual functions\n",
4629 free_some_resources(adapter
);
4631 iounmap(adapter
->regs
);
4635 pci_disable_pcie_error_reporting(pdev
);
4636 pci_disable_device(pdev
);
4637 out_release_regions
:
4638 pci_release_regions(pdev
);
4639 pci_set_drvdata(pdev
, NULL
);
4643 static void __devexit
remove_one(struct pci_dev
*pdev
)
4645 struct adapter
*adapter
= pci_get_drvdata(pdev
);
4647 #ifdef CONFIG_PCI_IOV
4648 pci_disable_sriov(pdev
);
4655 if (is_offload(adapter
))
4656 detach_ulds(adapter
);
4658 for_each_port(adapter
, i
)
4659 if (adapter
->port
[i
]->reg_state
== NETREG_REGISTERED
)
4660 unregister_netdev(adapter
->port
[i
]);
4662 if (adapter
->debugfs_root
)
4663 debugfs_remove_recursive(adapter
->debugfs_root
);
4665 if (adapter
->flags
& FULL_INIT_DONE
)
4668 free_some_resources(adapter
);
4669 iounmap(adapter
->regs
);
4671 pci_disable_pcie_error_reporting(pdev
);
4672 pci_disable_device(pdev
);
4673 pci_release_regions(pdev
);
4674 pci_set_drvdata(pdev
, NULL
);
4676 pci_release_regions(pdev
);
4679 static struct pci_driver cxgb4_driver
= {
4680 .name
= KBUILD_MODNAME
,
4681 .id_table
= cxgb4_pci_tbl
,
4683 .remove
= __devexit_p(remove_one
),
4684 .err_handler
= &cxgb4_eeh
,
4687 static int __init
cxgb4_init_module(void)
4691 workq
= create_singlethread_workqueue("cxgb4");
4695 /* Debugfs support is optional, just warn if this fails */
4696 cxgb4_debugfs_root
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
4697 if (!cxgb4_debugfs_root
)
4698 pr_warning("could not create debugfs entry, continuing\n");
4700 ret
= pci_register_driver(&cxgb4_driver
);
4702 debugfs_remove(cxgb4_debugfs_root
);
4706 static void __exit
cxgb4_cleanup_module(void)
4708 pci_unregister_driver(&cxgb4_driver
);
4709 debugfs_remove(cxgb4_debugfs_root
); /* NULL ok */
4710 flush_workqueue(workq
);
4711 destroy_workqueue(workq
);
4714 module_init(cxgb4_init_module
);
4715 module_exit(cxgb4_cleanup_module
);