2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <net/addrconf.h>
64 #include <asm/uaccess.h>
72 #include <../drivers/net/bonding/bonding.h>
77 #define DRV_VERSION "2.0.0-ko"
78 #define DRV_DESC "Chelsio T4/T5 Network Driver"
81 * Max interrupt hold-off timer value in us. Queues fall back to this value
82 * under extreme memory pressure so it's largish to give the system time to
85 #define MAX_SGE_TIMERVAL 200U
89 * Physical Function provisioning constants.
91 PFRES_NVI
= 4, /* # of Virtual Interfaces */
92 PFRES_NETHCTRL
= 128, /* # of EQs used for ETH or CTRL Qs */
93 PFRES_NIQFLINT
= 128, /* # of ingress Qs/w Free List(s)/intr
95 PFRES_NEQ
= 256, /* # of egress queues */
96 PFRES_NIQ
= 0, /* # of ingress queues */
97 PFRES_TC
= 0, /* PCI-E traffic class */
98 PFRES_NEXACTF
= 128, /* # of exact MPS filters */
100 PFRES_R_CAPS
= FW_CMD_CAP_PF
,
101 PFRES_WX_CAPS
= FW_CMD_CAP_PF
,
103 #ifdef CONFIG_PCI_IOV
105 * Virtual Function provisioning constants. We need two extra Ingress
106 * Queues with Interrupt capability to serve as the VF's Firmware
107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
108 * neither will have Free Lists associated with them). For each
109 * Ethernet/Control Egress Queue and for each Free List, we need an
112 VFRES_NPORTS
= 1, /* # of "ports" per VF */
113 VFRES_NQSETS
= 2, /* # of "Queue Sets" per VF */
115 VFRES_NVI
= VFRES_NPORTS
, /* # of Virtual Interfaces */
116 VFRES_NETHCTRL
= VFRES_NQSETS
, /* # of EQs used for ETH or CTRL Qs */
117 VFRES_NIQFLINT
= VFRES_NQSETS
+2,/* # of ingress Qs/w Free List(s)/intr */
118 VFRES_NEQ
= VFRES_NQSETS
*2, /* # of egress queues */
119 VFRES_NIQ
= 0, /* # of non-fl/int ingress queues */
120 VFRES_TC
= 0, /* PCI-E traffic class */
121 VFRES_NEXACTF
= 16, /* # of exact MPS filters */
123 VFRES_R_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
|FW_CMD_CAP_PORT
,
124 VFRES_WX_CAPS
= FW_CMD_CAP_DMAQ
|FW_CMD_CAP_VF
,
129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
130 * static and likely not to be useful in the long run. We really need to
131 * implement some form of persistent configuration which the firmware
134 static unsigned int pfvfres_pmask(struct adapter
*adapter
,
135 unsigned int pf
, unsigned int vf
)
137 unsigned int portn
, portvec
;
140 * Give PF's access to all of the ports.
143 return FW_PFVF_CMD_PMASK_MASK
;
146 * For VFs, we'll assign them access to the ports based purely on the
147 * PF. We assign active ports in order, wrapping around if there are
148 * fewer active ports than PFs: e.g. active port[pf % nports].
149 * Unfortunately the adapter's port_info structs haven't been
150 * initialized yet so we have to compute this.
152 if (adapter
->params
.nports
== 0)
155 portn
= pf
% adapter
->params
.nports
;
156 portvec
= adapter
->params
.portvec
;
159 * Isolate the lowest set bit in the port vector. If we're at
160 * the port number that we want, return that as the pmask.
161 * otherwise mask that bit out of the port vector and
162 * decrement our port number ...
164 unsigned int pmask
= portvec
^ (portvec
& (portvec
-1));
174 MAX_TXQ_ENTRIES
= 16384,
175 MAX_CTRL_TXQ_ENTRIES
= 1024,
176 MAX_RSPQ_ENTRIES
= 16384,
177 MAX_RX_BUFFERS
= 16384,
178 MIN_TXQ_ENTRIES
= 32,
179 MIN_CTRL_TXQ_ENTRIES
= 32,
180 MIN_RSPQ_ENTRIES
= 128,
184 /* Host shadow copy of ingress filter entry. This is in host native format
185 * and doesn't match the ordering or bit order, etc. of the hardware of the
186 * firmware command. The use of bit-field structure elements is purely to
187 * remind ourselves of the field size limitations and save memory in the case
188 * where the filter table is large.
190 struct filter_entry
{
191 /* Administrative fields for filter.
193 u32 valid
:1; /* filter allocated and valid */
194 u32 locked
:1; /* filter is administratively locked */
196 u32 pending
:1; /* filter action is pending firmware reply */
197 u32 smtidx
:8; /* Source MAC Table index for smac */
198 struct l2t_entry
*l2t
; /* Layer Two Table entry for dmac */
200 /* The filter itself. Most of this is a straight copy of information
201 * provided by the extended ioctl(). Some fields are translated to
202 * internal forms -- for instance the Ingress Queue ID passed in from
203 * the ioctl() is translated into the Absolute Ingress Queue ID.
205 struct ch_filter_specification fs
;
208 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
212 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
214 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl
) = {
215 CH_DEVICE(0xa000, 0), /* PE10K */
216 CH_DEVICE(0x4001, -1),
217 CH_DEVICE(0x4002, -1),
218 CH_DEVICE(0x4003, -1),
219 CH_DEVICE(0x4004, -1),
220 CH_DEVICE(0x4005, -1),
221 CH_DEVICE(0x4006, -1),
222 CH_DEVICE(0x4007, -1),
223 CH_DEVICE(0x4008, -1),
224 CH_DEVICE(0x4009, -1),
225 CH_DEVICE(0x400a, -1),
226 CH_DEVICE(0x4401, 4),
227 CH_DEVICE(0x4402, 4),
228 CH_DEVICE(0x4403, 4),
229 CH_DEVICE(0x4404, 4),
230 CH_DEVICE(0x4405, 4),
231 CH_DEVICE(0x4406, 4),
232 CH_DEVICE(0x4407, 4),
233 CH_DEVICE(0x4408, 4),
234 CH_DEVICE(0x4409, 4),
235 CH_DEVICE(0x440a, 4),
236 CH_DEVICE(0x440d, 4),
237 CH_DEVICE(0x440e, 4),
238 CH_DEVICE(0x5001, 4),
239 CH_DEVICE(0x5002, 4),
240 CH_DEVICE(0x5003, 4),
241 CH_DEVICE(0x5004, 4),
242 CH_DEVICE(0x5005, 4),
243 CH_DEVICE(0x5006, 4),
244 CH_DEVICE(0x5007, 4),
245 CH_DEVICE(0x5008, 4),
246 CH_DEVICE(0x5009, 4),
247 CH_DEVICE(0x500A, 4),
248 CH_DEVICE(0x500B, 4),
249 CH_DEVICE(0x500C, 4),
250 CH_DEVICE(0x500D, 4),
251 CH_DEVICE(0x500E, 4),
252 CH_DEVICE(0x500F, 4),
253 CH_DEVICE(0x5010, 4),
254 CH_DEVICE(0x5011, 4),
255 CH_DEVICE(0x5012, 4),
256 CH_DEVICE(0x5013, 4),
257 CH_DEVICE(0x5401, 4),
258 CH_DEVICE(0x5402, 4),
259 CH_DEVICE(0x5403, 4),
260 CH_DEVICE(0x5404, 4),
261 CH_DEVICE(0x5405, 4),
262 CH_DEVICE(0x5406, 4),
263 CH_DEVICE(0x5407, 4),
264 CH_DEVICE(0x5408, 4),
265 CH_DEVICE(0x5409, 4),
266 CH_DEVICE(0x540A, 4),
267 CH_DEVICE(0x540B, 4),
268 CH_DEVICE(0x540C, 4),
269 CH_DEVICE(0x540D, 4),
270 CH_DEVICE(0x540E, 4),
271 CH_DEVICE(0x540F, 4),
272 CH_DEVICE(0x5410, 4),
273 CH_DEVICE(0x5411, 4),
274 CH_DEVICE(0x5412, 4),
275 CH_DEVICE(0x5413, 4),
279 #define FW_FNAME "cxgb4/t4fw.bin"
280 #define FW5_FNAME "cxgb4/t5fw.bin"
281 #define FW_CFNAME "cxgb4/t4-config.txt"
282 #define FW5_CFNAME "cxgb4/t5-config.txt"
284 MODULE_DESCRIPTION(DRV_DESC
);
285 MODULE_AUTHOR("Chelsio Communications");
286 MODULE_LICENSE("Dual BSD/GPL");
287 MODULE_VERSION(DRV_VERSION
);
288 MODULE_DEVICE_TABLE(pci
, cxgb4_pci_tbl
);
289 MODULE_FIRMWARE(FW_FNAME
);
290 MODULE_FIRMWARE(FW5_FNAME
);
293 * Normally we're willing to become the firmware's Master PF but will be happy
294 * if another PF has already become the Master and initialized the adapter.
295 * Setting "force_init" will cause this driver to forcibly establish itself as
296 * the Master PF and initialize the adapter.
298 static uint force_init
;
300 module_param(force_init
, uint
, 0644);
301 MODULE_PARM_DESC(force_init
, "Forcibly become Master PF and initialize adapter");
304 * Normally if the firmware we connect to has Configuration File support, we
305 * use that and only fall back to the old Driver-based initialization if the
306 * Configuration File fails for some reason. If force_old_init is set, then
307 * we'll always use the old Driver-based initialization sequence.
309 static uint force_old_init
;
311 module_param(force_old_init
, uint
, 0644);
312 MODULE_PARM_DESC(force_old_init
, "Force old initialization sequence");
314 static int dflt_msg_enable
= DFLT_MSG_ENABLE
;
316 module_param(dflt_msg_enable
, int, 0644);
317 MODULE_PARM_DESC(dflt_msg_enable
, "Chelsio T4 default message enable bitmap");
320 * The driver uses the best interrupt scheme available on a platform in the
321 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
322 * of these schemes the driver may consider as follows:
324 * msi = 2: choose from among all three options
325 * msi = 1: only consider MSI and INTx interrupts
326 * msi = 0: force INTx interrupts
330 module_param(msi
, int, 0644);
331 MODULE_PARM_DESC(msi
, "whether to use INTx (0), MSI (1) or MSI-X (2)");
334 * Queue interrupt hold-off timer values. Queues default to the first of these
337 static unsigned int intr_holdoff
[SGE_NTIMERS
- 1] = { 5, 10, 20, 50, 100 };
339 module_param_array(intr_holdoff
, uint
, NULL
, 0644);
340 MODULE_PARM_DESC(intr_holdoff
, "values for queue interrupt hold-off timers "
341 "0..4 in microseconds");
343 static unsigned int intr_cnt
[SGE_NCOUNTERS
- 1] = { 4, 8, 16 };
345 module_param_array(intr_cnt
, uint
, NULL
, 0644);
346 MODULE_PARM_DESC(intr_cnt
,
347 "thresholds 1..3 for queue interrupt packet counters");
350 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
351 * offset by 2 bytes in order to have the IP headers line up on 4-byte
352 * boundaries. This is a requirement for many architectures which will throw
353 * a machine check fault if an attempt is made to access one of the 4-byte IP
354 * header fields on a non-4-byte boundary. And it's a major performance issue
355 * even on some architectures which allow it like some implementations of the
356 * x86 ISA. However, some architectures don't mind this and for some very
357 * edge-case performance sensitive applications (like forwarding large volumes
358 * of small packets), setting this DMA offset to 0 will decrease the number of
359 * PCI-E Bus transfers enough to measurably affect performance.
361 static int rx_dma_offset
= 2;
365 #ifdef CONFIG_PCI_IOV
366 module_param(vf_acls
, bool, 0644);
367 MODULE_PARM_DESC(vf_acls
, "if set enable virtualization L2 ACL enforcement");
369 /* Configure the number of PCI-E Virtual Function which are to be instantiated
370 * on SR-IOV Capable Physical Functions.
372 static unsigned int num_vf
[NUM_OF_PF_WITH_SRIOV
];
374 module_param_array(num_vf
, uint
, NULL
, 0644);
375 MODULE_PARM_DESC(num_vf
, "number of VFs for each of PFs 0-3");
379 * The filter TCAM has a fixed portion and a variable portion. The fixed
380 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
381 * ports. The variable portion is 36 bits which can include things like Exact
382 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
383 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
384 * far exceed the 36-bit budget for this "compressed" header portion of the
385 * filter. Thus, we have a scarce resource which must be carefully managed.
387 * By default we set this up to mostly match the set of filter matching
388 * capabilities of T3 but with accommodations for some of T4's more
389 * interesting features:
391 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
392 * [Inner] VLAN (17), Port (3), FCoE (1) }
395 TP_VLAN_PRI_MAP_DEFAULT
= HW_TPL_FR_MT_PR_IV_P_FC
,
396 TP_VLAN_PRI_MAP_FIRST
= FCOE_SHIFT
,
397 TP_VLAN_PRI_MAP_LAST
= FRAGMENTATION_SHIFT
,
400 static unsigned int tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
402 module_param(tp_vlan_pri_map
, uint
, 0644);
403 MODULE_PARM_DESC(tp_vlan_pri_map
, "global compressed filter configuration");
405 static struct dentry
*cxgb4_debugfs_root
;
407 static LIST_HEAD(adapter_list
);
408 static DEFINE_MUTEX(uld_mutex
);
409 /* Adapter list to be accessed from atomic context */
410 static LIST_HEAD(adap_rcu_list
);
411 static DEFINE_SPINLOCK(adap_rcu_lock
);
412 static struct cxgb4_uld_info ulds
[CXGB4_ULD_MAX
];
413 static const char *uld_str
[] = { "RDMA", "iSCSI" };
415 static void link_report(struct net_device
*dev
)
417 if (!netif_carrier_ok(dev
))
418 netdev_info(dev
, "link down\n");
420 static const char *fc
[] = { "no", "Rx", "Tx", "Tx/Rx" };
422 const char *s
= "10Mbps";
423 const struct port_info
*p
= netdev_priv(dev
);
425 switch (p
->link_cfg
.speed
) {
437 netdev_info(dev
, "link up, %s, full-duplex, %s PAUSE\n", s
,
442 void t4_os_link_changed(struct adapter
*adapter
, int port_id
, int link_stat
)
444 struct net_device
*dev
= adapter
->port
[port_id
];
446 /* Skip changes from disabled ports. */
447 if (netif_running(dev
) && link_stat
!= netif_carrier_ok(dev
)) {
449 netif_carrier_on(dev
);
451 netif_carrier_off(dev
);
457 void t4_os_portmod_changed(const struct adapter
*adap
, int port_id
)
459 static const char *mod_str
[] = {
460 NULL
, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
463 const struct net_device
*dev
= adap
->port
[port_id
];
464 const struct port_info
*pi
= netdev_priv(dev
);
466 if (pi
->mod_type
== FW_PORT_MOD_TYPE_NONE
)
467 netdev_info(dev
, "port module unplugged\n");
468 else if (pi
->mod_type
< ARRAY_SIZE(mod_str
))
469 netdev_info(dev
, "%s module inserted\n", mod_str
[pi
->mod_type
]);
473 * Configure the exact and hash address filters to handle a port's multicast
474 * and secondary unicast MAC addresses.
476 static int set_addr_filters(const struct net_device
*dev
, bool sleep
)
484 const struct netdev_hw_addr
*ha
;
485 int uc_cnt
= netdev_uc_count(dev
);
486 int mc_cnt
= netdev_mc_count(dev
);
487 const struct port_info
*pi
= netdev_priv(dev
);
488 unsigned int mb
= pi
->adapter
->fn
;
490 /* first do the secondary unicast addresses */
491 netdev_for_each_uc_addr(ha
, dev
) {
492 addr
[naddr
++] = ha
->addr
;
493 if (--uc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
494 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
495 naddr
, addr
, filt_idx
, &uhash
, sleep
);
504 /* next set up the multicast addresses */
505 netdev_for_each_mc_addr(ha
, dev
) {
506 addr
[naddr
++] = ha
->addr
;
507 if (--mc_cnt
== 0 || naddr
>= ARRAY_SIZE(addr
)) {
508 ret
= t4_alloc_mac_filt(pi
->adapter
, mb
, pi
->viid
, free
,
509 naddr
, addr
, filt_idx
, &mhash
, sleep
);
518 return t4_set_addr_hash(pi
->adapter
, mb
, pi
->viid
, uhash
!= 0,
519 uhash
| mhash
, sleep
);
522 int dbfifo_int_thresh
= 10; /* 10 == 640 entry threshold */
523 module_param(dbfifo_int_thresh
, int, 0644);
524 MODULE_PARM_DESC(dbfifo_int_thresh
, "doorbell fifo interrupt threshold");
527 * usecs to sleep while draining the dbfifo
529 static int dbfifo_drain_delay
= 1000;
530 module_param(dbfifo_drain_delay
, int, 0644);
531 MODULE_PARM_DESC(dbfifo_drain_delay
,
532 "usecs to sleep while draining the dbfifo");
535 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
536 * If @mtu is -1 it is left unchanged.
538 static int set_rxmode(struct net_device
*dev
, int mtu
, bool sleep_ok
)
541 struct port_info
*pi
= netdev_priv(dev
);
543 ret
= set_addr_filters(dev
, sleep_ok
);
545 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, mtu
,
546 (dev
->flags
& IFF_PROMISC
) ? 1 : 0,
547 (dev
->flags
& IFF_ALLMULTI
) ? 1 : 0, 1, -1,
552 static struct workqueue_struct
*workq
;
555 * link_start - enable a port
556 * @dev: the port to enable
558 * Performs the MAC and PHY actions needed to enable a port.
560 static int link_start(struct net_device
*dev
)
563 struct port_info
*pi
= netdev_priv(dev
);
564 unsigned int mb
= pi
->adapter
->fn
;
567 * We do not set address filters and promiscuity here, the stack does
568 * that step explicitly.
570 ret
= t4_set_rxmode(pi
->adapter
, mb
, pi
->viid
, dev
->mtu
, -1, -1, -1,
571 !!(dev
->features
& NETIF_F_HW_VLAN_CTAG_RX
), true);
573 ret
= t4_change_mac(pi
->adapter
, mb
, pi
->viid
,
574 pi
->xact_addr_filt
, dev
->dev_addr
, true,
577 pi
->xact_addr_filt
= ret
;
582 ret
= t4_link_start(pi
->adapter
, mb
, pi
->tx_chan
,
585 ret
= t4_enable_vi(pi
->adapter
, mb
, pi
->viid
, true, true);
589 /* Clear a filter and release any of its resources that we own. This also
590 * clears the filter's "pending" status.
592 static void clear_filter(struct adapter
*adap
, struct filter_entry
*f
)
594 /* If the new or old filter have loopback rewriteing rules then we'll
595 * need to free any existing Layer Two Table (L2T) entries of the old
596 * filter rule. The firmware will handle freeing up any Source MAC
597 * Table (SMT) entries used for rewriting Source MAC Addresses in
601 cxgb4_l2t_release(f
->l2t
);
603 /* The zeroing of the filter rule below clears the filter valid,
604 * pending, locked flags, l2t pointer, etc. so it's all we need for
607 memset(f
, 0, sizeof(*f
));
610 /* Handle a filter write/deletion reply.
612 static void filter_rpl(struct adapter
*adap
, const struct cpl_set_tcb_rpl
*rpl
)
614 unsigned int idx
= GET_TID(rpl
);
615 unsigned int nidx
= idx
- adap
->tids
.ftid_base
;
617 struct filter_entry
*f
;
619 if (idx
>= adap
->tids
.ftid_base
&& nidx
<
620 (adap
->tids
.nftids
+ adap
->tids
.nsftids
)) {
622 ret
= GET_TCB_COOKIE(rpl
->cookie
);
623 f
= &adap
->tids
.ftid_tab
[idx
];
625 if (ret
== FW_FILTER_WR_FLT_DELETED
) {
626 /* Clear the filter when we get confirmation from the
627 * hardware that the filter has been deleted.
629 clear_filter(adap
, f
);
630 } else if (ret
== FW_FILTER_WR_SMT_TBL_FULL
) {
631 dev_err(adap
->pdev_dev
, "filter %u setup failed due to full SMT\n",
633 clear_filter(adap
, f
);
634 } else if (ret
== FW_FILTER_WR_FLT_ADDED
) {
635 f
->smtidx
= (be64_to_cpu(rpl
->oldval
) >> 24) & 0xff;
636 f
->pending
= 0; /* asynchronous setup completed */
639 /* Something went wrong. Issue a warning about the
640 * problem and clear everything out.
642 dev_err(adap
->pdev_dev
, "filter %u setup failed with error %u\n",
644 clear_filter(adap
, f
);
649 /* Response queue handler for the FW event queue.
651 static int fwevtq_handler(struct sge_rspq
*q
, const __be64
*rsp
,
652 const struct pkt_gl
*gl
)
654 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
656 rsp
++; /* skip RSS header */
658 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
660 if (unlikely(opcode
== CPL_FW4_MSG
&&
661 ((const struct cpl_fw4_msg
*)rsp
)->type
== FW_TYPE_RSSCPL
)) {
663 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
665 if (opcode
!= CPL_SGE_EGR_UPDATE
) {
666 dev_err(q
->adap
->pdev_dev
, "unexpected FW4/CPL %#x on FW event queue\n"
672 if (likely(opcode
== CPL_SGE_EGR_UPDATE
)) {
673 const struct cpl_sge_egr_update
*p
= (void *)rsp
;
674 unsigned int qid
= EGR_QID(ntohl(p
->opcode_qid
));
677 txq
= q
->adap
->sge
.egr_map
[qid
- q
->adap
->sge
.egr_start
];
679 if ((u8
*)txq
< (u8
*)q
->adap
->sge
.ofldtxq
) {
680 struct sge_eth_txq
*eq
;
682 eq
= container_of(txq
, struct sge_eth_txq
, q
);
683 netif_tx_wake_queue(eq
->txq
);
685 struct sge_ofld_txq
*oq
;
687 oq
= container_of(txq
, struct sge_ofld_txq
, q
);
688 tasklet_schedule(&oq
->qresume_tsk
);
690 } else if (opcode
== CPL_FW6_MSG
|| opcode
== CPL_FW4_MSG
) {
691 const struct cpl_fw6_msg
*p
= (void *)rsp
;
694 t4_handle_fw_rpl(q
->adap
, p
->data
);
695 } else if (opcode
== CPL_L2T_WRITE_RPL
) {
696 const struct cpl_l2t_write_rpl
*p
= (void *)rsp
;
698 do_l2t_write_rpl(q
->adap
, p
);
699 } else if (opcode
== CPL_SET_TCB_RPL
) {
700 const struct cpl_set_tcb_rpl
*p
= (void *)rsp
;
702 filter_rpl(q
->adap
, p
);
704 dev_err(q
->adap
->pdev_dev
,
705 "unexpected CPL %#x on FW event queue\n", opcode
);
711 * uldrx_handler - response queue handler for ULD queues
712 * @q: the response queue that received the packet
713 * @rsp: the response queue descriptor holding the offload message
714 * @gl: the gather list of packet fragments
716 * Deliver an ingress offload packet to a ULD. All processing is done by
717 * the ULD, we just maintain statistics.
719 static int uldrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
720 const struct pkt_gl
*gl
)
722 struct sge_ofld_rxq
*rxq
= container_of(q
, struct sge_ofld_rxq
, rspq
);
724 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
726 if (((const struct rss_header
*)rsp
)->opcode
== CPL_FW4_MSG
&&
727 ((const struct cpl_fw4_msg
*)(rsp
+ 1))->type
== FW_TYPE_RSSCPL
)
730 if (ulds
[q
->uld
].rx_handler(q
->adap
->uld_handle
[q
->uld
], rsp
, gl
)) {
736 else if (gl
== CXGB4_MSG_AN
)
743 static void disable_msi(struct adapter
*adapter
)
745 if (adapter
->flags
& USING_MSIX
) {
746 pci_disable_msix(adapter
->pdev
);
747 adapter
->flags
&= ~USING_MSIX
;
748 } else if (adapter
->flags
& USING_MSI
) {
749 pci_disable_msi(adapter
->pdev
);
750 adapter
->flags
&= ~USING_MSI
;
755 * Interrupt handler for non-data events used with MSI-X.
757 static irqreturn_t
t4_nondata_intr(int irq
, void *cookie
)
759 struct adapter
*adap
= cookie
;
761 u32 v
= t4_read_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
));
764 t4_write_reg(adap
, MYPF_REG(PL_PF_INT_CAUSE
), v
);
766 t4_slow_intr_handler(adap
);
771 * Name the MSI-X interrupts.
773 static void name_msix_vecs(struct adapter
*adap
)
775 int i
, j
, msi_idx
= 2, n
= sizeof(adap
->msix_info
[0].desc
);
777 /* non-data interrupts */
778 snprintf(adap
->msix_info
[0].desc
, n
, "%s", adap
->port
[0]->name
);
781 snprintf(adap
->msix_info
[1].desc
, n
, "%s-FWeventq",
782 adap
->port
[0]->name
);
784 /* Ethernet queues */
785 for_each_port(adap
, j
) {
786 struct net_device
*d
= adap
->port
[j
];
787 const struct port_info
*pi
= netdev_priv(d
);
789 for (i
= 0; i
< pi
->nqsets
; i
++, msi_idx
++)
790 snprintf(adap
->msix_info
[msi_idx
].desc
, n
, "%s-Rx%d",
795 for_each_ofldrxq(&adap
->sge
, i
)
796 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-ofld%d",
797 adap
->port
[0]->name
, i
);
799 for_each_rdmarxq(&adap
->sge
, i
)
800 snprintf(adap
->msix_info
[msi_idx
++].desc
, n
, "%s-rdma%d",
801 adap
->port
[0]->name
, i
);
804 static int request_msix_queue_irqs(struct adapter
*adap
)
806 struct sge
*s
= &adap
->sge
;
807 int err
, ethqidx
, ofldqidx
= 0, rdmaqidx
= 0, msi_index
= 2;
809 err
= request_irq(adap
->msix_info
[1].vec
, t4_sge_intr_msix
, 0,
810 adap
->msix_info
[1].desc
, &s
->fw_evtq
);
814 for_each_ethrxq(s
, ethqidx
) {
815 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
817 adap
->msix_info
[msi_index
].desc
,
818 &s
->ethrxq
[ethqidx
].rspq
);
823 for_each_ofldrxq(s
, ofldqidx
) {
824 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
826 adap
->msix_info
[msi_index
].desc
,
827 &s
->ofldrxq
[ofldqidx
].rspq
);
832 for_each_rdmarxq(s
, rdmaqidx
) {
833 err
= request_irq(adap
->msix_info
[msi_index
].vec
,
835 adap
->msix_info
[msi_index
].desc
,
836 &s
->rdmarxq
[rdmaqidx
].rspq
);
844 while (--rdmaqidx
>= 0)
845 free_irq(adap
->msix_info
[--msi_index
].vec
,
846 &s
->rdmarxq
[rdmaqidx
].rspq
);
847 while (--ofldqidx
>= 0)
848 free_irq(adap
->msix_info
[--msi_index
].vec
,
849 &s
->ofldrxq
[ofldqidx
].rspq
);
850 while (--ethqidx
>= 0)
851 free_irq(adap
->msix_info
[--msi_index
].vec
,
852 &s
->ethrxq
[ethqidx
].rspq
);
853 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
857 static void free_msix_queue_irqs(struct adapter
*adap
)
859 int i
, msi_index
= 2;
860 struct sge
*s
= &adap
->sge
;
862 free_irq(adap
->msix_info
[1].vec
, &s
->fw_evtq
);
863 for_each_ethrxq(s
, i
)
864 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ethrxq
[i
].rspq
);
865 for_each_ofldrxq(s
, i
)
866 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->ofldrxq
[i
].rspq
);
867 for_each_rdmarxq(s
, i
)
868 free_irq(adap
->msix_info
[msi_index
++].vec
, &s
->rdmarxq
[i
].rspq
);
872 * write_rss - write the RSS table for a given port
874 * @queues: array of queue indices for RSS
876 * Sets up the portion of the HW RSS table for the port's VI to distribute
877 * packets to the Rx queues in @queues.
879 static int write_rss(const struct port_info
*pi
, const u16
*queues
)
883 const struct sge_eth_rxq
*q
= &pi
->adapter
->sge
.ethrxq
[pi
->first_qset
];
885 rss
= kmalloc(pi
->rss_size
* sizeof(u16
), GFP_KERNEL
);
889 /* map the queue indices to queue ids */
890 for (i
= 0; i
< pi
->rss_size
; i
++, queues
++)
891 rss
[i
] = q
[*queues
].rspq
.abs_id
;
893 err
= t4_config_rss_range(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, 0,
894 pi
->rss_size
, rss
, pi
->rss_size
);
900 * setup_rss - configure RSS
903 * Sets up RSS for each port.
905 static int setup_rss(struct adapter
*adap
)
909 for_each_port(adap
, i
) {
910 const struct port_info
*pi
= adap2pinfo(adap
, i
);
912 err
= write_rss(pi
, pi
->rss
);
920 * Return the channel of the ingress queue with the given qid.
922 static unsigned int rxq_to_chan(const struct sge
*p
, unsigned int qid
)
924 qid
-= p
->ingr_start
;
925 return netdev2pinfo(p
->ingr_map
[qid
]->netdev
)->tx_chan
;
929 * Wait until all NAPI handlers are descheduled.
931 static void quiesce_rx(struct adapter
*adap
)
935 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
936 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
939 napi_disable(&q
->napi
);
944 * Enable NAPI scheduling and interrupt generation for all Rx queues.
946 static void enable_rx(struct adapter
*adap
)
950 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ingr_map
); i
++) {
951 struct sge_rspq
*q
= adap
->sge
.ingr_map
[i
];
956 napi_enable(&q
->napi
);
957 /* 0-increment GTS to start the timer and enable interrupts */
958 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS
),
959 SEINTARM(q
->intr_params
) |
960 INGRESSQID(q
->cntxt_id
));
965 * setup_sge_queues - configure SGE Tx/Rx/response queues
968 * Determines how many sets of SGE queues to use and initializes them.
969 * We support multiple queue sets per port if we have MSI-X, otherwise
970 * just one queue set per port.
972 static int setup_sge_queues(struct adapter
*adap
)
974 int err
, msi_idx
, i
, j
;
975 struct sge
*s
= &adap
->sge
;
977 bitmap_zero(s
->starving_fl
, MAX_EGRQ
);
978 bitmap_zero(s
->txq_maperr
, MAX_EGRQ
);
980 if (adap
->flags
& USING_MSIX
)
981 msi_idx
= 1; /* vector 0 is for non-queue interrupts */
983 err
= t4_sge_alloc_rxq(adap
, &s
->intrq
, false, adap
->port
[0], 0,
987 msi_idx
= -((int)s
->intrq
.abs_id
+ 1);
990 err
= t4_sge_alloc_rxq(adap
, &s
->fw_evtq
, true, adap
->port
[0],
991 msi_idx
, NULL
, fwevtq_handler
);
993 freeout
: t4_free_sge_resources(adap
);
997 for_each_port(adap
, i
) {
998 struct net_device
*dev
= adap
->port
[i
];
999 struct port_info
*pi
= netdev_priv(dev
);
1000 struct sge_eth_rxq
*q
= &s
->ethrxq
[pi
->first_qset
];
1001 struct sge_eth_txq
*t
= &s
->ethtxq
[pi
->first_qset
];
1003 for (j
= 0; j
< pi
->nqsets
; j
++, q
++) {
1006 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
,
1012 memset(&q
->stats
, 0, sizeof(q
->stats
));
1014 for (j
= 0; j
< pi
->nqsets
; j
++, t
++) {
1015 err
= t4_sge_alloc_eth_txq(adap
, t
, dev
,
1016 netdev_get_tx_queue(dev
, j
),
1017 s
->fw_evtq
.cntxt_id
);
1023 j
= s
->ofldqsets
/ adap
->params
.nports
; /* ofld queues per channel */
1024 for_each_ofldrxq(s
, i
) {
1025 struct sge_ofld_rxq
*q
= &s
->ofldrxq
[i
];
1026 struct net_device
*dev
= adap
->port
[i
/ j
];
1030 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, dev
, msi_idx
,
1031 &q
->fl
, uldrx_handler
);
1034 memset(&q
->stats
, 0, sizeof(q
->stats
));
1035 s
->ofld_rxq
[i
] = q
->rspq
.abs_id
;
1036 err
= t4_sge_alloc_ofld_txq(adap
, &s
->ofldtxq
[i
], dev
,
1037 s
->fw_evtq
.cntxt_id
);
1042 for_each_rdmarxq(s
, i
) {
1043 struct sge_ofld_rxq
*q
= &s
->rdmarxq
[i
];
1047 err
= t4_sge_alloc_rxq(adap
, &q
->rspq
, false, adap
->port
[i
],
1048 msi_idx
, &q
->fl
, uldrx_handler
);
1051 memset(&q
->stats
, 0, sizeof(q
->stats
));
1052 s
->rdma_rxq
[i
] = q
->rspq
.abs_id
;
1055 for_each_port(adap
, i
) {
1057 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1058 * have RDMA queues, and that's the right value.
1060 err
= t4_sge_alloc_ctrl_txq(adap
, &s
->ctrlq
[i
], adap
->port
[i
],
1061 s
->fw_evtq
.cntxt_id
,
1062 s
->rdmarxq
[i
].rspq
.cntxt_id
);
1067 t4_write_reg(adap
, MPS_TRC_RSS_CONTROL
,
1068 RSSCONTROL(netdev2pinfo(adap
->port
[0])->tx_chan
) |
1069 QUEUENUMBER(s
->ethrxq
[0].rspq
.abs_id
));
1074 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
1075 * started but failed, and a negative errno if flash load couldn't start.
1077 static int upgrade_fw(struct adapter
*adap
)
1080 u32 vers
, exp_major
;
1081 const struct fw_hdr
*hdr
;
1082 const struct firmware
*fw
;
1083 struct device
*dev
= adap
->pdev_dev
;
1086 switch (CHELSIO_CHIP_VERSION(adap
->chip
)) {
1088 fw_file_name
= FW_FNAME
;
1089 exp_major
= FW_VERSION_MAJOR
;
1092 fw_file_name
= FW5_FNAME
;
1093 exp_major
= FW_VERSION_MAJOR_T5
;
1096 dev_err(dev
, "Unsupported chip type, %x\n", adap
->chip
);
1100 ret
= request_firmware(&fw
, fw_file_name
, dev
);
1102 dev_err(dev
, "unable to load firmware image %s, error %d\n",
1107 hdr
= (const struct fw_hdr
*)fw
->data
;
1108 vers
= ntohl(hdr
->fw_ver
);
1109 if (FW_HDR_FW_VER_MAJOR_GET(vers
) != exp_major
) {
1110 ret
= -EINVAL
; /* wrong major version, won't do */
1115 * If the flash FW is unusable or we found something newer, load it.
1117 if (FW_HDR_FW_VER_MAJOR_GET(adap
->params
.fw_vers
) != exp_major
||
1118 vers
> adap
->params
.fw_vers
) {
1119 dev_info(dev
, "upgrading firmware ...\n");
1120 ret
= t4_fw_upgrade(adap
, adap
->mbox
, fw
->data
, fw
->size
,
1124 "firmware upgraded to version %pI4 from %s\n",
1125 &hdr
->fw_ver
, fw_file_name
);
1127 dev_err(dev
, "firmware upgrade failed! err=%d\n", -ret
);
1130 * Tell our caller that we didn't upgrade the firmware.
1135 out
: release_firmware(fw
);
1140 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1141 * The allocated memory is cleared.
1143 void *t4_alloc_mem(size_t size
)
1145 void *p
= kzalloc(size
, GFP_KERNEL
| __GFP_NOWARN
);
1153 * Free memory allocated through alloc_mem().
1155 static void t4_free_mem(void *addr
)
1157 if (is_vmalloc_addr(addr
))
1163 /* Send a Work Request to write the filter at a specified index. We construct
1164 * a Firmware Filter Work Request to have the work done and put the indicated
1165 * filter into "pending" mode which will prevent any further actions against
1166 * it till we get a reply from the firmware on the completion status of the
1169 static int set_filter_wr(struct adapter
*adapter
, int fidx
)
1171 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[fidx
];
1172 struct sk_buff
*skb
;
1173 struct fw_filter_wr
*fwr
;
1176 /* If the new filter requires loopback Destination MAC and/or VLAN
1177 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1180 if (f
->fs
.newdmac
|| f
->fs
.newvlan
) {
1181 /* allocate L2T entry for new filter */
1182 f
->l2t
= t4_l2t_alloc_switching(adapter
->l2t
);
1185 if (t4_l2t_set_switching(adapter
, f
->l2t
, f
->fs
.vlan
,
1186 f
->fs
.eport
, f
->fs
.dmac
)) {
1187 cxgb4_l2t_release(f
->l2t
);
1193 ftid
= adapter
->tids
.ftid_base
+ fidx
;
1195 skb
= alloc_skb(sizeof(*fwr
), GFP_KERNEL
| __GFP_NOFAIL
);
1196 fwr
= (struct fw_filter_wr
*)__skb_put(skb
, sizeof(*fwr
));
1197 memset(fwr
, 0, sizeof(*fwr
));
1199 /* It would be nice to put most of the following in t4_hw.c but most
1200 * of the work is translating the cxgbtool ch_filter_specification
1201 * into the Work Request and the definition of that structure is
1202 * currently in cxgbtool.h which isn't appropriate to pull into the
1203 * common code. We may eventually try to come up with a more neutral
1204 * filter specification structure but for now it's easiest to simply
1205 * put this fairly direct code in line ...
1207 fwr
->op_pkd
= htonl(FW_WR_OP(FW_FILTER_WR
));
1208 fwr
->len16_pkd
= htonl(FW_WR_LEN16(sizeof(*fwr
)/16));
1210 htonl(V_FW_FILTER_WR_TID(ftid
) |
1211 V_FW_FILTER_WR_RQTYPE(f
->fs
.type
) |
1212 V_FW_FILTER_WR_NOREPLY(0) |
1213 V_FW_FILTER_WR_IQ(f
->fs
.iq
));
1214 fwr
->del_filter_to_l2tix
=
1215 htonl(V_FW_FILTER_WR_RPTTID(f
->fs
.rpttid
) |
1216 V_FW_FILTER_WR_DROP(f
->fs
.action
== FILTER_DROP
) |
1217 V_FW_FILTER_WR_DIRSTEER(f
->fs
.dirsteer
) |
1218 V_FW_FILTER_WR_MASKHASH(f
->fs
.maskhash
) |
1219 V_FW_FILTER_WR_DIRSTEERHASH(f
->fs
.dirsteerhash
) |
1220 V_FW_FILTER_WR_LPBK(f
->fs
.action
== FILTER_SWITCH
) |
1221 V_FW_FILTER_WR_DMAC(f
->fs
.newdmac
) |
1222 V_FW_FILTER_WR_SMAC(f
->fs
.newsmac
) |
1223 V_FW_FILTER_WR_INSVLAN(f
->fs
.newvlan
== VLAN_INSERT
||
1224 f
->fs
.newvlan
== VLAN_REWRITE
) |
1225 V_FW_FILTER_WR_RMVLAN(f
->fs
.newvlan
== VLAN_REMOVE
||
1226 f
->fs
.newvlan
== VLAN_REWRITE
) |
1227 V_FW_FILTER_WR_HITCNTS(f
->fs
.hitcnts
) |
1228 V_FW_FILTER_WR_TXCHAN(f
->fs
.eport
) |
1229 V_FW_FILTER_WR_PRIO(f
->fs
.prio
) |
1230 V_FW_FILTER_WR_L2TIX(f
->l2t
? f
->l2t
->idx
: 0));
1231 fwr
->ethtype
= htons(f
->fs
.val
.ethtype
);
1232 fwr
->ethtypem
= htons(f
->fs
.mask
.ethtype
);
1233 fwr
->frag_to_ovlan_vldm
=
1234 (V_FW_FILTER_WR_FRAG(f
->fs
.val
.frag
) |
1235 V_FW_FILTER_WR_FRAGM(f
->fs
.mask
.frag
) |
1236 V_FW_FILTER_WR_IVLAN_VLD(f
->fs
.val
.ivlan_vld
) |
1237 V_FW_FILTER_WR_OVLAN_VLD(f
->fs
.val
.ovlan_vld
) |
1238 V_FW_FILTER_WR_IVLAN_VLDM(f
->fs
.mask
.ivlan_vld
) |
1239 V_FW_FILTER_WR_OVLAN_VLDM(f
->fs
.mask
.ovlan_vld
));
1241 fwr
->rx_chan_rx_rpl_iq
=
1242 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1243 V_FW_FILTER_WR_RX_RPL_IQ(adapter
->sge
.fw_evtq
.abs_id
));
1244 fwr
->maci_to_matchtypem
=
1245 htonl(V_FW_FILTER_WR_MACI(f
->fs
.val
.macidx
) |
1246 V_FW_FILTER_WR_MACIM(f
->fs
.mask
.macidx
) |
1247 V_FW_FILTER_WR_FCOE(f
->fs
.val
.fcoe
) |
1248 V_FW_FILTER_WR_FCOEM(f
->fs
.mask
.fcoe
) |
1249 V_FW_FILTER_WR_PORT(f
->fs
.val
.iport
) |
1250 V_FW_FILTER_WR_PORTM(f
->fs
.mask
.iport
) |
1251 V_FW_FILTER_WR_MATCHTYPE(f
->fs
.val
.matchtype
) |
1252 V_FW_FILTER_WR_MATCHTYPEM(f
->fs
.mask
.matchtype
));
1253 fwr
->ptcl
= f
->fs
.val
.proto
;
1254 fwr
->ptclm
= f
->fs
.mask
.proto
;
1255 fwr
->ttyp
= f
->fs
.val
.tos
;
1256 fwr
->ttypm
= f
->fs
.mask
.tos
;
1257 fwr
->ivlan
= htons(f
->fs
.val
.ivlan
);
1258 fwr
->ivlanm
= htons(f
->fs
.mask
.ivlan
);
1259 fwr
->ovlan
= htons(f
->fs
.val
.ovlan
);
1260 fwr
->ovlanm
= htons(f
->fs
.mask
.ovlan
);
1261 memcpy(fwr
->lip
, f
->fs
.val
.lip
, sizeof(fwr
->lip
));
1262 memcpy(fwr
->lipm
, f
->fs
.mask
.lip
, sizeof(fwr
->lipm
));
1263 memcpy(fwr
->fip
, f
->fs
.val
.fip
, sizeof(fwr
->fip
));
1264 memcpy(fwr
->fipm
, f
->fs
.mask
.fip
, sizeof(fwr
->fipm
));
1265 fwr
->lp
= htons(f
->fs
.val
.lport
);
1266 fwr
->lpm
= htons(f
->fs
.mask
.lport
);
1267 fwr
->fp
= htons(f
->fs
.val
.fport
);
1268 fwr
->fpm
= htons(f
->fs
.mask
.fport
);
1270 memcpy(fwr
->sma
, f
->fs
.smac
, sizeof(fwr
->sma
));
1272 /* Mark the filter as "pending" and ship off the Filter Work Request.
1273 * When we get the Work Request Reply we'll clear the pending status.
1276 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, f
->fs
.val
.iport
& 0x3);
1277 t4_ofld_send(adapter
, skb
);
1281 /* Delete the filter at a specified index.
1283 static int del_filter_wr(struct adapter
*adapter
, int fidx
)
1285 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[fidx
];
1286 struct sk_buff
*skb
;
1287 struct fw_filter_wr
*fwr
;
1288 unsigned int len
, ftid
;
1291 ftid
= adapter
->tids
.ftid_base
+ fidx
;
1293 skb
= alloc_skb(len
, GFP_KERNEL
| __GFP_NOFAIL
);
1294 fwr
= (struct fw_filter_wr
*)__skb_put(skb
, len
);
1295 t4_mk_filtdelwr(ftid
, fwr
, adapter
->sge
.fw_evtq
.abs_id
);
1297 /* Mark the filter as "pending" and ship off the Filter Work Request.
1298 * When we get the Work Request Reply we'll clear the pending status.
1301 t4_mgmt_tx(adapter
, skb
);
1305 static inline int is_offload(const struct adapter
*adap
)
1307 return adap
->params
.offload
;
1311 * Implementation of ethtool operations.
1314 static u32
get_msglevel(struct net_device
*dev
)
1316 return netdev2adap(dev
)->msg_enable
;
1319 static void set_msglevel(struct net_device
*dev
, u32 val
)
1321 netdev2adap(dev
)->msg_enable
= val
;
1324 static char stats_strings
[][ETH_GSTRING_LEN
] = {
1327 "TxBroadcastFrames ",
1328 "TxMulticastFrames ",
1334 "TxFrames128To255 ",
1335 "TxFrames256To511 ",
1336 "TxFrames512To1023 ",
1337 "TxFrames1024To1518 ",
1338 "TxFrames1519ToMax ",
1353 "RxBroadcastFrames ",
1354 "RxMulticastFrames ",
1366 "RxFrames128To255 ",
1367 "RxFrames256To511 ",
1368 "RxFrames512To1023 ",
1369 "RxFrames1024To1518 ",
1370 "RxFrames1519ToMax ",
1382 "RxBG0FramesDropped ",
1383 "RxBG1FramesDropped ",
1384 "RxBG2FramesDropped ",
1385 "RxBG3FramesDropped ",
1386 "RxBG0FramesTrunc ",
1387 "RxBG1FramesTrunc ",
1388 "RxBG2FramesTrunc ",
1389 "RxBG3FramesTrunc ",
1398 "WriteCoalSuccess ",
1402 static int get_sset_count(struct net_device
*dev
, int sset
)
1406 return ARRAY_SIZE(stats_strings
);
1412 #define T4_REGMAP_SIZE (160 * 1024)
1413 #define T5_REGMAP_SIZE (332 * 1024)
1415 static int get_regs_len(struct net_device
*dev
)
1417 struct adapter
*adap
= netdev2adap(dev
);
1418 if (is_t4(adap
->chip
))
1419 return T4_REGMAP_SIZE
;
1421 return T5_REGMAP_SIZE
;
1424 static int get_eeprom_len(struct net_device
*dev
)
1429 static void get_drvinfo(struct net_device
*dev
, struct ethtool_drvinfo
*info
)
1431 struct adapter
*adapter
= netdev2adap(dev
);
1433 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
1434 strlcpy(info
->version
, DRV_VERSION
, sizeof(info
->version
));
1435 strlcpy(info
->bus_info
, pci_name(adapter
->pdev
),
1436 sizeof(info
->bus_info
));
1438 if (adapter
->params
.fw_vers
)
1439 snprintf(info
->fw_version
, sizeof(info
->fw_version
),
1440 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1441 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.fw_vers
),
1442 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.fw_vers
),
1443 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.fw_vers
),
1444 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.fw_vers
),
1445 FW_HDR_FW_VER_MAJOR_GET(adapter
->params
.tp_vers
),
1446 FW_HDR_FW_VER_MINOR_GET(adapter
->params
.tp_vers
),
1447 FW_HDR_FW_VER_MICRO_GET(adapter
->params
.tp_vers
),
1448 FW_HDR_FW_VER_BUILD_GET(adapter
->params
.tp_vers
));
1451 static void get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1453 if (stringset
== ETH_SS_STATS
)
1454 memcpy(data
, stats_strings
, sizeof(stats_strings
));
1458 * port stats maintained per queue of the port. They should be in the same
1459 * order as in stats_strings above.
1461 struct queue_port_stats
{
1471 static void collect_sge_port_stats(const struct adapter
*adap
,
1472 const struct port_info
*p
, struct queue_port_stats
*s
)
1475 const struct sge_eth_txq
*tx
= &adap
->sge
.ethtxq
[p
->first_qset
];
1476 const struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[p
->first_qset
];
1478 memset(s
, 0, sizeof(*s
));
1479 for (i
= 0; i
< p
->nqsets
; i
++, rx
++, tx
++) {
1481 s
->tx_csum
+= tx
->tx_cso
;
1482 s
->rx_csum
+= rx
->stats
.rx_cso
;
1483 s
->vlan_ex
+= rx
->stats
.vlan_ex
;
1484 s
->vlan_ins
+= tx
->vlan_ins
;
1485 s
->gro_pkts
+= rx
->stats
.lro_pkts
;
1486 s
->gro_merged
+= rx
->stats
.lro_merged
;
1490 static void get_stats(struct net_device
*dev
, struct ethtool_stats
*stats
,
1493 struct port_info
*pi
= netdev_priv(dev
);
1494 struct adapter
*adapter
= pi
->adapter
;
1497 t4_get_port_stats(adapter
, pi
->tx_chan
, (struct port_stats
*)data
);
1499 data
+= sizeof(struct port_stats
) / sizeof(u64
);
1500 collect_sge_port_stats(adapter
, pi
, (struct queue_port_stats
*)data
);
1501 data
+= sizeof(struct queue_port_stats
) / sizeof(u64
);
1502 if (!is_t4(adapter
->chip
)) {
1503 t4_write_reg(adapter
, SGE_STAT_CFG
, STATSOURCE_T5(7));
1504 val1
= t4_read_reg(adapter
, SGE_STAT_TOTAL
);
1505 val2
= t4_read_reg(adapter
, SGE_STAT_MATCH
);
1506 *data
= val1
- val2
;
1511 memset(data
, 0, 2 * sizeof(u64
));
1517 * Return a version number to identify the type of adapter. The scheme is:
1518 * - bits 0..9: chip version
1519 * - bits 10..15: chip revision
1520 * - bits 16..23: register dump version
1522 static inline unsigned int mk_adap_vers(const struct adapter
*ap
)
1524 return CHELSIO_CHIP_VERSION(ap
->chip
) |
1525 (CHELSIO_CHIP_RELEASE(ap
->chip
) << 10) | (1 << 16);
1528 static void reg_block_dump(struct adapter
*ap
, void *buf
, unsigned int start
,
1531 u32
*p
= buf
+ start
;
1533 for ( ; start
<= end
; start
+= sizeof(u32
))
1534 *p
++ = t4_read_reg(ap
, start
);
1537 static void get_regs(struct net_device
*dev
, struct ethtool_regs
*regs
,
1540 static const unsigned int t4_reg_ranges
[] = {
1760 static const unsigned int t5_reg_ranges
[] = {
2188 struct adapter
*ap
= netdev2adap(dev
);
2189 static const unsigned int *reg_ranges
;
2190 int arr_size
= 0, buf_size
= 0;
2192 if (is_t4(ap
->chip
)) {
2193 reg_ranges
= &t4_reg_ranges
[0];
2194 arr_size
= ARRAY_SIZE(t4_reg_ranges
);
2195 buf_size
= T4_REGMAP_SIZE
;
2197 reg_ranges
= &t5_reg_ranges
[0];
2198 arr_size
= ARRAY_SIZE(t5_reg_ranges
);
2199 buf_size
= T5_REGMAP_SIZE
;
2202 regs
->version
= mk_adap_vers(ap
);
2204 memset(buf
, 0, buf_size
);
2205 for (i
= 0; i
< arr_size
; i
+= 2)
2206 reg_block_dump(ap
, buf
, reg_ranges
[i
], reg_ranges
[i
+ 1]);
2209 static int restart_autoneg(struct net_device
*dev
)
2211 struct port_info
*p
= netdev_priv(dev
);
2213 if (!netif_running(dev
))
2215 if (p
->link_cfg
.autoneg
!= AUTONEG_ENABLE
)
2217 t4_restart_aneg(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
);
2221 static int identify_port(struct net_device
*dev
,
2222 enum ethtool_phys_id_state state
)
2225 struct adapter
*adap
= netdev2adap(dev
);
2227 if (state
== ETHTOOL_ID_ACTIVE
)
2229 else if (state
== ETHTOOL_ID_INACTIVE
)
2234 return t4_identify_port(adap
, adap
->fn
, netdev2pinfo(dev
)->viid
, val
);
2237 static unsigned int from_fw_linkcaps(unsigned int type
, unsigned int caps
)
2241 if (type
== FW_PORT_TYPE_BT_SGMII
|| type
== FW_PORT_TYPE_BT_XFI
||
2242 type
== FW_PORT_TYPE_BT_XAUI
) {
2244 if (caps
& FW_PORT_CAP_SPEED_100M
)
2245 v
|= SUPPORTED_100baseT_Full
;
2246 if (caps
& FW_PORT_CAP_SPEED_1G
)
2247 v
|= SUPPORTED_1000baseT_Full
;
2248 if (caps
& FW_PORT_CAP_SPEED_10G
)
2249 v
|= SUPPORTED_10000baseT_Full
;
2250 } else if (type
== FW_PORT_TYPE_KX4
|| type
== FW_PORT_TYPE_KX
) {
2251 v
|= SUPPORTED_Backplane
;
2252 if (caps
& FW_PORT_CAP_SPEED_1G
)
2253 v
|= SUPPORTED_1000baseKX_Full
;
2254 if (caps
& FW_PORT_CAP_SPEED_10G
)
2255 v
|= SUPPORTED_10000baseKX4_Full
;
2256 } else if (type
== FW_PORT_TYPE_KR
)
2257 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseKR_Full
;
2258 else if (type
== FW_PORT_TYPE_BP_AP
)
2259 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
2260 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
;
2261 else if (type
== FW_PORT_TYPE_BP4_AP
)
2262 v
|= SUPPORTED_Backplane
| SUPPORTED_10000baseR_FEC
|
2263 SUPPORTED_10000baseKR_Full
| SUPPORTED_1000baseKX_Full
|
2264 SUPPORTED_10000baseKX4_Full
;
2265 else if (type
== FW_PORT_TYPE_FIBER_XFI
||
2266 type
== FW_PORT_TYPE_FIBER_XAUI
|| type
== FW_PORT_TYPE_SFP
)
2267 v
|= SUPPORTED_FIBRE
;
2269 if (caps
& FW_PORT_CAP_ANEG
)
2270 v
|= SUPPORTED_Autoneg
;
2274 static unsigned int to_fw_linkcaps(unsigned int caps
)
2278 if (caps
& ADVERTISED_100baseT_Full
)
2279 v
|= FW_PORT_CAP_SPEED_100M
;
2280 if (caps
& ADVERTISED_1000baseT_Full
)
2281 v
|= FW_PORT_CAP_SPEED_1G
;
2282 if (caps
& ADVERTISED_10000baseT_Full
)
2283 v
|= FW_PORT_CAP_SPEED_10G
;
2287 static int get_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
2289 const struct port_info
*p
= netdev_priv(dev
);
2291 if (p
->port_type
== FW_PORT_TYPE_BT_SGMII
||
2292 p
->port_type
== FW_PORT_TYPE_BT_XFI
||
2293 p
->port_type
== FW_PORT_TYPE_BT_XAUI
)
2294 cmd
->port
= PORT_TP
;
2295 else if (p
->port_type
== FW_PORT_TYPE_FIBER_XFI
||
2296 p
->port_type
== FW_PORT_TYPE_FIBER_XAUI
)
2297 cmd
->port
= PORT_FIBRE
;
2298 else if (p
->port_type
== FW_PORT_TYPE_SFP
) {
2299 if (p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_PASSIVE
||
2300 p
->mod_type
== FW_PORT_MOD_TYPE_TWINAX_ACTIVE
)
2301 cmd
->port
= PORT_DA
;
2303 cmd
->port
= PORT_FIBRE
;
2305 cmd
->port
= PORT_OTHER
;
2307 if (p
->mdio_addr
>= 0) {
2308 cmd
->phy_address
= p
->mdio_addr
;
2309 cmd
->transceiver
= XCVR_EXTERNAL
;
2310 cmd
->mdio_support
= p
->port_type
== FW_PORT_TYPE_BT_SGMII
?
2311 MDIO_SUPPORTS_C22
: MDIO_SUPPORTS_C45
;
2313 cmd
->phy_address
= 0; /* not really, but no better option */
2314 cmd
->transceiver
= XCVR_INTERNAL
;
2315 cmd
->mdio_support
= 0;
2318 cmd
->supported
= from_fw_linkcaps(p
->port_type
, p
->link_cfg
.supported
);
2319 cmd
->advertising
= from_fw_linkcaps(p
->port_type
,
2320 p
->link_cfg
.advertising
);
2321 ethtool_cmd_speed_set(cmd
,
2322 netif_carrier_ok(dev
) ? p
->link_cfg
.speed
: 0);
2323 cmd
->duplex
= DUPLEX_FULL
;
2324 cmd
->autoneg
= p
->link_cfg
.autoneg
;
2330 static unsigned int speed_to_caps(int speed
)
2332 if (speed
== SPEED_100
)
2333 return FW_PORT_CAP_SPEED_100M
;
2334 if (speed
== SPEED_1000
)
2335 return FW_PORT_CAP_SPEED_1G
;
2336 if (speed
== SPEED_10000
)
2337 return FW_PORT_CAP_SPEED_10G
;
2341 static int set_settings(struct net_device
*dev
, struct ethtool_cmd
*cmd
)
2344 struct port_info
*p
= netdev_priv(dev
);
2345 struct link_config
*lc
= &p
->link_cfg
;
2346 u32 speed
= ethtool_cmd_speed(cmd
);
2348 if (cmd
->duplex
!= DUPLEX_FULL
) /* only full-duplex supported */
2351 if (!(lc
->supported
& FW_PORT_CAP_ANEG
)) {
2353 * PHY offers a single speed. See if that's what's
2356 if (cmd
->autoneg
== AUTONEG_DISABLE
&&
2357 (lc
->supported
& speed_to_caps(speed
)))
2362 if (cmd
->autoneg
== AUTONEG_DISABLE
) {
2363 cap
= speed_to_caps(speed
);
2365 if (!(lc
->supported
& cap
) || (speed
== SPEED_1000
) ||
2366 (speed
== SPEED_10000
))
2368 lc
->requested_speed
= cap
;
2369 lc
->advertising
= 0;
2371 cap
= to_fw_linkcaps(cmd
->advertising
);
2372 if (!(lc
->supported
& cap
))
2374 lc
->requested_speed
= 0;
2375 lc
->advertising
= cap
| FW_PORT_CAP_ANEG
;
2377 lc
->autoneg
= cmd
->autoneg
;
2379 if (netif_running(dev
))
2380 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
2385 static void get_pauseparam(struct net_device
*dev
,
2386 struct ethtool_pauseparam
*epause
)
2388 struct port_info
*p
= netdev_priv(dev
);
2390 epause
->autoneg
= (p
->link_cfg
.requested_fc
& PAUSE_AUTONEG
) != 0;
2391 epause
->rx_pause
= (p
->link_cfg
.fc
& PAUSE_RX
) != 0;
2392 epause
->tx_pause
= (p
->link_cfg
.fc
& PAUSE_TX
) != 0;
2395 static int set_pauseparam(struct net_device
*dev
,
2396 struct ethtool_pauseparam
*epause
)
2398 struct port_info
*p
= netdev_priv(dev
);
2399 struct link_config
*lc
= &p
->link_cfg
;
2401 if (epause
->autoneg
== AUTONEG_DISABLE
)
2402 lc
->requested_fc
= 0;
2403 else if (lc
->supported
& FW_PORT_CAP_ANEG
)
2404 lc
->requested_fc
= PAUSE_AUTONEG
;
2408 if (epause
->rx_pause
)
2409 lc
->requested_fc
|= PAUSE_RX
;
2410 if (epause
->tx_pause
)
2411 lc
->requested_fc
|= PAUSE_TX
;
2412 if (netif_running(dev
))
2413 return t4_link_start(p
->adapter
, p
->adapter
->fn
, p
->tx_chan
,
2418 static void get_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
2420 const struct port_info
*pi
= netdev_priv(dev
);
2421 const struct sge
*s
= &pi
->adapter
->sge
;
2423 e
->rx_max_pending
= MAX_RX_BUFFERS
;
2424 e
->rx_mini_max_pending
= MAX_RSPQ_ENTRIES
;
2425 e
->rx_jumbo_max_pending
= 0;
2426 e
->tx_max_pending
= MAX_TXQ_ENTRIES
;
2428 e
->rx_pending
= s
->ethrxq
[pi
->first_qset
].fl
.size
- 8;
2429 e
->rx_mini_pending
= s
->ethrxq
[pi
->first_qset
].rspq
.size
;
2430 e
->rx_jumbo_pending
= 0;
2431 e
->tx_pending
= s
->ethtxq
[pi
->first_qset
].q
.size
;
2434 static int set_sge_param(struct net_device
*dev
, struct ethtool_ringparam
*e
)
2437 const struct port_info
*pi
= netdev_priv(dev
);
2438 struct adapter
*adapter
= pi
->adapter
;
2439 struct sge
*s
= &adapter
->sge
;
2441 if (e
->rx_pending
> MAX_RX_BUFFERS
|| e
->rx_jumbo_pending
||
2442 e
->tx_pending
> MAX_TXQ_ENTRIES
||
2443 e
->rx_mini_pending
> MAX_RSPQ_ENTRIES
||
2444 e
->rx_mini_pending
< MIN_RSPQ_ENTRIES
||
2445 e
->rx_pending
< MIN_FL_ENTRIES
|| e
->tx_pending
< MIN_TXQ_ENTRIES
)
2448 if (adapter
->flags
& FULL_INIT_DONE
)
2451 for (i
= 0; i
< pi
->nqsets
; ++i
) {
2452 s
->ethtxq
[pi
->first_qset
+ i
].q
.size
= e
->tx_pending
;
2453 s
->ethrxq
[pi
->first_qset
+ i
].fl
.size
= e
->rx_pending
+ 8;
2454 s
->ethrxq
[pi
->first_qset
+ i
].rspq
.size
= e
->rx_mini_pending
;
2459 static int closest_timer(const struct sge
*s
, int time
)
2461 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
2463 for (i
= 0; i
< ARRAY_SIZE(s
->timer_val
); i
++) {
2464 delta
= time
- s
->timer_val
[i
];
2467 if (delta
< min_delta
) {
2475 static int closest_thres(const struct sge
*s
, int thres
)
2477 int i
, delta
, match
= 0, min_delta
= INT_MAX
;
2479 for (i
= 0; i
< ARRAY_SIZE(s
->counter_val
); i
++) {
2480 delta
= thres
- s
->counter_val
[i
];
2483 if (delta
< min_delta
) {
2492 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2494 static unsigned int qtimer_val(const struct adapter
*adap
,
2495 const struct sge_rspq
*q
)
2497 unsigned int idx
= q
->intr_params
>> 1;
2499 return idx
< SGE_NTIMERS
? adap
->sge
.timer_val
[idx
] : 0;
2503 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
2504 * @adap: the adapter
2506 * @us: the hold-off time in us, or 0 to disable timer
2507 * @cnt: the hold-off packet count, or 0 to disable counter
2509 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2510 * one of the two needs to be enabled for the queue to generate interrupts.
2512 static int set_rxq_intr_params(struct adapter
*adap
, struct sge_rspq
*q
,
2513 unsigned int us
, unsigned int cnt
)
2515 if ((us
| cnt
) == 0)
2522 new_idx
= closest_thres(&adap
->sge
, cnt
);
2523 if (q
->desc
&& q
->pktcnt_idx
!= new_idx
) {
2524 /* the queue has already been created, update it */
2525 v
= FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ
) |
2526 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH
) |
2527 FW_PARAMS_PARAM_YZ(q
->cntxt_id
);
2528 err
= t4_set_params(adap
, adap
->fn
, adap
->fn
, 0, 1, &v
,
2533 q
->pktcnt_idx
= new_idx
;
2536 us
= us
== 0 ? 6 : closest_timer(&adap
->sge
, us
);
2537 q
->intr_params
= QINTR_TIMER_IDX(us
) | (cnt
> 0 ? QINTR_CNT_EN
: 0);
2541 static int set_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
2543 const struct port_info
*pi
= netdev_priv(dev
);
2544 struct adapter
*adap
= pi
->adapter
;
2549 for (i
= pi
->first_qset
; i
< pi
->first_qset
+ pi
->nqsets
; i
++) {
2550 q
= &adap
->sge
.ethrxq
[i
].rspq
;
2551 r
= set_rxq_intr_params(adap
, q
, c
->rx_coalesce_usecs
,
2552 c
->rx_max_coalesced_frames
);
2554 dev_err(&dev
->dev
, "failed to set coalesce %d\n", r
);
2561 static int get_coalesce(struct net_device
*dev
, struct ethtool_coalesce
*c
)
2563 const struct port_info
*pi
= netdev_priv(dev
);
2564 const struct adapter
*adap
= pi
->adapter
;
2565 const struct sge_rspq
*rq
= &adap
->sge
.ethrxq
[pi
->first_qset
].rspq
;
2567 c
->rx_coalesce_usecs
= qtimer_val(adap
, rq
);
2568 c
->rx_max_coalesced_frames
= (rq
->intr_params
& QINTR_CNT_EN
) ?
2569 adap
->sge
.counter_val
[rq
->pktcnt_idx
] : 0;
2574 * eeprom_ptov - translate a physical EEPROM address to virtual
2575 * @phys_addr: the physical EEPROM address
2576 * @fn: the PCI function number
2577 * @sz: size of function-specific area
2579 * Translate a physical EEPROM address to virtual. The first 1K is
2580 * accessed through virtual addresses starting at 31K, the rest is
2581 * accessed through virtual addresses starting at 0.
2583 * The mapping is as follows:
2584 * [0..1K) -> [31K..32K)
2585 * [1K..1K+A) -> [31K-A..31K)
2586 * [1K+A..ES) -> [0..ES-A-1K)
2588 * where A = @fn * @sz, and ES = EEPROM size.
2590 static int eeprom_ptov(unsigned int phys_addr
, unsigned int fn
, unsigned int sz
)
2593 if (phys_addr
< 1024)
2594 return phys_addr
+ (31 << 10);
2595 if (phys_addr
< 1024 + fn
)
2596 return 31744 - fn
+ phys_addr
- 1024;
2597 if (phys_addr
< EEPROMSIZE
)
2598 return phys_addr
- 1024 - fn
;
2603 * The next two routines implement eeprom read/write from physical addresses.
2605 static int eeprom_rd_phys(struct adapter
*adap
, unsigned int phys_addr
, u32
*v
)
2607 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
2610 vaddr
= pci_read_vpd(adap
->pdev
, vaddr
, sizeof(u32
), v
);
2611 return vaddr
< 0 ? vaddr
: 0;
2614 static int eeprom_wr_phys(struct adapter
*adap
, unsigned int phys_addr
, u32 v
)
2616 int vaddr
= eeprom_ptov(phys_addr
, adap
->fn
, EEPROMPFSIZE
);
2619 vaddr
= pci_write_vpd(adap
->pdev
, vaddr
, sizeof(u32
), &v
);
2620 return vaddr
< 0 ? vaddr
: 0;
2623 #define EEPROM_MAGIC 0x38E2F10C
2625 static int get_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*e
,
2629 struct adapter
*adapter
= netdev2adap(dev
);
2631 u8
*buf
= kmalloc(EEPROMSIZE
, GFP_KERNEL
);
2635 e
->magic
= EEPROM_MAGIC
;
2636 for (i
= e
->offset
& ~3; !err
&& i
< e
->offset
+ e
->len
; i
+= 4)
2637 err
= eeprom_rd_phys(adapter
, i
, (u32
*)&buf
[i
]);
2640 memcpy(data
, buf
+ e
->offset
, e
->len
);
2645 static int set_eeprom(struct net_device
*dev
, struct ethtool_eeprom
*eeprom
,
2650 u32 aligned_offset
, aligned_len
, *p
;
2651 struct adapter
*adapter
= netdev2adap(dev
);
2653 if (eeprom
->magic
!= EEPROM_MAGIC
)
2656 aligned_offset
= eeprom
->offset
& ~3;
2657 aligned_len
= (eeprom
->len
+ (eeprom
->offset
& 3) + 3) & ~3;
2659 if (adapter
->fn
> 0) {
2660 u32 start
= 1024 + adapter
->fn
* EEPROMPFSIZE
;
2662 if (aligned_offset
< start
||
2663 aligned_offset
+ aligned_len
> start
+ EEPROMPFSIZE
)
2667 if (aligned_offset
!= eeprom
->offset
|| aligned_len
!= eeprom
->len
) {
2669 * RMW possibly needed for first or last words.
2671 buf
= kmalloc(aligned_len
, GFP_KERNEL
);
2674 err
= eeprom_rd_phys(adapter
, aligned_offset
, (u32
*)buf
);
2675 if (!err
&& aligned_len
> 4)
2676 err
= eeprom_rd_phys(adapter
,
2677 aligned_offset
+ aligned_len
- 4,
2678 (u32
*)&buf
[aligned_len
- 4]);
2681 memcpy(buf
+ (eeprom
->offset
& 3), data
, eeprom
->len
);
2685 err
= t4_seeprom_wp(adapter
, false);
2689 for (p
= (u32
*)buf
; !err
&& aligned_len
; aligned_len
-= 4, p
++) {
2690 err
= eeprom_wr_phys(adapter
, aligned_offset
, *p
);
2691 aligned_offset
+= 4;
2695 err
= t4_seeprom_wp(adapter
, true);
2702 static int set_flash(struct net_device
*netdev
, struct ethtool_flash
*ef
)
2705 const struct firmware
*fw
;
2706 struct adapter
*adap
= netdev2adap(netdev
);
2708 ef
->data
[sizeof(ef
->data
) - 1] = '\0';
2709 ret
= request_firmware(&fw
, ef
->data
, adap
->pdev_dev
);
2713 ret
= t4_load_fw(adap
, fw
->data
, fw
->size
);
2714 release_firmware(fw
);
2716 dev_info(adap
->pdev_dev
, "loaded firmware %s\n", ef
->data
);
2720 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2721 #define BCAST_CRC 0xa0ccc1a6
2723 static void get_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
2725 wol
->supported
= WAKE_BCAST
| WAKE_MAGIC
;
2726 wol
->wolopts
= netdev2adap(dev
)->wol
;
2727 memset(&wol
->sopass
, 0, sizeof(wol
->sopass
));
2730 static int set_wol(struct net_device
*dev
, struct ethtool_wolinfo
*wol
)
2733 struct port_info
*pi
= netdev_priv(dev
);
2735 if (wol
->wolopts
& ~WOL_SUPPORTED
)
2737 t4_wol_magic_enable(pi
->adapter
, pi
->tx_chan
,
2738 (wol
->wolopts
& WAKE_MAGIC
) ? dev
->dev_addr
: NULL
);
2739 if (wol
->wolopts
& WAKE_BCAST
) {
2740 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0xfe, ~0ULL,
2743 err
= t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 1,
2744 ~6ULL, ~0ULL, BCAST_CRC
, true);
2746 t4_wol_pat_enable(pi
->adapter
, pi
->tx_chan
, 0, 0, 0, 0, false);
2750 static int cxgb_set_features(struct net_device
*dev
, netdev_features_t features
)
2752 const struct port_info
*pi
= netdev_priv(dev
);
2753 netdev_features_t changed
= dev
->features
^ features
;
2756 if (!(changed
& NETIF_F_HW_VLAN_CTAG_RX
))
2759 err
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, -1,
2761 !!(features
& NETIF_F_HW_VLAN_CTAG_RX
), true);
2763 dev
->features
= features
^ NETIF_F_HW_VLAN_CTAG_RX
;
2767 static u32
get_rss_table_size(struct net_device
*dev
)
2769 const struct port_info
*pi
= netdev_priv(dev
);
2771 return pi
->rss_size
;
2774 static int get_rss_table(struct net_device
*dev
, u32
*p
)
2776 const struct port_info
*pi
= netdev_priv(dev
);
2777 unsigned int n
= pi
->rss_size
;
2784 static int set_rss_table(struct net_device
*dev
, const u32
*p
)
2787 struct port_info
*pi
= netdev_priv(dev
);
2789 for (i
= 0; i
< pi
->rss_size
; i
++)
2791 if (pi
->adapter
->flags
& FULL_INIT_DONE
)
2792 return write_rss(pi
, pi
->rss
);
2796 static int get_rxnfc(struct net_device
*dev
, struct ethtool_rxnfc
*info
,
2799 const struct port_info
*pi
= netdev_priv(dev
);
2801 switch (info
->cmd
) {
2802 case ETHTOOL_GRXFH
: {
2803 unsigned int v
= pi
->rss_mode
;
2806 switch (info
->flow_type
) {
2808 if (v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
)
2809 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2810 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2811 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2812 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2815 if ((v
& FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
) &&
2816 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
2817 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2818 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2819 else if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2820 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2823 case AH_ESP_V4_FLOW
:
2825 if (v
& FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
)
2826 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2829 if (v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
)
2830 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2831 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2832 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2833 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2836 if ((v
& FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
) &&
2837 (v
& FW_RSS_VI_CONFIG_CMD_UDPEN
))
2838 info
->data
= RXH_IP_SRC
| RXH_IP_DST
|
2839 RXH_L4_B_0_1
| RXH_L4_B_2_3
;
2840 else if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2841 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2844 case AH_ESP_V6_FLOW
:
2846 if (v
& FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
)
2847 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
2852 case ETHTOOL_GRXRINGS
:
2853 info
->data
= pi
->nqsets
;
2859 static const struct ethtool_ops cxgb_ethtool_ops
= {
2860 .get_settings
= get_settings
,
2861 .set_settings
= set_settings
,
2862 .get_drvinfo
= get_drvinfo
,
2863 .get_msglevel
= get_msglevel
,
2864 .set_msglevel
= set_msglevel
,
2865 .get_ringparam
= get_sge_param
,
2866 .set_ringparam
= set_sge_param
,
2867 .get_coalesce
= get_coalesce
,
2868 .set_coalesce
= set_coalesce
,
2869 .get_eeprom_len
= get_eeprom_len
,
2870 .get_eeprom
= get_eeprom
,
2871 .set_eeprom
= set_eeprom
,
2872 .get_pauseparam
= get_pauseparam
,
2873 .set_pauseparam
= set_pauseparam
,
2874 .get_link
= ethtool_op_get_link
,
2875 .get_strings
= get_strings
,
2876 .set_phys_id
= identify_port
,
2877 .nway_reset
= restart_autoneg
,
2878 .get_sset_count
= get_sset_count
,
2879 .get_ethtool_stats
= get_stats
,
2880 .get_regs_len
= get_regs_len
,
2881 .get_regs
= get_regs
,
2884 .get_rxnfc
= get_rxnfc
,
2885 .get_rxfh_indir_size
= get_rss_table_size
,
2886 .get_rxfh_indir
= get_rss_table
,
2887 .set_rxfh_indir
= set_rss_table
,
2888 .flash_device
= set_flash
,
2894 static ssize_t
mem_read(struct file
*file
, char __user
*buf
, size_t count
,
2898 loff_t avail
= file_inode(file
)->i_size
;
2899 unsigned int mem
= (uintptr_t)file
->private_data
& 3;
2900 struct adapter
*adap
= file
->private_data
- mem
;
2906 if (count
> avail
- pos
)
2907 count
= avail
- pos
;
2914 if ((mem
== MEM_MC
) || (mem
== MEM_MC1
))
2915 ret
= t4_mc_read(adap
, mem
% MEM_MC
, pos
, data
, NULL
);
2917 ret
= t4_edc_read(adap
, mem
, pos
, data
, NULL
);
2921 ofst
= pos
% sizeof(data
);
2922 len
= min(count
, sizeof(data
) - ofst
);
2923 if (copy_to_user(buf
, (u8
*)data
+ ofst
, len
))
2930 count
= pos
- *ppos
;
2935 static const struct file_operations mem_debugfs_fops
= {
2936 .owner
= THIS_MODULE
,
2937 .open
= simple_open
,
2939 .llseek
= default_llseek
,
2942 static void add_debugfs_mem(struct adapter
*adap
, const char *name
,
2943 unsigned int idx
, unsigned int size_mb
)
2947 de
= debugfs_create_file(name
, S_IRUSR
, adap
->debugfs_root
,
2948 (void *)adap
+ idx
, &mem_debugfs_fops
);
2949 if (de
&& de
->d_inode
)
2950 de
->d_inode
->i_size
= size_mb
<< 20;
2953 static int setup_debugfs(struct adapter
*adap
)
2958 if (IS_ERR_OR_NULL(adap
->debugfs_root
))
2961 i
= t4_read_reg(adap
, MA_TARGET_MEM_ENABLE
);
2962 if (i
& EDRAM0_ENABLE
) {
2963 size
= t4_read_reg(adap
, MA_EDRAM0_BAR
);
2964 add_debugfs_mem(adap
, "edc0", MEM_EDC0
, EDRAM_SIZE_GET(size
));
2966 if (i
& EDRAM1_ENABLE
) {
2967 size
= t4_read_reg(adap
, MA_EDRAM1_BAR
);
2968 add_debugfs_mem(adap
, "edc1", MEM_EDC1
, EDRAM_SIZE_GET(size
));
2970 if (is_t4(adap
->chip
)) {
2971 size
= t4_read_reg(adap
, MA_EXT_MEMORY_BAR
);
2972 if (i
& EXT_MEM_ENABLE
)
2973 add_debugfs_mem(adap
, "mc", MEM_MC
,
2974 EXT_MEM_SIZE_GET(size
));
2976 if (i
& EXT_MEM_ENABLE
) {
2977 size
= t4_read_reg(adap
, MA_EXT_MEMORY_BAR
);
2978 add_debugfs_mem(adap
, "mc0", MEM_MC0
,
2979 EXT_MEM_SIZE_GET(size
));
2981 if (i
& EXT_MEM1_ENABLE
) {
2982 size
= t4_read_reg(adap
, MA_EXT_MEMORY1_BAR
);
2983 add_debugfs_mem(adap
, "mc1", MEM_MC1
,
2984 EXT_MEM_SIZE_GET(size
));
2988 debugfs_create_file("l2t", S_IRUSR
, adap
->debugfs_root
, adap
,
2994 * upper-layer driver support
2998 * Allocate an active-open TID and set it to the supplied value.
3000 int cxgb4_alloc_atid(struct tid_info
*t
, void *data
)
3004 spin_lock_bh(&t
->atid_lock
);
3006 union aopen_entry
*p
= t
->afree
;
3008 atid
= (p
- t
->atid_tab
) + t
->atid_base
;
3013 spin_unlock_bh(&t
->atid_lock
);
3016 EXPORT_SYMBOL(cxgb4_alloc_atid
);
3019 * Release an active-open TID.
3021 void cxgb4_free_atid(struct tid_info
*t
, unsigned int atid
)
3023 union aopen_entry
*p
= &t
->atid_tab
[atid
- t
->atid_base
];
3025 spin_lock_bh(&t
->atid_lock
);
3029 spin_unlock_bh(&t
->atid_lock
);
3031 EXPORT_SYMBOL(cxgb4_free_atid
);
3034 * Allocate a server TID and set it to the supplied value.
3036 int cxgb4_alloc_stid(struct tid_info
*t
, int family
, void *data
)
3040 spin_lock_bh(&t
->stid_lock
);
3041 if (family
== PF_INET
) {
3042 stid
= find_first_zero_bit(t
->stid_bmap
, t
->nstids
);
3043 if (stid
< t
->nstids
)
3044 __set_bit(stid
, t
->stid_bmap
);
3048 stid
= bitmap_find_free_region(t
->stid_bmap
, t
->nstids
, 2);
3053 t
->stid_tab
[stid
].data
= data
;
3054 stid
+= t
->stid_base
;
3057 spin_unlock_bh(&t
->stid_lock
);
3060 EXPORT_SYMBOL(cxgb4_alloc_stid
);
3062 /* Allocate a server filter TID and set it to the supplied value.
3064 int cxgb4_alloc_sftid(struct tid_info
*t
, int family
, void *data
)
3068 spin_lock_bh(&t
->stid_lock
);
3069 if (family
== PF_INET
) {
3070 stid
= find_next_zero_bit(t
->stid_bmap
,
3071 t
->nstids
+ t
->nsftids
, t
->nstids
);
3072 if (stid
< (t
->nstids
+ t
->nsftids
))
3073 __set_bit(stid
, t
->stid_bmap
);
3080 t
->stid_tab
[stid
].data
= data
;
3081 stid
+= t
->stid_base
;
3084 spin_unlock_bh(&t
->stid_lock
);
3087 EXPORT_SYMBOL(cxgb4_alloc_sftid
);
3089 /* Release a server TID.
3091 void cxgb4_free_stid(struct tid_info
*t
, unsigned int stid
, int family
)
3093 stid
-= t
->stid_base
;
3094 spin_lock_bh(&t
->stid_lock
);
3095 if (family
== PF_INET
)
3096 __clear_bit(stid
, t
->stid_bmap
);
3098 bitmap_release_region(t
->stid_bmap
, stid
, 2);
3099 t
->stid_tab
[stid
].data
= NULL
;
3101 spin_unlock_bh(&t
->stid_lock
);
3103 EXPORT_SYMBOL(cxgb4_free_stid
);
3106 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3108 static void mk_tid_release(struct sk_buff
*skb
, unsigned int chan
,
3111 struct cpl_tid_release
*req
;
3113 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, chan
);
3114 req
= (struct cpl_tid_release
*)__skb_put(skb
, sizeof(*req
));
3115 INIT_TP_WR(req
, tid
);
3116 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE
, tid
));
3120 * Queue a TID release request and if necessary schedule a work queue to
3123 static void cxgb4_queue_tid_release(struct tid_info
*t
, unsigned int chan
,
3126 void **p
= &t
->tid_tab
[tid
];
3127 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
3129 spin_lock_bh(&adap
->tid_release_lock
);
3130 *p
= adap
->tid_release_head
;
3131 /* Low 2 bits encode the Tx channel number */
3132 adap
->tid_release_head
= (void **)((uintptr_t)p
| chan
);
3133 if (!adap
->tid_release_task_busy
) {
3134 adap
->tid_release_task_busy
= true;
3135 queue_work(workq
, &adap
->tid_release_task
);
3137 spin_unlock_bh(&adap
->tid_release_lock
);
3141 * Process the list of pending TID release requests.
3143 static void process_tid_release_list(struct work_struct
*work
)
3145 struct sk_buff
*skb
;
3146 struct adapter
*adap
;
3148 adap
= container_of(work
, struct adapter
, tid_release_task
);
3150 spin_lock_bh(&adap
->tid_release_lock
);
3151 while (adap
->tid_release_head
) {
3152 void **p
= adap
->tid_release_head
;
3153 unsigned int chan
= (uintptr_t)p
& 3;
3154 p
= (void *)p
- chan
;
3156 adap
->tid_release_head
= *p
;
3158 spin_unlock_bh(&adap
->tid_release_lock
);
3160 while (!(skb
= alloc_skb(sizeof(struct cpl_tid_release
),
3162 schedule_timeout_uninterruptible(1);
3164 mk_tid_release(skb
, chan
, p
- adap
->tids
.tid_tab
);
3165 t4_ofld_send(adap
, skb
);
3166 spin_lock_bh(&adap
->tid_release_lock
);
3168 adap
->tid_release_task_busy
= false;
3169 spin_unlock_bh(&adap
->tid_release_lock
);
3173 * Release a TID and inform HW. If we are unable to allocate the release
3174 * message we defer to a work queue.
3176 void cxgb4_remove_tid(struct tid_info
*t
, unsigned int chan
, unsigned int tid
)
3179 struct sk_buff
*skb
;
3180 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
3182 old
= t
->tid_tab
[tid
];
3183 skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_ATOMIC
);
3185 t
->tid_tab
[tid
] = NULL
;
3186 mk_tid_release(skb
, chan
, tid
);
3187 t4_ofld_send(adap
, skb
);
3189 cxgb4_queue_tid_release(t
, chan
, tid
);
3191 atomic_dec(&t
->tids_in_use
);
3193 EXPORT_SYMBOL(cxgb4_remove_tid
);
3196 * Allocate and initialize the TID tables. Returns 0 on success.
3198 static int tid_init(struct tid_info
*t
)
3201 unsigned int stid_bmap_size
;
3202 unsigned int natids
= t
->natids
;
3204 stid_bmap_size
= BITS_TO_LONGS(t
->nstids
+ t
->nsftids
);
3205 size
= t
->ntids
* sizeof(*t
->tid_tab
) +
3206 natids
* sizeof(*t
->atid_tab
) +
3207 t
->nstids
* sizeof(*t
->stid_tab
) +
3208 t
->nsftids
* sizeof(*t
->stid_tab
) +
3209 stid_bmap_size
* sizeof(long) +
3210 t
->nftids
* sizeof(*t
->ftid_tab
) +
3211 t
->nsftids
* sizeof(*t
->ftid_tab
);
3213 t
->tid_tab
= t4_alloc_mem(size
);
3217 t
->atid_tab
= (union aopen_entry
*)&t
->tid_tab
[t
->ntids
];
3218 t
->stid_tab
= (struct serv_entry
*)&t
->atid_tab
[natids
];
3219 t
->stid_bmap
= (unsigned long *)&t
->stid_tab
[t
->nstids
+ t
->nsftids
];
3220 t
->ftid_tab
= (struct filter_entry
*)&t
->stid_bmap
[stid_bmap_size
];
3221 spin_lock_init(&t
->stid_lock
);
3222 spin_lock_init(&t
->atid_lock
);
3224 t
->stids_in_use
= 0;
3226 t
->atids_in_use
= 0;
3227 atomic_set(&t
->tids_in_use
, 0);
3229 /* Setup the free list for atid_tab and clear the stid bitmap. */
3232 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
3233 t
->afree
= t
->atid_tab
;
3235 bitmap_zero(t
->stid_bmap
, t
->nstids
+ t
->nsftids
);
3239 static int cxgb4_clip_get(const struct net_device
*dev
,
3240 const struct in6_addr
*lip
)
3242 struct adapter
*adap
;
3243 struct fw_clip_cmd c
;
3245 adap
= netdev2adap(dev
);
3246 memset(&c
, 0, sizeof(c
));
3247 c
.op_to_write
= htonl(FW_CMD_OP(FW_CLIP_CMD
) |
3248 FW_CMD_REQUEST
| FW_CMD_WRITE
);
3249 c
.alloc_to_len16
= htonl(F_FW_CLIP_CMD_ALLOC
| FW_LEN16(c
));
3250 *(__be64
*)&c
.ip_hi
= *(__be64
*)(lip
->s6_addr
);
3251 *(__be64
*)&c
.ip_lo
= *(__be64
*)(lip
->s6_addr
+ 8);
3252 return t4_wr_mbox_meat(adap
, adap
->mbox
, &c
, sizeof(c
), &c
, false);
3255 static int cxgb4_clip_release(const struct net_device
*dev
,
3256 const struct in6_addr
*lip
)
3258 struct adapter
*adap
;
3259 struct fw_clip_cmd c
;
3261 adap
= netdev2adap(dev
);
3262 memset(&c
, 0, sizeof(c
));
3263 c
.op_to_write
= htonl(FW_CMD_OP(FW_CLIP_CMD
) |
3264 FW_CMD_REQUEST
| FW_CMD_READ
);
3265 c
.alloc_to_len16
= htonl(F_FW_CLIP_CMD_FREE
| FW_LEN16(c
));
3266 *(__be64
*)&c
.ip_hi
= *(__be64
*)(lip
->s6_addr
);
3267 *(__be64
*)&c
.ip_lo
= *(__be64
*)(lip
->s6_addr
+ 8);
3268 return t4_wr_mbox_meat(adap
, adap
->mbox
, &c
, sizeof(c
), &c
, false);
3272 * cxgb4_create_server - create an IP server
3274 * @stid: the server TID
3275 * @sip: local IP address to bind server to
3276 * @sport: the server's TCP port
3277 * @queue: queue to direct messages from this server to
3279 * Create an IP server for the given port and address.
3280 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3282 int cxgb4_create_server(const struct net_device
*dev
, unsigned int stid
,
3283 __be32 sip
, __be16 sport
, __be16 vlan
,
3287 struct sk_buff
*skb
;
3288 struct adapter
*adap
;
3289 struct cpl_pass_open_req
*req
;
3292 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3296 adap
= netdev2adap(dev
);
3297 req
= (struct cpl_pass_open_req
*)__skb_put(skb
, sizeof(*req
));
3299 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ
, stid
));
3300 req
->local_port
= sport
;
3301 req
->peer_port
= htons(0);
3302 req
->local_ip
= sip
;
3303 req
->peer_ip
= htonl(0);
3304 chan
= rxq_to_chan(&adap
->sge
, queue
);
3305 req
->opt0
= cpu_to_be64(TX_CHAN(chan
));
3306 req
->opt1
= cpu_to_be64(CONN_POLICY_ASK
|
3307 SYN_RSS_ENABLE
| SYN_RSS_QUEUE(queue
));
3308 ret
= t4_mgmt_tx(adap
, skb
);
3309 return net_xmit_eval(ret
);
3311 EXPORT_SYMBOL(cxgb4_create_server
);
3313 /* cxgb4_create_server6 - create an IPv6 server
3315 * @stid: the server TID
3316 * @sip: local IPv6 address to bind server to
3317 * @sport: the server's TCP port
3318 * @queue: queue to direct messages from this server to
3320 * Create an IPv6 server for the given port and address.
3321 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3323 int cxgb4_create_server6(const struct net_device
*dev
, unsigned int stid
,
3324 const struct in6_addr
*sip
, __be16 sport
,
3328 struct sk_buff
*skb
;
3329 struct adapter
*adap
;
3330 struct cpl_pass_open_req6
*req
;
3333 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3337 adap
= netdev2adap(dev
);
3338 req
= (struct cpl_pass_open_req6
*)__skb_put(skb
, sizeof(*req
));
3340 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6
, stid
));
3341 req
->local_port
= sport
;
3342 req
->peer_port
= htons(0);
3343 req
->local_ip_hi
= *(__be64
*)(sip
->s6_addr
);
3344 req
->local_ip_lo
= *(__be64
*)(sip
->s6_addr
+ 8);
3345 req
->peer_ip_hi
= cpu_to_be64(0);
3346 req
->peer_ip_lo
= cpu_to_be64(0);
3347 chan
= rxq_to_chan(&adap
->sge
, queue
);
3348 req
->opt0
= cpu_to_be64(TX_CHAN(chan
));
3349 req
->opt1
= cpu_to_be64(CONN_POLICY_ASK
|
3350 SYN_RSS_ENABLE
| SYN_RSS_QUEUE(queue
));
3351 ret
= t4_mgmt_tx(adap
, skb
);
3352 return net_xmit_eval(ret
);
3354 EXPORT_SYMBOL(cxgb4_create_server6
);
3356 int cxgb4_remove_server(const struct net_device
*dev
, unsigned int stid
,
3357 unsigned int queue
, bool ipv6
)
3359 struct sk_buff
*skb
;
3360 struct adapter
*adap
;
3361 struct cpl_close_listsvr_req
*req
;
3364 adap
= netdev2adap(dev
);
3366 skb
= alloc_skb(sizeof(*req
), GFP_KERNEL
);
3370 req
= (struct cpl_close_listsvr_req
*)__skb_put(skb
, sizeof(*req
));
3372 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ
, stid
));
3373 req
->reply_ctrl
= htons(NO_REPLY(0) | (ipv6
? LISTSVR_IPV6(1) :
3374 LISTSVR_IPV6(0)) | QUEUENO(queue
));
3375 ret
= t4_mgmt_tx(adap
, skb
);
3376 return net_xmit_eval(ret
);
3378 EXPORT_SYMBOL(cxgb4_remove_server
);
3381 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3382 * @mtus: the HW MTU table
3383 * @mtu: the target MTU
3384 * @idx: index of selected entry in the MTU table
3386 * Returns the index and the value in the HW MTU table that is closest to
3387 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3388 * table, in which case that smallest available value is selected.
3390 unsigned int cxgb4_best_mtu(const unsigned short *mtus
, unsigned short mtu
,
3395 while (i
< NMTUS
- 1 && mtus
[i
+ 1] <= mtu
)
3401 EXPORT_SYMBOL(cxgb4_best_mtu
);
3404 * cxgb4_port_chan - get the HW channel of a port
3405 * @dev: the net device for the port
3407 * Return the HW Tx channel of the given port.
3409 unsigned int cxgb4_port_chan(const struct net_device
*dev
)
3411 return netdev2pinfo(dev
)->tx_chan
;
3413 EXPORT_SYMBOL(cxgb4_port_chan
);
3415 unsigned int cxgb4_dbfifo_count(const struct net_device
*dev
, int lpfifo
)
3417 struct adapter
*adap
= netdev2adap(dev
);
3418 u32 v1
, v2
, lp_count
, hp_count
;
3420 v1
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
3421 v2
= t4_read_reg(adap
, SGE_DBFIFO_STATUS2
);
3422 if (is_t4(adap
->chip
)) {
3423 lp_count
= G_LP_COUNT(v1
);
3424 hp_count
= G_HP_COUNT(v1
);
3426 lp_count
= G_LP_COUNT_T5(v1
);
3427 hp_count
= G_HP_COUNT_T5(v2
);
3429 return lpfifo
? lp_count
: hp_count
;
3431 EXPORT_SYMBOL(cxgb4_dbfifo_count
);
3434 * cxgb4_port_viid - get the VI id of a port
3435 * @dev: the net device for the port
3437 * Return the VI id of the given port.
3439 unsigned int cxgb4_port_viid(const struct net_device
*dev
)
3441 return netdev2pinfo(dev
)->viid
;
3443 EXPORT_SYMBOL(cxgb4_port_viid
);
3446 * cxgb4_port_idx - get the index of a port
3447 * @dev: the net device for the port
3449 * Return the index of the given port.
3451 unsigned int cxgb4_port_idx(const struct net_device
*dev
)
3453 return netdev2pinfo(dev
)->port_id
;
3455 EXPORT_SYMBOL(cxgb4_port_idx
);
3457 void cxgb4_get_tcp_stats(struct pci_dev
*pdev
, struct tp_tcp_stats
*v4
,
3458 struct tp_tcp_stats
*v6
)
3460 struct adapter
*adap
= pci_get_drvdata(pdev
);
3462 spin_lock(&adap
->stats_lock
);
3463 t4_tp_get_tcp_stats(adap
, v4
, v6
);
3464 spin_unlock(&adap
->stats_lock
);
3466 EXPORT_SYMBOL(cxgb4_get_tcp_stats
);
3468 void cxgb4_iscsi_init(struct net_device
*dev
, unsigned int tag_mask
,
3469 const unsigned int *pgsz_order
)
3471 struct adapter
*adap
= netdev2adap(dev
);
3473 t4_write_reg(adap
, ULP_RX_ISCSI_TAGMASK
, tag_mask
);
3474 t4_write_reg(adap
, ULP_RX_ISCSI_PSZ
, HPZ0(pgsz_order
[0]) |
3475 HPZ1(pgsz_order
[1]) | HPZ2(pgsz_order
[2]) |
3476 HPZ3(pgsz_order
[3]));
3478 EXPORT_SYMBOL(cxgb4_iscsi_init
);
3480 int cxgb4_flush_eq_cache(struct net_device
*dev
)
3482 struct adapter
*adap
= netdev2adap(dev
);
3485 ret
= t4_fwaddrspace_write(adap
, adap
->mbox
,
3486 0xe1000000 + A_SGE_CTXT_CMD
, 0x20000000);
3489 EXPORT_SYMBOL(cxgb4_flush_eq_cache
);
3491 static int read_eq_indices(struct adapter
*adap
, u16 qid
, u16
*pidx
, u16
*cidx
)
3493 u32 addr
= t4_read_reg(adap
, A_SGE_DBQ_CTXT_BADDR
) + 24 * qid
+ 8;
3497 ret
= t4_mem_win_read_len(adap
, addr
, (__be32
*)&indices
, 8);
3499 *cidx
= (be64_to_cpu(indices
) >> 25) & 0xffff;
3500 *pidx
= (be64_to_cpu(indices
) >> 9) & 0xffff;
3505 int cxgb4_sync_txq_pidx(struct net_device
*dev
, u16 qid
, u16 pidx
,
3508 struct adapter
*adap
= netdev2adap(dev
);
3509 u16 hw_pidx
, hw_cidx
;
3512 ret
= read_eq_indices(adap
, qid
, &hw_pidx
, &hw_cidx
);
3516 if (pidx
!= hw_pidx
) {
3519 if (pidx
>= hw_pidx
)
3520 delta
= pidx
- hw_pidx
;
3522 delta
= size
- hw_pidx
+ pidx
;
3524 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
3525 QID(qid
) | PIDX(delta
));
3530 EXPORT_SYMBOL(cxgb4_sync_txq_pidx
);
3532 void cxgb4_disable_db_coalescing(struct net_device
*dev
)
3534 struct adapter
*adap
;
3536 adap
= netdev2adap(dev
);
3537 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_NOCOALESCE
,
3540 EXPORT_SYMBOL(cxgb4_disable_db_coalescing
);
3542 void cxgb4_enable_db_coalescing(struct net_device
*dev
)
3544 struct adapter
*adap
;
3546 adap
= netdev2adap(dev
);
3547 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_NOCOALESCE
, 0);
3549 EXPORT_SYMBOL(cxgb4_enable_db_coalescing
);
3551 static struct pci_driver cxgb4_driver
;
3553 static void check_neigh_update(struct neighbour
*neigh
)
3555 const struct device
*parent
;
3556 const struct net_device
*netdev
= neigh
->dev
;
3558 if (netdev
->priv_flags
& IFF_802_1Q_VLAN
)
3559 netdev
= vlan_dev_real_dev(netdev
);
3560 parent
= netdev
->dev
.parent
;
3561 if (parent
&& parent
->driver
== &cxgb4_driver
.driver
)
3562 t4_l2t_update(dev_get_drvdata(parent
), neigh
);
3565 static int netevent_cb(struct notifier_block
*nb
, unsigned long event
,
3569 case NETEVENT_NEIGH_UPDATE
:
3570 check_neigh_update(data
);
3572 case NETEVENT_REDIRECT
:
3579 static bool netevent_registered
;
3580 static struct notifier_block cxgb4_netevent_nb
= {
3581 .notifier_call
= netevent_cb
3584 static void drain_db_fifo(struct adapter
*adap
, int usecs
)
3586 u32 v1
, v2
, lp_count
, hp_count
;
3589 v1
= t4_read_reg(adap
, A_SGE_DBFIFO_STATUS
);
3590 v2
= t4_read_reg(adap
, SGE_DBFIFO_STATUS2
);
3591 if (is_t4(adap
->chip
)) {
3592 lp_count
= G_LP_COUNT(v1
);
3593 hp_count
= G_HP_COUNT(v1
);
3595 lp_count
= G_LP_COUNT_T5(v1
);
3596 hp_count
= G_HP_COUNT_T5(v2
);
3599 if (lp_count
== 0 && hp_count
== 0)
3601 set_current_state(TASK_UNINTERRUPTIBLE
);
3602 schedule_timeout(usecs_to_jiffies(usecs
));
3606 static void disable_txq_db(struct sge_txq
*q
)
3608 spin_lock_irq(&q
->db_lock
);
3610 spin_unlock_irq(&q
->db_lock
);
3613 static void enable_txq_db(struct sge_txq
*q
)
3615 spin_lock_irq(&q
->db_lock
);
3617 spin_unlock_irq(&q
->db_lock
);
3620 static void disable_dbs(struct adapter
*adap
)
3624 for_each_ethrxq(&adap
->sge
, i
)
3625 disable_txq_db(&adap
->sge
.ethtxq
[i
].q
);
3626 for_each_ofldrxq(&adap
->sge
, i
)
3627 disable_txq_db(&adap
->sge
.ofldtxq
[i
].q
);
3628 for_each_port(adap
, i
)
3629 disable_txq_db(&adap
->sge
.ctrlq
[i
].q
);
3632 static void enable_dbs(struct adapter
*adap
)
3636 for_each_ethrxq(&adap
->sge
, i
)
3637 enable_txq_db(&adap
->sge
.ethtxq
[i
].q
);
3638 for_each_ofldrxq(&adap
->sge
, i
)
3639 enable_txq_db(&adap
->sge
.ofldtxq
[i
].q
);
3640 for_each_port(adap
, i
)
3641 enable_txq_db(&adap
->sge
.ctrlq
[i
].q
);
3644 static void sync_txq_pidx(struct adapter
*adap
, struct sge_txq
*q
)
3646 u16 hw_pidx
, hw_cidx
;
3649 spin_lock_bh(&q
->db_lock
);
3650 ret
= read_eq_indices(adap
, (u16
)q
->cntxt_id
, &hw_pidx
, &hw_cidx
);
3653 if (q
->db_pidx
!= hw_pidx
) {
3656 if (q
->db_pidx
>= hw_pidx
)
3657 delta
= q
->db_pidx
- hw_pidx
;
3659 delta
= q
->size
- hw_pidx
+ q
->db_pidx
;
3661 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL
),
3662 QID(q
->cntxt_id
) | PIDX(delta
));
3666 spin_unlock_bh(&q
->db_lock
);
3668 CH_WARN(adap
, "DB drop recovery failed.\n");
3670 static void recover_all_queues(struct adapter
*adap
)
3674 for_each_ethrxq(&adap
->sge
, i
)
3675 sync_txq_pidx(adap
, &adap
->sge
.ethtxq
[i
].q
);
3676 for_each_ofldrxq(&adap
->sge
, i
)
3677 sync_txq_pidx(adap
, &adap
->sge
.ofldtxq
[i
].q
);
3678 for_each_port(adap
, i
)
3679 sync_txq_pidx(adap
, &adap
->sge
.ctrlq
[i
].q
);
3682 static void notify_rdma_uld(struct adapter
*adap
, enum cxgb4_control cmd
)
3684 mutex_lock(&uld_mutex
);
3685 if (adap
->uld_handle
[CXGB4_ULD_RDMA
])
3686 ulds
[CXGB4_ULD_RDMA
].control(adap
->uld_handle
[CXGB4_ULD_RDMA
],
3688 mutex_unlock(&uld_mutex
);
3691 static void process_db_full(struct work_struct
*work
)
3693 struct adapter
*adap
;
3695 adap
= container_of(work
, struct adapter
, db_full_task
);
3697 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_FULL
);
3698 drain_db_fifo(adap
, dbfifo_drain_delay
);
3699 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
3700 DBFIFO_HP_INT
| DBFIFO_LP_INT
,
3701 DBFIFO_HP_INT
| DBFIFO_LP_INT
);
3702 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_EMPTY
);
3705 static void process_db_drop(struct work_struct
*work
)
3707 struct adapter
*adap
;
3709 adap
= container_of(work
, struct adapter
, db_drop_task
);
3711 if (is_t4(adap
->chip
)) {
3713 notify_rdma_uld(adap
, CXGB4_CONTROL_DB_DROP
);
3714 drain_db_fifo(adap
, 1);
3715 recover_all_queues(adap
);
3718 u32 dropped_db
= t4_read_reg(adap
, 0x010ac);
3719 u16 qid
= (dropped_db
>> 15) & 0x1ffff;
3720 u16 pidx_inc
= dropped_db
& 0x1fff;
3722 unsigned short udb_density
;
3723 unsigned long qpshift
;
3727 dev_warn(adap
->pdev_dev
,
3728 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3730 (dropped_db
>> 14) & 1,
3731 (dropped_db
>> 13) & 1,
3734 drain_db_fifo(adap
, 1);
3736 s_qpp
= QUEUESPERPAGEPF1
* adap
->fn
;
3737 udb_density
= 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap
,
3738 SGE_EGRESS_QUEUES_PER_PAGE_PF
) >> s_qpp
);
3739 qpshift
= PAGE_SHIFT
- ilog2(udb_density
);
3740 udb
= qid
<< qpshift
;
3742 page
= udb
/ PAGE_SIZE
;
3743 udb
+= (qid
- (page
* udb_density
)) * 128;
3745 writel(PIDX(pidx_inc
), adap
->bar2
+ udb
+ 8);
3747 /* Re-enable BAR2 WC */
3748 t4_set_reg_field(adap
, 0x10b0, 1<<15, 1<<15);
3751 t4_set_reg_field(adap
, A_SGE_DOORBELL_CONTROL
, F_DROPPED_DB
, 0);
3754 void t4_db_full(struct adapter
*adap
)
3756 if (is_t4(adap
->chip
)) {
3757 t4_set_reg_field(adap
, SGE_INT_ENABLE3
,
3758 DBFIFO_HP_INT
| DBFIFO_LP_INT
, 0);
3759 queue_work(workq
, &adap
->db_full_task
);
3763 void t4_db_dropped(struct adapter
*adap
)
3765 if (is_t4(adap
->chip
))
3766 queue_work(workq
, &adap
->db_drop_task
);
3769 static void uld_attach(struct adapter
*adap
, unsigned int uld
)
3772 struct cxgb4_lld_info lli
;
3775 lli
.pdev
= adap
->pdev
;
3776 lli
.l2t
= adap
->l2t
;
3777 lli
.tids
= &adap
->tids
;
3778 lli
.ports
= adap
->port
;
3779 lli
.vr
= &adap
->vres
;
3780 lli
.mtus
= adap
->params
.mtus
;
3781 if (uld
== CXGB4_ULD_RDMA
) {
3782 lli
.rxq_ids
= adap
->sge
.rdma_rxq
;
3783 lli
.nrxq
= adap
->sge
.rdmaqs
;
3784 } else if (uld
== CXGB4_ULD_ISCSI
) {
3785 lli
.rxq_ids
= adap
->sge
.ofld_rxq
;
3786 lli
.nrxq
= adap
->sge
.ofldqsets
;
3788 lli
.ntxq
= adap
->sge
.ofldqsets
;
3789 lli
.nchan
= adap
->params
.nports
;
3790 lli
.nports
= adap
->params
.nports
;
3791 lli
.wr_cred
= adap
->params
.ofldq_wr_cred
;
3792 lli
.adapter_type
= adap
->params
.rev
;
3793 lli
.iscsi_iolen
= MAXRXDATA_GET(t4_read_reg(adap
, TP_PARA_REG2
));
3794 lli
.udb_density
= 1 << QUEUESPERPAGEPF0_GET(
3795 t4_read_reg(adap
, SGE_EGRESS_QUEUES_PER_PAGE_PF
) >>
3797 lli
.ucq_density
= 1 << QUEUESPERPAGEPF0_GET(
3798 t4_read_reg(adap
, SGE_INGRESS_QUEUES_PER_PAGE_PF
) >>
3800 lli
.filt_mode
= adap
->filter_mode
;
3801 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3802 for (i
= 0; i
< NCHAN
; i
++)
3804 lli
.gts_reg
= adap
->regs
+ MYPF_REG(SGE_PF_GTS
);
3805 lli
.db_reg
= adap
->regs
+ MYPF_REG(SGE_PF_KDOORBELL
);
3806 lli
.fw_vers
= adap
->params
.fw_vers
;
3807 lli
.dbfifo_int_thresh
= dbfifo_int_thresh
;
3808 lli
.sge_pktshift
= adap
->sge
.pktshift
;
3809 lli
.enable_fw_ofld_conn
= adap
->flags
& FW_OFLD_CONN
;
3811 handle
= ulds
[uld
].add(&lli
);
3812 if (IS_ERR(handle
)) {
3813 dev_warn(adap
->pdev_dev
,
3814 "could not attach to the %s driver, error %ld\n",
3815 uld_str
[uld
], PTR_ERR(handle
));
3819 adap
->uld_handle
[uld
] = handle
;
3821 if (!netevent_registered
) {
3822 register_netevent_notifier(&cxgb4_netevent_nb
);
3823 netevent_registered
= true;
3826 if (adap
->flags
& FULL_INIT_DONE
)
3827 ulds
[uld
].state_change(handle
, CXGB4_STATE_UP
);
3830 static void attach_ulds(struct adapter
*adap
)
3834 spin_lock(&adap_rcu_lock
);
3835 list_add_tail_rcu(&adap
->rcu_node
, &adap_rcu_list
);
3836 spin_unlock(&adap_rcu_lock
);
3838 mutex_lock(&uld_mutex
);
3839 list_add_tail(&adap
->list_node
, &adapter_list
);
3840 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
3842 uld_attach(adap
, i
);
3843 mutex_unlock(&uld_mutex
);
3846 static void detach_ulds(struct adapter
*adap
)
3850 mutex_lock(&uld_mutex
);
3851 list_del(&adap
->list_node
);
3852 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
3853 if (adap
->uld_handle
[i
]) {
3854 ulds
[i
].state_change(adap
->uld_handle
[i
],
3855 CXGB4_STATE_DETACH
);
3856 adap
->uld_handle
[i
] = NULL
;
3858 if (netevent_registered
&& list_empty(&adapter_list
)) {
3859 unregister_netevent_notifier(&cxgb4_netevent_nb
);
3860 netevent_registered
= false;
3862 mutex_unlock(&uld_mutex
);
3864 spin_lock(&adap_rcu_lock
);
3865 list_del_rcu(&adap
->rcu_node
);
3866 spin_unlock(&adap_rcu_lock
);
3869 static void notify_ulds(struct adapter
*adap
, enum cxgb4_state new_state
)
3873 mutex_lock(&uld_mutex
);
3874 for (i
= 0; i
< CXGB4_ULD_MAX
; i
++)
3875 if (adap
->uld_handle
[i
])
3876 ulds
[i
].state_change(adap
->uld_handle
[i
], new_state
);
3877 mutex_unlock(&uld_mutex
);
3881 * cxgb4_register_uld - register an upper-layer driver
3882 * @type: the ULD type
3883 * @p: the ULD methods
3885 * Registers an upper-layer driver with this driver and notifies the ULD
3886 * about any presently available devices that support its type. Returns
3887 * %-EBUSY if a ULD of the same type is already registered.
3889 int cxgb4_register_uld(enum cxgb4_uld type
, const struct cxgb4_uld_info
*p
)
3892 struct adapter
*adap
;
3894 if (type
>= CXGB4_ULD_MAX
)
3896 mutex_lock(&uld_mutex
);
3897 if (ulds
[type
].add
) {
3902 list_for_each_entry(adap
, &adapter_list
, list_node
)
3903 uld_attach(adap
, type
);
3904 out
: mutex_unlock(&uld_mutex
);
3907 EXPORT_SYMBOL(cxgb4_register_uld
);
3910 * cxgb4_unregister_uld - unregister an upper-layer driver
3911 * @type: the ULD type
3913 * Unregisters an existing upper-layer driver.
3915 int cxgb4_unregister_uld(enum cxgb4_uld type
)
3917 struct adapter
*adap
;
3919 if (type
>= CXGB4_ULD_MAX
)
3921 mutex_lock(&uld_mutex
);
3922 list_for_each_entry(adap
, &adapter_list
, list_node
)
3923 adap
->uld_handle
[type
] = NULL
;
3924 ulds
[type
].add
= NULL
;
3925 mutex_unlock(&uld_mutex
);
3928 EXPORT_SYMBOL(cxgb4_unregister_uld
);
3930 /* Check if netdev on which event is occured belongs to us or not. Return
3931 * suceess (1) if it belongs otherwise failure (0).
3933 static int cxgb4_netdev(struct net_device
*netdev
)
3935 struct adapter
*adap
;
3938 spin_lock(&adap_rcu_lock
);
3939 list_for_each_entry_rcu(adap
, &adap_rcu_list
, rcu_node
)
3940 for (i
= 0; i
< MAX_NPORTS
; i
++)
3941 if (adap
->port
[i
] == netdev
) {
3942 spin_unlock(&adap_rcu_lock
);
3945 spin_unlock(&adap_rcu_lock
);
3949 static int clip_add(struct net_device
*event_dev
, struct inet6_ifaddr
*ifa
,
3950 unsigned long event
)
3952 int ret
= NOTIFY_DONE
;
3955 if (cxgb4_netdev(event_dev
)) {
3958 ret
= cxgb4_clip_get(event_dev
,
3959 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
3967 cxgb4_clip_release(event_dev
,
3968 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
3979 static int cxgb4_inet6addr_handler(struct notifier_block
*this,
3980 unsigned long event
, void *data
)
3982 struct inet6_ifaddr
*ifa
= data
;
3983 struct net_device
*event_dev
;
3984 int ret
= NOTIFY_DONE
;
3985 struct bonding
*bond
= netdev_priv(ifa
->idev
->dev
);
3986 struct list_head
*iter
;
3987 struct slave
*slave
;
3988 struct pci_dev
*first_pdev
= NULL
;
3990 if (ifa
->idev
->dev
->priv_flags
& IFF_802_1Q_VLAN
) {
3991 event_dev
= vlan_dev_real_dev(ifa
->idev
->dev
);
3992 ret
= clip_add(event_dev
, ifa
, event
);
3993 } else if (ifa
->idev
->dev
->flags
& IFF_MASTER
) {
3994 /* It is possible that two different adapters are bonded in one
3995 * bond. We need to find such different adapters and add clip
3996 * in all of them only once.
3998 read_lock(&bond
->lock
);
3999 bond_for_each_slave(bond
, slave
, iter
) {
4001 ret
= clip_add(slave
->dev
, ifa
, event
);
4002 /* If clip_add is success then only initialize
4003 * first_pdev since it means it is our device
4005 if (ret
== NOTIFY_OK
)
4006 first_pdev
= to_pci_dev(
4007 slave
->dev
->dev
.parent
);
4008 } else if (first_pdev
!=
4009 to_pci_dev(slave
->dev
->dev
.parent
))
4010 ret
= clip_add(slave
->dev
, ifa
, event
);
4012 read_unlock(&bond
->lock
);
4014 ret
= clip_add(ifa
->idev
->dev
, ifa
, event
);
4019 static struct notifier_block cxgb4_inet6addr_notifier
= {
4020 .notifier_call
= cxgb4_inet6addr_handler
4023 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4024 * a physical device.
4025 * The physical device reference is needed to send the actul CLIP command.
4027 static int update_dev_clip(struct net_device
*root_dev
, struct net_device
*dev
)
4029 struct inet6_dev
*idev
= NULL
;
4030 struct inet6_ifaddr
*ifa
;
4033 idev
= __in6_dev_get(root_dev
);
4037 read_lock_bh(&idev
->lock
);
4038 list_for_each_entry(ifa
, &idev
->addr_list
, if_list
) {
4039 ret
= cxgb4_clip_get(dev
,
4040 (const struct in6_addr
*)ifa
->addr
.s6_addr
);
4044 read_unlock_bh(&idev
->lock
);
4049 static int update_root_dev_clip(struct net_device
*dev
)
4051 struct net_device
*root_dev
= NULL
;
4054 /* First populate the real net device's IPv6 addresses */
4055 ret
= update_dev_clip(dev
, dev
);
4059 /* Parse all bond and vlan devices layered on top of the physical dev */
4060 for (i
= 0; i
< VLAN_N_VID
; i
++) {
4061 root_dev
= __vlan_find_dev_deep(dev
, htons(ETH_P_8021Q
), i
);
4065 ret
= update_dev_clip(root_dev
, dev
);
4072 static void update_clip(const struct adapter
*adap
)
4075 struct net_device
*dev
;
4080 for (i
= 0; i
< MAX_NPORTS
; i
++) {
4081 dev
= adap
->port
[i
];
4085 ret
= update_root_dev_clip(dev
);
4094 * cxgb_up - enable the adapter
4095 * @adap: adapter being enabled
4097 * Called when the first port is enabled, this function performs the
4098 * actions necessary to make an adapter operational, such as completing
4099 * the initialization of HW modules, and enabling interrupts.
4101 * Must be called with the rtnl lock held.
4103 static int cxgb_up(struct adapter
*adap
)
4107 err
= setup_sge_queues(adap
);
4110 err
= setup_rss(adap
);
4114 if (adap
->flags
& USING_MSIX
) {
4115 name_msix_vecs(adap
);
4116 err
= request_irq(adap
->msix_info
[0].vec
, t4_nondata_intr
, 0,
4117 adap
->msix_info
[0].desc
, adap
);
4121 err
= request_msix_queue_irqs(adap
);
4123 free_irq(adap
->msix_info
[0].vec
, adap
);
4127 err
= request_irq(adap
->pdev
->irq
, t4_intr_handler(adap
),
4128 (adap
->flags
& USING_MSI
) ? 0 : IRQF_SHARED
,
4129 adap
->port
[0]->name
, adap
);
4135 t4_intr_enable(adap
);
4136 adap
->flags
|= FULL_INIT_DONE
;
4137 notify_ulds(adap
, CXGB4_STATE_UP
);
4142 dev_err(adap
->pdev_dev
, "request_irq failed, err %d\n", err
);
4144 t4_free_sge_resources(adap
);
4148 static void cxgb_down(struct adapter
*adapter
)
4150 t4_intr_disable(adapter
);
4151 cancel_work_sync(&adapter
->tid_release_task
);
4152 cancel_work_sync(&adapter
->db_full_task
);
4153 cancel_work_sync(&adapter
->db_drop_task
);
4154 adapter
->tid_release_task_busy
= false;
4155 adapter
->tid_release_head
= NULL
;
4157 if (adapter
->flags
& USING_MSIX
) {
4158 free_msix_queue_irqs(adapter
);
4159 free_irq(adapter
->msix_info
[0].vec
, adapter
);
4161 free_irq(adapter
->pdev
->irq
, adapter
);
4162 quiesce_rx(adapter
);
4163 t4_sge_stop(adapter
);
4164 t4_free_sge_resources(adapter
);
4165 adapter
->flags
&= ~FULL_INIT_DONE
;
4169 * net_device operations
4171 static int cxgb_open(struct net_device
*dev
)
4174 struct port_info
*pi
= netdev_priv(dev
);
4175 struct adapter
*adapter
= pi
->adapter
;
4177 netif_carrier_off(dev
);
4179 if (!(adapter
->flags
& FULL_INIT_DONE
)) {
4180 err
= cxgb_up(adapter
);
4185 err
= link_start(dev
);
4187 netif_tx_start_all_queues(dev
);
4191 static int cxgb_close(struct net_device
*dev
)
4193 struct port_info
*pi
= netdev_priv(dev
);
4194 struct adapter
*adapter
= pi
->adapter
;
4196 netif_tx_stop_all_queues(dev
);
4197 netif_carrier_off(dev
);
4198 return t4_enable_vi(adapter
, adapter
->fn
, pi
->viid
, false, false);
4201 /* Return an error number if the indicated filter isn't writable ...
4203 static int writable_filter(struct filter_entry
*f
)
4213 /* Delete the filter at the specified index (if valid). The checks for all
4214 * the common problems with doing this like the filter being locked, currently
4215 * pending in another operation, etc.
4217 static int delete_filter(struct adapter
*adapter
, unsigned int fidx
)
4219 struct filter_entry
*f
;
4222 if (fidx
>= adapter
->tids
.nftids
+ adapter
->tids
.nsftids
)
4225 f
= &adapter
->tids
.ftid_tab
[fidx
];
4226 ret
= writable_filter(f
);
4230 return del_filter_wr(adapter
, fidx
);
4235 int cxgb4_create_server_filter(const struct net_device
*dev
, unsigned int stid
,
4236 __be32 sip
, __be16 sport
, __be16 vlan
,
4237 unsigned int queue
, unsigned char port
, unsigned char mask
)
4240 struct filter_entry
*f
;
4241 struct adapter
*adap
;
4245 adap
= netdev2adap(dev
);
4247 /* Adjust stid to correct filter index */
4248 stid
-= adap
->tids
.nstids
;
4249 stid
+= adap
->tids
.nftids
;
4251 /* Check to make sure the filter requested is writable ...
4253 f
= &adap
->tids
.ftid_tab
[stid
];
4254 ret
= writable_filter(f
);
4258 /* Clear out any old resources being used by the filter before
4259 * we start constructing the new filter.
4262 clear_filter(adap
, f
);
4264 /* Clear out filter specifications */
4265 memset(&f
->fs
, 0, sizeof(struct ch_filter_specification
));
4266 f
->fs
.val
.lport
= cpu_to_be16(sport
);
4267 f
->fs
.mask
.lport
= ~0;
4269 if ((val
[0] | val
[1] | val
[2] | val
[3]) != 0) {
4270 for (i
= 0; i
< 4; i
++) {
4271 f
->fs
.val
.lip
[i
] = val
[i
];
4272 f
->fs
.mask
.lip
[i
] = ~0;
4274 if (adap
->filter_mode
& F_PORT
) {
4275 f
->fs
.val
.iport
= port
;
4276 f
->fs
.mask
.iport
= mask
;
4282 /* Mark filter as locked */
4286 ret
= set_filter_wr(adap
, stid
);
4288 clear_filter(adap
, f
);
4294 EXPORT_SYMBOL(cxgb4_create_server_filter
);
4296 int cxgb4_remove_server_filter(const struct net_device
*dev
, unsigned int stid
,
4297 unsigned int queue
, bool ipv6
)
4300 struct filter_entry
*f
;
4301 struct adapter
*adap
;
4303 adap
= netdev2adap(dev
);
4305 /* Adjust stid to correct filter index */
4306 stid
-= adap
->tids
.nstids
;
4307 stid
+= adap
->tids
.nftids
;
4309 f
= &adap
->tids
.ftid_tab
[stid
];
4310 /* Unlock the filter */
4313 ret
= delete_filter(adap
, stid
);
4319 EXPORT_SYMBOL(cxgb4_remove_server_filter
);
4321 static struct rtnl_link_stats64
*cxgb_get_stats(struct net_device
*dev
,
4322 struct rtnl_link_stats64
*ns
)
4324 struct port_stats stats
;
4325 struct port_info
*p
= netdev_priv(dev
);
4326 struct adapter
*adapter
= p
->adapter
;
4328 spin_lock(&adapter
->stats_lock
);
4329 t4_get_port_stats(adapter
, p
->tx_chan
, &stats
);
4330 spin_unlock(&adapter
->stats_lock
);
4332 ns
->tx_bytes
= stats
.tx_octets
;
4333 ns
->tx_packets
= stats
.tx_frames
;
4334 ns
->rx_bytes
= stats
.rx_octets
;
4335 ns
->rx_packets
= stats
.rx_frames
;
4336 ns
->multicast
= stats
.rx_mcast_frames
;
4338 /* detailed rx_errors */
4339 ns
->rx_length_errors
= stats
.rx_jabber
+ stats
.rx_too_long
+
4341 ns
->rx_over_errors
= 0;
4342 ns
->rx_crc_errors
= stats
.rx_fcs_err
;
4343 ns
->rx_frame_errors
= stats
.rx_symbol_err
;
4344 ns
->rx_fifo_errors
= stats
.rx_ovflow0
+ stats
.rx_ovflow1
+
4345 stats
.rx_ovflow2
+ stats
.rx_ovflow3
+
4346 stats
.rx_trunc0
+ stats
.rx_trunc1
+
4347 stats
.rx_trunc2
+ stats
.rx_trunc3
;
4348 ns
->rx_missed_errors
= 0;
4350 /* detailed tx_errors */
4351 ns
->tx_aborted_errors
= 0;
4352 ns
->tx_carrier_errors
= 0;
4353 ns
->tx_fifo_errors
= 0;
4354 ns
->tx_heartbeat_errors
= 0;
4355 ns
->tx_window_errors
= 0;
4357 ns
->tx_errors
= stats
.tx_error_frames
;
4358 ns
->rx_errors
= stats
.rx_symbol_err
+ stats
.rx_fcs_err
+
4359 ns
->rx_length_errors
+ stats
.rx_len_err
+ ns
->rx_fifo_errors
;
4363 static int cxgb_ioctl(struct net_device
*dev
, struct ifreq
*req
, int cmd
)
4366 int ret
= 0, prtad
, devad
;
4367 struct port_info
*pi
= netdev_priv(dev
);
4368 struct mii_ioctl_data
*data
= (struct mii_ioctl_data
*)&req
->ifr_data
;
4372 if (pi
->mdio_addr
< 0)
4374 data
->phy_id
= pi
->mdio_addr
;
4378 if (mdio_phy_id_is_c45(data
->phy_id
)) {
4379 prtad
= mdio_phy_id_prtad(data
->phy_id
);
4380 devad
= mdio_phy_id_devad(data
->phy_id
);
4381 } else if (data
->phy_id
< 32) {
4382 prtad
= data
->phy_id
;
4384 data
->reg_num
&= 0x1f;
4388 mbox
= pi
->adapter
->fn
;
4389 if (cmd
== SIOCGMIIREG
)
4390 ret
= t4_mdio_rd(pi
->adapter
, mbox
, prtad
, devad
,
4391 data
->reg_num
, &data
->val_out
);
4393 ret
= t4_mdio_wr(pi
->adapter
, mbox
, prtad
, devad
,
4394 data
->reg_num
, data
->val_in
);
4402 static void cxgb_set_rxmode(struct net_device
*dev
)
4404 /* unfortunately we can't return errors to the stack */
4405 set_rxmode(dev
, -1, false);
4408 static int cxgb_change_mtu(struct net_device
*dev
, int new_mtu
)
4411 struct port_info
*pi
= netdev_priv(dev
);
4413 if (new_mtu
< 81 || new_mtu
> MAX_MTU
) /* accommodate SACK */
4415 ret
= t4_set_rxmode(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
, new_mtu
, -1,
4422 static int cxgb_set_mac_addr(struct net_device
*dev
, void *p
)
4425 struct sockaddr
*addr
= p
;
4426 struct port_info
*pi
= netdev_priv(dev
);
4428 if (!is_valid_ether_addr(addr
->sa_data
))
4429 return -EADDRNOTAVAIL
;
4431 ret
= t4_change_mac(pi
->adapter
, pi
->adapter
->fn
, pi
->viid
,
4432 pi
->xact_addr_filt
, addr
->sa_data
, true, true);
4436 memcpy(dev
->dev_addr
, addr
->sa_data
, dev
->addr_len
);
4437 pi
->xact_addr_filt
= ret
;
4441 #ifdef CONFIG_NET_POLL_CONTROLLER
4442 static void cxgb_netpoll(struct net_device
*dev
)
4444 struct port_info
*pi
= netdev_priv(dev
);
4445 struct adapter
*adap
= pi
->adapter
;
4447 if (adap
->flags
& USING_MSIX
) {
4449 struct sge_eth_rxq
*rx
= &adap
->sge
.ethrxq
[pi
->first_qset
];
4451 for (i
= pi
->nqsets
; i
; i
--, rx
++)
4452 t4_sge_intr_msix(0, &rx
->rspq
);
4454 t4_intr_handler(adap
)(0, adap
);
4458 static const struct net_device_ops cxgb4_netdev_ops
= {
4459 .ndo_open
= cxgb_open
,
4460 .ndo_stop
= cxgb_close
,
4461 .ndo_start_xmit
= t4_eth_xmit
,
4462 .ndo_get_stats64
= cxgb_get_stats
,
4463 .ndo_set_rx_mode
= cxgb_set_rxmode
,
4464 .ndo_set_mac_address
= cxgb_set_mac_addr
,
4465 .ndo_set_features
= cxgb_set_features
,
4466 .ndo_validate_addr
= eth_validate_addr
,
4467 .ndo_do_ioctl
= cxgb_ioctl
,
4468 .ndo_change_mtu
= cxgb_change_mtu
,
4469 #ifdef CONFIG_NET_POLL_CONTROLLER
4470 .ndo_poll_controller
= cxgb_netpoll
,
4474 void t4_fatal_err(struct adapter
*adap
)
4476 t4_set_reg_field(adap
, SGE_CONTROL
, GLOBALENABLE
, 0);
4477 t4_intr_disable(adap
);
4478 dev_alert(adap
->pdev_dev
, "encountered fatal error, adapter stopped\n");
4481 static void setup_memwin(struct adapter
*adap
)
4483 u32 bar0
, mem_win0_base
, mem_win1_base
, mem_win2_base
;
4485 bar0
= pci_resource_start(adap
->pdev
, 0); /* truncation intentional */
4486 if (is_t4(adap
->chip
)) {
4487 mem_win0_base
= bar0
+ MEMWIN0_BASE
;
4488 mem_win1_base
= bar0
+ MEMWIN1_BASE
;
4489 mem_win2_base
= bar0
+ MEMWIN2_BASE
;
4491 /* For T5, only relative offset inside the PCIe BAR is passed */
4492 mem_win0_base
= MEMWIN0_BASE
;
4493 mem_win1_base
= MEMWIN1_BASE_T5
;
4494 mem_win2_base
= MEMWIN2_BASE_T5
;
4496 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 0),
4497 mem_win0_base
| BIR(0) |
4498 WINDOW(ilog2(MEMWIN0_APERTURE
) - 10));
4499 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 1),
4500 mem_win1_base
| BIR(0) |
4501 WINDOW(ilog2(MEMWIN1_APERTURE
) - 10));
4502 t4_write_reg(adap
, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 2),
4503 mem_win2_base
| BIR(0) |
4504 WINDOW(ilog2(MEMWIN2_APERTURE
) - 10));
4507 static void setup_memwin_rdma(struct adapter
*adap
)
4509 if (adap
->vres
.ocq
.size
) {
4510 unsigned int start
, sz_kb
;
4512 start
= pci_resource_start(adap
->pdev
, 2) +
4513 OCQ_WIN_OFFSET(adap
->pdev
, &adap
->vres
);
4514 sz_kb
= roundup_pow_of_two(adap
->vres
.ocq
.size
) >> 10;
4516 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN
, 3),
4517 start
| BIR(1) | WINDOW(ilog2(sz_kb
)));
4519 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3),
4520 adap
->vres
.ocq
.start
);
4522 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET
, 3));
4526 static int adap_init1(struct adapter
*adap
, struct fw_caps_config_cmd
*c
)
4531 /* get device capabilities */
4532 memset(c
, 0, sizeof(*c
));
4533 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4534 FW_CMD_REQUEST
| FW_CMD_READ
);
4535 c
->cfvalid_to_len16
= htonl(FW_LEN16(*c
));
4536 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), c
);
4540 /* select capabilities we'll be using */
4541 if (c
->niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
4543 c
->niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
4545 c
->niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
4546 } else if (vf_acls
) {
4547 dev_err(adap
->pdev_dev
, "virtualization ACLs not supported");
4550 c
->op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4551 FW_CMD_REQUEST
| FW_CMD_WRITE
);
4552 ret
= t4_wr_mbox(adap
, adap
->fn
, c
, sizeof(*c
), NULL
);
4556 ret
= t4_config_glbl_rss(adap
, adap
->fn
,
4557 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
4558 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
4559 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
);
4563 ret
= t4_cfg_pfvf(adap
, adap
->fn
, adap
->fn
, 0, MAX_EGRQ
, 64, MAX_INGQ
,
4564 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF
, FW_CMD_CAP_PF
);
4570 /* tweak some settings */
4571 t4_write_reg(adap
, TP_SHIFT_CNT
, 0x64f8849);
4572 t4_write_reg(adap
, ULP_RX_TDDP_PSZ
, HPZ0(PAGE_SHIFT
- 12));
4573 t4_write_reg(adap
, TP_PIO_ADDR
, TP_INGRESS_CONFIG
);
4574 v
= t4_read_reg(adap
, TP_PIO_DATA
);
4575 t4_write_reg(adap
, TP_PIO_DATA
, v
& ~CSUM_HAS_PSEUDO_HDR
);
4577 /* first 4 Tx modulation queues point to consecutive Tx channels */
4578 adap
->params
.tp
.tx_modq_map
= 0xE4;
4579 t4_write_reg(adap
, A_TP_TX_MOD_QUEUE_REQ_MAP
,
4580 V_TX_MOD_QUEUE_REQ_MAP(adap
->params
.tp
.tx_modq_map
));
4582 /* associate each Tx modulation queue with consecutive Tx channels */
4584 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
4585 &v
, 1, A_TP_TX_SCHED_HDR
);
4586 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
4587 &v
, 1, A_TP_TX_SCHED_FIFO
);
4588 t4_write_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
4589 &v
, 1, A_TP_TX_SCHED_PCMD
);
4591 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4592 if (is_offload(adap
)) {
4593 t4_write_reg(adap
, A_TP_TX_MOD_QUEUE_WEIGHT0
,
4594 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4595 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4596 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4597 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT
));
4598 t4_write_reg(adap
, A_TP_TX_MOD_CHANNEL_WEIGHT
,
4599 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4600 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4601 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT
) |
4602 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT
));
4605 /* get basic stuff going */
4606 return t4_early_init(adap
, adap
->fn
);
4610 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4612 #define MAX_ATIDS 8192U
4615 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4617 * If the firmware we're dealing with has Configuration File support, then
4618 * we use that to perform all configuration
4622 * Tweak configuration based on module parameters, etc. Most of these have
4623 * defaults assigned to them by Firmware Configuration Files (if we're using
4624 * them) but need to be explicitly set if we're using hard-coded
4625 * initialization. But even in the case of using Firmware Configuration
4626 * Files, we'd like to expose the ability to change these via module
4627 * parameters so these are essentially common tweaks/settings for
4628 * Configuration Files and hard-coded initialization ...
4630 static int adap_init0_tweaks(struct adapter
*adapter
)
4633 * Fix up various Host-Dependent Parameters like Page Size, Cache
4634 * Line Size, etc. The firmware default is for a 4KB Page Size and
4635 * 64B Cache Line Size ...
4637 t4_fixup_host_params(adapter
, PAGE_SIZE
, L1_CACHE_BYTES
);
4640 * Process module parameters which affect early initialization.
4642 if (rx_dma_offset
!= 2 && rx_dma_offset
!= 0) {
4643 dev_err(&adapter
->pdev
->dev
,
4644 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4648 t4_set_reg_field(adapter
, SGE_CONTROL
,
4650 PKTSHIFT(rx_dma_offset
));
4653 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4654 * adds the pseudo header itself.
4656 t4_tp_wr_bits_indirect(adapter
, TP_INGRESS_CONFIG
,
4657 CSUM_HAS_PSEUDO_HDR
, 0);
4663 * Attempt to initialize the adapter via a Firmware Configuration File.
4665 static int adap_init0_config(struct adapter
*adapter
, int reset
)
4667 struct fw_caps_config_cmd caps_cmd
;
4668 const struct firmware
*cf
;
4669 unsigned long mtype
= 0, maddr
= 0;
4670 u32 finiver
, finicsum
, cfcsum
;
4671 int ret
, using_flash
;
4672 char *fw_config_file
, fw_config_file_path
[256];
4675 * Reset device if necessary.
4678 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
4679 PIORSTMODE
| PIORST
);
4685 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4686 * then use that. Otherwise, use the configuration file stored
4687 * in the adapter flash ...
4689 switch (CHELSIO_CHIP_VERSION(adapter
->chip
)) {
4691 fw_config_file
= FW_CFNAME
;
4694 fw_config_file
= FW5_CFNAME
;
4697 dev_err(adapter
->pdev_dev
, "Device %d is not supported\n",
4698 adapter
->pdev
->device
);
4703 ret
= request_firmware(&cf
, fw_config_file
, adapter
->pdev_dev
);
4706 mtype
= FW_MEMTYPE_CF_FLASH
;
4707 maddr
= t4_flash_cfg_addr(adapter
);
4709 u32 params
[7], val
[7];
4712 if (cf
->size
>= FLASH_CFG_MAX_SIZE
)
4715 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
4716 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
4717 ret
= t4_query_params(adapter
, adapter
->mbox
,
4718 adapter
->fn
, 0, 1, params
, val
);
4721 * For t4_memory_write() below addresses and
4722 * sizes have to be in terms of multiples of 4
4723 * bytes. So, if the Configuration File isn't
4724 * a multiple of 4 bytes in length we'll have
4725 * to write that out separately since we can't
4726 * guarantee that the bytes following the
4727 * residual byte in the buffer returned by
4728 * request_firmware() are zeroed out ...
4730 size_t resid
= cf
->size
& 0x3;
4731 size_t size
= cf
->size
& ~0x3;
4732 __be32
*data
= (__be32
*)cf
->data
;
4734 mtype
= FW_PARAMS_PARAM_Y_GET(val
[0]);
4735 maddr
= FW_PARAMS_PARAM_Z_GET(val
[0]) << 16;
4737 ret
= t4_memory_write(adapter
, mtype
, maddr
,
4739 if (ret
== 0 && resid
!= 0) {
4746 last
.word
= data
[size
>> 2];
4747 for (i
= resid
; i
< 4; i
++)
4749 ret
= t4_memory_write(adapter
, mtype
,
4756 release_firmware(cf
);
4762 * Issue a Capability Configuration command to the firmware to get it
4763 * to parse the Configuration File. We don't use t4_fw_config_file()
4764 * because we want the ability to modify various features after we've
4765 * processed the configuration file ...
4767 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
4768 caps_cmd
.op_to_write
=
4769 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4772 caps_cmd
.cfvalid_to_len16
=
4773 htonl(FW_CAPS_CONFIG_CMD_CFVALID
|
4774 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype
) |
4775 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr
>> 16) |
4776 FW_LEN16(caps_cmd
));
4777 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
4782 finiver
= ntohl(caps_cmd
.finiver
);
4783 finicsum
= ntohl(caps_cmd
.finicsum
);
4784 cfcsum
= ntohl(caps_cmd
.cfcsum
);
4785 if (finicsum
!= cfcsum
)
4786 dev_warn(adapter
->pdev_dev
, "Configuration File checksum "\
4787 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4791 * And now tell the firmware to use the configuration we just loaded.
4793 caps_cmd
.op_to_write
=
4794 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4797 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
4798 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
4804 * Tweak configuration based on system architecture, module
4807 ret
= adap_init0_tweaks(adapter
);
4812 * And finally tell the firmware to initialize itself using the
4813 * parameters from the Configuration File.
4815 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
4819 sprintf(fw_config_file_path
, "/lib/firmware/%s", fw_config_file
);
4821 * Return successfully and note that we're operating with parameters
4822 * not supplied by the driver, rather than from hard-wired
4823 * initialization constants burried in the driver.
4825 adapter
->flags
|= USING_SOFT_PARAMS
;
4826 dev_info(adapter
->pdev_dev
, "Successfully configured using Firmware "\
4827 "Configuration File %s, version %#x, computed checksum %#x\n",
4830 : fw_config_file_path
),
4835 * Something bad happened. Return the error ... (If the "error"
4836 * is that there's no Configuration File on the adapter we don't
4837 * want to issue a warning since this is fairly common.)
4841 dev_warn(adapter
->pdev_dev
, "Configuration file error %d\n",
4847 * Attempt to initialize the adapter via hard-coded, driver supplied
4850 static int adap_init0_no_config(struct adapter
*adapter
, int reset
)
4852 struct sge
*s
= &adapter
->sge
;
4853 struct fw_caps_config_cmd caps_cmd
;
4858 * Reset device if necessary
4861 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
4862 PIORSTMODE
| PIORST
);
4868 * Get device capabilities and select which we'll be using.
4870 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
4871 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4872 FW_CMD_REQUEST
| FW_CMD_READ
);
4873 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
4874 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
4879 if (caps_cmd
.niccaps
& htons(FW_CAPS_CONFIG_NIC_VM
)) {
4881 caps_cmd
.niccaps
^= htons(FW_CAPS_CONFIG_NIC_VM
);
4883 caps_cmd
.niccaps
= htons(FW_CAPS_CONFIG_NIC_VM
);
4884 } else if (vf_acls
) {
4885 dev_err(adapter
->pdev_dev
, "virtualization ACLs not supported");
4888 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
4889 FW_CMD_REQUEST
| FW_CMD_WRITE
);
4890 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
4896 * Tweak configuration based on system architecture, module
4899 ret
= adap_init0_tweaks(adapter
);
4904 * Select RSS Global Mode we want to use. We use "Basic Virtual"
4905 * mode which maps each Virtual Interface to its own section of
4906 * the RSS Table and we turn on all map and hash enables ...
4908 adapter
->flags
|= RSS_TNLALLLOOKUP
;
4909 ret
= t4_config_glbl_rss(adapter
, adapter
->mbox
,
4910 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL
,
4911 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN
|
4912 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ
|
4913 ((adapter
->flags
& RSS_TNLALLLOOKUP
) ?
4914 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP
: 0));
4919 * Set up our own fundamental resource provisioning ...
4921 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
, adapter
->fn
, 0,
4922 PFRES_NEQ
, PFRES_NETHCTRL
,
4923 PFRES_NIQFLINT
, PFRES_NIQ
,
4924 PFRES_TC
, PFRES_NVI
,
4925 FW_PFVF_CMD_CMASK_MASK
,
4926 pfvfres_pmask(adapter
, adapter
->fn
, 0),
4928 PFRES_R_CAPS
, PFRES_WX_CAPS
);
4933 * Perform low level SGE initialization. We need to do this before we
4934 * send the firmware the INITIALIZE command because that will cause
4935 * any other PF Drivers which are waiting for the Master
4936 * Initialization to proceed forward.
4938 for (i
= 0; i
< SGE_NTIMERS
- 1; i
++)
4939 s
->timer_val
[i
] = min(intr_holdoff
[i
], MAX_SGE_TIMERVAL
);
4940 s
->timer_val
[SGE_NTIMERS
- 1] = MAX_SGE_TIMERVAL
;
4941 s
->counter_val
[0] = 1;
4942 for (i
= 1; i
< SGE_NCOUNTERS
; i
++)
4943 s
->counter_val
[i
] = min(intr_cnt
[i
- 1],
4944 THRESHOLD_0_GET(THRESHOLD_0_MASK
));
4945 t4_sge_init(adapter
);
4947 #ifdef CONFIG_PCI_IOV
4949 * Provision resource limits for Virtual Functions. We currently
4950 * grant them all the same static resource limits except for the Port
4951 * Access Rights Mask which we're assigning based on the PF. All of
4952 * the static provisioning stuff for both the PF and VF really needs
4953 * to be managed in a persistent manner for each device which the
4954 * firmware controls.
4959 for (pf
= 0; pf
< ARRAY_SIZE(num_vf
); pf
++) {
4960 if (num_vf
[pf
] <= 0)
4963 /* VF numbering starts at 1! */
4964 for (vf
= 1; vf
<= num_vf
[pf
]; vf
++) {
4965 ret
= t4_cfg_pfvf(adapter
, adapter
->mbox
,
4967 VFRES_NEQ
, VFRES_NETHCTRL
,
4968 VFRES_NIQFLINT
, VFRES_NIQ
,
4969 VFRES_TC
, VFRES_NVI
,
4970 FW_PFVF_CMD_CMASK_MASK
,
4974 VFRES_R_CAPS
, VFRES_WX_CAPS
);
4976 dev_warn(adapter
->pdev_dev
,
4978 "provision pf/vf=%d/%d; "
4979 "err=%d\n", pf
, vf
, ret
);
4986 * Set up the default filter mode. Later we'll want to implement this
4987 * via a firmware command, etc. ... This needs to be done before the
4988 * firmare initialization command ... If the selected set of fields
4989 * isn't equal to the default value, we'll need to make sure that the
4990 * field selections will fit in the 36-bit budget.
4992 if (tp_vlan_pri_map
!= TP_VLAN_PRI_MAP_DEFAULT
) {
4995 for (j
= TP_VLAN_PRI_MAP_FIRST
; j
<= TP_VLAN_PRI_MAP_LAST
; j
++)
4996 switch (tp_vlan_pri_map
& (1 << j
)) {
4998 /* compressed filter field not enabled */
5018 case ETHERTYPE_MASK
:
5024 case MPSHITTYPE_MASK
:
5027 case FRAGMENTATION_MASK
:
5033 dev_err(adapter
->pdev_dev
,
5034 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5035 " using %#x\n", tp_vlan_pri_map
, bits
,
5036 TP_VLAN_PRI_MAP_DEFAULT
);
5037 tp_vlan_pri_map
= TP_VLAN_PRI_MAP_DEFAULT
;
5040 v
= tp_vlan_pri_map
;
5041 t4_write_indirect(adapter
, TP_PIO_ADDR
, TP_PIO_DATA
,
5042 &v
, 1, TP_VLAN_PRI_MAP
);
5045 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5046 * to support any of the compressed filter fields above. Newer
5047 * versions of the firmware do this automatically but it doesn't hurt
5048 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5049 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5050 * since the firmware automatically turns this on and off when we have
5051 * a non-zero number of filters active (since it does have a
5052 * performance impact).
5054 if (tp_vlan_pri_map
)
5055 t4_set_reg_field(adapter
, TP_GLOBAL_CONFIG
,
5056 FIVETUPLELOOKUP_MASK
,
5057 FIVETUPLELOOKUP_MASK
);
5060 * Tweak some settings.
5062 t4_write_reg(adapter
, TP_SHIFT_CNT
, SYNSHIFTMAX(6) |
5063 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5064 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5065 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5068 * Get basic stuff going by issuing the Firmware Initialize command.
5069 * Note that this _must_ be after all PFVF commands ...
5071 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
5076 * Return successfully!
5078 dev_info(adapter
->pdev_dev
, "Successfully configured using built-in "\
5079 "driver parameters\n");
5083 * Something bad happened. Return the error ...
5090 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5092 static int adap_init0(struct adapter
*adap
)
5096 enum dev_state state
;
5097 u32 params
[7], val
[7];
5098 struct fw_caps_config_cmd caps_cmd
;
5102 * Contact FW, advertising Master capability (and potentially forcing
5103 * ourselves as the Master PF if our module parameter force_init is
5106 ret
= t4_fw_hello(adap
, adap
->mbox
, adap
->fn
,
5107 force_init
? MASTER_MUST
: MASTER_MAY
,
5110 dev_err(adap
->pdev_dev
, "could not connect to FW, error %d\n",
5114 if (ret
== adap
->mbox
)
5115 adap
->flags
|= MASTER_PF
;
5116 if (force_init
&& state
== DEV_STATE_INIT
)
5117 state
= DEV_STATE_UNINIT
;
5120 * If we're the Master PF Driver and the device is uninitialized,
5121 * then let's consider upgrading the firmware ... (We always want
5122 * to check the firmware version number in order to A. get it for
5123 * later reporting and B. to warn if the currently loaded firmware
5124 * is excessively mismatched relative to the driver.)
5126 ret
= t4_check_fw_version(adap
);
5128 /* The error code -EFAULT is returned by t4_check_fw_version() if
5129 * firmware on adapter < supported firmware. If firmware on adapter
5130 * is too old (not supported by driver) and we're the MASTER_PF set
5131 * adapter state to DEV_STATE_UNINIT to force firmware upgrade
5132 * and reinitialization.
5134 if ((adap
->flags
& MASTER_PF
) && ret
== -EFAULT
)
5135 state
= DEV_STATE_UNINIT
;
5136 if ((adap
->flags
& MASTER_PF
) && state
!= DEV_STATE_INIT
) {
5137 if (ret
== -EINVAL
|| ret
== -EFAULT
|| ret
> 0) {
5138 if (upgrade_fw(adap
) >= 0) {
5140 * Note that the chip was reset as part of the
5141 * firmware upgrade so we don't reset it again
5142 * below and grab the new firmware version.
5145 ret
= t4_check_fw_version(adap
);
5147 if (ret
== -EFAULT
) {
5149 * Firmware is old but still might
5150 * work if we force reinitialization
5151 * of the adapter. Ignoring FW upgrade
5154 dev_warn(adap
->pdev_dev
,
5155 "Ignoring firmware upgrade "
5156 "failure, and forcing driver "
5157 "to reinitialize the "
5167 * Grab VPD parameters. This should be done after we establish a
5168 * connection to the firmware since some of the VPD parameters
5169 * (notably the Core Clock frequency) are retrieved via requests to
5170 * the firmware. On the other hand, we need these fairly early on
5171 * so we do this right after getting ahold of the firmware.
5173 ret
= get_vpd_params(adap
, &adap
->params
.vpd
);
5178 * Find out what ports are available to us. Note that we need to do
5179 * this before calling adap_init0_no_config() since it needs nports
5183 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
5184 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC
);
5185 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1, &v
, &port_vec
);
5189 adap
->params
.nports
= hweight32(port_vec
);
5190 adap
->params
.portvec
= port_vec
;
5193 * If the firmware is initialized already (and we're not forcing a
5194 * master initialization), note that we're living with existing
5195 * adapter parameters. Otherwise, it's time to try initializing the
5198 if (state
== DEV_STATE_INIT
) {
5199 dev_info(adap
->pdev_dev
, "Coming up as %s: "\
5200 "Adapter already initialized\n",
5201 adap
->flags
& MASTER_PF
? "MASTER" : "SLAVE");
5202 adap
->flags
|= USING_SOFT_PARAMS
;
5204 dev_info(adap
->pdev_dev
, "Coming up as MASTER: "\
5205 "Initializing adapter\n");
5208 * If the firmware doesn't support Configuration
5209 * Files warn user and exit,
5212 dev_warn(adap
->pdev_dev
, "Firmware doesn't support "
5213 "configuration file.\n");
5215 ret
= adap_init0_no_config(adap
, reset
);
5218 * Find out whether we're dealing with a version of
5219 * the firmware which has configuration file support.
5221 params
[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
5222 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF
));
5223 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 1,
5227 * If the firmware doesn't support Configuration
5228 * Files, use the old Driver-based, hard-wired
5229 * initialization. Otherwise, try using the
5230 * Configuration File support and fall back to the
5231 * Driver-based initialization if there's no
5232 * Configuration File found.
5235 ret
= adap_init0_no_config(adap
, reset
);
5238 * The firmware provides us with a memory
5239 * buffer where we can load a Configuration
5240 * File from the host if we want to override
5241 * the Configuration File in flash.
5244 ret
= adap_init0_config(adap
, reset
);
5245 if (ret
== -ENOENT
) {
5246 dev_info(adap
->pdev_dev
,
5247 "No Configuration File present "
5248 "on adapter. Using hard-wired "
5249 "configuration parameters.\n");
5250 ret
= adap_init0_no_config(adap
, reset
);
5255 dev_err(adap
->pdev_dev
,
5256 "could not initialize adapter, error %d\n",
5263 * If we're living with non-hard-coded parameters (either from a
5264 * Firmware Configuration File or values programmed by a different PF
5265 * Driver), give the SGE code a chance to pull in anything that it
5266 * needs ... Note that this must be called after we retrieve our VPD
5267 * parameters in order to know how to convert core ticks to seconds.
5269 if (adap
->flags
& USING_SOFT_PARAMS
) {
5270 ret
= t4_sge_init(adap
);
5275 if (is_bypass_device(adap
->pdev
->device
))
5276 adap
->params
.bypass
= 1;
5279 * Grab some of our basic fundamental operating parameters.
5281 #define FW_PARAM_DEV(param) \
5282 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5283 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5285 #define FW_PARAM_PFVF(param) \
5286 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5287 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5288 FW_PARAMS_PARAM_Y(0) | \
5289 FW_PARAMS_PARAM_Z(0)
5291 params
[0] = FW_PARAM_PFVF(EQ_START
);
5292 params
[1] = FW_PARAM_PFVF(L2T_START
);
5293 params
[2] = FW_PARAM_PFVF(L2T_END
);
5294 params
[3] = FW_PARAM_PFVF(FILTER_START
);
5295 params
[4] = FW_PARAM_PFVF(FILTER_END
);
5296 params
[5] = FW_PARAM_PFVF(IQFLINT_START
);
5297 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6, params
, val
);
5300 adap
->sge
.egr_start
= val
[0];
5301 adap
->l2t_start
= val
[1];
5302 adap
->l2t_end
= val
[2];
5303 adap
->tids
.ftid_base
= val
[3];
5304 adap
->tids
.nftids
= val
[4] - val
[3] + 1;
5305 adap
->sge
.ingr_start
= val
[5];
5307 /* query params related to active filter region */
5308 params
[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START
);
5309 params
[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END
);
5310 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2, params
, val
);
5311 /* If Active filter size is set we enable establishing
5312 * offload connection through firmware work request
5314 if ((val
[0] != val
[1]) && (ret
>= 0)) {
5315 adap
->flags
|= FW_OFLD_CONN
;
5316 adap
->tids
.aftid_base
= val
[0];
5317 adap
->tids
.aftid_end
= val
[1];
5320 /* If we're running on newer firmware, let it know that we're
5321 * prepared to deal with encapsulated CPL messages. Older
5322 * firmware won't understand this and we'll just get
5323 * unencapsulated messages ...
5325 params
[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP
);
5327 (void) t4_set_params(adap
, adap
->mbox
, adap
->fn
, 0, 1, params
, val
);
5330 * Get device capabilities so we can determine what resources we need
5333 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
5334 caps_cmd
.op_to_write
= htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
5335 FW_CMD_REQUEST
| FW_CMD_READ
);
5336 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
5337 ret
= t4_wr_mbox(adap
, adap
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
5342 if (caps_cmd
.ofldcaps
) {
5343 /* query offload-related parameters */
5344 params
[0] = FW_PARAM_DEV(NTID
);
5345 params
[1] = FW_PARAM_PFVF(SERVER_START
);
5346 params
[2] = FW_PARAM_PFVF(SERVER_END
);
5347 params
[3] = FW_PARAM_PFVF(TDDP_START
);
5348 params
[4] = FW_PARAM_PFVF(TDDP_END
);
5349 params
[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ
);
5350 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
5354 adap
->tids
.ntids
= val
[0];
5355 adap
->tids
.natids
= min(adap
->tids
.ntids
/ 2, MAX_ATIDS
);
5356 adap
->tids
.stid_base
= val
[1];
5357 adap
->tids
.nstids
= val
[2] - val
[1] + 1;
5359 * Setup server filter region. Divide the availble filter
5360 * region into two parts. Regular filters get 1/3rd and server
5361 * filters get 2/3rd part. This is only enabled if workarond
5363 * 1. For regular filters.
5364 * 2. Server filter: This are special filters which are used
5365 * to redirect SYN packets to offload queue.
5367 if (adap
->flags
& FW_OFLD_CONN
&& !is_bypass(adap
)) {
5368 adap
->tids
.sftid_base
= adap
->tids
.ftid_base
+
5369 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
5370 adap
->tids
.nsftids
= adap
->tids
.nftids
-
5371 DIV_ROUND_UP(adap
->tids
.nftids
, 3);
5372 adap
->tids
.nftids
= adap
->tids
.sftid_base
-
5373 adap
->tids
.ftid_base
;
5375 adap
->vres
.ddp
.start
= val
[3];
5376 adap
->vres
.ddp
.size
= val
[4] - val
[3] + 1;
5377 adap
->params
.ofldq_wr_cred
= val
[5];
5379 adap
->params
.offload
= 1;
5381 if (caps_cmd
.rdmacaps
) {
5382 params
[0] = FW_PARAM_PFVF(STAG_START
);
5383 params
[1] = FW_PARAM_PFVF(STAG_END
);
5384 params
[2] = FW_PARAM_PFVF(RQ_START
);
5385 params
[3] = FW_PARAM_PFVF(RQ_END
);
5386 params
[4] = FW_PARAM_PFVF(PBL_START
);
5387 params
[5] = FW_PARAM_PFVF(PBL_END
);
5388 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 6,
5392 adap
->vres
.stag
.start
= val
[0];
5393 adap
->vres
.stag
.size
= val
[1] - val
[0] + 1;
5394 adap
->vres
.rq
.start
= val
[2];
5395 adap
->vres
.rq
.size
= val
[3] - val
[2] + 1;
5396 adap
->vres
.pbl
.start
= val
[4];
5397 adap
->vres
.pbl
.size
= val
[5] - val
[4] + 1;
5399 params
[0] = FW_PARAM_PFVF(SQRQ_START
);
5400 params
[1] = FW_PARAM_PFVF(SQRQ_END
);
5401 params
[2] = FW_PARAM_PFVF(CQ_START
);
5402 params
[3] = FW_PARAM_PFVF(CQ_END
);
5403 params
[4] = FW_PARAM_PFVF(OCQ_START
);
5404 params
[5] = FW_PARAM_PFVF(OCQ_END
);
5405 ret
= t4_query_params(adap
, 0, 0, 0, 6, params
, val
);
5408 adap
->vres
.qp
.start
= val
[0];
5409 adap
->vres
.qp
.size
= val
[1] - val
[0] + 1;
5410 adap
->vres
.cq
.start
= val
[2];
5411 adap
->vres
.cq
.size
= val
[3] - val
[2] + 1;
5412 adap
->vres
.ocq
.start
= val
[4];
5413 adap
->vres
.ocq
.size
= val
[5] - val
[4] + 1;
5415 if (caps_cmd
.iscsicaps
) {
5416 params
[0] = FW_PARAM_PFVF(ISCSI_START
);
5417 params
[1] = FW_PARAM_PFVF(ISCSI_END
);
5418 ret
= t4_query_params(adap
, adap
->mbox
, adap
->fn
, 0, 2,
5422 adap
->vres
.iscsi
.start
= val
[0];
5423 adap
->vres
.iscsi
.size
= val
[1] - val
[0] + 1;
5425 #undef FW_PARAM_PFVF
5429 * These are finalized by FW initialization, load their values now.
5431 v
= t4_read_reg(adap
, TP_TIMER_RESOLUTION
);
5432 adap
->params
.tp
.tre
= TIMERRESOLUTION_GET(v
);
5433 adap
->params
.tp
.dack_re
= DELAYEDACKRESOLUTION_GET(v
);
5434 t4_read_mtu_tbl(adap
, adap
->params
.mtus
, NULL
);
5435 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
5436 adap
->params
.b_wnd
);
5438 /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
5439 for (j
= 0; j
< NCHAN
; j
++)
5440 adap
->params
.tp
.tx_modq
[j
] = j
;
5442 t4_read_indirect(adap
, TP_PIO_ADDR
, TP_PIO_DATA
,
5443 &adap
->filter_mode
, 1,
5446 adap
->flags
|= FW_OK
;
5450 * Something bad happened. If a command timed out or failed with EIO
5451 * FW does not operate within its spec or something catastrophic
5452 * happened to HW/FW, stop issuing commands.
5455 if (ret
!= -ETIMEDOUT
&& ret
!= -EIO
)
5456 t4_fw_bye(adap
, adap
->mbox
);
5462 static pci_ers_result_t
eeh_err_detected(struct pci_dev
*pdev
,
5463 pci_channel_state_t state
)
5466 struct adapter
*adap
= pci_get_drvdata(pdev
);
5472 adap
->flags
&= ~FW_OK
;
5473 notify_ulds(adap
, CXGB4_STATE_START_RECOVERY
);
5474 for_each_port(adap
, i
) {
5475 struct net_device
*dev
= adap
->port
[i
];
5477 netif_device_detach(dev
);
5478 netif_carrier_off(dev
);
5480 if (adap
->flags
& FULL_INIT_DONE
)
5483 pci_disable_device(pdev
);
5484 out
: return state
== pci_channel_io_perm_failure
?
5485 PCI_ERS_RESULT_DISCONNECT
: PCI_ERS_RESULT_NEED_RESET
;
5488 static pci_ers_result_t
eeh_slot_reset(struct pci_dev
*pdev
)
5491 struct fw_caps_config_cmd c
;
5492 struct adapter
*adap
= pci_get_drvdata(pdev
);
5495 pci_restore_state(pdev
);
5496 pci_save_state(pdev
);
5497 return PCI_ERS_RESULT_RECOVERED
;
5500 if (pci_enable_device(pdev
)) {
5501 dev_err(&pdev
->dev
, "cannot reenable PCI device after reset\n");
5502 return PCI_ERS_RESULT_DISCONNECT
;
5505 pci_set_master(pdev
);
5506 pci_restore_state(pdev
);
5507 pci_save_state(pdev
);
5508 pci_cleanup_aer_uncorrect_error_status(pdev
);
5510 if (t4_wait_dev_ready(adap
) < 0)
5511 return PCI_ERS_RESULT_DISCONNECT
;
5512 if (t4_fw_hello(adap
, adap
->fn
, adap
->fn
, MASTER_MUST
, NULL
) < 0)
5513 return PCI_ERS_RESULT_DISCONNECT
;
5514 adap
->flags
|= FW_OK
;
5515 if (adap_init1(adap
, &c
))
5516 return PCI_ERS_RESULT_DISCONNECT
;
5518 for_each_port(adap
, i
) {
5519 struct port_info
*p
= adap2pinfo(adap
, i
);
5521 ret
= t4_alloc_vi(adap
, adap
->fn
, p
->tx_chan
, adap
->fn
, 0, 1,
5524 return PCI_ERS_RESULT_DISCONNECT
;
5526 p
->xact_addr_filt
= -1;
5529 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
5530 adap
->params
.b_wnd
);
5533 return PCI_ERS_RESULT_DISCONNECT
;
5534 return PCI_ERS_RESULT_RECOVERED
;
5537 static void eeh_resume(struct pci_dev
*pdev
)
5540 struct adapter
*adap
= pci_get_drvdata(pdev
);
5546 for_each_port(adap
, i
) {
5547 struct net_device
*dev
= adap
->port
[i
];
5549 if (netif_running(dev
)) {
5551 cxgb_set_rxmode(dev
);
5553 netif_device_attach(dev
);
5558 static const struct pci_error_handlers cxgb4_eeh
= {
5559 .error_detected
= eeh_err_detected
,
5560 .slot_reset
= eeh_slot_reset
,
5561 .resume
= eeh_resume
,
5564 static inline bool is_10g_port(const struct link_config
*lc
)
5566 return (lc
->supported
& FW_PORT_CAP_SPEED_10G
) != 0;
5569 static inline void init_rspq(struct sge_rspq
*q
, u8 timer_idx
, u8 pkt_cnt_idx
,
5570 unsigned int size
, unsigned int iqe_size
)
5572 q
->intr_params
= QINTR_TIMER_IDX(timer_idx
) |
5573 (pkt_cnt_idx
< SGE_NCOUNTERS
? QINTR_CNT_EN
: 0);
5574 q
->pktcnt_idx
= pkt_cnt_idx
< SGE_NCOUNTERS
? pkt_cnt_idx
: 0;
5575 q
->iqe_len
= iqe_size
;
5580 * Perform default configuration of DMA queues depending on the number and type
5581 * of ports we found and the number of available CPUs. Most settings can be
5582 * modified by the admin prior to actual use.
5584 static void cfg_queues(struct adapter
*adap
)
5586 struct sge
*s
= &adap
->sge
;
5587 int i
, q10g
= 0, n10g
= 0, qidx
= 0;
5589 for_each_port(adap
, i
)
5590 n10g
+= is_10g_port(&adap2pinfo(adap
, i
)->link_cfg
);
5593 * We default to 1 queue per non-10G port and up to # of cores queues
5597 q10g
= (MAX_ETH_QSETS
- (adap
->params
.nports
- n10g
)) / n10g
;
5598 if (q10g
> netif_get_num_default_rss_queues())
5599 q10g
= netif_get_num_default_rss_queues();
5601 for_each_port(adap
, i
) {
5602 struct port_info
*pi
= adap2pinfo(adap
, i
);
5604 pi
->first_qset
= qidx
;
5605 pi
->nqsets
= is_10g_port(&pi
->link_cfg
) ? q10g
: 1;
5610 s
->max_ethqsets
= qidx
; /* MSI-X may lower it later */
5612 if (is_offload(adap
)) {
5614 * For offload we use 1 queue/channel if all ports are up to 1G,
5615 * otherwise we divide all available queues amongst the channels
5616 * capped by the number of available cores.
5619 i
= min_t(int, ARRAY_SIZE(s
->ofldrxq
),
5621 s
->ofldqsets
= roundup(i
, adap
->params
.nports
);
5623 s
->ofldqsets
= adap
->params
.nports
;
5624 /* For RDMA one Rx queue per channel suffices */
5625 s
->rdmaqs
= adap
->params
.nports
;
5628 for (i
= 0; i
< ARRAY_SIZE(s
->ethrxq
); i
++) {
5629 struct sge_eth_rxq
*r
= &s
->ethrxq
[i
];
5631 init_rspq(&r
->rspq
, 0, 0, 1024, 64);
5635 for (i
= 0; i
< ARRAY_SIZE(s
->ethtxq
); i
++)
5636 s
->ethtxq
[i
].q
.size
= 1024;
5638 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++)
5639 s
->ctrlq
[i
].q
.size
= 512;
5641 for (i
= 0; i
< ARRAY_SIZE(s
->ofldtxq
); i
++)
5642 s
->ofldtxq
[i
].q
.size
= 1024;
5644 for (i
= 0; i
< ARRAY_SIZE(s
->ofldrxq
); i
++) {
5645 struct sge_ofld_rxq
*r
= &s
->ofldrxq
[i
];
5647 init_rspq(&r
->rspq
, 0, 0, 1024, 64);
5648 r
->rspq
.uld
= CXGB4_ULD_ISCSI
;
5652 for (i
= 0; i
< ARRAY_SIZE(s
->rdmarxq
); i
++) {
5653 struct sge_ofld_rxq
*r
= &s
->rdmarxq
[i
];
5655 init_rspq(&r
->rspq
, 0, 0, 511, 64);
5656 r
->rspq
.uld
= CXGB4_ULD_RDMA
;
5660 init_rspq(&s
->fw_evtq
, 6, 0, 512, 64);
5661 init_rspq(&s
->intrq
, 6, 0, 2 * MAX_INGQ
, 64);
5665 * Reduce the number of Ethernet queues across all ports to at most n.
5666 * n provides at least one queue per port.
5668 static void reduce_ethqs(struct adapter
*adap
, int n
)
5671 struct port_info
*pi
;
5673 while (n
< adap
->sge
.ethqsets
)
5674 for_each_port(adap
, i
) {
5675 pi
= adap2pinfo(adap
, i
);
5676 if (pi
->nqsets
> 1) {
5678 adap
->sge
.ethqsets
--;
5679 if (adap
->sge
.ethqsets
<= n
)
5685 for_each_port(adap
, i
) {
5686 pi
= adap2pinfo(adap
, i
);
5692 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5693 #define EXTRA_VECS 2
5695 static int enable_msix(struct adapter
*adap
)
5698 int i
, err
, want
, need
;
5699 struct sge
*s
= &adap
->sge
;
5700 unsigned int nchan
= adap
->params
.nports
;
5701 struct msix_entry entries
[MAX_INGQ
+ 1];
5703 for (i
= 0; i
< ARRAY_SIZE(entries
); ++i
)
5704 entries
[i
].entry
= i
;
5706 want
= s
->max_ethqsets
+ EXTRA_VECS
;
5707 if (is_offload(adap
)) {
5708 want
+= s
->rdmaqs
+ s
->ofldqsets
;
5709 /* need nchan for each possible ULD */
5710 ofld_need
= 2 * nchan
;
5712 need
= adap
->params
.nports
+ EXTRA_VECS
+ ofld_need
;
5714 while ((err
= pci_enable_msix(adap
->pdev
, entries
, want
)) >= need
)
5719 * Distribute available vectors to the various queue groups.
5720 * Every group gets its minimum requirement and NIC gets top
5721 * priority for leftovers.
5723 i
= want
- EXTRA_VECS
- ofld_need
;
5724 if (i
< s
->max_ethqsets
) {
5725 s
->max_ethqsets
= i
;
5726 if (i
< s
->ethqsets
)
5727 reduce_ethqs(adap
, i
);
5729 if (is_offload(adap
)) {
5730 i
= want
- EXTRA_VECS
- s
->max_ethqsets
;
5731 i
-= ofld_need
- nchan
;
5732 s
->ofldqsets
= (i
/ nchan
) * nchan
; /* round down */
5734 for (i
= 0; i
< want
; ++i
)
5735 adap
->msix_info
[i
].vec
= entries
[i
].vector
;
5737 dev_info(adap
->pdev_dev
,
5738 "only %d MSI-X vectors left, not using MSI-X\n", err
);
5744 static int init_rss(struct adapter
*adap
)
5748 for_each_port(adap
, i
) {
5749 struct port_info
*pi
= adap2pinfo(adap
, i
);
5751 pi
->rss
= kcalloc(pi
->rss_size
, sizeof(u16
), GFP_KERNEL
);
5754 for (j
= 0; j
< pi
->rss_size
; j
++)
5755 pi
->rss
[j
] = ethtool_rxfh_indir_default(j
, pi
->nqsets
);
5760 static void print_port_info(const struct net_device
*dev
)
5762 static const char *base
[] = {
5763 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
5764 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
5769 const char *spd
= "";
5770 const struct port_info
*pi
= netdev_priv(dev
);
5771 const struct adapter
*adap
= pi
->adapter
;
5773 if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_2_5GB
)
5775 else if (adap
->params
.pci
.speed
== PCI_EXP_LNKSTA_CLS_5_0GB
)
5778 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_100M
)
5779 bufp
+= sprintf(bufp
, "100/");
5780 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_1G
)
5781 bufp
+= sprintf(bufp
, "1000/");
5782 if (pi
->link_cfg
.supported
& FW_PORT_CAP_SPEED_10G
)
5783 bufp
+= sprintf(bufp
, "10G/");
5786 sprintf(bufp
, "BASE-%s", base
[pi
->port_type
]);
5788 netdev_info(dev
, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5789 adap
->params
.vpd
.id
,
5790 CHELSIO_CHIP_RELEASE(adap
->params
.rev
), buf
,
5791 is_offload(adap
) ? "R" : "", adap
->params
.pci
.width
, spd
,
5792 (adap
->flags
& USING_MSIX
) ? " MSI-X" :
5793 (adap
->flags
& USING_MSI
) ? " MSI" : "");
5794 netdev_info(dev
, "S/N: %s, E/C: %s\n",
5795 adap
->params
.vpd
.sn
, adap
->params
.vpd
.ec
);
5798 static void enable_pcie_relaxed_ordering(struct pci_dev
*dev
)
5800 pcie_capability_set_word(dev
, PCI_EXP_DEVCTL
, PCI_EXP_DEVCTL_RELAX_EN
);
5804 * Free the following resources:
5805 * - memory used for tables
5808 * - resources FW is holding for us
5810 static void free_some_resources(struct adapter
*adapter
)
5814 t4_free_mem(adapter
->l2t
);
5815 t4_free_mem(adapter
->tids
.tid_tab
);
5816 disable_msi(adapter
);
5818 for_each_port(adapter
, i
)
5819 if (adapter
->port
[i
]) {
5820 kfree(adap2pinfo(adapter
, i
)->rss
);
5821 free_netdev(adapter
->port
[i
]);
5823 if (adapter
->flags
& FW_OK
)
5824 t4_fw_bye(adapter
, adapter
->fn
);
5827 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5828 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5829 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5830 #define SEGMENT_SIZE 128
5832 static int init_one(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
5834 int func
, i
, err
, s_qpp
, qpp
, num_seg
;
5835 struct port_info
*pi
;
5836 bool highdma
= false;
5837 struct adapter
*adapter
= NULL
;
5839 printk_once(KERN_INFO
"%s - version %s\n", DRV_DESC
, DRV_VERSION
);
5841 err
= pci_request_regions(pdev
, KBUILD_MODNAME
);
5843 /* Just info, some other driver may have claimed the device. */
5844 dev_info(&pdev
->dev
, "cannot obtain PCI resources\n");
5848 /* We control everything through one PF */
5849 func
= PCI_FUNC(pdev
->devfn
);
5850 if (func
!= ent
->driver_data
) {
5851 pci_save_state(pdev
); /* to restore SR-IOV later */
5855 err
= pci_enable_device(pdev
);
5857 dev_err(&pdev
->dev
, "cannot enable PCI device\n");
5858 goto out_release_regions
;
5861 if (!pci_set_dma_mask(pdev
, DMA_BIT_MASK(64))) {
5863 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
5865 dev_err(&pdev
->dev
, "unable to obtain 64-bit DMA for "
5866 "coherent allocations\n");
5867 goto out_disable_device
;
5870 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
5872 dev_err(&pdev
->dev
, "no usable DMA configuration\n");
5873 goto out_disable_device
;
5877 pci_enable_pcie_error_reporting(pdev
);
5878 enable_pcie_relaxed_ordering(pdev
);
5879 pci_set_master(pdev
);
5880 pci_save_state(pdev
);
5882 adapter
= kzalloc(sizeof(*adapter
), GFP_KERNEL
);
5885 goto out_disable_device
;
5888 adapter
->regs
= pci_ioremap_bar(pdev
, 0);
5889 if (!adapter
->regs
) {
5890 dev_err(&pdev
->dev
, "cannot map device registers\n");
5892 goto out_free_adapter
;
5895 adapter
->pdev
= pdev
;
5896 adapter
->pdev_dev
= &pdev
->dev
;
5897 adapter
->mbox
= func
;
5899 adapter
->msg_enable
= dflt_msg_enable
;
5900 memset(adapter
->chan_map
, 0xff, sizeof(adapter
->chan_map
));
5902 spin_lock_init(&adapter
->stats_lock
);
5903 spin_lock_init(&adapter
->tid_release_lock
);
5905 INIT_WORK(&adapter
->tid_release_task
, process_tid_release_list
);
5906 INIT_WORK(&adapter
->db_full_task
, process_db_full
);
5907 INIT_WORK(&adapter
->db_drop_task
, process_db_drop
);
5909 err
= t4_prep_adapter(adapter
);
5911 goto out_unmap_bar0
;
5913 if (!is_t4(adapter
->chip
)) {
5914 s_qpp
= QUEUESPERPAGEPF1
* adapter
->fn
;
5915 qpp
= 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter
,
5916 SGE_EGRESS_QUEUES_PER_PAGE_PF
) >> s_qpp
);
5917 num_seg
= PAGE_SIZE
/ SEGMENT_SIZE
;
5919 /* Each segment size is 128B. Write coalescing is enabled only
5920 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
5921 * queue is less no of segments that can be accommodated in
5924 if (qpp
> num_seg
) {
5926 "Incorrect number of egress queues per page\n");
5928 goto out_unmap_bar0
;
5930 adapter
->bar2
= ioremap_wc(pci_resource_start(pdev
, 2),
5931 pci_resource_len(pdev
, 2));
5932 if (!adapter
->bar2
) {
5933 dev_err(&pdev
->dev
, "cannot map device bar2 region\n");
5935 goto out_unmap_bar0
;
5939 setup_memwin(adapter
);
5940 err
= adap_init0(adapter
);
5941 setup_memwin_rdma(adapter
);
5945 for_each_port(adapter
, i
) {
5946 struct net_device
*netdev
;
5948 netdev
= alloc_etherdev_mq(sizeof(struct port_info
),
5955 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
5957 adapter
->port
[i
] = netdev
;
5958 pi
= netdev_priv(netdev
);
5959 pi
->adapter
= adapter
;
5960 pi
->xact_addr_filt
= -1;
5962 netdev
->irq
= pdev
->irq
;
5964 netdev
->hw_features
= NETIF_F_SG
| TSO_FLAGS
|
5965 NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
5966 NETIF_F_RXCSUM
| NETIF_F_RXHASH
|
5967 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
;
5969 netdev
->hw_features
|= NETIF_F_HIGHDMA
;
5970 netdev
->features
|= netdev
->hw_features
;
5971 netdev
->vlan_features
= netdev
->features
& VLAN_FEAT
;
5973 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
5975 netdev
->netdev_ops
= &cxgb4_netdev_ops
;
5976 SET_ETHTOOL_OPS(netdev
, &cxgb_ethtool_ops
);
5979 pci_set_drvdata(pdev
, adapter
);
5981 if (adapter
->flags
& FW_OK
) {
5982 err
= t4_port_init(adapter
, func
, func
, 0);
5988 * Configure queues and allocate tables now, they can be needed as
5989 * soon as the first register_netdev completes.
5991 cfg_queues(adapter
);
5993 adapter
->l2t
= t4_init_l2t();
5994 if (!adapter
->l2t
) {
5995 /* We tolerate a lack of L2T, giving up some functionality */
5996 dev_warn(&pdev
->dev
, "could not allocate L2T, continuing\n");
5997 adapter
->params
.offload
= 0;
6000 if (is_offload(adapter
) && tid_init(&adapter
->tids
) < 0) {
6001 dev_warn(&pdev
->dev
, "could not allocate TID table, "
6003 adapter
->params
.offload
= 0;
6006 /* See what interrupts we'll be using */
6007 if (msi
> 1 && enable_msix(adapter
) == 0)
6008 adapter
->flags
|= USING_MSIX
;
6009 else if (msi
> 0 && pci_enable_msi(pdev
) == 0)
6010 adapter
->flags
|= USING_MSI
;
6012 err
= init_rss(adapter
);
6017 * The card is now ready to go. If any errors occur during device
6018 * registration we do not fail the whole card but rather proceed only
6019 * with the ports we manage to register successfully. However we must
6020 * register at least one net device.
6022 for_each_port(adapter
, i
) {
6023 pi
= adap2pinfo(adapter
, i
);
6024 netif_set_real_num_tx_queues(adapter
->port
[i
], pi
->nqsets
);
6025 netif_set_real_num_rx_queues(adapter
->port
[i
], pi
->nqsets
);
6027 err
= register_netdev(adapter
->port
[i
]);
6030 adapter
->chan_map
[pi
->tx_chan
] = i
;
6031 print_port_info(adapter
->port
[i
]);
6034 dev_err(&pdev
->dev
, "could not register any net devices\n");
6038 dev_warn(&pdev
->dev
, "only %d net devices registered\n", i
);
6042 if (cxgb4_debugfs_root
) {
6043 adapter
->debugfs_root
= debugfs_create_dir(pci_name(pdev
),
6044 cxgb4_debugfs_root
);
6045 setup_debugfs(adapter
);
6048 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6049 pdev
->needs_freset
= 1;
6051 if (is_offload(adapter
))
6052 attach_ulds(adapter
);
6055 #ifdef CONFIG_PCI_IOV
6056 if (func
< ARRAY_SIZE(num_vf
) && num_vf
[func
] > 0)
6057 if (pci_enable_sriov(pdev
, num_vf
[func
]) == 0)
6058 dev_info(&pdev
->dev
,
6059 "instantiated %u virtual functions\n",
6065 free_some_resources(adapter
);
6067 if (!is_t4(adapter
->chip
))
6068 iounmap(adapter
->bar2
);
6070 iounmap(adapter
->regs
);
6074 pci_disable_pcie_error_reporting(pdev
);
6075 pci_disable_device(pdev
);
6076 out_release_regions
:
6077 pci_release_regions(pdev
);
6081 static void remove_one(struct pci_dev
*pdev
)
6083 struct adapter
*adapter
= pci_get_drvdata(pdev
);
6085 #ifdef CONFIG_PCI_IOV
6086 pci_disable_sriov(pdev
);
6093 if (is_offload(adapter
))
6094 detach_ulds(adapter
);
6096 for_each_port(adapter
, i
)
6097 if (adapter
->port
[i
]->reg_state
== NETREG_REGISTERED
)
6098 unregister_netdev(adapter
->port
[i
]);
6100 if (adapter
->debugfs_root
)
6101 debugfs_remove_recursive(adapter
->debugfs_root
);
6103 /* If we allocated filters, free up state associated with any
6106 if (adapter
->tids
.ftid_tab
) {
6107 struct filter_entry
*f
= &adapter
->tids
.ftid_tab
[0];
6108 for (i
= 0; i
< (adapter
->tids
.nftids
+
6109 adapter
->tids
.nsftids
); i
++, f
++)
6111 clear_filter(adapter
, f
);
6114 if (adapter
->flags
& FULL_INIT_DONE
)
6117 free_some_resources(adapter
);
6118 iounmap(adapter
->regs
);
6119 if (!is_t4(adapter
->chip
))
6120 iounmap(adapter
->bar2
);
6122 pci_disable_pcie_error_reporting(pdev
);
6123 pci_disable_device(pdev
);
6124 pci_release_regions(pdev
);
6126 pci_release_regions(pdev
);
6129 static struct pci_driver cxgb4_driver
= {
6130 .name
= KBUILD_MODNAME
,
6131 .id_table
= cxgb4_pci_tbl
,
6133 .remove
= remove_one
,
6134 .err_handler
= &cxgb4_eeh
,
6137 static int __init
cxgb4_init_module(void)
6141 workq
= create_singlethread_workqueue("cxgb4");
6145 /* Debugfs support is optional, just warn if this fails */
6146 cxgb4_debugfs_root
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
6147 if (!cxgb4_debugfs_root
)
6148 pr_warn("could not create debugfs entry, continuing\n");
6150 ret
= pci_register_driver(&cxgb4_driver
);
6152 debugfs_remove(cxgb4_debugfs_root
);
6153 destroy_workqueue(workq
);
6156 register_inet6addr_notifier(&cxgb4_inet6addr_notifier
);
6161 static void __exit
cxgb4_cleanup_module(void)
6163 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier
);
6164 pci_unregister_driver(&cxgb4_driver
);
6165 debugfs_remove(cxgb4_debugfs_root
); /* NULL ok */
6166 flush_workqueue(workq
);
6167 destroy_workqueue(workq
);
6170 module_init(cxgb4_init_module
);
6171 module_exit(cxgb4_cleanup_module
);