?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ethtool: Change ethtool_op_set_flags to validate flags
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / net / cxgb4 / cxgb4_main.c
blob55a720e4abdc63301e707b3cbd218b7a7e2c9cec
1 /*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if_vlan.h>
45 #include <linux/init.h>
46 #include <linux/log2.h>
47 #include <linux/mdio.h>
48 #include <linux/module.h>
49 #include <linux/moduleparam.h>
50 #include <linux/mutex.h>
51 #include <linux/netdevice.h>
52 #include <linux/pci.h>
53 #include <linux/aer.h>
54 #include <linux/rtnetlink.h>
55 #include <linux/sched.h>
56 #include <linux/seq_file.h>
57 #include <linux/sockios.h>
58 #include <linux/vmalloc.h>
59 #include <linux/workqueue.h>
60 #include <net/neighbour.h>
61 #include <net/netevent.h>
62 #include <asm/uaccess.h>
63
64 #include "cxgb4.h"
65 #include "t4_regs.h"
66 #include "t4_msg.h"
67 #include "t4fw_api.h"
68 #include "l2t.h"
69
70 #define DRV_VERSION "1.0.0-ko"
71 #define DRV_DESC "Chelsio T4 Network Driver"
72
73 /*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78 #define MAX_SGE_TIMERVAL 200U
79
80 #ifdef CONFIG_PCI_IOV
81 /*
82 * Virtual Function provisioning constants. We need two extra Ingress Queues
83 * with Interrupt capability to serve as the VF's Firmware Event Queue and
84 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85 * Lists associated with them). For each Ethernet/Control Egress Queue and
86 * for each Free List, we need an Egress Context.
87 */
88 enum {
89 VFRES_NPORTS = 1, /* # of "ports" per VF */
90 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
91
92 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
93 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
94 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
96 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
97 VFRES_TC = 0, /* PCI-E traffic class */
98 VFRES_NEXACTF = 16, /* # of exact MPS filters */
99
100 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102 };
103
104 /*
105 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
106 * static and likely not to be useful in the long run. We really need to
107 * implement some form of persistent configuration which the firmware
108 * controls.
109 */
110 static unsigned int pfvfres_pmask(struct adapter *adapter,
111 unsigned int pf, unsigned int vf)
112 {
113 unsigned int portn, portvec;
114
115 /*
116 * Give PF's access to all of the ports.
117 */
118 if (vf == 0)
119 return FW_PFVF_CMD_PMASK_MASK;
120
121 /*
122 * For VFs, we'll assign them access to the ports based purely on the
123 * PF. We assign active ports in order, wrapping around if there are
124 * fewer active ports than PFs: e.g. active port[pf % nports].
125 * Unfortunately the adapter's port_info structs haven't been
126 * initialized yet so we have to compute this.
127 */
128 if (adapter->params.nports == 0)
129 return 0;
130
131 portn = pf % adapter->params.nports;
132 portvec = adapter->params.portvec;
133 for (;;) {
134 /*
135 * Isolate the lowest set bit in the port vector. If we're at
136 * the port number that we want, return that as the pmask.
137 * otherwise mask that bit out of the port vector and
138 * decrement our port number ...
139 */
140 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141 if (portn == 0)
142 return pmask;
143 portn--;
144 portvec &= ~pmask;
145 }
146 /*NOTREACHED*/
147 }
148 #endif
149
150 enum {
151 MEMWIN0_APERTURE = 65536,
152 MEMWIN0_BASE = 0x30000,
153 MEMWIN1_APERTURE = 32768,
154 MEMWIN1_BASE = 0x28000,
155 MEMWIN2_APERTURE = 2048,
156 MEMWIN2_BASE = 0x1b800,
157 };
158
159 enum {
160 MAX_TXQ_ENTRIES = 16384,
161 MAX_CTRL_TXQ_ENTRIES = 1024,
162 MAX_RSPQ_ENTRIES = 16384,
163 MAX_RX_BUFFERS = 16384,
164 MIN_TXQ_ENTRIES = 32,
165 MIN_CTRL_TXQ_ENTRIES = 32,
166 MIN_RSPQ_ENTRIES = 128,
167 MIN_FL_ENTRIES = 16
168 };
169
170 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
174 #define CH_DEVICE(devid) { PCI_VDEVICE(CHELSIO, devid), 0 }
175
176 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
177 CH_DEVICE(0xa000), /* PE10K */
178 { 0, }
179 };
180
181 #define FW_FNAME "cxgb4/t4fw.bin"
182
183 MODULE_DESCRIPTION(DRV_DESC);
184 MODULE_AUTHOR("Chelsio Communications");
185 MODULE_LICENSE("Dual BSD/GPL");
186 MODULE_VERSION(DRV_VERSION);
187 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
188 MODULE_FIRMWARE(FW_FNAME);
189
190 static int dflt_msg_enable = DFLT_MSG_ENABLE;
191
192 module_param(dflt_msg_enable, int, 0644);
193 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
194
195 /*
196 * The driver uses the best interrupt scheme available on a platform in the
197 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
198 * of these schemes the driver may consider as follows:
199 *
200 * msi = 2: choose from among all three options
201 * msi = 1: only consider MSI and INTx interrupts
202 * msi = 0: force INTx interrupts
203 */
204 static int msi = 2;
205
206 module_param(msi, int, 0644);
207 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
208
209 /*
210 * Queue interrupt hold-off timer values. Queues default to the first of these
211 * upon creation.
212 */
213 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
214
215 module_param_array(intr_holdoff, uint, NULL, 0644);
216 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
217 "0..4 in microseconds");
218
219 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
220
221 module_param_array(intr_cnt, uint, NULL, 0644);
222 MODULE_PARM_DESC(intr_cnt,
223 "thresholds 1..3 for queue interrupt packet counters");
224
225 static int vf_acls;
226
227 #ifdef CONFIG_PCI_IOV
228 module_param(vf_acls, bool, 0644);
229 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
230
231 static unsigned int num_vf[4];
232
233 module_param_array(num_vf, uint, NULL, 0644);
234 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
235 #endif
236
237 static struct dentry *cxgb4_debugfs_root;
238
239 static LIST_HEAD(adapter_list);
240 static DEFINE_MUTEX(uld_mutex);
241 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
242 static const char *uld_str[] = { "RDMA", "iSCSI" };
243
244 static void link_report(struct net_device *dev)
245 {
246 if (!netif_carrier_ok(dev))
247 netdev_info(dev, "link down\n");
248 else {
249 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
250
251 const char *s = "10Mbps";
252 const struct port_info *p = netdev_priv(dev);
253
254 switch (p->link_cfg.speed) {
255 case SPEED_10000:
256 s = "10Gbps";
257 break;
258 case SPEED_1000:
259 s = "1000Mbps";
260 break;
261 case SPEED_100:
262 s = "100Mbps";
263 break;
264 }
265
266 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
267 fc[p->link_cfg.fc]);
268 }
269 }
270
271 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
272 {
273 struct net_device *dev = adapter->port[port_id];
274
275 /* Skip changes from disabled ports. */
276 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
277 if (link_stat)
278 netif_carrier_on(dev);
279 else
280 netif_carrier_off(dev);
281
282 link_report(dev);
283 }
284 }
285
286 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
287 {
288 static const char *mod_str[] = {
289 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
290 };
291
292 const struct net_device *dev = adap->port[port_id];
293 const struct port_info *pi = netdev_priv(dev);
294
295 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
296 netdev_info(dev, "port module unplugged\n");
297 else if (pi->mod_type < ARRAY_SIZE(mod_str))
298 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
299 }
300
301 /*
302 * Configure the exact and hash address filters to handle a port's multicast
303 * and secondary unicast MAC addresses.
304 */
305 static int set_addr_filters(const struct net_device *dev, bool sleep)
306 {
307 u64 mhash = 0;
308 u64 uhash = 0;
309 bool free = true;
310 u16 filt_idx[7];
311 const u8 *addr[7];
312 int ret, naddr = 0;
313 const struct netdev_hw_addr *ha;
314 int uc_cnt = netdev_uc_count(dev);
315 int mc_cnt = netdev_mc_count(dev);
316 const struct port_info *pi = netdev_priv(dev);
317
318 /* first do the secondary unicast addresses */
319 netdev_for_each_uc_addr(ha, dev) {
320 addr[naddr++] = ha->addr;
321 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
322 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
323 naddr, addr, filt_idx, &uhash, sleep);
324 if (ret < 0)
325 return ret;
326
327 free = false;
328 naddr = 0;
329 }
330 }
331
332 /* next set up the multicast addresses */
333 netdev_for_each_mc_addr(ha, dev) {
334 addr[naddr++] = ha->addr;
335 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
336 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
337 naddr, addr, filt_idx, &mhash, sleep);
338 if (ret < 0)
339 return ret;
340
341 free = false;
342 naddr = 0;
343 }
344 }
345
346 return t4_set_addr_hash(pi->adapter, 0, pi->viid, uhash != 0,
347 uhash | mhash, sleep);
348 }
349
350 /*
351 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
352 * If @mtu is -1 it is left unchanged.
353 */
354 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
355 {
356 int ret;
357 struct port_info *pi = netdev_priv(dev);
358
359 ret = set_addr_filters(dev, sleep_ok);
360 if (ret == 0)
361 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, mtu,
362 (dev->flags & IFF_PROMISC) ? 1 : 0,
363 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
364 sleep_ok);
365 return ret;
366 }
367
368 /**
369 * link_start - enable a port
370 * @dev: the port to enable
371 *
372 * Performs the MAC and PHY actions needed to enable a port.
373 */
374 static int link_start(struct net_device *dev)
375 {
376 int ret;
377 struct port_info *pi = netdev_priv(dev);
378
379 /*
380 * We do not set address filters and promiscuity here, the stack does
381 * that step explicitly.
382 */
383 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, dev->mtu, -1, -1, -1,
384 pi->vlan_grp != NULL, true);
385 if (ret == 0) {
386 ret = t4_change_mac(pi->adapter, 0, pi->viid,
387 pi->xact_addr_filt, dev->dev_addr, true,
388 true);
389 if (ret >= 0) {
390 pi->xact_addr_filt = ret;
391 ret = 0;
392 }
393 }
394 if (ret == 0)
395 ret = t4_link_start(pi->adapter, 0, pi->tx_chan, &pi->link_cfg);
396 if (ret == 0)
397 ret = t4_enable_vi(pi->adapter, 0, pi->viid, true, true);
398 return ret;
399 }
400
401 /*
402 * Response queue handler for the FW event queue.
403 */
404 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
405 const struct pkt_gl *gl)
406 {
407 u8 opcode = ((const struct rss_header *)rsp)->opcode;
408
409 rsp++; /* skip RSS header */
410 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
411 const struct cpl_sge_egr_update *p = (void *)rsp;
412 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
413 struct sge_txq *txq = q->adap->sge.egr_map[qid];
414
415 txq->restarts++;
416 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
417 struct sge_eth_txq *eq;
418
419 eq = container_of(txq, struct sge_eth_txq, q);
420 netif_tx_wake_queue(eq->txq);
421 } else {
422 struct sge_ofld_txq *oq;
423
424 oq = container_of(txq, struct sge_ofld_txq, q);
425 tasklet_schedule(&oq->qresume_tsk);
426 }
427 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
428 const struct cpl_fw6_msg *p = (void *)rsp;
429
430 if (p->type == 0)
431 t4_handle_fw_rpl(q->adap, p->data);
432 } else if (opcode == CPL_L2T_WRITE_RPL) {
433 const struct cpl_l2t_write_rpl *p = (void *)rsp;
434
435 do_l2t_write_rpl(q->adap, p);
436 } else
437 dev_err(q->adap->pdev_dev,
438 "unexpected CPL %#x on FW event queue\n", opcode);
439 return 0;
440 }
441
442 /**
443 * uldrx_handler - response queue handler for ULD queues
444 * @q: the response queue that received the packet
445 * @rsp: the response queue descriptor holding the offload message
446 * @gl: the gather list of packet fragments
447 *
448 * Deliver an ingress offload packet to a ULD. All processing is done by
449 * the ULD, we just maintain statistics.
450 */
451 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
452 const struct pkt_gl *gl)
453 {
454 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
455
456 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
457 rxq->stats.nomem++;
458 return -1;
459 }
460 if (gl == NULL)
461 rxq->stats.imm++;
462 else if (gl == CXGB4_MSG_AN)
463 rxq->stats.an++;
464 else
465 rxq->stats.pkts++;
466 return 0;
467 }
468
469 static void disable_msi(struct adapter *adapter)
470 {
471 if (adapter->flags & USING_MSIX) {
472 pci_disable_msix(adapter->pdev);
473 adapter->flags &= ~USING_MSIX;
474 } else if (adapter->flags & USING_MSI) {
475 pci_disable_msi(adapter->pdev);
476 adapter->flags &= ~USING_MSI;
477 }
478 }
479
480 /*
481 * Interrupt handler for non-data events used with MSI-X.
482 */
483 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
484 {
485 struct adapter *adap = cookie;
486
487 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
488 if (v & PFSW) {
489 adap->swintr = 1;
490 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
491 }
492 t4_slow_intr_handler(adap);
493 return IRQ_HANDLED;
494 }
495
496 /*
497 * Name the MSI-X interrupts.
498 */
499 static void name_msix_vecs(struct adapter *adap)
500 {
501 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
502
503 /* non-data interrupts */
504 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
505 adap->msix_info[0].desc[n] = 0;
506
507 /* FW events */
508 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
509 adap->msix_info[1].desc[n] = 0;
510
511 /* Ethernet queues */
512 for_each_port(adap, j) {
513 struct net_device *d = adap->port[j];
514 const struct port_info *pi = netdev_priv(d);
515
516 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
517 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
518 d->name, i);
519 adap->msix_info[msi_idx].desc[n] = 0;
520 }
521 }
522
523 /* offload queues */
524 for_each_ofldrxq(&adap->sge, i) {
525 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
526 adap->name, i);
527 adap->msix_info[msi_idx++].desc[n] = 0;
528 }
529 for_each_rdmarxq(&adap->sge, i) {
530 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
531 adap->name, i);
532 adap->msix_info[msi_idx++].desc[n] = 0;
533 }
534 }
535
536 static int request_msix_queue_irqs(struct adapter *adap)
537 {
538 struct sge *s = &adap->sge;
539 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
540
541 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
542 adap->msix_info[1].desc, &s->fw_evtq);
543 if (err)
544 return err;
545
546 for_each_ethrxq(s, ethqidx) {
547 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
548 adap->msix_info[msi].desc,
549 &s->ethrxq[ethqidx].rspq);
550 if (err)
551 goto unwind;
552 msi++;
553 }
554 for_each_ofldrxq(s, ofldqidx) {
555 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
556 adap->msix_info[msi].desc,
557 &s->ofldrxq[ofldqidx].rspq);
558 if (err)
559 goto unwind;
560 msi++;
561 }
562 for_each_rdmarxq(s, rdmaqidx) {
563 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
564 adap->msix_info[msi].desc,
565 &s->rdmarxq[rdmaqidx].rspq);
566 if (err)
567 goto unwind;
568 msi++;
569 }
570 return 0;
571
572 unwind:
573 while (--rdmaqidx >= 0)
574 free_irq(adap->msix_info[--msi].vec,
575 &s->rdmarxq[rdmaqidx].rspq);
576 while (--ofldqidx >= 0)
577 free_irq(adap->msix_info[--msi].vec,
578 &s->ofldrxq[ofldqidx].rspq);
579 while (--ethqidx >= 0)
580 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
581 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
582 return err;
583 }
584
585 static void free_msix_queue_irqs(struct adapter *adap)
586 {
587 int i, msi = 2;
588 struct sge *s = &adap->sge;
589
590 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
591 for_each_ethrxq(s, i)
592 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
593 for_each_ofldrxq(s, i)
594 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
595 for_each_rdmarxq(s, i)
596 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
597 }
598
599 /**
600 * setup_rss - configure RSS
601 * @adap: the adapter
602 *
603 * Sets up RSS to distribute packets to multiple receive queues. We
604 * configure the RSS CPU lookup table to distribute to the number of HW
605 * receive queues, and the response queue lookup table to narrow that
606 * down to the response queues actually configured for each port.
607 * We always configure the RSS mapping for all ports since the mapping
608 * table has plenty of entries.
609 */
610 static int setup_rss(struct adapter *adap)
611 {
612 int i, j, err;
613 u16 rss[MAX_ETH_QSETS];
614
615 for_each_port(adap, i) {
616 const struct port_info *pi = adap2pinfo(adap, i);
617 const struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
618
619 for (j = 0; j < pi->nqsets; j++)
620 rss[j] = q[j].rspq.abs_id;
621
622 err = t4_config_rss_range(adap, 0, pi->viid, 0, pi->rss_size,
623 rss, pi->nqsets);
624 if (err)
625 return err;
626 }
627 return 0;
628 }
629
630 /*
631 * Wait until all NAPI handlers are descheduled.
632 */
633 static void quiesce_rx(struct adapter *adap)
634 {
635 int i;
636
637 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
638 struct sge_rspq *q = adap->sge.ingr_map[i];
639
640 if (q && q->handler)
641 napi_disable(&q->napi);
642 }
643 }
644
645 /*
646 * Enable NAPI scheduling and interrupt generation for all Rx queues.
647 */
648 static void enable_rx(struct adapter *adap)
649 {
650 int i;
651
652 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
653 struct sge_rspq *q = adap->sge.ingr_map[i];
654
655 if (!q)
656 continue;
657 if (q->handler)
658 napi_enable(&q->napi);
659 /* 0-increment GTS to start the timer and enable interrupts */
660 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
661 SEINTARM(q->intr_params) |
662 INGRESSQID(q->cntxt_id));
663 }
664 }
665
666 /**
667 * setup_sge_queues - configure SGE Tx/Rx/response queues
668 * @adap: the adapter
669 *
670 * Determines how many sets of SGE queues to use and initializes them.
671 * We support multiple queue sets per port if we have MSI-X, otherwise
672 * just one queue set per port.
673 */
674 static int setup_sge_queues(struct adapter *adap)
675 {
676 int err, msi_idx, i, j;
677 struct sge *s = &adap->sge;
678
679 bitmap_zero(s->starving_fl, MAX_EGRQ);
680 bitmap_zero(s->txq_maperr, MAX_EGRQ);
681
682 if (adap->flags & USING_MSIX)
683 msi_idx = 1; /* vector 0 is for non-queue interrupts */
684 else {
685 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
686 NULL, NULL);
687 if (err)
688 return err;
689 msi_idx = -((int)s->intrq.abs_id + 1);
690 }
691
692 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
693 msi_idx, NULL, fwevtq_handler);
694 if (err) {
695 freeout: t4_free_sge_resources(adap);
696 return err;
697 }
698
699 for_each_port(adap, i) {
700 struct net_device *dev = adap->port[i];
701 struct port_info *pi = netdev_priv(dev);
702 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
703 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
704
705 for (j = 0; j < pi->nqsets; j++, q++) {
706 if (msi_idx > 0)
707 msi_idx++;
708 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
709 msi_idx, &q->fl,
710 t4_ethrx_handler);
711 if (err)
712 goto freeout;
713 q->rspq.idx = j;
714 memset(&q->stats, 0, sizeof(q->stats));
715 }
716 for (j = 0; j < pi->nqsets; j++, t++) {
717 err = t4_sge_alloc_eth_txq(adap, t, dev,
718 netdev_get_tx_queue(dev, j),
719 s->fw_evtq.cntxt_id);
720 if (err)
721 goto freeout;
722 }
723 }
724
725 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
726 for_each_ofldrxq(s, i) {
727 struct sge_ofld_rxq *q = &s->ofldrxq[i];
728 struct net_device *dev = adap->port[i / j];
729
730 if (msi_idx > 0)
731 msi_idx++;
732 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
733 &q->fl, uldrx_handler);
734 if (err)
735 goto freeout;
736 memset(&q->stats, 0, sizeof(q->stats));
737 s->ofld_rxq[i] = q->rspq.abs_id;
738 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
739 s->fw_evtq.cntxt_id);
740 if (err)
741 goto freeout;
742 }
743
744 for_each_rdmarxq(s, i) {
745 struct sge_ofld_rxq *q = &s->rdmarxq[i];
746
747 if (msi_idx > 0)
748 msi_idx++;
749 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
750 msi_idx, &q->fl, uldrx_handler);
751 if (err)
752 goto freeout;
753 memset(&q->stats, 0, sizeof(q->stats));
754 s->rdma_rxq[i] = q->rspq.abs_id;
755 }
756
757 for_each_port(adap, i) {
758 /*
759 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
760 * have RDMA queues, and that's the right value.
761 */
762 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
763 s->fw_evtq.cntxt_id,
764 s->rdmarxq[i].rspq.cntxt_id);
765 if (err)
766 goto freeout;
767 }
768
769 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
770 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
771 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
772 return 0;
773 }
774
775 /*
776 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
777 * started but failed, and a negative errno if flash load couldn't start.
778 */
779 static int upgrade_fw(struct adapter *adap)
780 {
781 int ret;
782 u32 vers;
783 const struct fw_hdr *hdr;
784 const struct firmware *fw;
785 struct device *dev = adap->pdev_dev;
786
787 ret = request_firmware(&fw, FW_FNAME, dev);
788 if (ret < 0) {
789 dev_err(dev, "unable to load firmware image " FW_FNAME
790 ", error %d\n", ret);
791 return ret;
792 }
793
794 hdr = (const struct fw_hdr *)fw->data;
795 vers = ntohl(hdr->fw_ver);
796 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
797 ret = -EINVAL; /* wrong major version, won't do */
798 goto out;
799 }
800
801 /*
802 * If the flash FW is unusable or we found something newer, load it.
803 */
804 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
805 vers > adap->params.fw_vers) {
806 ret = -t4_load_fw(adap, fw->data, fw->size);
807 if (!ret)
808 dev_info(dev, "firmware upgraded to version %pI4 from "
809 FW_FNAME "\n", &hdr->fw_ver);
810 }
811 out: release_firmware(fw);
812 return ret;
813 }
814
815 /*
816 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
817 * The allocated memory is cleared.
818 */
819 void *t4_alloc_mem(size_t size)
820 {
821 void *p = kmalloc(size, GFP_KERNEL);
822
823 if (!p)
824 p = vmalloc(size);
825 if (p)
826 memset(p, 0, size);
827 return p;
828 }
829
830 /*
831 * Free memory allocated through alloc_mem().
832 */
833 void t4_free_mem(void *addr)
834 {
835 if (is_vmalloc_addr(addr))
836 vfree(addr);
837 else
838 kfree(addr);
839 }
840
841 static inline int is_offload(const struct adapter *adap)
842 {
843 return adap->params.offload;
844 }
845
846 /*
847 * Implementation of ethtool operations.
848 */
849
850 static u32 get_msglevel(struct net_device *dev)
851 {
852 return netdev2adap(dev)->msg_enable;
853 }
854
855 static void set_msglevel(struct net_device *dev, u32 val)
856 {
857 netdev2adap(dev)->msg_enable = val;
858 }
859
860 static char stats_strings[][ETH_GSTRING_LEN] = {
861 "TxOctetsOK ",
862 "TxFramesOK ",
863 "TxBroadcastFrames ",
864 "TxMulticastFrames ",
865 "TxUnicastFrames ",
866 "TxErrorFrames ",
867
868 "TxFrames64 ",
869 "TxFrames65To127 ",
870 "TxFrames128To255 ",
871 "TxFrames256To511 ",
872 "TxFrames512To1023 ",
873 "TxFrames1024To1518 ",
874 "TxFrames1519ToMax ",
875
876 "TxFramesDropped ",
877 "TxPauseFrames ",
878 "TxPPP0Frames ",
879 "TxPPP1Frames ",
880 "TxPPP2Frames ",
881 "TxPPP3Frames ",
882 "TxPPP4Frames ",
883 "TxPPP5Frames ",
884 "TxPPP6Frames ",
885 "TxPPP7Frames ",
886
887 "RxOctetsOK ",
888 "RxFramesOK ",
889 "RxBroadcastFrames ",
890 "RxMulticastFrames ",
891 "RxUnicastFrames ",
892
893 "RxFramesTooLong ",
894 "RxJabberErrors ",
895 "RxFCSErrors ",
896 "RxLengthErrors ",
897 "RxSymbolErrors ",
898 "RxRuntFrames ",
899
900 "RxFrames64 ",
901 "RxFrames65To127 ",
902 "RxFrames128To255 ",
903 "RxFrames256To511 ",
904 "RxFrames512To1023 ",
905 "RxFrames1024To1518 ",
906 "RxFrames1519ToMax ",
907
908 "RxPauseFrames ",
909 "RxPPP0Frames ",
910 "RxPPP1Frames ",
911 "RxPPP2Frames ",
912 "RxPPP3Frames ",
913 "RxPPP4Frames ",
914 "RxPPP5Frames ",
915 "RxPPP6Frames ",
916 "RxPPP7Frames ",
917
918 "RxBG0FramesDropped ",
919 "RxBG1FramesDropped ",
920 "RxBG2FramesDropped ",
921 "RxBG3FramesDropped ",
922 "RxBG0FramesTrunc ",
923 "RxBG1FramesTrunc ",
924 "RxBG2FramesTrunc ",
925 "RxBG3FramesTrunc ",
926
927 "TSO ",
928 "TxCsumOffload ",
929 "RxCsumGood ",
930 "VLANextractions ",
931 "VLANinsertions ",
932 "GROpackets ",
933 "GROmerged ",
934 };
935
936 static int get_sset_count(struct net_device *dev, int sset)
937 {
938 switch (sset) {
939 case ETH_SS_STATS:
940 return ARRAY_SIZE(stats_strings);
941 default:
942 return -EOPNOTSUPP;
943 }
944 }
945
946 #define T4_REGMAP_SIZE (160 * 1024)
947
948 static int get_regs_len(struct net_device *dev)
949 {
950 return T4_REGMAP_SIZE;
951 }
952
953 static int get_eeprom_len(struct net_device *dev)
954 {
955 return EEPROMSIZE;
956 }
957
958 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
959 {
960 struct adapter *adapter = netdev2adap(dev);
961
962 strcpy(info->driver, KBUILD_MODNAME);
963 strcpy(info->version, DRV_VERSION);
964 strcpy(info->bus_info, pci_name(adapter->pdev));
965
966 if (!adapter->params.fw_vers)
967 strcpy(info->fw_version, "N/A");
968 else
969 snprintf(info->fw_version, sizeof(info->fw_version),
970 "%u.%u.%u.%u, TP %u.%u.%u.%u",
971 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
972 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
973 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
974 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
975 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
976 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
977 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
978 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
979 }
980
981 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
982 {
983 if (stringset == ETH_SS_STATS)
984 memcpy(data, stats_strings, sizeof(stats_strings));
985 }
986
987 /*
988 * port stats maintained per queue of the port. They should be in the same
989 * order as in stats_strings above.
990 */
991 struct queue_port_stats {
992 u64 tso;
993 u64 tx_csum;
994 u64 rx_csum;
995 u64 vlan_ex;
996 u64 vlan_ins;
997 u64 gro_pkts;
998 u64 gro_merged;
999 };
1000
1001 static void collect_sge_port_stats(const struct adapter *adap,
1002 const struct port_info *p, struct queue_port_stats *s)
1003 {
1004 int i;
1005 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1006 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1007
1008 memset(s, 0, sizeof(*s));
1009 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1010 s->tso += tx->tso;
1011 s->tx_csum += tx->tx_cso;
1012 s->rx_csum += rx->stats.rx_cso;
1013 s->vlan_ex += rx->stats.vlan_ex;
1014 s->vlan_ins += tx->vlan_ins;
1015 s->gro_pkts += rx->stats.lro_pkts;
1016 s->gro_merged += rx->stats.lro_merged;
1017 }
1018 }
1019
1020 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1021 u64 *data)
1022 {
1023 struct port_info *pi = netdev_priv(dev);
1024 struct adapter *adapter = pi->adapter;
1025
1026 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1027
1028 data += sizeof(struct port_stats) / sizeof(u64);
1029 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1030 }
1031
1032 /*
1033 * Return a version number to identify the type of adapter. The scheme is:
1034 * - bits 0..9: chip version
1035 * - bits 10..15: chip revision
1036 */
1037 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1038 {
1039 return 4 | (ap->params.rev << 10);
1040 }
1041
1042 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1043 unsigned int end)
1044 {
1045 u32 *p = buf + start;
1046
1047 for ( ; start <= end; start += sizeof(u32))
1048 *p++ = t4_read_reg(ap, start);
1049 }
1050
1051 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1052 void *buf)
1053 {
1054 static const unsigned int reg_ranges[] = {
1055 0x1008, 0x1108,
1056 0x1180, 0x11b4,
1057 0x11fc, 0x123c,
1058 0x1300, 0x173c,
1059 0x1800, 0x18fc,
1060 0x3000, 0x30d8,
1061 0x30e0, 0x5924,
1062 0x5960, 0x59d4,
1063 0x5a00, 0x5af8,
1064 0x6000, 0x6098,
1065 0x6100, 0x6150,
1066 0x6200, 0x6208,
1067 0x6240, 0x6248,
1068 0x6280, 0x6338,
1069 0x6370, 0x638c,
1070 0x6400, 0x643c,
1071 0x6500, 0x6524,
1072 0x6a00, 0x6a38,
1073 0x6a60, 0x6a78,
1074 0x6b00, 0x6b84,
1075 0x6bf0, 0x6c84,
1076 0x6cf0, 0x6d84,
1077 0x6df0, 0x6e84,
1078 0x6ef0, 0x6f84,
1079 0x6ff0, 0x7084,
1080 0x70f0, 0x7184,
1081 0x71f0, 0x7284,
1082 0x72f0, 0x7384,
1083 0x73f0, 0x7450,
1084 0x7500, 0x7530,
1085 0x7600, 0x761c,
1086 0x7680, 0x76cc,
1087 0x7700, 0x7798,
1088 0x77c0, 0x77fc,
1089 0x7900, 0x79fc,
1090 0x7b00, 0x7c38,
1091 0x7d00, 0x7efc,
1092 0x8dc0, 0x8e1c,
1093 0x8e30, 0x8e78,
1094 0x8ea0, 0x8f6c,
1095 0x8fc0, 0x9074,
1096 0x90fc, 0x90fc,
1097 0x9400, 0x9458,
1098 0x9600, 0x96bc,
1099 0x9800, 0x9808,
1100 0x9820, 0x983c,
1101 0x9850, 0x9864,
1102 0x9c00, 0x9c6c,
1103 0x9c80, 0x9cec,
1104 0x9d00, 0x9d6c,
1105 0x9d80, 0x9dec,
1106 0x9e00, 0x9e6c,
1107 0x9e80, 0x9eec,
1108 0x9f00, 0x9f6c,
1109 0x9f80, 0x9fec,
1110 0xd004, 0xd03c,
1111 0xdfc0, 0xdfe0,
1112 0xe000, 0xea7c,
1113 0xf000, 0x11190,
1114 0x19040, 0x19124,
1115 0x19150, 0x191b0,
1116 0x191d0, 0x191e8,
1117 0x19238, 0x1924c,
1118 0x193f8, 0x19474,
1119 0x19490, 0x194f8,
1120 0x19800, 0x19f30,
1121 0x1a000, 0x1a06c,
1122 0x1a0b0, 0x1a120,
1123 0x1a128, 0x1a138,
1124 0x1a190, 0x1a1c4,
1125 0x1a1fc, 0x1a1fc,
1126 0x1e040, 0x1e04c,
1127 0x1e240, 0x1e28c,
1128 0x1e2c0, 0x1e2c0,
1129 0x1e2e0, 0x1e2e0,
1130 0x1e300, 0x1e384,
1131 0x1e3c0, 0x1e3c8,
1132 0x1e440, 0x1e44c,
1133 0x1e640, 0x1e68c,
1134 0x1e6c0, 0x1e6c0,
1135 0x1e6e0, 0x1e6e0,
1136 0x1e700, 0x1e784,
1137 0x1e7c0, 0x1e7c8,
1138 0x1e840, 0x1e84c,
1139 0x1ea40, 0x1ea8c,
1140 0x1eac0, 0x1eac0,
1141 0x1eae0, 0x1eae0,
1142 0x1eb00, 0x1eb84,
1143 0x1ebc0, 0x1ebc8,
1144 0x1ec40, 0x1ec4c,
1145 0x1ee40, 0x1ee8c,
1146 0x1eec0, 0x1eec0,
1147 0x1eee0, 0x1eee0,
1148 0x1ef00, 0x1ef84,
1149 0x1efc0, 0x1efc8,
1150 0x1f040, 0x1f04c,
1151 0x1f240, 0x1f28c,
1152 0x1f2c0, 0x1f2c0,
1153 0x1f2e0, 0x1f2e0,
1154 0x1f300, 0x1f384,
1155 0x1f3c0, 0x1f3c8,
1156 0x1f440, 0x1f44c,
1157 0x1f640, 0x1f68c,
1158 0x1f6c0, 0x1f6c0,
1159 0x1f6e0, 0x1f6e0,
1160 0x1f700, 0x1f784,
1161 0x1f7c0, 0x1f7c8,
1162 0x1f840, 0x1f84c,
1163 0x1fa40, 0x1fa8c,
1164 0x1fac0, 0x1fac0,
1165 0x1fae0, 0x1fae0,
1166 0x1fb00, 0x1fb84,
1167 0x1fbc0, 0x1fbc8,
1168 0x1fc40, 0x1fc4c,
1169 0x1fe40, 0x1fe8c,
1170 0x1fec0, 0x1fec0,
1171 0x1fee0, 0x1fee0,
1172 0x1ff00, 0x1ff84,
1173 0x1ffc0, 0x1ffc8,
1174 0x20000, 0x2002c,
1175 0x20100, 0x2013c,
1176 0x20190, 0x201c8,
1177 0x20200, 0x20318,
1178 0x20400, 0x20528,
1179 0x20540, 0x20614,
1180 0x21000, 0x21040,
1181 0x2104c, 0x21060,
1182 0x210c0, 0x210ec,
1183 0x21200, 0x21268,
1184 0x21270, 0x21284,
1185 0x212fc, 0x21388,
1186 0x21400, 0x21404,
1187 0x21500, 0x21518,
1188 0x2152c, 0x2153c,
1189 0x21550, 0x21554,
1190 0x21600, 0x21600,
1191 0x21608, 0x21628,
1192 0x21630, 0x2163c,
1193 0x21700, 0x2171c,
1194 0x21780, 0x2178c,
1195 0x21800, 0x21c38,
1196 0x21c80, 0x21d7c,
1197 0x21e00, 0x21e04,
1198 0x22000, 0x2202c,
1199 0x22100, 0x2213c,
1200 0x22190, 0x221c8,
1201 0x22200, 0x22318,
1202 0x22400, 0x22528,
1203 0x22540, 0x22614,
1204 0x23000, 0x23040,
1205 0x2304c, 0x23060,
1206 0x230c0, 0x230ec,
1207 0x23200, 0x23268,
1208 0x23270, 0x23284,
1209 0x232fc, 0x23388,
1210 0x23400, 0x23404,
1211 0x23500, 0x23518,
1212 0x2352c, 0x2353c,
1213 0x23550, 0x23554,
1214 0x23600, 0x23600,
1215 0x23608, 0x23628,
1216 0x23630, 0x2363c,
1217 0x23700, 0x2371c,
1218 0x23780, 0x2378c,
1219 0x23800, 0x23c38,
1220 0x23c80, 0x23d7c,
1221 0x23e00, 0x23e04,
1222 0x24000, 0x2402c,
1223 0x24100, 0x2413c,
1224 0x24190, 0x241c8,
1225 0x24200, 0x24318,
1226 0x24400, 0x24528,
1227 0x24540, 0x24614,
1228 0x25000, 0x25040,
1229 0x2504c, 0x25060,
1230 0x250c0, 0x250ec,
1231 0x25200, 0x25268,
1232 0x25270, 0x25284,
1233 0x252fc, 0x25388,
1234 0x25400, 0x25404,
1235 0x25500, 0x25518,
1236 0x2552c, 0x2553c,
1237 0x25550, 0x25554,
1238 0x25600, 0x25600,
1239 0x25608, 0x25628,
1240 0x25630, 0x2563c,
1241 0x25700, 0x2571c,
1242 0x25780, 0x2578c,
1243 0x25800, 0x25c38,
1244 0x25c80, 0x25d7c,
1245 0x25e00, 0x25e04,
1246 0x26000, 0x2602c,
1247 0x26100, 0x2613c,
1248 0x26190, 0x261c8,
1249 0x26200, 0x26318,
1250 0x26400, 0x26528,
1251 0x26540, 0x26614,
1252 0x27000, 0x27040,
1253 0x2704c, 0x27060,
1254 0x270c0, 0x270ec,
1255 0x27200, 0x27268,
1256 0x27270, 0x27284,
1257 0x272fc, 0x27388,
1258 0x27400, 0x27404,
1259 0x27500, 0x27518,
1260 0x2752c, 0x2753c,
1261 0x27550, 0x27554,
1262 0x27600, 0x27600,
1263 0x27608, 0x27628,
1264 0x27630, 0x2763c,
1265 0x27700, 0x2771c,
1266 0x27780, 0x2778c,
1267 0x27800, 0x27c38,
1268 0x27c80, 0x27d7c,
1269 0x27e00, 0x27e04
1270 };
1271
1272 int i;
1273 struct adapter *ap = netdev2adap(dev);
1274
1275 regs->version = mk_adap_vers(ap);
1276
1277 memset(buf, 0, T4_REGMAP_SIZE);
1278 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1279 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1280 }
1281
1282 static int restart_autoneg(struct net_device *dev)
1283 {
1284 struct port_info *p = netdev_priv(dev);
1285
1286 if (!netif_running(dev))
1287 return -EAGAIN;
1288 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1289 return -EINVAL;
1290 t4_restart_aneg(p->adapter, 0, p->tx_chan);
1291 return 0;
1292 }
1293
1294 static int identify_port(struct net_device *dev, u32 data)
1295 {
1296 if (data == 0)
1297 data = 2; /* default to 2 seconds */
1298
1299 return t4_identify_port(netdev2adap(dev), 0, netdev2pinfo(dev)->viid,
1300 data * 5);
1301 }
1302
1303 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1304 {
1305 unsigned int v = 0;
1306
1307 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1308 type == FW_PORT_TYPE_BT_XAUI) {
1309 v |= SUPPORTED_TP;
1310 if (caps & FW_PORT_CAP_SPEED_100M)
1311 v |= SUPPORTED_100baseT_Full;
1312 if (caps & FW_PORT_CAP_SPEED_1G)
1313 v |= SUPPORTED_1000baseT_Full;
1314 if (caps & FW_PORT_CAP_SPEED_10G)
1315 v |= SUPPORTED_10000baseT_Full;
1316 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1317 v |= SUPPORTED_Backplane;
1318 if (caps & FW_PORT_CAP_SPEED_1G)
1319 v |= SUPPORTED_1000baseKX_Full;
1320 if (caps & FW_PORT_CAP_SPEED_10G)
1321 v |= SUPPORTED_10000baseKX4_Full;
1322 } else if (type == FW_PORT_TYPE_KR)
1323 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1324 else if (type == FW_PORT_TYPE_BP_AP)
1325 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1326 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1327 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1328 v |= SUPPORTED_FIBRE;
1329
1330 if (caps & FW_PORT_CAP_ANEG)
1331 v |= SUPPORTED_Autoneg;
1332 return v;
1333 }
1334
1335 static unsigned int to_fw_linkcaps(unsigned int caps)
1336 {
1337 unsigned int v = 0;
1338
1339 if (caps & ADVERTISED_100baseT_Full)
1340 v |= FW_PORT_CAP_SPEED_100M;
1341 if (caps & ADVERTISED_1000baseT_Full)
1342 v |= FW_PORT_CAP_SPEED_1G;
1343 if (caps & ADVERTISED_10000baseT_Full)
1344 v |= FW_PORT_CAP_SPEED_10G;
1345 return v;
1346 }
1347
1348 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1349 {
1350 const struct port_info *p = netdev_priv(dev);
1351
1352 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1353 p->port_type == FW_PORT_TYPE_BT_XFI ||
1354 p->port_type == FW_PORT_TYPE_BT_XAUI)
1355 cmd->port = PORT_TP;
1356 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1357 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1358 cmd->port = PORT_FIBRE;
1359 else if (p->port_type == FW_PORT_TYPE_SFP) {
1360 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1361 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1362 cmd->port = PORT_DA;
1363 else
1364 cmd->port = PORT_FIBRE;
1365 } else
1366 cmd->port = PORT_OTHER;
1367
1368 if (p->mdio_addr >= 0) {
1369 cmd->phy_address = p->mdio_addr;
1370 cmd->transceiver = XCVR_EXTERNAL;
1371 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1372 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1373 } else {
1374 cmd->phy_address = 0; /* not really, but no better option */
1375 cmd->transceiver = XCVR_INTERNAL;
1376 cmd->mdio_support = 0;
1377 }
1378
1379 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1380 cmd->advertising = from_fw_linkcaps(p->port_type,
1381 p->link_cfg.advertising);
1382 cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1383 cmd->duplex = DUPLEX_FULL;
1384 cmd->autoneg = p->link_cfg.autoneg;
1385 cmd->maxtxpkt = 0;
1386 cmd->maxrxpkt = 0;
1387 return 0;
1388 }
1389
1390 static unsigned int speed_to_caps(int speed)
1391 {
1392 if (speed == SPEED_100)
1393 return FW_PORT_CAP_SPEED_100M;
1394 if (speed == SPEED_1000)
1395 return FW_PORT_CAP_SPEED_1G;
1396 if (speed == SPEED_10000)
1397 return FW_PORT_CAP_SPEED_10G;
1398 return 0;
1399 }
1400
1401 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1402 {
1403 unsigned int cap;
1404 struct port_info *p = netdev_priv(dev);
1405 struct link_config *lc = &p->link_cfg;
1406
1407 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1408 return -EINVAL;
1409
1410 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1411 /*
1412 * PHY offers a single speed. See if that's what's
1413 * being requested.
1414 */
1415 if (cmd->autoneg == AUTONEG_DISABLE &&
1416 (lc->supported & speed_to_caps(cmd->speed)))
1417 return 0;
1418 return -EINVAL;
1419 }
1420
1421 if (cmd->autoneg == AUTONEG_DISABLE) {
1422 cap = speed_to_caps(cmd->speed);
1423
1424 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1425 cmd->speed == SPEED_10000)
1426 return -EINVAL;
1427 lc->requested_speed = cap;
1428 lc->advertising = 0;
1429 } else {
1430 cap = to_fw_linkcaps(cmd->advertising);
1431 if (!(lc->supported & cap))
1432 return -EINVAL;
1433 lc->requested_speed = 0;
1434 lc->advertising = cap | FW_PORT_CAP_ANEG;
1435 }
1436 lc->autoneg = cmd->autoneg;
1437
1438 if (netif_running(dev))
1439 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1440 return 0;
1441 }
1442
1443 static void get_pauseparam(struct net_device *dev,
1444 struct ethtool_pauseparam *epause)
1445 {
1446 struct port_info *p = netdev_priv(dev);
1447
1448 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1449 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1450 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1451 }
1452
1453 static int set_pauseparam(struct net_device *dev,
1454 struct ethtool_pauseparam *epause)
1455 {
1456 struct port_info *p = netdev_priv(dev);
1457 struct link_config *lc = &p->link_cfg;
1458
1459 if (epause->autoneg == AUTONEG_DISABLE)
1460 lc->requested_fc = 0;
1461 else if (lc->supported & FW_PORT_CAP_ANEG)
1462 lc->requested_fc = PAUSE_AUTONEG;
1463 else
1464 return -EINVAL;
1465
1466 if (epause->rx_pause)
1467 lc->requested_fc |= PAUSE_RX;
1468 if (epause->tx_pause)
1469 lc->requested_fc |= PAUSE_TX;
1470 if (netif_running(dev))
1471 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1472 return 0;
1473 }
1474
1475 static u32 get_rx_csum(struct net_device *dev)
1476 {
1477 struct port_info *p = netdev_priv(dev);
1478
1479 return p->rx_offload & RX_CSO;
1480 }
1481
1482 static int set_rx_csum(struct net_device *dev, u32 data)
1483 {
1484 struct port_info *p = netdev_priv(dev);
1485
1486 if (data)
1487 p->rx_offload |= RX_CSO;
1488 else
1489 p->rx_offload &= ~RX_CSO;
1490 return 0;
1491 }
1492
1493 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1494 {
1495 const struct port_info *pi = netdev_priv(dev);
1496 const struct sge *s = &pi->adapter->sge;
1497
1498 e->rx_max_pending = MAX_RX_BUFFERS;
1499 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1500 e->rx_jumbo_max_pending = 0;
1501 e->tx_max_pending = MAX_TXQ_ENTRIES;
1502
1503 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1504 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1505 e->rx_jumbo_pending = 0;
1506 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1507 }
1508
1509 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1510 {
1511 int i;
1512 const struct port_info *pi = netdev_priv(dev);
1513 struct adapter *adapter = pi->adapter;
1514 struct sge *s = &adapter->sge;
1515
1516 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1517 e->tx_pending > MAX_TXQ_ENTRIES ||
1518 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1519 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1520 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1521 return -EINVAL;
1522
1523 if (adapter->flags & FULL_INIT_DONE)
1524 return -EBUSY;
1525
1526 for (i = 0; i < pi->nqsets; ++i) {
1527 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1528 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1529 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1530 }
1531 return 0;
1532 }
1533
1534 static int closest_timer(const struct sge *s, int time)
1535 {
1536 int i, delta, match = 0, min_delta = INT_MAX;
1537
1538 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1539 delta = time - s->timer_val[i];
1540 if (delta < 0)
1541 delta = -delta;
1542 if (delta < min_delta) {
1543 min_delta = delta;
1544 match = i;
1545 }
1546 }
1547 return match;
1548 }
1549
1550 static int closest_thres(const struct sge *s, int thres)
1551 {
1552 int i, delta, match = 0, min_delta = INT_MAX;
1553
1554 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1555 delta = thres - s->counter_val[i];
1556 if (delta < 0)
1557 delta = -delta;
1558 if (delta < min_delta) {
1559 min_delta = delta;
1560 match = i;
1561 }
1562 }
1563 return match;
1564 }
1565
1566 /*
1567 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1568 */
1569 static unsigned int qtimer_val(const struct adapter *adap,
1570 const struct sge_rspq *q)
1571 {
1572 unsigned int idx = q->intr_params >> 1;
1573
1574 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1575 }
1576
1577 /**
1578 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1579 * @adap: the adapter
1580 * @q: the Rx queue
1581 * @us: the hold-off time in us, or 0 to disable timer
1582 * @cnt: the hold-off packet count, or 0 to disable counter
1583 *
1584 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1585 * one of the two needs to be enabled for the queue to generate interrupts.
1586 */
1587 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1588 unsigned int us, unsigned int cnt)
1589 {
1590 if ((us | cnt) == 0)
1591 cnt = 1;
1592
1593 if (cnt) {
1594 int err;
1595 u32 v, new_idx;
1596
1597 new_idx = closest_thres(&adap->sge, cnt);
1598 if (q->desc && q->pktcnt_idx != new_idx) {
1599 /* the queue has already been created, update it */
1600 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1601 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1602 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1603 err = t4_set_params(adap, 0, 0, 0, 1, &v, &new_idx);
1604 if (err)
1605 return err;
1606 }
1607 q->pktcnt_idx = new_idx;
1608 }
1609
1610 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1611 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1612 return 0;
1613 }
1614
1615 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1616 {
1617 const struct port_info *pi = netdev_priv(dev);
1618 struct adapter *adap = pi->adapter;
1619
1620 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1621 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1622 }
1623
1624 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1625 {
1626 const struct port_info *pi = netdev_priv(dev);
1627 const struct adapter *adap = pi->adapter;
1628 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1629
1630 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1631 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1632 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1633 return 0;
1634 }
1635
1636 /*
1637 * Translate a physical EEPROM address to virtual. The first 1K is accessed
1638 * through virtual addresses starting at 31K, the rest is accessed through
1639 * virtual addresses starting at 0. This mapping is correct only for PF0.
1640 */
1641 static int eeprom_ptov(unsigned int phys_addr)
1642 {
1643 if (phys_addr < 1024)
1644 return phys_addr + (31 << 10);
1645 if (phys_addr < EEPROMSIZE)
1646 return phys_addr - 1024;
1647 return -EINVAL;
1648 }
1649
1650 /*
1651 * The next two routines implement eeprom read/write from physical addresses.
1652 * The physical->virtual translation is correct only for PF0.
1653 */
1654 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1655 {
1656 int vaddr = eeprom_ptov(phys_addr);
1657
1658 if (vaddr >= 0)
1659 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1660 return vaddr < 0 ? vaddr : 0;
1661 }
1662
1663 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1664 {
1665 int vaddr = eeprom_ptov(phys_addr);
1666
1667 if (vaddr >= 0)
1668 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1669 return vaddr < 0 ? vaddr : 0;
1670 }
1671
1672 #define EEPROM_MAGIC 0x38E2F10C
1673
1674 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1675 u8 *data)
1676 {
1677 int i, err = 0;
1678 struct adapter *adapter = netdev2adap(dev);
1679
1680 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1681 if (!buf)
1682 return -ENOMEM;
1683
1684 e->magic = EEPROM_MAGIC;
1685 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1686 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1687
1688 if (!err)
1689 memcpy(data, buf + e->offset, e->len);
1690 kfree(buf);
1691 return err;
1692 }
1693
1694 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1695 u8 *data)
1696 {
1697 u8 *buf;
1698 int err = 0;
1699 u32 aligned_offset, aligned_len, *p;
1700 struct adapter *adapter = netdev2adap(dev);
1701
1702 if (eeprom->magic != EEPROM_MAGIC)
1703 return -EINVAL;
1704
1705 aligned_offset = eeprom->offset & ~3;
1706 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1707
1708 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1709 /*
1710 * RMW possibly needed for first or last words.
1711 */
1712 buf = kmalloc(aligned_len, GFP_KERNEL);
1713 if (!buf)
1714 return -ENOMEM;
1715 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1716 if (!err && aligned_len > 4)
1717 err = eeprom_rd_phys(adapter,
1718 aligned_offset + aligned_len - 4,
1719 (u32 *)&buf[aligned_len - 4]);
1720 if (err)
1721 goto out;
1722 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1723 } else
1724 buf = data;
1725
1726 err = t4_seeprom_wp(adapter, false);
1727 if (err)
1728 goto out;
1729
1730 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1731 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1732 aligned_offset += 4;
1733 }
1734
1735 if (!err)
1736 err = t4_seeprom_wp(adapter, true);
1737 out:
1738 if (buf != data)
1739 kfree(buf);
1740 return err;
1741 }
1742
1743 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1744 {
1745 int ret;
1746 const struct firmware *fw;
1747 struct adapter *adap = netdev2adap(netdev);
1748
1749 ef->data[sizeof(ef->data) - 1] = '\0';
1750 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1751 if (ret < 0)
1752 return ret;
1753
1754 ret = t4_load_fw(adap, fw->data, fw->size);
1755 release_firmware(fw);
1756 if (!ret)
1757 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1758 return ret;
1759 }
1760
1761 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1762 #define BCAST_CRC 0xa0ccc1a6
1763
1764 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1765 {
1766 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1767 wol->wolopts = netdev2adap(dev)->wol;
1768 memset(&wol->sopass, 0, sizeof(wol->sopass));
1769 }
1770
1771 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1772 {
1773 int err = 0;
1774 struct port_info *pi = netdev_priv(dev);
1775
1776 if (wol->wolopts & ~WOL_SUPPORTED)
1777 return -EINVAL;
1778 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1779 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1780 if (wol->wolopts & WAKE_BCAST) {
1781 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1782 ~0ULL, 0, false);
1783 if (!err)
1784 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1785 ~6ULL, ~0ULL, BCAST_CRC, true);
1786 } else
1787 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1788 return err;
1789 }
1790
1791 static int set_tso(struct net_device *dev, u32 value)
1792 {
1793 if (value)
1794 dev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1795 else
1796 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
1797 return 0;
1798 }
1799
1800 static int set_flags(struct net_device *dev, u32 flags)
1801 {
1802 return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
1803 }
1804
1805 static struct ethtool_ops cxgb_ethtool_ops = {
1806 .get_settings = get_settings,
1807 .set_settings = set_settings,
1808 .get_drvinfo = get_drvinfo,
1809 .get_msglevel = get_msglevel,
1810 .set_msglevel = set_msglevel,
1811 .get_ringparam = get_sge_param,
1812 .set_ringparam = set_sge_param,
1813 .get_coalesce = get_coalesce,
1814 .set_coalesce = set_coalesce,
1815 .get_eeprom_len = get_eeprom_len,
1816 .get_eeprom = get_eeprom,
1817 .set_eeprom = set_eeprom,
1818 .get_pauseparam = get_pauseparam,
1819 .set_pauseparam = set_pauseparam,
1820 .get_rx_csum = get_rx_csum,
1821 .set_rx_csum = set_rx_csum,
1822 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
1823 .set_sg = ethtool_op_set_sg,
1824 .get_link = ethtool_op_get_link,
1825 .get_strings = get_strings,
1826 .phys_id = identify_port,
1827 .nway_reset = restart_autoneg,
1828 .get_sset_count = get_sset_count,
1829 .get_ethtool_stats = get_stats,
1830 .get_regs_len = get_regs_len,
1831 .get_regs = get_regs,
1832 .get_wol = get_wol,
1833 .set_wol = set_wol,
1834 .set_tso = set_tso,
1835 .set_flags = set_flags,
1836 .flash_device = set_flash,
1837 };
1838
1839 /*
1840 * debugfs support
1841 */
1842
1843 static int mem_open(struct inode *inode, struct file *file)
1844 {
1845 file->private_data = inode->i_private;
1846 return 0;
1847 }
1848
1849 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1850 loff_t *ppos)
1851 {
1852 loff_t pos = *ppos;
1853 loff_t avail = file->f_path.dentry->d_inode->i_size;
1854 unsigned int mem = (uintptr_t)file->private_data & 3;
1855 struct adapter *adap = file->private_data - mem;
1856
1857 if (pos < 0)
1858 return -EINVAL;
1859 if (pos >= avail)
1860 return 0;
1861 if (count > avail - pos)
1862 count = avail - pos;
1863
1864 while (count) {
1865 size_t len;
1866 int ret, ofst;
1867 __be32 data[16];
1868
1869 if (mem == MEM_MC)
1870 ret = t4_mc_read(adap, pos, data, NULL);
1871 else
1872 ret = t4_edc_read(adap, mem, pos, data, NULL);
1873 if (ret)
1874 return ret;
1875
1876 ofst = pos % sizeof(data);
1877 len = min(count, sizeof(data) - ofst);
1878 if (copy_to_user(buf, (u8 *)data + ofst, len))
1879 return -EFAULT;
1880
1881 buf += len;
1882 pos += len;
1883 count -= len;
1884 }
1885 count = pos - *ppos;
1886 *ppos = pos;
1887 return count;
1888 }
1889
1890 static const struct file_operations mem_debugfs_fops = {
1891 .owner = THIS_MODULE,
1892 .open = mem_open,
1893 .read = mem_read,
1894 };
1895
1896 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
1897 unsigned int idx, unsigned int size_mb)
1898 {
1899 struct dentry *de;
1900
1901 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
1902 (void *)adap + idx, &mem_debugfs_fops);
1903 if (de && de->d_inode)
1904 de->d_inode->i_size = size_mb << 20;
1905 }
1906
1907 static int __devinit setup_debugfs(struct adapter *adap)
1908 {
1909 int i;
1910
1911 if (IS_ERR_OR_NULL(adap->debugfs_root))
1912 return -1;
1913
1914 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
1915 if (i & EDRAM0_ENABLE)
1916 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
1917 if (i & EDRAM1_ENABLE)
1918 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
1919 if (i & EXT_MEM_ENABLE)
1920 add_debugfs_mem(adap, "mc", MEM_MC,
1921 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
1922 if (adap->l2t)
1923 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
1924 &t4_l2t_fops);
1925 return 0;
1926 }
1927
1928 /*
1929 * upper-layer driver support
1930 */
1931
1932 /*
1933 * Allocate an active-open TID and set it to the supplied value.
1934 */
1935 int cxgb4_alloc_atid(struct tid_info *t, void *data)
1936 {
1937 int atid = -1;
1938
1939 spin_lock_bh(&t->atid_lock);
1940 if (t->afree) {
1941 union aopen_entry *p = t->afree;
1942
1943 atid = p - t->atid_tab;
1944 t->afree = p->next;
1945 p->data = data;
1946 t->atids_in_use++;
1947 }
1948 spin_unlock_bh(&t->atid_lock);
1949 return atid;
1950 }
1951 EXPORT_SYMBOL(cxgb4_alloc_atid);
1952
1953 /*
1954 * Release an active-open TID.
1955 */
1956 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
1957 {
1958 union aopen_entry *p = &t->atid_tab[atid];
1959
1960 spin_lock_bh(&t->atid_lock);
1961 p->next = t->afree;
1962 t->afree = p;
1963 t->atids_in_use--;
1964 spin_unlock_bh(&t->atid_lock);
1965 }
1966 EXPORT_SYMBOL(cxgb4_free_atid);
1967
1968 /*
1969 * Allocate a server TID and set it to the supplied value.
1970 */
1971 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
1972 {
1973 int stid;
1974
1975 spin_lock_bh(&t->stid_lock);
1976 if (family == PF_INET) {
1977 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
1978 if (stid < t->nstids)
1979 __set_bit(stid, t->stid_bmap);
1980 else
1981 stid = -1;
1982 } else {
1983 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
1984 if (stid < 0)
1985 stid = -1;
1986 }
1987 if (stid >= 0) {
1988 t->stid_tab[stid].data = data;
1989 stid += t->stid_base;
1990 t->stids_in_use++;
1991 }
1992 spin_unlock_bh(&t->stid_lock);
1993 return stid;
1994 }
1995 EXPORT_SYMBOL(cxgb4_alloc_stid);
1996
1997 /*
1998 * Release a server TID.
1999 */
2000 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2001 {
2002 stid -= t->stid_base;
2003 spin_lock_bh(&t->stid_lock);
2004 if (family == PF_INET)
2005 __clear_bit(stid, t->stid_bmap);
2006 else
2007 bitmap_release_region(t->stid_bmap, stid, 2);
2008 t->stid_tab[stid].data = NULL;
2009 t->stids_in_use--;
2010 spin_unlock_bh(&t->stid_lock);
2011 }
2012 EXPORT_SYMBOL(cxgb4_free_stid);
2013
2014 /*
2015 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2016 */
2017 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2018 unsigned int tid)
2019 {
2020 struct cpl_tid_release *req;
2021
2022 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2023 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2024 INIT_TP_WR(req, tid);
2025 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2026 }
2027
2028 /*
2029 * Queue a TID release request and if necessary schedule a work queue to
2030 * process it.
2031 */
2032 void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2033 unsigned int tid)
2034 {
2035 void **p = &t->tid_tab[tid];
2036 struct adapter *adap = container_of(t, struct adapter, tids);
2037
2038 spin_lock_bh(&adap->tid_release_lock);
2039 *p = adap->tid_release_head;
2040 /* Low 2 bits encode the Tx channel number */
2041 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2042 if (!adap->tid_release_task_busy) {
2043 adap->tid_release_task_busy = true;
2044 schedule_work(&adap->tid_release_task);
2045 }
2046 spin_unlock_bh(&adap->tid_release_lock);
2047 }
2048 EXPORT_SYMBOL(cxgb4_queue_tid_release);
2049
2050 /*
2051 * Process the list of pending TID release requests.
2052 */
2053 static void process_tid_release_list(struct work_struct *work)
2054 {
2055 struct sk_buff *skb;
2056 struct adapter *adap;
2057
2058 adap = container_of(work, struct adapter, tid_release_task);
2059
2060 spin_lock_bh(&adap->tid_release_lock);
2061 while (adap->tid_release_head) {
2062 void **p = adap->tid_release_head;
2063 unsigned int chan = (uintptr_t)p & 3;
2064 p = (void *)p - chan;
2065
2066 adap->tid_release_head = *p;
2067 *p = NULL;
2068 spin_unlock_bh(&adap->tid_release_lock);
2069
2070 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2071 GFP_KERNEL)))
2072 schedule_timeout_uninterruptible(1);
2073
2074 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2075 t4_ofld_send(adap, skb);
2076 spin_lock_bh(&adap->tid_release_lock);
2077 }
2078 adap->tid_release_task_busy = false;
2079 spin_unlock_bh(&adap->tid_release_lock);
2080 }
2081
2082 /*
2083 * Release a TID and inform HW. If we are unable to allocate the release
2084 * message we defer to a work queue.
2085 */
2086 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2087 {
2088 void *old;
2089 struct sk_buff *skb;
2090 struct adapter *adap = container_of(t, struct adapter, tids);
2091
2092 old = t->tid_tab[tid];
2093 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2094 if (likely(skb)) {
2095 t->tid_tab[tid] = NULL;
2096 mk_tid_release(skb, chan, tid);
2097 t4_ofld_send(adap, skb);
2098 } else
2099 cxgb4_queue_tid_release(t, chan, tid);
2100 if (old)
2101 atomic_dec(&t->tids_in_use);
2102 }
2103 EXPORT_SYMBOL(cxgb4_remove_tid);
2104
2105 /*
2106 * Allocate and initialize the TID tables. Returns 0 on success.
2107 */
2108 static int tid_init(struct tid_info *t)
2109 {
2110 size_t size;
2111 unsigned int natids = t->natids;
2112
2113 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2114 t->nstids * sizeof(*t->stid_tab) +
2115 BITS_TO_LONGS(t->nstids) * sizeof(long);
2116 t->tid_tab = t4_alloc_mem(size);
2117 if (!t->tid_tab)
2118 return -ENOMEM;
2119
2120 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2121 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2122 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2123 spin_lock_init(&t->stid_lock);
2124 spin_lock_init(&t->atid_lock);
2125
2126 t->stids_in_use = 0;
2127 t->afree = NULL;
2128 t->atids_in_use = 0;
2129 atomic_set(&t->tids_in_use, 0);
2130
2131 /* Setup the free list for atid_tab and clear the stid bitmap. */
2132 if (natids) {
2133 while (--natids)
2134 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2135 t->afree = t->atid_tab;
2136 }
2137 bitmap_zero(t->stid_bmap, t->nstids);
2138 return 0;
2139 }
2140
2141 /**
2142 * cxgb4_create_server - create an IP server
2143 * @dev: the device
2144 * @stid: the server TID
2145 * @sip: local IP address to bind server to
2146 * @sport: the server's TCP port
2147 * @queue: queue to direct messages from this server to
2148 *
2149 * Create an IP server for the given port and address.
2150 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2151 */
2152 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2153 __be32 sip, __be16 sport, unsigned int queue)
2154 {
2155 unsigned int chan;
2156 struct sk_buff *skb;
2157 struct adapter *adap;
2158 struct cpl_pass_open_req *req;
2159
2160 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2161 if (!skb)
2162 return -ENOMEM;
2163
2164 adap = netdev2adap(dev);
2165 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2166 INIT_TP_WR(req, 0);
2167 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2168 req->local_port = sport;
2169 req->peer_port = htons(0);
2170 req->local_ip = sip;
2171 req->peer_ip = htonl(0);
2172 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2173 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2174 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2175 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2176 return t4_mgmt_tx(adap, skb);
2177 }
2178 EXPORT_SYMBOL(cxgb4_create_server);
2179
2180 /**
2181 * cxgb4_create_server6 - create an IPv6 server
2182 * @dev: the device
2183 * @stid: the server TID
2184 * @sip: local IPv6 address to bind server to
2185 * @sport: the server's TCP port
2186 * @queue: queue to direct messages from this server to
2187 *
2188 * Create an IPv6 server for the given port and address.
2189 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2190 */
2191 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2192 const struct in6_addr *sip, __be16 sport,
2193 unsigned int queue)
2194 {
2195 unsigned int chan;
2196 struct sk_buff *skb;
2197 struct adapter *adap;
2198 struct cpl_pass_open_req6 *req;
2199
2200 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2201 if (!skb)
2202 return -ENOMEM;
2203
2204 adap = netdev2adap(dev);
2205 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2206 INIT_TP_WR(req, 0);
2207 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2208 req->local_port = sport;
2209 req->peer_port = htons(0);
2210 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2211 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2212 req->peer_ip_hi = cpu_to_be64(0);
2213 req->peer_ip_lo = cpu_to_be64(0);
2214 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2215 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2216 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2217 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2218 return t4_mgmt_tx(adap, skb);
2219 }
2220 EXPORT_SYMBOL(cxgb4_create_server6);
2221
2222 /**
2223 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2224 * @mtus: the HW MTU table
2225 * @mtu: the target MTU
2226 * @idx: index of selected entry in the MTU table
2227 *
2228 * Returns the index and the value in the HW MTU table that is closest to
2229 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2230 * table, in which case that smallest available value is selected.
2231 */
2232 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2233 unsigned int *idx)
2234 {
2235 unsigned int i = 0;
2236
2237 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2238 ++i;
2239 if (idx)
2240 *idx = i;
2241 return mtus[i];
2242 }
2243 EXPORT_SYMBOL(cxgb4_best_mtu);
2244
2245 /**
2246 * cxgb4_port_chan - get the HW channel of a port
2247 * @dev: the net device for the port
2248 *
2249 * Return the HW Tx channel of the given port.
2250 */
2251 unsigned int cxgb4_port_chan(const struct net_device *dev)
2252 {
2253 return netdev2pinfo(dev)->tx_chan;
2254 }
2255 EXPORT_SYMBOL(cxgb4_port_chan);
2256
2257 /**
2258 * cxgb4_port_viid - get the VI id of a port
2259 * @dev: the net device for the port
2260 *
2261 * Return the VI id of the given port.
2262 */
2263 unsigned int cxgb4_port_viid(const struct net_device *dev)
2264 {
2265 return netdev2pinfo(dev)->viid;
2266 }
2267 EXPORT_SYMBOL(cxgb4_port_viid);
2268
2269 /**
2270 * cxgb4_port_idx - get the index of a port
2271 * @dev: the net device for the port
2272 *
2273 * Return the index of the given port.
2274 */
2275 unsigned int cxgb4_port_idx(const struct net_device *dev)
2276 {
2277 return netdev2pinfo(dev)->port_id;
2278 }
2279 EXPORT_SYMBOL(cxgb4_port_idx);
2280
2281 /**
2282 * cxgb4_netdev_by_hwid - return the net device of a HW port
2283 * @pdev: identifies the adapter
2284 * @id: the HW port id
2285 *
2286 * Return the net device associated with the interface with the given HW
2287 * id.
2288 */
2289 struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2290 {
2291 const struct adapter *adap = pci_get_drvdata(pdev);
2292
2293 if (!adap || id >= NCHAN)
2294 return NULL;
2295 id = adap->chan_map[id];
2296 return id < MAX_NPORTS ? adap->port[id] : NULL;
2297 }
2298 EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2299
2300 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2301 struct tp_tcp_stats *v6)
2302 {
2303 struct adapter *adap = pci_get_drvdata(pdev);
2304
2305 spin_lock(&adap->stats_lock);
2306 t4_tp_get_tcp_stats(adap, v4, v6);
2307 spin_unlock(&adap->stats_lock);
2308 }
2309 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2310
2311 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2312 const unsigned int *pgsz_order)
2313 {
2314 struct adapter *adap = netdev2adap(dev);
2315
2316 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2317 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2318 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2319 HPZ3(pgsz_order[3]));
2320 }
2321 EXPORT_SYMBOL(cxgb4_iscsi_init);
2322
2323 static struct pci_driver cxgb4_driver;
2324
2325 static void check_neigh_update(struct neighbour *neigh)
2326 {
2327 const struct device *parent;
2328 const struct net_device *netdev = neigh->dev;
2329
2330 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2331 netdev = vlan_dev_real_dev(netdev);
2332 parent = netdev->dev.parent;
2333 if (parent && parent->driver == &cxgb4_driver.driver)
2334 t4_l2t_update(dev_get_drvdata(parent), neigh);
2335 }
2336
2337 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2338 void *data)
2339 {
2340 switch (event) {
2341 case NETEVENT_NEIGH_UPDATE:
2342 check_neigh_update(data);
2343 break;
2344 case NETEVENT_PMTU_UPDATE:
2345 case NETEVENT_REDIRECT:
2346 default:
2347 break;
2348 }
2349 return 0;
2350 }
2351
2352 static bool netevent_registered;
2353 static struct notifier_block cxgb4_netevent_nb = {
2354 .notifier_call = netevent_cb
2355 };
2356
2357 static void uld_attach(struct adapter *adap, unsigned int uld)
2358 {
2359 void *handle;
2360 struct cxgb4_lld_info lli;
2361
2362 lli.pdev = adap->pdev;
2363 lli.l2t = adap->l2t;
2364 lli.tids = &adap->tids;
2365 lli.ports = adap->port;
2366 lli.vr = &adap->vres;
2367 lli.mtus = adap->params.mtus;
2368 if (uld == CXGB4_ULD_RDMA) {
2369 lli.rxq_ids = adap->sge.rdma_rxq;
2370 lli.nrxq = adap->sge.rdmaqs;
2371 } else if (uld == CXGB4_ULD_ISCSI) {
2372 lli.rxq_ids = adap->sge.ofld_rxq;
2373 lli.nrxq = adap->sge.ofldqsets;
2374 }
2375 lli.ntxq = adap->sge.ofldqsets;
2376 lli.nchan = adap->params.nports;
2377 lli.nports = adap->params.nports;
2378 lli.wr_cred = adap->params.ofldq_wr_cred;
2379 lli.adapter_type = adap->params.rev;
2380 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2381 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2382 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF));
2383 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2384 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF));
2385 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2386 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2387 lli.fw_vers = adap->params.fw_vers;
2388
2389 handle = ulds[uld].add(&lli);
2390 if (IS_ERR(handle)) {
2391 dev_warn(adap->pdev_dev,
2392 "could not attach to the %s driver, error %ld\n",
2393 uld_str[uld], PTR_ERR(handle));
2394 return;
2395 }
2396
2397 adap->uld_handle[uld] = handle;
2398
2399 if (!netevent_registered) {
2400 register_netevent_notifier(&cxgb4_netevent_nb);
2401 netevent_registered = true;
2402 }
2403
2404 if (adap->flags & FULL_INIT_DONE)
2405 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2406 }
2407
2408 static void attach_ulds(struct adapter *adap)
2409 {
2410 unsigned int i;
2411
2412 mutex_lock(&uld_mutex);
2413 list_add_tail(&adap->list_node, &adapter_list);
2414 for (i = 0; i < CXGB4_ULD_MAX; i++)
2415 if (ulds[i].add)
2416 uld_attach(adap, i);
2417 mutex_unlock(&uld_mutex);
2418 }
2419
2420 static void detach_ulds(struct adapter *adap)
2421 {
2422 unsigned int i;
2423
2424 mutex_lock(&uld_mutex);
2425 list_del(&adap->list_node);
2426 for (i = 0; i < CXGB4_ULD_MAX; i++)
2427 if (adap->uld_handle[i]) {
2428 ulds[i].state_change(adap->uld_handle[i],
2429 CXGB4_STATE_DETACH);
2430 adap->uld_handle[i] = NULL;
2431 }
2432 if (netevent_registered && list_empty(&adapter_list)) {
2433 unregister_netevent_notifier(&cxgb4_netevent_nb);
2434 netevent_registered = false;
2435 }
2436 mutex_unlock(&uld_mutex);
2437 }
2438
2439 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2440 {
2441 unsigned int i;
2442
2443 mutex_lock(&uld_mutex);
2444 for (i = 0; i < CXGB4_ULD_MAX; i++)
2445 if (adap->uld_handle[i])
2446 ulds[i].state_change(adap->uld_handle[i], new_state);
2447 mutex_unlock(&uld_mutex);
2448 }
2449
2450 /**
2451 * cxgb4_register_uld - register an upper-layer driver
2452 * @type: the ULD type
2453 * @p: the ULD methods
2454 *
2455 * Registers an upper-layer driver with this driver and notifies the ULD
2456 * about any presently available devices that support its type. Returns
2457 * %-EBUSY if a ULD of the same type is already registered.
2458 */
2459 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2460 {
2461 int ret = 0;
2462 struct adapter *adap;
2463
2464 if (type >= CXGB4_ULD_MAX)
2465 return -EINVAL;
2466 mutex_lock(&uld_mutex);
2467 if (ulds[type].add) {
2468 ret = -EBUSY;
2469 goto out;
2470 }
2471 ulds[type] = *p;
2472 list_for_each_entry(adap, &adapter_list, list_node)
2473 uld_attach(adap, type);
2474 out: mutex_unlock(&uld_mutex);
2475 return ret;
2476 }
2477 EXPORT_SYMBOL(cxgb4_register_uld);
2478
2479 /**
2480 * cxgb4_unregister_uld - unregister an upper-layer driver
2481 * @type: the ULD type
2482 *
2483 * Unregisters an existing upper-layer driver.
2484 */
2485 int cxgb4_unregister_uld(enum cxgb4_uld type)
2486 {
2487 struct adapter *adap;
2488
2489 if (type >= CXGB4_ULD_MAX)
2490 return -EINVAL;
2491 mutex_lock(&uld_mutex);
2492 list_for_each_entry(adap, &adapter_list, list_node)
2493 adap->uld_handle[type] = NULL;
2494 ulds[type].add = NULL;
2495 mutex_unlock(&uld_mutex);
2496 return 0;
2497 }
2498 EXPORT_SYMBOL(cxgb4_unregister_uld);
2499
2500 /**
2501 * cxgb_up - enable the adapter
2502 * @adap: adapter being enabled
2503 *
2504 * Called when the first port is enabled, this function performs the
2505 * actions necessary to make an adapter operational, such as completing
2506 * the initialization of HW modules, and enabling interrupts.
2507 *
2508 * Must be called with the rtnl lock held.
2509 */
2510 static int cxgb_up(struct adapter *adap)
2511 {
2512 int err;
2513
2514 err = setup_sge_queues(adap);
2515 if (err)
2516 goto out;
2517 err = setup_rss(adap);
2518 if (err)
2519 goto freeq;
2520
2521 if (adap->flags & USING_MSIX) {
2522 name_msix_vecs(adap);
2523 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2524 adap->msix_info[0].desc, adap);
2525 if (err)
2526 goto irq_err;
2527
2528 err = request_msix_queue_irqs(adap);
2529 if (err) {
2530 free_irq(adap->msix_info[0].vec, adap);
2531 goto irq_err;
2532 }
2533 } else {
2534 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2535 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2536 adap->name, adap);
2537 if (err)
2538 goto irq_err;
2539 }
2540 enable_rx(adap);
2541 t4_sge_start(adap);
2542 t4_intr_enable(adap);
2543 adap->flags |= FULL_INIT_DONE;
2544 notify_ulds(adap, CXGB4_STATE_UP);
2545 out:
2546 return err;
2547 irq_err:
2548 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2549 freeq:
2550 t4_free_sge_resources(adap);
2551 goto out;
2552 }
2553
2554 static void cxgb_down(struct adapter *adapter)
2555 {
2556 t4_intr_disable(adapter);
2557 cancel_work_sync(&adapter->tid_release_task);
2558 adapter->tid_release_task_busy = false;
2559 adapter->tid_release_head = NULL;
2560
2561 if (adapter->flags & USING_MSIX) {
2562 free_msix_queue_irqs(adapter);
2563 free_irq(adapter->msix_info[0].vec, adapter);
2564 } else
2565 free_irq(adapter->pdev->irq, adapter);
2566 quiesce_rx(adapter);
2567 t4_sge_stop(adapter);
2568 t4_free_sge_resources(adapter);
2569 adapter->flags &= ~FULL_INIT_DONE;
2570 }
2571
2572 /*
2573 * net_device operations
2574 */
2575 static int cxgb_open(struct net_device *dev)
2576 {
2577 int err;
2578 struct port_info *pi = netdev_priv(dev);
2579 struct adapter *adapter = pi->adapter;
2580
2581 if (!(adapter->flags & FULL_INIT_DONE)) {
2582 err = cxgb_up(adapter);
2583 if (err < 0)
2584 return err;
2585 }
2586
2587 dev->real_num_tx_queues = pi->nqsets;
2588 err = link_start(dev);
2589 if (!err)
2590 netif_tx_start_all_queues(dev);
2591 return err;
2592 }
2593
2594 static int cxgb_close(struct net_device *dev)
2595 {
2596 struct port_info *pi = netdev_priv(dev);
2597 struct adapter *adapter = pi->adapter;
2598
2599 netif_tx_stop_all_queues(dev);
2600 netif_carrier_off(dev);
2601 return t4_enable_vi(adapter, 0, pi->viid, false, false);
2602 }
2603
2604 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev)
2605 {
2606 struct port_stats stats;
2607 struct port_info *p = netdev_priv(dev);
2608 struct adapter *adapter = p->adapter;
2609 struct rtnl_link_stats64 *ns = &dev->stats64;
2610
2611 spin_lock(&adapter->stats_lock);
2612 t4_get_port_stats(adapter, p->tx_chan, &stats);
2613 spin_unlock(&adapter->stats_lock);
2614
2615 ns->tx_bytes = stats.tx_octets;
2616 ns->tx_packets = stats.tx_frames;
2617 ns->rx_bytes = stats.rx_octets;
2618 ns->rx_packets = stats.rx_frames;
2619 ns->multicast = stats.rx_mcast_frames;
2620
2621 /* detailed rx_errors */
2622 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2623 stats.rx_runt;
2624 ns->rx_over_errors = 0;
2625 ns->rx_crc_errors = stats.rx_fcs_err;
2626 ns->rx_frame_errors = stats.rx_symbol_err;
2627 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2628 stats.rx_ovflow2 + stats.rx_ovflow3 +
2629 stats.rx_trunc0 + stats.rx_trunc1 +
2630 stats.rx_trunc2 + stats.rx_trunc3;
2631 ns->rx_missed_errors = 0;
2632
2633 /* detailed tx_errors */
2634 ns->tx_aborted_errors = 0;
2635 ns->tx_carrier_errors = 0;
2636 ns->tx_fifo_errors = 0;
2637 ns->tx_heartbeat_errors = 0;
2638 ns->tx_window_errors = 0;
2639
2640 ns->tx_errors = stats.tx_error_frames;
2641 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2642 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2643 return ns;
2644 }
2645
2646 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2647 {
2648 int ret = 0, prtad, devad;
2649 struct port_info *pi = netdev_priv(dev);
2650 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2651
2652 switch (cmd) {
2653 case SIOCGMIIPHY:
2654 if (pi->mdio_addr < 0)
2655 return -EOPNOTSUPP;
2656 data->phy_id = pi->mdio_addr;
2657 break;
2658 case SIOCGMIIREG:
2659 case SIOCSMIIREG:
2660 if (mdio_phy_id_is_c45(data->phy_id)) {
2661 prtad = mdio_phy_id_prtad(data->phy_id);
2662 devad = mdio_phy_id_devad(data->phy_id);
2663 } else if (data->phy_id < 32) {
2664 prtad = data->phy_id;
2665 devad = 0;
2666 data->reg_num &= 0x1f;
2667 } else
2668 return -EINVAL;
2669
2670 if (cmd == SIOCGMIIREG)
2671 ret = t4_mdio_rd(pi->adapter, 0, prtad, devad,
2672 data->reg_num, &data->val_out);
2673 else
2674 ret = t4_mdio_wr(pi->adapter, 0, prtad, devad,
2675 data->reg_num, data->val_in);
2676 break;
2677 default:
2678 return -EOPNOTSUPP;
2679 }
2680 return ret;
2681 }
2682
2683 static void cxgb_set_rxmode(struct net_device *dev)
2684 {
2685 /* unfortunately we can't return errors to the stack */
2686 set_rxmode(dev, -1, false);
2687 }
2688
2689 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2690 {
2691 int ret;
2692 struct port_info *pi = netdev_priv(dev);
2693
2694 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2695 return -EINVAL;
2696 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, new_mtu, -1, -1, -1, -1,
2697 true);
2698 if (!ret)
2699 dev->mtu = new_mtu;
2700 return ret;
2701 }
2702
2703 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2704 {
2705 int ret;
2706 struct sockaddr *addr = p;
2707 struct port_info *pi = netdev_priv(dev);
2708
2709 if (!is_valid_ether_addr(addr->sa_data))
2710 return -EINVAL;
2711
2712 ret = t4_change_mac(pi->adapter, 0, pi->viid, pi->xact_addr_filt,
2713 addr->sa_data, true, true);
2714 if (ret < 0)
2715 return ret;
2716
2717 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2718 pi->xact_addr_filt = ret;
2719 return 0;
2720 }
2721
2722 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2723 {
2724 struct port_info *pi = netdev_priv(dev);
2725
2726 pi->vlan_grp = grp;
2727 t4_set_rxmode(pi->adapter, 0, pi->viid, -1, -1, -1, -1, grp != NULL,
2728 true);
2729 }
2730
2731 #ifdef CONFIG_NET_POLL_CONTROLLER
2732 static void cxgb_netpoll(struct net_device *dev)
2733 {
2734 struct port_info *pi = netdev_priv(dev);
2735 struct adapter *adap = pi->adapter;
2736
2737 if (adap->flags & USING_MSIX) {
2738 int i;
2739 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2740
2741 for (i = pi->nqsets; i; i--, rx++)
2742 t4_sge_intr_msix(0, &rx->rspq);
2743 } else
2744 t4_intr_handler(adap)(0, adap);
2745 }
2746 #endif
2747
2748 static const struct net_device_ops cxgb4_netdev_ops = {
2749 .ndo_open = cxgb_open,
2750 .ndo_stop = cxgb_close,
2751 .ndo_start_xmit = t4_eth_xmit,
2752 .ndo_get_stats64 = cxgb_get_stats,
2753 .ndo_set_rx_mode = cxgb_set_rxmode,
2754 .ndo_set_mac_address = cxgb_set_mac_addr,
2755 .ndo_validate_addr = eth_validate_addr,
2756 .ndo_do_ioctl = cxgb_ioctl,
2757 .ndo_change_mtu = cxgb_change_mtu,
2758 .ndo_vlan_rx_register = vlan_rx_register,
2759 #ifdef CONFIG_NET_POLL_CONTROLLER
2760 .ndo_poll_controller = cxgb_netpoll,
2761 #endif
2762 };
2763
2764 void t4_fatal_err(struct adapter *adap)
2765 {
2766 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2767 t4_intr_disable(adap);
2768 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2769 }
2770
2771 static void setup_memwin(struct adapter *adap)
2772 {
2773 u32 bar0;
2774
2775 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2776 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2777 (bar0 + MEMWIN0_BASE) | BIR(0) |
2778 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2779 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2780 (bar0 + MEMWIN1_BASE) | BIR(0) |
2781 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2782 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2783 (bar0 + MEMWIN2_BASE) | BIR(0) |
2784 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2785 }
2786
2787 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2788 {
2789 u32 v;
2790 int ret;
2791
2792 /* get device capabilities */
2793 memset(c, 0, sizeof(*c));
2794 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2795 FW_CMD_REQUEST | FW_CMD_READ);
2796 c->retval_len16 = htonl(FW_LEN16(*c));
2797 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), c);
2798 if (ret < 0)
2799 return ret;
2800
2801 /* select capabilities we'll be using */
2802 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2803 if (!vf_acls)
2804 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2805 else
2806 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2807 } else if (vf_acls) {
2808 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2809 return ret;
2810 }
2811 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2812 FW_CMD_REQUEST | FW_CMD_WRITE);
2813 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), NULL);
2814 if (ret < 0)
2815 return ret;
2816
2817 ret = t4_config_glbl_rss(adap, 0,
2818 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2819 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2820 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2821 if (ret < 0)
2822 return ret;
2823
2824 ret = t4_cfg_pfvf(adap, 0, 0, 0, MAX_EGRQ, 64, MAX_INGQ, 0, 0, 4,
2825 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2826 if (ret < 0)
2827 return ret;
2828
2829 t4_sge_init(adap);
2830
2831 /* get basic stuff going */
2832 ret = t4_early_init(adap, 0);
2833 if (ret < 0)
2834 return ret;
2835
2836 /* tweak some settings */
2837 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2838 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2839 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2840 v = t4_read_reg(adap, TP_PIO_DATA);
2841 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2842 setup_memwin(adap);
2843 return 0;
2844 }
2845
2846 /*
2847 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2848 */
2849 #define MAX_ATIDS 8192U
2850
2851 /*
2852 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2853 */
2854 static int adap_init0(struct adapter *adap)
2855 {
2856 int ret;
2857 u32 v, port_vec;
2858 enum dev_state state;
2859 u32 params[7], val[7];
2860 struct fw_caps_config_cmd c;
2861
2862 ret = t4_check_fw_version(adap);
2863 if (ret == -EINVAL || ret > 0) {
2864 if (upgrade_fw(adap) >= 0) /* recache FW version */
2865 ret = t4_check_fw_version(adap);
2866 }
2867 if (ret < 0)
2868 return ret;
2869
2870 /* contact FW, request master */
2871 ret = t4_fw_hello(adap, 0, 0, MASTER_MUST, &state);
2872 if (ret < 0) {
2873 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2874 ret);
2875 return ret;
2876 }
2877
2878 /* reset device */
2879 ret = t4_fw_reset(adap, 0, PIORSTMODE | PIORST);
2880 if (ret < 0)
2881 goto bye;
2882
2883 for (v = 0; v < SGE_NTIMERS - 1; v++)
2884 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2885 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2886 adap->sge.counter_val[0] = 1;
2887 for (v = 1; v < SGE_NCOUNTERS; v++)
2888 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2889 THRESHOLD_3_MASK);
2890 #define FW_PARAM_DEV(param) \
2891 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2892 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2893
2894 params[0] = FW_PARAM_DEV(CCLK);
2895 ret = t4_query_params(adap, 0, 0, 0, 1, params, val);
2896 if (ret < 0)
2897 goto bye;
2898 adap->params.vpd.cclk = val[0];
2899
2900 ret = adap_init1(adap, &c);
2901 if (ret < 0)
2902 goto bye;
2903
2904 #define FW_PARAM_PFVF(param) \
2905 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2906 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2907
2908 params[0] = FW_PARAM_DEV(PORTVEC);
2909 params[1] = FW_PARAM_PFVF(L2T_START);
2910 params[2] = FW_PARAM_PFVF(L2T_END);
2911 params[3] = FW_PARAM_PFVF(FILTER_START);
2912 params[4] = FW_PARAM_PFVF(FILTER_END);
2913 ret = t4_query_params(adap, 0, 0, 0, 5, params, val);
2914 if (ret < 0)
2915 goto bye;
2916 port_vec = val[0];
2917 adap->tids.ftid_base = val[3];
2918 adap->tids.nftids = val[4] - val[3] + 1;
2919
2920 if (c.ofldcaps) {
2921 /* query offload-related parameters */
2922 params[0] = FW_PARAM_DEV(NTID);
2923 params[1] = FW_PARAM_PFVF(SERVER_START);
2924 params[2] = FW_PARAM_PFVF(SERVER_END);
2925 params[3] = FW_PARAM_PFVF(TDDP_START);
2926 params[4] = FW_PARAM_PFVF(TDDP_END);
2927 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2928 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
2929 if (ret < 0)
2930 goto bye;
2931 adap->tids.ntids = val[0];
2932 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
2933 adap->tids.stid_base = val[1];
2934 adap->tids.nstids = val[2] - val[1] + 1;
2935 adap->vres.ddp.start = val[3];
2936 adap->vres.ddp.size = val[4] - val[3] + 1;
2937 adap->params.ofldq_wr_cred = val[5];
2938 adap->params.offload = 1;
2939 }
2940 if (c.rdmacaps) {
2941 params[0] = FW_PARAM_PFVF(STAG_START);
2942 params[1] = FW_PARAM_PFVF(STAG_END);
2943 params[2] = FW_PARAM_PFVF(RQ_START);
2944 params[3] = FW_PARAM_PFVF(RQ_END);
2945 params[4] = FW_PARAM_PFVF(PBL_START);
2946 params[5] = FW_PARAM_PFVF(PBL_END);
2947 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
2948 if (ret < 0)
2949 goto bye;
2950 adap->vres.stag.start = val[0];
2951 adap->vres.stag.size = val[1] - val[0] + 1;
2952 adap->vres.rq.start = val[2];
2953 adap->vres.rq.size = val[3] - val[2] + 1;
2954 adap->vres.pbl.start = val[4];
2955 adap->vres.pbl.size = val[5] - val[4] + 1;
2956
2957 params[0] = FW_PARAM_PFVF(SQRQ_START);
2958 params[1] = FW_PARAM_PFVF(SQRQ_END);
2959 params[2] = FW_PARAM_PFVF(CQ_START);
2960 params[3] = FW_PARAM_PFVF(CQ_END);
2961 ret = t4_query_params(adap, 0, 0, 0, 4, params, val);
2962 if (ret < 0)
2963 goto bye;
2964 adap->vres.qp.start = val[0];
2965 adap->vres.qp.size = val[1] - val[0] + 1;
2966 adap->vres.cq.start = val[2];
2967 adap->vres.cq.size = val[3] - val[2] + 1;
2968 }
2969 if (c.iscsicaps) {
2970 params[0] = FW_PARAM_PFVF(ISCSI_START);
2971 params[1] = FW_PARAM_PFVF(ISCSI_END);
2972 ret = t4_query_params(adap, 0, 0, 0, 2, params, val);
2973 if (ret < 0)
2974 goto bye;
2975 adap->vres.iscsi.start = val[0];
2976 adap->vres.iscsi.size = val[1] - val[0] + 1;
2977 }
2978 #undef FW_PARAM_PFVF
2979 #undef FW_PARAM_DEV
2980
2981 adap->params.nports = hweight32(port_vec);
2982 adap->params.portvec = port_vec;
2983 adap->flags |= FW_OK;
2984
2985 /* These are finalized by FW initialization, load their values now */
2986 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
2987 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
2988 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
2989 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
2990 adap->params.b_wnd);
2991
2992 #ifdef CONFIG_PCI_IOV
2993 /*
2994 * Provision resource limits for Virtual Functions. We currently
2995 * grant them all the same static resource limits except for the Port
2996 * Access Rights Mask which we're assigning based on the PF. All of
2997 * the static provisioning stuff for both the PF and VF really needs
2998 * to be managed in a persistent manner for each device which the
2999 * firmware controls.
3000 */
3001 {
3002 int pf, vf;
3003
3004 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3005 if (num_vf[pf] <= 0)
3006 continue;
3007
3008 /* VF numbering starts at 1! */
3009 for (vf = 1; vf <= num_vf[pf]; vf++) {
3010 ret = t4_cfg_pfvf(adap, 0, pf, vf,
3011 VFRES_NEQ, VFRES_NETHCTRL,
3012 VFRES_NIQFLINT, VFRES_NIQ,
3013 VFRES_TC, VFRES_NVI,
3014 FW_PFVF_CMD_CMASK_MASK,
3015 pfvfres_pmask(adap, pf, vf),
3016 VFRES_NEXACTF,
3017 VFRES_R_CAPS, VFRES_WX_CAPS);
3018 if (ret < 0)
3019 dev_warn(adap->pdev_dev, "failed to "
3020 "provision pf/vf=%d/%d; "
3021 "err=%d\n", pf, vf, ret);
3022 }
3023 }
3024 }
3025 #endif
3026
3027 return 0;
3028
3029 /*
3030 * If a command timed out or failed with EIO FW does not operate within
3031 * its spec or something catastrophic happened to HW/FW, stop issuing
3032 * commands.
3033 */
3034 bye: if (ret != -ETIMEDOUT && ret != -EIO)
3035 t4_fw_bye(adap, 0);
3036 return ret;
3037 }
3038
3039 /* EEH callbacks */
3040
3041 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3042 pci_channel_state_t state)
3043 {
3044 int i;
3045 struct adapter *adap = pci_get_drvdata(pdev);
3046
3047 if (!adap)
3048 goto out;
3049
3050 rtnl_lock();
3051 adap->flags &= ~FW_OK;
3052 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3053 for_each_port(adap, i) {
3054 struct net_device *dev = adap->port[i];
3055
3056 netif_device_detach(dev);
3057 netif_carrier_off(dev);
3058 }
3059 if (adap->flags & FULL_INIT_DONE)
3060 cxgb_down(adap);
3061 rtnl_unlock();
3062 pci_disable_device(pdev);
3063 out: return state == pci_channel_io_perm_failure ?
3064 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3065 }
3066
3067 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3068 {
3069 int i, ret;
3070 struct fw_caps_config_cmd c;
3071 struct adapter *adap = pci_get_drvdata(pdev);
3072
3073 if (!adap) {
3074 pci_restore_state(pdev);
3075 pci_save_state(pdev);
3076 return PCI_ERS_RESULT_RECOVERED;
3077 }
3078
3079 if (pci_enable_device(pdev)) {
3080 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3081 return PCI_ERS_RESULT_DISCONNECT;
3082 }
3083
3084 pci_set_master(pdev);
3085 pci_restore_state(pdev);
3086 pci_save_state(pdev);
3087 pci_cleanup_aer_uncorrect_error_status(pdev);
3088
3089 if (t4_wait_dev_ready(adap) < 0)
3090 return PCI_ERS_RESULT_DISCONNECT;
3091 if (t4_fw_hello(adap, 0, 0, MASTER_MUST, NULL))
3092 return PCI_ERS_RESULT_DISCONNECT;
3093 adap->flags |= FW_OK;
3094 if (adap_init1(adap, &c))
3095 return PCI_ERS_RESULT_DISCONNECT;
3096
3097 for_each_port(adap, i) {
3098 struct port_info *p = adap2pinfo(adap, i);
3099
3100 ret = t4_alloc_vi(adap, 0, p->tx_chan, 0, 0, 1, NULL, NULL);
3101 if (ret < 0)
3102 return PCI_ERS_RESULT_DISCONNECT;
3103 p->viid = ret;
3104 p->xact_addr_filt = -1;
3105 }
3106
3107 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3108 adap->params.b_wnd);
3109 if (cxgb_up(adap))
3110 return PCI_ERS_RESULT_DISCONNECT;
3111 return PCI_ERS_RESULT_RECOVERED;
3112 }
3113
3114 static void eeh_resume(struct pci_dev *pdev)
3115 {
3116 int i;
3117 struct adapter *adap = pci_get_drvdata(pdev);
3118
3119 if (!adap)
3120 return;
3121
3122 rtnl_lock();
3123 for_each_port(adap, i) {
3124 struct net_device *dev = adap->port[i];
3125
3126 if (netif_running(dev)) {
3127 link_start(dev);
3128 cxgb_set_rxmode(dev);
3129 }
3130 netif_device_attach(dev);
3131 }
3132 rtnl_unlock();
3133 }
3134
3135 static struct pci_error_handlers cxgb4_eeh = {
3136 .error_detected = eeh_err_detected,
3137 .slot_reset = eeh_slot_reset,
3138 .resume = eeh_resume,
3139 };
3140
3141 static inline bool is_10g_port(const struct link_config *lc)
3142 {
3143 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3144 }
3145
3146 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3147 unsigned int size, unsigned int iqe_size)
3148 {
3149 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3150 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3151 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3152 q->iqe_len = iqe_size;
3153 q->size = size;
3154 }
3155
3156 /*
3157 * Perform default configuration of DMA queues depending on the number and type
3158 * of ports we found and the number of available CPUs. Most settings can be
3159 * modified by the admin prior to actual use.
3160 */
3161 static void __devinit cfg_queues(struct adapter *adap)
3162 {
3163 struct sge *s = &adap->sge;
3164 int i, q10g = 0, n10g = 0, qidx = 0;
3165
3166 for_each_port(adap, i)
3167 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3168
3169 /*
3170 * We default to 1 queue per non-10G port and up to # of cores queues
3171 * per 10G port.
3172 */
3173 if (n10g)
3174 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3175 if (q10g > num_online_cpus())
3176 q10g = num_online_cpus();
3177
3178 for_each_port(adap, i) {
3179 struct port_info *pi = adap2pinfo(adap, i);
3180
3181 pi->first_qset = qidx;
3182 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3183 qidx += pi->nqsets;
3184 }
3185
3186 s->ethqsets = qidx;
3187 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3188
3189 if (is_offload(adap)) {
3190 /*
3191 * For offload we use 1 queue/channel if all ports are up to 1G,
3192 * otherwise we divide all available queues amongst the channels
3193 * capped by the number of available cores.
3194 */
3195 if (n10g) {
3196 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3197 num_online_cpus());
3198 s->ofldqsets = roundup(i, adap->params.nports);
3199 } else
3200 s->ofldqsets = adap->params.nports;
3201 /* For RDMA one Rx queue per channel suffices */
3202 s->rdmaqs = adap->params.nports;
3203 }
3204
3205 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3206 struct sge_eth_rxq *r = &s->ethrxq[i];
3207
3208 init_rspq(&r->rspq, 0, 0, 1024, 64);
3209 r->fl.size = 72;
3210 }
3211
3212 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3213 s->ethtxq[i].q.size = 1024;
3214
3215 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3216 s->ctrlq[i].q.size = 512;
3217
3218 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3219 s->ofldtxq[i].q.size = 1024;
3220
3221 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3222 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3223
3224 init_rspq(&r->rspq, 0, 0, 1024, 64);
3225 r->rspq.uld = CXGB4_ULD_ISCSI;
3226 r->fl.size = 72;
3227 }
3228
3229 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3230 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3231
3232 init_rspq(&r->rspq, 0, 0, 511, 64);
3233 r->rspq.uld = CXGB4_ULD_RDMA;
3234 r->fl.size = 72;
3235 }
3236
3237 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3238 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3239 }
3240
3241 /*
3242 * Reduce the number of Ethernet queues across all ports to at most n.
3243 * n provides at least one queue per port.
3244 */
3245 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3246 {
3247 int i;
3248 struct port_info *pi;
3249
3250 while (n < adap->sge.ethqsets)
3251 for_each_port(adap, i) {
3252 pi = adap2pinfo(adap, i);
3253 if (pi->nqsets > 1) {
3254 pi->nqsets--;
3255 adap->sge.ethqsets--;
3256 if (adap->sge.ethqsets <= n)
3257 break;
3258 }
3259 }
3260
3261 n = 0;
3262 for_each_port(adap, i) {
3263 pi = adap2pinfo(adap, i);
3264 pi->first_qset = n;
3265 n += pi->nqsets;
3266 }
3267 }
3268
3269 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3270 #define EXTRA_VECS 2
3271
3272 static int __devinit enable_msix(struct adapter *adap)
3273 {
3274 int ofld_need = 0;
3275 int i, err, want, need;
3276 struct sge *s = &adap->sge;
3277 unsigned int nchan = adap->params.nports;
3278 struct msix_entry entries[MAX_INGQ + 1];
3279
3280 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3281 entries[i].entry = i;
3282
3283 want = s->max_ethqsets + EXTRA_VECS;
3284 if (is_offload(adap)) {
3285 want += s->rdmaqs + s->ofldqsets;
3286 /* need nchan for each possible ULD */
3287 ofld_need = 2 * nchan;
3288 }
3289 need = adap->params.nports + EXTRA_VECS + ofld_need;
3290
3291 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3292 want = err;
3293
3294 if (!err) {
3295 /*
3296 * Distribute available vectors to the various queue groups.
3297 * Every group gets its minimum requirement and NIC gets top
3298 * priority for leftovers.
3299 */
3300 i = want - EXTRA_VECS - ofld_need;
3301 if (i < s->max_ethqsets) {
3302 s->max_ethqsets = i;
3303 if (i < s->ethqsets)
3304 reduce_ethqs(adap, i);
3305 }
3306 if (is_offload(adap)) {
3307 i = want - EXTRA_VECS - s->max_ethqsets;
3308 i -= ofld_need - nchan;
3309 s->ofldqsets = (i / nchan) * nchan; /* round down */
3310 }
3311 for (i = 0; i < want; ++i)
3312 adap->msix_info[i].vec = entries[i].vector;
3313 } else if (err > 0)
3314 dev_info(adap->pdev_dev,
3315 "only %d MSI-X vectors left, not using MSI-X\n", err);
3316 return err;
3317 }
3318
3319 #undef EXTRA_VECS
3320
3321 static void __devinit print_port_info(struct adapter *adap)
3322 {
3323 static const char *base[] = {
3324 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3325 "KX", "KR", "KR SFP+", "KR FEC"
3326 };
3327
3328 int i;
3329 char buf[80];
3330 const char *spd = "";
3331
3332 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3333 spd = " 2.5 GT/s";
3334 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3335 spd = " 5 GT/s";
3336
3337 for_each_port(adap, i) {
3338 struct net_device *dev = adap->port[i];
3339 const struct port_info *pi = netdev_priv(dev);
3340 char *bufp = buf;
3341
3342 if (!test_bit(i, &adap->registered_device_map))
3343 continue;
3344
3345 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3346 bufp += sprintf(bufp, "100/");
3347 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3348 bufp += sprintf(bufp, "1000/");
3349 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3350 bufp += sprintf(bufp, "10G/");
3351 if (bufp != buf)
3352 --bufp;
3353 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3354
3355 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3356 adap->params.vpd.id, adap->params.rev,
3357 buf, is_offload(adap) ? "R" : "",
3358 adap->params.pci.width, spd,
3359 (adap->flags & USING_MSIX) ? " MSI-X" :
3360 (adap->flags & USING_MSI) ? " MSI" : "");
3361 if (adap->name == dev->name)
3362 netdev_info(dev, "S/N: %s, E/C: %s\n",
3363 adap->params.vpd.sn, adap->params.vpd.ec);
3364 }
3365 }
3366
3367 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | NETIF_F_TSO6 |\
3368 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3369
3370 static int __devinit init_one(struct pci_dev *pdev,
3371 const struct pci_device_id *ent)
3372 {
3373 int func, i, err;
3374 struct port_info *pi;
3375 unsigned int highdma = 0;
3376 struct adapter *adapter = NULL;
3377
3378 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3379
3380 err = pci_request_regions(pdev, KBUILD_MODNAME);
3381 if (err) {
3382 /* Just info, some other driver may have claimed the device. */
3383 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3384 return err;
3385 }
3386
3387 /* We control everything through PF 0 */
3388 func = PCI_FUNC(pdev->devfn);
3389 if (func > 0) {
3390 pci_save_state(pdev); /* to restore SR-IOV later */
3391 goto sriov;
3392 }
3393
3394 err = pci_enable_device(pdev);
3395 if (err) {
3396 dev_err(&pdev->dev, "cannot enable PCI device\n");
3397 goto out_release_regions;
3398 }
3399
3400 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3401 highdma = NETIF_F_HIGHDMA;
3402 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3403 if (err) {
3404 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3405 "coherent allocations\n");
3406 goto out_disable_device;
3407 }
3408 } else {
3409 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3410 if (err) {
3411 dev_err(&pdev->dev, "no usable DMA configuration\n");
3412 goto out_disable_device;
3413 }
3414 }
3415
3416 pci_enable_pcie_error_reporting(pdev);
3417 pci_set_master(pdev);
3418 pci_save_state(pdev);
3419
3420 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3421 if (!adapter) {
3422 err = -ENOMEM;
3423 goto out_disable_device;
3424 }
3425
3426 adapter->regs = pci_ioremap_bar(pdev, 0);
3427 if (!adapter->regs) {
3428 dev_err(&pdev->dev, "cannot map device registers\n");
3429 err = -ENOMEM;
3430 goto out_free_adapter;
3431 }
3432
3433 adapter->pdev = pdev;
3434 adapter->pdev_dev = &pdev->dev;
3435 adapter->name = pci_name(pdev);
3436 adapter->msg_enable = dflt_msg_enable;
3437 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3438
3439 spin_lock_init(&adapter->stats_lock);
3440 spin_lock_init(&adapter->tid_release_lock);
3441
3442 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3443
3444 err = t4_prep_adapter(adapter);
3445 if (err)
3446 goto out_unmap_bar;
3447 err = adap_init0(adapter);
3448 if (err)
3449 goto out_unmap_bar;
3450
3451 for_each_port(adapter, i) {
3452 struct net_device *netdev;
3453
3454 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3455 MAX_ETH_QSETS);
3456 if (!netdev) {
3457 err = -ENOMEM;
3458 goto out_free_dev;
3459 }
3460
3461 SET_NETDEV_DEV(netdev, &pdev->dev);
3462
3463 adapter->port[i] = netdev;
3464 pi = netdev_priv(netdev);
3465 pi->adapter = adapter;
3466 pi->xact_addr_filt = -1;
3467 pi->rx_offload = RX_CSO;
3468 pi->port_id = i;
3469 netif_carrier_off(netdev);
3470 netif_tx_stop_all_queues(netdev);
3471 netdev->irq = pdev->irq;
3472
3473 netdev->features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
3474 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3475 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
3476 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3477 netdev->vlan_features = netdev->features & VLAN_FEAT;
3478
3479 netdev->netdev_ops = &cxgb4_netdev_ops;
3480 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3481 }
3482
3483 pci_set_drvdata(pdev, adapter);
3484
3485 if (adapter->flags & FW_OK) {
3486 err = t4_port_init(adapter, 0, 0, 0);
3487 if (err)
3488 goto out_free_dev;
3489 }
3490
3491 /*
3492 * Configure queues and allocate tables now, they can be needed as
3493 * soon as the first register_netdev completes.
3494 */
3495 cfg_queues(adapter);
3496
3497 adapter->l2t = t4_init_l2t();
3498 if (!adapter->l2t) {
3499 /* We tolerate a lack of L2T, giving up some functionality */
3500 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3501 adapter->params.offload = 0;
3502 }
3503
3504 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3505 dev_warn(&pdev->dev, "could not allocate TID table, "
3506 "continuing\n");
3507 adapter->params.offload = 0;
3508 }
3509
3510 /*
3511 * The card is now ready to go. If any errors occur during device
3512 * registration we do not fail the whole card but rather proceed only
3513 * with the ports we manage to register successfully. However we must
3514 * register at least one net device.
3515 */
3516 for_each_port(adapter, i) {
3517 err = register_netdev(adapter->port[i]);
3518 if (err)
3519 dev_warn(&pdev->dev,
3520 "cannot register net device %s, skipping\n",
3521 adapter->port[i]->name);
3522 else {
3523 /*
3524 * Change the name we use for messages to the name of
3525 * the first successfully registered interface.
3526 */
3527 if (!adapter->registered_device_map)
3528 adapter->name = adapter->port[i]->name;
3529
3530 __set_bit(i, &adapter->registered_device_map);
3531 adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3532 }
3533 }
3534 if (!adapter->registered_device_map) {
3535 dev_err(&pdev->dev, "could not register any net devices\n");
3536 goto out_free_dev;
3537 }
3538
3539 if (cxgb4_debugfs_root) {
3540 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3541 cxgb4_debugfs_root);
3542 setup_debugfs(adapter);
3543 }
3544
3545 /* See what interrupts we'll be using */
3546 if (msi > 1 && enable_msix(adapter) == 0)
3547 adapter->flags |= USING_MSIX;
3548 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3549 adapter->flags |= USING_MSI;
3550
3551 if (is_offload(adapter))
3552 attach_ulds(adapter);
3553
3554 print_port_info(adapter);
3555
3556 sriov:
3557 #ifdef CONFIG_PCI_IOV
3558 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3559 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3560 dev_info(&pdev->dev,
3561 "instantiated %u virtual functions\n",
3562 num_vf[func]);
3563 #endif
3564 return 0;
3565
3566 out_free_dev:
3567 t4_free_mem(adapter->tids.tid_tab);
3568 t4_free_mem(adapter->l2t);
3569 for_each_port(adapter, i)
3570 if (adapter->port[i])
3571 free_netdev(adapter->port[i]);
3572 if (adapter->flags & FW_OK)
3573 t4_fw_bye(adapter, 0);
3574 out_unmap_bar:
3575 iounmap(adapter->regs);
3576 out_free_adapter:
3577 kfree(adapter);
3578 out_disable_device:
3579 pci_disable_pcie_error_reporting(pdev);
3580 pci_disable_device(pdev);
3581 out_release_regions:
3582 pci_release_regions(pdev);
3583 pci_set_drvdata(pdev, NULL);
3584 return err;
3585 }
3586
3587 static void __devexit remove_one(struct pci_dev *pdev)
3588 {
3589 struct adapter *adapter = pci_get_drvdata(pdev);
3590
3591 pci_disable_sriov(pdev);
3592
3593 if (adapter) {
3594 int i;
3595
3596 if (is_offload(adapter))
3597 detach_ulds(adapter);
3598
3599 for_each_port(adapter, i)
3600 if (test_bit(i, &adapter->registered_device_map))
3601 unregister_netdev(adapter->port[i]);
3602
3603 if (adapter->debugfs_root)
3604 debugfs_remove_recursive(adapter->debugfs_root);
3605
3606 if (adapter->flags & FULL_INIT_DONE)
3607 cxgb_down(adapter);
3608 t4_free_mem(adapter->l2t);
3609 t4_free_mem(adapter->tids.tid_tab);
3610 disable_msi(adapter);
3611
3612 for_each_port(adapter, i)
3613 if (adapter->port[i])
3614 free_netdev(adapter->port[i]);
3615
3616 if (adapter->flags & FW_OK)
3617 t4_fw_bye(adapter, 0);
3618 iounmap(adapter->regs);
3619 kfree(adapter);
3620 pci_disable_pcie_error_reporting(pdev);
3621 pci_disable_device(pdev);
3622 pci_release_regions(pdev);
3623 pci_set_drvdata(pdev, NULL);
3624 } else if (PCI_FUNC(pdev->devfn) > 0)
3625 pci_release_regions(pdev);
3626 }
3627
3628 static struct pci_driver cxgb4_driver = {
3629 .name = KBUILD_MODNAME,
3630 .id_table = cxgb4_pci_tbl,
3631 .probe = init_one,
3632 .remove = __devexit_p(remove_one),
3633 .err_handler = &cxgb4_eeh,
3634 };
3635
3636 static int __init cxgb4_init_module(void)
3637 {
3638 int ret;
3639
3640 /* Debugfs support is optional, just warn if this fails */
3641 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3642 if (!cxgb4_debugfs_root)
3643 pr_warning("could not create debugfs entry, continuing\n");
3644
3645 ret = pci_register_driver(&cxgb4_driver);
3646 if (ret < 0)
3647 debugfs_remove(cxgb4_debugfs_root);
3648 return ret;
3649 }
3650
3651 static void __exit cxgb4_cleanup_module(void)
3652 {
3653 pci_unregister_driver(&cxgb4_driver);
3654 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3655 }
3656
3657 module_init(cxgb4_init_module);
3658 module_exit(cxgb4_cleanup_module);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree