?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
cxgb4: set maximal number of default RSS queues
[linux-2.6.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
blob5ed49af23d6ae37f5f7f7136412c9c0f20ac3e56
1 /*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <asm/uaccess.h>
64
65 #include "cxgb4.h"
66 #include "t4_regs.h"
67 #include "t4_msg.h"
68 #include "t4fw_api.h"
69 #include "l2t.h"
70
71 #define DRV_VERSION "1.3.0-ko"
72 #define DRV_DESC "Chelsio T4 Network Driver"
73
74 /*
75 * Max interrupt hold-off timer value in us. Queues fall back to this value
76 * under extreme memory pressure so it's largish to give the system time to
77 * recover.
78 */
79 #define MAX_SGE_TIMERVAL 200U
80
81 #ifdef CONFIG_PCI_IOV
82 /*
83 * Virtual Function provisioning constants. We need two extra Ingress Queues
84 * with Interrupt capability to serve as the VF's Firmware Event Queue and
85 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
86 * Lists associated with them). For each Ethernet/Control Egress Queue and
87 * for each Free List, we need an Egress Context.
88 */
89 enum {
90 VFRES_NPORTS = 1, /* # of "ports" per VF */
91 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
92
93 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
94 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
95 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
96 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
97 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
98 VFRES_TC = 0, /* PCI-E traffic class */
99 VFRES_NEXACTF = 16, /* # of exact MPS filters */
100
101 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
102 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
103 };
104
105 /*
106 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
107 * static and likely not to be useful in the long run. We really need to
108 * implement some form of persistent configuration which the firmware
109 * controls.
110 */
111 static unsigned int pfvfres_pmask(struct adapter *adapter,
112 unsigned int pf, unsigned int vf)
113 {
114 unsigned int portn, portvec;
115
116 /*
117 * Give PF's access to all of the ports.
118 */
119 if (vf == 0)
120 return FW_PFVF_CMD_PMASK_MASK;
121
122 /*
123 * For VFs, we'll assign them access to the ports based purely on the
124 * PF. We assign active ports in order, wrapping around if there are
125 * fewer active ports than PFs: e.g. active port[pf % nports].
126 * Unfortunately the adapter's port_info structs haven't been
127 * initialized yet so we have to compute this.
128 */
129 if (adapter->params.nports == 0)
130 return 0;
131
132 portn = pf % adapter->params.nports;
133 portvec = adapter->params.portvec;
134 for (;;) {
135 /*
136 * Isolate the lowest set bit in the port vector. If we're at
137 * the port number that we want, return that as the pmask.
138 * otherwise mask that bit out of the port vector and
139 * decrement our port number ...
140 */
141 unsigned int pmask = portvec ^ (portvec & (portvec-1));
142 if (portn == 0)
143 return pmask;
144 portn--;
145 portvec &= ~pmask;
146 }
147 /*NOTREACHED*/
148 }
149 #endif
150
151 enum {
152 MAX_TXQ_ENTRIES = 16384,
153 MAX_CTRL_TXQ_ENTRIES = 1024,
154 MAX_RSPQ_ENTRIES = 16384,
155 MAX_RX_BUFFERS = 16384,
156 MIN_TXQ_ENTRIES = 32,
157 MIN_CTRL_TXQ_ENTRIES = 32,
158 MIN_RSPQ_ENTRIES = 128,
159 MIN_FL_ENTRIES = 16
160 };
161
162 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
163 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
164 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
165
166 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
167
168 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
169 CH_DEVICE(0xa000, 0), /* PE10K */
170 CH_DEVICE(0x4001, -1),
171 CH_DEVICE(0x4002, -1),
172 CH_DEVICE(0x4003, -1),
173 CH_DEVICE(0x4004, -1),
174 CH_DEVICE(0x4005, -1),
175 CH_DEVICE(0x4006, -1),
176 CH_DEVICE(0x4007, -1),
177 CH_DEVICE(0x4008, -1),
178 CH_DEVICE(0x4009, -1),
179 CH_DEVICE(0x400a, -1),
180 CH_DEVICE(0x4401, 4),
181 CH_DEVICE(0x4402, 4),
182 CH_DEVICE(0x4403, 4),
183 CH_DEVICE(0x4404, 4),
184 CH_DEVICE(0x4405, 4),
185 CH_DEVICE(0x4406, 4),
186 CH_DEVICE(0x4407, 4),
187 CH_DEVICE(0x4408, 4),
188 CH_DEVICE(0x4409, 4),
189 CH_DEVICE(0x440a, 4),
190 CH_DEVICE(0x440d, 4),
191 CH_DEVICE(0x440e, 4),
192 { 0, }
193 };
194
195 #define FW_FNAME "cxgb4/t4fw.bin"
196
197 MODULE_DESCRIPTION(DRV_DESC);
198 MODULE_AUTHOR("Chelsio Communications");
199 MODULE_LICENSE("Dual BSD/GPL");
200 MODULE_VERSION(DRV_VERSION);
201 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
202 MODULE_FIRMWARE(FW_FNAME);
203
204 static int dflt_msg_enable = DFLT_MSG_ENABLE;
205
206 module_param(dflt_msg_enable, int, 0644);
207 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
208
209 /*
210 * The driver uses the best interrupt scheme available on a platform in the
211 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
212 * of these schemes the driver may consider as follows:
213 *
214 * msi = 2: choose from among all three options
215 * msi = 1: only consider MSI and INTx interrupts
216 * msi = 0: force INTx interrupts
217 */
218 static int msi = 2;
219
220 module_param(msi, int, 0644);
221 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
222
223 /*
224 * Queue interrupt hold-off timer values. Queues default to the first of these
225 * upon creation.
226 */
227 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
228
229 module_param_array(intr_holdoff, uint, NULL, 0644);
230 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
231 "0..4 in microseconds");
232
233 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
234
235 module_param_array(intr_cnt, uint, NULL, 0644);
236 MODULE_PARM_DESC(intr_cnt,
237 "thresholds 1..3 for queue interrupt packet counters");
238
239 static bool vf_acls;
240
241 #ifdef CONFIG_PCI_IOV
242 module_param(vf_acls, bool, 0644);
243 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
244
245 static unsigned int num_vf[4];
246
247 module_param_array(num_vf, uint, NULL, 0644);
248 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
249 #endif
250
251 static struct dentry *cxgb4_debugfs_root;
252
253 static LIST_HEAD(adapter_list);
254 static DEFINE_MUTEX(uld_mutex);
255 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
256 static const char *uld_str[] = { "RDMA", "iSCSI" };
257
258 static void link_report(struct net_device *dev)
259 {
260 if (!netif_carrier_ok(dev))
261 netdev_info(dev, "link down\n");
262 else {
263 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
264
265 const char *s = "10Mbps";
266 const struct port_info *p = netdev_priv(dev);
267
268 switch (p->link_cfg.speed) {
269 case SPEED_10000:
270 s = "10Gbps";
271 break;
272 case SPEED_1000:
273 s = "1000Mbps";
274 break;
275 case SPEED_100:
276 s = "100Mbps";
277 break;
278 }
279
280 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
281 fc[p->link_cfg.fc]);
282 }
283 }
284
285 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
286 {
287 struct net_device *dev = adapter->port[port_id];
288
289 /* Skip changes from disabled ports. */
290 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
291 if (link_stat)
292 netif_carrier_on(dev);
293 else
294 netif_carrier_off(dev);
295
296 link_report(dev);
297 }
298 }
299
300 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
301 {
302 static const char *mod_str[] = {
303 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
304 };
305
306 const struct net_device *dev = adap->port[port_id];
307 const struct port_info *pi = netdev_priv(dev);
308
309 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
310 netdev_info(dev, "port module unplugged\n");
311 else if (pi->mod_type < ARRAY_SIZE(mod_str))
312 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
313 }
314
315 /*
316 * Configure the exact and hash address filters to handle a port's multicast
317 * and secondary unicast MAC addresses.
318 */
319 static int set_addr_filters(const struct net_device *dev, bool sleep)
320 {
321 u64 mhash = 0;
322 u64 uhash = 0;
323 bool free = true;
324 u16 filt_idx[7];
325 const u8 *addr[7];
326 int ret, naddr = 0;
327 const struct netdev_hw_addr *ha;
328 int uc_cnt = netdev_uc_count(dev);
329 int mc_cnt = netdev_mc_count(dev);
330 const struct port_info *pi = netdev_priv(dev);
331 unsigned int mb = pi->adapter->fn;
332
333 /* first do the secondary unicast addresses */
334 netdev_for_each_uc_addr(ha, dev) {
335 addr[naddr++] = ha->addr;
336 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
337 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
338 naddr, addr, filt_idx, &uhash, sleep);
339 if (ret < 0)
340 return ret;
341
342 free = false;
343 naddr = 0;
344 }
345 }
346
347 /* next set up the multicast addresses */
348 netdev_for_each_mc_addr(ha, dev) {
349 addr[naddr++] = ha->addr;
350 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
351 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
352 naddr, addr, filt_idx, &mhash, sleep);
353 if (ret < 0)
354 return ret;
355
356 free = false;
357 naddr = 0;
358 }
359 }
360
361 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
362 uhash | mhash, sleep);
363 }
364
365 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
366 module_param(dbfifo_int_thresh, int, 0644);
367 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
368
369 int dbfifo_drain_delay = 1000; /* usecs to sleep while draining the dbfifo */
370 module_param(dbfifo_drain_delay, int, 0644);
371 MODULE_PARM_DESC(dbfifo_drain_delay,
372 "usecs to sleep while draining the dbfifo");
373
374 /*
375 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
376 * If @mtu is -1 it is left unchanged.
377 */
378 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
379 {
380 int ret;
381 struct port_info *pi = netdev_priv(dev);
382
383 ret = set_addr_filters(dev, sleep_ok);
384 if (ret == 0)
385 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
386 (dev->flags & IFF_PROMISC) ? 1 : 0,
387 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
388 sleep_ok);
389 return ret;
390 }
391
392 static struct workqueue_struct *workq;
393
394 /**
395 * link_start - enable a port
396 * @dev: the port to enable
397 *
398 * Performs the MAC and PHY actions needed to enable a port.
399 */
400 static int link_start(struct net_device *dev)
401 {
402 int ret;
403 struct port_info *pi = netdev_priv(dev);
404 unsigned int mb = pi->adapter->fn;
405
406 /*
407 * We do not set address filters and promiscuity here, the stack does
408 * that step explicitly.
409 */
410 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
411 !!(dev->features & NETIF_F_HW_VLAN_RX), true);
412 if (ret == 0) {
413 ret = t4_change_mac(pi->adapter, mb, pi->viid,
414 pi->xact_addr_filt, dev->dev_addr, true,
415 true);
416 if (ret >= 0) {
417 pi->xact_addr_filt = ret;
418 ret = 0;
419 }
420 }
421 if (ret == 0)
422 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
423 &pi->link_cfg);
424 if (ret == 0)
425 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
426 return ret;
427 }
428
429 /*
430 * Response queue handler for the FW event queue.
431 */
432 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
433 const struct pkt_gl *gl)
434 {
435 u8 opcode = ((const struct rss_header *)rsp)->opcode;
436
437 rsp++; /* skip RSS header */
438 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
439 const struct cpl_sge_egr_update *p = (void *)rsp;
440 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
441 struct sge_txq *txq;
442
443 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
444 txq->restarts++;
445 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
446 struct sge_eth_txq *eq;
447
448 eq = container_of(txq, struct sge_eth_txq, q);
449 netif_tx_wake_queue(eq->txq);
450 } else {
451 struct sge_ofld_txq *oq;
452
453 oq = container_of(txq, struct sge_ofld_txq, q);
454 tasklet_schedule(&oq->qresume_tsk);
455 }
456 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
457 const struct cpl_fw6_msg *p = (void *)rsp;
458
459 if (p->type == 0)
460 t4_handle_fw_rpl(q->adap, p->data);
461 } else if (opcode == CPL_L2T_WRITE_RPL) {
462 const struct cpl_l2t_write_rpl *p = (void *)rsp;
463
464 do_l2t_write_rpl(q->adap, p);
465 } else
466 dev_err(q->adap->pdev_dev,
467 "unexpected CPL %#x on FW event queue\n", opcode);
468 return 0;
469 }
470
471 /**
472 * uldrx_handler - response queue handler for ULD queues
473 * @q: the response queue that received the packet
474 * @rsp: the response queue descriptor holding the offload message
475 * @gl: the gather list of packet fragments
476 *
477 * Deliver an ingress offload packet to a ULD. All processing is done by
478 * the ULD, we just maintain statistics.
479 */
480 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
481 const struct pkt_gl *gl)
482 {
483 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
484
485 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
486 rxq->stats.nomem++;
487 return -1;
488 }
489 if (gl == NULL)
490 rxq->stats.imm++;
491 else if (gl == CXGB4_MSG_AN)
492 rxq->stats.an++;
493 else
494 rxq->stats.pkts++;
495 return 0;
496 }
497
498 static void disable_msi(struct adapter *adapter)
499 {
500 if (adapter->flags & USING_MSIX) {
501 pci_disable_msix(adapter->pdev);
502 adapter->flags &= ~USING_MSIX;
503 } else if (adapter->flags & USING_MSI) {
504 pci_disable_msi(adapter->pdev);
505 adapter->flags &= ~USING_MSI;
506 }
507 }
508
509 /*
510 * Interrupt handler for non-data events used with MSI-X.
511 */
512 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
513 {
514 struct adapter *adap = cookie;
515
516 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
517 if (v & PFSW) {
518 adap->swintr = 1;
519 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
520 }
521 t4_slow_intr_handler(adap);
522 return IRQ_HANDLED;
523 }
524
525 /*
526 * Name the MSI-X interrupts.
527 */
528 static void name_msix_vecs(struct adapter *adap)
529 {
530 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
531
532 /* non-data interrupts */
533 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
534
535 /* FW events */
536 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
537 adap->port[0]->name);
538
539 /* Ethernet queues */
540 for_each_port(adap, j) {
541 struct net_device *d = adap->port[j];
542 const struct port_info *pi = netdev_priv(d);
543
544 for (i = 0; i < pi->nqsets; i++, msi_idx++)
545 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
546 d->name, i);
547 }
548
549 /* offload queues */
550 for_each_ofldrxq(&adap->sge, i)
551 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
552 adap->port[0]->name, i);
553
554 for_each_rdmarxq(&adap->sge, i)
555 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
556 adap->port[0]->name, i);
557 }
558
559 static int request_msix_queue_irqs(struct adapter *adap)
560 {
561 struct sge *s = &adap->sge;
562 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
563
564 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
565 adap->msix_info[1].desc, &s->fw_evtq);
566 if (err)
567 return err;
568
569 for_each_ethrxq(s, ethqidx) {
570 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
571 adap->msix_info[msi].desc,
572 &s->ethrxq[ethqidx].rspq);
573 if (err)
574 goto unwind;
575 msi++;
576 }
577 for_each_ofldrxq(s, ofldqidx) {
578 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
579 adap->msix_info[msi].desc,
580 &s->ofldrxq[ofldqidx].rspq);
581 if (err)
582 goto unwind;
583 msi++;
584 }
585 for_each_rdmarxq(s, rdmaqidx) {
586 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
587 adap->msix_info[msi].desc,
588 &s->rdmarxq[rdmaqidx].rspq);
589 if (err)
590 goto unwind;
591 msi++;
592 }
593 return 0;
594
595 unwind:
596 while (--rdmaqidx >= 0)
597 free_irq(adap->msix_info[--msi].vec,
598 &s->rdmarxq[rdmaqidx].rspq);
599 while (--ofldqidx >= 0)
600 free_irq(adap->msix_info[--msi].vec,
601 &s->ofldrxq[ofldqidx].rspq);
602 while (--ethqidx >= 0)
603 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
604 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
605 return err;
606 }
607
608 static void free_msix_queue_irqs(struct adapter *adap)
609 {
610 int i, msi = 2;
611 struct sge *s = &adap->sge;
612
613 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
614 for_each_ethrxq(s, i)
615 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
616 for_each_ofldrxq(s, i)
617 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
618 for_each_rdmarxq(s, i)
619 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
620 }
621
622 /**
623 * write_rss - write the RSS table for a given port
624 * @pi: the port
625 * @queues: array of queue indices for RSS
626 *
627 * Sets up the portion of the HW RSS table for the port's VI to distribute
628 * packets to the Rx queues in @queues.
629 */
630 static int write_rss(const struct port_info *pi, const u16 *queues)
631 {
632 u16 *rss;
633 int i, err;
634 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
635
636 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
637 if (!rss)
638 return -ENOMEM;
639
640 /* map the queue indices to queue ids */
641 for (i = 0; i < pi->rss_size; i++, queues++)
642 rss[i] = q[*queues].rspq.abs_id;
643
644 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
645 pi->rss_size, rss, pi->rss_size);
646 kfree(rss);
647 return err;
648 }
649
650 /**
651 * setup_rss - configure RSS
652 * @adap: the adapter
653 *
654 * Sets up RSS for each port.
655 */
656 static int setup_rss(struct adapter *adap)
657 {
658 int i, err;
659
660 for_each_port(adap, i) {
661 const struct port_info *pi = adap2pinfo(adap, i);
662
663 err = write_rss(pi, pi->rss);
664 if (err)
665 return err;
666 }
667 return 0;
668 }
669
670 /*
671 * Return the channel of the ingress queue with the given qid.
672 */
673 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
674 {
675 qid -= p->ingr_start;
676 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
677 }
678
679 /*
680 * Wait until all NAPI handlers are descheduled.
681 */
682 static void quiesce_rx(struct adapter *adap)
683 {
684 int i;
685
686 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
687 struct sge_rspq *q = adap->sge.ingr_map[i];
688
689 if (q && q->handler)
690 napi_disable(&q->napi);
691 }
692 }
693
694 /*
695 * Enable NAPI scheduling and interrupt generation for all Rx queues.
696 */
697 static void enable_rx(struct adapter *adap)
698 {
699 int i;
700
701 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
702 struct sge_rspq *q = adap->sge.ingr_map[i];
703
704 if (!q)
705 continue;
706 if (q->handler)
707 napi_enable(&q->napi);
708 /* 0-increment GTS to start the timer and enable interrupts */
709 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
710 SEINTARM(q->intr_params) |
711 INGRESSQID(q->cntxt_id));
712 }
713 }
714
715 /**
716 * setup_sge_queues - configure SGE Tx/Rx/response queues
717 * @adap: the adapter
718 *
719 * Determines how many sets of SGE queues to use and initializes them.
720 * We support multiple queue sets per port if we have MSI-X, otherwise
721 * just one queue set per port.
722 */
723 static int setup_sge_queues(struct adapter *adap)
724 {
725 int err, msi_idx, i, j;
726 struct sge *s = &adap->sge;
727
728 bitmap_zero(s->starving_fl, MAX_EGRQ);
729 bitmap_zero(s->txq_maperr, MAX_EGRQ);
730
731 if (adap->flags & USING_MSIX)
732 msi_idx = 1; /* vector 0 is for non-queue interrupts */
733 else {
734 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
735 NULL, NULL);
736 if (err)
737 return err;
738 msi_idx = -((int)s->intrq.abs_id + 1);
739 }
740
741 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
742 msi_idx, NULL, fwevtq_handler);
743 if (err) {
744 freeout: t4_free_sge_resources(adap);
745 return err;
746 }
747
748 for_each_port(adap, i) {
749 struct net_device *dev = adap->port[i];
750 struct port_info *pi = netdev_priv(dev);
751 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
752 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
753
754 for (j = 0; j < pi->nqsets; j++, q++) {
755 if (msi_idx > 0)
756 msi_idx++;
757 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
758 msi_idx, &q->fl,
759 t4_ethrx_handler);
760 if (err)
761 goto freeout;
762 q->rspq.idx = j;
763 memset(&q->stats, 0, sizeof(q->stats));
764 }
765 for (j = 0; j < pi->nqsets; j++, t++) {
766 err = t4_sge_alloc_eth_txq(adap, t, dev,
767 netdev_get_tx_queue(dev, j),
768 s->fw_evtq.cntxt_id);
769 if (err)
770 goto freeout;
771 }
772 }
773
774 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
775 for_each_ofldrxq(s, i) {
776 struct sge_ofld_rxq *q = &s->ofldrxq[i];
777 struct net_device *dev = adap->port[i / j];
778
779 if (msi_idx > 0)
780 msi_idx++;
781 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
782 &q->fl, uldrx_handler);
783 if (err)
784 goto freeout;
785 memset(&q->stats, 0, sizeof(q->stats));
786 s->ofld_rxq[i] = q->rspq.abs_id;
787 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
788 s->fw_evtq.cntxt_id);
789 if (err)
790 goto freeout;
791 }
792
793 for_each_rdmarxq(s, i) {
794 struct sge_ofld_rxq *q = &s->rdmarxq[i];
795
796 if (msi_idx > 0)
797 msi_idx++;
798 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
799 msi_idx, &q->fl, uldrx_handler);
800 if (err)
801 goto freeout;
802 memset(&q->stats, 0, sizeof(q->stats));
803 s->rdma_rxq[i] = q->rspq.abs_id;
804 }
805
806 for_each_port(adap, i) {
807 /*
808 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
809 * have RDMA queues, and that's the right value.
810 */
811 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
812 s->fw_evtq.cntxt_id,
813 s->rdmarxq[i].rspq.cntxt_id);
814 if (err)
815 goto freeout;
816 }
817
818 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
819 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
820 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
821 return 0;
822 }
823
824 /*
825 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
826 * started but failed, and a negative errno if flash load couldn't start.
827 */
828 static int upgrade_fw(struct adapter *adap)
829 {
830 int ret;
831 u32 vers;
832 const struct fw_hdr *hdr;
833 const struct firmware *fw;
834 struct device *dev = adap->pdev_dev;
835
836 ret = request_firmware(&fw, FW_FNAME, dev);
837 if (ret < 0) {
838 dev_err(dev, "unable to load firmware image " FW_FNAME
839 ", error %d\n", ret);
840 return ret;
841 }
842
843 hdr = (const struct fw_hdr *)fw->data;
844 vers = ntohl(hdr->fw_ver);
845 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
846 ret = -EINVAL; /* wrong major version, won't do */
847 goto out;
848 }
849
850 /*
851 * If the flash FW is unusable or we found something newer, load it.
852 */
853 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
854 vers > adap->params.fw_vers) {
855 ret = -t4_load_fw(adap, fw->data, fw->size);
856 if (!ret)
857 dev_info(dev, "firmware upgraded to version %pI4 from "
858 FW_FNAME "\n", &hdr->fw_ver);
859 }
860 out: release_firmware(fw);
861 return ret;
862 }
863
864 /*
865 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
866 * The allocated memory is cleared.
867 */
868 void *t4_alloc_mem(size_t size)
869 {
870 void *p = kzalloc(size, GFP_KERNEL);
871
872 if (!p)
873 p = vzalloc(size);
874 return p;
875 }
876
877 /*
878 * Free memory allocated through alloc_mem().
879 */
880 static void t4_free_mem(void *addr)
881 {
882 if (is_vmalloc_addr(addr))
883 vfree(addr);
884 else
885 kfree(addr);
886 }
887
888 static inline int is_offload(const struct adapter *adap)
889 {
890 return adap->params.offload;
891 }
892
893 /*
894 * Implementation of ethtool operations.
895 */
896
897 static u32 get_msglevel(struct net_device *dev)
898 {
899 return netdev2adap(dev)->msg_enable;
900 }
901
902 static void set_msglevel(struct net_device *dev, u32 val)
903 {
904 netdev2adap(dev)->msg_enable = val;
905 }
906
907 static char stats_strings[][ETH_GSTRING_LEN] = {
908 "TxOctetsOK ",
909 "TxFramesOK ",
910 "TxBroadcastFrames ",
911 "TxMulticastFrames ",
912 "TxUnicastFrames ",
913 "TxErrorFrames ",
914
915 "TxFrames64 ",
916 "TxFrames65To127 ",
917 "TxFrames128To255 ",
918 "TxFrames256To511 ",
919 "TxFrames512To1023 ",
920 "TxFrames1024To1518 ",
921 "TxFrames1519ToMax ",
922
923 "TxFramesDropped ",
924 "TxPauseFrames ",
925 "TxPPP0Frames ",
926 "TxPPP1Frames ",
927 "TxPPP2Frames ",
928 "TxPPP3Frames ",
929 "TxPPP4Frames ",
930 "TxPPP5Frames ",
931 "TxPPP6Frames ",
932 "TxPPP7Frames ",
933
934 "RxOctetsOK ",
935 "RxFramesOK ",
936 "RxBroadcastFrames ",
937 "RxMulticastFrames ",
938 "RxUnicastFrames ",
939
940 "RxFramesTooLong ",
941 "RxJabberErrors ",
942 "RxFCSErrors ",
943 "RxLengthErrors ",
944 "RxSymbolErrors ",
945 "RxRuntFrames ",
946
947 "RxFrames64 ",
948 "RxFrames65To127 ",
949 "RxFrames128To255 ",
950 "RxFrames256To511 ",
951 "RxFrames512To1023 ",
952 "RxFrames1024To1518 ",
953 "RxFrames1519ToMax ",
954
955 "RxPauseFrames ",
956 "RxPPP0Frames ",
957 "RxPPP1Frames ",
958 "RxPPP2Frames ",
959 "RxPPP3Frames ",
960 "RxPPP4Frames ",
961 "RxPPP5Frames ",
962 "RxPPP6Frames ",
963 "RxPPP7Frames ",
964
965 "RxBG0FramesDropped ",
966 "RxBG1FramesDropped ",
967 "RxBG2FramesDropped ",
968 "RxBG3FramesDropped ",
969 "RxBG0FramesTrunc ",
970 "RxBG1FramesTrunc ",
971 "RxBG2FramesTrunc ",
972 "RxBG3FramesTrunc ",
973
974 "TSO ",
975 "TxCsumOffload ",
976 "RxCsumGood ",
977 "VLANextractions ",
978 "VLANinsertions ",
979 "GROpackets ",
980 "GROmerged ",
981 };
982
983 static int get_sset_count(struct net_device *dev, int sset)
984 {
985 switch (sset) {
986 case ETH_SS_STATS:
987 return ARRAY_SIZE(stats_strings);
988 default:
989 return -EOPNOTSUPP;
990 }
991 }
992
993 #define T4_REGMAP_SIZE (160 * 1024)
994
995 static int get_regs_len(struct net_device *dev)
996 {
997 return T4_REGMAP_SIZE;
998 }
999
1000 static int get_eeprom_len(struct net_device *dev)
1001 {
1002 return EEPROMSIZE;
1003 }
1004
1005 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1006 {
1007 struct adapter *adapter = netdev2adap(dev);
1008
1009 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1010 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1011 strlcpy(info->bus_info, pci_name(adapter->pdev),
1012 sizeof(info->bus_info));
1013
1014 if (adapter->params.fw_vers)
1015 snprintf(info->fw_version, sizeof(info->fw_version),
1016 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1017 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1018 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1019 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1020 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1021 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1022 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1023 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1024 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1025 }
1026
1027 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1028 {
1029 if (stringset == ETH_SS_STATS)
1030 memcpy(data, stats_strings, sizeof(stats_strings));
1031 }
1032
1033 /*
1034 * port stats maintained per queue of the port. They should be in the same
1035 * order as in stats_strings above.
1036 */
1037 struct queue_port_stats {
1038 u64 tso;
1039 u64 tx_csum;
1040 u64 rx_csum;
1041 u64 vlan_ex;
1042 u64 vlan_ins;
1043 u64 gro_pkts;
1044 u64 gro_merged;
1045 };
1046
1047 static void collect_sge_port_stats(const struct adapter *adap,
1048 const struct port_info *p, struct queue_port_stats *s)
1049 {
1050 int i;
1051 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1052 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1053
1054 memset(s, 0, sizeof(*s));
1055 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1056 s->tso += tx->tso;
1057 s->tx_csum += tx->tx_cso;
1058 s->rx_csum += rx->stats.rx_cso;
1059 s->vlan_ex += rx->stats.vlan_ex;
1060 s->vlan_ins += tx->vlan_ins;
1061 s->gro_pkts += rx->stats.lro_pkts;
1062 s->gro_merged += rx->stats.lro_merged;
1063 }
1064 }
1065
1066 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1067 u64 *data)
1068 {
1069 struct port_info *pi = netdev_priv(dev);
1070 struct adapter *adapter = pi->adapter;
1071
1072 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1073
1074 data += sizeof(struct port_stats) / sizeof(u64);
1075 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1076 }
1077
1078 /*
1079 * Return a version number to identify the type of adapter. The scheme is:
1080 * - bits 0..9: chip version
1081 * - bits 10..15: chip revision
1082 * - bits 16..23: register dump version
1083 */
1084 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1085 {
1086 return 4 | (ap->params.rev << 10) | (1 << 16);
1087 }
1088
1089 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1090 unsigned int end)
1091 {
1092 u32 *p = buf + start;
1093
1094 for ( ; start <= end; start += sizeof(u32))
1095 *p++ = t4_read_reg(ap, start);
1096 }
1097
1098 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1099 void *buf)
1100 {
1101 static const unsigned int reg_ranges[] = {
1102 0x1008, 0x1108,
1103 0x1180, 0x11b4,
1104 0x11fc, 0x123c,
1105 0x1300, 0x173c,
1106 0x1800, 0x18fc,
1107 0x3000, 0x30d8,
1108 0x30e0, 0x5924,
1109 0x5960, 0x59d4,
1110 0x5a00, 0x5af8,
1111 0x6000, 0x6098,
1112 0x6100, 0x6150,
1113 0x6200, 0x6208,
1114 0x6240, 0x6248,
1115 0x6280, 0x6338,
1116 0x6370, 0x638c,
1117 0x6400, 0x643c,
1118 0x6500, 0x6524,
1119 0x6a00, 0x6a38,
1120 0x6a60, 0x6a78,
1121 0x6b00, 0x6b84,
1122 0x6bf0, 0x6c84,
1123 0x6cf0, 0x6d84,
1124 0x6df0, 0x6e84,
1125 0x6ef0, 0x6f84,
1126 0x6ff0, 0x7084,
1127 0x70f0, 0x7184,
1128 0x71f0, 0x7284,
1129 0x72f0, 0x7384,
1130 0x73f0, 0x7450,
1131 0x7500, 0x7530,
1132 0x7600, 0x761c,
1133 0x7680, 0x76cc,
1134 0x7700, 0x7798,
1135 0x77c0, 0x77fc,
1136 0x7900, 0x79fc,
1137 0x7b00, 0x7c38,
1138 0x7d00, 0x7efc,
1139 0x8dc0, 0x8e1c,
1140 0x8e30, 0x8e78,
1141 0x8ea0, 0x8f6c,
1142 0x8fc0, 0x9074,
1143 0x90fc, 0x90fc,
1144 0x9400, 0x9458,
1145 0x9600, 0x96bc,
1146 0x9800, 0x9808,
1147 0x9820, 0x983c,
1148 0x9850, 0x9864,
1149 0x9c00, 0x9c6c,
1150 0x9c80, 0x9cec,
1151 0x9d00, 0x9d6c,
1152 0x9d80, 0x9dec,
1153 0x9e00, 0x9e6c,
1154 0x9e80, 0x9eec,
1155 0x9f00, 0x9f6c,
1156 0x9f80, 0x9fec,
1157 0xd004, 0xd03c,
1158 0xdfc0, 0xdfe0,
1159 0xe000, 0xea7c,
1160 0xf000, 0x11190,
1161 0x19040, 0x1906c,
1162 0x19078, 0x19080,
1163 0x1908c, 0x19124,
1164 0x19150, 0x191b0,
1165 0x191d0, 0x191e8,
1166 0x19238, 0x1924c,
1167 0x193f8, 0x19474,
1168 0x19490, 0x194f8,
1169 0x19800, 0x19f30,
1170 0x1a000, 0x1a06c,
1171 0x1a0b0, 0x1a120,
1172 0x1a128, 0x1a138,
1173 0x1a190, 0x1a1c4,
1174 0x1a1fc, 0x1a1fc,
1175 0x1e040, 0x1e04c,
1176 0x1e284, 0x1e28c,
1177 0x1e2c0, 0x1e2c0,
1178 0x1e2e0, 0x1e2e0,
1179 0x1e300, 0x1e384,
1180 0x1e3c0, 0x1e3c8,
1181 0x1e440, 0x1e44c,
1182 0x1e684, 0x1e68c,
1183 0x1e6c0, 0x1e6c0,
1184 0x1e6e0, 0x1e6e0,
1185 0x1e700, 0x1e784,
1186 0x1e7c0, 0x1e7c8,
1187 0x1e840, 0x1e84c,
1188 0x1ea84, 0x1ea8c,
1189 0x1eac0, 0x1eac0,
1190 0x1eae0, 0x1eae0,
1191 0x1eb00, 0x1eb84,
1192 0x1ebc0, 0x1ebc8,
1193 0x1ec40, 0x1ec4c,
1194 0x1ee84, 0x1ee8c,
1195 0x1eec0, 0x1eec0,
1196 0x1eee0, 0x1eee0,
1197 0x1ef00, 0x1ef84,
1198 0x1efc0, 0x1efc8,
1199 0x1f040, 0x1f04c,
1200 0x1f284, 0x1f28c,
1201 0x1f2c0, 0x1f2c0,
1202 0x1f2e0, 0x1f2e0,
1203 0x1f300, 0x1f384,
1204 0x1f3c0, 0x1f3c8,
1205 0x1f440, 0x1f44c,
1206 0x1f684, 0x1f68c,
1207 0x1f6c0, 0x1f6c0,
1208 0x1f6e0, 0x1f6e0,
1209 0x1f700, 0x1f784,
1210 0x1f7c0, 0x1f7c8,
1211 0x1f840, 0x1f84c,
1212 0x1fa84, 0x1fa8c,
1213 0x1fac0, 0x1fac0,
1214 0x1fae0, 0x1fae0,
1215 0x1fb00, 0x1fb84,
1216 0x1fbc0, 0x1fbc8,
1217 0x1fc40, 0x1fc4c,
1218 0x1fe84, 0x1fe8c,
1219 0x1fec0, 0x1fec0,
1220 0x1fee0, 0x1fee0,
1221 0x1ff00, 0x1ff84,
1222 0x1ffc0, 0x1ffc8,
1223 0x20000, 0x2002c,
1224 0x20100, 0x2013c,
1225 0x20190, 0x201c8,
1226 0x20200, 0x20318,
1227 0x20400, 0x20528,
1228 0x20540, 0x20614,
1229 0x21000, 0x21040,
1230 0x2104c, 0x21060,
1231 0x210c0, 0x210ec,
1232 0x21200, 0x21268,
1233 0x21270, 0x21284,
1234 0x212fc, 0x21388,
1235 0x21400, 0x21404,
1236 0x21500, 0x21518,
1237 0x2152c, 0x2153c,
1238 0x21550, 0x21554,
1239 0x21600, 0x21600,
1240 0x21608, 0x21628,
1241 0x21630, 0x2163c,
1242 0x21700, 0x2171c,
1243 0x21780, 0x2178c,
1244 0x21800, 0x21c38,
1245 0x21c80, 0x21d7c,
1246 0x21e00, 0x21e04,
1247 0x22000, 0x2202c,
1248 0x22100, 0x2213c,
1249 0x22190, 0x221c8,
1250 0x22200, 0x22318,
1251 0x22400, 0x22528,
1252 0x22540, 0x22614,
1253 0x23000, 0x23040,
1254 0x2304c, 0x23060,
1255 0x230c0, 0x230ec,
1256 0x23200, 0x23268,
1257 0x23270, 0x23284,
1258 0x232fc, 0x23388,
1259 0x23400, 0x23404,
1260 0x23500, 0x23518,
1261 0x2352c, 0x2353c,
1262 0x23550, 0x23554,
1263 0x23600, 0x23600,
1264 0x23608, 0x23628,
1265 0x23630, 0x2363c,
1266 0x23700, 0x2371c,
1267 0x23780, 0x2378c,
1268 0x23800, 0x23c38,
1269 0x23c80, 0x23d7c,
1270 0x23e00, 0x23e04,
1271 0x24000, 0x2402c,
1272 0x24100, 0x2413c,
1273 0x24190, 0x241c8,
1274 0x24200, 0x24318,
1275 0x24400, 0x24528,
1276 0x24540, 0x24614,
1277 0x25000, 0x25040,
1278 0x2504c, 0x25060,
1279 0x250c0, 0x250ec,
1280 0x25200, 0x25268,
1281 0x25270, 0x25284,
1282 0x252fc, 0x25388,
1283 0x25400, 0x25404,
1284 0x25500, 0x25518,
1285 0x2552c, 0x2553c,
1286 0x25550, 0x25554,
1287 0x25600, 0x25600,
1288 0x25608, 0x25628,
1289 0x25630, 0x2563c,
1290 0x25700, 0x2571c,
1291 0x25780, 0x2578c,
1292 0x25800, 0x25c38,
1293 0x25c80, 0x25d7c,
1294 0x25e00, 0x25e04,
1295 0x26000, 0x2602c,
1296 0x26100, 0x2613c,
1297 0x26190, 0x261c8,
1298 0x26200, 0x26318,
1299 0x26400, 0x26528,
1300 0x26540, 0x26614,
1301 0x27000, 0x27040,
1302 0x2704c, 0x27060,
1303 0x270c0, 0x270ec,
1304 0x27200, 0x27268,
1305 0x27270, 0x27284,
1306 0x272fc, 0x27388,
1307 0x27400, 0x27404,
1308 0x27500, 0x27518,
1309 0x2752c, 0x2753c,
1310 0x27550, 0x27554,
1311 0x27600, 0x27600,
1312 0x27608, 0x27628,
1313 0x27630, 0x2763c,
1314 0x27700, 0x2771c,
1315 0x27780, 0x2778c,
1316 0x27800, 0x27c38,
1317 0x27c80, 0x27d7c,
1318 0x27e00, 0x27e04
1319 };
1320
1321 int i;
1322 struct adapter *ap = netdev2adap(dev);
1323
1324 regs->version = mk_adap_vers(ap);
1325
1326 memset(buf, 0, T4_REGMAP_SIZE);
1327 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1328 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1329 }
1330
1331 static int restart_autoneg(struct net_device *dev)
1332 {
1333 struct port_info *p = netdev_priv(dev);
1334
1335 if (!netif_running(dev))
1336 return -EAGAIN;
1337 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1338 return -EINVAL;
1339 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1340 return 0;
1341 }
1342
1343 static int identify_port(struct net_device *dev,
1344 enum ethtool_phys_id_state state)
1345 {
1346 unsigned int val;
1347 struct adapter *adap = netdev2adap(dev);
1348
1349 if (state == ETHTOOL_ID_ACTIVE)
1350 val = 0xffff;
1351 else if (state == ETHTOOL_ID_INACTIVE)
1352 val = 0;
1353 else
1354 return -EINVAL;
1355
1356 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1357 }
1358
1359 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1360 {
1361 unsigned int v = 0;
1362
1363 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1364 type == FW_PORT_TYPE_BT_XAUI) {
1365 v |= SUPPORTED_TP;
1366 if (caps & FW_PORT_CAP_SPEED_100M)
1367 v |= SUPPORTED_100baseT_Full;
1368 if (caps & FW_PORT_CAP_SPEED_1G)
1369 v |= SUPPORTED_1000baseT_Full;
1370 if (caps & FW_PORT_CAP_SPEED_10G)
1371 v |= SUPPORTED_10000baseT_Full;
1372 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1373 v |= SUPPORTED_Backplane;
1374 if (caps & FW_PORT_CAP_SPEED_1G)
1375 v |= SUPPORTED_1000baseKX_Full;
1376 if (caps & FW_PORT_CAP_SPEED_10G)
1377 v |= SUPPORTED_10000baseKX4_Full;
1378 } else if (type == FW_PORT_TYPE_KR)
1379 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1380 else if (type == FW_PORT_TYPE_BP_AP)
1381 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1382 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1383 else if (type == FW_PORT_TYPE_BP4_AP)
1384 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1385 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1386 SUPPORTED_10000baseKX4_Full;
1387 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1388 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1389 v |= SUPPORTED_FIBRE;
1390
1391 if (caps & FW_PORT_CAP_ANEG)
1392 v |= SUPPORTED_Autoneg;
1393 return v;
1394 }
1395
1396 static unsigned int to_fw_linkcaps(unsigned int caps)
1397 {
1398 unsigned int v = 0;
1399
1400 if (caps & ADVERTISED_100baseT_Full)
1401 v |= FW_PORT_CAP_SPEED_100M;
1402 if (caps & ADVERTISED_1000baseT_Full)
1403 v |= FW_PORT_CAP_SPEED_1G;
1404 if (caps & ADVERTISED_10000baseT_Full)
1405 v |= FW_PORT_CAP_SPEED_10G;
1406 return v;
1407 }
1408
1409 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1410 {
1411 const struct port_info *p = netdev_priv(dev);
1412
1413 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1414 p->port_type == FW_PORT_TYPE_BT_XFI ||
1415 p->port_type == FW_PORT_TYPE_BT_XAUI)
1416 cmd->port = PORT_TP;
1417 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1418 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1419 cmd->port = PORT_FIBRE;
1420 else if (p->port_type == FW_PORT_TYPE_SFP) {
1421 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1422 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1423 cmd->port = PORT_DA;
1424 else
1425 cmd->port = PORT_FIBRE;
1426 } else
1427 cmd->port = PORT_OTHER;
1428
1429 if (p->mdio_addr >= 0) {
1430 cmd->phy_address = p->mdio_addr;
1431 cmd->transceiver = XCVR_EXTERNAL;
1432 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1433 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1434 } else {
1435 cmd->phy_address = 0; /* not really, but no better option */
1436 cmd->transceiver = XCVR_INTERNAL;
1437 cmd->mdio_support = 0;
1438 }
1439
1440 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1441 cmd->advertising = from_fw_linkcaps(p->port_type,
1442 p->link_cfg.advertising);
1443 ethtool_cmd_speed_set(cmd,
1444 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1445 cmd->duplex = DUPLEX_FULL;
1446 cmd->autoneg = p->link_cfg.autoneg;
1447 cmd->maxtxpkt = 0;
1448 cmd->maxrxpkt = 0;
1449 return 0;
1450 }
1451
1452 static unsigned int speed_to_caps(int speed)
1453 {
1454 if (speed == SPEED_100)
1455 return FW_PORT_CAP_SPEED_100M;
1456 if (speed == SPEED_1000)
1457 return FW_PORT_CAP_SPEED_1G;
1458 if (speed == SPEED_10000)
1459 return FW_PORT_CAP_SPEED_10G;
1460 return 0;
1461 }
1462
1463 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1464 {
1465 unsigned int cap;
1466 struct port_info *p = netdev_priv(dev);
1467 struct link_config *lc = &p->link_cfg;
1468 u32 speed = ethtool_cmd_speed(cmd);
1469
1470 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1471 return -EINVAL;
1472
1473 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1474 /*
1475 * PHY offers a single speed. See if that's what's
1476 * being requested.
1477 */
1478 if (cmd->autoneg == AUTONEG_DISABLE &&
1479 (lc->supported & speed_to_caps(speed)))
1480 return 0;
1481 return -EINVAL;
1482 }
1483
1484 if (cmd->autoneg == AUTONEG_DISABLE) {
1485 cap = speed_to_caps(speed);
1486
1487 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1488 (speed == SPEED_10000))
1489 return -EINVAL;
1490 lc->requested_speed = cap;
1491 lc->advertising = 0;
1492 } else {
1493 cap = to_fw_linkcaps(cmd->advertising);
1494 if (!(lc->supported & cap))
1495 return -EINVAL;
1496 lc->requested_speed = 0;
1497 lc->advertising = cap | FW_PORT_CAP_ANEG;
1498 }
1499 lc->autoneg = cmd->autoneg;
1500
1501 if (netif_running(dev))
1502 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1503 lc);
1504 return 0;
1505 }
1506
1507 static void get_pauseparam(struct net_device *dev,
1508 struct ethtool_pauseparam *epause)
1509 {
1510 struct port_info *p = netdev_priv(dev);
1511
1512 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1513 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1514 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1515 }
1516
1517 static int set_pauseparam(struct net_device *dev,
1518 struct ethtool_pauseparam *epause)
1519 {
1520 struct port_info *p = netdev_priv(dev);
1521 struct link_config *lc = &p->link_cfg;
1522
1523 if (epause->autoneg == AUTONEG_DISABLE)
1524 lc->requested_fc = 0;
1525 else if (lc->supported & FW_PORT_CAP_ANEG)
1526 lc->requested_fc = PAUSE_AUTONEG;
1527 else
1528 return -EINVAL;
1529
1530 if (epause->rx_pause)
1531 lc->requested_fc |= PAUSE_RX;
1532 if (epause->tx_pause)
1533 lc->requested_fc |= PAUSE_TX;
1534 if (netif_running(dev))
1535 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1536 lc);
1537 return 0;
1538 }
1539
1540 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1541 {
1542 const struct port_info *pi = netdev_priv(dev);
1543 const struct sge *s = &pi->adapter->sge;
1544
1545 e->rx_max_pending = MAX_RX_BUFFERS;
1546 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1547 e->rx_jumbo_max_pending = 0;
1548 e->tx_max_pending = MAX_TXQ_ENTRIES;
1549
1550 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1551 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1552 e->rx_jumbo_pending = 0;
1553 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1554 }
1555
1556 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1557 {
1558 int i;
1559 const struct port_info *pi = netdev_priv(dev);
1560 struct adapter *adapter = pi->adapter;
1561 struct sge *s = &adapter->sge;
1562
1563 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1564 e->tx_pending > MAX_TXQ_ENTRIES ||
1565 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1566 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1567 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1568 return -EINVAL;
1569
1570 if (adapter->flags & FULL_INIT_DONE)
1571 return -EBUSY;
1572
1573 for (i = 0; i < pi->nqsets; ++i) {
1574 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1575 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1576 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1577 }
1578 return 0;
1579 }
1580
1581 static int closest_timer(const struct sge *s, int time)
1582 {
1583 int i, delta, match = 0, min_delta = INT_MAX;
1584
1585 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1586 delta = time - s->timer_val[i];
1587 if (delta < 0)
1588 delta = -delta;
1589 if (delta < min_delta) {
1590 min_delta = delta;
1591 match = i;
1592 }
1593 }
1594 return match;
1595 }
1596
1597 static int closest_thres(const struct sge *s, int thres)
1598 {
1599 int i, delta, match = 0, min_delta = INT_MAX;
1600
1601 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1602 delta = thres - s->counter_val[i];
1603 if (delta < 0)
1604 delta = -delta;
1605 if (delta < min_delta) {
1606 min_delta = delta;
1607 match = i;
1608 }
1609 }
1610 return match;
1611 }
1612
1613 /*
1614 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1615 */
1616 static unsigned int qtimer_val(const struct adapter *adap,
1617 const struct sge_rspq *q)
1618 {
1619 unsigned int idx = q->intr_params >> 1;
1620
1621 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1622 }
1623
1624 /**
1625 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1626 * @adap: the adapter
1627 * @q: the Rx queue
1628 * @us: the hold-off time in us, or 0 to disable timer
1629 * @cnt: the hold-off packet count, or 0 to disable counter
1630 *
1631 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1632 * one of the two needs to be enabled for the queue to generate interrupts.
1633 */
1634 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1635 unsigned int us, unsigned int cnt)
1636 {
1637 if ((us | cnt) == 0)
1638 cnt = 1;
1639
1640 if (cnt) {
1641 int err;
1642 u32 v, new_idx;
1643
1644 new_idx = closest_thres(&adap->sge, cnt);
1645 if (q->desc && q->pktcnt_idx != new_idx) {
1646 /* the queue has already been created, update it */
1647 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1648 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1649 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1650 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1651 &new_idx);
1652 if (err)
1653 return err;
1654 }
1655 q->pktcnt_idx = new_idx;
1656 }
1657
1658 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1659 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1660 return 0;
1661 }
1662
1663 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1664 {
1665 const struct port_info *pi = netdev_priv(dev);
1666 struct adapter *adap = pi->adapter;
1667
1668 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1669 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1670 }
1671
1672 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1673 {
1674 const struct port_info *pi = netdev_priv(dev);
1675 const struct adapter *adap = pi->adapter;
1676 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1677
1678 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1679 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1680 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1681 return 0;
1682 }
1683
1684 /**
1685 * eeprom_ptov - translate a physical EEPROM address to virtual
1686 * @phys_addr: the physical EEPROM address
1687 * @fn: the PCI function number
1688 * @sz: size of function-specific area
1689 *
1690 * Translate a physical EEPROM address to virtual. The first 1K is
1691 * accessed through virtual addresses starting at 31K, the rest is
1692 * accessed through virtual addresses starting at 0.
1693 *
1694 * The mapping is as follows:
1695 * [0..1K) -> [31K..32K)
1696 * [1K..1K+A) -> [31K-A..31K)
1697 * [1K+A..ES) -> [0..ES-A-1K)
1698 *
1699 * where A = @fn * @sz, and ES = EEPROM size.
1700 */
1701 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1702 {
1703 fn *= sz;
1704 if (phys_addr < 1024)
1705 return phys_addr + (31 << 10);
1706 if (phys_addr < 1024 + fn)
1707 return 31744 - fn + phys_addr - 1024;
1708 if (phys_addr < EEPROMSIZE)
1709 return phys_addr - 1024 - fn;
1710 return -EINVAL;
1711 }
1712
1713 /*
1714 * The next two routines implement eeprom read/write from physical addresses.
1715 */
1716 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1717 {
1718 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1719
1720 if (vaddr >= 0)
1721 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1722 return vaddr < 0 ? vaddr : 0;
1723 }
1724
1725 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1726 {
1727 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1728
1729 if (vaddr >= 0)
1730 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1731 return vaddr < 0 ? vaddr : 0;
1732 }
1733
1734 #define EEPROM_MAGIC 0x38E2F10C
1735
1736 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1737 u8 *data)
1738 {
1739 int i, err = 0;
1740 struct adapter *adapter = netdev2adap(dev);
1741
1742 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1743 if (!buf)
1744 return -ENOMEM;
1745
1746 e->magic = EEPROM_MAGIC;
1747 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1748 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1749
1750 if (!err)
1751 memcpy(data, buf + e->offset, e->len);
1752 kfree(buf);
1753 return err;
1754 }
1755
1756 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1757 u8 *data)
1758 {
1759 u8 *buf;
1760 int err = 0;
1761 u32 aligned_offset, aligned_len, *p;
1762 struct adapter *adapter = netdev2adap(dev);
1763
1764 if (eeprom->magic != EEPROM_MAGIC)
1765 return -EINVAL;
1766
1767 aligned_offset = eeprom->offset & ~3;
1768 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1769
1770 if (adapter->fn > 0) {
1771 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1772
1773 if (aligned_offset < start ||
1774 aligned_offset + aligned_len > start + EEPROMPFSIZE)
1775 return -EPERM;
1776 }
1777
1778 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1779 /*
1780 * RMW possibly needed for first or last words.
1781 */
1782 buf = kmalloc(aligned_len, GFP_KERNEL);
1783 if (!buf)
1784 return -ENOMEM;
1785 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1786 if (!err && aligned_len > 4)
1787 err = eeprom_rd_phys(adapter,
1788 aligned_offset + aligned_len - 4,
1789 (u32 *)&buf[aligned_len - 4]);
1790 if (err)
1791 goto out;
1792 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1793 } else
1794 buf = data;
1795
1796 err = t4_seeprom_wp(adapter, false);
1797 if (err)
1798 goto out;
1799
1800 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1801 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1802 aligned_offset += 4;
1803 }
1804
1805 if (!err)
1806 err = t4_seeprom_wp(adapter, true);
1807 out:
1808 if (buf != data)
1809 kfree(buf);
1810 return err;
1811 }
1812
1813 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1814 {
1815 int ret;
1816 const struct firmware *fw;
1817 struct adapter *adap = netdev2adap(netdev);
1818
1819 ef->data[sizeof(ef->data) - 1] = '\0';
1820 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1821 if (ret < 0)
1822 return ret;
1823
1824 ret = t4_load_fw(adap, fw->data, fw->size);
1825 release_firmware(fw);
1826 if (!ret)
1827 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1828 return ret;
1829 }
1830
1831 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1832 #define BCAST_CRC 0xa0ccc1a6
1833
1834 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1835 {
1836 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1837 wol->wolopts = netdev2adap(dev)->wol;
1838 memset(&wol->sopass, 0, sizeof(wol->sopass));
1839 }
1840
1841 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1842 {
1843 int err = 0;
1844 struct port_info *pi = netdev_priv(dev);
1845
1846 if (wol->wolopts & ~WOL_SUPPORTED)
1847 return -EINVAL;
1848 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1849 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1850 if (wol->wolopts & WAKE_BCAST) {
1851 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1852 ~0ULL, 0, false);
1853 if (!err)
1854 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1855 ~6ULL, ~0ULL, BCAST_CRC, true);
1856 } else
1857 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1858 return err;
1859 }
1860
1861 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
1862 {
1863 const struct port_info *pi = netdev_priv(dev);
1864 netdev_features_t changed = dev->features ^ features;
1865 int err;
1866
1867 if (!(changed & NETIF_F_HW_VLAN_RX))
1868 return 0;
1869
1870 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1871 -1, -1, -1,
1872 !!(features & NETIF_F_HW_VLAN_RX), true);
1873 if (unlikely(err))
1874 dev->features = features ^ NETIF_F_HW_VLAN_RX;
1875 return err;
1876 }
1877
1878 static u32 get_rss_table_size(struct net_device *dev)
1879 {
1880 const struct port_info *pi = netdev_priv(dev);
1881
1882 return pi->rss_size;
1883 }
1884
1885 static int get_rss_table(struct net_device *dev, u32 *p)
1886 {
1887 const struct port_info *pi = netdev_priv(dev);
1888 unsigned int n = pi->rss_size;
1889
1890 while (n--)
1891 p[n] = pi->rss[n];
1892 return 0;
1893 }
1894
1895 static int set_rss_table(struct net_device *dev, const u32 *p)
1896 {
1897 unsigned int i;
1898 struct port_info *pi = netdev_priv(dev);
1899
1900 for (i = 0; i < pi->rss_size; i++)
1901 pi->rss[i] = p[i];
1902 if (pi->adapter->flags & FULL_INIT_DONE)
1903 return write_rss(pi, pi->rss);
1904 return 0;
1905 }
1906
1907 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1908 u32 *rules)
1909 {
1910 const struct port_info *pi = netdev_priv(dev);
1911
1912 switch (info->cmd) {
1913 case ETHTOOL_GRXFH: {
1914 unsigned int v = pi->rss_mode;
1915
1916 info->data = 0;
1917 switch (info->flow_type) {
1918 case TCP_V4_FLOW:
1919 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1920 info->data = RXH_IP_SRC | RXH_IP_DST |
1921 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1922 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1923 info->data = RXH_IP_SRC | RXH_IP_DST;
1924 break;
1925 case UDP_V4_FLOW:
1926 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1927 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1928 info->data = RXH_IP_SRC | RXH_IP_DST |
1929 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1930 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1931 info->data = RXH_IP_SRC | RXH_IP_DST;
1932 break;
1933 case SCTP_V4_FLOW:
1934 case AH_ESP_V4_FLOW:
1935 case IPV4_FLOW:
1936 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1937 info->data = RXH_IP_SRC | RXH_IP_DST;
1938 break;
1939 case TCP_V6_FLOW:
1940 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1941 info->data = RXH_IP_SRC | RXH_IP_DST |
1942 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1943 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1944 info->data = RXH_IP_SRC | RXH_IP_DST;
1945 break;
1946 case UDP_V6_FLOW:
1947 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1948 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1949 info->data = RXH_IP_SRC | RXH_IP_DST |
1950 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1951 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1952 info->data = RXH_IP_SRC | RXH_IP_DST;
1953 break;
1954 case SCTP_V6_FLOW:
1955 case AH_ESP_V6_FLOW:
1956 case IPV6_FLOW:
1957 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1958 info->data = RXH_IP_SRC | RXH_IP_DST;
1959 break;
1960 }
1961 return 0;
1962 }
1963 case ETHTOOL_GRXRINGS:
1964 info->data = pi->nqsets;
1965 return 0;
1966 }
1967 return -EOPNOTSUPP;
1968 }
1969
1970 static const struct ethtool_ops cxgb_ethtool_ops = {
1971 .get_settings = get_settings,
1972 .set_settings = set_settings,
1973 .get_drvinfo = get_drvinfo,
1974 .get_msglevel = get_msglevel,
1975 .set_msglevel = set_msglevel,
1976 .get_ringparam = get_sge_param,
1977 .set_ringparam = set_sge_param,
1978 .get_coalesce = get_coalesce,
1979 .set_coalesce = set_coalesce,
1980 .get_eeprom_len = get_eeprom_len,
1981 .get_eeprom = get_eeprom,
1982 .set_eeprom = set_eeprom,
1983 .get_pauseparam = get_pauseparam,
1984 .set_pauseparam = set_pauseparam,
1985 .get_link = ethtool_op_get_link,
1986 .get_strings = get_strings,
1987 .set_phys_id = identify_port,
1988 .nway_reset = restart_autoneg,
1989 .get_sset_count = get_sset_count,
1990 .get_ethtool_stats = get_stats,
1991 .get_regs_len = get_regs_len,
1992 .get_regs = get_regs,
1993 .get_wol = get_wol,
1994 .set_wol = set_wol,
1995 .get_rxnfc = get_rxnfc,
1996 .get_rxfh_indir_size = get_rss_table_size,
1997 .get_rxfh_indir = get_rss_table,
1998 .set_rxfh_indir = set_rss_table,
1999 .flash_device = set_flash,
2000 };
2001
2002 /*
2003 * debugfs support
2004 */
2005 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2006 loff_t *ppos)
2007 {
2008 loff_t pos = *ppos;
2009 loff_t avail = file->f_path.dentry->d_inode->i_size;
2010 unsigned int mem = (uintptr_t)file->private_data & 3;
2011 struct adapter *adap = file->private_data - mem;
2012
2013 if (pos < 0)
2014 return -EINVAL;
2015 if (pos >= avail)
2016 return 0;
2017 if (count > avail - pos)
2018 count = avail - pos;
2019
2020 while (count) {
2021 size_t len;
2022 int ret, ofst;
2023 __be32 data[16];
2024
2025 if (mem == MEM_MC)
2026 ret = t4_mc_read(adap, pos, data, NULL);
2027 else
2028 ret = t4_edc_read(adap, mem, pos, data, NULL);
2029 if (ret)
2030 return ret;
2031
2032 ofst = pos % sizeof(data);
2033 len = min(count, sizeof(data) - ofst);
2034 if (copy_to_user(buf, (u8 *)data + ofst, len))
2035 return -EFAULT;
2036
2037 buf += len;
2038 pos += len;
2039 count -= len;
2040 }
2041 count = pos - *ppos;
2042 *ppos = pos;
2043 return count;
2044 }
2045
2046 static const struct file_operations mem_debugfs_fops = {
2047 .owner = THIS_MODULE,
2048 .open = simple_open,
2049 .read = mem_read,
2050 .llseek = default_llseek,
2051 };
2052
2053 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2054 unsigned int idx, unsigned int size_mb)
2055 {
2056 struct dentry *de;
2057
2058 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2059 (void *)adap + idx, &mem_debugfs_fops);
2060 if (de && de->d_inode)
2061 de->d_inode->i_size = size_mb << 20;
2062 }
2063
2064 static int __devinit setup_debugfs(struct adapter *adap)
2065 {
2066 int i;
2067
2068 if (IS_ERR_OR_NULL(adap->debugfs_root))
2069 return -1;
2070
2071 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2072 if (i & EDRAM0_ENABLE)
2073 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2074 if (i & EDRAM1_ENABLE)
2075 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2076 if (i & EXT_MEM_ENABLE)
2077 add_debugfs_mem(adap, "mc", MEM_MC,
2078 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2079 if (adap->l2t)
2080 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2081 &t4_l2t_fops);
2082 return 0;
2083 }
2084
2085 /*
2086 * upper-layer driver support
2087 */
2088
2089 /*
2090 * Allocate an active-open TID and set it to the supplied value.
2091 */
2092 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2093 {
2094 int atid = -1;
2095
2096 spin_lock_bh(&t->atid_lock);
2097 if (t->afree) {
2098 union aopen_entry *p = t->afree;
2099
2100 atid = p - t->atid_tab;
2101 t->afree = p->next;
2102 p->data = data;
2103 t->atids_in_use++;
2104 }
2105 spin_unlock_bh(&t->atid_lock);
2106 return atid;
2107 }
2108 EXPORT_SYMBOL(cxgb4_alloc_atid);
2109
2110 /*
2111 * Release an active-open TID.
2112 */
2113 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2114 {
2115 union aopen_entry *p = &t->atid_tab[atid];
2116
2117 spin_lock_bh(&t->atid_lock);
2118 p->next = t->afree;
2119 t->afree = p;
2120 t->atids_in_use--;
2121 spin_unlock_bh(&t->atid_lock);
2122 }
2123 EXPORT_SYMBOL(cxgb4_free_atid);
2124
2125 /*
2126 * Allocate a server TID and set it to the supplied value.
2127 */
2128 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2129 {
2130 int stid;
2131
2132 spin_lock_bh(&t->stid_lock);
2133 if (family == PF_INET) {
2134 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2135 if (stid < t->nstids)
2136 __set_bit(stid, t->stid_bmap);
2137 else
2138 stid = -1;
2139 } else {
2140 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2141 if (stid < 0)
2142 stid = -1;
2143 }
2144 if (stid >= 0) {
2145 t->stid_tab[stid].data = data;
2146 stid += t->stid_base;
2147 t->stids_in_use++;
2148 }
2149 spin_unlock_bh(&t->stid_lock);
2150 return stid;
2151 }
2152 EXPORT_SYMBOL(cxgb4_alloc_stid);
2153
2154 /*
2155 * Release a server TID.
2156 */
2157 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2158 {
2159 stid -= t->stid_base;
2160 spin_lock_bh(&t->stid_lock);
2161 if (family == PF_INET)
2162 __clear_bit(stid, t->stid_bmap);
2163 else
2164 bitmap_release_region(t->stid_bmap, stid, 2);
2165 t->stid_tab[stid].data = NULL;
2166 t->stids_in_use--;
2167 spin_unlock_bh(&t->stid_lock);
2168 }
2169 EXPORT_SYMBOL(cxgb4_free_stid);
2170
2171 /*
2172 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2173 */
2174 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2175 unsigned int tid)
2176 {
2177 struct cpl_tid_release *req;
2178
2179 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2180 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2181 INIT_TP_WR(req, tid);
2182 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2183 }
2184
2185 /*
2186 * Queue a TID release request and if necessary schedule a work queue to
2187 * process it.
2188 */
2189 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2190 unsigned int tid)
2191 {
2192 void **p = &t->tid_tab[tid];
2193 struct adapter *adap = container_of(t, struct adapter, tids);
2194
2195 spin_lock_bh(&adap->tid_release_lock);
2196 *p = adap->tid_release_head;
2197 /* Low 2 bits encode the Tx channel number */
2198 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2199 if (!adap->tid_release_task_busy) {
2200 adap->tid_release_task_busy = true;
2201 queue_work(workq, &adap->tid_release_task);
2202 }
2203 spin_unlock_bh(&adap->tid_release_lock);
2204 }
2205
2206 /*
2207 * Process the list of pending TID release requests.
2208 */
2209 static void process_tid_release_list(struct work_struct *work)
2210 {
2211 struct sk_buff *skb;
2212 struct adapter *adap;
2213
2214 adap = container_of(work, struct adapter, tid_release_task);
2215
2216 spin_lock_bh(&adap->tid_release_lock);
2217 while (adap->tid_release_head) {
2218 void **p = adap->tid_release_head;
2219 unsigned int chan = (uintptr_t)p & 3;
2220 p = (void *)p - chan;
2221
2222 adap->tid_release_head = *p;
2223 *p = NULL;
2224 spin_unlock_bh(&adap->tid_release_lock);
2225
2226 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2227 GFP_KERNEL)))
2228 schedule_timeout_uninterruptible(1);
2229
2230 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2231 t4_ofld_send(adap, skb);
2232 spin_lock_bh(&adap->tid_release_lock);
2233 }
2234 adap->tid_release_task_busy = false;
2235 spin_unlock_bh(&adap->tid_release_lock);
2236 }
2237
2238 /*
2239 * Release a TID and inform HW. If we are unable to allocate the release
2240 * message we defer to a work queue.
2241 */
2242 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2243 {
2244 void *old;
2245 struct sk_buff *skb;
2246 struct adapter *adap = container_of(t, struct adapter, tids);
2247
2248 old = t->tid_tab[tid];
2249 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2250 if (likely(skb)) {
2251 t->tid_tab[tid] = NULL;
2252 mk_tid_release(skb, chan, tid);
2253 t4_ofld_send(adap, skb);
2254 } else
2255 cxgb4_queue_tid_release(t, chan, tid);
2256 if (old)
2257 atomic_dec(&t->tids_in_use);
2258 }
2259 EXPORT_SYMBOL(cxgb4_remove_tid);
2260
2261 /*
2262 * Allocate and initialize the TID tables. Returns 0 on success.
2263 */
2264 static int tid_init(struct tid_info *t)
2265 {
2266 size_t size;
2267 unsigned int natids = t->natids;
2268
2269 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2270 t->nstids * sizeof(*t->stid_tab) +
2271 BITS_TO_LONGS(t->nstids) * sizeof(long);
2272 t->tid_tab = t4_alloc_mem(size);
2273 if (!t->tid_tab)
2274 return -ENOMEM;
2275
2276 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2277 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2278 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2279 spin_lock_init(&t->stid_lock);
2280 spin_lock_init(&t->atid_lock);
2281
2282 t->stids_in_use = 0;
2283 t->afree = NULL;
2284 t->atids_in_use = 0;
2285 atomic_set(&t->tids_in_use, 0);
2286
2287 /* Setup the free list for atid_tab and clear the stid bitmap. */
2288 if (natids) {
2289 while (--natids)
2290 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2291 t->afree = t->atid_tab;
2292 }
2293 bitmap_zero(t->stid_bmap, t->nstids);
2294 return 0;
2295 }
2296
2297 /**
2298 * cxgb4_create_server - create an IP server
2299 * @dev: the device
2300 * @stid: the server TID
2301 * @sip: local IP address to bind server to
2302 * @sport: the server's TCP port
2303 * @queue: queue to direct messages from this server to
2304 *
2305 * Create an IP server for the given port and address.
2306 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2307 */
2308 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2309 __be32 sip, __be16 sport, unsigned int queue)
2310 {
2311 unsigned int chan;
2312 struct sk_buff *skb;
2313 struct adapter *adap;
2314 struct cpl_pass_open_req *req;
2315
2316 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2317 if (!skb)
2318 return -ENOMEM;
2319
2320 adap = netdev2adap(dev);
2321 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2322 INIT_TP_WR(req, 0);
2323 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2324 req->local_port = sport;
2325 req->peer_port = htons(0);
2326 req->local_ip = sip;
2327 req->peer_ip = htonl(0);
2328 chan = rxq_to_chan(&adap->sge, queue);
2329 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2330 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2331 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2332 return t4_mgmt_tx(adap, skb);
2333 }
2334 EXPORT_SYMBOL(cxgb4_create_server);
2335
2336 /**
2337 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2338 * @mtus: the HW MTU table
2339 * @mtu: the target MTU
2340 * @idx: index of selected entry in the MTU table
2341 *
2342 * Returns the index and the value in the HW MTU table that is closest to
2343 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2344 * table, in which case that smallest available value is selected.
2345 */
2346 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2347 unsigned int *idx)
2348 {
2349 unsigned int i = 0;
2350
2351 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2352 ++i;
2353 if (idx)
2354 *idx = i;
2355 return mtus[i];
2356 }
2357 EXPORT_SYMBOL(cxgb4_best_mtu);
2358
2359 /**
2360 * cxgb4_port_chan - get the HW channel of a port
2361 * @dev: the net device for the port
2362 *
2363 * Return the HW Tx channel of the given port.
2364 */
2365 unsigned int cxgb4_port_chan(const struct net_device *dev)
2366 {
2367 return netdev2pinfo(dev)->tx_chan;
2368 }
2369 EXPORT_SYMBOL(cxgb4_port_chan);
2370
2371 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
2372 {
2373 struct adapter *adap = netdev2adap(dev);
2374 u32 v;
2375
2376 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2377 return lpfifo ? G_LP_COUNT(v) : G_HP_COUNT(v);
2378 }
2379 EXPORT_SYMBOL(cxgb4_dbfifo_count);
2380
2381 /**
2382 * cxgb4_port_viid - get the VI id of a port
2383 * @dev: the net device for the port
2384 *
2385 * Return the VI id of the given port.
2386 */
2387 unsigned int cxgb4_port_viid(const struct net_device *dev)
2388 {
2389 return netdev2pinfo(dev)->viid;
2390 }
2391 EXPORT_SYMBOL(cxgb4_port_viid);
2392
2393 /**
2394 * cxgb4_port_idx - get the index of a port
2395 * @dev: the net device for the port
2396 *
2397 * Return the index of the given port.
2398 */
2399 unsigned int cxgb4_port_idx(const struct net_device *dev)
2400 {
2401 return netdev2pinfo(dev)->port_id;
2402 }
2403 EXPORT_SYMBOL(cxgb4_port_idx);
2404
2405 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2406 struct tp_tcp_stats *v6)
2407 {
2408 struct adapter *adap = pci_get_drvdata(pdev);
2409
2410 spin_lock(&adap->stats_lock);
2411 t4_tp_get_tcp_stats(adap, v4, v6);
2412 spin_unlock(&adap->stats_lock);
2413 }
2414 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2415
2416 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2417 const unsigned int *pgsz_order)
2418 {
2419 struct adapter *adap = netdev2adap(dev);
2420
2421 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2422 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2423 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2424 HPZ3(pgsz_order[3]));
2425 }
2426 EXPORT_SYMBOL(cxgb4_iscsi_init);
2427
2428 int cxgb4_flush_eq_cache(struct net_device *dev)
2429 {
2430 struct adapter *adap = netdev2adap(dev);
2431 int ret;
2432
2433 ret = t4_fwaddrspace_write(adap, adap->mbox,
2434 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
2435 return ret;
2436 }
2437 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
2438
2439 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
2440 {
2441 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
2442 __be64 indices;
2443 int ret;
2444
2445 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
2446 if (!ret) {
2447 indices = be64_to_cpu(indices);
2448 *cidx = (indices >> 25) & 0xffff;
2449 *pidx = (indices >> 9) & 0xffff;
2450 }
2451 return ret;
2452 }
2453
2454 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
2455 u16 size)
2456 {
2457 struct adapter *adap = netdev2adap(dev);
2458 u16 hw_pidx, hw_cidx;
2459 int ret;
2460
2461 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
2462 if (ret)
2463 goto out;
2464
2465 if (pidx != hw_pidx) {
2466 u16 delta;
2467
2468 if (pidx >= hw_pidx)
2469 delta = pidx - hw_pidx;
2470 else
2471 delta = size - hw_pidx + pidx;
2472 wmb();
2473 t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL),
2474 V_QID(qid) | V_PIDX(delta));
2475 }
2476 out:
2477 return ret;
2478 }
2479 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
2480
2481 static struct pci_driver cxgb4_driver;
2482
2483 static void check_neigh_update(struct neighbour *neigh)
2484 {
2485 const struct device *parent;
2486 const struct net_device *netdev = neigh->dev;
2487
2488 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2489 netdev = vlan_dev_real_dev(netdev);
2490 parent = netdev->dev.parent;
2491 if (parent && parent->driver == &cxgb4_driver.driver)
2492 t4_l2t_update(dev_get_drvdata(parent), neigh);
2493 }
2494
2495 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2496 void *data)
2497 {
2498 switch (event) {
2499 case NETEVENT_NEIGH_UPDATE:
2500 check_neigh_update(data);
2501 break;
2502 case NETEVENT_REDIRECT:
2503 default:
2504 break;
2505 }
2506 return 0;
2507 }
2508
2509 static bool netevent_registered;
2510 static struct notifier_block cxgb4_netevent_nb = {
2511 .notifier_call = netevent_cb
2512 };
2513
2514 static void drain_db_fifo(struct adapter *adap, int usecs)
2515 {
2516 u32 v;
2517
2518 do {
2519 set_current_state(TASK_UNINTERRUPTIBLE);
2520 schedule_timeout(usecs_to_jiffies(usecs));
2521 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2522 if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0)
2523 break;
2524 } while (1);
2525 }
2526
2527 static void disable_txq_db(struct sge_txq *q)
2528 {
2529 spin_lock_irq(&q->db_lock);
2530 q->db_disabled = 1;
2531 spin_unlock_irq(&q->db_lock);
2532 }
2533
2534 static void enable_txq_db(struct sge_txq *q)
2535 {
2536 spin_lock_irq(&q->db_lock);
2537 q->db_disabled = 0;
2538 spin_unlock_irq(&q->db_lock);
2539 }
2540
2541 static void disable_dbs(struct adapter *adap)
2542 {
2543 int i;
2544
2545 for_each_ethrxq(&adap->sge, i)
2546 disable_txq_db(&adap->sge.ethtxq[i].q);
2547 for_each_ofldrxq(&adap->sge, i)
2548 disable_txq_db(&adap->sge.ofldtxq[i].q);
2549 for_each_port(adap, i)
2550 disable_txq_db(&adap->sge.ctrlq[i].q);
2551 }
2552
2553 static void enable_dbs(struct adapter *adap)
2554 {
2555 int i;
2556
2557 for_each_ethrxq(&adap->sge, i)
2558 enable_txq_db(&adap->sge.ethtxq[i].q);
2559 for_each_ofldrxq(&adap->sge, i)
2560 enable_txq_db(&adap->sge.ofldtxq[i].q);
2561 for_each_port(adap, i)
2562 enable_txq_db(&adap->sge.ctrlq[i].q);
2563 }
2564
2565 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
2566 {
2567 u16 hw_pidx, hw_cidx;
2568 int ret;
2569
2570 spin_lock_bh(&q->db_lock);
2571 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
2572 if (ret)
2573 goto out;
2574 if (q->db_pidx != hw_pidx) {
2575 u16 delta;
2576
2577 if (q->db_pidx >= hw_pidx)
2578 delta = q->db_pidx - hw_pidx;
2579 else
2580 delta = q->size - hw_pidx + q->db_pidx;
2581 wmb();
2582 t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL),
2583 V_QID(q->cntxt_id) | V_PIDX(delta));
2584 }
2585 out:
2586 q->db_disabled = 0;
2587 spin_unlock_bh(&q->db_lock);
2588 if (ret)
2589 CH_WARN(adap, "DB drop recovery failed.\n");
2590 }
2591 static void recover_all_queues(struct adapter *adap)
2592 {
2593 int i;
2594
2595 for_each_ethrxq(&adap->sge, i)
2596 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2597 for_each_ofldrxq(&adap->sge, i)
2598 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
2599 for_each_port(adap, i)
2600 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
2601 }
2602
2603 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
2604 {
2605 mutex_lock(&uld_mutex);
2606 if (adap->uld_handle[CXGB4_ULD_RDMA])
2607 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
2608 cmd);
2609 mutex_unlock(&uld_mutex);
2610 }
2611
2612 static void process_db_full(struct work_struct *work)
2613 {
2614 struct adapter *adap;
2615
2616 adap = container_of(work, struct adapter, db_full_task);
2617
2618 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2619 drain_db_fifo(adap, dbfifo_drain_delay);
2620 t4_set_reg_field(adap, A_SGE_INT_ENABLE3,
2621 F_DBFIFO_HP_INT | F_DBFIFO_LP_INT,
2622 F_DBFIFO_HP_INT | F_DBFIFO_LP_INT);
2623 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2624 }
2625
2626 static void process_db_drop(struct work_struct *work)
2627 {
2628 struct adapter *adap;
2629
2630 adap = container_of(work, struct adapter, db_drop_task);
2631
2632 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
2633 disable_dbs(adap);
2634 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
2635 drain_db_fifo(adap, 1);
2636 recover_all_queues(adap);
2637 enable_dbs(adap);
2638 }
2639
2640 void t4_db_full(struct adapter *adap)
2641 {
2642 t4_set_reg_field(adap, A_SGE_INT_ENABLE3,
2643 F_DBFIFO_HP_INT | F_DBFIFO_LP_INT, 0);
2644 queue_work(workq, &adap->db_full_task);
2645 }
2646
2647 void t4_db_dropped(struct adapter *adap)
2648 {
2649 queue_work(workq, &adap->db_drop_task);
2650 }
2651
2652 static void uld_attach(struct adapter *adap, unsigned int uld)
2653 {
2654 void *handle;
2655 struct cxgb4_lld_info lli;
2656
2657 lli.pdev = adap->pdev;
2658 lli.l2t = adap->l2t;
2659 lli.tids = &adap->tids;
2660 lli.ports = adap->port;
2661 lli.vr = &adap->vres;
2662 lli.mtus = adap->params.mtus;
2663 if (uld == CXGB4_ULD_RDMA) {
2664 lli.rxq_ids = adap->sge.rdma_rxq;
2665 lli.nrxq = adap->sge.rdmaqs;
2666 } else if (uld == CXGB4_ULD_ISCSI) {
2667 lli.rxq_ids = adap->sge.ofld_rxq;
2668 lli.nrxq = adap->sge.ofldqsets;
2669 }
2670 lli.ntxq = adap->sge.ofldqsets;
2671 lli.nchan = adap->params.nports;
2672 lli.nports = adap->params.nports;
2673 lli.wr_cred = adap->params.ofldq_wr_cred;
2674 lli.adapter_type = adap->params.rev;
2675 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2676 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2677 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2678 (adap->fn * 4));
2679 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2680 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2681 (adap->fn * 4));
2682 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2683 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2684 lli.fw_vers = adap->params.fw_vers;
2685 lli.dbfifo_int_thresh = dbfifo_int_thresh;
2686
2687 handle = ulds[uld].add(&lli);
2688 if (IS_ERR(handle)) {
2689 dev_warn(adap->pdev_dev,
2690 "could not attach to the %s driver, error %ld\n",
2691 uld_str[uld], PTR_ERR(handle));
2692 return;
2693 }
2694
2695 adap->uld_handle[uld] = handle;
2696
2697 if (!netevent_registered) {
2698 register_netevent_notifier(&cxgb4_netevent_nb);
2699 netevent_registered = true;
2700 }
2701
2702 if (adap->flags & FULL_INIT_DONE)
2703 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2704 }
2705
2706 static void attach_ulds(struct adapter *adap)
2707 {
2708 unsigned int i;
2709
2710 mutex_lock(&uld_mutex);
2711 list_add_tail(&adap->list_node, &adapter_list);
2712 for (i = 0; i < CXGB4_ULD_MAX; i++)
2713 if (ulds[i].add)
2714 uld_attach(adap, i);
2715 mutex_unlock(&uld_mutex);
2716 }
2717
2718 static void detach_ulds(struct adapter *adap)
2719 {
2720 unsigned int i;
2721
2722 mutex_lock(&uld_mutex);
2723 list_del(&adap->list_node);
2724 for (i = 0; i < CXGB4_ULD_MAX; i++)
2725 if (adap->uld_handle[i]) {
2726 ulds[i].state_change(adap->uld_handle[i],
2727 CXGB4_STATE_DETACH);
2728 adap->uld_handle[i] = NULL;
2729 }
2730 if (netevent_registered && list_empty(&adapter_list)) {
2731 unregister_netevent_notifier(&cxgb4_netevent_nb);
2732 netevent_registered = false;
2733 }
2734 mutex_unlock(&uld_mutex);
2735 }
2736
2737 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2738 {
2739 unsigned int i;
2740
2741 mutex_lock(&uld_mutex);
2742 for (i = 0; i < CXGB4_ULD_MAX; i++)
2743 if (adap->uld_handle[i])
2744 ulds[i].state_change(adap->uld_handle[i], new_state);
2745 mutex_unlock(&uld_mutex);
2746 }
2747
2748 /**
2749 * cxgb4_register_uld - register an upper-layer driver
2750 * @type: the ULD type
2751 * @p: the ULD methods
2752 *
2753 * Registers an upper-layer driver with this driver and notifies the ULD
2754 * about any presently available devices that support its type. Returns
2755 * %-EBUSY if a ULD of the same type is already registered.
2756 */
2757 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2758 {
2759 int ret = 0;
2760 struct adapter *adap;
2761
2762 if (type >= CXGB4_ULD_MAX)
2763 return -EINVAL;
2764 mutex_lock(&uld_mutex);
2765 if (ulds[type].add) {
2766 ret = -EBUSY;
2767 goto out;
2768 }
2769 ulds[type] = *p;
2770 list_for_each_entry(adap, &adapter_list, list_node)
2771 uld_attach(adap, type);
2772 out: mutex_unlock(&uld_mutex);
2773 return ret;
2774 }
2775 EXPORT_SYMBOL(cxgb4_register_uld);
2776
2777 /**
2778 * cxgb4_unregister_uld - unregister an upper-layer driver
2779 * @type: the ULD type
2780 *
2781 * Unregisters an existing upper-layer driver.
2782 */
2783 int cxgb4_unregister_uld(enum cxgb4_uld type)
2784 {
2785 struct adapter *adap;
2786
2787 if (type >= CXGB4_ULD_MAX)
2788 return -EINVAL;
2789 mutex_lock(&uld_mutex);
2790 list_for_each_entry(adap, &adapter_list, list_node)
2791 adap->uld_handle[type] = NULL;
2792 ulds[type].add = NULL;
2793 mutex_unlock(&uld_mutex);
2794 return 0;
2795 }
2796 EXPORT_SYMBOL(cxgb4_unregister_uld);
2797
2798 /**
2799 * cxgb_up - enable the adapter
2800 * @adap: adapter being enabled
2801 *
2802 * Called when the first port is enabled, this function performs the
2803 * actions necessary to make an adapter operational, such as completing
2804 * the initialization of HW modules, and enabling interrupts.
2805 *
2806 * Must be called with the rtnl lock held.
2807 */
2808 static int cxgb_up(struct adapter *adap)
2809 {
2810 int err;
2811
2812 err = setup_sge_queues(adap);
2813 if (err)
2814 goto out;
2815 err = setup_rss(adap);
2816 if (err)
2817 goto freeq;
2818
2819 if (adap->flags & USING_MSIX) {
2820 name_msix_vecs(adap);
2821 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2822 adap->msix_info[0].desc, adap);
2823 if (err)
2824 goto irq_err;
2825
2826 err = request_msix_queue_irqs(adap);
2827 if (err) {
2828 free_irq(adap->msix_info[0].vec, adap);
2829 goto irq_err;
2830 }
2831 } else {
2832 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2833 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2834 adap->port[0]->name, adap);
2835 if (err)
2836 goto irq_err;
2837 }
2838 enable_rx(adap);
2839 t4_sge_start(adap);
2840 t4_intr_enable(adap);
2841 adap->flags |= FULL_INIT_DONE;
2842 notify_ulds(adap, CXGB4_STATE_UP);
2843 out:
2844 return err;
2845 irq_err:
2846 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2847 freeq:
2848 t4_free_sge_resources(adap);
2849 goto out;
2850 }
2851
2852 static void cxgb_down(struct adapter *adapter)
2853 {
2854 t4_intr_disable(adapter);
2855 cancel_work_sync(&adapter->tid_release_task);
2856 cancel_work_sync(&adapter->db_full_task);
2857 cancel_work_sync(&adapter->db_drop_task);
2858 adapter->tid_release_task_busy = false;
2859 adapter->tid_release_head = NULL;
2860
2861 if (adapter->flags & USING_MSIX) {
2862 free_msix_queue_irqs(adapter);
2863 free_irq(adapter->msix_info[0].vec, adapter);
2864 } else
2865 free_irq(adapter->pdev->irq, adapter);
2866 quiesce_rx(adapter);
2867 t4_sge_stop(adapter);
2868 t4_free_sge_resources(adapter);
2869 adapter->flags &= ~FULL_INIT_DONE;
2870 }
2871
2872 /*
2873 * net_device operations
2874 */
2875 static int cxgb_open(struct net_device *dev)
2876 {
2877 int err;
2878 struct port_info *pi = netdev_priv(dev);
2879 struct adapter *adapter = pi->adapter;
2880
2881 netif_carrier_off(dev);
2882
2883 if (!(adapter->flags & FULL_INIT_DONE)) {
2884 err = cxgb_up(adapter);
2885 if (err < 0)
2886 return err;
2887 }
2888
2889 err = link_start(dev);
2890 if (!err)
2891 netif_tx_start_all_queues(dev);
2892 return err;
2893 }
2894
2895 static int cxgb_close(struct net_device *dev)
2896 {
2897 struct port_info *pi = netdev_priv(dev);
2898 struct adapter *adapter = pi->adapter;
2899
2900 netif_tx_stop_all_queues(dev);
2901 netif_carrier_off(dev);
2902 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2903 }
2904
2905 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2906 struct rtnl_link_stats64 *ns)
2907 {
2908 struct port_stats stats;
2909 struct port_info *p = netdev_priv(dev);
2910 struct adapter *adapter = p->adapter;
2911
2912 spin_lock(&adapter->stats_lock);
2913 t4_get_port_stats(adapter, p->tx_chan, &stats);
2914 spin_unlock(&adapter->stats_lock);
2915
2916 ns->tx_bytes = stats.tx_octets;
2917 ns->tx_packets = stats.tx_frames;
2918 ns->rx_bytes = stats.rx_octets;
2919 ns->rx_packets = stats.rx_frames;
2920 ns->multicast = stats.rx_mcast_frames;
2921
2922 /* detailed rx_errors */
2923 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2924 stats.rx_runt;
2925 ns->rx_over_errors = 0;
2926 ns->rx_crc_errors = stats.rx_fcs_err;
2927 ns->rx_frame_errors = stats.rx_symbol_err;
2928 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2929 stats.rx_ovflow2 + stats.rx_ovflow3 +
2930 stats.rx_trunc0 + stats.rx_trunc1 +
2931 stats.rx_trunc2 + stats.rx_trunc3;
2932 ns->rx_missed_errors = 0;
2933
2934 /* detailed tx_errors */
2935 ns->tx_aborted_errors = 0;
2936 ns->tx_carrier_errors = 0;
2937 ns->tx_fifo_errors = 0;
2938 ns->tx_heartbeat_errors = 0;
2939 ns->tx_window_errors = 0;
2940
2941 ns->tx_errors = stats.tx_error_frames;
2942 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2943 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2944 return ns;
2945 }
2946
2947 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2948 {
2949 unsigned int mbox;
2950 int ret = 0, prtad, devad;
2951 struct port_info *pi = netdev_priv(dev);
2952 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2953
2954 switch (cmd) {
2955 case SIOCGMIIPHY:
2956 if (pi->mdio_addr < 0)
2957 return -EOPNOTSUPP;
2958 data->phy_id = pi->mdio_addr;
2959 break;
2960 case SIOCGMIIREG:
2961 case SIOCSMIIREG:
2962 if (mdio_phy_id_is_c45(data->phy_id)) {
2963 prtad = mdio_phy_id_prtad(data->phy_id);
2964 devad = mdio_phy_id_devad(data->phy_id);
2965 } else if (data->phy_id < 32) {
2966 prtad = data->phy_id;
2967 devad = 0;
2968 data->reg_num &= 0x1f;
2969 } else
2970 return -EINVAL;
2971
2972 mbox = pi->adapter->fn;
2973 if (cmd == SIOCGMIIREG)
2974 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2975 data->reg_num, &data->val_out);
2976 else
2977 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2978 data->reg_num, data->val_in);
2979 break;
2980 default:
2981 return -EOPNOTSUPP;
2982 }
2983 return ret;
2984 }
2985
2986 static void cxgb_set_rxmode(struct net_device *dev)
2987 {
2988 /* unfortunately we can't return errors to the stack */
2989 set_rxmode(dev, -1, false);
2990 }
2991
2992 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2993 {
2994 int ret;
2995 struct port_info *pi = netdev_priv(dev);
2996
2997 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2998 return -EINVAL;
2999 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
3000 -1, -1, -1, true);
3001 if (!ret)
3002 dev->mtu = new_mtu;
3003 return ret;
3004 }
3005
3006 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
3007 {
3008 int ret;
3009 struct sockaddr *addr = p;
3010 struct port_info *pi = netdev_priv(dev);
3011
3012 if (!is_valid_ether_addr(addr->sa_data))
3013 return -EADDRNOTAVAIL;
3014
3015 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
3016 pi->xact_addr_filt, addr->sa_data, true, true);
3017 if (ret < 0)
3018 return ret;
3019
3020 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3021 pi->xact_addr_filt = ret;
3022 return 0;
3023 }
3024
3025 #ifdef CONFIG_NET_POLL_CONTROLLER
3026 static void cxgb_netpoll(struct net_device *dev)
3027 {
3028 struct port_info *pi = netdev_priv(dev);
3029 struct adapter *adap = pi->adapter;
3030
3031 if (adap->flags & USING_MSIX) {
3032 int i;
3033 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
3034
3035 for (i = pi->nqsets; i; i--, rx++)
3036 t4_sge_intr_msix(0, &rx->rspq);
3037 } else
3038 t4_intr_handler(adap)(0, adap);
3039 }
3040 #endif
3041
3042 static const struct net_device_ops cxgb4_netdev_ops = {
3043 .ndo_open = cxgb_open,
3044 .ndo_stop = cxgb_close,
3045 .ndo_start_xmit = t4_eth_xmit,
3046 .ndo_get_stats64 = cxgb_get_stats,
3047 .ndo_set_rx_mode = cxgb_set_rxmode,
3048 .ndo_set_mac_address = cxgb_set_mac_addr,
3049 .ndo_set_features = cxgb_set_features,
3050 .ndo_validate_addr = eth_validate_addr,
3051 .ndo_do_ioctl = cxgb_ioctl,
3052 .ndo_change_mtu = cxgb_change_mtu,
3053 #ifdef CONFIG_NET_POLL_CONTROLLER
3054 .ndo_poll_controller = cxgb_netpoll,
3055 #endif
3056 };
3057
3058 void t4_fatal_err(struct adapter *adap)
3059 {
3060 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
3061 t4_intr_disable(adap);
3062 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
3063 }
3064
3065 static void setup_memwin(struct adapter *adap)
3066 {
3067 u32 bar0;
3068
3069 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
3070 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
3071 (bar0 + MEMWIN0_BASE) | BIR(0) |
3072 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
3073 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
3074 (bar0 + MEMWIN1_BASE) | BIR(0) |
3075 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
3076 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
3077 (bar0 + MEMWIN2_BASE) | BIR(0) |
3078 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
3079 if (adap->vres.ocq.size) {
3080 unsigned int start, sz_kb;
3081
3082 start = pci_resource_start(adap->pdev, 2) +
3083 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
3084 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
3085 t4_write_reg(adap,
3086 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
3087 start | BIR(1) | WINDOW(ilog2(sz_kb)));
3088 t4_write_reg(adap,
3089 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
3090 adap->vres.ocq.start);
3091 t4_read_reg(adap,
3092 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
3093 }
3094 }
3095
3096 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
3097 {
3098 u32 v;
3099 int ret;
3100
3101 /* get device capabilities */
3102 memset(c, 0, sizeof(*c));
3103 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3104 FW_CMD_REQUEST | FW_CMD_READ);
3105 c->retval_len16 = htonl(FW_LEN16(*c));
3106 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
3107 if (ret < 0)
3108 return ret;
3109
3110 /* select capabilities we'll be using */
3111 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3112 if (!vf_acls)
3113 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3114 else
3115 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3116 } else if (vf_acls) {
3117 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
3118 return ret;
3119 }
3120 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3121 FW_CMD_REQUEST | FW_CMD_WRITE);
3122 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
3123 if (ret < 0)
3124 return ret;
3125
3126 ret = t4_config_glbl_rss(adap, adap->fn,
3127 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3128 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3129 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
3130 if (ret < 0)
3131 return ret;
3132
3133 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
3134 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
3135 if (ret < 0)
3136 return ret;
3137
3138 t4_sge_init(adap);
3139
3140 /* tweak some settings */
3141 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
3142 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
3143 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
3144 v = t4_read_reg(adap, TP_PIO_DATA);
3145 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
3146
3147 /* get basic stuff going */
3148 return t4_early_init(adap, adap->fn);
3149 }
3150
3151 /*
3152 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
3153 */
3154 #define MAX_ATIDS 8192U
3155
3156 /*
3157 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3158 */
3159 static int adap_init0(struct adapter *adap)
3160 {
3161 int ret;
3162 u32 v, port_vec;
3163 enum dev_state state;
3164 u32 params[7], val[7];
3165 struct fw_caps_config_cmd c;
3166
3167 ret = t4_check_fw_version(adap);
3168 if (ret == -EINVAL || ret > 0) {
3169 if (upgrade_fw(adap) >= 0) /* recache FW version */
3170 ret = t4_check_fw_version(adap);
3171 }
3172 if (ret < 0)
3173 return ret;
3174
3175 /* contact FW, request master */
3176 ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
3177 if (ret < 0) {
3178 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3179 ret);
3180 return ret;
3181 }
3182
3183 /* reset device */
3184 ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
3185 if (ret < 0)
3186 goto bye;
3187
3188 for (v = 0; v < SGE_NTIMERS - 1; v++)
3189 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3190 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3191 adap->sge.counter_val[0] = 1;
3192 for (v = 1; v < SGE_NCOUNTERS; v++)
3193 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3194 THRESHOLD_3_MASK);
3195 #define FW_PARAM_DEV(param) \
3196 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3197 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3198
3199 params[0] = FW_PARAM_DEV(CCLK);
3200 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
3201 if (ret < 0)
3202 goto bye;
3203 adap->params.vpd.cclk = val[0];
3204
3205 ret = adap_init1(adap, &c);
3206 if (ret < 0)
3207 goto bye;
3208
3209 #define FW_PARAM_PFVF(param) \
3210 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3211 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3212 FW_PARAMS_PARAM_Y(adap->fn))
3213
3214 params[0] = FW_PARAM_DEV(PORTVEC);
3215 params[1] = FW_PARAM_PFVF(L2T_START);
3216 params[2] = FW_PARAM_PFVF(L2T_END);
3217 params[3] = FW_PARAM_PFVF(FILTER_START);
3218 params[4] = FW_PARAM_PFVF(FILTER_END);
3219 params[5] = FW_PARAM_PFVF(IQFLINT_START);
3220 params[6] = FW_PARAM_PFVF(EQ_START);
3221 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
3222 if (ret < 0)
3223 goto bye;
3224 port_vec = val[0];
3225 adap->tids.ftid_base = val[3];
3226 adap->tids.nftids = val[4] - val[3] + 1;
3227 adap->sge.ingr_start = val[5];
3228 adap->sge.egr_start = val[6];
3229
3230 if (c.ofldcaps) {
3231 /* query offload-related parameters */
3232 params[0] = FW_PARAM_DEV(NTID);
3233 params[1] = FW_PARAM_PFVF(SERVER_START);
3234 params[2] = FW_PARAM_PFVF(SERVER_END);
3235 params[3] = FW_PARAM_PFVF(TDDP_START);
3236 params[4] = FW_PARAM_PFVF(TDDP_END);
3237 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3238 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3239 val);
3240 if (ret < 0)
3241 goto bye;
3242 adap->tids.ntids = val[0];
3243 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3244 adap->tids.stid_base = val[1];
3245 adap->tids.nstids = val[2] - val[1] + 1;
3246 adap->vres.ddp.start = val[3];
3247 adap->vres.ddp.size = val[4] - val[3] + 1;
3248 adap->params.ofldq_wr_cred = val[5];
3249 adap->params.offload = 1;
3250 }
3251 if (c.rdmacaps) {
3252 params[0] = FW_PARAM_PFVF(STAG_START);
3253 params[1] = FW_PARAM_PFVF(STAG_END);
3254 params[2] = FW_PARAM_PFVF(RQ_START);
3255 params[3] = FW_PARAM_PFVF(RQ_END);
3256 params[4] = FW_PARAM_PFVF(PBL_START);
3257 params[5] = FW_PARAM_PFVF(PBL_END);
3258 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3259 val);
3260 if (ret < 0)
3261 goto bye;
3262 adap->vres.stag.start = val[0];
3263 adap->vres.stag.size = val[1] - val[0] + 1;
3264 adap->vres.rq.start = val[2];
3265 adap->vres.rq.size = val[3] - val[2] + 1;
3266 adap->vres.pbl.start = val[4];
3267 adap->vres.pbl.size = val[5] - val[4] + 1;
3268
3269 params[0] = FW_PARAM_PFVF(SQRQ_START);
3270 params[1] = FW_PARAM_PFVF(SQRQ_END);
3271 params[2] = FW_PARAM_PFVF(CQ_START);
3272 params[3] = FW_PARAM_PFVF(CQ_END);
3273 params[4] = FW_PARAM_PFVF(OCQ_START);
3274 params[5] = FW_PARAM_PFVF(OCQ_END);
3275 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3276 val);
3277 if (ret < 0)
3278 goto bye;
3279 adap->vres.qp.start = val[0];
3280 adap->vres.qp.size = val[1] - val[0] + 1;
3281 adap->vres.cq.start = val[2];
3282 adap->vres.cq.size = val[3] - val[2] + 1;
3283 adap->vres.ocq.start = val[4];
3284 adap->vres.ocq.size = val[5] - val[4] + 1;
3285 }
3286 if (c.iscsicaps) {
3287 params[0] = FW_PARAM_PFVF(ISCSI_START);
3288 params[1] = FW_PARAM_PFVF(ISCSI_END);
3289 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3290 val);
3291 if (ret < 0)
3292 goto bye;
3293 adap->vres.iscsi.start = val[0];
3294 adap->vres.iscsi.size = val[1] - val[0] + 1;
3295 }
3296 #undef FW_PARAM_PFVF
3297 #undef FW_PARAM_DEV
3298
3299 adap->params.nports = hweight32(port_vec);
3300 adap->params.portvec = port_vec;
3301 adap->flags |= FW_OK;
3302
3303 /* These are finalized by FW initialization, load their values now */
3304 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3305 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3306 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3307 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3308 adap->params.b_wnd);
3309
3310 #ifdef CONFIG_PCI_IOV
3311 /*
3312 * Provision resource limits for Virtual Functions. We currently
3313 * grant them all the same static resource limits except for the Port
3314 * Access Rights Mask which we're assigning based on the PF. All of
3315 * the static provisioning stuff for both the PF and VF really needs
3316 * to be managed in a persistent manner for each device which the
3317 * firmware controls.
3318 */
3319 {
3320 int pf, vf;
3321
3322 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3323 if (num_vf[pf] <= 0)
3324 continue;
3325
3326 /* VF numbering starts at 1! */
3327 for (vf = 1; vf <= num_vf[pf]; vf++) {
3328 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3329 VFRES_NEQ, VFRES_NETHCTRL,
3330 VFRES_NIQFLINT, VFRES_NIQ,
3331 VFRES_TC, VFRES_NVI,
3332 FW_PFVF_CMD_CMASK_MASK,
3333 pfvfres_pmask(adap, pf, vf),
3334 VFRES_NEXACTF,
3335 VFRES_R_CAPS, VFRES_WX_CAPS);
3336 if (ret < 0)
3337 dev_warn(adap->pdev_dev, "failed to "
3338 "provision pf/vf=%d/%d; "
3339 "err=%d\n", pf, vf, ret);
3340 }
3341 }
3342 }
3343 #endif
3344
3345 setup_memwin(adap);
3346 return 0;
3347
3348 /*
3349 * If a command timed out or failed with EIO FW does not operate within
3350 * its spec or something catastrophic happened to HW/FW, stop issuing
3351 * commands.
3352 */
3353 bye: if (ret != -ETIMEDOUT && ret != -EIO)
3354 t4_fw_bye(adap, adap->fn);
3355 return ret;
3356 }
3357
3358 /* EEH callbacks */
3359
3360 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3361 pci_channel_state_t state)
3362 {
3363 int i;
3364 struct adapter *adap = pci_get_drvdata(pdev);
3365
3366 if (!adap)
3367 goto out;
3368
3369 rtnl_lock();
3370 adap->flags &= ~FW_OK;
3371 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3372 for_each_port(adap, i) {
3373 struct net_device *dev = adap->port[i];
3374
3375 netif_device_detach(dev);
3376 netif_carrier_off(dev);
3377 }
3378 if (adap->flags & FULL_INIT_DONE)
3379 cxgb_down(adap);
3380 rtnl_unlock();
3381 pci_disable_device(pdev);
3382 out: return state == pci_channel_io_perm_failure ?
3383 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3384 }
3385
3386 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3387 {
3388 int i, ret;
3389 struct fw_caps_config_cmd c;
3390 struct adapter *adap = pci_get_drvdata(pdev);
3391
3392 if (!adap) {
3393 pci_restore_state(pdev);
3394 pci_save_state(pdev);
3395 return PCI_ERS_RESULT_RECOVERED;
3396 }
3397
3398 if (pci_enable_device(pdev)) {
3399 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3400 return PCI_ERS_RESULT_DISCONNECT;
3401 }
3402
3403 pci_set_master(pdev);
3404 pci_restore_state(pdev);
3405 pci_save_state(pdev);
3406 pci_cleanup_aer_uncorrect_error_status(pdev);
3407
3408 if (t4_wait_dev_ready(adap) < 0)
3409 return PCI_ERS_RESULT_DISCONNECT;
3410 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3411 return PCI_ERS_RESULT_DISCONNECT;
3412 adap->flags |= FW_OK;
3413 if (adap_init1(adap, &c))
3414 return PCI_ERS_RESULT_DISCONNECT;
3415
3416 for_each_port(adap, i) {
3417 struct port_info *p = adap2pinfo(adap, i);
3418
3419 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3420 NULL, NULL);
3421 if (ret < 0)
3422 return PCI_ERS_RESULT_DISCONNECT;
3423 p->viid = ret;
3424 p->xact_addr_filt = -1;
3425 }
3426
3427 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3428 adap->params.b_wnd);
3429 setup_memwin(adap);
3430 if (cxgb_up(adap))
3431 return PCI_ERS_RESULT_DISCONNECT;
3432 return PCI_ERS_RESULT_RECOVERED;
3433 }
3434
3435 static void eeh_resume(struct pci_dev *pdev)
3436 {
3437 int i;
3438 struct adapter *adap = pci_get_drvdata(pdev);
3439
3440 if (!adap)
3441 return;
3442
3443 rtnl_lock();
3444 for_each_port(adap, i) {
3445 struct net_device *dev = adap->port[i];
3446
3447 if (netif_running(dev)) {
3448 link_start(dev);
3449 cxgb_set_rxmode(dev);
3450 }
3451 netif_device_attach(dev);
3452 }
3453 rtnl_unlock();
3454 }
3455
3456 static struct pci_error_handlers cxgb4_eeh = {
3457 .error_detected = eeh_err_detected,
3458 .slot_reset = eeh_slot_reset,
3459 .resume = eeh_resume,
3460 };
3461
3462 static inline bool is_10g_port(const struct link_config *lc)
3463 {
3464 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3465 }
3466
3467 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3468 unsigned int size, unsigned int iqe_size)
3469 {
3470 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3471 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3472 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3473 q->iqe_len = iqe_size;
3474 q->size = size;
3475 }
3476
3477 /*
3478 * Perform default configuration of DMA queues depending on the number and type
3479 * of ports we found and the number of available CPUs. Most settings can be
3480 * modified by the admin prior to actual use.
3481 */
3482 static void __devinit cfg_queues(struct adapter *adap)
3483 {
3484 struct sge *s = &adap->sge;
3485 int i, q10g = 0, n10g = 0, qidx = 0;
3486
3487 for_each_port(adap, i)
3488 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3489
3490 /*
3491 * We default to 1 queue per non-10G port and up to # of cores queues
3492 * per 10G port.
3493 */
3494 if (n10g)
3495 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3496 if (q10g > netif_get_num_default_rss_queues())
3497 q10g = netif_get_num_default_rss_queues();
3498
3499 for_each_port(adap, i) {
3500 struct port_info *pi = adap2pinfo(adap, i);
3501
3502 pi->first_qset = qidx;
3503 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3504 qidx += pi->nqsets;
3505 }
3506
3507 s->ethqsets = qidx;
3508 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3509
3510 if (is_offload(adap)) {
3511 /*
3512 * For offload we use 1 queue/channel if all ports are up to 1G,
3513 * otherwise we divide all available queues amongst the channels
3514 * capped by the number of available cores.
3515 */
3516 if (n10g) {
3517 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3518 num_online_cpus());
3519 s->ofldqsets = roundup(i, adap->params.nports);
3520 } else
3521 s->ofldqsets = adap->params.nports;
3522 /* For RDMA one Rx queue per channel suffices */
3523 s->rdmaqs = adap->params.nports;
3524 }
3525
3526 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3527 struct sge_eth_rxq *r = &s->ethrxq[i];
3528
3529 init_rspq(&r->rspq, 0, 0, 1024, 64);
3530 r->fl.size = 72;
3531 }
3532
3533 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3534 s->ethtxq[i].q.size = 1024;
3535
3536 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3537 s->ctrlq[i].q.size = 512;
3538
3539 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3540 s->ofldtxq[i].q.size = 1024;
3541
3542 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3543 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3544
3545 init_rspq(&r->rspq, 0, 0, 1024, 64);
3546 r->rspq.uld = CXGB4_ULD_ISCSI;
3547 r->fl.size = 72;
3548 }
3549
3550 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3551 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3552
3553 init_rspq(&r->rspq, 0, 0, 511, 64);
3554 r->rspq.uld = CXGB4_ULD_RDMA;
3555 r->fl.size = 72;
3556 }
3557
3558 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3559 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3560 }
3561
3562 /*
3563 * Reduce the number of Ethernet queues across all ports to at most n.
3564 * n provides at least one queue per port.
3565 */
3566 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3567 {
3568 int i;
3569 struct port_info *pi;
3570
3571 while (n < adap->sge.ethqsets)
3572 for_each_port(adap, i) {
3573 pi = adap2pinfo(adap, i);
3574 if (pi->nqsets > 1) {
3575 pi->nqsets--;
3576 adap->sge.ethqsets--;
3577 if (adap->sge.ethqsets <= n)
3578 break;
3579 }
3580 }
3581
3582 n = 0;
3583 for_each_port(adap, i) {
3584 pi = adap2pinfo(adap, i);
3585 pi->first_qset = n;
3586 n += pi->nqsets;
3587 }
3588 }
3589
3590 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3591 #define EXTRA_VECS 2
3592
3593 static int __devinit enable_msix(struct adapter *adap)
3594 {
3595 int ofld_need = 0;
3596 int i, err, want, need;
3597 struct sge *s = &adap->sge;
3598 unsigned int nchan = adap->params.nports;
3599 struct msix_entry entries[MAX_INGQ + 1];
3600
3601 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3602 entries[i].entry = i;
3603
3604 want = s->max_ethqsets + EXTRA_VECS;
3605 if (is_offload(adap)) {
3606 want += s->rdmaqs + s->ofldqsets;
3607 /* need nchan for each possible ULD */
3608 ofld_need = 2 * nchan;
3609 }
3610 need = adap->params.nports + EXTRA_VECS + ofld_need;
3611
3612 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3613 want = err;
3614
3615 if (!err) {
3616 /*
3617 * Distribute available vectors to the various queue groups.
3618 * Every group gets its minimum requirement and NIC gets top
3619 * priority for leftovers.
3620 */
3621 i = want - EXTRA_VECS - ofld_need;
3622 if (i < s->max_ethqsets) {
3623 s->max_ethqsets = i;
3624 if (i < s->ethqsets)
3625 reduce_ethqs(adap, i);
3626 }
3627 if (is_offload(adap)) {
3628 i = want - EXTRA_VECS - s->max_ethqsets;
3629 i -= ofld_need - nchan;
3630 s->ofldqsets = (i / nchan) * nchan; /* round down */
3631 }
3632 for (i = 0; i < want; ++i)
3633 adap->msix_info[i].vec = entries[i].vector;
3634 } else if (err > 0)
3635 dev_info(adap->pdev_dev,
3636 "only %d MSI-X vectors left, not using MSI-X\n", err);
3637 return err;
3638 }
3639
3640 #undef EXTRA_VECS
3641
3642 static int __devinit init_rss(struct adapter *adap)
3643 {
3644 unsigned int i, j;
3645
3646 for_each_port(adap, i) {
3647 struct port_info *pi = adap2pinfo(adap, i);
3648
3649 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3650 if (!pi->rss)
3651 return -ENOMEM;
3652 for (j = 0; j < pi->rss_size; j++)
3653 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
3654 }
3655 return 0;
3656 }
3657
3658 static void __devinit print_port_info(const struct net_device *dev)
3659 {
3660 static const char *base[] = {
3661 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3662 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
3663 };
3664
3665 char buf[80];
3666 char *bufp = buf;
3667 const char *spd = "";
3668 const struct port_info *pi = netdev_priv(dev);
3669 const struct adapter *adap = pi->adapter;
3670
3671 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3672 spd = " 2.5 GT/s";
3673 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3674 spd = " 5 GT/s";
3675
3676 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3677 bufp += sprintf(bufp, "100/");
3678 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3679 bufp += sprintf(bufp, "1000/");
3680 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3681 bufp += sprintf(bufp, "10G/");
3682 if (bufp != buf)
3683 --bufp;
3684 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3685
3686 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3687 adap->params.vpd.id, adap->params.rev, buf,
3688 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
3689 (adap->flags & USING_MSIX) ? " MSI-X" :
3690 (adap->flags & USING_MSI) ? " MSI" : "");
3691 netdev_info(dev, "S/N: %s, E/C: %s\n",
3692 adap->params.vpd.sn, adap->params.vpd.ec);
3693 }
3694
3695 static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
3696 {
3697 u16 v;
3698 int pos;
3699
3700 pos = pci_pcie_cap(dev);
3701 if (pos > 0) {
3702 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
3703 v |= PCI_EXP_DEVCTL_RELAX_EN;
3704 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
3705 }
3706 }
3707
3708 /*
3709 * Free the following resources:
3710 * - memory used for tables
3711 * - MSI/MSI-X
3712 * - net devices
3713 * - resources FW is holding for us
3714 */
3715 static void free_some_resources(struct adapter *adapter)
3716 {
3717 unsigned int i;
3718
3719 t4_free_mem(adapter->l2t);
3720 t4_free_mem(adapter->tids.tid_tab);
3721 disable_msi(adapter);
3722
3723 for_each_port(adapter, i)
3724 if (adapter->port[i]) {
3725 kfree(adap2pinfo(adapter, i)->rss);
3726 free_netdev(adapter->port[i]);
3727 }
3728 if (adapter->flags & FW_OK)
3729 t4_fw_bye(adapter, adapter->fn);
3730 }
3731
3732 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3733 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3734 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3735
3736 static int __devinit init_one(struct pci_dev *pdev,
3737 const struct pci_device_id *ent)
3738 {
3739 int func, i, err;
3740 struct port_info *pi;
3741 bool highdma = false;
3742 struct adapter *adapter = NULL;
3743
3744 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3745
3746 err = pci_request_regions(pdev, KBUILD_MODNAME);
3747 if (err) {
3748 /* Just info, some other driver may have claimed the device. */
3749 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3750 return err;
3751 }
3752
3753 /* We control everything through one PF */
3754 func = PCI_FUNC(pdev->devfn);
3755 if (func != ent->driver_data) {
3756 pci_save_state(pdev); /* to restore SR-IOV later */
3757 goto sriov;
3758 }
3759
3760 err = pci_enable_device(pdev);
3761 if (err) {
3762 dev_err(&pdev->dev, "cannot enable PCI device\n");
3763 goto out_release_regions;
3764 }
3765
3766 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3767 highdma = true;
3768 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3769 if (err) {
3770 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3771 "coherent allocations\n");
3772 goto out_disable_device;
3773 }
3774 } else {
3775 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3776 if (err) {
3777 dev_err(&pdev->dev, "no usable DMA configuration\n");
3778 goto out_disable_device;
3779 }
3780 }
3781
3782 pci_enable_pcie_error_reporting(pdev);
3783 enable_pcie_relaxed_ordering(pdev);
3784 pci_set_master(pdev);
3785 pci_save_state(pdev);
3786
3787 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3788 if (!adapter) {
3789 err = -ENOMEM;
3790 goto out_disable_device;
3791 }
3792
3793 adapter->regs = pci_ioremap_bar(pdev, 0);
3794 if (!adapter->regs) {
3795 dev_err(&pdev->dev, "cannot map device registers\n");
3796 err = -ENOMEM;
3797 goto out_free_adapter;
3798 }
3799
3800 adapter->pdev = pdev;
3801 adapter->pdev_dev = &pdev->dev;
3802 adapter->mbox = func;
3803 adapter->fn = func;
3804 adapter->msg_enable = dflt_msg_enable;
3805 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3806
3807 spin_lock_init(&adapter->stats_lock);
3808 spin_lock_init(&adapter->tid_release_lock);
3809
3810 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3811 INIT_WORK(&adapter->db_full_task, process_db_full);
3812 INIT_WORK(&adapter->db_drop_task, process_db_drop);
3813
3814 err = t4_prep_adapter(adapter);
3815 if (err)
3816 goto out_unmap_bar;
3817 err = adap_init0(adapter);
3818 if (err)
3819 goto out_unmap_bar;
3820
3821 for_each_port(adapter, i) {
3822 struct net_device *netdev;
3823
3824 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3825 MAX_ETH_QSETS);
3826 if (!netdev) {
3827 err = -ENOMEM;
3828 goto out_free_dev;
3829 }
3830
3831 SET_NETDEV_DEV(netdev, &pdev->dev);
3832
3833 adapter->port[i] = netdev;
3834 pi = netdev_priv(netdev);
3835 pi->adapter = adapter;
3836 pi->xact_addr_filt = -1;
3837 pi->port_id = i;
3838 netdev->irq = pdev->irq;
3839
3840 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3841 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3842 NETIF_F_RXCSUM | NETIF_F_RXHASH |
3843 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3844 if (highdma)
3845 netdev->hw_features |= NETIF_F_HIGHDMA;
3846 netdev->features |= netdev->hw_features;
3847 netdev->vlan_features = netdev->features & VLAN_FEAT;
3848
3849 netdev->priv_flags |= IFF_UNICAST_FLT;
3850
3851 netdev->netdev_ops = &cxgb4_netdev_ops;
3852 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3853 }
3854
3855 pci_set_drvdata(pdev, adapter);
3856
3857 if (adapter->flags & FW_OK) {
3858 err = t4_port_init(adapter, func, func, 0);
3859 if (err)
3860 goto out_free_dev;
3861 }
3862
3863 /*
3864 * Configure queues and allocate tables now, they can be needed as
3865 * soon as the first register_netdev completes.
3866 */
3867 cfg_queues(adapter);
3868
3869 adapter->l2t = t4_init_l2t();
3870 if (!adapter->l2t) {
3871 /* We tolerate a lack of L2T, giving up some functionality */
3872 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3873 adapter->params.offload = 0;
3874 }
3875
3876 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3877 dev_warn(&pdev->dev, "could not allocate TID table, "
3878 "continuing\n");
3879 adapter->params.offload = 0;
3880 }
3881
3882 /* See what interrupts we'll be using */
3883 if (msi > 1 && enable_msix(adapter) == 0)
3884 adapter->flags |= USING_MSIX;
3885 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3886 adapter->flags |= USING_MSI;
3887
3888 err = init_rss(adapter);
3889 if (err)
3890 goto out_free_dev;
3891
3892 /*
3893 * The card is now ready to go. If any errors occur during device
3894 * registration we do not fail the whole card but rather proceed only
3895 * with the ports we manage to register successfully. However we must
3896 * register at least one net device.
3897 */
3898 for_each_port(adapter, i) {
3899 pi = adap2pinfo(adapter, i);
3900 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
3901 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
3902
3903 err = register_netdev(adapter->port[i]);
3904 if (err)
3905 break;
3906 adapter->chan_map[pi->tx_chan] = i;
3907 print_port_info(adapter->port[i]);
3908 }
3909 if (i == 0) {
3910 dev_err(&pdev->dev, "could not register any net devices\n");
3911 goto out_free_dev;
3912 }
3913 if (err) {
3914 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
3915 err = 0;
3916 }
3917
3918 if (cxgb4_debugfs_root) {
3919 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3920 cxgb4_debugfs_root);
3921 setup_debugfs(adapter);
3922 }
3923
3924 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
3925 pdev->needs_freset = 1;
3926
3927 if (is_offload(adapter))
3928 attach_ulds(adapter);
3929
3930 sriov:
3931 #ifdef CONFIG_PCI_IOV
3932 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3933 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3934 dev_info(&pdev->dev,
3935 "instantiated %u virtual functions\n",
3936 num_vf[func]);
3937 #endif
3938 return 0;
3939
3940 out_free_dev:
3941 free_some_resources(adapter);
3942 out_unmap_bar:
3943 iounmap(adapter->regs);
3944 out_free_adapter:
3945 kfree(adapter);
3946 out_disable_device:
3947 pci_disable_pcie_error_reporting(pdev);
3948 pci_disable_device(pdev);
3949 out_release_regions:
3950 pci_release_regions(pdev);
3951 pci_set_drvdata(pdev, NULL);
3952 return err;
3953 }
3954
3955 static void __devexit remove_one(struct pci_dev *pdev)
3956 {
3957 struct adapter *adapter = pci_get_drvdata(pdev);
3958
3959 pci_disable_sriov(pdev);
3960
3961 if (adapter) {
3962 int i;
3963
3964 if (is_offload(adapter))
3965 detach_ulds(adapter);
3966
3967 for_each_port(adapter, i)
3968 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
3969 unregister_netdev(adapter->port[i]);
3970
3971 if (adapter->debugfs_root)
3972 debugfs_remove_recursive(adapter->debugfs_root);
3973
3974 if (adapter->flags & FULL_INIT_DONE)
3975 cxgb_down(adapter);
3976
3977 free_some_resources(adapter);
3978 iounmap(adapter->regs);
3979 kfree(adapter);
3980 pci_disable_pcie_error_reporting(pdev);
3981 pci_disable_device(pdev);
3982 pci_release_regions(pdev);
3983 pci_set_drvdata(pdev, NULL);
3984 } else
3985 pci_release_regions(pdev);
3986 }
3987
3988 static struct pci_driver cxgb4_driver = {
3989 .name = KBUILD_MODNAME,
3990 .id_table = cxgb4_pci_tbl,
3991 .probe = init_one,
3992 .remove = __devexit_p(remove_one),
3993 .err_handler = &cxgb4_eeh,
3994 };
3995
3996 static int __init cxgb4_init_module(void)
3997 {
3998 int ret;
3999
4000 workq = create_singlethread_workqueue("cxgb4");
4001 if (!workq)
4002 return -ENOMEM;
4003
4004 /* Debugfs support is optional, just warn if this fails */
4005 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
4006 if (!cxgb4_debugfs_root)
4007 pr_warning("could not create debugfs entry, continuing\n");
4008
4009 ret = pci_register_driver(&cxgb4_driver);
4010 if (ret < 0)
4011 debugfs_remove(cxgb4_debugfs_root);
4012 return ret;
4013 }
4014
4015 static void __exit cxgb4_cleanup_module(void)
4016 {
4017 pci_unregister_driver(&cxgb4_driver);
4018 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
4019 flush_workqueue(workq);
4020 destroy_workqueue(workq);
4021 }
4022
4023 module_init(cxgb4_init_module);
4024 module_exit(cxgb4_cleanup_module);
Linus' kernel tree