?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
IMPORT openssh-9.8p1
[dragonfly.git] / sys / dev / netif / mxge / if_mxge.c
blobe037891e2e9c45766074b362ef610258d4d86c48
1 /******************************************************************************
2
3 Copyright (c) 2006-2013, Myricom Inc.
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Myricom Inc, nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
27
28 $FreeBSD: head/sys/dev/mxge/if_mxge.c 254263 2013-08-12 23:30:01Z scottl $
29
30 ***************************************************************************/
31
32 #include "opt_ifpoll.h"
33 #include "opt_inet.h"
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/linker.h>
38 #include <sys/firmware.h>
39 #include <sys/endian.h>
40 #include <sys/in_cksum.h>
41 #include <sys/sockio.h>
42 #include <sys/mbuf.h>
43 #include <sys/malloc.h>
44 #include <sys/kernel.h>
45 #include <sys/module.h>
46 #include <sys/serialize.h>
47 #include <sys/socket.h>
48 #include <sys/sysctl.h>
49
50 #include <net/if.h>
51 #include <net/if_arp.h>
52 #include <net/ifq_var.h>
53 #include <net/if_ringmap.h>
54 #include <net/ethernet.h>
55 #include <net/if_dl.h>
56 #include <net/if_media.h>
57 #include <net/if_poll.h>
58
59 #include <net/bpf.h>
60
61 #include <net/if_types.h>
62 #include <net/vlan/if_vlan_var.h>
63 #include <net/zlib.h>
64 #include <net/toeplitz.h>
65
66 #include <netinet/in_systm.h>
67 #include <netinet/in.h>
68 #include <netinet/ip.h>
69 #include <netinet/tcp.h>
70
71 #include <sys/bus.h>
72 #include <sys/rman.h>
73
74 #include <bus/pci/pcireg.h>
75 #include <bus/pci/pcivar.h>
76 #include <bus/pci/pci_private.h> /* XXX for pci_cfg_restore */
77
78 #include <vm/vm.h> /* for pmap_mapdev() */
79 #include <vm/pmap.h>
80
81 #if defined(__x86_64__)
82 #include <machine/specialreg.h>
83 #endif
84
85 #include <dev/netif/mxge/mxge_mcp.h>
86 #include <dev/netif/mxge/mcp_gen_header.h>
87 #include <dev/netif/mxge/if_mxge_var.h>
88
89 #define MXGE_IFM (IFM_ETHER | IFM_FDX | IFM_ETH_FORCEPAUSE)
90
91 #define MXGE_RX_SMALL_BUFLEN (MHLEN - MXGEFW_PAD)
92 #define MXGE_HWRSS_KEYLEN 16
93
94 /* Tunable params */
95 static int mxge_nvidia_ecrc_enable = 1;
96 static int mxge_force_firmware = 0;
97 static int mxge_intr_coal_delay = MXGE_INTR_COAL_DELAY;
98 static int mxge_deassert_wait = 1;
99 static int mxge_ticks;
100 static int mxge_num_slices = 0;
101 static int mxge_always_promisc = 0;
102 static int mxge_throttle = 0;
103 static int mxge_msi_enable = 1;
104 static int mxge_msix_enable = 1;
105 static int mxge_multi_tx = 1;
106 /*
107 * Don't use RSS by default, its just too slow
108 */
109 static int mxge_use_rss = 0;
110
111 static char mxge_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_FORCE_NONE;
112
113 static const char *mxge_fw_unaligned = "mxge_ethp_z8e";
114 static const char *mxge_fw_aligned = "mxge_eth_z8e";
115 static const char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e";
116 static const char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e";
117
118 TUNABLE_INT("hw.mxge.num_slices", &mxge_num_slices);
119 TUNABLE_INT("hw.mxge.intr_coal_delay", &mxge_intr_coal_delay);
120 TUNABLE_INT("hw.mxge.nvidia_ecrc_enable", &mxge_nvidia_ecrc_enable);
121 TUNABLE_INT("hw.mxge.force_firmware", &mxge_force_firmware);
122 TUNABLE_INT("hw.mxge.deassert_wait", &mxge_deassert_wait);
123 TUNABLE_INT("hw.mxge.ticks", &mxge_ticks);
124 TUNABLE_INT("hw.mxge.always_promisc", &mxge_always_promisc);
125 TUNABLE_INT("hw.mxge.throttle", &mxge_throttle);
126 TUNABLE_INT("hw.mxge.multi_tx", &mxge_multi_tx);
127 TUNABLE_INT("hw.mxge.use_rss", &mxge_use_rss);
128 TUNABLE_INT("hw.mxge.msi.enable", &mxge_msi_enable);
129 TUNABLE_INT("hw.mxge.msix.enable", &mxge_msix_enable);
130 TUNABLE_STR("hw.mxge.flow_ctrl", mxge_flowctrl, sizeof(mxge_flowctrl));
131
132 static int mxge_probe(device_t dev);
133 static int mxge_attach(device_t dev);
134 static int mxge_detach(device_t dev);
135 static int mxge_shutdown(device_t dev);
136
137 static int mxge_alloc_intr(struct mxge_softc *sc);
138 static void mxge_free_intr(struct mxge_softc *sc);
139 static int mxge_setup_intr(struct mxge_softc *sc);
140 static void mxge_teardown_intr(struct mxge_softc *sc, int cnt);
141
142 static device_method_t mxge_methods[] = {
143 /* Device interface */
144 DEVMETHOD(device_probe, mxge_probe),
145 DEVMETHOD(device_attach, mxge_attach),
146 DEVMETHOD(device_detach, mxge_detach),
147 DEVMETHOD(device_shutdown, mxge_shutdown),
148 DEVMETHOD_END
149 };
150
151 static driver_t mxge_driver = {
152 "mxge",
153 mxge_methods,
154 sizeof(mxge_softc_t),
155 };
156
157 static devclass_t mxge_devclass;
158
159 /* Declare ourselves to be a child of the PCI bus.*/
160 DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, NULL, NULL);
161 MODULE_DEPEND(mxge, firmware, 1, 1, 1);
162 MODULE_DEPEND(mxge, zlib, 1, 1, 1);
163
164 static int mxge_load_firmware(mxge_softc_t *sc, int adopt);
165 static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
166 static void mxge_close(mxge_softc_t *sc, int down);
167 static int mxge_open(mxge_softc_t *sc);
168 static void mxge_tick(void *arg);
169 static void mxge_watchdog_reset(mxge_softc_t *sc);
170 static void mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice);
171
172 static int
173 mxge_probe(device_t dev)
174 {
175 if (pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM &&
176 (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E ||
177 pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9)) {
178 int rev = pci_get_revid(dev);
179
180 switch (rev) {
181 case MXGE_PCI_REV_Z8E:
182 device_set_desc(dev, "Myri10G-PCIE-8A");
183 break;
184 case MXGE_PCI_REV_Z8ES:
185 device_set_desc(dev, "Myri10G-PCIE-8B");
186 break;
187 default:
188 device_set_desc(dev, "Myri10G-PCIE-8??");
189 device_printf(dev, "Unrecognized rev %d NIC\n", rev);
190 break;
191 }
192 return 0;
193 }
194 return ENXIO;
195 }
196
197 static void
198 mxge_enable_wc(mxge_softc_t *sc)
199 {
200 #if defined(__x86_64__)
201 vm_offset_t len;
202
203 sc->wc = 1;
204 len = rman_get_size(sc->mem_res);
205 pmap_change_attr((vm_offset_t) sc->sram, len / PAGE_SIZE,
206 PAT_WRITE_COMBINING);
207 #endif
208 }
209
210 static int
211 mxge_dma_alloc(mxge_softc_t *sc, bus_dmamem_t *dma, size_t bytes,
212 bus_size_t alignment)
213 {
214 bus_size_t boundary;
215 int err;
216
217 if (bytes > 4096 && alignment == 4096)
218 boundary = 0;
219 else
220 boundary = 4096;
221
222 err = bus_dmamem_coherent(sc->parent_dmat, alignment, boundary,
223 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, bytes,
224 BUS_DMA_WAITOK | BUS_DMA_ZERO, dma);
225 if (err != 0) {
226 device_printf(sc->dev, "bus_dmamem_coherent failed: %d\n", err);
227 return err;
228 }
229 return 0;
230 }
231
232 static void
233 mxge_dma_free(bus_dmamem_t *dma)
234 {
235 bus_dmamap_unload(dma->dmem_tag, dma->dmem_map);
236 bus_dmamem_free(dma->dmem_tag, dma->dmem_addr, dma->dmem_map);
237 bus_dma_tag_destroy(dma->dmem_tag);
238 }
239
240 /*
241 * The eeprom strings on the lanaiX have the format
242 * SN=x\0
243 * MAC=x:x:x:x:x:x\0
244 * PC=text\0
245 */
246 static int
247 mxge_parse_strings(mxge_softc_t *sc)
248 {
249 const char *ptr;
250 int i, found_mac, found_sn2;
251 char *endptr;
252
253 ptr = sc->eeprom_strings;
254 found_mac = 0;
255 found_sn2 = 0;
256 while (*ptr != '\0') {
257 if (strncmp(ptr, "MAC=", 4) == 0) {
258 ptr += 4;
259 for (i = 0;;) {
260 sc->mac_addr[i] = strtoul(ptr, &endptr, 16);
261 if (endptr - ptr != 2)
262 goto abort;
263 ptr = endptr;
264 if (++i == 6)
265 break;
266 if (*ptr++ != ':')
267 goto abort;
268 }
269 found_mac = 1;
270 } else if (strncmp(ptr, "PC=", 3) == 0) {
271 ptr += 3;
272 strlcpy(sc->product_code_string, ptr,
273 sizeof(sc->product_code_string));
274 } else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) {
275 ptr += 3;
276 strlcpy(sc->serial_number_string, ptr,
277 sizeof(sc->serial_number_string));
278 } else if (strncmp(ptr, "SN2=", 4) == 0) {
279 /* SN2 takes precedence over SN */
280 ptr += 4;
281 found_sn2 = 1;
282 strlcpy(sc->serial_number_string, ptr,
283 sizeof(sc->serial_number_string));
284 }
285 while (*ptr++ != '\0') {}
286 }
287
288 if (found_mac)
289 return 0;
290
291 abort:
292 device_printf(sc->dev, "failed to parse eeprom_strings\n");
293 return ENXIO;
294 }
295
296 #if defined(__x86_64__)
297
298 static void
299 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
300 {
301 uint32_t val;
302 unsigned long base, off;
303 char *va, *cfgptr;
304 device_t pdev, mcp55;
305 uint16_t vendor_id, device_id, word;
306 uintptr_t bus, slot, func, ivend, idev;
307 uint32_t *ptr32;
308
309 if (!mxge_nvidia_ecrc_enable)
310 return;
311
312 pdev = device_get_parent(device_get_parent(sc->dev));
313 if (pdev == NULL) {
314 device_printf(sc->dev, "could not find parent?\n");
315 return;
316 }
317 vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2);
318 device_id = pci_read_config(pdev, PCIR_DEVICE, 2);
319
320 if (vendor_id != 0x10de)
321 return;
322
323 base = 0;
324
325 if (device_id == 0x005d) {
326 /* ck804, base address is magic */
327 base = 0xe0000000UL;
328 } else if (device_id >= 0x0374 && device_id <= 0x378) {
329 /* mcp55, base address stored in chipset */
330 mcp55 = pci_find_bsf(0, 0, 0);
331 if (mcp55 &&
332 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) &&
333 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) {
334 word = pci_read_config(mcp55, 0x90, 2);
335 base = ((unsigned long)word & 0x7ffeU) << 25;
336 }
337 }
338 if (!base)
339 return;
340
341 /*
342 * XXXX
343 * Test below is commented because it is believed that doing
344 * config read/write beyond 0xff will access the config space
345 * for the next larger function. Uncomment this and remove
346 * the hacky pmap_mapdev() way of accessing config space when
347 * DragonFly grows support for extended pcie config space access.
348 */
349 #if 0
350 /*
351 * See if we can, by some miracle, access the extended
352 * config space
353 */
354 val = pci_read_config(pdev, 0x178, 4);
355 if (val != 0xffffffff) {
356 val |= 0x40;
357 pci_write_config(pdev, 0x178, val, 4);
358 return;
359 }
360 #endif
361 /*
362 * Rather than using normal pci config space writes, we must
363 * map the Nvidia config space ourselves. This is because on
364 * opteron/nvidia class machine the 0xe000000 mapping is
365 * handled by the nvidia chipset, that means the internal PCI
366 * device (the on-chip northbridge), or the amd-8131 bridge
367 * and things behind them are not visible by this method.
368 */
369
370 BUS_READ_IVAR(device_get_parent(pdev), pdev,
371 PCI_IVAR_BUS, &bus);
372 BUS_READ_IVAR(device_get_parent(pdev), pdev,
373 PCI_IVAR_SLOT, &slot);
374 BUS_READ_IVAR(device_get_parent(pdev), pdev,
375 PCI_IVAR_FUNCTION, &func);
376 BUS_READ_IVAR(device_get_parent(pdev), pdev,
377 PCI_IVAR_VENDOR, &ivend);
378 BUS_READ_IVAR(device_get_parent(pdev), pdev,
379 PCI_IVAR_DEVICE, &idev);
380
381 off = base + 0x00100000UL * (unsigned long)bus +
382 0x00001000UL * (unsigned long)(func + 8 * slot);
383
384 /* map it into the kernel */
385 va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE);
386 if (va == NULL) {
387 device_printf(sc->dev, "pmap_kenter_temporary didn't\n");
388 return;
389 }
390 /* get a pointer to the config space mapped into the kernel */
391 cfgptr = va + (off & PAGE_MASK);
392
393 /* make sure that we can really access it */
394 vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR);
395 device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE);
396 if (!(vendor_id == ivend && device_id == idev)) {
397 device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n",
398 vendor_id, device_id);
399 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
400 return;
401 }
402
403 ptr32 = (uint32_t*)(cfgptr + 0x178);
404 val = *ptr32;
405
406 if (val == 0xffffffff) {
407 device_printf(sc->dev, "extended mapping failed\n");
408 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
409 return;
410 }
411 *ptr32 = val | 0x40;
412 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
413 if (bootverbose) {
414 device_printf(sc->dev, "Enabled ECRC on upstream "
415 "Nvidia bridge at %d:%d:%d\n",
416 (int)bus, (int)slot, (int)func);
417 }
418 }
419
420 #else /* __x86_64__ */
421
422 static void
423 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
424 {
425 device_printf(sc->dev, "Nforce 4 chipset on non-x86/x86_64!?!?!\n");
426 }
427
428 #endif
429
430 static int
431 mxge_dma_test(mxge_softc_t *sc, int test_type)
432 {
433 mxge_cmd_t cmd;
434 bus_addr_t dmatest_bus = sc->dmabench_dma.dmem_busaddr;
435 int status;
436 uint32_t len;
437 const char *test = " ";
438
439 /*
440 * Run a small DMA test.
441 * The magic multipliers to the length tell the firmware
442 * to do DMA read, write, or read+write tests. The
443 * results are returned in cmd.data0. The upper 16
444 * bits of the return is the number of transfers completed.
445 * The lower 16 bits is the time in 0.5us ticks that the
446 * transfers took to complete.
447 */
448
449 len = sc->tx_boundary;
450
451 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
452 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
453 cmd.data2 = len * 0x10000;
454 status = mxge_send_cmd(sc, test_type, &cmd);
455 if (status != 0) {
456 test = "read";
457 goto abort;
458 }
459 sc->read_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
460
461 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
462 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
463 cmd.data2 = len * 0x1;
464 status = mxge_send_cmd(sc, test_type, &cmd);
465 if (status != 0) {
466 test = "write";
467 goto abort;
468 }
469 sc->write_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
470
471 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
472 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
473 cmd.data2 = len * 0x10001;
474 status = mxge_send_cmd(sc, test_type, &cmd);
475 if (status != 0) {
476 test = "read/write";
477 goto abort;
478 }
479 sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
480 (cmd.data0 & 0xffff);
481
482 abort:
483 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) {
484 device_printf(sc->dev, "DMA %s benchmark failed: %d\n",
485 test, status);
486 }
487 return status;
488 }
489
490 /*
491 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
492 * when the PCI-E Completion packets are aligned on an 8-byte
493 * boundary. Some PCI-E chip sets always align Completion packets; on
494 * the ones that do not, the alignment can be enforced by enabling
495 * ECRC generation (if supported).
496 *
497 * When PCI-E Completion packets are not aligned, it is actually more
498 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
499 *
500 * If the driver can neither enable ECRC nor verify that it has
501 * already been enabled, then it must use a firmware image which works
502 * around unaligned completion packets (ethp_z8e.dat), and it should
503 * also ensure that it never gives the device a Read-DMA which is
504 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is
505 * enabled, then the driver should use the aligned (eth_z8e.dat)
506 * firmware image, and set tx_boundary to 4KB.
507 */
508 static int
509 mxge_firmware_probe(mxge_softc_t *sc)
510 {
511 device_t dev = sc->dev;
512 int reg, status;
513 uint16_t pectl;
514
515 sc->tx_boundary = 4096;
516
517 /*
518 * Verify the max read request size was set to 4KB
519 * before trying the test with 4KB.
520 */
521 if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
522 pectl = pci_read_config(dev, reg + 0x8, 2);
523 if ((pectl & (5 << 12)) != (5 << 12)) {
524 device_printf(dev, "Max Read Req. size != 4k (0x%x)\n",
525 pectl);
526 sc->tx_boundary = 2048;
527 }
528 }
529
530 /*
531 * Load the optimized firmware (which assumes aligned PCIe
532 * completions) in order to see if it works on this host.
533 */
534 sc->fw_name = mxge_fw_aligned;
535 status = mxge_load_firmware(sc, 1);
536 if (status != 0)
537 return status;
538
539 /*
540 * Enable ECRC if possible
541 */
542 mxge_enable_nvidia_ecrc(sc);
543
544 /*
545 * Run a DMA test which watches for unaligned completions and
546 * aborts on the first one seen. Not required on Z8ES or newer.
547 */
548 if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES)
549 return 0;
550
551 status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST);
552 if (status == 0)
553 return 0; /* keep the aligned firmware */
554
555 if (status != E2BIG)
556 device_printf(dev, "DMA test failed: %d\n", status);
557 if (status == ENOSYS) {
558 device_printf(dev, "Falling back to ethp! "
559 "Please install up to date fw\n");
560 }
561 return status;
562 }
563
564 static int
565 mxge_select_firmware(mxge_softc_t *sc)
566 {
567 int aligned = 0;
568 int force_firmware = mxge_force_firmware;
569
570 if (sc->throttle)
571 force_firmware = sc->throttle;
572
573 if (force_firmware != 0) {
574 if (force_firmware == 1)
575 aligned = 1;
576 else
577 aligned = 0;
578 if (bootverbose) {
579 device_printf(sc->dev,
580 "Assuming %s completions (forced)\n",
581 aligned ? "aligned" : "unaligned");
582 }
583 goto abort;
584 }
585
586 /*
587 * If the PCIe link width is 4 or less, we can use the aligned
588 * firmware and skip any checks
589 */
590 if (sc->link_width != 0 && sc->link_width <= 4) {
591 device_printf(sc->dev, "PCIe x%d Link, "
592 "expect reduced performance\n", sc->link_width);
593 aligned = 1;
594 goto abort;
595 }
596
597 if (mxge_firmware_probe(sc) == 0)
598 return 0;
599
600 abort:
601 if (aligned) {
602 sc->fw_name = mxge_fw_aligned;
603 sc->tx_boundary = 4096;
604 } else {
605 sc->fw_name = mxge_fw_unaligned;
606 sc->tx_boundary = 2048;
607 }
608 return mxge_load_firmware(sc, 0);
609 }
610
611 static int
612 mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr)
613 {
614 if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) {
615 if_printf(sc->ifp, "Bad firmware type: 0x%x\n",
616 be32toh(hdr->mcp_type));
617 return EIO;
618 }
619
620 /* Save firmware version for sysctl */
621 strlcpy(sc->fw_version, hdr->version, sizeof(sc->fw_version));
622 if (bootverbose)
623 if_printf(sc->ifp, "firmware id: %s\n", hdr->version);
624
625 ksscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major,
626 &sc->fw_ver_minor, &sc->fw_ver_tiny);
627
628 if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR &&
629 sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
630 if_printf(sc->ifp, "Found firmware version %s\n",
631 sc->fw_version);
632 if_printf(sc->ifp, "Driver needs %d.%d\n",
633 MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR);
634 return EINVAL;
635 }
636 return 0;
637 }
638
639 static void *
640 z_alloc(void *nil, u_int items, u_int size)
641 {
642 return kmalloc(items * size, M_TEMP, M_WAITOK);
643 }
644
645 static void
646 z_free(void *nil, void *ptr)
647 {
648 kfree(ptr, M_TEMP);
649 }
650
651 static int
652 mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit)
653 {
654 z_stream zs;
655 char *inflate_buffer;
656 const struct firmware *fw;
657 const mcp_gen_header_t *hdr;
658 unsigned hdr_offset;
659 int status;
660 unsigned int i;
661 char dummy;
662 size_t fw_len;
663
664 fw = firmware_get(sc->fw_name);
665 if (fw == NULL) {
666 if_printf(sc->ifp, "Could not find firmware image %s\n",
667 sc->fw_name);
668 return ENOENT;
669 }
670
671 /* Setup zlib and decompress f/w */
672 bzero(&zs, sizeof(zs));
673 zs.zalloc = z_alloc;
674 zs.zfree = z_free;
675 status = inflateInit(&zs);
676 if (status != Z_OK) {
677 status = EIO;
678 goto abort_with_fw;
679 }
680
681 /*
682 * The uncompressed size is stored as the firmware version,
683 * which would otherwise go unused
684 */
685 fw_len = (size_t)fw->version;
686 inflate_buffer = kmalloc(fw_len, M_TEMP, M_WAITOK);
687 zs.avail_in = fw->datasize;
688 zs.next_in = __DECONST(char *, fw->data);
689 zs.avail_out = fw_len;
690 zs.next_out = inflate_buffer;
691 status = inflate(&zs, Z_FINISH);
692 if (status != Z_STREAM_END) {
693 if_printf(sc->ifp, "zlib %d\n", status);
694 status = EIO;
695 goto abort_with_buffer;
696 }
697
698 /* Check id */
699 hdr_offset =
700 htobe32(*(const uint32_t *)(inflate_buffer + MCP_HEADER_PTR_OFFSET));
701 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) {
702 if_printf(sc->ifp, "Bad firmware file");
703 status = EIO;
704 goto abort_with_buffer;
705 }
706 hdr = (const void*)(inflate_buffer + hdr_offset);
707
708 status = mxge_validate_firmware(sc, hdr);
709 if (status != 0)
710 goto abort_with_buffer;
711
712 /* Copy the inflated firmware to NIC SRAM. */
713 for (i = 0; i < fw_len; i += 256) {
714 mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, inflate_buffer + i,
715 min(256U, (unsigned)(fw_len - i)));
716 wmb();
717 dummy = *sc->sram;
718 wmb();
719 }
720
721 *limit = fw_len;
722 status = 0;
723 abort_with_buffer:
724 kfree(inflate_buffer, M_TEMP);
725 inflateEnd(&zs);
726 abort_with_fw:
727 firmware_put(fw, FIRMWARE_UNLOAD);
728 return status;
729 }
730
731 /*
732 * Enable or disable periodic RDMAs from the host to make certain
733 * chipsets resend dropped PCIe messages
734 */
735 static void
736 mxge_dummy_rdma(mxge_softc_t *sc, int enable)
737 {
738 char buf_bytes[72];
739 volatile uint32_t *confirm;
740 volatile char *submit;
741 uint32_t *buf, dma_low, dma_high;
742 int i;
743
744 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
745
746 /* Clear confirmation addr */
747 confirm = (volatile uint32_t *)sc->cmd;
748 *confirm = 0;
749 wmb();
750
751 /*
752 * Send an rdma command to the PCIe engine, and wait for the
753 * response in the confirmation address. The firmware should
754 * write a -1 there to indicate it is alive and well
755 */
756 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.dmem_busaddr);
757 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.dmem_busaddr);
758 buf[0] = htobe32(dma_high); /* confirm addr MSW */
759 buf[1] = htobe32(dma_low); /* confirm addr LSW */
760 buf[2] = htobe32(0xffffffff); /* confirm data */
761 dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.dmem_busaddr);
762 dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.dmem_busaddr);
763 buf[3] = htobe32(dma_high); /* dummy addr MSW */
764 buf[4] = htobe32(dma_low); /* dummy addr LSW */
765 buf[5] = htobe32(enable); /* enable? */
766
767 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA);
768
769 mxge_pio_copy(submit, buf, 64);
770 wmb();
771 DELAY(1000);
772 wmb();
773 i = 0;
774 while (*confirm != 0xffffffff && i < 20) {
775 DELAY(1000);
776 i++;
777 }
778 if (*confirm != 0xffffffff) {
779 if_printf(sc->ifp, "dummy rdma %s failed (%p = 0x%x)",
780 (enable ? "enable" : "disable"), confirm, *confirm);
781 }
782 }
783
784 static int
785 mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data)
786 {
787 mcp_cmd_t *buf;
788 char buf_bytes[sizeof(*buf) + 8];
789 volatile mcp_cmd_response_t *response = sc->cmd;
790 volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD;
791 uint32_t dma_low, dma_high;
792 int err, sleep_total = 0;
793
794 /* Ensure buf is aligned to 8 bytes */
795 buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
796
797 buf->data0 = htobe32(data->data0);
798 buf->data1 = htobe32(data->data1);
799 buf->data2 = htobe32(data->data2);
800 buf->cmd = htobe32(cmd);
801 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.dmem_busaddr);
802 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.dmem_busaddr);
803
804 buf->response_addr.low = htobe32(dma_low);
805 buf->response_addr.high = htobe32(dma_high);
806
807 response->result = 0xffffffff;
808 wmb();
809 mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf));
810
811 /*
812 * Wait up to 20ms
813 */
814 err = EAGAIN;
815 for (sleep_total = 0; sleep_total < 20; sleep_total++) {
816 wmb();
817 switch (be32toh(response->result)) {
818 case 0:
819 data->data0 = be32toh(response->data);
820 err = 0;
821 break;
822 case 0xffffffff:
823 DELAY(1000);
824 break;
825 case MXGEFW_CMD_UNKNOWN:
826 err = ENOSYS;
827 break;
828 case MXGEFW_CMD_ERROR_UNALIGNED:
829 err = E2BIG;
830 break;
831 case MXGEFW_CMD_ERROR_BUSY:
832 err = EBUSY;
833 break;
834 case MXGEFW_CMD_ERROR_I2C_ABSENT:
835 err = ENXIO;
836 break;
837 default:
838 if_printf(sc->ifp, "command %d failed, result = %d\n",
839 cmd, be32toh(response->result));
840 err = ENXIO;
841 break;
842 }
843 if (err != EAGAIN)
844 break;
845 }
846 if (err == EAGAIN) {
847 if_printf(sc->ifp, "command %d timed out result = %d\n",
848 cmd, be32toh(response->result));
849 }
850 return err;
851 }
852
853 static int
854 mxge_adopt_running_firmware(mxge_softc_t *sc)
855 {
856 struct mcp_gen_header *hdr;
857 const size_t bytes = sizeof(struct mcp_gen_header);
858 size_t hdr_offset;
859 int status;
860
861 /*
862 * Find running firmware header
863 */
864 hdr_offset =
865 htobe32(*(volatile uint32_t *)(sc->sram + MCP_HEADER_PTR_OFFSET));
866
867 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) {
868 if_printf(sc->ifp, "Running firmware has bad header offset "
869 "(%zu)\n", hdr_offset);
870 return EIO;
871 }
872
873 /*
874 * Copy header of running firmware from SRAM to host memory to
875 * validate firmware
876 */
877 hdr = kmalloc(bytes, M_DEVBUF, M_WAITOK);
878 bus_space_read_region_1(rman_get_bustag(sc->mem_res),
879 rman_get_bushandle(sc->mem_res), hdr_offset, (char *)hdr, bytes);
880 status = mxge_validate_firmware(sc, hdr);
881 kfree(hdr, M_DEVBUF);
882
883 /*
884 * Check to see if adopted firmware has bug where adopting
885 * it will cause broadcasts to be filtered unless the NIC
886 * is kept in ALLMULTI mode
887 */
888 if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
889 sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) {
890 sc->adopted_rx_filter_bug = 1;
891 if_printf(sc->ifp, "Adopting fw %d.%d.%d: "
892 "working around rx filter bug\n",
893 sc->fw_ver_major, sc->fw_ver_minor, sc->fw_ver_tiny);
894 }
895
896 return status;
897 }
898
899 static int
900 mxge_load_firmware(mxge_softc_t *sc, int adopt)
901 {
902 volatile uint32_t *confirm;
903 volatile char *submit;
904 char buf_bytes[72];
905 uint32_t *buf, size, dma_low, dma_high;
906 int status, i;
907
908 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
909
910 size = sc->sram_size;
911 status = mxge_load_firmware_helper(sc, &size);
912 if (status) {
913 if (!adopt)
914 return status;
915
916 /*
917 * Try to use the currently running firmware, if
918 * it is new enough
919 */
920 status = mxge_adopt_running_firmware(sc);
921 if (status) {
922 if_printf(sc->ifp,
923 "failed to adopt running firmware\n");
924 return status;
925 }
926 if_printf(sc->ifp, "Successfully adopted running firmware\n");
927
928 if (sc->tx_boundary == 4096) {
929 if_printf(sc->ifp,
930 "Using firmware currently running on NIC. "
931 "For optimal\n");
932 if_printf(sc->ifp, "performance consider loading "
933 "optimized firmware\n");
934 }
935 sc->fw_name = mxge_fw_unaligned;
936 sc->tx_boundary = 2048;
937 return 0;
938 }
939
940 /* Clear confirmation addr */
941 confirm = (volatile uint32_t *)sc->cmd;
942 *confirm = 0;
943 wmb();
944
945 /*
946 * Send a reload command to the bootstrap MCP, and wait for the
947 * response in the confirmation address. The firmware should
948 * write a -1 there to indicate it is alive and well
949 */
950
951 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.dmem_busaddr);
952 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.dmem_busaddr);
953
954 buf[0] = htobe32(dma_high); /* confirm addr MSW */
955 buf[1] = htobe32(dma_low); /* confirm addr LSW */
956 buf[2] = htobe32(0xffffffff); /* confirm data */
957
958 /*
959 * FIX: All newest firmware should un-protect the bottom of
960 * the sram before handoff. However, the very first interfaces
961 * do not. Therefore the handoff copy must skip the first 8 bytes
962 */
963 /* where the code starts*/
964 buf[3] = htobe32(MXGE_FW_OFFSET + 8);
965 buf[4] = htobe32(size - 8); /* length of code */
966 buf[5] = htobe32(8); /* where to copy to */
967 buf[6] = htobe32(0); /* where to jump to */
968
969 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF);
970 mxge_pio_copy(submit, buf, 64);
971 wmb();
972 DELAY(1000);
973 wmb();
974 i = 0;
975 while (*confirm != 0xffffffff && i < 20) {
976 DELAY(1000*10);
977 i++;
978 }
979 if (*confirm != 0xffffffff) {
980 if_printf(sc->ifp,"handoff failed (%p = 0x%x)",
981 confirm, *confirm);
982 return ENXIO;
983 }
984 return 0;
985 }
986
987 static int
988 mxge_update_mac_address(mxge_softc_t *sc)
989 {
990 mxge_cmd_t cmd;
991 uint8_t *addr = sc->mac_addr;
992
993 cmd.data0 = (addr[0] << 24) | (addr[1] << 16) |
994 (addr[2] << 8) | addr[3];
995 cmd.data1 = (addr[4] << 8) | (addr[5]);
996 return mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd);
997 }
998
999 static int
1000 mxge_change_pause(mxge_softc_t *sc, int pause)
1001 {
1002 mxge_cmd_t cmd;
1003 int status;
1004
1005 bzero(&cmd, sizeof(cmd)); /* silence gcc warning */
1006 if (pause)
1007 status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL, &cmd);
1008 else
1009 status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL, &cmd);
1010 if (status) {
1011 if_printf(sc->ifp, "Failed to set flow control mode\n");
1012 return ENXIO;
1013 }
1014 sc->pause = pause;
1015 return 0;
1016 }
1017
1018 static void
1019 mxge_change_promisc(mxge_softc_t *sc, int promisc)
1020 {
1021 mxge_cmd_t cmd;
1022 int status;
1023
1024 bzero(&cmd, sizeof(cmd)); /* avoid gcc warning */
1025 if (mxge_always_promisc)
1026 promisc = 1;
1027
1028 if (promisc)
1029 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC, &cmd);
1030 else
1031 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC, &cmd);
1032 if (status)
1033 if_printf(sc->ifp, "Failed to set promisc mode\n");
1034 }
1035
1036 static void
1037 mxge_set_multicast_list(mxge_softc_t *sc)
1038 {
1039 mxge_cmd_t cmd;
1040 struct ifmultiaddr *ifma;
1041 struct ifnet *ifp = sc->ifp;
1042 int err;
1043
1044 /* This firmware is known to not support multicast */
1045 if (!sc->fw_multicast_support)
1046 return;
1047
1048 /* Disable multicast filtering while we play with the lists*/
1049 bzero(&cmd, sizeof(cmd)); /* silence gcc warning */
1050 err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd);
1051 if (err != 0) {
1052 if_printf(ifp, "Failed MXGEFW_ENABLE_ALLMULTI, "
1053 "error status: %d\n", err);
1054 return;
1055 }
1056
1057 if (sc->adopted_rx_filter_bug)
1058 return;
1059
1060 if (ifp->if_flags & IFF_ALLMULTI) {
1061 /* Request to disable multicast filtering, so quit here */
1062 return;
1063 }
1064
1065 /* Flush all the filters */
1066 err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd);
1067 if (err != 0) {
1068 if_printf(ifp, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, "
1069 "error status: %d\n", err);
1070 return;
1071 }
1072
1073 /*
1074 * Walk the multicast list, and add each address
1075 */
1076 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1077 if (ifma->ifma_addr->sa_family != AF_LINK)
1078 continue;
1079
1080 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1081 &cmd.data0, 4);
1082 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4,
1083 &cmd.data1, 2);
1084 cmd.data0 = htonl(cmd.data0);
1085 cmd.data1 = htonl(cmd.data1);
1086 err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd);
1087 if (err != 0) {
1088 if_printf(ifp, "Failed MXGEFW_JOIN_MULTICAST_GROUP, "
1089 "error status: %d\n", err);
1090 /* Abort, leaving multicast filtering off */
1091 return;
1092 }
1093 }
1094
1095 /* Enable multicast filtering */
1096 err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd);
1097 if (err != 0) {
1098 if_printf(ifp, "Failed MXGEFW_DISABLE_ALLMULTI, "
1099 "error status: %d\n", err);
1100 }
1101 }
1102
1103 #if 0
1104 static int
1105 mxge_max_mtu(mxge_softc_t *sc)
1106 {
1107 mxge_cmd_t cmd;
1108 int status;
1109
1110 if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU)
1111 return MXGEFW_MAX_MTU - MXGEFW_PAD;
1112
1113 /* try to set nbufs to see if it we can
1114 use virtually contiguous jumbos */
1115 cmd.data0 = 0;
1116 status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
1117 &cmd);
1118 if (status == 0)
1119 return MXGEFW_MAX_MTU - MXGEFW_PAD;
1120
1121 /* otherwise, we're limited to MJUMPAGESIZE */
1122 return MJUMPAGESIZE - MXGEFW_PAD;
1123 }
1124 #endif
1125
1126 static int
1127 mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1128 {
1129 struct mxge_slice_state *ss;
1130 mxge_rx_done_t *rx_done;
1131 volatile uint32_t *irq_claim;
1132 mxge_cmd_t cmd;
1133 int slice, status, rx_intr_size;
1134
1135 /*
1136 * Try to send a reset command to the card to see if it
1137 * is alive
1138 */
1139 memset(&cmd, 0, sizeof (cmd));
1140 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1141 if (status != 0) {
1142 if_printf(sc->ifp, "failed reset\n");
1143 return ENXIO;
1144 }
1145
1146 mxge_dummy_rdma(sc, 1);
1147
1148 /*
1149 * Set the intrq size
1150 * XXX assume 4byte mcp_slot
1151 */
1152 rx_intr_size = sc->rx_intr_slots * sizeof(mcp_slot_t);
1153 cmd.data0 = rx_intr_size;
1154 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1155
1156 /*
1157 * Even though we already know how many slices are supported
1158 * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES
1159 * has magic side effects, and must be called after a reset.
1160 * It must be called prior to calling any RSS related cmds,
1161 * including assigning an interrupt queue for anything but
1162 * slice 0. It must also be called *after*
1163 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
1164 * the firmware to compute offsets.
1165 */
1166 if (sc->num_slices > 1) {
1167 /* Ask the maximum number of slices it supports */
1168 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
1169 if (status != 0) {
1170 if_printf(sc->ifp, "failed to get number of slices\n");
1171 return status;
1172 }
1173
1174 /*
1175 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
1176 * to setting up the interrupt queue DMA
1177 */
1178 cmd.data0 = sc->num_slices;
1179 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
1180 if (sc->num_tx_rings > 1)
1181 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
1182 status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES, &cmd);
1183 if (status != 0) {
1184 if_printf(sc->ifp, "failed to set number of slices\n");
1185 return status;
1186 }
1187 }
1188
1189 if (interrupts_setup) {
1190 /* Now exchange information about interrupts */
1191 for (slice = 0; slice < sc->num_slices; slice++) {
1192 ss = &sc->ss[slice];
1193
1194 rx_done = &ss->rx_data.rx_done;
1195 memset(rx_done->entry, 0, rx_intr_size);
1196
1197 cmd.data0 =
1198 MXGE_LOWPART_TO_U32(ss->rx_done_dma.dmem_busaddr);
1199 cmd.data1 =
1200 MXGE_HIGHPART_TO_U32(ss->rx_done_dma.dmem_busaddr);
1201 cmd.data2 = slice;
1202 status |= mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_DMA,
1203 &cmd);
1204 }
1205 }
1206
1207 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET,
1208 &cmd);
1209 sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
1210
1211 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1212 irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
1213
1214 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, &cmd);
1215 sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1216
1217 if (status != 0) {
1218 if_printf(sc->ifp, "failed set interrupt parameters\n");
1219 return status;
1220 }
1221
1222 *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
1223
1224 /* Run a DMA benchmark */
1225 mxge_dma_test(sc, MXGEFW_DMA_TEST);
1226
1227 for (slice = 0; slice < sc->num_slices; slice++) {
1228 ss = &sc->ss[slice];
1229
1230 ss->irq_claim = irq_claim + (2 * slice);
1231
1232 /* Reset mcp/driver shared state back to 0 */
1233 ss->rx_data.rx_done.idx = 0;
1234 ss->tx.req = 0;
1235 ss->tx.done = 0;
1236 ss->tx.pkt_done = 0;
1237 ss->tx.queue_active = 0;
1238 ss->tx.activate = 0;
1239 ss->tx.deactivate = 0;
1240 ss->rx_data.rx_big.cnt = 0;
1241 ss->rx_data.rx_small.cnt = 0;
1242 if (ss->fw_stats != NULL)
1243 bzero(ss->fw_stats, sizeof(*ss->fw_stats));
1244 }
1245 sc->rdma_tags_available = 15;
1246
1247 status = mxge_update_mac_address(sc);
1248 mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC);
1249 mxge_change_pause(sc, sc->pause);
1250 mxge_set_multicast_list(sc);
1251
1252 if (sc->throttle) {
1253 cmd.data0 = sc->throttle;
1254 if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd))
1255 if_printf(sc->ifp, "can't enable throttle\n");
1256 }
1257 return status;
1258 }
1259
1260 static int
1261 mxge_change_throttle(SYSCTL_HANDLER_ARGS)
1262 {
1263 mxge_cmd_t cmd;
1264 mxge_softc_t *sc;
1265 int err;
1266 unsigned int throttle;
1267
1268 sc = arg1;
1269 throttle = sc->throttle;
1270 err = sysctl_handle_int(oidp, &throttle, arg2, req);
1271 if (err != 0)
1272 return err;
1273
1274 if (throttle == sc->throttle)
1275 return 0;
1276
1277 if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE)
1278 return EINVAL;
1279
1280 ifnet_serialize_all(sc->ifp);
1281
1282 cmd.data0 = throttle;
1283 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd);
1284 if (err == 0)
1285 sc->throttle = throttle;
1286
1287 ifnet_deserialize_all(sc->ifp);
1288 return err;
1289 }
1290
1291 static int
1292 mxge_change_use_rss(SYSCTL_HANDLER_ARGS)
1293 {
1294 mxge_softc_t *sc;
1295 int err, use_rss;
1296
1297 sc = arg1;
1298 use_rss = sc->use_rss;
1299 err = sysctl_handle_int(oidp, &use_rss, arg2, req);
1300 if (err != 0)
1301 return err;
1302
1303 if (use_rss == sc->use_rss)
1304 return 0;
1305
1306 ifnet_serialize_all(sc->ifp);
1307
1308 sc->use_rss = use_rss;
1309 if (sc->ifp->if_flags & IFF_RUNNING) {
1310 mxge_close(sc, 0);
1311 mxge_open(sc);
1312 }
1313
1314 ifnet_deserialize_all(sc->ifp);
1315 return err;
1316 }
1317
1318 static int
1319 mxge_change_intr_coal(SYSCTL_HANDLER_ARGS)
1320 {
1321 mxge_softc_t *sc;
1322 unsigned int intr_coal_delay;
1323 int err;
1324
1325 sc = arg1;
1326 intr_coal_delay = sc->intr_coal_delay;
1327 err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req);
1328 if (err != 0)
1329 return err;
1330
1331 if (intr_coal_delay == sc->intr_coal_delay)
1332 return 0;
1333
1334 if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000)
1335 return EINVAL;
1336
1337 ifnet_serialize_all(sc->ifp);
1338
1339 *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay);
1340 sc->intr_coal_delay = intr_coal_delay;
1341
1342 ifnet_deserialize_all(sc->ifp);
1343 return err;
1344 }
1345
1346 static int
1347 mxge_handle_be32(SYSCTL_HANDLER_ARGS)
1348 {
1349 int err;
1350
1351 if (arg1 == NULL)
1352 return EFAULT;
1353 arg2 = be32toh(*(int *)arg1);
1354 arg1 = NULL;
1355 err = sysctl_handle_int(oidp, arg1, arg2, req);
1356
1357 return err;
1358 }
1359
1360 static void
1361 mxge_rem_sysctls(mxge_softc_t *sc)
1362 {
1363 if (sc->ss != NULL) {
1364 struct mxge_slice_state *ss;
1365 int slice;
1366
1367 for (slice = 0; slice < sc->num_slices; slice++) {
1368 ss = &sc->ss[slice];
1369 if (ss->sysctl_tree != NULL) {
1370 sysctl_ctx_free(&ss->sysctl_ctx);
1371 ss->sysctl_tree = NULL;
1372 }
1373 }
1374 }
1375
1376 if (sc->slice_sysctl_tree != NULL) {
1377 sysctl_ctx_free(&sc->slice_sysctl_ctx);
1378 sc->slice_sysctl_tree = NULL;
1379 }
1380 }
1381
1382 static void
1383 mxge_add_sysctls(mxge_softc_t *sc)
1384 {
1385 struct sysctl_ctx_list *ctx;
1386 struct sysctl_oid_list *children;
1387 mcp_irq_data_t *fw;
1388 struct mxge_slice_state *ss;
1389 int slice;
1390 char slice_num[8];
1391
1392 ctx = device_get_sysctl_ctx(sc->dev);
1393 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
1394 fw = sc->ss[0].fw_stats;
1395
1396 /*
1397 * Random information
1398 */
1399 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
1400 CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
1401
1402 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "serial_number",
1403 CTLFLAG_RD, &sc->serial_number_string, 0, "serial number");
1404
1405 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "product_code",
1406 CTLFLAG_RD, &sc->product_code_string, 0, "product code");
1407
1408 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcie_link_width",
1409 CTLFLAG_RD, &sc->link_width, 0, "link width");
1410
1411 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_boundary",
1412 CTLFLAG_RD, &sc->tx_boundary, 0, "tx boundary");
1413
1414 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "write_combine",
1415 CTLFLAG_RD, &sc->wc, 0, "write combining PIO");
1416
1417 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "read_dma_MBs",
1418 CTLFLAG_RD, &sc->read_dma, 0, "DMA Read speed in MB/s");
1419
1420 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "write_dma_MBs",
1421 CTLFLAG_RD, &sc->write_dma, 0, "DMA Write speed in MB/s");
1422
1423 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "read_write_dma_MBs",
1424 CTLFLAG_RD, &sc->read_write_dma, 0,
1425 "DMA concurrent Read/Write speed in MB/s");
1426
1427 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "watchdog_resets",
1428 CTLFLAG_RD, &sc->watchdog_resets, 0,
1429 "Number of times NIC was reset");
1430
1431 if (sc->num_slices > 1) {
1432 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "slice_cpumap",
1433 CTLTYPE_OPAQUE | CTLFLAG_RD, sc->ring_map, 0,
1434 if_ringmap_cpumap_sysctl, "I", "slice CPU map");
1435 }
1436
1437 /*
1438 * Performance related tunables
1439 */
1440 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_delay",
1441 CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_intr_coal, "I",
1442 "Interrupt coalescing delay in usecs");
1443
1444 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "throttle",
1445 CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_throttle, "I",
1446 "Transmit throttling");
1447
1448 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "use_rss",
1449 CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_use_rss, "I",
1450 "Use RSS");
1451
1452 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "deassert_wait",
1453 CTLFLAG_RW, &mxge_deassert_wait, 0,
1454 "Wait for IRQ line to go low in ihandler");
1455
1456 /*
1457 * Stats block from firmware is in network byte order.
1458 * Need to swap it
1459 */
1460 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "link_up",
1461 CTLTYPE_INT|CTLFLAG_RD, &fw->link_up, 0,
1462 mxge_handle_be32, "I", "link up");
1463
1464 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_tags_available",
1465 CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available, 0,
1466 mxge_handle_be32, "I", "rdma_tags_available");
1467
1468 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_bad_crc32",
1469 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_bad_crc32, 0,
1470 mxge_handle_be32, "I", "dropped_bad_crc32");
1471
1472 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_bad_phy",
1473 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_bad_phy, 0,
1474 mxge_handle_be32, "I", "dropped_bad_phy");
1475
1476 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_link_error_or_filtered",
1477 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_error_or_filtered, 0,
1478 mxge_handle_be32, "I", "dropped_link_error_or_filtered");
1479
1480 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_link_overflow",
1481 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow, 0,
1482 mxge_handle_be32, "I", "dropped_link_overflow");
1483
1484 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_multicast_filtered",
1485 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_multicast_filtered, 0,
1486 mxge_handle_be32, "I", "dropped_multicast_filtered");
1487
1488 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_no_big_buffer",
1489 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer, 0,
1490 mxge_handle_be32, "I", "dropped_no_big_buffer");
1491
1492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_no_small_buffer",
1493 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_small_buffer, 0,
1494 mxge_handle_be32, "I", "dropped_no_small_buffer");
1495
1496 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_overrun",
1497 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun, 0,
1498 mxge_handle_be32, "I", "dropped_overrun");
1499
1500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_pause",
1501 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_pause, 0,
1502 mxge_handle_be32, "I", "dropped_pause");
1503
1504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_runt",
1505 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt, 0,
1506 mxge_handle_be32, "I", "dropped_runt");
1507
1508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_unicast_filtered",
1509 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered, 0,
1510 mxge_handle_be32, "I", "dropped_unicast_filtered");
1511
1512 /* add counters exported for debugging from all slices */
1513 sysctl_ctx_init(&sc->slice_sysctl_ctx);
1514 sc->slice_sysctl_tree = SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx,
1515 children, OID_AUTO, "slice", CTLFLAG_RD, 0, "");
1516 if (sc->slice_sysctl_tree == NULL) {
1517 device_printf(sc->dev, "can't add slice sysctl node\n");
1518 return;
1519 }
1520
1521 for (slice = 0; slice < sc->num_slices; slice++) {
1522 ss = &sc->ss[slice];
1523 sysctl_ctx_init(&ss->sysctl_ctx);
1524 ctx = &ss->sysctl_ctx;
1525 children = SYSCTL_CHILDREN(sc->slice_sysctl_tree);
1526 ksprintf(slice_num, "%d", slice);
1527 ss->sysctl_tree = SYSCTL_ADD_NODE(ctx, children, OID_AUTO,
1528 slice_num, CTLFLAG_RD, 0, "");
1529 if (ss->sysctl_tree == NULL) {
1530 device_printf(sc->dev,
1531 "can't add %d slice sysctl node\n", slice);
1532 return; /* XXX continue? */
1533 }
1534 children = SYSCTL_CHILDREN(ss->sysctl_tree);
1535
1536 /*
1537 * XXX change to ULONG
1538 */
1539
1540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_small_cnt",
1541 CTLFLAG_RD, &ss->rx_data.rx_small.cnt, 0, "rx_small_cnt");
1542
1543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_big_cnt",
1544 CTLFLAG_RD, &ss->rx_data.rx_big.cnt, 0, "rx_small_cnt");
1545
1546 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_req",
1547 CTLFLAG_RD, &ss->tx.req, 0, "tx_req");
1548
1549 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_done",
1550 CTLFLAG_RD, &ss->tx.done, 0, "tx_done");
1551
1552 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_pkt_done",
1553 CTLFLAG_RD, &ss->tx.pkt_done, 0, "tx_done");
1554
1555 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_queue_active",
1556 CTLFLAG_RD, &ss->tx.queue_active, 0, "tx_queue_active");
1557
1558 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_activate",
1559 CTLFLAG_RD, &ss->tx.activate, 0, "tx_activate");
1560
1561 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_deactivate",
1562 CTLFLAG_RD, &ss->tx.deactivate, 0, "tx_deactivate");
1563 }
1564 }
1565
1566 /*
1567 * Copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1568 * backwards one at a time and handle ring wraps
1569 */
1570 static __inline void
1571 mxge_submit_req_backwards(mxge_tx_ring_t *tx,
1572 mcp_kreq_ether_send_t *src, int cnt)
1573 {
1574 int idx, starting_slot;
1575
1576 starting_slot = tx->req;
1577 while (cnt > 1) {
1578 cnt--;
1579 idx = (starting_slot + cnt) & tx->mask;
1580 mxge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src));
1581 wmb();
1582 }
1583 }
1584
1585 /*
1586 * Copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1587 * at most 32 bytes at a time, so as to avoid involving the software
1588 * pio handler in the nic. We re-write the first segment's flags
1589 * to mark them valid only after writing the entire chain
1590 */
1591 static __inline void
1592 mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, int cnt)
1593 {
1594 int idx, i;
1595 uint32_t *src_ints;
1596 volatile uint32_t *dst_ints;
1597 mcp_kreq_ether_send_t *srcp;
1598 volatile mcp_kreq_ether_send_t *dstp, *dst;
1599 uint8_t last_flags;
1600
1601 idx = tx->req & tx->mask;
1602
1603 last_flags = src->flags;
1604 src->flags = 0;
1605 wmb();
1606 dst = dstp = &tx->lanai[idx];
1607 srcp = src;
1608
1609 if ((idx + cnt) < tx->mask) {
1610 for (i = 0; i < cnt - 1; i += 2) {
1611 mxge_pio_copy(dstp, srcp, 2 * sizeof(*src));
1612 wmb(); /* force write every 32 bytes */
1613 srcp += 2;
1614 dstp += 2;
1615 }
1616 } else {
1617 /*
1618 * Submit all but the first request, and ensure
1619 * that it is submitted below
1620 */
1621 mxge_submit_req_backwards(tx, src, cnt);
1622 i = 0;
1623 }
1624 if (i < cnt) {
1625 /* Submit the first request */
1626 mxge_pio_copy(dstp, srcp, sizeof(*src));
1627 wmb(); /* barrier before setting valid flag */
1628 }
1629
1630 /* Re-write the last 32-bits with the valid flags */
1631 src->flags = last_flags;
1632 src_ints = (uint32_t *)src;
1633 src_ints+=3;
1634 dst_ints = (volatile uint32_t *)dst;
1635 dst_ints+=3;
1636 *dst_ints = *src_ints;
1637 tx->req += cnt;
1638 wmb();
1639 }
1640
1641 static int
1642 mxge_pullup_tso(struct mbuf **mp)
1643 {
1644 int hoff, iphlen, thoff;
1645 struct mbuf *m;
1646
1647 m = *mp;
1648 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
1649
1650 iphlen = m->m_pkthdr.csum_iphlen;
1651 thoff = m->m_pkthdr.csum_thlen;
1652 hoff = m->m_pkthdr.csum_lhlen;
1653
1654 KASSERT(iphlen > 0, ("invalid ip hlen"));
1655 KASSERT(thoff > 0, ("invalid tcp hlen"));
1656 KASSERT(hoff > 0, ("invalid ether hlen"));
1657
1658 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
1659 m = m_pullup(m, hoff + iphlen + thoff);
1660 if (m == NULL) {
1661 *mp = NULL;
1662 return ENOBUFS;
1663 }
1664 *mp = m;
1665 }
1666 return 0;
1667 }
1668
1669 static int
1670 mxge_encap_tso(mxge_tx_ring_t *tx, struct mxge_buffer_state *info_map,
1671 struct mbuf *m, int busdma_seg_cnt)
1672 {
1673 mcp_kreq_ether_send_t *req;
1674 bus_dma_segment_t *seg;
1675 uint32_t low, high_swapped;
1676 int len, seglen, cum_len, cum_len_next;
1677 int next_is_first, chop, cnt, rdma_count, small;
1678 uint16_t pseudo_hdr_offset, cksum_offset, mss;
1679 uint8_t flags, flags_next;
1680 struct mxge_buffer_state *info_last;
1681 bus_dmamap_t map = info_map->map;
1682
1683 mss = m->m_pkthdr.tso_segsz;
1684
1685 /*
1686 * Negative cum_len signifies to the send loop that we are
1687 * still in the header portion of the TSO packet.
1688 */
1689 cum_len = -(m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen +
1690 m->m_pkthdr.csum_thlen);
1691
1692 /*
1693 * TSO implies checksum offload on this hardware
1694 */
1695 cksum_offset = m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen;
1696 flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1697
1698 /*
1699 * For TSO, pseudo_hdr_offset holds mss. The firmware figures
1700 * out where to put the checksum by parsing the header.
1701 */
1702 pseudo_hdr_offset = htobe16(mss);
1703
1704 req = tx->req_list;
1705 seg = tx->seg_list;
1706 cnt = 0;
1707 rdma_count = 0;
1708
1709 /*
1710 * "rdma_count" is the number of RDMAs belonging to the current
1711 * packet BEFORE the current send request. For non-TSO packets,
1712 * this is equal to "count".
1713 *
1714 * For TSO packets, rdma_count needs to be reset to 0 after a
1715 * segment cut.
1716 *
1717 * The rdma_count field of the send request is the number of
1718 * RDMAs of the packet starting at that request. For TSO send
1719 * requests with one ore more cuts in the middle, this is the
1720 * number of RDMAs starting after the last cut in the request.
1721 * All previous segments before the last cut implicitly have 1
1722 * RDMA.
1723 *
1724 * Since the number of RDMAs is not known beforehand, it must be
1725 * filled-in retroactively - after each segmentation cut or at
1726 * the end of the entire packet.
1727 */
1728
1729 while (busdma_seg_cnt) {
1730 /*
1731 * Break the busdma segment up into pieces
1732 */
1733 low = MXGE_LOWPART_TO_U32(seg->ds_addr);
1734 high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1735 len = seg->ds_len;
1736
1737 while (len) {
1738 flags_next = flags & ~MXGEFW_FLAGS_FIRST;
1739 seglen = len;
1740 cum_len_next = cum_len + seglen;
1741 (req - rdma_count)->rdma_count = rdma_count + 1;
1742 if (__predict_true(cum_len >= 0)) {
1743 /* Payload */
1744 chop = (cum_len_next > mss);
1745 cum_len_next = cum_len_next % mss;
1746 next_is_first = (cum_len_next == 0);
1747 flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
1748 flags_next |=
1749 next_is_first * MXGEFW_FLAGS_FIRST;
1750 rdma_count |= -(chop | next_is_first);
1751 rdma_count += chop & !next_is_first;
1752 } else if (cum_len_next >= 0) {
1753 /* Header ends */
1754 rdma_count = -1;
1755 cum_len_next = 0;
1756 seglen = -cum_len;
1757 small = (mss <= MXGEFW_SEND_SMALL_SIZE);
1758 flags_next = MXGEFW_FLAGS_TSO_PLD |
1759 MXGEFW_FLAGS_FIRST |
1760 (small * MXGEFW_FLAGS_SMALL);
1761 }
1762
1763 req->addr_high = high_swapped;
1764 req->addr_low = htobe32(low);
1765 req->pseudo_hdr_offset = pseudo_hdr_offset;
1766 req->pad = 0;
1767 req->rdma_count = 1;
1768 req->length = htobe16(seglen);
1769 req->cksum_offset = cksum_offset;
1770 req->flags =
1771 flags | ((cum_len & 1) * MXGEFW_FLAGS_ALIGN_ODD);
1772 low += seglen;
1773 len -= seglen;
1774 cum_len = cum_len_next;
1775 flags = flags_next;
1776 req++;
1777 cnt++;
1778 rdma_count++;
1779 if (__predict_false(cksum_offset > seglen))
1780 cksum_offset -= seglen;
1781 else
1782 cksum_offset = 0;
1783 if (__predict_false(cnt > tx->max_desc))
1784 goto drop;
1785 }
1786 busdma_seg_cnt--;
1787 seg++;
1788 }
1789 (req - rdma_count)->rdma_count = rdma_count;
1790
1791 do {
1792 req--;
1793 req->flags |= MXGEFW_FLAGS_TSO_LAST;
1794 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST)));
1795
1796 info_last = &tx->info[((cnt - 1) + tx->req) & tx->mask];
1797
1798 info_map->map = info_last->map;
1799 info_last->map = map;
1800 info_last->m = m;
1801
1802 mxge_submit_req(tx, tx->req_list, cnt);
1803
1804 if (tx->send_go != NULL && tx->queue_active == 0) {
1805 /* Tell the NIC to start polling this slice */
1806 *tx->send_go = 1;
1807 tx->queue_active = 1;
1808 tx->activate++;
1809 wmb();
1810 }
1811 return 0;
1812
1813 drop:
1814 bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
1815 m_freem(m);
1816 return ENOBUFS;
1817 }
1818
1819 static int
1820 mxge_encap(mxge_tx_ring_t *tx, struct mbuf *m, bus_addr_t zeropad)
1821 {
1822 mcp_kreq_ether_send_t *req;
1823 bus_dma_segment_t *seg;
1824 bus_dmamap_t map;
1825 int cnt, cum_len, err, i, idx, odd_flag;
1826 uint16_t pseudo_hdr_offset;
1827 uint8_t flags, cksum_offset;
1828 struct mxge_buffer_state *info_map, *info_last;
1829
1830 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1831 err = mxge_pullup_tso(&m);
1832 if (__predict_false(err))
1833 return err;
1834 }
1835
1836 /*
1837 * Map the frame for DMA
1838 */
1839 idx = tx->req & tx->mask;
1840 info_map = &tx->info[idx];
1841 map = info_map->map;
1842
1843 err = bus_dmamap_load_mbuf_defrag(tx->dmat, map, &m,
1844 tx->seg_list, tx->max_desc - 2, &cnt, BUS_DMA_NOWAIT);
1845 if (__predict_false(err != 0))
1846 goto drop;
1847 bus_dmamap_sync(tx->dmat, map, BUS_DMASYNC_PREWRITE);
1848
1849 /*
1850 * TSO is different enough, we handle it in another routine
1851 */
1852 if (m->m_pkthdr.csum_flags & CSUM_TSO)
1853 return mxge_encap_tso(tx, info_map, m, cnt);
1854
1855 req = tx->req_list;
1856 cksum_offset = 0;
1857 pseudo_hdr_offset = 0;
1858 flags = MXGEFW_FLAGS_NO_TSO;
1859
1860 /*
1861 * Checksum offloading
1862 */
1863 if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
1864 cksum_offset = m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen;
1865 pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data;
1866 pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
1867 req->cksum_offset = cksum_offset;
1868 flags |= MXGEFW_FLAGS_CKSUM;
1869 odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
1870 } else {
1871 odd_flag = 0;
1872 }
1873 if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE)
1874 flags |= MXGEFW_FLAGS_SMALL;
1875
1876 /*
1877 * Convert segments into a request list
1878 */
1879 cum_len = 0;
1880 seg = tx->seg_list;
1881 req->flags = MXGEFW_FLAGS_FIRST;
1882 for (i = 0; i < cnt; i++) {
1883 req->addr_low = htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
1884 req->addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1885 req->length = htobe16(seg->ds_len);
1886 req->cksum_offset = cksum_offset;
1887 if (cksum_offset > seg->ds_len)
1888 cksum_offset -= seg->ds_len;
1889 else
1890 cksum_offset = 0;
1891 req->pseudo_hdr_offset = pseudo_hdr_offset;
1892 req->pad = 0; /* complete solid 16-byte block */
1893 req->rdma_count = 1;
1894 req->flags |= flags | ((cum_len & 1) * odd_flag);
1895 cum_len += seg->ds_len;
1896 seg++;
1897 req++;
1898 req->flags = 0;
1899 }
1900 req--;
1901
1902 /*
1903 * Pad runt to 60 bytes
1904 */
1905 if (cum_len < 60) {
1906 req++;
1907 req->addr_low = htobe32(MXGE_LOWPART_TO_U32(zeropad));
1908 req->addr_high = htobe32(MXGE_HIGHPART_TO_U32(zeropad));
1909 req->length = htobe16(60 - cum_len);
1910 req->cksum_offset = 0;
1911 req->pseudo_hdr_offset = pseudo_hdr_offset;
1912 req->pad = 0; /* complete solid 16-byte block */
1913 req->rdma_count = 1;
1914 req->flags |= flags | ((cum_len & 1) * odd_flag);
1915 cnt++;
1916 }
1917
1918 tx->req_list[0].rdma_count = cnt;
1919 #if 0
1920 /* print what the firmware will see */
1921 for (i = 0; i < cnt; i++) {
1922 kprintf("%d: addr: 0x%x 0x%x len:%d pso%d,"
1923 "cso:%d, flags:0x%x, rdma:%d\n",
1924 i, (int)ntohl(tx->req_list[i].addr_high),
1925 (int)ntohl(tx->req_list[i].addr_low),
1926 (int)ntohs(tx->req_list[i].length),
1927 (int)ntohs(tx->req_list[i].pseudo_hdr_offset),
1928 tx->req_list[i].cksum_offset, tx->req_list[i].flags,
1929 tx->req_list[i].rdma_count);
1930 }
1931 kprintf("--------------\n");
1932 #endif
1933 info_last = &tx->info[((cnt - 1) + tx->req) & tx->mask];
1934
1935 info_map->map = info_last->map;
1936 info_last->map = map;
1937 info_last->m = m;
1938
1939 mxge_submit_req(tx, tx->req_list, cnt);
1940
1941 if (tx->send_go != NULL && tx->queue_active == 0) {
1942 /* Tell the NIC to start polling this slice */
1943 *tx->send_go = 1;
1944 tx->queue_active = 1;
1945 tx->activate++;
1946 wmb();
1947 }
1948 return 0;
1949
1950 drop:
1951 m_freem(m);
1952 return err;
1953 }
1954
1955 static void
1956 mxge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1957 {
1958 mxge_softc_t *sc = ifp->if_softc;
1959 mxge_tx_ring_t *tx = ifsq_get_priv(ifsq);
1960 bus_addr_t zeropad;
1961 int encap = 0;
1962
1963 KKASSERT(tx->ifsq == ifsq);
1964 ASSERT_SERIALIZED(&tx->tx_serialize);
1965
1966 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
1967 return;
1968
1969 zeropad = sc->zeropad_dma.dmem_busaddr;
1970 while (tx->mask - (tx->req - tx->done) > tx->max_desc) {
1971 struct mbuf *m;
1972 int error;
1973
1974 m = ifsq_dequeue(ifsq);
1975 if (m == NULL)
1976 goto done;
1977
1978 BPF_MTAP(ifp, m);
1979 error = mxge_encap(tx, m, zeropad);
1980 if (!error)
1981 encap = 1;
1982 else
1983 IFNET_STAT_INC(ifp, oerrors, 1);
1984 }
1985
1986 /* Ran out of transmit slots */
1987 ifsq_set_oactive(ifsq);
1988 done:
1989 if (encap)
1990 ifsq_watchdog_set_count(&tx->watchdog, 5);
1991 }
1992
1993 static void
1994 mxge_watchdog(struct ifaltq_subque *ifsq)
1995 {
1996 struct ifnet *ifp = ifsq_get_ifp(ifsq);
1997 struct mxge_softc *sc = ifp->if_softc;
1998 uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
1999 mxge_tx_ring_t *tx = ifsq_get_priv(ifsq);
2000
2001 ASSERT_IFNET_SERIALIZED_ALL(ifp);
2002
2003 /* Check for pause blocking before resetting */
2004 if (tx->watchdog_rx_pause == rx_pause) {
2005 mxge_warn_stuck(sc, tx, 0);
2006 mxge_watchdog_reset(sc);
2007 return;
2008 } else {
2009 if_printf(ifp, "Flow control blocking xmits, "
2010 "check link partner\n");
2011 }
2012 tx->watchdog_rx_pause = rx_pause;
2013 }
2014
2015 /*
2016 * Copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy
2017 * at most 32 bytes at a time, so as to avoid involving the software
2018 * pio handler in the nic. We re-write the first segment's low
2019 * DMA address to mark it valid only after we write the entire chunk
2020 * in a burst
2021 */
2022 static __inline void
2023 mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst,
2024 mcp_kreq_ether_recv_t *src)
2025 {
2026 uint32_t low;
2027
2028 low = src->addr_low;
2029 src->addr_low = 0xffffffff;
2030 mxge_pio_copy(dst, src, 4 * sizeof (*src));
2031 wmb();
2032 mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
2033 wmb();
2034 src->addr_low = low;
2035 dst->addr_low = low;
2036 wmb();
2037 }
2038
2039 static int
2040 mxge_get_buf_small(mxge_rx_ring_t *rx, bus_dmamap_t map, int idx,
2041 boolean_t init)
2042 {
2043 bus_dma_segment_t seg;
2044 struct mbuf *m;
2045 int cnt, err, mflag;
2046
2047 mflag = M_NOWAIT;
2048 if (__predict_false(init))
2049 mflag = M_WAITOK;
2050
2051 m = m_gethdr(mflag, MT_DATA);
2052 if (m == NULL) {
2053 err = ENOBUFS;
2054 if (__predict_false(init)) {
2055 /*
2056 * During initialization, there
2057 * is nothing to setup; bail out
2058 */
2059 return err;
2060 }
2061 goto done;
2062 }
2063 m->m_len = m->m_pkthdr.len = MHLEN;
2064
2065 err = bus_dmamap_load_mbuf_segment(rx->dmat, map, m,
2066 &seg, 1, &cnt, BUS_DMA_NOWAIT);
2067 if (err != 0) {
2068 m_freem(m);
2069 if (__predict_false(init)) {
2070 /*
2071 * During initialization, there
2072 * is nothing to setup; bail out
2073 */
2074 return err;
2075 }
2076 goto done;
2077 }
2078
2079 rx->info[idx].m = m;
2080 rx->shadow[idx].addr_low = htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr));
2081 rx->shadow[idx].addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr));
2082
2083 done:
2084 if ((idx & 7) == 7)
2085 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2086 return err;
2087 }
2088
2089 static int
2090 mxge_get_buf_big(mxge_rx_ring_t *rx, bus_dmamap_t map, int idx,
2091 boolean_t init)
2092 {
2093 bus_dma_segment_t seg;
2094 struct mbuf *m;
2095 int cnt, err, mflag;
2096
2097 mflag = M_NOWAIT;
2098 if (__predict_false(init))
2099 mflag = M_WAITOK;
2100
2101 if (rx->cl_size == MCLBYTES)
2102 m = m_getcl(mflag, MT_DATA, M_PKTHDR);
2103 else
2104 m = m_getjcl(mflag, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
2105 if (m == NULL) {
2106 err = ENOBUFS;
2107 if (__predict_false(init)) {
2108 /*
2109 * During initialization, there
2110 * is nothing to setup; bail out
2111 */
2112 return err;
2113 }
2114 goto done;
2115 }
2116 m->m_len = m->m_pkthdr.len = rx->cl_size;
2117
2118 err = bus_dmamap_load_mbuf_segment(rx->dmat, map, m,
2119 &seg, 1, &cnt, BUS_DMA_NOWAIT);
2120 if (err != 0) {
2121 m_freem(m);
2122 if (__predict_false(init)) {
2123 /*
2124 * During initialization, there
2125 * is nothing to setup; bail out
2126 */
2127 return err;
2128 }
2129 goto done;
2130 }
2131
2132 rx->info[idx].m = m;
2133 rx->shadow[idx].addr_low = htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr));
2134 rx->shadow[idx].addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr));
2135
2136 done:
2137 if ((idx & 7) == 7)
2138 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2139 return err;
2140 }
2141
2142 /*
2143 * Myri10GE hardware checksums are not valid if the sender
2144 * padded the frame with non-zero padding. This is because
2145 * the firmware just does a simple 16-bit 1s complement
2146 * checksum across the entire frame, excluding the first 14
2147 * bytes. It is best to simply to check the checksum and
2148 * tell the stack about it only if the checksum is good
2149 */
2150 static __inline uint16_t
2151 mxge_rx_csum(struct mbuf *m, int csum)
2152 {
2153 const struct ether_header *eh;
2154 const struct ip *ip;
2155 uint16_t c;
2156
2157 eh = mtod(m, const struct ether_header *);
2158
2159 /* Only deal with IPv4 TCP & UDP for now */
2160 if (__predict_false(eh->ether_type != htons(ETHERTYPE_IP)))
2161 return 1;
2162
2163 ip = (const struct ip *)(eh + 1);
2164 if (__predict_false(ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP))
2165 return 1;
2166
2167 #ifdef INET
2168 c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2169 htonl(ntohs(csum) + ntohs(ip->ip_len) +
2170 - (ip->ip_hl << 2) + ip->ip_p));
2171 #else
2172 c = 1;
2173 #endif
2174 c ^= 0xffff;
2175 return c;
2176 }
2177
2178 static void
2179 mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum)
2180 {
2181 struct ether_vlan_header *evl;
2182 uint32_t partial;
2183
2184 evl = mtod(m, struct ether_vlan_header *);
2185
2186 /*
2187 * Fix checksum by subtracting EVL_ENCAPLEN bytes after
2188 * what the firmware thought was the end of the ethernet
2189 * header.
2190 */
2191
2192 /* Put checksum into host byte order */
2193 *csum = ntohs(*csum);
2194
2195 partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN));
2196 *csum += ~partial;
2197 *csum += ((*csum) < ~partial);
2198 *csum = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2199 *csum = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2200
2201 /*
2202 * Restore checksum to network byte order;
2203 * later consumers expect this
2204 */
2205 *csum = htons(*csum);
2206
2207 /* save the tag */
2208 m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
2209 m->m_flags |= M_VLANTAG;
2210
2211 /*
2212 * Remove the 802.1q header by copying the Ethernet
2213 * addresses over it and adjusting the beginning of
2214 * the data in the mbuf. The encapsulated Ethernet
2215 * type field is already in place.
2216 */
2217 bcopy((char *)evl, (char *)evl + EVL_ENCAPLEN,
2218 ETHER_HDR_LEN - ETHER_TYPE_LEN);
2219 m_adj(m, EVL_ENCAPLEN);
2220 }
2221
2222
2223 static __inline void
2224 mxge_rx_done_big(struct ifnet *ifp, mxge_rx_ring_t *rx,
2225 uint32_t len, uint32_t csum)
2226 {
2227 struct mbuf *m;
2228 const struct ether_header *eh;
2229 bus_dmamap_t old_map;
2230 int idx;
2231
2232 idx = rx->cnt & rx->mask;
2233 rx->cnt++;
2234
2235 /* Save a pointer to the received mbuf */
2236 m = rx->info[idx].m;
2237
2238 /* Try to replace the received mbuf */
2239 if (mxge_get_buf_big(rx, rx->extra_map, idx, FALSE)) {
2240 /* Drop the frame -- the old mbuf is re-cycled */
2241 IFNET_STAT_INC(ifp, ierrors, 1);
2242 return;
2243 }
2244
2245 /* Unmap the received buffer */
2246 old_map = rx->info[idx].map;
2247 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2248 bus_dmamap_unload(rx->dmat, old_map);
2249
2250 /* Swap the bus_dmamap_t's */
2251 rx->info[idx].map = rx->extra_map;
2252 rx->extra_map = old_map;
2253
2254 /*
2255 * mcp implicitly skips 1st 2 bytes so that packet is properly
2256 * aligned
2257 */
2258 m->m_data += MXGEFW_PAD;
2259
2260 m->m_pkthdr.rcvif = ifp;
2261 m->m_len = m->m_pkthdr.len = len;
2262
2263 IFNET_STAT_INC(ifp, ipackets, 1);
2264
2265 eh = mtod(m, const struct ether_header *);
2266 if (eh->ether_type == htons(ETHERTYPE_VLAN))
2267 mxge_vlan_tag_remove(m, &csum);
2268
2269 /* If the checksum is valid, mark it in the mbuf header */
2270 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2271 mxge_rx_csum(m, csum) == 0) {
2272 /* Tell the stack that the checksum is good */
2273 m->m_pkthdr.csum_data = 0xffff;
2274 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
2275 CSUM_DATA_VALID;
2276 }
2277 ifp->if_input(ifp, m, NULL, -1);
2278 }
2279
2280 static __inline void
2281 mxge_rx_done_small(struct ifnet *ifp, mxge_rx_ring_t *rx,
2282 uint32_t len, uint32_t csum)
2283 {
2284 const struct ether_header *eh;
2285 struct mbuf *m;
2286 bus_dmamap_t old_map;
2287 int idx;
2288
2289 idx = rx->cnt & rx->mask;
2290 rx->cnt++;
2291
2292 /* Save a pointer to the received mbuf */
2293 m = rx->info[idx].m;
2294
2295 /* Try to replace the received mbuf */
2296 if (mxge_get_buf_small(rx, rx->extra_map, idx, FALSE)) {
2297 /* Drop the frame -- the old mbuf is re-cycled */
2298 IFNET_STAT_INC(ifp, ierrors, 1);
2299 return;
2300 }
2301
2302 /* Unmap the received buffer */
2303 old_map = rx->info[idx].map;
2304 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2305 bus_dmamap_unload(rx->dmat, old_map);
2306
2307 /* Swap the bus_dmamap_t's */
2308 rx->info[idx].map = rx->extra_map;
2309 rx->extra_map = old_map;
2310
2311 /*
2312 * mcp implicitly skips 1st 2 bytes so that packet is properly
2313 * aligned
2314 */
2315 m->m_data += MXGEFW_PAD;
2316
2317 m->m_pkthdr.rcvif = ifp;
2318 m->m_len = m->m_pkthdr.len = len;
2319
2320 IFNET_STAT_INC(ifp, ipackets, 1);
2321
2322 eh = mtod(m, const struct ether_header *);
2323 if (eh->ether_type == htons(ETHERTYPE_VLAN))
2324 mxge_vlan_tag_remove(m, &csum);
2325
2326 /* If the checksum is valid, mark it in the mbuf header */
2327 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2328 mxge_rx_csum(m, csum) == 0) {
2329 /* Tell the stack that the checksum is good */
2330 m->m_pkthdr.csum_data = 0xffff;
2331 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
2332 CSUM_DATA_VALID;
2333 }
2334 ifp->if_input(ifp, m, NULL, -1);
2335 }
2336
2337 static __inline void
2338 mxge_clean_rx_done(struct ifnet *ifp, struct mxge_rx_data *rx_data, int cycle)
2339 {
2340 mxge_rx_done_t *rx_done = &rx_data->rx_done;
2341
2342 while (rx_done->entry[rx_done->idx].length != 0 && cycle != 0) {
2343 uint16_t length, checksum;
2344
2345 length = ntohs(rx_done->entry[rx_done->idx].length);
2346 rx_done->entry[rx_done->idx].length = 0;
2347
2348 checksum = rx_done->entry[rx_done->idx].checksum;
2349
2350 if (length <= MXGE_RX_SMALL_BUFLEN) {
2351 mxge_rx_done_small(ifp, &rx_data->rx_small,
2352 length, checksum);
2353 } else {
2354 mxge_rx_done_big(ifp, &rx_data->rx_big,
2355 length, checksum);
2356 }
2357
2358 rx_done->idx++;
2359 rx_done->idx &= rx_done->mask;
2360 --cycle;
2361 }
2362 }
2363
2364 static __inline void
2365 mxge_tx_done(struct ifnet *ifp, mxge_tx_ring_t *tx, uint32_t mcp_idx)
2366 {
2367 ASSERT_SERIALIZED(&tx->tx_serialize);
2368
2369 while (tx->pkt_done != mcp_idx) {
2370 struct mbuf *m;
2371 int idx;
2372
2373 idx = tx->done & tx->mask;
2374 tx->done++;
2375
2376 m = tx->info[idx].m;
2377 /*
2378 * mbuf and DMA map only attached to the first
2379 * segment per-mbuf.
2380 */
2381 if (m != NULL) {
2382 tx->pkt_done++;
2383 IFNET_STAT_INC(ifp, opackets, 1);
2384 tx->info[idx].m = NULL;
2385 bus_dmamap_unload(tx->dmat, tx->info[idx].map);
2386 m_freem(m);
2387 }
2388 }
2389
2390 /*
2391 * If we have space, clear OACTIVE to tell the stack that
2392 * its OK to send packets
2393 */
2394 if (tx->req - tx->done < (tx->mask + 1) / 2) {
2395 ifsq_clr_oactive(tx->ifsq);
2396 if (tx->req == tx->done) {
2397 /* Reset watchdog */
2398 ifsq_watchdog_set_count(&tx->watchdog, 0);
2399 }
2400 }
2401
2402 if (!ifsq_is_empty(tx->ifsq))
2403 ifsq_devstart(tx->ifsq);
2404
2405 if (tx->send_stop != NULL && tx->req == tx->done) {
2406 /*
2407 * Let the NIC stop polling this queue, since there
2408 * are no more transmits pending
2409 */
2410 *tx->send_stop = 1;
2411 tx->queue_active = 0;
2412 tx->deactivate++;
2413 wmb();
2414 }
2415 }
2416
2417 static struct mxge_media_type mxge_xfp_media_types[] = {
2418 {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"},
2419 {IFM_10G_SR, (1 << 7), "10GBASE-SR"},
2420 {IFM_10G_LR, (1 << 6), "10GBASE-LR"},
2421 {IFM_NONE, (1 << 5), "10GBASE-ER"},
2422 {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"},
2423 {IFM_NONE, (1 << 3), "10GBASE-SW"},
2424 {IFM_NONE, (1 << 2), "10GBASE-LW"},
2425 {IFM_NONE, (1 << 1), "10GBASE-EW"},
2426 {IFM_NONE, (1 << 0), "Reserved"}
2427 };
2428
2429 static struct mxge_media_type mxge_sfp_media_types[] = {
2430 {IFM_10G_TWINAX, 0, "10GBASE-Twinax"},
2431 {IFM_NONE, (1 << 7), "Reserved"},
2432 {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"},
2433 {IFM_10G_LR, (1 << 5), "10GBASE-LR"},
2434 {IFM_10G_SR, (1 << 4), "10GBASE-SR"},
2435 {IFM_10G_TWINAX,(1 << 0), "10GBASE-Twinax"}
2436 };
2437
2438 static void
2439 mxge_media_set(mxge_softc_t *sc, int media_type)
2440 {
2441 int fc_opt = 0;
2442
2443 if (media_type == IFM_NONE)
2444 return;
2445
2446 if (sc->pause)
2447 fc_opt = IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE;
2448
2449 ifmedia_add(&sc->media, MXGE_IFM | media_type, 0, NULL);
2450 ifmedia_set(&sc->media, MXGE_IFM | media_type | fc_opt);
2451
2452 sc->current_media = media_type;
2453 }
2454
2455 static void
2456 mxge_media_unset(mxge_softc_t *sc)
2457 {
2458 ifmedia_removeall(&sc->media);
2459 sc->current_media = IFM_NONE;
2460 }
2461
2462 static void
2463 mxge_media_init(mxge_softc_t *sc)
2464 {
2465 const char *ptr;
2466 int i;
2467
2468 mxge_media_unset(sc);
2469
2470 /*
2471 * Parse the product code to deterimine the interface type
2472 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
2473 * after the 3rd dash in the driver's cached copy of the
2474 * EEPROM's product code string.
2475 */
2476 ptr = sc->product_code_string;
2477 if (ptr == NULL) {
2478 if_printf(sc->ifp, "Missing product code\n");
2479 return;
2480 }
2481
2482 for (i = 0; i < 3; i++, ptr++) {
2483 ptr = strchr(ptr, '-');
2484 if (ptr == NULL) {
2485 if_printf(sc->ifp, "only %d dashes in PC?!?\n", i);
2486 return;
2487 }
2488 }
2489 if (*ptr == 'C' || *(ptr +1) == 'C') {
2490 /* -C is CX4 */
2491 sc->connector = MXGE_CX4;
2492 mxge_media_set(sc, IFM_10G_CX4);
2493 } else if (*ptr == 'Q') {
2494 /* -Q is Quad Ribbon Fiber */
2495 sc->connector = MXGE_QRF;
2496 if_printf(sc->ifp, "Quad Ribbon Fiber Media\n");
2497 /* DragonFly has no media type for Quad ribbon fiber */
2498 } else if (*ptr == 'R') {
2499 /* -R is XFP */
2500 sc->connector = MXGE_XFP;
2501 /* NOTE: ifmedia will be installed later */
2502 } else if (*ptr == 'S' || *(ptr +1) == 'S') {
2503 /* -S or -2S is SFP+ */
2504 sc->connector = MXGE_SFP;
2505 /* NOTE: ifmedia will be installed later */
2506 } else {
2507 sc->connector = MXGE_UNK;
2508 if_printf(sc->ifp, "Unknown media type: %c\n", *ptr);
2509 }
2510 }
2511
2512 /*
2513 * Determine the media type for a NIC. Some XFPs will identify
2514 * themselves only when their link is up, so this is initiated via a
2515 * link up interrupt. However, this can potentially take up to
2516 * several milliseconds, so it is run via the watchdog routine, rather
2517 * than in the interrupt handler itself.
2518 */
2519 static void
2520 mxge_media_probe(mxge_softc_t *sc)
2521 {
2522 mxge_cmd_t cmd;
2523 const char *cage_type;
2524 struct mxge_media_type *mxge_media_types = NULL;
2525 int i, err, ms, mxge_media_type_entries;
2526 uint32_t byte;
2527
2528 sc->need_media_probe = 0;
2529
2530 if (sc->connector == MXGE_XFP) {
2531 /* -R is XFP */
2532 mxge_media_types = mxge_xfp_media_types;
2533 mxge_media_type_entries = NELEM(mxge_xfp_media_types);
2534 byte = MXGE_XFP_COMPLIANCE_BYTE;
2535 cage_type = "XFP";
2536 } else if (sc->connector == MXGE_SFP) {
2537 /* -S or -2S is SFP+ */
2538 mxge_media_types = mxge_sfp_media_types;
2539 mxge_media_type_entries = NELEM(mxge_sfp_media_types);
2540 cage_type = "SFP+";
2541 byte = 3;
2542 } else {
2543 /* nothing to do; media type cannot change */
2544 return;
2545 }
2546
2547 /*
2548 * At this point we know the NIC has an XFP cage, so now we
2549 * try to determine what is in the cage by using the
2550 * firmware's XFP I2C commands to read the XFP 10GbE compilance
2551 * register. We read just one byte, which may take over
2552 * a millisecond
2553 */
2554
2555 bzero(&cmd, sizeof(cmd)); /* silence gcc warning */
2556 cmd.data0 = 0; /* just fetch 1 byte, not all 256 */
2557 cmd.data1 = byte;
2558 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd);
2559 if (err != MXGEFW_CMD_OK) {
2560 if (err == MXGEFW_CMD_ERROR_I2C_FAILURE)
2561 if_printf(sc->ifp, "failed to read XFP\n");
2562 else if (err == MXGEFW_CMD_ERROR_I2C_ABSENT)
2563 if_printf(sc->ifp, "Type R/S with no XFP!?!?\n");
2564 else
2565 if_printf(sc->ifp, "I2C read failed, err: %d", err);
2566 mxge_media_unset(sc);
2567 return;
2568 }
2569
2570 /* Now we wait for the data to be cached */
2571 cmd.data0 = byte;
2572 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2573 for (ms = 0; err == EBUSY && ms < 50; ms++) {
2574 DELAY(1000);
2575 cmd.data0 = byte;
2576 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2577 }
2578 if (err != MXGEFW_CMD_OK) {
2579 if_printf(sc->ifp, "failed to read %s (%d, %dms)\n",
2580 cage_type, err, ms);
2581 mxge_media_unset(sc);
2582 return;
2583 }
2584
2585 if (cmd.data0 == mxge_media_types[0].bitmask) {
2586 if (bootverbose) {
2587 if_printf(sc->ifp, "%s:%s\n", cage_type,
2588 mxge_media_types[0].name);
2589 }
2590 if (sc->current_media != mxge_media_types[0].flag) {
2591 mxge_media_unset(sc);
2592 mxge_media_set(sc, mxge_media_types[0].flag);
2593 }
2594 return;
2595 }
2596 for (i = 1; i < mxge_media_type_entries; i++) {
2597 if (cmd.data0 & mxge_media_types[i].bitmask) {
2598 if (bootverbose) {
2599 if_printf(sc->ifp, "%s:%s\n", cage_type,
2600 mxge_media_types[i].name);
2601 }
2602
2603 if (sc->current_media != mxge_media_types[i].flag) {
2604 mxge_media_unset(sc);
2605 mxge_media_set(sc, mxge_media_types[i].flag);
2606 }
2607 return;
2608 }
2609 }
2610 mxge_media_unset(sc);
2611 if (bootverbose) {
2612 if_printf(sc->ifp, "%s media 0x%x unknown\n", cage_type,
2613 cmd.data0);
2614 }
2615 }
2616
2617 static void
2618 mxge_intr_status(struct mxge_softc *sc, const mcp_irq_data_t *stats)
2619 {
2620 if (sc->link_state != stats->link_up) {
2621 sc->link_state = stats->link_up;
2622 if (sc->link_state) {
2623 sc->ifp->if_link_state = LINK_STATE_UP;
2624 if_link_state_change(sc->ifp);
2625 if (bootverbose)
2626 if_printf(sc->ifp, "link up\n");
2627 } else {
2628 sc->ifp->if_link_state = LINK_STATE_DOWN;
2629 if_link_state_change(sc->ifp);
2630 if (bootverbose)
2631 if_printf(sc->ifp, "link down\n");
2632 }
2633 sc->need_media_probe = 1;
2634 }
2635
2636 if (sc->rdma_tags_available != be32toh(stats->rdma_tags_available)) {
2637 sc->rdma_tags_available = be32toh(stats->rdma_tags_available);
2638 if_printf(sc->ifp, "RDMA timed out! %d tags left\n",
2639 sc->rdma_tags_available);
2640 }
2641
2642 if (stats->link_down) {
2643 sc->down_cnt += stats->link_down;
2644 sc->link_state = 0;
2645 sc->ifp->if_link_state = LINK_STATE_DOWN;
2646 if_link_state_change(sc->ifp);
2647 }
2648 }
2649
2650 static void
2651 mxge_serialize_skipmain(struct mxge_softc *sc)
2652 {
2653 lwkt_serialize_array_enter(sc->serializes, sc->nserialize, 1);
2654 }
2655
2656 static void
2657 mxge_deserialize_skipmain(struct mxge_softc *sc)
2658 {
2659 lwkt_serialize_array_exit(sc->serializes, sc->nserialize, 1);
2660 }
2661
2662 static void
2663 mxge_legacy(void *arg)
2664 {
2665 struct mxge_slice_state *ss = arg;
2666 mxge_softc_t *sc = ss->sc;
2667 mcp_irq_data_t *stats = ss->fw_stats;
2668 mxge_tx_ring_t *tx = &ss->tx;
2669 mxge_rx_done_t *rx_done = &ss->rx_data.rx_done;
2670 uint32_t send_done_count;
2671 uint8_t valid;
2672
2673 ASSERT_SERIALIZED(&sc->main_serialize);
2674
2675 /* Make sure the DMA has finished */
2676 if (!stats->valid)
2677 return;
2678 valid = stats->valid;
2679
2680 /* Lower legacy IRQ */
2681 *sc->irq_deassert = 0;
2682 if (!mxge_deassert_wait) {
2683 /* Don't wait for conf. that irq is low */
2684 stats->valid = 0;
2685 }
2686
2687 mxge_serialize_skipmain(sc);
2688
2689 /*
2690 * Loop while waiting for legacy irq deassertion
2691 * XXX do we really want to loop?
2692 */
2693 do {
2694 /* Check for transmit completes and receives */
2695 send_done_count = be32toh(stats->send_done_count);
2696 while ((send_done_count != tx->pkt_done) ||
2697 (rx_done->entry[rx_done->idx].length != 0)) {
2698 if (send_done_count != tx->pkt_done) {
2699 mxge_tx_done(&sc->arpcom.ac_if, tx,
2700 (int)send_done_count);
2701 }
2702 mxge_clean_rx_done(&sc->arpcom.ac_if, &ss->rx_data, -1);
2703 send_done_count = be32toh(stats->send_done_count);
2704 }
2705 if (mxge_deassert_wait)
2706 wmb();
2707 } while (*((volatile uint8_t *)&stats->valid));
2708
2709 mxge_deserialize_skipmain(sc);
2710
2711 /* Fw link & error stats meaningful only on the first slice */
2712 if (__predict_false(stats->stats_updated))
2713 mxge_intr_status(sc, stats);
2714
2715 /* Check to see if we have rx token to pass back */
2716 if (valid & 0x1)
2717 *ss->irq_claim = be32toh(3);
2718 *(ss->irq_claim + 1) = be32toh(3);
2719 }
2720
2721 static void
2722 mxge_msi(void *arg)
2723 {
2724 struct mxge_slice_state *ss = arg;
2725 mxge_softc_t *sc = ss->sc;
2726 mcp_irq_data_t *stats = ss->fw_stats;
2727 mxge_tx_ring_t *tx = &ss->tx;
2728 mxge_rx_done_t *rx_done = &ss->rx_data.rx_done;
2729 uint32_t send_done_count;
2730 uint8_t valid;
2731 #ifndef IFPOLL_ENABLE
2732 const boolean_t polling = FALSE;
2733 #else
2734 boolean_t polling = FALSE;
2735 #endif
2736
2737 ASSERT_SERIALIZED(&sc->main_serialize);
2738
2739 /* Make sure the DMA has finished */
2740 if (__predict_false(!stats->valid))
2741 return;
2742
2743 valid = stats->valid;
2744 stats->valid = 0;
2745
2746 #ifdef IFPOLL_ENABLE
2747 if (sc->arpcom.ac_if.if_flags & IFF_NPOLLING)
2748 polling = TRUE;
2749 #endif
2750
2751 if (!polling) {
2752 /* Check for receives */
2753 lwkt_serialize_enter(&ss->rx_data.rx_serialize);
2754 if (rx_done->entry[rx_done->idx].length != 0)
2755 mxge_clean_rx_done(&sc->arpcom.ac_if, &ss->rx_data, -1);
2756 lwkt_serialize_exit(&ss->rx_data.rx_serialize);
2757 }
2758
2759 /*
2760 * Check for transmit completes
2761 *
2762 * NOTE:
2763 * Since pkt_done is only changed by mxge_tx_done(),
2764 * which is called only in interrupt handler, the
2765 * check w/o holding tx serializer is MPSAFE.
2766 */
2767 send_done_count = be32toh(stats->send_done_count);
2768 if (send_done_count != tx->pkt_done) {
2769 lwkt_serialize_enter(&tx->tx_serialize);
2770 mxge_tx_done(&sc->arpcom.ac_if, tx, (int)send_done_count);
2771 lwkt_serialize_exit(&tx->tx_serialize);
2772 }
2773
2774 if (__predict_false(stats->stats_updated))
2775 mxge_intr_status(sc, stats);
2776
2777 /* Check to see if we have rx token to pass back */
2778 if (!polling && (valid & 0x1))
2779 *ss->irq_claim = be32toh(3);
2780 *(ss->irq_claim + 1) = be32toh(3);
2781 }
2782
2783 static void
2784 mxge_msix_rx(void *arg)
2785 {
2786 struct mxge_slice_state *ss = arg;
2787 mxge_rx_done_t *rx_done = &ss->rx_data.rx_done;
2788
2789 #ifdef IFPOLL_ENABLE
2790 if (ss->sc->arpcom.ac_if.if_flags & IFF_NPOLLING)
2791 return;
2792 #endif
2793
2794 ASSERT_SERIALIZED(&ss->rx_data.rx_serialize);
2795
2796 if (rx_done->entry[rx_done->idx].length != 0)
2797 mxge_clean_rx_done(&ss->sc->arpcom.ac_if, &ss->rx_data, -1);
2798
2799 *ss->irq_claim = be32toh(3);
2800 }
2801
2802 static void
2803 mxge_msix_rxtx(void *arg)
2804 {
2805 struct mxge_slice_state *ss = arg;
2806 mxge_softc_t *sc = ss->sc;
2807 mcp_irq_data_t *stats = ss->fw_stats;
2808 mxge_tx_ring_t *tx = &ss->tx;
2809 mxge_rx_done_t *rx_done = &ss->rx_data.rx_done;
2810 uint32_t send_done_count;
2811 uint8_t valid;
2812 #ifndef IFPOLL_ENABLE
2813 const boolean_t polling = FALSE;
2814 #else
2815 boolean_t polling = FALSE;
2816 #endif
2817
2818 ASSERT_SERIALIZED(&ss->rx_data.rx_serialize);
2819
2820 /* Make sure the DMA has finished */
2821 if (__predict_false(!stats->valid))
2822 return;
2823
2824 valid = stats->valid;
2825 stats->valid = 0;
2826
2827 #ifdef IFPOLL_ENABLE
2828 if (sc->arpcom.ac_if.if_flags & IFF_NPOLLING)
2829 polling = TRUE;
2830 #endif
2831
2832 /* Check for receives */
2833 if (!polling && rx_done->entry[rx_done->idx].length != 0)
2834 mxge_clean_rx_done(&sc->arpcom.ac_if, &ss->rx_data, -1);
2835
2836 /*
2837 * Check for transmit completes
2838 *
2839 * NOTE:
2840 * Since pkt_done is only changed by mxge_tx_done(),
2841 * which is called only in interrupt handler, the
2842 * check w/o holding tx serializer is MPSAFE.
2843 */
2844 send_done_count = be32toh(stats->send_done_count);
2845 if (send_done_count != tx->pkt_done) {
2846 lwkt_serialize_enter(&tx->tx_serialize);
2847 mxge_tx_done(&sc->arpcom.ac_if, tx, (int)send_done_count);
2848 lwkt_serialize_exit(&tx->tx_serialize);
2849 }
2850
2851 /* Check to see if we have rx token to pass back */
2852 if (!polling && (valid & 0x1))
2853 *ss->irq_claim = be32toh(3);
2854 *(ss->irq_claim + 1) = be32toh(3);
2855 }
2856
2857 static void
2858 mxge_init(void *arg)
2859 {
2860 struct mxge_softc *sc = arg;
2861
2862 ASSERT_IFNET_SERIALIZED_ALL(sc->ifp);
2863 if ((sc->ifp->if_flags & IFF_RUNNING) == 0)
2864 mxge_open(sc);
2865 }
2866
2867 static void
2868 mxge_free_slice_mbufs(struct mxge_slice_state *ss)
2869 {
2870 int i;
2871
2872 for (i = 0; i <= ss->rx_data.rx_big.mask; i++) {
2873 if (ss->rx_data.rx_big.info[i].m == NULL)
2874 continue;
2875 bus_dmamap_unload(ss->rx_data.rx_big.dmat,
2876 ss->rx_data.rx_big.info[i].map);
2877 m_freem(ss->rx_data.rx_big.info[i].m);
2878 ss->rx_data.rx_big.info[i].m = NULL;
2879 }
2880
2881 for (i = 0; i <= ss->rx_data.rx_small.mask; i++) {
2882 if (ss->rx_data.rx_small.info[i].m == NULL)
2883 continue;
2884 bus_dmamap_unload(ss->rx_data.rx_small.dmat,
2885 ss->rx_data.rx_small.info[i].map);
2886 m_freem(ss->rx_data.rx_small.info[i].m);
2887 ss->rx_data.rx_small.info[i].m = NULL;
2888 }
2889
2890 /* Transmit ring used only on the first slice */
2891 if (ss->tx.info == NULL)
2892 return;
2893
2894 for (i = 0; i <= ss->tx.mask; i++) {
2895 if (ss->tx.info[i].m == NULL)
2896 continue;
2897 bus_dmamap_unload(ss->tx.dmat, ss->tx.info[i].map);
2898 m_freem(ss->tx.info[i].m);
2899 ss->tx.info[i].m = NULL;
2900 }
2901 }
2902
2903 static void
2904 mxge_free_mbufs(mxge_softc_t *sc)
2905 {
2906 int slice;
2907
2908 for (slice = 0; slice < sc->num_slices; slice++)
2909 mxge_free_slice_mbufs(&sc->ss[slice]);
2910 }
2911
2912 static void
2913 mxge_free_slice_rings(struct mxge_slice_state *ss)
2914 {
2915 int i;
2916
2917 if (ss->rx_data.rx_done.entry != NULL) {
2918 mxge_dma_free(&ss->rx_done_dma);
2919 ss->rx_data.rx_done.entry = NULL;
2920 }
2921
2922 if (ss->tx.req_list != NULL) {
2923 kfree(ss->tx.req_list, M_DEVBUF);
2924 ss->tx.req_list = NULL;
2925 }
2926
2927 if (ss->tx.seg_list != NULL) {
2928 kfree(ss->tx.seg_list, M_DEVBUF);
2929 ss->tx.seg_list = NULL;
2930 }
2931
2932 if (ss->rx_data.rx_small.shadow != NULL) {
2933 kfree(ss->rx_data.rx_small.shadow, M_DEVBUF);
2934 ss->rx_data.rx_small.shadow = NULL;
2935 }
2936
2937 if (ss->rx_data.rx_big.shadow != NULL) {
2938 kfree(ss->rx_data.rx_big.shadow, M_DEVBUF);
2939 ss->rx_data.rx_big.shadow = NULL;
2940 }
2941
2942 if (ss->tx.info != NULL) {
2943 if (ss->tx.dmat != NULL) {
2944 for (i = 0; i <= ss->tx.mask; i++) {
2945 bus_dmamap_destroy(ss->tx.dmat,
2946 ss->tx.info[i].map);
2947 }
2948 bus_dma_tag_destroy(ss->tx.dmat);
2949 }
2950 kfree(ss->tx.info, M_DEVBUF);
2951 ss->tx.info = NULL;
2952 }
2953
2954 if (ss->rx_data.rx_small.info != NULL) {
2955 if (ss->rx_data.rx_small.dmat != NULL) {
2956 for (i = 0; i <= ss->rx_data.rx_small.mask; i++) {
2957 bus_dmamap_destroy(ss->rx_data.rx_small.dmat,
2958 ss->rx_data.rx_small.info[i].map);
2959 }
2960 bus_dmamap_destroy(ss->rx_data.rx_small.dmat,
2961 ss->rx_data.rx_small.extra_map);
2962 bus_dma_tag_destroy(ss->rx_data.rx_small.dmat);
2963 }
2964 kfree(ss->rx_data.rx_small.info, M_DEVBUF);
2965 ss->rx_data.rx_small.info = NULL;
2966 }
2967
2968 if (ss->rx_data.rx_big.info != NULL) {
2969 if (ss->rx_data.rx_big.dmat != NULL) {
2970 for (i = 0; i <= ss->rx_data.rx_big.mask; i++) {
2971 bus_dmamap_destroy(ss->rx_data.rx_big.dmat,
2972 ss->rx_data.rx_big.info[i].map);
2973 }
2974 bus_dmamap_destroy(ss->rx_data.rx_big.dmat,
2975 ss->rx_data.rx_big.extra_map);
2976 bus_dma_tag_destroy(ss->rx_data.rx_big.dmat);
2977 }
2978 kfree(ss->rx_data.rx_big.info, M_DEVBUF);
2979 ss->rx_data.rx_big.info = NULL;
2980 }
2981 }
2982
2983 static void
2984 mxge_free_rings(mxge_softc_t *sc)
2985 {
2986 int slice;
2987
2988 if (sc->ss == NULL)
2989 return;
2990
2991 for (slice = 0; slice < sc->num_slices; slice++)
2992 mxge_free_slice_rings(&sc->ss[slice]);
2993 }
2994
2995 static int
2996 mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries,
2997 int tx_ring_entries)
2998 {
2999 mxge_softc_t *sc = ss->sc;
3000 size_t bytes;
3001 int err, i;
3002
3003 /*
3004 * Allocate per-slice receive resources
3005 */
3006
3007 ss->rx_data.rx_small.mask = ss->rx_data.rx_big.mask =
3008 rx_ring_entries - 1;
3009 ss->rx_data.rx_done.mask = (2 * rx_ring_entries) - 1;
3010
3011 /* Allocate the rx shadow rings */
3012 bytes = rx_ring_entries * sizeof(*ss->rx_data.rx_small.shadow);
3013 ss->rx_data.rx_small.shadow = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3014
3015 bytes = rx_ring_entries * sizeof(*ss->rx_data.rx_big.shadow);
3016 ss->rx_data.rx_big.shadow = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3017
3018 /* Allocate the rx host info rings */
3019 bytes = rx_ring_entries * sizeof(*ss->rx_data.rx_small.info);
3020 ss->rx_data.rx_small.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3021
3022 bytes = rx_ring_entries * sizeof(*ss->rx_data.rx_big.info);
3023 ss->rx_data.rx_big.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3024
3025 /* Allocate the rx busdma resources */
3026 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3027 1, /* alignment */
3028 4096, /* boundary */
3029 BUS_SPACE_MAXADDR, /* low */
3030 BUS_SPACE_MAXADDR, /* high */
3031 MHLEN, /* maxsize */
3032 1, /* num segs */
3033 MHLEN, /* maxsegsize */
3034 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
3035 /* flags */
3036 &ss->rx_data.rx_small.dmat); /* tag */
3037 if (err != 0) {
3038 device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
3039 err);
3040 return err;
3041 }
3042
3043 err = bus_dmamap_create(ss->rx_data.rx_small.dmat, BUS_DMA_WAITOK,
3044 &ss->rx_data.rx_small.extra_map);
3045 if (err != 0) {
3046 device_printf(sc->dev, "Err %d extra rx_small dmamap\n", err);
3047 bus_dma_tag_destroy(ss->rx_data.rx_small.dmat);
3048 ss->rx_data.rx_small.dmat = NULL;
3049 return err;
3050 }
3051 for (i = 0; i <= ss->rx_data.rx_small.mask; i++) {
3052 err = bus_dmamap_create(ss->rx_data.rx_small.dmat,
3053 BUS_DMA_WAITOK, &ss->rx_data.rx_small.info[i].map);
3054 if (err != 0) {
3055 int j;
3056
3057 device_printf(sc->dev, "Err %d rx_small dmamap\n", err);
3058
3059 for (j = 0; j < i; ++j) {
3060 bus_dmamap_destroy(ss->rx_data.rx_small.dmat,
3061 ss->rx_data.rx_small.info[j].map);
3062 }
3063 bus_dmamap_destroy(ss->rx_data.rx_small.dmat,
3064 ss->rx_data.rx_small.extra_map);
3065 bus_dma_tag_destroy(ss->rx_data.rx_small.dmat);
3066 ss->rx_data.rx_small.dmat = NULL;
3067 return err;
3068 }
3069 }
3070
3071 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3072 1, /* alignment */
3073 4096, /* boundary */
3074 BUS_SPACE_MAXADDR, /* low */
3075 BUS_SPACE_MAXADDR, /* high */
3076 4096, /* maxsize */
3077 1, /* num segs */
3078 4096, /* maxsegsize*/
3079 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
3080 /* flags */
3081 &ss->rx_data.rx_big.dmat); /* tag */
3082 if (err != 0) {
3083 device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
3084 err);
3085 return err;
3086 }
3087
3088 err = bus_dmamap_create(ss->rx_data.rx_big.dmat, BUS_DMA_WAITOK,
3089 &ss->rx_data.rx_big.extra_map);
3090 if (err != 0) {
3091 device_printf(sc->dev, "Err %d extra rx_big dmamap\n", err);
3092 bus_dma_tag_destroy(ss->rx_data.rx_big.dmat);
3093 ss->rx_data.rx_big.dmat = NULL;
3094 return err;
3095 }
3096 for (i = 0; i <= ss->rx_data.rx_big.mask; i++) {
3097 err = bus_dmamap_create(ss->rx_data.rx_big.dmat, BUS_DMA_WAITOK,
3098 &ss->rx_data.rx_big.info[i].map);
3099 if (err != 0) {
3100 int j;
3101
3102 device_printf(sc->dev, "Err %d rx_big dmamap\n", err);
3103 for (j = 0; j < i; ++j) {
3104 bus_dmamap_destroy(ss->rx_data.rx_big.dmat,
3105 ss->rx_data.rx_big.info[j].map);
3106 }
3107 bus_dmamap_destroy(ss->rx_data.rx_big.dmat,
3108 ss->rx_data.rx_big.extra_map);
3109 bus_dma_tag_destroy(ss->rx_data.rx_big.dmat);
3110 ss->rx_data.rx_big.dmat = NULL;
3111 return err;
3112 }
3113 }
3114
3115 /*
3116 * Now allocate TX resources
3117 */
3118
3119 ss->tx.mask = tx_ring_entries - 1;
3120 ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
3121
3122 /*
3123 * Allocate the tx request copy block; MUST be at least 8 bytes
3124 * aligned
3125 */
3126 bytes = sizeof(*ss->tx.req_list) * (ss->tx.max_desc + 4);
3127 ss->tx.req_list = kmalloc(__VM_CACHELINE_ALIGN(bytes),
3128 M_DEVBUF,
3129 M_WAITOK | M_CACHEALIGN);
3130
3131 /* Allocate the tx busdma segment list */
3132 bytes = sizeof(*ss->tx.seg_list) * ss->tx.max_desc;
3133 ss->tx.seg_list = kmalloc(bytes, M_DEVBUF, M_WAITOK);
3134
3135 /* Allocate the tx host info ring */
3136 bytes = tx_ring_entries * sizeof(*ss->tx.info);
3137 ss->tx.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3138
3139 /* Allocate the tx busdma resources */
3140 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3141 1, /* alignment */
3142 sc->tx_boundary, /* boundary */
3143 BUS_SPACE_MAXADDR, /* low */
3144 BUS_SPACE_MAXADDR, /* high */
3145 IP_MAXPACKET +
3146 sizeof(struct ether_vlan_header),
3147 /* maxsize */
3148 ss->tx.max_desc - 2, /* num segs */
3149 sc->tx_boundary, /* maxsegsz */
3150 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
3151 BUS_DMA_ONEBPAGE, /* flags */
3152 &ss->tx.dmat); /* tag */
3153 if (err != 0) {
3154 device_printf(sc->dev, "Err %d allocating tx dmat\n", err);
3155 return err;
3156 }
3157
3158 /*
3159 * Now use these tags to setup DMA maps for each slot in the ring
3160 */
3161 for (i = 0; i <= ss->tx.mask; i++) {
3162 err = bus_dmamap_create(ss->tx.dmat,
3163 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &ss->tx.info[i].map);
3164 if (err != 0) {
3165 int j;
3166
3167 device_printf(sc->dev, "Err %d tx dmamap\n", err);
3168 for (j = 0; j < i; ++j) {
3169 bus_dmamap_destroy(ss->tx.dmat,
3170 ss->tx.info[j].map);
3171 }
3172 bus_dma_tag_destroy(ss->tx.dmat);
3173 ss->tx.dmat = NULL;
3174 return err;
3175 }
3176 }
3177 return 0;
3178 }
3179
3180 static int
3181 mxge_alloc_rings(mxge_softc_t *sc)
3182 {
3183 mxge_cmd_t cmd;
3184 int tx_ring_size;
3185 int tx_ring_entries, rx_ring_entries;
3186 int err, slice;
3187
3188 /* Get ring sizes */
3189 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
3190 if (err != 0) {
3191 device_printf(sc->dev, "Cannot determine tx ring sizes\n");
3192 return err;
3193 }
3194 tx_ring_size = cmd.data0;
3195
3196 tx_ring_entries = tx_ring_size / sizeof(mcp_kreq_ether_send_t);
3197 rx_ring_entries = sc->rx_intr_slots / 2;
3198
3199 if (bootverbose) {
3200 device_printf(sc->dev, "tx desc %d, rx desc %d\n",
3201 tx_ring_entries, rx_ring_entries);
3202 }
3203
3204 sc->ifp->if_nmbclusters = rx_ring_entries * sc->num_slices;
3205 sc->ifp->if_nmbjclusters = sc->ifp->if_nmbclusters;
3206
3207 ifq_set_maxlen(&sc->ifp->if_snd, tx_ring_entries - 1);
3208 ifq_set_ready(&sc->ifp->if_snd);
3209 ifq_set_subq_cnt(&sc->ifp->if_snd, sc->num_tx_rings);
3210
3211 if (sc->num_tx_rings > 1) {
3212 sc->ifp->if_mapsubq = ifq_mapsubq_modulo;
3213 ifq_set_subq_divisor(&sc->ifp->if_snd, sc->num_tx_rings);
3214 }
3215
3216 for (slice = 0; slice < sc->num_slices; slice++) {
3217 err = mxge_alloc_slice_rings(&sc->ss[slice],
3218 rx_ring_entries, tx_ring_entries);
3219 if (err != 0) {
3220 device_printf(sc->dev,
3221 "alloc %d slice rings failed\n", slice);
3222 return err;
3223 }
3224 }
3225 return 0;
3226 }
3227
3228 static void
3229 mxge_choose_params(int mtu, int *cl_size)
3230 {
3231 int bufsize = mtu + ETHER_HDR_LEN + EVL_ENCAPLEN + MXGEFW_PAD;
3232
3233 if (bufsize < MCLBYTES) {
3234 *cl_size = MCLBYTES;
3235 } else {
3236 KASSERT(bufsize < MJUMPAGESIZE, ("invalid MTU %d", mtu));
3237 *cl_size = MJUMPAGESIZE;
3238 }
3239 }
3240
3241 static int
3242 mxge_slice_open(struct mxge_slice_state *ss, int cl_size)
3243 {
3244 mxge_cmd_t cmd;
3245 int err, i, slice;
3246
3247 slice = ss - ss->sc->ss;
3248
3249 /*
3250 * Get the lanai pointers to the send and receive rings
3251 */
3252 err = 0;
3253
3254 bzero(&cmd, sizeof(cmd)); /* silence gcc warning */
3255 if (ss->sc->num_tx_rings == 1) {
3256 if (slice == 0) {
3257 cmd.data0 = slice;
3258 err = mxge_send_cmd(ss->sc, MXGEFW_CMD_GET_SEND_OFFSET,
3259 &cmd);
3260 ss->tx.lanai = (volatile mcp_kreq_ether_send_t *)
3261 (ss->sc->sram + cmd.data0);
3262 /* Leave send_go and send_stop as NULL */
3263 }
3264 } else {
3265 cmd.data0 = slice;
3266 err = mxge_send_cmd(ss->sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
3267 ss->tx.lanai = (volatile mcp_kreq_ether_send_t *)
3268 (ss->sc->sram + cmd.data0);
3269 ss->tx.send_go = (volatile uint32_t *)
3270 (ss->sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
3271 ss->tx.send_stop = (volatile uint32_t *)
3272 (ss->sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
3273 }
3274
3275 cmd.data0 = slice;
3276 err |= mxge_send_cmd(ss->sc, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
3277 ss->rx_data.rx_small.lanai =
3278 (volatile mcp_kreq_ether_recv_t *)(ss->sc->sram + cmd.data0);
3279
3280 cmd.data0 = slice;
3281 err |= mxge_send_cmd(ss->sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
3282 ss->rx_data.rx_big.lanai =
3283 (volatile mcp_kreq_ether_recv_t *)(ss->sc->sram + cmd.data0);
3284
3285 if (err != 0) {
3286 if_printf(ss->sc->ifp,
3287 "failed to get ring sizes or locations\n");
3288 return EIO;
3289 }
3290
3291 /*
3292 * Stock small receive ring
3293 */
3294 for (i = 0; i <= ss->rx_data.rx_small.mask; i++) {
3295 err = mxge_get_buf_small(&ss->rx_data.rx_small,
3296 ss->rx_data.rx_small.info[i].map, i, TRUE);
3297 if (err) {
3298 if_printf(ss->sc->ifp, "alloced %d/%d smalls\n", i,
3299 ss->rx_data.rx_small.mask + 1);
3300 return ENOMEM;
3301 }
3302 }
3303
3304 /*
3305 * Stock big receive ring
3306 */
3307 for (i = 0; i <= ss->rx_data.rx_big.mask; i++) {
3308 ss->rx_data.rx_big.shadow[i].addr_low = 0xffffffff;
3309 ss->rx_data.rx_big.shadow[i].addr_high = 0xffffffff;
3310 }
3311
3312 ss->rx_data.rx_big.cl_size = cl_size;
3313
3314 for (i = 0; i <= ss->rx_data.rx_big.mask; i++) {
3315 err = mxge_get_buf_big(&ss->rx_data.rx_big,
3316 ss->rx_data.rx_big.info[i].map, i, TRUE);
3317 if (err) {
3318 if_printf(ss->sc->ifp, "alloced %d/%d bigs\n", i,
3319 ss->rx_data.rx_big.mask + 1);
3320 return ENOMEM;
3321 }
3322 }
3323 return 0;
3324 }
3325
3326 static int
3327 mxge_open(mxge_softc_t *sc)
3328 {
3329 struct ifnet *ifp = sc->ifp;
3330 mxge_cmd_t cmd;
3331 int err, slice, cl_size, i;
3332 bus_addr_t bus;
3333 volatile uint8_t *itable;
3334 struct mxge_slice_state *ss;
3335
3336 ASSERT_IFNET_SERIALIZED_ALL(ifp);
3337
3338 /* Copy the MAC address in case it was overridden */
3339 bcopy(IF_LLADDR(ifp), sc->mac_addr, ETHER_ADDR_LEN);
3340
3341 err = mxge_reset(sc, 1);
3342 if (err != 0) {
3343 if_printf(ifp, "failed to reset\n");
3344 return EIO;
3345 }
3346
3347 if (sc->num_slices > 1) {
3348 /*
3349 * Setup the indirect table.
3350 */
3351 if_ringmap_rdrtable(sc->ring_map, sc->rdr_table, NETISR_CPUMAX);
3352
3353 cmd.data0 = NETISR_CPUMAX;
3354 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE, &cmd);
3355
3356 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET, &cmd);
3357 if (err != 0) {
3358 if_printf(ifp, "failed to setup rss tables\n");
3359 return err;
3360 }
3361
3362 itable = sc->sram + cmd.data0;
3363 for (i = 0; i < NETISR_CPUMAX; i++)
3364 itable[i] = sc->rdr_table[i];
3365
3366 if (sc->use_rss) {
3367 volatile uint8_t *hwkey;
3368 uint8_t swkey[MXGE_HWRSS_KEYLEN];
3369
3370 /*
3371 * Setup Toeplitz key.
3372 */
3373 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_KEY_OFFSET,
3374 &cmd);
3375 if (err != 0) {
3376 if_printf(ifp, "failed to get rsskey\n");
3377 return err;
3378 }
3379 hwkey = sc->sram + cmd.data0;
3380
3381 toeplitz_get_key(swkey, MXGE_HWRSS_KEYLEN);
3382 for (i = 0; i < MXGE_HWRSS_KEYLEN; ++i)
3383 hwkey[i] = swkey[i];
3384 wmb();
3385
3386 err = mxge_send_cmd(sc, MXGEFW_CMD_RSS_KEY_UPDATED,
3387 &cmd);
3388 if (err != 0) {
3389 if_printf(ifp, "failed to update rsskey\n");
3390 return err;
3391 }
3392 if (bootverbose)
3393 if_printf(ifp, "RSS key updated\n");
3394 }
3395
3396 cmd.data0 = 1;
3397 if (sc->use_rss) {
3398 if (bootverbose)
3399 if_printf(ifp, "input hash: RSS\n");
3400 cmd.data1 = MXGEFW_RSS_HASH_TYPE_IPV4 |
3401 MXGEFW_RSS_HASH_TYPE_TCP_IPV4;
3402 } else {
3403 if (bootverbose)
3404 if_printf(ifp, "input hash: SRC_DST_PORT\n");
3405 cmd.data1 = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
3406 }
3407 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd);
3408 if (err != 0) {
3409 if_printf(ifp, "failed to enable slices\n");
3410 return err;
3411 }
3412 }
3413
3414 cmd.data0 = MXGEFW_TSO_MODE_NDIS;
3415 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_TSO_MODE, &cmd);
3416 if (err) {
3417 /*
3418 * Can't change TSO mode to NDIS, never allow TSO then
3419 */
3420 if_printf(ifp, "failed to set TSO mode\n");
3421 ifp->if_capenable &= ~IFCAP_TSO;
3422 ifp->if_capabilities &= ~IFCAP_TSO;
3423 ifp->if_hwassist &= ~CSUM_TSO;
3424 }
3425
3426 mxge_choose_params(ifp->if_mtu, &cl_size);
3427
3428 cmd.data0 = 1;
3429 err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, &cmd);
3430 /*
3431 * Error is only meaningful if we're trying to set
3432 * MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1
3433 */
3434
3435 /*
3436 * Give the firmware the mtu and the big and small buffer
3437 * sizes. The firmware wants the big buf size to be a power
3438 * of two. Luckily, DragonFly's clusters are powers of two
3439 */
3440 cmd.data0 = ifp->if_mtu + ETHER_HDR_LEN + EVL_ENCAPLEN;
3441 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
3442
3443 cmd.data0 = MXGE_RX_SMALL_BUFLEN;
3444 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd);
3445
3446 cmd.data0 = cl_size;
3447 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
3448
3449 if (err != 0) {
3450 if_printf(ifp, "failed to setup params\n");
3451 goto abort;
3452 }
3453
3454 /* Now give him the pointer to the stats block */
3455 for (slice = 0; slice < sc->num_slices; slice++) {
3456 ss = &sc->ss[slice];
3457 cmd.data0 = MXGE_LOWPART_TO_U32(ss->fw_stats_dma.dmem_busaddr);
3458 cmd.data1 = MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.dmem_busaddr);
3459 cmd.data2 = sizeof(struct mcp_irq_data);
3460 cmd.data2 |= (slice << 16);
3461 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3462 }
3463
3464 if (err != 0) {
3465 bus = sc->ss->fw_stats_dma.dmem_busaddr;
3466 bus += offsetof(struct mcp_irq_data, send_done_count);
3467 cmd.data0 = MXGE_LOWPART_TO_U32(bus);
3468 cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
3469 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
3470 &cmd);
3471
3472 /* Firmware cannot support multicast without STATS_DMA_V2 */
3473 sc->fw_multicast_support = 0;
3474 } else {
3475 sc->fw_multicast_support = 1;
3476 }
3477
3478 if (err != 0) {
3479 if_printf(ifp, "failed to setup params\n");
3480 goto abort;
3481 }
3482
3483 for (slice = 0; slice < sc->num_slices; slice++) {
3484 err = mxge_slice_open(&sc->ss[slice], cl_size);
3485 if (err != 0) {
3486 if_printf(ifp, "couldn't open slice %d\n", slice);
3487 goto abort;
3488 }
3489 }
3490
3491 /* Finally, start the firmware running */
3492 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3493 if (err) {
3494 if_printf(ifp, "Couldn't bring up link\n");
3495 goto abort;
3496 }
3497
3498 ifp->if_flags |= IFF_RUNNING;
3499 for (i = 0; i < sc->num_tx_rings; ++i) {
3500 mxge_tx_ring_t *tx = &sc->ss[i].tx;
3501
3502 ifsq_clr_oactive(tx->ifsq);
3503 ifsq_watchdog_start(&tx->watchdog);
3504 }
3505
3506 return 0;
3507
3508 abort:
3509 mxge_free_mbufs(sc);
3510 return err;
3511 }
3512
3513 static void
3514 mxge_close(mxge_softc_t *sc, int down)
3515 {
3516 struct ifnet *ifp = sc->ifp;
3517 mxge_cmd_t cmd;
3518 int err, old_down_cnt, i;
3519
3520 ASSERT_IFNET_SERIALIZED_ALL(ifp);
3521
3522 if (!down) {
3523 old_down_cnt = sc->down_cnt;
3524 wmb();
3525
3526 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3527 if (err)
3528 if_printf(ifp, "Couldn't bring down link\n");
3529
3530 if (old_down_cnt == sc->down_cnt) {
3531 /*
3532 * Wait for down irq
3533 * XXX racy
3534 */
3535 ifnet_deserialize_all(ifp);
3536 DELAY(10 * sc->intr_coal_delay);
3537 ifnet_serialize_all(ifp);
3538 }
3539
3540 wmb();
3541 if (old_down_cnt == sc->down_cnt)
3542 if_printf(ifp, "never got down irq\n");
3543 }
3544 mxge_free_mbufs(sc);
3545
3546 ifp->if_flags &= ~IFF_RUNNING;
3547 for (i = 0; i < sc->num_tx_rings; ++i) {
3548 mxge_tx_ring_t *tx = &sc->ss[i].tx;
3549
3550 ifsq_clr_oactive(tx->ifsq);
3551 ifsq_watchdog_stop(&tx->watchdog);
3552 }
3553 }
3554
3555 static void
3556 mxge_setup_cfg_space(mxge_softc_t *sc)
3557 {
3558 device_t dev = sc->dev;
3559 int reg;
3560 uint16_t lnk, pectl;
3561
3562 /* Find the PCIe link width and set max read request to 4KB */
3563 if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
3564 lnk = pci_read_config(dev, reg + 0x12, 2);
3565 sc->link_width = (lnk >> 4) & 0x3f;
3566
3567 if (sc->pectl == 0) {
3568 pectl = pci_read_config(dev, reg + 0x8, 2);
3569 pectl = (pectl & ~0x7000) | (5 << 12);
3570 pci_write_config(dev, reg + 0x8, pectl, 2);
3571 sc->pectl = pectl;
3572 } else {
3573 /* Restore saved pectl after watchdog reset */
3574 pci_write_config(dev, reg + 0x8, sc->pectl, 2);
3575 }
3576 }
3577
3578 /* Enable DMA and memory space access */
3579 pci_enable_busmaster(dev);
3580 }
3581
3582 static uint32_t
3583 mxge_read_reboot(mxge_softc_t *sc)
3584 {
3585 device_t dev = sc->dev;
3586 uint32_t vs;
3587
3588 /* Find the vendor specific offset */
3589 if (pci_find_extcap(dev, PCIY_VENDOR, &vs) != 0) {
3590 if_printf(sc->ifp, "could not find vendor specific offset\n");
3591 return (uint32_t)-1;
3592 }
3593 /* Enable read32 mode */
3594 pci_write_config(dev, vs + 0x10, 0x3, 1);
3595 /* Tell NIC which register to read */
3596 pci_write_config(dev, vs + 0x18, 0xfffffff0, 4);
3597 return pci_read_config(dev, vs + 0x14, 4);
3598 }
3599
3600 static void
3601 mxge_watchdog_reset(mxge_softc_t *sc)
3602 {
3603 struct pci_devinfo *dinfo;
3604 int err, running;
3605 uint32_t reboot;
3606 uint16_t cmd;
3607
3608 err = ENXIO;
3609
3610 if_printf(sc->ifp, "Watchdog reset!\n");
3611
3612 /*
3613 * Check to see if the NIC rebooted. If it did, then all of
3614 * PCI config space has been reset, and things like the
3615 * busmaster bit will be zero. If this is the case, then we
3616 * must restore PCI config space before the NIC can be used
3617 * again
3618 */
3619 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3620 if (cmd == 0xffff) {
3621 /*
3622 * Maybe the watchdog caught the NIC rebooting; wait
3623 * up to 100ms for it to finish. If it does not come
3624 * back, then give up
3625 */
3626 DELAY(1000*100);
3627 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3628 if (cmd == 0xffff)
3629 if_printf(sc->ifp, "NIC disappeared!\n");
3630 }
3631 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3632 /* Print the reboot status */
3633 reboot = mxge_read_reboot(sc);
3634 if_printf(sc->ifp, "NIC rebooted, status = 0x%x\n", reboot);
3635
3636 running = sc->ifp->if_flags & IFF_RUNNING;
3637 if (running) {
3638 /*
3639 * Quiesce NIC so that TX routines will not try to
3640 * xmit after restoration of BAR
3641 */
3642
3643 /* Mark the link as down */
3644 if (sc->link_state) {
3645 sc->ifp->if_link_state = LINK_STATE_DOWN;
3646 if_link_state_change(sc->ifp);
3647 }
3648 mxge_close(sc, 1);
3649 }
3650 /* Restore PCI configuration space */
3651 dinfo = device_get_ivars(sc->dev);
3652 pci_cfg_restore(sc->dev, dinfo);
3653
3654 /* And redo any changes we made to our config space */
3655 mxge_setup_cfg_space(sc);
3656
3657 /* Reload f/w */
3658 err = mxge_load_firmware(sc, 0);
3659 if (err)
3660 if_printf(sc->ifp, "Unable to re-load f/w\n");
3661 if (running && !err) {
3662 int i;
3663
3664 err = mxge_open(sc);
3665
3666 for (i = 0; i < sc->num_tx_rings; ++i)
3667 ifsq_devstart_sched(sc->ss[i].tx.ifsq);
3668 }
3669 sc->watchdog_resets++;
3670 } else {
3671 if_printf(sc->ifp, "NIC did not reboot, not resetting\n");
3672 err = 0;
3673 }
3674 if (err) {
3675 if_printf(sc->ifp, "watchdog reset failed\n");
3676 } else {
3677 if (sc->dying == 2)
3678 sc->dying = 0;
3679 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
3680 }
3681 }
3682
3683 static void
3684 mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice)
3685 {
3686 if_printf(sc->ifp, "slice %d struck? ring state:\n", slice);
3687 if_printf(sc->ifp, "tx.req=%d tx.done=%d, tx.queue_active=%d\n",
3688 tx->req, tx->done, tx->queue_active);
3689 if_printf(sc->ifp, "tx.activate=%d tx.deactivate=%d\n",
3690 tx->activate, tx->deactivate);
3691 if_printf(sc->ifp, "pkt_done=%d fw=%d\n",
3692 tx->pkt_done, be32toh(sc->ss->fw_stats->send_done_count));
3693 }
3694
3695 static u_long
3696 mxge_update_stats(mxge_softc_t *sc)
3697 {
3698 u_long ipackets, opackets, pkts;
3699
3700 IFNET_STAT_GET(sc->ifp, ipackets, ipackets);
3701 IFNET_STAT_GET(sc->ifp, opackets, opackets);
3702
3703 pkts = ipackets - sc->ipackets;
3704 pkts += opackets - sc->opackets;
3705
3706 sc->ipackets = ipackets;
3707 sc->opackets = opackets;
3708
3709 return pkts;
3710 }
3711
3712 static void
3713 mxge_tick(void *arg)
3714 {
3715 mxge_softc_t *sc = arg;
3716 u_long pkts = 0;
3717 int err = 0;
3718 int ticks;
3719
3720 lwkt_serialize_enter(&sc->main_serialize);
3721
3722 ticks = mxge_ticks;
3723 if (sc->ifp->if_flags & IFF_RUNNING) {
3724 /* Aggregate stats from different slices */
3725 pkts = mxge_update_stats(sc);
3726 if (sc->need_media_probe)
3727 mxge_media_probe(sc);
3728 }
3729 if (pkts == 0) {
3730 uint16_t cmd;
3731
3732 /* Ensure NIC did not suffer h/w fault while idle */
3733 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3734 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3735 sc->dying = 2;
3736 mxge_serialize_skipmain(sc);
3737 mxge_watchdog_reset(sc);
3738 mxge_deserialize_skipmain(sc);
3739 err = ENXIO;
3740 }
3741
3742 /* Look less often if NIC is idle */
3743 ticks *= 4;
3744 }
3745
3746 if (err == 0)
3747 callout_reset(&sc->co_hdl, ticks, mxge_tick, sc);
3748
3749 lwkt_serialize_exit(&sc->main_serialize);
3750 }
3751
3752 static int
3753 mxge_media_change(struct ifnet *ifp)
3754 {
3755 mxge_softc_t *sc = ifp->if_softc;
3756 const struct ifmedia *ifm = &sc->media;
3757 int pause;
3758
3759 if (IFM_OPTIONS(ifm->ifm_media) & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
3760 if (sc->pause)
3761 return 0;
3762 pause = 1;
3763 } else {
3764 if (!sc->pause)
3765 return 0;
3766 pause = 0;
3767 }
3768 return mxge_change_pause(sc, pause);
3769 }
3770
3771 static int
3772 mxge_change_mtu(mxge_softc_t *sc, int mtu)
3773 {
3774 struct ifnet *ifp = sc->ifp;
3775 int real_mtu, old_mtu;
3776 int err = 0;
3777
3778 real_mtu = mtu + ETHER_HDR_LEN + EVL_ENCAPLEN;
3779 if (mtu > sc->max_mtu || real_mtu < 60)
3780 return EINVAL;
3781
3782 old_mtu = ifp->if_mtu;
3783 ifp->if_mtu = mtu;
3784 if (ifp->if_flags & IFF_RUNNING) {
3785 mxge_close(sc, 0);
3786 err = mxge_open(sc);
3787 if (err != 0) {
3788 ifp->if_mtu = old_mtu;
3789 mxge_close(sc, 0);
3790 mxge_open(sc);
3791 }
3792 }
3793 return err;
3794 }
3795
3796 static void
3797 mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3798 {
3799 mxge_softc_t *sc = ifp->if_softc;
3800
3801 ifmr->ifm_status = IFM_AVALID;
3802 ifmr->ifm_active = IFM_ETHER;
3803
3804 if (sc->link_state)
3805 ifmr->ifm_status |= IFM_ACTIVE;
3806
3807 /*
3808 * Autoselect is not supported, so the current media
3809 * should be delivered.
3810 */
3811 ifmr->ifm_active |= sc->current_media;
3812 if (sc->current_media != IFM_NONE) {
3813 ifmr->ifm_active |= MXGE_IFM;
3814 if (sc->pause)
3815 ifmr->ifm_active |= IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE;
3816 }
3817 }
3818
3819 static int
3820 mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data,
3821 struct ucred *cr __unused)
3822 {
3823 mxge_softc_t *sc = ifp->if_softc;
3824 struct ifreq *ifr = (struct ifreq *)data;
3825 int err, mask;
3826
3827 ASSERT_IFNET_SERIALIZED_ALL(ifp);
3828 err = 0;
3829
3830 switch (command) {
3831 case SIOCSIFMTU:
3832 err = mxge_change_mtu(sc, ifr->ifr_mtu);
3833 break;
3834
3835 case SIOCSIFFLAGS:
3836 if (sc->dying)
3837 return EINVAL;
3838
3839 if (ifp->if_flags & IFF_UP) {
3840 if (!(ifp->if_flags & IFF_RUNNING)) {
3841 err = mxge_open(sc);
3842 } else {
3843 /*
3844 * Take care of PROMISC and ALLMULTI
3845 * flag changes
3846 */
3847 mxge_change_promisc(sc,
3848 ifp->if_flags & IFF_PROMISC);
3849 mxge_set_multicast_list(sc);
3850 }
3851 } else {
3852 if (ifp->if_flags & IFF_RUNNING)
3853 mxge_close(sc, 0);
3854 }
3855 break;
3856
3857 case SIOCADDMULTI:
3858 case SIOCDELMULTI:
3859 mxge_set_multicast_list(sc);
3860 break;
3861
3862 case SIOCSIFCAP:
3863 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3864 if (mask & IFCAP_TXCSUM) {
3865 ifp->if_capenable ^= IFCAP_TXCSUM;
3866 if (ifp->if_capenable & IFCAP_TXCSUM)
3867 ifp->if_hwassist |= CSUM_TCP | CSUM_UDP;
3868 else
3869 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP);
3870 }
3871 if (mask & IFCAP_TSO) {
3872 ifp->if_capenable ^= IFCAP_TSO;
3873 if (ifp->if_capenable & IFCAP_TSO)
3874 ifp->if_hwassist |= CSUM_TSO;
3875 else
3876 ifp->if_hwassist &= ~CSUM_TSO;
3877 }
3878 if (mask & IFCAP_RXCSUM)
3879 ifp->if_capenable ^= IFCAP_RXCSUM;
3880 if (mask & IFCAP_VLAN_HWTAGGING)
3881 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3882 break;
3883
3884 case SIOCGIFMEDIA:
3885 case SIOCSIFMEDIA:
3886 err = ifmedia_ioctl(ifp, (struct ifreq *)data,
3887 &sc->media, command);
3888 break;
3889
3890 default:
3891 err = ether_ioctl(ifp, command, data);
3892 break;
3893 }
3894 return err;
3895 }
3896
3897 static void
3898 mxge_fetch_tunables(mxge_softc_t *sc)
3899 {
3900 int ifm;
3901
3902 sc->intr_coal_delay = mxge_intr_coal_delay;
3903 if (sc->intr_coal_delay < 0 || sc->intr_coal_delay > (10 * 1000))
3904 sc->intr_coal_delay = MXGE_INTR_COAL_DELAY;
3905
3906 /* XXX */
3907 if (mxge_ticks == 0)
3908 mxge_ticks = hz / 2;
3909
3910 ifm = ifmedia_str2ethfc(mxge_flowctrl);
3911 if (ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE))
3912 sc->pause = 1;
3913
3914 sc->use_rss = mxge_use_rss;
3915
3916 sc->throttle = mxge_throttle;
3917 if (sc->throttle && sc->throttle > MXGE_MAX_THROTTLE)
3918 sc->throttle = MXGE_MAX_THROTTLE;
3919 if (sc->throttle && sc->throttle < MXGE_MIN_THROTTLE)
3920 sc->throttle = MXGE_MIN_THROTTLE;
3921 }
3922
3923 static void
3924 mxge_free_slices(mxge_softc_t *sc)
3925 {
3926 struct mxge_slice_state *ss;
3927 int i;
3928
3929 if (sc->ss == NULL)
3930 return;
3931
3932 for (i = 0; i < sc->num_slices; i++) {
3933 ss = &sc->ss[i];
3934 if (ss->fw_stats != NULL) {
3935 mxge_dma_free(&ss->fw_stats_dma);
3936 ss->fw_stats = NULL;
3937 }
3938 if (ss->rx_data.rx_done.entry != NULL) {
3939 mxge_dma_free(&ss->rx_done_dma);
3940 ss->rx_data.rx_done.entry = NULL;
3941 }
3942 }
3943 kfree(sc->ss, M_DEVBUF);
3944 sc->ss = NULL;
3945 }
3946
3947 static int
3948 mxge_alloc_slices(mxge_softc_t *sc)
3949 {
3950 mxge_cmd_t cmd;
3951 struct mxge_slice_state *ss;
3952 size_t bytes;
3953 int err, i, rx_ring_size;
3954
3955 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
3956 if (err != 0) {
3957 device_printf(sc->dev, "Cannot determine rx ring size\n");
3958 return err;
3959 }
3960 rx_ring_size = cmd.data0;
3961 sc->rx_intr_slots = 2 * (rx_ring_size / sizeof (mcp_dma_addr_t));
3962
3963 bytes = sizeof(*sc->ss) * sc->num_slices;
3964 sc->ss = kmalloc(bytes, M_DEVBUF,
3965 M_WAITOK | M_ZERO | M_CACHEALIGN);
3966
3967 for (i = 0; i < sc->num_slices; i++) {
3968 ss = &sc->ss[i];
3969
3970 ss->sc = sc;
3971
3972 lwkt_serialize_init(&ss->rx_data.rx_serialize);
3973 lwkt_serialize_init(&ss->tx.tx_serialize);
3974 ss->intr_rid = -1;
3975
3976 /*
3977 * Allocate per-slice rx interrupt queue
3978 * XXX assume 4bytes mcp_slot
3979 */
3980 bytes = sc->rx_intr_slots * sizeof(mcp_slot_t);
3981 err = mxge_dma_alloc(sc, &ss->rx_done_dma, bytes, 4096);
3982 if (err != 0) {
3983 device_printf(sc->dev,
3984 "alloc %d slice rx_done failed\n", i);
3985 return err;
3986 }
3987 ss->rx_data.rx_done.entry = ss->rx_done_dma.dmem_addr;
3988
3989 /*
3990 * Allocate the per-slice firmware stats
3991 */
3992 bytes = sizeof(*ss->fw_stats);
3993 err = mxge_dma_alloc(sc, &ss->fw_stats_dma,
3994 sizeof(*ss->fw_stats), 64);
3995 if (err != 0) {
3996 device_printf(sc->dev,
3997 "alloc %d fw_stats failed\n", i);
3998 return err;
3999 }
4000 ss->fw_stats = ss->fw_stats_dma.dmem_addr;
4001 }
4002 return 0;
4003 }
4004
4005 static void
4006 mxge_slice_probe(mxge_softc_t *sc)
4007 {
4008 int status, max_intr_slots, max_slices, num_slices;
4009 int msix_cnt, msix_enable, multi_tx;
4010 mxge_cmd_t cmd;
4011 const char *old_fw;
4012
4013 sc->num_slices = 1;
4014 sc->num_tx_rings = 1;
4015
4016 num_slices = device_getenv_int(sc->dev, "num_slices", mxge_num_slices);
4017 if (num_slices == 1)
4018 return;
4019
4020 if (netisr_ncpus == 1)
4021 return;
4022
4023 msix_enable = device_getenv_int(sc->dev, "msix.enable",
4024 mxge_msix_enable);
4025 if (!msix_enable)
4026 return;
4027
4028 msix_cnt = pci_msix_count(sc->dev);
4029 if (msix_cnt < 2)
4030 return;
4031 if (bootverbose)
4032 device_printf(sc->dev, "MSI-X count %d\n", msix_cnt);
4033
4034 /*
4035 * Now load the slice aware firmware see what it supports
4036 */
4037 old_fw = sc->fw_name;
4038 if (old_fw == mxge_fw_aligned)
4039 sc->fw_name = mxge_fw_rss_aligned;
4040 else
4041 sc->fw_name = mxge_fw_rss_unaligned;
4042 status = mxge_load_firmware(sc, 0);
4043 if (status != 0) {
4044 device_printf(sc->dev, "Falling back to a single slice\n");
4045 return;
4046 }
4047
4048 /*
4049 * Try to send a reset command to the card to see if it is alive
4050 */
4051 memset(&cmd, 0, sizeof(cmd));
4052 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
4053 if (status != 0) {
4054 device_printf(sc->dev, "failed reset\n");
4055 goto abort_with_fw;
4056 }
4057
4058 /*
4059 * Get rx ring size to calculate rx interrupt queue size
4060 */
4061 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
4062 if (status != 0) {
4063 device_printf(sc->dev, "Cannot determine rx ring size\n");
4064 goto abort_with_fw;
4065 }
4066 max_intr_slots = 2 * (cmd.data0 / sizeof(mcp_dma_addr_t));
4067
4068 /*
4069 * Tell it the size of the rx interrupt queue
4070 */
4071 cmd.data0 = max_intr_slots * sizeof(struct mcp_slot);
4072 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
4073 if (status != 0) {
4074 device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
4075 goto abort_with_fw;
4076 }
4077
4078 /*
4079 * Ask the maximum number of slices it supports
4080 */
4081 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
4082 if (status != 0) {
4083 device_printf(sc->dev,
4084 "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n");
4085 goto abort_with_fw;
4086 }
4087 max_slices = cmd.data0;
4088 if (bootverbose)
4089 device_printf(sc->dev, "max slices %d\n", max_slices);
4090
4091 if (max_slices > msix_cnt)
4092 max_slices = msix_cnt;
4093
4094 sc->ring_map = if_ringmap_alloc(sc->dev, num_slices, max_slices);
4095 sc->num_slices = if_ringmap_count(sc->ring_map);
4096
4097 multi_tx = device_getenv_int(sc->dev, "multi_tx", mxge_multi_tx);
4098 if (multi_tx)
4099 sc->num_tx_rings = sc->num_slices;
4100
4101 if (bootverbose) {
4102 device_printf(sc->dev, "using %d slices, max %d\n",
4103 sc->num_slices, max_slices);
4104 }
4105
4106 if (sc->num_slices == 1)
4107 goto abort_with_fw;
4108 return;
4109
4110 abort_with_fw:
4111 sc->fw_name = old_fw;
4112 mxge_load_firmware(sc, 0);
4113 }
4114
4115 static void
4116 mxge_setup_serialize(struct mxge_softc *sc)
4117 {
4118 int i = 0, slice;
4119
4120 /* Main + rx + tx */
4121 sc->nserialize = (2 * sc->num_slices) + 1;
4122 sc->serializes =
4123 kmalloc(sc->nserialize * sizeof(struct lwkt_serialize *),
4124 M_DEVBUF, M_WAITOK | M_ZERO);
4125
4126 /*
4127 * Setup serializes
4128 *
4129 * NOTE: Order is critical
4130 */
4131
4132 KKASSERT(i < sc->nserialize);
4133 sc->serializes[i++] = &sc->main_serialize;
4134
4135 for (slice = 0; slice < sc->num_slices; ++slice) {
4136 KKASSERT(i < sc->nserialize);
4137 sc->serializes[i++] = &sc->ss[slice].rx_data.rx_serialize;
4138 }
4139
4140 for (slice = 0; slice < sc->num_slices; ++slice) {
4141 KKASSERT(i < sc->nserialize);
4142 sc->serializes[i++] = &sc->ss[slice].tx.tx_serialize;
4143 }
4144
4145 KKASSERT(i == sc->nserialize);
4146 }
4147
4148 static void
4149 mxge_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
4150 {
4151 struct mxge_softc *sc = ifp->if_softc;
4152
4153 ifnet_serialize_array_enter(sc->serializes, sc->nserialize, slz);
4154 }
4155
4156 static void
4157 mxge_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
4158 {
4159 struct mxge_softc *sc = ifp->if_softc;
4160
4161 ifnet_serialize_array_exit(sc->serializes, sc->nserialize, slz);
4162 }
4163
4164 static int
4165 mxge_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
4166 {
4167 struct mxge_softc *sc = ifp->if_softc;
4168
4169 return ifnet_serialize_array_try(sc->serializes, sc->nserialize, slz);
4170 }
4171
4172 #ifdef INVARIANTS
4173
4174 static void
4175 mxge_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
4176 boolean_t serialized)
4177 {
4178 struct mxge_softc *sc = ifp->if_softc;
4179
4180 ifnet_serialize_array_assert(sc->serializes, sc->nserialize,
4181 slz, serialized);
4182 }
4183
4184 #endif /* INVARIANTS */
4185
4186 #ifdef IFPOLL_ENABLE
4187
4188 static void
4189 mxge_npoll_rx(struct ifnet *ifp, void *xss, int cycle)
4190 {
4191 struct mxge_slice_state *ss = xss;
4192 mxge_rx_done_t *rx_done = &ss->rx_data.rx_done;
4193
4194 ASSERT_SERIALIZED(&ss->rx_data.rx_serialize);
4195
4196 if (rx_done->entry[rx_done->idx].length != 0) {
4197 mxge_clean_rx_done(&ss->sc->arpcom.ac_if, &ss->rx_data, cycle);
4198 } else {
4199 /*
4200 * XXX
4201 * This register writting obviously has cost,
4202 * however, if we don't hand back the rx token,
4203 * the upcoming packets may suffer rediculously
4204 * large delay, as observed on 8AL-C using ping(8).
4205 */
4206 *ss->irq_claim = be32toh(3);
4207 }
4208 }
4209
4210 static void
4211 mxge_npoll(struct ifnet *ifp, struct ifpoll_info *info)
4212 {
4213 struct mxge_softc *sc = ifp->if_softc;
4214 int i;
4215
4216 if (info == NULL)
4217 return;
4218
4219 /*
4220 * Only poll rx; polling tx and status don't seem to work
4221 */
4222 for (i = 0; i < sc->num_slices; ++i) {
4223 struct mxge_slice_state *ss = &sc->ss[i];
4224 int cpu = ss->intr_cpuid;
4225
4226 KKASSERT(cpu < netisr_ncpus);
4227 info->ifpi_rx[cpu].poll_func = mxge_npoll_rx;
4228 info->ifpi_rx[cpu].arg = ss;
4229 info->ifpi_rx[cpu].serializer = &ss->rx_data.rx_serialize;
4230 }
4231 }
4232
4233 #endif /* IFPOLL_ENABLE */
4234
4235 static int
4236 mxge_attach(device_t dev)
4237 {
4238 mxge_softc_t *sc = device_get_softc(dev);
4239 struct ifnet *ifp = &sc->arpcom.ac_if;
4240 int err, rid, i;
4241
4242 /*
4243 * Avoid rewriting half the lines in this file to use
4244 * &sc->arpcom.ac_if instead
4245 */
4246 sc->ifp = ifp;
4247 sc->dev = dev;
4248 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4249
4250 /* IFM_ETH_FORCEPAUSE can't be changed */
4251 ifmedia_init(&sc->media, IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE,
4252 mxge_media_change, mxge_media_status);
4253
4254 lwkt_serialize_init(&sc->main_serialize);
4255
4256 mxge_fetch_tunables(sc);
4257
4258 err = bus_dma_tag_create(NULL, /* parent */
4259 1, /* alignment */
4260 0, /* boundary */
4261 BUS_SPACE_MAXADDR, /* low */
4262 BUS_SPACE_MAXADDR, /* high */
4263 BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
4264 0, /* num segs */
4265 BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
4266 0, /* flags */
4267 &sc->parent_dmat); /* tag */
4268 if (err != 0) {
4269 device_printf(dev, "Err %d allocating parent dmat\n", err);
4270 goto failed;
4271 }
4272
4273 callout_init_mp(&sc->co_hdl);
4274
4275 mxge_setup_cfg_space(sc);
4276
4277 /*
4278 * Map the board into the kernel
4279 */
4280 rid = PCIR_BARS;
4281 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
4282 &rid, RF_ACTIVE);
4283 if (sc->mem_res == NULL) {
4284 device_printf(dev, "could not map memory\n");
4285 err = ENXIO;
4286 goto failed;
4287 }
4288
4289 sc->sram = rman_get_virtual(sc->mem_res);
4290 sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100;
4291 if (sc->sram_size > rman_get_size(sc->mem_res)) {
4292 device_printf(dev, "impossible memory region size %ld\n",
4293 rman_get_size(sc->mem_res));
4294 err = ENXIO;
4295 goto failed;
4296 }
4297
4298 /*
4299 * Make NULL terminated copy of the EEPROM strings section of
4300 * lanai SRAM
4301 */
4302 bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE);
4303 bus_space_read_region_1(rman_get_bustag(sc->mem_res),
4304 rman_get_bushandle(sc->mem_res),
4305 sc->sram_size - MXGE_EEPROM_STRINGS_SIZE,
4306 sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE - 2);
4307 err = mxge_parse_strings(sc);
4308 if (err != 0) {
4309 device_printf(dev, "parse EEPROM string failed\n");
4310 goto failed;
4311 }
4312
4313 /*
4314 * Enable write combining for efficient use of PCIe bus
4315 */
4316 mxge_enable_wc(sc);
4317
4318 /*
4319 * Allocate the out of band DMA memory
4320 */
4321 err = mxge_dma_alloc(sc, &sc->cmd_dma, sizeof(mxge_cmd_t), 64);
4322 if (err != 0) {
4323 device_printf(dev, "alloc cmd DMA buf failed\n");
4324 goto failed;
4325 }
4326 sc->cmd = sc->cmd_dma.dmem_addr;
4327
4328 err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
4329 if (err != 0) {
4330 device_printf(dev, "alloc zeropad DMA buf failed\n");
4331 goto failed;
4332 }
4333
4334 err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
4335 if (err != 0) {
4336 device_printf(dev, "alloc dmabench DMA buf failed\n");
4337 goto failed;
4338 }
4339
4340 /* Select & load the firmware */
4341 err = mxge_select_firmware(sc);
4342 if (err != 0) {
4343 device_printf(dev, "select firmware failed\n");
4344 goto failed;
4345 }
4346
4347 mxge_slice_probe(sc);
4348 err = mxge_alloc_slices(sc);
4349 if (err != 0) {
4350 device_printf(dev, "alloc slices failed\n");
4351 goto failed;
4352 }
4353
4354 err = mxge_alloc_intr(sc);
4355 if (err != 0) {
4356 device_printf(dev, "alloc intr failed\n");
4357 goto failed;
4358 }
4359
4360 /* Setup serializes */
4361 mxge_setup_serialize(sc);
4362
4363 err = mxge_reset(sc, 0);
4364 if (err != 0) {
4365 device_printf(dev, "reset failed\n");
4366 goto failed;
4367 }
4368
4369 err = mxge_alloc_rings(sc);
4370 if (err != 0) {
4371 device_printf(dev, "failed to allocate rings\n");
4372 goto failed;
4373 }
4374
4375 ifp->if_baudrate = IF_Gbps(10UL);
4376 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO;
4377 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
4378
4379 ifp->if_capabilities |= IFCAP_VLAN_MTU;
4380 #if 0
4381 /* Well, its software, sigh */
4382 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
4383 #endif
4384 ifp->if_capenable = ifp->if_capabilities;
4385
4386 ifp->if_softc = sc;
4387 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
4388 ifp->if_init = mxge_init;
4389 ifp->if_ioctl = mxge_ioctl;
4390 ifp->if_start = mxge_start;
4391 #ifdef IFPOLL_ENABLE
4392 if (sc->intr_type != PCI_INTR_TYPE_LEGACY)
4393 ifp->if_npoll = mxge_npoll;
4394 #endif
4395 ifp->if_serialize = mxge_serialize;
4396 ifp->if_deserialize = mxge_deserialize;
4397 ifp->if_tryserialize = mxge_tryserialize;
4398 #ifdef INVARIANTS
4399 ifp->if_serialize_assert = mxge_serialize_assert;
4400 #endif
4401
4402 /* Increase TSO burst length */
4403 ifp->if_tsolen = (32 * ETHERMTU);
4404
4405 /* Initialise the ifmedia structure */
4406 mxge_media_init(sc);
4407 mxge_media_probe(sc);
4408
4409 ether_ifattach(ifp, sc->mac_addr, NULL);
4410
4411 /* Setup TX rings and subqueues */
4412 for (i = 0; i < sc->num_tx_rings; ++i) {
4413 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
4414 struct mxge_slice_state *ss = &sc->ss[i];
4415
4416 ifsq_set_cpuid(ifsq, ss->intr_cpuid);
4417 ifsq_set_hw_serialize(ifsq, &ss->tx.tx_serialize);
4418 ifsq_set_priv(ifsq, &ss->tx);
4419 ss->tx.ifsq = ifsq;
4420
4421 ifsq_watchdog_init(&ss->tx.watchdog, ifsq, mxge_watchdog, 0);
4422 }
4423
4424 /*
4425 * XXX
4426 * We are not ready to do "gather" jumbo frame, so
4427 * limit MTU to MJUMPAGESIZE
4428 */
4429 sc->max_mtu = MJUMPAGESIZE -
4430 ETHER_HDR_LEN - EVL_ENCAPLEN - MXGEFW_PAD - 1;
4431 sc->dying = 0;
4432
4433 err = mxge_setup_intr(sc);
4434 if (err != 0) {
4435 device_printf(dev, "alloc and setup intr failed\n");
4436 ether_ifdetach(ifp);
4437 goto failed;
4438 }
4439
4440 mxge_add_sysctls(sc);
4441
4442 /* Increase non-cluster mbuf limit; used by small RX rings */
4443 mb_inclimit(ifp->if_nmbclusters);
4444
4445 callout_reset_bycpu(&sc->co_hdl, mxge_ticks, mxge_tick, sc,
4446 sc->ss[0].intr_cpuid);
4447 return 0;
4448
4449 failed:
4450 mxge_detach(dev);
4451 return err;
4452 }
4453
4454 static int
4455 mxge_detach(device_t dev)
4456 {
4457 mxge_softc_t *sc = device_get_softc(dev);
4458
4459 if (device_is_attached(dev)) {
4460 struct ifnet *ifp = sc->ifp;
4461 int mblimit = ifp->if_nmbclusters;
4462
4463 ifnet_serialize_all(ifp);
4464
4465 sc->dying = 1;
4466 if (ifp->if_flags & IFF_RUNNING)
4467 mxge_close(sc, 1);
4468 callout_stop(&sc->co_hdl);
4469
4470 mxge_teardown_intr(sc, sc->num_slices);
4471
4472 ifnet_deserialize_all(ifp);
4473
4474 callout_terminate(&sc->co_hdl);
4475
4476 ether_ifdetach(ifp);
4477
4478 /* Decrease non-cluster mbuf limit increased by us */
4479 mb_inclimit(-mblimit);
4480 }
4481 ifmedia_removeall(&sc->media);
4482
4483 if (sc->cmd != NULL && sc->zeropad_dma.dmem_addr != NULL &&
4484 sc->sram != NULL)
4485 mxge_dummy_rdma(sc, 0);
4486
4487 mxge_free_intr(sc);
4488 mxge_rem_sysctls(sc);
4489 mxge_free_rings(sc);
4490
4491 /* MUST after sysctls, intr and rings are freed */
4492 mxge_free_slices(sc);
4493
4494 if (sc->dmabench_dma.dmem_addr != NULL)
4495 mxge_dma_free(&sc->dmabench_dma);
4496 if (sc->zeropad_dma.dmem_addr != NULL)
4497 mxge_dma_free(&sc->zeropad_dma);
4498 if (sc->cmd_dma.dmem_addr != NULL)
4499 mxge_dma_free(&sc->cmd_dma);
4500
4501 if (sc->msix_table_res != NULL) {
4502 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(2),
4503 sc->msix_table_res);
4504 }
4505 if (sc->mem_res != NULL) {
4506 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS,
4507 sc->mem_res);
4508 }
4509
4510 if (sc->parent_dmat != NULL)
4511 bus_dma_tag_destroy(sc->parent_dmat);
4512
4513 if (sc->ring_map != NULL)
4514 if_ringmap_free(sc->ring_map);
4515
4516 return 0;
4517 }
4518
4519 static int
4520 mxge_shutdown(device_t dev)
4521 {
4522 return 0;
4523 }
4524
4525 static void
4526 mxge_free_msix(struct mxge_softc *sc, boolean_t setup)
4527 {
4528 int i;
4529
4530 KKASSERT(sc->num_slices > 1);
4531
4532 for (i = 0; i < sc->num_slices; ++i) {
4533 struct mxge_slice_state *ss = &sc->ss[i];
4534
4535 if (ss->intr_res != NULL) {
4536 bus_release_resource(sc->dev, SYS_RES_IRQ,
4537 ss->intr_rid, ss->intr_res);
4538 }
4539 if (ss->intr_rid >= 0)
4540 pci_release_msix_vector(sc->dev, ss->intr_rid);
4541 }
4542 if (setup)
4543 pci_teardown_msix(sc->dev);
4544 }
4545
4546 static int
4547 mxge_alloc_msix(struct mxge_softc *sc)
4548 {
4549 struct mxge_slice_state *ss;
4550 int rid, error, i;
4551 boolean_t setup = FALSE;
4552
4553 KKASSERT(sc->num_slices > 1);
4554
4555 ss = &sc->ss[0];
4556
4557 ss->intr_serialize = &sc->main_serialize;
4558 ss->intr_func = mxge_msi;
4559 ksnprintf(ss->intr_desc0, sizeof(ss->intr_desc0),
4560 "%s comb", device_get_nameunit(sc->dev));
4561 ss->intr_desc = ss->intr_desc0;
4562 ss->intr_cpuid = if_ringmap_cpumap(sc->ring_map, 0);
4563
4564 for (i = 1; i < sc->num_slices; ++i) {
4565 ss = &sc->ss[i];
4566
4567 ss->intr_serialize = &ss->rx_data.rx_serialize;
4568 if (sc->num_tx_rings == 1) {
4569 ss->intr_func = mxge_msix_rx;
4570 ksnprintf(ss->intr_desc0, sizeof(ss->intr_desc0),
4571 "%s rx%d", device_get_nameunit(sc->dev), i);
4572 } else {
4573 ss->intr_func = mxge_msix_rxtx;
4574 ksnprintf(ss->intr_desc0, sizeof(ss->intr_desc0),
4575 "%s rxtx%d", device_get_nameunit(sc->dev), i);
4576 }
4577 ss->intr_desc = ss->intr_desc0;
4578 ss->intr_cpuid = if_ringmap_cpumap(sc->ring_map, i);
4579 }
4580
4581 rid = PCIR_BAR(2);
4582 sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
4583 &rid, RF_ACTIVE);
4584 if (sc->msix_table_res == NULL) {
4585 device_printf(sc->dev, "couldn't alloc MSI-X table res\n");
4586 return ENXIO;
4587 }
4588
4589 error = pci_setup_msix(sc->dev);
4590 if (error) {
4591 device_printf(sc->dev, "could not setup MSI-X\n");
4592 goto back;
4593 }
4594 setup = TRUE;
4595
4596 for (i = 0; i < sc->num_slices; ++i) {
4597 ss = &sc->ss[i];
4598
4599 error = pci_alloc_msix_vector(sc->dev, i, &ss->intr_rid,
4600 ss->intr_cpuid);
4601 if (error) {
4602 device_printf(sc->dev, "could not alloc "
4603 "MSI-X %d on cpu%d\n", i, ss->intr_cpuid);
4604 goto back;
4605 }
4606
4607 ss->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
4608 &ss->intr_rid, RF_ACTIVE);
4609 if (ss->intr_res == NULL) {
4610 device_printf(sc->dev, "could not alloc "
4611 "MSI-X %d resource\n", i);
4612 error = ENXIO;
4613 goto back;
4614 }
4615 }
4616
4617 pci_enable_msix(sc->dev);
4618 sc->intr_type = PCI_INTR_TYPE_MSIX;
4619 back:
4620 if (error)
4621 mxge_free_msix(sc, setup);
4622 return error;
4623 }
4624
4625 static int
4626 mxge_alloc_intr(struct mxge_softc *sc)
4627 {
4628 struct mxge_slice_state *ss;
4629 u_int irq_flags;
4630
4631 if (sc->num_slices > 1) {
4632 int error;
4633
4634 error = mxge_alloc_msix(sc);
4635 if (error)
4636 return error;
4637 KKASSERT(sc->intr_type == PCI_INTR_TYPE_MSIX);
4638 return 0;
4639 }
4640
4641 ss = &sc->ss[0];
4642
4643 sc->intr_type = pci_alloc_1intr(sc->dev, mxge_msi_enable,
4644 &ss->intr_rid, &irq_flags);
4645
4646 ss->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
4647 &ss->intr_rid, irq_flags);
4648 if (ss->intr_res == NULL) {
4649 device_printf(sc->dev, "could not alloc interrupt\n");
4650 return ENXIO;
4651 }
4652
4653 if (sc->intr_type == PCI_INTR_TYPE_LEGACY)
4654 ss->intr_func = mxge_legacy;
4655 else
4656 ss->intr_func = mxge_msi;
4657 ss->intr_serialize = &sc->main_serialize;
4658 ss->intr_cpuid = rman_get_cpuid(ss->intr_res);
4659
4660 return 0;
4661 }
4662
4663 static int
4664 mxge_setup_intr(struct mxge_softc *sc)
4665 {
4666 int i;
4667
4668 for (i = 0; i < sc->num_slices; ++i) {
4669 struct mxge_slice_state *ss = &sc->ss[i];
4670 int error;
4671
4672 error = bus_setup_intr_descr(sc->dev, ss->intr_res,
4673 INTR_MPSAFE, ss->intr_func, ss, &ss->intr_hand,
4674 ss->intr_serialize, ss->intr_desc);
4675 if (error) {
4676 device_printf(sc->dev, "can't setup %dth intr\n", i);
4677 mxge_teardown_intr(sc, i);
4678 return error;
4679 }
4680 }
4681 return 0;
4682 }
4683
4684 static void
4685 mxge_teardown_intr(struct mxge_softc *sc, int cnt)
4686 {
4687 int i;
4688
4689 if (sc->ss == NULL)
4690 return;
4691
4692 for (i = 0; i < cnt; ++i) {
4693 struct mxge_slice_state *ss = &sc->ss[i];
4694
4695 bus_teardown_intr(sc->dev, ss->intr_res, ss->intr_hand);
4696 }
4697 }
4698
4699 static void
4700 mxge_free_intr(struct mxge_softc *sc)
4701 {
4702 if (sc->ss == NULL)
4703 return;
4704
4705 if (sc->intr_type != PCI_INTR_TYPE_MSIX) {
4706 struct mxge_slice_state *ss = &sc->ss[0];
4707
4708 if (ss->intr_res != NULL) {
4709 bus_release_resource(sc->dev, SYS_RES_IRQ,
4710 ss->intr_rid, ss->intr_res);
4711 }
4712 if (sc->intr_type == PCI_INTR_TYPE_MSI)
4713 pci_release_msi(sc->dev);
4714 } else {
4715 mxge_free_msix(sc, TRUE);
4716 }
4717 }
DragonFly BSD