?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dsynth - Make dummy /usr/packages directory for pkg compatibility
[dragonfly.git] / sys / dev / netif / ix / if_ix.c
blob1307ae5a663d334de4c419ec92711cc7a2724070
1 /*
2 * Copyright (c) 2001-2017, Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the Intel Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include "opt_ifpoll.h"
33 #include "opt_ix.h"
34
35 #include <sys/param.h>
36 #include <sys/bus.h>
37 #include <sys/endian.h>
38 #include <sys/interrupt.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/proc.h>
43 #include <sys/rman.h>
44 #include <sys/serialize.h>
45 #include <sys/serialize2.h>
46 #include <sys/socket.h>
47 #include <sys/sockio.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 #include <sys/taskqueue.h>
51
52 #include <net/bpf.h>
53 #include <net/ethernet.h>
54 #include <net/if.h>
55 #include <net/if_arp.h>
56 #include <net/if_dl.h>
57 #include <net/if_media.h>
58 #include <net/ifq_var.h>
59 #include <net/if_ringmap.h>
60 #include <net/toeplitz.h>
61 #include <net/toeplitz2.h>
62 #include <net/vlan/if_vlan_var.h>
63 #include <net/vlan/if_vlan_ether.h>
64 #include <net/if_poll.h>
65
66 #include <netinet/in_systm.h>
67 #include <netinet/in.h>
68 #include <netinet/ip.h>
69
70 #include <bus/pci/pcivar.h>
71 #include <bus/pci/pcireg.h>
72
73 #include <dev/netif/ix/ixgbe_common.h>
74 #include <dev/netif/ix/ixgbe_api.h>
75 #include <dev/netif/ix/if_ix.h>
76
77 #define IX_IFM_DEFAULT (IFM_ETHER | IFM_AUTO)
78
79 #ifdef IX_RSS_DEBUG
80 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) \
81 do { \
82 if (sc->rss_debug >= lvl) \
83 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
84 } while (0)
85 #else /* !IX_RSS_DEBUG */
86 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0)
87 #endif /* IX_RSS_DEBUG */
88
89 #define IX_NAME "Intel(R) PRO/10GbE "
90 #define IX_DEVICE(id) \
91 { IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_##id, IX_NAME #id }
92 #define IX_DEVICE_NULL { 0, 0, NULL }
93
94 static struct ix_device {
95 uint16_t vid;
96 uint16_t did;
97 const char *desc;
98 } ix_devices[] = {
99 IX_DEVICE(82598AF_DUAL_PORT),
100 IX_DEVICE(82598AF_SINGLE_PORT),
101 IX_DEVICE(82598EB_CX4),
102 IX_DEVICE(82598AT),
103 IX_DEVICE(82598AT2),
104 IX_DEVICE(82598),
105 IX_DEVICE(82598_DA_DUAL_PORT),
106 IX_DEVICE(82598_CX4_DUAL_PORT),
107 IX_DEVICE(82598EB_XF_LR),
108 IX_DEVICE(82598_SR_DUAL_PORT_EM),
109 IX_DEVICE(82598EB_SFP_LOM),
110 IX_DEVICE(82599_KX4),
111 IX_DEVICE(82599_KX4_MEZZ),
112 IX_DEVICE(82599_SFP),
113 IX_DEVICE(82599_XAUI_LOM),
114 IX_DEVICE(82599_CX4),
115 IX_DEVICE(82599_T3_LOM),
116 IX_DEVICE(82599_COMBO_BACKPLANE),
117 IX_DEVICE(82599_BACKPLANE_FCOE),
118 IX_DEVICE(82599_SFP_SF2),
119 IX_DEVICE(82599_SFP_FCOE),
120 IX_DEVICE(82599EN_SFP),
121 IX_DEVICE(82599_SFP_SF_QP),
122 IX_DEVICE(82599_QSFP_SF_QP),
123 IX_DEVICE(X540T),
124 IX_DEVICE(X540T1),
125 IX_DEVICE(X550T),
126 IX_DEVICE(X550T1),
127 IX_DEVICE(X550EM_X_KR),
128 IX_DEVICE(X550EM_X_KX4),
129 IX_DEVICE(X550EM_X_10G_T),
130 IX_DEVICE(X550EM_X_1G_T),
131 IX_DEVICE(X550EM_X_SFP),
132 IX_DEVICE(X550EM_A_KR),
133 IX_DEVICE(X550EM_A_KR_L),
134 IX_DEVICE(X550EM_A_SFP),
135 IX_DEVICE(X550EM_A_SFP_N),
136 IX_DEVICE(X550EM_A_SGMII),
137 IX_DEVICE(X550EM_A_SGMII_L),
138 IX_DEVICE(X550EM_A_10G_T),
139 IX_DEVICE(X550EM_A_1G_T),
140 IX_DEVICE(X550EM_A_1G_T_L),
141 #if 0
142 IX_DEVICE(X540_BYPASS),
143 IX_DEVICE(82599_BYPASS),
144 #endif
145
146 /* required last entry */
147 IX_DEVICE_NULL
148 };
149
150 static int ix_probe(device_t);
151 static int ix_attach(device_t);
152 static int ix_detach(device_t);
153 static int ix_shutdown(device_t);
154
155 static void ix_serialize(struct ifnet *, enum ifnet_serialize);
156 static void ix_deserialize(struct ifnet *, enum ifnet_serialize);
157 static int ix_tryserialize(struct ifnet *, enum ifnet_serialize);
158 #ifdef INVARIANTS
159 static void ix_serialize_assert(struct ifnet *, enum ifnet_serialize,
160 boolean_t);
161 #endif
162 static void ix_start(struct ifnet *, struct ifaltq_subque *);
163 static void ix_watchdog(struct ifaltq_subque *);
164 static int ix_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
165 static void ix_init(void *);
166 static void ix_stop(struct ix_softc *);
167 static void ix_media_status(struct ifnet *, struct ifmediareq *);
168 static int ix_media_change(struct ifnet *);
169 static void ix_timer(void *);
170 static void ix_fw_timer(void *);
171 #ifdef IFPOLL_ENABLE
172 static void ix_npoll(struct ifnet *, struct ifpoll_info *);
173 static void ix_npoll_rx(struct ifnet *, void *, int);
174 static void ix_npoll_rx_direct(struct ifnet *, void *, int);
175 static void ix_npoll_tx(struct ifnet *, void *, int);
176 static void ix_npoll_status(struct ifnet *);
177 #endif
178
179 static void ix_add_sysctl(struct ix_softc *);
180 static void ix_add_intr_rate_sysctl(struct ix_softc *, int,
181 const char *, int (*)(SYSCTL_HANDLER_ARGS), const char *);
182 static int ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS);
183 static int ix_sysctl_tx_nmbuf(SYSCTL_HANDLER_ARGS);
184 static int ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS);
185 static int ix_sysctl_txd(SYSCTL_HANDLER_ARGS);
186 static int ix_sysctl_rxd(SYSCTL_HANDLER_ARGS);
187 static int ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS);
188 static int ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int);
189 static int ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS);
190 static int ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS);
191 static int ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS);
192 static int ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS);
193 #if 0
194 static void ix_add_hw_stats(struct ix_softc *);
195 #endif
196
197 static void ix_watchdog_reset(struct ix_softc *);
198 static void ix_watchdog_task(void *, int);
199 static void ix_sync_netisr(struct ix_softc *, int);
200 static void ix_slot_info(struct ix_softc *);
201 static int ix_alloc_rings(struct ix_softc *);
202 static void ix_free_rings(struct ix_softc *);
203 static void ix_setup_ifp(struct ix_softc *);
204 static void ix_setup_serialize(struct ix_softc *);
205 static void ix_setup_caps(struct ix_softc *);
206 static void ix_set_ring_inuse(struct ix_softc *, boolean_t);
207 static int ix_get_timer_cpuid(const struct ix_softc *, boolean_t);
208 static void ix_update_stats(struct ix_softc *);
209 static void ix_detect_fanfail(struct ix_softc *, uint32_t, boolean_t);
210
211 static void ix_set_promisc(struct ix_softc *);
212 static void ix_set_multi(struct ix_softc *);
213 static void ix_set_vlan(struct ix_softc *);
214 static uint8_t *ix_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
215 static enum ixgbe_fc_mode ix_ifmedia2fc(int);
216 static const char *ix_ifmedia2str(int);
217 static const char *ix_fc2str(enum ixgbe_fc_mode);
218
219 static void ix_get_txring_cnt(const struct ix_softc *, int *, int *);
220 static int ix_get_txring_inuse(const struct ix_softc *, boolean_t);
221 static void ix_init_tx_ring(struct ix_tx_ring *);
222 static void ix_free_tx_ring(struct ix_tx_ring *);
223 static int ix_create_tx_ring(struct ix_tx_ring *);
224 static void ix_destroy_tx_ring(struct ix_tx_ring *, int);
225 static void ix_init_tx_unit(struct ix_softc *);
226 static int ix_encap(struct ix_tx_ring *, struct mbuf **,
227 uint16_t *, int *);
228 static int ix_tx_ctx_setup(struct ix_tx_ring *,
229 const struct mbuf *, uint32_t *, uint32_t *);
230 static int ix_tso_ctx_setup(struct ix_tx_ring *,
231 const struct mbuf *, uint32_t *, uint32_t *);
232 static void ix_txeof(struct ix_tx_ring *, int);
233 static void ix_txgc(struct ix_tx_ring *);
234 static void ix_txgc_timer(void *);
235
236 static void ix_get_rxring_cnt(const struct ix_softc *, int *, int *);
237 static int ix_get_rxring_inuse(const struct ix_softc *, boolean_t);
238 static int ix_init_rx_ring(struct ix_rx_ring *);
239 static void ix_free_rx_ring(struct ix_rx_ring *);
240 static int ix_create_rx_ring(struct ix_rx_ring *);
241 static void ix_destroy_rx_ring(struct ix_rx_ring *, int);
242 static void ix_init_rx_unit(struct ix_softc *, boolean_t);
243 #if 0
244 static void ix_setup_hw_rsc(struct ix_rx_ring *);
245 #endif
246 static int ix_newbuf(struct ix_rx_ring *, int, boolean_t);
247 static void ix_rxeof(struct ix_rx_ring *, int);
248 static void ix_rx_discard(struct ix_rx_ring *, int, boolean_t);
249 static void ix_enable_rx_drop(struct ix_softc *);
250 static void ix_disable_rx_drop(struct ix_softc *);
251
252 static void ix_config_gpie(struct ix_softc *);
253 static void ix_alloc_msix(struct ix_softc *);
254 static void ix_free_msix(struct ix_softc *, boolean_t);
255 static void ix_setup_msix_eims(const struct ix_softc *, int,
256 uint32_t *, uint32_t *);
257 static int ix_alloc_intr(struct ix_softc *);
258 static void ix_free_intr(struct ix_softc *);
259 static int ix_setup_intr(struct ix_softc *);
260 static void ix_teardown_intr(struct ix_softc *, int);
261 static void ix_enable_intr(struct ix_softc *);
262 static void ix_disable_intr(struct ix_softc *);
263 static void ix_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t);
264 static void ix_set_eitr(struct ix_softc *, int, int);
265 static void ix_intr_status(struct ix_softc *, uint32_t);
266 static void ix_intr_82598(void *);
267 static void ix_intr(void *);
268 static void ix_msix_rxtx(void *);
269 static void ix_msix_rx(void *);
270 static void ix_msix_tx(void *);
271 static void ix_msix_status(void *);
272
273 static void ix_config_link(struct ix_softc *);
274 static boolean_t ix_sfp_probe(struct ix_softc *);
275 static boolean_t ix_is_sfp(struct ixgbe_hw *);
276 static void ix_update_link_status(struct ix_softc *);
277 static void ix_handle_link(struct ix_softc *);
278 static void ix_handle_mod(struct ix_softc *);
279 static void ix_handle_msf(struct ix_softc *);
280 static void ix_handle_phy(struct ix_softc *);
281 static int ix_powerdown(struct ix_softc *);
282 static void ix_config_flowctrl(struct ix_softc *);
283 static void ix_config_dmac(struct ix_softc *);
284 static void ix_init_media(struct ix_softc *);
285
286 static void ix_serialize_skipmain(struct ix_softc *);
287 static void ix_deserialize_skipmain(struct ix_softc *);
288
289 static device_method_t ix_methods[] = {
290 /* Device interface */
291 DEVMETHOD(device_probe, ix_probe),
292 DEVMETHOD(device_attach, ix_attach),
293 DEVMETHOD(device_detach, ix_detach),
294 DEVMETHOD(device_shutdown, ix_shutdown),
295 DEVMETHOD_END
296 };
297
298 static driver_t ix_driver = {
299 "ix",
300 ix_methods,
301 sizeof(struct ix_softc)
302 };
303
304 static devclass_t ix_devclass;
305
306 DECLARE_DUMMY_MODULE(if_ix);
307 DRIVER_MODULE(if_ix, pci, ix_driver, ix_devclass, NULL, NULL);
308
309 static int ix_msi_enable = 1;
310 static int ix_msix_enable = 1;
311 static int ix_rxr = 0;
312 static int ix_txr = 0;
313 static int ix_txd = IX_PERF_TXD;
314 static int ix_rxd = IX_PERF_RXD;
315 static int ix_unsupported_sfp = 0;
316 static int ix_direct_input = 1;
317
318 static char ix_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_NONE;
319
320 TUNABLE_INT("hw.ix.msi.enable", &ix_msi_enable);
321 TUNABLE_INT("hw.ix.msix.enable", &ix_msix_enable);
322 TUNABLE_INT("hw.ix.rxr", &ix_rxr);
323 TUNABLE_INT("hw.ix.txr", &ix_txr);
324 TUNABLE_INT("hw.ix.txd", &ix_txd);
325 TUNABLE_INT("hw.ix.rxd", &ix_rxd);
326 TUNABLE_INT("hw.ix.unsupported_sfp", &ix_unsupported_sfp);
327 TUNABLE_STR("hw.ix.flow_ctrl", ix_flowctrl, sizeof(ix_flowctrl));
328 TUNABLE_INT("hw.ix.direct_input", &ix_direct_input);
329
330 /*
331 * Smart speed setting, default to on. This only works
332 * as a compile option right now as its during attach,
333 * set this to 'ixgbe_smart_speed_off' to disable.
334 */
335 static const enum ixgbe_smart_speed ix_smart_speed =
336 ixgbe_smart_speed_on;
337
338 static __inline void
339 ix_try_txgc(struct ix_tx_ring *txr, int8_t dec)
340 {
341
342 if (txr->tx_running > 0) {
343 txr->tx_running -= dec;
344 if (txr->tx_running <= 0 && txr->tx_nmbuf &&
345 txr->tx_avail < txr->tx_ndesc &&
346 txr->tx_avail + txr->tx_intr_nsegs > txr->tx_ndesc)
347 ix_txgc(txr);
348 }
349 }
350
351 static void
352 ix_txgc_timer(void *xtxr)
353 {
354 struct ix_tx_ring *txr = xtxr;
355 struct ifnet *ifp = &txr->tx_sc->arpcom.ac_if;
356
357 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) !=
358 (IFF_RUNNING | IFF_UP))
359 return;
360
361 if (!lwkt_serialize_try(&txr->tx_serialize))
362 goto done;
363
364 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) !=
365 (IFF_RUNNING | IFF_UP)) {
366 lwkt_serialize_exit(&txr->tx_serialize);
367 return;
368 }
369 ix_try_txgc(txr, IX_TX_RUNNING_DEC);
370
371 lwkt_serialize_exit(&txr->tx_serialize);
372 done:
373 callout_reset(&txr->tx_gc_timer, 1, ix_txgc_timer, txr);
374 }
375
376 static __inline void
377 ix_tx_intr(struct ix_tx_ring *txr, int hdr)
378 {
379
380 ix_txeof(txr, hdr);
381 if (!ifsq_is_empty(txr->tx_ifsq))
382 ifsq_devstart(txr->tx_ifsq);
383 }
384
385 static __inline void
386 ix_free_txbuf(struct ix_tx_ring *txr, struct ix_tx_buf *txbuf)
387 {
388
389 KKASSERT(txbuf->m_head != NULL);
390 KKASSERT(txr->tx_nmbuf > 0);
391 txr->tx_nmbuf--;
392
393 bus_dmamap_unload(txr->tx_tag, txbuf->map);
394 m_freem(txbuf->m_head);
395 txbuf->m_head = NULL;
396 }
397
398 static int
399 ix_probe(device_t dev)
400 {
401 const struct ix_device *d;
402 uint16_t vid, did;
403
404 vid = pci_get_vendor(dev);
405 did = pci_get_device(dev);
406
407 for (d = ix_devices; d->desc != NULL; ++d) {
408 if (vid == d->vid && did == d->did) {
409 device_set_desc(dev, d->desc);
410 return 0;
411 }
412 }
413 return ENXIO;
414 }
415
416 static void
417 ix_get_rxring_cnt(const struct ix_softc *sc, int *ring_cnt, int *ring_cntmax)
418 {
419
420 switch (sc->hw.mac.type) {
421 case ixgbe_mac_X550:
422 case ixgbe_mac_X550EM_x:
423 case ixgbe_mac_X550EM_a:
424 *ring_cntmax = IX_MAX_RXRING_X550;
425 break;
426
427 default:
428 *ring_cntmax = IX_MAX_RXRING;
429 break;
430 }
431 *ring_cnt = device_getenv_int(sc->dev, "rxr", ix_rxr);
432 }
433
434 static void
435 ix_get_txring_cnt(const struct ix_softc *sc, int *ring_cnt, int *ring_cntmax)
436 {
437
438 switch (sc->hw.mac.type) {
439 case ixgbe_mac_82598EB:
440 *ring_cntmax = IX_MAX_TXRING_82598;
441 break;
442
443 case ixgbe_mac_82599EB:
444 *ring_cntmax = IX_MAX_TXRING_82599;
445 break;
446
447 case ixgbe_mac_X540:
448 *ring_cntmax = IX_MAX_TXRING_X540;
449 break;
450
451 case ixgbe_mac_X550:
452 case ixgbe_mac_X550EM_x:
453 case ixgbe_mac_X550EM_a:
454 *ring_cntmax = IX_MAX_TXRING_X550;
455 break;
456
457 default:
458 *ring_cntmax = IX_MAX_TXRING;
459 break;
460 }
461 *ring_cnt = device_getenv_int(sc->dev, "txr", ix_txr);
462 }
463
464 static int
465 ix_attach(device_t dev)
466 {
467 struct ix_softc *sc = device_get_softc(dev);
468 struct ixgbe_hw *hw;
469 int error, ring_cnt, ring_cntmax;
470 uint32_t ctrl_ext;
471 char flowctrl[IFM_ETH_FC_STRLEN];
472
473 sc->dev = dev;
474 hw = &sc->hw;
475 hw->back = sc;
476
477 if_initname(&sc->arpcom.ac_if, device_get_name(dev),
478 device_get_unit(dev));
479 ifmedia_init(&sc->media, IFM_IMASK | IFM_ETH_FCMASK,
480 ix_media_change, ix_media_status);
481
482 /* Save frame size */
483 sc->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
484
485 sc->direct_input = ix_direct_input;
486 TASK_INIT(&sc->wdog_task, 0, ix_watchdog_task, sc);
487
488 callout_init_mp(&sc->fw_timer);
489 callout_init_mp(&sc->timer);
490 lwkt_serialize_init(&sc->main_serialize);
491
492 /*
493 * Save off the information about this board
494 */
495 hw->vendor_id = pci_get_vendor(dev);
496 hw->device_id = pci_get_device(dev);
497 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
498 hw->subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2);
499 hw->subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2);
500
501 /* Enable bus mastering */
502 pci_enable_busmaster(dev);
503
504 /*
505 * Allocate IO memory
506 */
507 sc->mem_rid = PCIR_BAR(0);
508 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
509 &sc->mem_rid, RF_ACTIVE);
510 if (sc->mem_res == NULL) {
511 device_printf(dev, "Unable to allocate bus resource: memory\n");
512 error = ENXIO;
513 goto failed;
514 }
515
516 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->mem_res);
517 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->mem_res);
518
519 sc->hw.hw_addr = (uint8_t *)&sc->osdep.mem_bus_space_handle;
520
521 /* Let hardware know driver is loaded */
522 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
523 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
524 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
525
526 /*
527 * Initialize the shared code
528 */
529 if (ixgbe_init_shared_code(hw)) {
530 device_printf(dev, "Unable to initialize the shared code\n");
531 error = ENXIO;
532 goto failed;
533 }
534
535 if (hw->mbx.ops.init_params)
536 hw->mbx.ops.init_params(hw);
537
538 hw->allow_unsupported_sfp = ix_unsupported_sfp;
539
540 /* Pick up the 82599 settings */
541 if (hw->mac.type != ixgbe_mac_82598EB)
542 hw->phy.smart_speed = ix_smart_speed;
543
544 /* Setup hardware capabilities */
545 ix_setup_caps(sc);
546
547 /* Allocate multicast array memory. */
548 sc->mta = kmalloc(sizeof(*sc->mta) * IX_MAX_MCASTADDR,
549 M_DEVBUF, M_WAITOK);
550
551 /* Save initial wake up filter configuration; WOL is disabled. */
552 sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);
553
554 /* Verify adapter fan is still functional (if applicable) */
555 if (sc->caps & IX_CAP_DETECT_FANFAIL)
556 ix_detect_fanfail(sc, IXGBE_READ_REG(hw, IXGBE_ESDP), FALSE);
557
558 /* Ensure SW/FW semaphore is free */
559 ixgbe_init_swfw_semaphore(hw);
560
561 #ifdef notyet
562 /* Enable EEE power saving */
563 if (sc->caps & IX_CAP_EEE)
564 hw->mac.ops.setup_eee(hw, true);
565 #endif
566
567 /*
568 * Configure total supported RX/TX ring count
569 */
570 ix_get_rxring_cnt(sc, &ring_cnt, &ring_cntmax);
571 sc->rx_rmap = if_ringmap_alloc(dev, ring_cnt, ring_cntmax);
572 ix_get_txring_cnt(sc, &ring_cnt, &ring_cntmax);
573 sc->tx_rmap = if_ringmap_alloc(dev, ring_cnt, ring_cntmax);
574 if_ringmap_match(dev, sc->rx_rmap, sc->tx_rmap);
575
576 sc->rx_ring_cnt = if_ringmap_count(sc->rx_rmap);
577 sc->rx_ring_inuse = sc->rx_ring_cnt;
578 sc->tx_ring_cnt = if_ringmap_count(sc->tx_rmap);
579 sc->tx_ring_inuse = sc->tx_ring_cnt;
580
581 /* Allocate TX/RX rings */
582 error = ix_alloc_rings(sc);
583 if (error)
584 goto failed;
585
586 /* Allocate interrupt */
587 error = ix_alloc_intr(sc);
588 if (error)
589 goto failed;
590
591 /* Setup serializes */
592 ix_setup_serialize(sc);
593
594 hw->phy.reset_if_overtemp = TRUE;
595 error = ixgbe_reset_hw(hw);
596 hw->phy.reset_if_overtemp = FALSE;
597 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
598 /*
599 * No optics in this port; ask timer routine
600 * to probe for later insertion.
601 */
602 sc->sfp_probe = TRUE;
603 error = 0;
604 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
605 device_printf(dev, "Unsupported SFP+ module detected!\n");
606 error = EIO;
607 goto failed;
608 } else if (error) {
609 device_printf(dev, "Hardware initialization failed\n");
610 error = EIO;
611 goto failed;
612 }
613
614 /* Make sure we have a good EEPROM before we read from it */
615 if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) {
616 device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
617 error = EIO;
618 goto failed;
619 }
620
621 error = ixgbe_start_hw(hw);
622 if (error == IXGBE_ERR_EEPROM_VERSION) {
623 device_printf(dev, "Pre-production device detected\n");
624 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
625 device_printf(dev, "Unsupported SFP+ Module\n");
626 error = EIO;
627 goto failed;
628 } else if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
629 device_printf(dev, "No SFP+ Module found\n");
630 }
631
632 /* Enable the optics for 82599 SFP+ fiber */
633 ixgbe_enable_tx_laser(hw);
634
635 /* Enable power to the phy. */
636 ixgbe_set_phy_power(hw, TRUE);
637
638 sc->ifm_media = IX_IFM_DEFAULT;
639 /* Get default flow control settings */
640 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl),
641 ix_flowctrl);
642 sc->ifm_media |= ifmedia_str2ethfc(flowctrl);
643 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN;
644
645 /* Setup OS specific network interface */
646 ix_setup_ifp(sc);
647
648 /* Add sysctl tree */
649 ix_add_sysctl(sc);
650
651 error = ix_setup_intr(sc);
652 if (error) {
653 ether_ifdetach(&sc->arpcom.ac_if);
654 goto failed;
655 }
656
657 /* Initialize statistics */
658 ix_update_stats(sc);
659
660 /* Check PCIE slot type/speed/width */
661 ix_slot_info(sc);
662
663 if (sc->caps & IX_CAP_FW_RECOVERY) {
664 device_printf(dev, "start fw timer\n");
665 callout_reset_bycpu(&sc->fw_timer, hz,
666 ix_fw_timer, sc, ix_get_timer_cpuid(sc, FALSE));
667 }
668
669 return 0;
670 failed:
671 ix_detach(dev);
672 return error;
673 }
674
675 static int
676 ix_detach(device_t dev)
677 {
678 struct ix_softc *sc = device_get_softc(dev);
679
680 if (device_is_attached(dev)) {
681 struct ifnet *ifp = &sc->arpcom.ac_if;
682
683 ix_sync_netisr(sc, IFF_UP);
684 taskqueue_drain(taskqueue_thread[0], &sc->wdog_task);
685
686 ifnet_serialize_all(ifp);
687
688 ix_powerdown(sc);
689 ix_teardown_intr(sc, sc->intr_cnt);
690
691 ifnet_deserialize_all(ifp);
692
693 callout_terminate(&sc->timer);
694 ether_ifdetach(ifp);
695 }
696 callout_terminate(&sc->fw_timer);
697
698 if (sc->mem_res != NULL) {
699 uint32_t ctrl_ext;
700
701 /* Let hardware know driver is unloading */
702 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
703 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
704 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
705 }
706
707 ifmedia_removeall(&sc->media);
708 bus_generic_detach(dev);
709
710 ix_free_intr(sc);
711
712 if (sc->msix_mem_res != NULL) {
713 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_mem_rid,
714 sc->msix_mem_res);
715 }
716 if (sc->mem_res != NULL) {
717 bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid,
718 sc->mem_res);
719 }
720
721 ix_free_rings(sc);
722
723 if (sc->mta != NULL)
724 kfree(sc->mta, M_DEVBUF);
725 if (sc->serializes != NULL)
726 kfree(sc->serializes, M_DEVBUF);
727
728 if (sc->rx_rmap != NULL)
729 if_ringmap_free(sc->rx_rmap);
730 if (sc->rx_rmap_intr != NULL)
731 if_ringmap_free(sc->rx_rmap_intr);
732 if (sc->tx_rmap != NULL)
733 if_ringmap_free(sc->tx_rmap);
734 if (sc->tx_rmap_intr != NULL)
735 if_ringmap_free(sc->tx_rmap_intr);
736
737 return 0;
738 }
739
740 static int
741 ix_shutdown(device_t dev)
742 {
743 struct ix_softc *sc = device_get_softc(dev);
744 struct ifnet *ifp = &sc->arpcom.ac_if;
745
746 ix_sync_netisr(sc, IFF_UP);
747 taskqueue_drain(taskqueue_thread[0], &sc->wdog_task);
748
749 ifnet_serialize_all(ifp);
750 ix_powerdown(sc);
751 ifnet_deserialize_all(ifp);
752
753 return 0;
754 }
755
756 static void
757 ix_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
758 {
759 struct ix_softc *sc = ifp->if_softc;
760 struct ix_tx_ring *txr = ifsq_get_priv(ifsq);
761 int idx = -1;
762 uint16_t nsegs;
763
764 KKASSERT(txr->tx_ifsq == ifsq);
765 ASSERT_SERIALIZED(&txr->tx_serialize);
766
767 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
768 return;
769
770 if (!sc->link_active || (txr->tx_flags & IX_TXFLAG_ENABLED) == 0) {
771 ifsq_purge(ifsq);
772 return;
773 }
774
775 while (!ifsq_is_empty(ifsq)) {
776 struct mbuf *m_head;
777
778 if (txr->tx_avail <= IX_MAX_SCATTER + IX_TX_RESERVED) {
779 ifsq_set_oactive(ifsq);
780 ifsq_watchdog_set_count(&txr->tx_watchdog, 5);
781 break;
782 }
783
784 m_head = ifsq_dequeue(ifsq);
785 if (m_head == NULL)
786 break;
787
788 if (ix_encap(txr, &m_head, &nsegs, &idx)) {
789 IFNET_STAT_INC(ifp, oerrors, 1);
790 continue;
791 }
792
793 /*
794 * TX interrupt are aggressively aggregated, so increasing
795 * opackets at TX interrupt time will make the opackets
796 * statistics vastly inaccurate; we do the opackets increment
797 * now.
798 */
799 IFNET_STAT_INC(ifp, opackets, 1);
800
801 if (nsegs >= txr->tx_wreg_nsegs) {
802 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx);
803 nsegs = 0;
804 idx = -1;
805 }
806
807 ETHER_BPF_MTAP(ifp, m_head);
808 }
809 if (idx >= 0)
810 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->tx_idx), idx);
811 txr->tx_running = IX_TX_RUNNING;
812 }
813
814 static int
815 ix_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
816 {
817 struct ix_softc *sc = ifp->if_softc;
818 struct ifreq *ifr = (struct ifreq *) data;
819 int error = 0, mask, reinit;
820
821 ASSERT_IFNET_SERIALIZED_ALL(ifp);
822
823 switch (command) {
824 case SIOCSIFMTU:
825 if (ifr->ifr_mtu > IX_MAX_MTU) {
826 error = EINVAL;
827 } else {
828 ifp->if_mtu = ifr->ifr_mtu;
829 sc->max_frame_size = ifp->if_mtu + IX_MTU_HDR;
830 ix_init(sc);
831 }
832 break;
833
834 case SIOCSIFFLAGS:
835 if (ifp->if_flags & IFF_UP) {
836 if (ifp->if_flags & IFF_RUNNING) {
837 if ((ifp->if_flags ^ sc->if_flags) &
838 (IFF_PROMISC | IFF_ALLMULTI))
839 ix_set_promisc(sc);
840 } else {
841 ix_init(sc);
842 }
843 } else if (ifp->if_flags & IFF_RUNNING) {
844 ix_stop(sc);
845 }
846 sc->if_flags = ifp->if_flags;
847 break;
848
849 case SIOCADDMULTI:
850 case SIOCDELMULTI:
851 if (ifp->if_flags & IFF_RUNNING) {
852 ix_disable_intr(sc);
853 ix_set_multi(sc);
854 #ifdef IFPOLL_ENABLE
855 if ((ifp->if_flags & IFF_NPOLLING) == 0)
856 #endif
857 ix_enable_intr(sc);
858 }
859 break;
860
861 case SIOCSIFMEDIA:
862 case SIOCGIFMEDIA:
863 error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
864 break;
865
866 case SIOCSIFCAP:
867 reinit = 0;
868 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
869 if (mask & IFCAP_RXCSUM) {
870 ifp->if_capenable ^= IFCAP_RXCSUM;
871 reinit = 1;
872 }
873 if (mask & IFCAP_VLAN_HWTAGGING) {
874 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
875 reinit = 1;
876 }
877 if (mask & IFCAP_TXCSUM) {
878 ifp->if_capenable ^= IFCAP_TXCSUM;
879 if (ifp->if_capenable & IFCAP_TXCSUM)
880 ifp->if_hwassist |= CSUM_OFFLOAD;
881 else
882 ifp->if_hwassist &= ~CSUM_OFFLOAD;
883 }
884 if (mask & IFCAP_TSO) {
885 ifp->if_capenable ^= IFCAP_TSO;
886 if (ifp->if_capenable & IFCAP_TSO)
887 ifp->if_hwassist |= CSUM_TSO;
888 else
889 ifp->if_hwassist &= ~CSUM_TSO;
890 }
891 if (mask & IFCAP_RSS)
892 ifp->if_capenable ^= IFCAP_RSS;
893 if (reinit && (ifp->if_flags & IFF_RUNNING))
894 ix_init(sc);
895 break;
896
897 #if 0
898 case SIOCGI2C:
899 {
900 struct ixgbe_i2c_req i2c;
901 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
902 if (error)
903 break;
904 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
905 error = EINVAL;
906 break;
907 }
908 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
909 i2c.dev_addr, i2c.data);
910 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
911 break;
912 }
913 #endif
914
915 default:
916 error = ether_ioctl(ifp, command, data);
917 break;
918 }
919 return error;
920 }
921
922 #define IXGBE_MHADD_MFS_SHIFT 16
923
924 static void
925 ix_init(void *xsc)
926 {
927 struct ix_softc *sc = xsc;
928 struct ifnet *ifp = &sc->arpcom.ac_if;
929 struct ixgbe_hw *hw = &sc->hw;
930 uint32_t rxctrl;
931 int i, error;
932 boolean_t polling;
933
934 ASSERT_IFNET_SERIALIZED_ALL(ifp);
935
936 ix_stop(sc);
937
938 if (sc->flags & IX_FLAG_FW_RECOVERY)
939 return;
940
941 polling = FALSE;
942 #ifdef IFPOLL_ENABLE
943 if (ifp->if_flags & IFF_NPOLLING)
944 polling = TRUE;
945 #endif
946
947 /* Configure # of used RX/TX rings */
948 ix_set_ring_inuse(sc, polling);
949 ifq_set_subq_divisor(&ifp->if_snd, sc->tx_ring_inuse);
950
951 /* Get the latest mac address, User can use a LAA */
952 bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
953 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
954 hw->addr_ctrl.rar_used_count = 1;
955
956 /* Prepare transmit descriptors and buffers */
957 for (i = 0; i < sc->tx_ring_inuse; ++i)
958 ix_init_tx_ring(&sc->tx_rings[i]);
959
960 ixgbe_init_hw(hw);
961 ix_init_tx_unit(sc);
962
963 /* Setup Multicast table */
964 ix_set_multi(sc);
965
966 /* Prepare receive descriptors and buffers */
967 for (i = 0; i < sc->rx_ring_inuse; ++i) {
968 error = ix_init_rx_ring(&sc->rx_rings[i]);
969 if (error) {
970 if_printf(ifp, "Could not initialize RX ring%d\n", i);
971 ix_stop(sc);
972 return;
973 }
974 }
975
976 /* Configure RX settings */
977 ix_init_rx_unit(sc, polling);
978
979 /* Enable SDP & MSI-X interrupts based on adapter */
980 ix_config_gpie(sc);
981
982 /* Set MTU size */
983 if (ifp->if_mtu > ETHERMTU) {
984 uint32_t mhadd;
985
986 /* aka IXGBE_MAXFRS on 82599 and newer */
987 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
988 mhadd &= ~IXGBE_MHADD_MFS_MASK;
989 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
990 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
991 }
992
993 /*
994 * Enable TX rings
995 */
996 for (i = 0; i < sc->tx_ring_inuse; ++i) {
997 uint32_t txdctl;
998
999 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1000 txdctl |= IXGBE_TXDCTL_ENABLE;
1001
1002 /*
1003 * Set WTHRESH to 0, since TX head write-back is used
1004 */
1005 txdctl &= ~(0x7f << 16);
1006
1007 /*
1008 * When the internal queue falls below PTHRESH (32),
1009 * start prefetching as long as there are at least
1010 * HTHRESH (1) buffers ready. The values are taken
1011 * from the Intel linux driver 3.8.21.
1012 * Prefetching enables tx line rate even with 1 queue.
1013 */
1014 txdctl |= (32 << 0) | (1 << 8);
1015 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1016 }
1017
1018 /*
1019 * Enable RX rings
1020 */
1021 for (i = 0; i < sc->rx_ring_inuse; ++i) {
1022 uint32_t rxdctl;
1023 int k;
1024
1025 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1026 if (hw->mac.type == ixgbe_mac_82598EB) {
1027 /*
1028 * PTHRESH = 21
1029 * HTHRESH = 4
1030 * WTHRESH = 8
1031 */
1032 rxdctl &= ~0x3FFFFF;
1033 rxdctl |= 0x080420;
1034 }
1035 rxdctl |= IXGBE_RXDCTL_ENABLE;
1036 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1037 for (k = 0; k < 10; ++k) {
1038 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1039 IXGBE_RXDCTL_ENABLE)
1040 break;
1041 else
1042 msec_delay(1);
1043 }
1044 wmb();
1045 IXGBE_WRITE_REG(hw, IXGBE_RDT(i),
1046 sc->rx_rings[0].rx_ndesc - 1);
1047 }
1048
1049 /* Enable Receive engine */
1050 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1051 if (hw->mac.type == ixgbe_mac_82598EB)
1052 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1053 rxctrl |= IXGBE_RXCTRL_RXEN;
1054 ixgbe_enable_rx_dma(hw, rxctrl);
1055
1056 for (i = 0; i < sc->tx_ring_inuse; ++i) {
1057 const struct ix_tx_ring *txr = &sc->tx_rings[i];
1058
1059 if (txr->tx_intr_vec >= 0) {
1060 ix_set_ivar(sc, i, txr->tx_intr_vec, 1);
1061 } else if (!polling) {
1062 /*
1063 * Unconfigured TX interrupt vector could only
1064 * happen for MSI-X.
1065 */
1066 KASSERT(sc->intr_type == PCI_INTR_TYPE_MSIX,
1067 ("TX intr vector is not set"));
1068 if (bootverbose)
1069 if_printf(ifp, "IVAR skips TX ring %d\n", i);
1070 }
1071 }
1072 for (i = 0; i < sc->rx_ring_inuse; ++i) {
1073 const struct ix_rx_ring *rxr = &sc->rx_rings[i];
1074
1075 if (polling && rxr->rx_intr_vec < 0)
1076 continue;
1077
1078 KKASSERT(rxr->rx_intr_vec >= 0);
1079 ix_set_ivar(sc, i, rxr->rx_intr_vec, 0);
1080 if (rxr->rx_txr != NULL) {
1081 /*
1082 * Piggyback the TX ring interrupt onto the RX
1083 * ring interrupt vector.
1084 */
1085 KASSERT(rxr->rx_txr->tx_intr_vec < 0,
1086 ("piggybacked TX ring configured intr vector"));
1087 ix_set_ivar(sc, rxr->rx_txr->tx_idx,
1088 rxr->rx_intr_vec, 1);
1089 if (bootverbose) {
1090 if_printf(ifp, "IVAR RX ring %d piggybacks "
1091 "TX ring %u\n", i, rxr->rx_txr->tx_idx);
1092 }
1093 }
1094 }
1095 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
1096 /* Set up status MSI-X vector; it is using fixed entry 1 */
1097 ix_set_ivar(sc, 1, sc->sts_msix_vec, -1);
1098
1099 /* Set up auto-mask for TX and RX rings */
1100 if (hw->mac.type == ixgbe_mac_82598EB) {
1101 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE);
1102 } else {
1103 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1104 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1105 }
1106 } else {
1107 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EIMS_RTX_QUEUE);
1108 }
1109 for (i = 0; i < sc->intr_cnt; ++i)
1110 ix_set_eitr(sc, i, sc->intr_data[i].intr_rate);
1111
1112 /*
1113 * Check on any SFP devices that need to be kick-started
1114 */
1115 if (hw->phy.type == ixgbe_phy_none) {
1116 error = hw->phy.ops.identify(hw);
1117 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1118 if_printf(ifp,
1119 "Unsupported SFP+ module type was detected.\n");
1120 /* XXX stop */
1121 return;
1122 }
1123 }
1124
1125 /* Config/Enable Link */
1126 ix_config_link(sc);
1127
1128 /* Hardware Packet Buffer & Flow Control setup */
1129 ix_config_flowctrl(sc);
1130
1131 /* Initialize the FC settings */
1132 ixgbe_start_hw(hw);
1133
1134 /* Set up VLAN support and filter */
1135 ix_set_vlan(sc);
1136
1137 /* Setup DMA Coalescing */
1138 ix_config_dmac(sc);
1139
1140 /*
1141 * Only enable interrupts if we are not polling, make sure
1142 * they are off otherwise.
1143 */
1144 if (polling)
1145 ix_disable_intr(sc);
1146 else
1147 ix_enable_intr(sc);
1148
1149 ifp->if_flags |= IFF_RUNNING;
1150 for (i = 0; i < sc->tx_ring_inuse; ++i) {
1151 struct ix_tx_ring *txr = &sc->tx_rings[i];
1152
1153 ifsq_clr_oactive(txr->tx_ifsq);
1154 ifsq_watchdog_start(&txr->tx_watchdog);
1155
1156 if (!polling) {
1157 callout_reset_bycpu(&txr->tx_gc_timer, 1,
1158 ix_txgc_timer, txr, txr->tx_intr_cpuid);
1159 }
1160 }
1161
1162 sc->timer_cpuid = ix_get_timer_cpuid(sc, polling);
1163 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid);
1164 }
1165
1166 static void
1167 ix_intr(void *xsc)
1168 {
1169 struct ix_softc *sc = xsc;
1170 struct ixgbe_hw *hw = &sc->hw;
1171 uint32_t eicr;
1172
1173 ASSERT_SERIALIZED(&sc->main_serialize);
1174
1175 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1176 if (eicr == 0) {
1177 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask);
1178 return;
1179 }
1180
1181 if (eicr & IX_RX0_INTR_MASK) {
1182 struct ix_rx_ring *rxr = &sc->rx_rings[0];
1183
1184 lwkt_serialize_enter(&rxr->rx_serialize);
1185 ix_rxeof(rxr, -1);
1186 lwkt_serialize_exit(&rxr->rx_serialize);
1187 }
1188 if (eicr & IX_RX1_INTR_MASK) {
1189 struct ix_rx_ring *rxr;
1190
1191 KKASSERT(sc->rx_ring_inuse == IX_MIN_RXRING_RSS);
1192 rxr = &sc->rx_rings[1];
1193
1194 lwkt_serialize_enter(&rxr->rx_serialize);
1195 ix_rxeof(rxr, -1);
1196 lwkt_serialize_exit(&rxr->rx_serialize);
1197 }
1198
1199 if (eicr & IX_TX_INTR_MASK) {
1200 struct ix_tx_ring *txr = &sc->tx_rings[0];
1201
1202 lwkt_serialize_enter(&txr->tx_serialize);
1203 ix_tx_intr(txr, *(txr->tx_hdr));
1204 lwkt_serialize_exit(&txr->tx_serialize);
1205 }
1206
1207 if (__predict_false(eicr & IX_EICR_STATUS))
1208 ix_intr_status(sc, eicr);
1209
1210 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask);
1211 }
1212
1213 static void
1214 ix_intr_82598(void *xsc)
1215 {
1216 struct ix_softc *sc = xsc;
1217
1218 ASSERT_SERIALIZED(&sc->main_serialize);
1219
1220 /* Software workaround for 82598 errata #26 */
1221 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
1222
1223 ix_intr(sc);
1224 }
1225
1226 static void
1227 ix_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1228 {
1229 struct ix_softc *sc = ifp->if_softc;
1230 struct ifmedia *ifm = &sc->media;
1231 int layer;
1232 boolean_t link_active;
1233
1234 if (sc->flags & IX_FLAG_FW_RECOVERY) {
1235 link_active = FALSE;
1236 } else {
1237 ix_update_link_status(sc);
1238 link_active = sc->link_active;
1239 }
1240
1241 ifmr->ifm_status = IFM_AVALID;
1242 ifmr->ifm_active = IFM_ETHER;
1243
1244 if (!link_active) {
1245 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO)
1246 ifmr->ifm_active |= ifm->ifm_media;
1247 else
1248 ifmr->ifm_active |= IFM_NONE;
1249 return;
1250 }
1251 ifmr->ifm_status |= IFM_ACTIVE;
1252
1253 layer = sc->phy_layer;
1254
1255 if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) ||
1256 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) ||
1257 (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) ||
1258 (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)) {
1259 switch (sc->link_speed) {
1260 case IXGBE_LINK_SPEED_10GB_FULL:
1261 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1262 break;
1263 case IXGBE_LINK_SPEED_1GB_FULL:
1264 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1265 break;
1266 case IXGBE_LINK_SPEED_100_FULL:
1267 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1268 break;
1269 case IXGBE_LINK_SPEED_10_FULL:
1270 ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1271 break;
1272 }
1273 } else if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) ||
1274 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) {
1275 switch (sc->link_speed) {
1276 case IXGBE_LINK_SPEED_10GB_FULL:
1277 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
1278 break;
1279 }
1280 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
1281 switch (sc->link_speed) {
1282 case IXGBE_LINK_SPEED_10GB_FULL:
1283 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
1284 break;
1285 case IXGBE_LINK_SPEED_1GB_FULL:
1286 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1287 break;
1288 }
1289 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) {
1290 switch (sc->link_speed) {
1291 case IXGBE_LINK_SPEED_10GB_FULL:
1292 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
1293 break;
1294 case IXGBE_LINK_SPEED_1GB_FULL:
1295 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1296 break;
1297 }
1298 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) ||
1299 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)) {
1300 switch (sc->link_speed) {
1301 case IXGBE_LINK_SPEED_10GB_FULL:
1302 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1303 break;
1304 case IXGBE_LINK_SPEED_1GB_FULL:
1305 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1306 break;
1307 }
1308 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) {
1309 switch (sc->link_speed) {
1310 case IXGBE_LINK_SPEED_10GB_FULL:
1311 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
1312 break;
1313 }
1314 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
1315 /*
1316 * XXX: These need to use the proper media types once
1317 * they're added.
1318 */
1319 switch (sc->link_speed) {
1320 case IXGBE_LINK_SPEED_10GB_FULL:
1321 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1322 break;
1323 case IXGBE_LINK_SPEED_2_5GB_FULL:
1324 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
1325 break;
1326 case IXGBE_LINK_SPEED_1GB_FULL:
1327 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
1328 break;
1329 }
1330 } else if ((layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) ||
1331 (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) ||
1332 (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)) {
1333 /*
1334 * XXX: These need to use the proper media types once
1335 * they're added.
1336 */
1337 switch (sc->link_speed) {
1338 case IXGBE_LINK_SPEED_10GB_FULL:
1339 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
1340 break;
1341 case IXGBE_LINK_SPEED_2_5GB_FULL:
1342 ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
1343 break;
1344 case IXGBE_LINK_SPEED_1GB_FULL:
1345 ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
1346 break;
1347 }
1348 }
1349
1350 /* If nothing is recognized... */
1351 if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
1352 ifmr->ifm_active |= IFM_NONE;
1353
1354 if (sc->ifm_media & IFM_ETH_FORCEPAUSE)
1355 ifmr->ifm_active |= (sc->ifm_media & IFM_ETH_FCMASK);
1356
1357 switch (sc->hw.fc.current_mode) {
1358 case ixgbe_fc_full:
1359 ifmr->ifm_active |= IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE;
1360 break;
1361 case ixgbe_fc_rx_pause:
1362 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1363 break;
1364 case ixgbe_fc_tx_pause:
1365 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1366 break;
1367 default:
1368 break;
1369 }
1370 }
1371
1372 static int
1373 ix_media_change(struct ifnet *ifp)
1374 {
1375 struct ix_softc *sc = ifp->if_softc;
1376 struct ifmedia *ifm = &sc->media;
1377 struct ixgbe_hw *hw = &sc->hw;
1378
1379 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1380 return (EINVAL);
1381
1382 if (hw->phy.media_type == ixgbe_media_type_backplane ||
1383 hw->mac.ops.setup_link == NULL) {
1384 if ((ifm->ifm_media ^ sc->ifm_media) & IFM_ETH_FCMASK) {
1385 /* Only flow control setting changes are allowed */
1386 return (EOPNOTSUPP);
1387 }
1388 }
1389
1390 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1391 case IFM_AUTO:
1392 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN;
1393 break;
1394
1395 case IFM_10G_T:
1396 case IFM_10G_LRM:
1397 case IFM_10G_SR: /* XXX also KR */
1398 case IFM_10G_LR:
1399 case IFM_10G_CX4: /* XXX also KX4 */
1400 case IFM_10G_TWINAX:
1401 sc->advspeed = IXGBE_LINK_SPEED_10GB_FULL;
1402 break;
1403
1404 case IFM_1000_T:
1405 case IFM_1000_LX:
1406 case IFM_1000_SX:
1407 case IFM_1000_CX: /* XXX is KX */
1408 sc->advspeed = IXGBE_LINK_SPEED_1GB_FULL;
1409 break;
1410
1411 case IFM_100_TX:
1412 sc->advspeed = IXGBE_LINK_SPEED_100_FULL;
1413 break;
1414
1415 default:
1416 if (bootverbose) {
1417 if_printf(ifp, "Invalid media type %d!\n",
1418 ifm->ifm_media);
1419 }
1420 return EINVAL;
1421 }
1422 sc->ifm_media = ifm->ifm_media;
1423
1424 #if 0
1425 if (hw->mac.ops.setup_link != NULL) {
1426 hw->mac.autotry_restart = TRUE;
1427 hw->mac.ops.setup_link(hw, sc->advspeed, TRUE);
1428 }
1429 #else
1430 if (ifp->if_flags & IFF_RUNNING)
1431 ix_init(sc);
1432 #endif
1433 return 0;
1434 }
1435
1436 static __inline int
1437 ix_tso_pullup(struct mbuf **mp)
1438 {
1439 int hoff, iphlen, thoff;
1440 struct mbuf *m;
1441
1442 m = *mp;
1443 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
1444
1445 iphlen = m->m_pkthdr.csum_iphlen;
1446 thoff = m->m_pkthdr.csum_thlen;
1447 hoff = m->m_pkthdr.csum_lhlen;
1448
1449 KASSERT(iphlen > 0, ("invalid ip hlen"));
1450 KASSERT(thoff > 0, ("invalid tcp hlen"));
1451 KASSERT(hoff > 0, ("invalid ether hlen"));
1452
1453 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
1454 m = m_pullup(m, hoff + iphlen + thoff);
1455 if (m == NULL) {
1456 *mp = NULL;
1457 return ENOBUFS;
1458 }
1459 *mp = m;
1460 }
1461 return 0;
1462 }
1463
1464 static int
1465 ix_encap(struct ix_tx_ring *txr, struct mbuf **m_headp,
1466 uint16_t *segs_used, int *idx)
1467 {
1468 uint32_t olinfo_status = 0, cmd_type_len, cmd_rs = 0;
1469 int i, j, error, nsegs, first, maxsegs;
1470 struct mbuf *m_head = *m_headp;
1471 bus_dma_segment_t segs[IX_MAX_SCATTER];
1472 bus_dmamap_t map;
1473 struct ix_tx_buf *txbuf;
1474 union ixgbe_adv_tx_desc *txd = NULL;
1475
1476 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1477 error = ix_tso_pullup(m_headp);
1478 if (__predict_false(error))
1479 return error;
1480 m_head = *m_headp;
1481 }
1482
1483 /* Basic descriptor defines */
1484 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1485 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1486
1487 if (m_head->m_flags & M_VLANTAG)
1488 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1489
1490 /*
1491 * Important to capture the first descriptor
1492 * used because it will contain the index of
1493 * the one we tell the hardware to report back
1494 */
1495 first = txr->tx_next_avail;
1496 txbuf = &txr->tx_buf[first];
1497 map = txbuf->map;
1498
1499 /*
1500 * Map the packet for DMA.
1501 */
1502 maxsegs = txr->tx_avail - IX_TX_RESERVED;
1503 if (maxsegs > IX_MAX_SCATTER)
1504 maxsegs = IX_MAX_SCATTER;
1505
1506 error = bus_dmamap_load_mbuf_defrag(txr->tx_tag, map, m_headp,
1507 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1508 if (__predict_false(error)) {
1509 m_freem(*m_headp);
1510 *m_headp = NULL;
1511 return error;
1512 }
1513 bus_dmamap_sync(txr->tx_tag, map, BUS_DMASYNC_PREWRITE);
1514
1515 m_head = *m_headp;
1516
1517 /*
1518 * Set up the appropriate offload context if requested,
1519 * this may consume one TX descriptor.
1520 */
1521 if (ix_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status)) {
1522 (*segs_used)++;
1523 txr->tx_nsegs++;
1524 }
1525
1526 *segs_used += nsegs;
1527 txr->tx_nsegs += nsegs;
1528 if (txr->tx_nsegs >= txr->tx_intr_nsegs) {
1529 /*
1530 * Report Status (RS) is turned on every intr_nsegs
1531 * descriptors (roughly).
1532 */
1533 txr->tx_nsegs = 0;
1534 cmd_rs = IXGBE_TXD_CMD_RS;
1535 }
1536
1537 i = txr->tx_next_avail;
1538 for (j = 0; j < nsegs; j++) {
1539 bus_size_t seglen;
1540 bus_addr_t segaddr;
1541
1542 txbuf = &txr->tx_buf[i];
1543 txd = &txr->tx_base[i];
1544 seglen = segs[j].ds_len;
1545 segaddr = htole64(segs[j].ds_addr);
1546
1547 txd->read.buffer_addr = segaddr;
1548 txd->read.cmd_type_len = htole32(IXGBE_TXD_CMD_IFCS |
1549 cmd_type_len |seglen);
1550 txd->read.olinfo_status = htole32(olinfo_status);
1551
1552 if (++i == txr->tx_ndesc)
1553 i = 0;
1554 }
1555 txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | cmd_rs);
1556
1557 txr->tx_avail -= nsegs;
1558 txr->tx_next_avail = i;
1559 txr->tx_nmbuf++;
1560
1561 txbuf->m_head = m_head;
1562 txr->tx_buf[first].map = txbuf->map;
1563 txbuf->map = map;
1564
1565 /*
1566 * Defer TDT updating, until enough descrptors are setup
1567 */
1568 *idx = i;
1569
1570 return 0;
1571 }
1572
1573 static void
1574 ix_set_promisc(struct ix_softc *sc)
1575 {
1576 struct ifnet *ifp = &sc->arpcom.ac_if;
1577 uint32_t reg_rctl;
1578 int mcnt = 0;
1579
1580 reg_rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
1581 reg_rctl &= ~IXGBE_FCTRL_UPE;
1582 if (ifp->if_flags & IFF_ALLMULTI) {
1583 mcnt = IX_MAX_MCASTADDR;
1584 } else {
1585 struct ifmultiaddr *ifma;
1586
1587 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1588 if (ifma->ifma_addr->sa_family != AF_LINK)
1589 continue;
1590 if (mcnt == IX_MAX_MCASTADDR)
1591 break;
1592 mcnt++;
1593 }
1594 }
1595 if (mcnt < IX_MAX_MCASTADDR)
1596 reg_rctl &= ~IXGBE_FCTRL_MPE;
1597 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl);
1598
1599 if (ifp->if_flags & IFF_PROMISC) {
1600 reg_rctl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE;
1601 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl);
1602 } else if (ifp->if_flags & IFF_ALLMULTI) {
1603 reg_rctl |= IXGBE_FCTRL_MPE;
1604 reg_rctl &= ~IXGBE_FCTRL_UPE;
1605 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, reg_rctl);
1606 }
1607 }
1608
1609 static void
1610 ix_set_multi(struct ix_softc *sc)
1611 {
1612 struct ifnet *ifp = &sc->arpcom.ac_if;
1613 struct ifmultiaddr *ifma;
1614 uint32_t fctrl;
1615 struct ix_mc_addr *mta;
1616 int mcnt = 0;
1617
1618 mta = sc->mta;
1619 bzero(mta, sizeof(*mta) * IX_MAX_MCASTADDR);
1620
1621 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1622 if (ifma->ifma_addr->sa_family != AF_LINK)
1623 continue;
1624 if (mcnt == IX_MAX_MCASTADDR)
1625 break;
1626 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1627 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
1628 mcnt++;
1629 }
1630
1631 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
1632 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1633 if (ifp->if_flags & IFF_PROMISC) {
1634 fctrl |= IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE;
1635 } else if (mcnt >= IX_MAX_MCASTADDR || (ifp->if_flags & IFF_ALLMULTI)) {
1636 fctrl |= IXGBE_FCTRL_MPE;
1637 fctrl &= ~IXGBE_FCTRL_UPE;
1638 } else {
1639 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1640 }
1641 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);
1642
1643 if (mcnt < IX_MAX_MCASTADDR) {
1644 ixgbe_update_mc_addr_list(&sc->hw,
1645 (uint8_t *)mta, mcnt, ix_mc_array_itr, TRUE);
1646 }
1647 }
1648
1649 /*
1650 * This is an iterator function now needed by the multicast
1651 * shared code. It simply feeds the shared code routine the
1652 * addresses in the array of ix_set_multi() one by one.
1653 */
1654 static uint8_t *
1655 ix_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq)
1656 {
1657 struct ix_mc_addr *mta = (struct ix_mc_addr *)*update_ptr;
1658
1659 *vmdq = mta->vmdq;
1660 *update_ptr = (uint8_t *)(mta + 1);
1661
1662 return (mta->addr);
1663 }
1664
1665 static void
1666 ix_timer(void *arg)
1667 {
1668 struct ix_softc *sc = arg;
1669
1670 lwkt_serialize_enter(&sc->main_serialize);
1671
1672 if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0) {
1673 lwkt_serialize_exit(&sc->main_serialize);
1674 return;
1675 }
1676
1677 /* Check for pluggable optics */
1678 if (sc->sfp_probe) {
1679 if (!ix_sfp_probe(sc))
1680 goto done; /* Nothing to do */
1681 }
1682
1683 ix_update_link_status(sc);
1684 ix_update_stats(sc);
1685
1686 done:
1687 callout_reset_bycpu(&sc->timer, hz, ix_timer, sc, sc->timer_cpuid);
1688 lwkt_serialize_exit(&sc->main_serialize);
1689 }
1690
1691 static void
1692 ix_update_link_status(struct ix_softc *sc)
1693 {
1694 struct ifnet *ifp = &sc->arpcom.ac_if;
1695
1696 if (sc->link_up) {
1697 if (sc->link_active == FALSE) {
1698 if (bootverbose) {
1699 if_printf(ifp, "Link is up %d Gbps %s\n",
1700 sc->link_speed == 128 ? 10 : 1,
1701 "Full Duplex");
1702 }
1703
1704 /*
1705 * Update any Flow Control changes
1706 */
1707 ixgbe_fc_enable(&sc->hw);
1708 /* MUST after ixgbe_fc_enable() */
1709 if (sc->rx_ring_inuse > 1) {
1710 switch (sc->hw.fc.current_mode) {
1711 case ixgbe_fc_rx_pause:
1712 case ixgbe_fc_tx_pause:
1713 case ixgbe_fc_full:
1714 ix_disable_rx_drop(sc);
1715 break;
1716
1717 case ixgbe_fc_none:
1718 ix_enable_rx_drop(sc);
1719 break;
1720
1721 default:
1722 break;
1723 }
1724 }
1725
1726 /* Update DMA coalescing config */
1727 ix_config_dmac(sc);
1728
1729 sc->link_active = TRUE;
1730
1731 ifp->if_link_state = LINK_STATE_UP;
1732 if_link_state_change(ifp);
1733 }
1734 } else { /* Link down */
1735 if (sc->link_active == TRUE) {
1736 if (bootverbose)
1737 if_printf(ifp, "Link is Down\n");
1738 ifp->if_link_state = LINK_STATE_DOWN;
1739 if_link_state_change(ifp);
1740
1741 sc->link_active = FALSE;
1742 }
1743 }
1744 }
1745
1746 static void
1747 ix_stop(struct ix_softc *sc)
1748 {
1749 struct ixgbe_hw *hw = &sc->hw;
1750 struct ifnet *ifp = &sc->arpcom.ac_if;
1751 int i;
1752
1753 ASSERT_IFNET_SERIALIZED_ALL(ifp);
1754
1755 ix_disable_intr(sc);
1756 callout_stop(&sc->timer);
1757
1758 ifp->if_flags &= ~IFF_RUNNING;
1759 for (i = 0; i < sc->tx_ring_cnt; ++i) {
1760 struct ix_tx_ring *txr = &sc->tx_rings[i];
1761
1762 ifsq_clr_oactive(txr->tx_ifsq);
1763 ifsq_watchdog_stop(&txr->tx_watchdog);
1764 txr->tx_flags &= ~IX_TXFLAG_ENABLED;
1765
1766 txr->tx_running = 0;
1767 callout_stop(&txr->tx_gc_timer);
1768 }
1769
1770 ixgbe_reset_hw(hw);
1771 hw->adapter_stopped = FALSE;
1772 ixgbe_stop_adapter(hw);
1773 if (hw->mac.type == ixgbe_mac_82599EB)
1774 ixgbe_stop_mac_link_on_d3_82599(hw);
1775 /* Turn off the laser - noop with no optics */
1776 ixgbe_disable_tx_laser(hw);
1777
1778 /* Update the stack */
1779 sc->link_up = FALSE;
1780 ix_update_link_status(sc);
1781
1782 /* Reprogram the RAR[0] in case user changed it. */
1783 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1784
1785 for (i = 0; i < sc->tx_ring_cnt; ++i)
1786 ix_free_tx_ring(&sc->tx_rings[i]);
1787
1788 for (i = 0; i < sc->rx_ring_cnt; ++i)
1789 ix_free_rx_ring(&sc->rx_rings[i]);
1790 }
1791
1792 static void
1793 ix_setup_ifp(struct ix_softc *sc)
1794 {
1795 struct ixgbe_hw *hw = &sc->hw;
1796 struct ifnet *ifp = &sc->arpcom.ac_if;
1797 int i;
1798
1799 ifp->if_baudrate = IF_Gbps(10UL);
1800
1801 ifp->if_softc = sc;
1802 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1803 ifp->if_init = ix_init;
1804 ifp->if_ioctl = ix_ioctl;
1805 ifp->if_start = ix_start;
1806 ifp->if_serialize = ix_serialize;
1807 ifp->if_deserialize = ix_deserialize;
1808 ifp->if_tryserialize = ix_tryserialize;
1809 #ifdef INVARIANTS
1810 ifp->if_serialize_assert = ix_serialize_assert;
1811 #endif
1812 #ifdef IFPOLL_ENABLE
1813 ifp->if_npoll = ix_npoll;
1814 #endif
1815
1816 /* Increase TSO burst length */
1817 ifp->if_tsolen = (8 * ETHERMTU);
1818
1819 ifp->if_nmbclusters = sc->rx_ring_cnt * sc->rx_rings[0].rx_ndesc;
1820 ifp->if_nmbjclusters = ifp->if_nmbclusters;
1821
1822 ifq_set_maxlen(&ifp->if_snd, sc->tx_rings[0].tx_ndesc - 2);
1823 ifq_set_ready(&ifp->if_snd);
1824 ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt);
1825
1826 ifp->if_mapsubq = ifq_mapsubq_modulo;
1827 ifq_set_subq_divisor(&ifp->if_snd, 1);
1828
1829 ether_ifattach(ifp, hw->mac.addr, NULL);
1830
1831 ifp->if_capabilities =
1832 IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1833 if (IX_ENABLE_HWRSS(sc))
1834 ifp->if_capabilities |= IFCAP_RSS;
1835 ifp->if_capenable = ifp->if_capabilities;
1836 ifp->if_hwassist = CSUM_OFFLOAD | CSUM_TSO;
1837
1838 /*
1839 * Tell the upper layer(s) we support long frames.
1840 */
1841 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1842
1843 /* Setup TX rings and subqueues */
1844 for (i = 0; i < sc->tx_ring_cnt; ++i) {
1845 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
1846 struct ix_tx_ring *txr = &sc->tx_rings[i];
1847
1848 ifsq_set_cpuid(ifsq, txr->tx_intr_cpuid);
1849 ifsq_set_priv(ifsq, txr);
1850 ifsq_set_hw_serialize(ifsq, &txr->tx_serialize);
1851 txr->tx_ifsq = ifsq;
1852
1853 ifsq_watchdog_init(&txr->tx_watchdog, ifsq, ix_watchdog, 0);
1854 }
1855
1856 /* Specify the media types supported by this adapter */
1857 sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
1858 ix_init_media(sc);
1859 }
1860
1861 static boolean_t
1862 ix_is_sfp(struct ixgbe_hw *hw)
1863 {
1864 switch (hw->mac.type) {
1865 case ixgbe_mac_82598EB:
1866 if (hw->phy.type == ixgbe_phy_nl)
1867 return TRUE;
1868 return FALSE;
1869
1870 case ixgbe_mac_82599EB:
1871 switch (hw->mac.ops.get_media_type(hw)) {
1872 case ixgbe_media_type_fiber:
1873 case ixgbe_media_type_fiber_qsfp:
1874 return TRUE;
1875 default:
1876 return FALSE;
1877 }
1878
1879 case ixgbe_mac_X550EM_x:
1880 case ixgbe_mac_X550EM_a:
1881 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber)
1882 return TRUE;
1883 return FALSE;
1884
1885 default:
1886 return FALSE;
1887 }
1888 }
1889
1890 static void
1891 ix_config_link(struct ix_softc *sc)
1892 {
1893 struct ixgbe_hw *hw = &sc->hw;
1894 boolean_t sfp;
1895
1896 sfp = ix_is_sfp(hw);
1897 if (sfp) {
1898 if (hw->phy.multispeed_fiber)
1899 ixgbe_enable_tx_laser(hw);
1900 ix_handle_mod(sc);
1901 } else {
1902 uint32_t autoneg, err = 0;
1903
1904 if (hw->mac.ops.check_link != NULL) {
1905 err = ixgbe_check_link(hw, &sc->link_speed,
1906 &sc->link_up, FALSE);
1907 if (err)
1908 return;
1909 }
1910
1911 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN)
1912 autoneg = sc->advspeed;
1913 else
1914 autoneg = hw->phy.autoneg_advertised;
1915 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) {
1916 bool negotiate;
1917
1918 err = hw->mac.ops.get_link_capabilities(hw,
1919 &autoneg, &negotiate);
1920 if (err)
1921 return;
1922 }
1923
1924 if (hw->mac.ops.setup_link != NULL) {
1925 err = hw->mac.ops.setup_link(hw,
1926 autoneg, sc->link_up);
1927 if (err)
1928 return;
1929 }
1930 }
1931 }
1932
1933 static int
1934 ix_alloc_rings(struct ix_softc *sc)
1935 {
1936 int error, i;
1937
1938 /*
1939 * Create top level busdma tag
1940 */
1941 error = bus_dma_tag_create(NULL, 1, 0,
1942 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1943 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0,
1944 &sc->parent_tag);
1945 if (error) {
1946 device_printf(sc->dev, "could not create top level DMA tag\n");
1947 return error;
1948 }
1949
1950 /*
1951 * Allocate TX descriptor rings and buffers
1952 */
1953 sc->tx_rings = kmalloc(sizeof(struct ix_tx_ring) * sc->tx_ring_cnt,
1954 M_DEVBUF,
1955 M_WAITOK | M_ZERO | M_CACHEALIGN);
1956 for (i = 0; i < sc->tx_ring_cnt; ++i) {
1957 struct ix_tx_ring *txr = &sc->tx_rings[i];
1958
1959 txr->tx_sc = sc;
1960 txr->tx_idx = i;
1961 txr->tx_intr_vec = -1;
1962 txr->tx_intr_cpuid = -1;
1963 lwkt_serialize_init(&txr->tx_serialize);
1964 callout_init_mp(&txr->tx_gc_timer);
1965
1966 error = ix_create_tx_ring(txr);
1967 if (error)
1968 return error;
1969 }
1970
1971 /*
1972 * Allocate RX descriptor rings and buffers
1973 */
1974 sc->rx_rings = kmalloc(sizeof(struct ix_rx_ring) * sc->rx_ring_cnt,
1975 M_DEVBUF,
1976 M_WAITOK | M_ZERO | M_CACHEALIGN);
1977 for (i = 0; i < sc->rx_ring_cnt; ++i) {
1978 struct ix_rx_ring *rxr = &sc->rx_rings[i];
1979
1980 rxr->rx_sc = sc;
1981 rxr->rx_idx = i;
1982 rxr->rx_intr_vec = -1;
1983 lwkt_serialize_init(&rxr->rx_serialize);
1984
1985 error = ix_create_rx_ring(rxr);
1986 if (error)
1987 return error;
1988 }
1989
1990 return 0;
1991 }
1992
1993 static int
1994 ix_create_tx_ring(struct ix_tx_ring *txr)
1995 {
1996 int error, i, tsize, ntxd;
1997
1998 /*
1999 * Validate number of transmit descriptors. It must not exceed
2000 * hardware maximum, and must be multiple of IX_DBA_ALIGN.
2001 */
2002 ntxd = device_getenv_int(txr->tx_sc->dev, "txd", ix_txd);
2003 if (((ntxd * sizeof(union ixgbe_adv_tx_desc)) % IX_DBA_ALIGN) != 0 ||
2004 ntxd < IX_MIN_TXD || ntxd > IX_MAX_TXD) {
2005 device_printf(txr->tx_sc->dev,
2006 "Using %d TX descriptors instead of %d!\n",
2007 IX_DEF_TXD, ntxd);
2008 txr->tx_ndesc = IX_DEF_TXD;
2009 } else {
2010 txr->tx_ndesc = ntxd;
2011 }
2012
2013 /*
2014 * Allocate TX head write-back buffer
2015 */
2016 txr->tx_hdr = bus_dmamem_coherent_any(txr->tx_sc->parent_tag,
2017 __VM_CACHELINE_SIZE, __VM_CACHELINE_SIZE, BUS_DMA_WAITOK,
2018 &txr->tx_hdr_dtag, &txr->tx_hdr_map, &txr->tx_hdr_paddr);
2019 if (txr->tx_hdr == NULL) {
2020 device_printf(txr->tx_sc->dev,
2021 "Unable to allocate TX head write-back buffer\n");
2022 return ENOMEM;
2023 }
2024
2025 /*
2026 * Allocate TX descriptor ring
2027 */
2028 tsize = roundup2(txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc),
2029 IX_DBA_ALIGN);
2030 txr->tx_base = bus_dmamem_coherent_any(txr->tx_sc->parent_tag,
2031 IX_DBA_ALIGN, tsize, BUS_DMA_WAITOK | BUS_DMA_ZERO,
2032 &txr->tx_base_dtag, &txr->tx_base_map, &txr->tx_base_paddr);
2033 if (txr->tx_base == NULL) {
2034 device_printf(txr->tx_sc->dev,
2035 "Unable to allocate TX Descriptor memory\n");
2036 return ENOMEM;
2037 }
2038
2039 tsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_tx_buf) * txr->tx_ndesc);
2040 txr->tx_buf = kmalloc(tsize, M_DEVBUF,
2041 M_WAITOK | M_ZERO | M_CACHEALIGN);
2042
2043 /*
2044 * Create DMA tag for TX buffers
2045 */
2046 error = bus_dma_tag_create(txr->tx_sc->parent_tag,
2047 1, 0, /* alignment, bounds */
2048 BUS_SPACE_MAXADDR, /* lowaddr */
2049 BUS_SPACE_MAXADDR, /* highaddr */
2050 IX_TSO_SIZE, /* maxsize */
2051 IX_MAX_SCATTER, /* nsegments */
2052 PAGE_SIZE, /* maxsegsize */
2053 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
2054 BUS_DMA_ONEBPAGE, /* flags */
2055 &txr->tx_tag);
2056 if (error) {
2057 device_printf(txr->tx_sc->dev,
2058 "Unable to allocate TX DMA tag\n");
2059 kfree(txr->tx_buf, M_DEVBUF);
2060 txr->tx_buf = NULL;
2061 return error;
2062 }
2063
2064 /*
2065 * Create DMA maps for TX buffers
2066 */
2067 for (i = 0; i < txr->tx_ndesc; ++i) {
2068 struct ix_tx_buf *txbuf = &txr->tx_buf[i];
2069
2070 error = bus_dmamap_create(txr->tx_tag,
2071 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &txbuf->map);
2072 if (error) {
2073 device_printf(txr->tx_sc->dev,
2074 "Unable to create TX DMA map\n");
2075 ix_destroy_tx_ring(txr, i);
2076 return error;
2077 }
2078 }
2079
2080 /*
2081 * Initialize various watermark
2082 */
2083 txr->tx_wreg_nsegs = IX_DEF_TXWREG_NSEGS;
2084 txr->tx_intr_nsegs = txr->tx_ndesc / 16;
2085
2086 return 0;
2087 }
2088
2089 static void
2090 ix_destroy_tx_ring(struct ix_tx_ring *txr, int ndesc)
2091 {
2092 int i;
2093
2094 if (txr->tx_hdr != NULL) {
2095 bus_dmamap_unload(txr->tx_hdr_dtag, txr->tx_hdr_map);
2096 bus_dmamem_free(txr->tx_hdr_dtag,
2097 __DEVOLATILE(void *, txr->tx_hdr), txr->tx_hdr_map);
2098 bus_dma_tag_destroy(txr->tx_hdr_dtag);
2099 txr->tx_hdr = NULL;
2100 }
2101
2102 if (txr->tx_base != NULL) {
2103 bus_dmamap_unload(txr->tx_base_dtag, txr->tx_base_map);
2104 bus_dmamem_free(txr->tx_base_dtag, txr->tx_base,
2105 txr->tx_base_map);
2106 bus_dma_tag_destroy(txr->tx_base_dtag);
2107 txr->tx_base = NULL;
2108 }
2109
2110 if (txr->tx_buf == NULL)
2111 return;
2112
2113 for (i = 0; i < ndesc; ++i) {
2114 struct ix_tx_buf *txbuf = &txr->tx_buf[i];
2115
2116 KKASSERT(txbuf->m_head == NULL);
2117 bus_dmamap_destroy(txr->tx_tag, txbuf->map);
2118 }
2119 bus_dma_tag_destroy(txr->tx_tag);
2120
2121 kfree(txr->tx_buf, M_DEVBUF);
2122 txr->tx_buf = NULL;
2123 }
2124
2125 static void
2126 ix_init_tx_ring(struct ix_tx_ring *txr)
2127 {
2128 /* Clear the old ring contents */
2129 bzero(txr->tx_base, sizeof(union ixgbe_adv_tx_desc) * txr->tx_ndesc);
2130
2131 /* Clear TX head write-back buffer */
2132 *(txr->tx_hdr) = 0;
2133
2134 /* Reset indices */
2135 txr->tx_next_avail = 0;
2136 txr->tx_next_clean = 0;
2137 txr->tx_nsegs = 0;
2138 txr->tx_nmbuf = 0;
2139 txr->tx_running = 0;
2140
2141 /* Set number of descriptors available */
2142 txr->tx_avail = txr->tx_ndesc;
2143
2144 /* Enable this TX ring */
2145 txr->tx_flags |= IX_TXFLAG_ENABLED;
2146 }
2147
2148 static void
2149 ix_init_tx_unit(struct ix_softc *sc)
2150 {
2151 struct ixgbe_hw *hw = &sc->hw;
2152 int i;
2153
2154 /*
2155 * Setup the Base and Length of the Tx Descriptor Ring
2156 */
2157 for (i = 0; i < sc->tx_ring_inuse; ++i) {
2158 struct ix_tx_ring *txr = &sc->tx_rings[i];
2159 uint64_t tdba = txr->tx_base_paddr;
2160 uint64_t hdr_paddr = txr->tx_hdr_paddr;
2161 uint32_t txctrl;
2162
2163 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), (uint32_t)tdba);
2164 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (uint32_t)(tdba >> 32));
2165 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2166 txr->tx_ndesc * sizeof(union ixgbe_adv_tx_desc));
2167
2168 /* Setup the HW Tx Head and Tail descriptor pointers */
2169 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2170 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2171
2172 /* Disable TX head write-back relax ordering */
2173 switch (hw->mac.type) {
2174 case ixgbe_mac_82598EB:
2175 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2176 break;
2177 default:
2178 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2179 break;
2180 }
2181 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2182 switch (hw->mac.type) {
2183 case ixgbe_mac_82598EB:
2184 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2185 break;
2186 default:
2187 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2188 break;
2189 }
2190
2191 /* Enable TX head write-back */
2192 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i),
2193 (uint32_t)(hdr_paddr >> 32));
2194 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i),
2195 ((uint32_t)hdr_paddr) | IXGBE_TDWBAL_HEAD_WB_ENABLE);
2196 }
2197
2198 if (hw->mac.type != ixgbe_mac_82598EB) {
2199 uint32_t dmatxctl, rttdcs;
2200
2201 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2202 dmatxctl |= IXGBE_DMATXCTL_TE;
2203 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2204
2205 /* Disable arbiter to set MTQC */
2206 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2207 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2208 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2209
2210 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2211
2212 /* Reenable aribter */
2213 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2214 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2215 }
2216 }
2217
2218 static int
2219 ix_tx_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp,
2220 uint32_t *cmd_type_len, uint32_t *olinfo_status)
2221 {
2222 struct ixgbe_adv_tx_context_desc *TXD;
2223 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2224 int ehdrlen, ip_hlen = 0, ctxd;
2225 boolean_t offload = TRUE;
2226
2227 /* First check if TSO is to be used */
2228 if (mp->m_pkthdr.csum_flags & CSUM_TSO) {
2229 return ix_tso_ctx_setup(txr, mp,
2230 cmd_type_len, olinfo_status);
2231 }
2232
2233 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2234 offload = FALSE;
2235
2236 /* Indicate the whole packet as payload when not doing TSO */
2237 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
2238
2239 /*
2240 * In advanced descriptors the vlan tag must be placed into the
2241 * context descriptor. Hence we need to make one even if not
2242 * doing checksum offloads.
2243 */
2244 if (mp->m_flags & M_VLANTAG) {
2245 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) <<
2246 IXGBE_ADVTXD_VLAN_SHIFT;
2247 } else if (!offload) {
2248 /* No TX descriptor is consumed */
2249 return 0;
2250 }
2251
2252 /* Set the ether header length */
2253 ehdrlen = mp->m_pkthdr.csum_lhlen;
2254 KASSERT(ehdrlen > 0, ("invalid ether hlen"));
2255 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2256
2257 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
2258 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
2259 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2260 ip_hlen = mp->m_pkthdr.csum_iphlen;
2261 KASSERT(ip_hlen > 0, ("invalid ip hlen"));
2262 }
2263 vlan_macip_lens |= ip_hlen;
2264
2265 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2266 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
2267 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2268 else if (mp->m_pkthdr.csum_flags & CSUM_UDP)
2269 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
2270
2271 if (mp->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2272 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
2273
2274 /* Now ready a context descriptor */
2275 ctxd = txr->tx_next_avail;
2276 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd];
2277
2278 /* Now copy bits into descriptor */
2279 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
2280 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
2281 TXD->seqnum_seed = htole32(0);
2282 TXD->mss_l4len_idx = htole32(0);
2283
2284 /* We've consumed the first desc, adjust counters */
2285 if (++ctxd == txr->tx_ndesc)
2286 ctxd = 0;
2287 txr->tx_next_avail = ctxd;
2288 --txr->tx_avail;
2289
2290 /* One TX descriptor is consumed */
2291 return 1;
2292 }
2293
2294 static int
2295 ix_tso_ctx_setup(struct ix_tx_ring *txr, const struct mbuf *mp,
2296 uint32_t *cmd_type_len, uint32_t *olinfo_status)
2297 {
2298 struct ixgbe_adv_tx_context_desc *TXD;
2299 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2300 uint32_t mss_l4len_idx = 0, paylen;
2301 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
2302
2303 ehdrlen = mp->m_pkthdr.csum_lhlen;
2304 KASSERT(ehdrlen > 0, ("invalid ether hlen"));
2305
2306 ip_hlen = mp->m_pkthdr.csum_iphlen;
2307 KASSERT(ip_hlen > 0, ("invalid ip hlen"));
2308
2309 tcp_hlen = mp->m_pkthdr.csum_thlen;
2310 KASSERT(tcp_hlen > 0, ("invalid tcp hlen"));
2311
2312 ctxd = txr->tx_next_avail;
2313 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2314
2315 if (mp->m_flags & M_VLANTAG) {
2316 vlan_macip_lens |= htole16(mp->m_pkthdr.ether_vlantag) <<
2317 IXGBE_ADVTXD_VLAN_SHIFT;
2318 }
2319 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2320 vlan_macip_lens |= ip_hlen;
2321 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
2322
2323 /* ADV DTYPE TUCMD */
2324 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2325 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2326 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2327 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
2328
2329 /* MSS L4LEN IDX */
2330 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
2331 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
2332 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
2333
2334 TXD->seqnum_seed = htole32(0);
2335
2336 if (++ctxd == txr->tx_ndesc)
2337 ctxd = 0;
2338
2339 txr->tx_avail--;
2340 txr->tx_next_avail = ctxd;
2341
2342 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2343
2344 /* This is used in the transmit desc in encap */
2345 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
2346
2347 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
2348 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
2349 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
2350
2351 /* One TX descriptor is consumed */
2352 return 1;
2353 }
2354
2355 static void
2356 ix_txeof(struct ix_tx_ring *txr, int hdr)
2357 {
2358 int first, avail;
2359
2360 if (txr->tx_avail == txr->tx_ndesc)
2361 return;
2362
2363 first = txr->tx_next_clean;
2364 if (first == hdr)
2365 return;
2366
2367 avail = txr->tx_avail;
2368 while (first != hdr) {
2369 struct ix_tx_buf *txbuf = &txr->tx_buf[first];
2370
2371 KKASSERT(avail < txr->tx_ndesc);
2372 ++avail;
2373
2374 if (txbuf->m_head != NULL)
2375 ix_free_txbuf(txr, txbuf);
2376 if (++first == txr->tx_ndesc)
2377 first = 0;
2378 }
2379 txr->tx_next_clean = first;
2380 txr->tx_avail = avail;
2381
2382 if (txr->tx_avail > IX_MAX_SCATTER + IX_TX_RESERVED) {
2383 ifsq_clr_oactive(txr->tx_ifsq);
2384 ifsq_watchdog_set_count(&txr->tx_watchdog, 0);
2385 }
2386 txr->tx_running = IX_TX_RUNNING;
2387 }
2388
2389 static void
2390 ix_txgc(struct ix_tx_ring *txr)
2391 {
2392 int first, hdr;
2393 #ifdef INVARIANTS
2394 int avail;
2395 #endif
2396
2397 if (txr->tx_avail == txr->tx_ndesc)
2398 return;
2399
2400 hdr = IXGBE_READ_REG(&txr->tx_sc->hw, IXGBE_TDH(txr->tx_idx));
2401 first = txr->tx_next_clean;
2402 if (first == hdr)
2403 goto done;
2404 txr->tx_gc++;
2405
2406 #ifdef INVARIANTS
2407 avail = txr->tx_avail;
2408 #endif
2409 while (first != hdr) {
2410 struct ix_tx_buf *txbuf = &txr->tx_buf[first];
2411
2412 #ifdef INVARIANTS
2413 KKASSERT(avail < txr->tx_ndesc);
2414 ++avail;
2415 #endif
2416 if (txbuf->m_head != NULL)
2417 ix_free_txbuf(txr, txbuf);
2418 if (++first == txr->tx_ndesc)
2419 first = 0;
2420 }
2421 done:
2422 if (txr->tx_nmbuf)
2423 txr->tx_running = IX_TX_RUNNING;
2424 }
2425
2426 static int
2427 ix_create_rx_ring(struct ix_rx_ring *rxr)
2428 {
2429 int i, rsize, error, nrxd;
2430
2431 /*
2432 * Validate number of receive descriptors. It must not exceed
2433 * hardware maximum, and must be multiple of IX_DBA_ALIGN.
2434 */
2435 nrxd = device_getenv_int(rxr->rx_sc->dev, "rxd", ix_rxd);
2436 if (((nrxd * sizeof(union ixgbe_adv_rx_desc)) % IX_DBA_ALIGN) != 0 ||
2437 nrxd < IX_MIN_RXD || nrxd > IX_MAX_RXD) {
2438 device_printf(rxr->rx_sc->dev,
2439 "Using %d RX descriptors instead of %d!\n",
2440 IX_DEF_RXD, nrxd);
2441 rxr->rx_ndesc = IX_DEF_RXD;
2442 } else {
2443 rxr->rx_ndesc = nrxd;
2444 }
2445
2446 /*
2447 * Allocate RX descriptor ring
2448 */
2449 rsize = roundup2(rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc),
2450 IX_DBA_ALIGN);
2451 rxr->rx_base = bus_dmamem_coherent_any(rxr->rx_sc->parent_tag,
2452 IX_DBA_ALIGN, rsize, BUS_DMA_WAITOK | BUS_DMA_ZERO,
2453 &rxr->rx_base_dtag, &rxr->rx_base_map, &rxr->rx_base_paddr);
2454 if (rxr->rx_base == NULL) {
2455 device_printf(rxr->rx_sc->dev,
2456 "Unable to allocate TX Descriptor memory\n");
2457 return ENOMEM;
2458 }
2459
2460 rsize = __VM_CACHELINE_ALIGN(sizeof(struct ix_rx_buf) * rxr->rx_ndesc);
2461 rxr->rx_buf = kmalloc(rsize, M_DEVBUF,
2462 M_WAITOK | M_ZERO | M_CACHEALIGN);
2463
2464 /*
2465 * Create DMA tag for RX buffers
2466 */
2467 error = bus_dma_tag_create(rxr->rx_sc->parent_tag,
2468 1, 0, /* alignment, bounds */
2469 BUS_SPACE_MAXADDR, /* lowaddr */
2470 BUS_SPACE_MAXADDR, /* highaddr */
2471 PAGE_SIZE, /* maxsize */
2472 1, /* nsegments */
2473 PAGE_SIZE, /* maxsegsize */
2474 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */
2475 &rxr->rx_tag);
2476 if (error) {
2477 device_printf(rxr->rx_sc->dev,
2478 "Unable to create RX DMA tag\n");
2479 kfree(rxr->rx_buf, M_DEVBUF);
2480 rxr->rx_buf = NULL;
2481 return error;
2482 }
2483
2484 /*
2485 * Create spare DMA map for RX buffers
2486 */
2487 error = bus_dmamap_create(rxr->rx_tag, BUS_DMA_WAITOK,
2488 &rxr->rx_sparemap);
2489 if (error) {
2490 device_printf(rxr->rx_sc->dev,
2491 "Unable to create spare RX DMA map\n");
2492 bus_dma_tag_destroy(rxr->rx_tag);
2493 kfree(rxr->rx_buf, M_DEVBUF);
2494 rxr->rx_buf = NULL;
2495 return error;
2496 }
2497
2498 /*
2499 * Create DMA maps for RX buffers
2500 */
2501 for (i = 0; i < rxr->rx_ndesc; ++i) {
2502 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i];
2503
2504 error = bus_dmamap_create(rxr->rx_tag,
2505 BUS_DMA_WAITOK, &rxbuf->map);
2506 if (error) {
2507 device_printf(rxr->rx_sc->dev,
2508 "Unable to create RX dma map\n");
2509 ix_destroy_rx_ring(rxr, i);
2510 return error;
2511 }
2512 }
2513
2514 /*
2515 * Initialize various watermark
2516 */
2517 rxr->rx_wreg_nsegs = IX_DEF_RXWREG_NSEGS;
2518
2519 return 0;
2520 }
2521
2522 static void
2523 ix_destroy_rx_ring(struct ix_rx_ring *rxr, int ndesc)
2524 {
2525 int i;
2526
2527 if (rxr->rx_base != NULL) {
2528 bus_dmamap_unload(rxr->rx_base_dtag, rxr->rx_base_map);
2529 bus_dmamem_free(rxr->rx_base_dtag, rxr->rx_base,
2530 rxr->rx_base_map);
2531 bus_dma_tag_destroy(rxr->rx_base_dtag);
2532 rxr->rx_base = NULL;
2533 }
2534
2535 if (rxr->rx_buf == NULL)
2536 return;
2537
2538 for (i = 0; i < ndesc; ++i) {
2539 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i];
2540
2541 KKASSERT(rxbuf->m_head == NULL);
2542 bus_dmamap_destroy(rxr->rx_tag, rxbuf->map);
2543 }
2544 bus_dmamap_destroy(rxr->rx_tag, rxr->rx_sparemap);
2545 bus_dma_tag_destroy(rxr->rx_tag);
2546
2547 kfree(rxr->rx_buf, M_DEVBUF);
2548 rxr->rx_buf = NULL;
2549 }
2550
2551 /*
2552 ** Used to detect a descriptor that has
2553 ** been merged by Hardware RSC.
2554 */
2555 static __inline uint32_t
2556 ix_rsc_count(union ixgbe_adv_rx_desc *rx)
2557 {
2558 return (le32toh(rx->wb.lower.lo_dword.data) &
2559 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
2560 }
2561
2562 #if 0
2563 /*********************************************************************
2564 *
2565 * Initialize Hardware RSC (LRO) feature on 82599
2566 * for an RX ring, this is toggled by the LRO capability
2567 * even though it is transparent to the stack.
2568 *
2569 * NOTE: since this HW feature only works with IPV4 and
2570 * our testing has shown soft LRO to be as effective
2571 * I have decided to disable this by default.
2572 *
2573 **********************************************************************/
2574 static void
2575 ix_setup_hw_rsc(struct ix_rx_ring *rxr)
2576 {
2577 struct ix_softc *sc = rxr->rx_sc;
2578 struct ixgbe_hw *hw = &sc->hw;
2579 uint32_t rscctrl, rdrxctl;
2580
2581 #if 0
2582 /* If turning LRO/RSC off we need to disable it */
2583 if ((sc->arpcom.ac_if.if_capenable & IFCAP_LRO) == 0) {
2584 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
2585 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
2586 return;
2587 }
2588 #endif
2589
2590 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2591 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2592 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
2593 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
2594 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
2595
2596 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
2597 rscctrl |= IXGBE_RSCCTL_RSCEN;
2598 /*
2599 ** Limit the total number of descriptors that
2600 ** can be combined, so it does not exceed 64K
2601 */
2602 if (rxr->mbuf_sz == MCLBYTES)
2603 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2604 else if (rxr->mbuf_sz == MJUMPAGESIZE)
2605 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2606 else if (rxr->mbuf_sz == MJUM9BYTES)
2607 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2608 else /* Using 16K cluster */
2609 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2610
2611 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
2612
2613 /* Enable TCP header recognition */
2614 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
2615 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
2616 IXGBE_PSRTYPE_TCPHDR));
2617
2618 /* Disable RSC for ACK packets */
2619 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
2620 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
2621
2622 rxr->hw_rsc = TRUE;
2623 }
2624 #endif
2625
2626 static int
2627 ix_init_rx_ring(struct ix_rx_ring *rxr)
2628 {
2629 int i;
2630
2631 /* Clear the ring contents */
2632 bzero(rxr->rx_base, rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc));
2633
2634 /* XXX we need JUMPAGESIZE for RSC too */
2635 if (rxr->rx_sc->max_frame_size <= MCLBYTES)
2636 rxr->rx_mbuf_sz = MCLBYTES;
2637 else
2638 rxr->rx_mbuf_sz = MJUMPAGESIZE;
2639
2640 /* Now replenish the mbufs */
2641 for (i = 0; i < rxr->rx_ndesc; ++i) {
2642 int error;
2643
2644 error = ix_newbuf(rxr, i, TRUE);
2645 if (error)
2646 return error;
2647 }
2648
2649 /* Setup our descriptor indices */
2650 rxr->rx_next_check = 0;
2651 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC;
2652
2653 #if 0
2654 /*
2655 ** Now set up the LRO interface:
2656 */
2657 if (ixgbe_rsc_enable)
2658 ix_setup_hw_rsc(rxr);
2659 #endif
2660
2661 return 0;
2662 }
2663
2664 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2665
2666 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
2667
2668 static void
2669 ix_init_rx_unit(struct ix_softc *sc, boolean_t polling)
2670 {
2671 struct ixgbe_hw *hw = &sc->hw;
2672 struct ifnet *ifp = &sc->arpcom.ac_if;
2673 uint32_t bufsz, fctrl, rxcsum, hlreg;
2674 int i;
2675
2676 /*
2677 * Make sure receives are disabled while setting up the descriptor ring
2678 */
2679 ixgbe_disable_rx(hw);
2680
2681 /* Enable broadcasts */
2682 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2683 fctrl |= IXGBE_FCTRL_BAM;
2684 if (hw->mac.type == ixgbe_mac_82598EB) {
2685 fctrl |= IXGBE_FCTRL_DPF;
2686 fctrl |= IXGBE_FCTRL_PMCF;
2687 }
2688 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2689
2690 /* Set for Jumbo Frames? */
2691 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2692 if (ifp->if_mtu > ETHERMTU)
2693 hlreg |= IXGBE_HLREG0_JUMBOEN;
2694 else
2695 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
2696 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
2697
2698 KKASSERT(sc->rx_rings[0].rx_mbuf_sz >= MCLBYTES);
2699 bufsz = (sc->rx_rings[0].rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
2700 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2701
2702 for (i = 0; i < sc->rx_ring_inuse; ++i) {
2703 struct ix_rx_ring *rxr = &sc->rx_rings[i];
2704 uint64_t rdba = rxr->rx_base_paddr;
2705 uint32_t srrctl;
2706
2707 /* Setup the Base and Length of the Rx Descriptor Ring */
2708 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), (uint32_t)rdba);
2709 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (uint32_t)(rdba >> 32));
2710 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
2711 rxr->rx_ndesc * sizeof(union ixgbe_adv_rx_desc));
2712
2713 /*
2714 * Set up the SRRCTL register
2715 */
2716 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
2717
2718 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2719 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2720 srrctl |= bufsz;
2721 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2722 if (sc->rx_ring_inuse > 1) {
2723 /* See the commend near ix_enable_rx_drop() */
2724 if (sc->ifm_media &
2725 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
2726 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
2727 if (i == 0 && bootverbose) {
2728 if_printf(ifp, "flow control %s, "
2729 "disable RX drop\n",
2730 ix_ifmedia2str(sc->ifm_media));
2731 }
2732 } else {
2733 srrctl |= IXGBE_SRRCTL_DROP_EN;
2734 if (i == 0 && bootverbose) {
2735 if_printf(ifp, "flow control %s, "
2736 "enable RX drop\n",
2737 ix_ifmedia2str(sc->ifm_media));
2738 }
2739 }
2740 }
2741 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
2742
2743 /* Setup the HW Rx Head and Tail Descriptor Pointers */
2744 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
2745 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
2746 }
2747
2748 if (sc->hw.mac.type != ixgbe_mac_82598EB)
2749 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 0);
2750
2751 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2752
2753 /*
2754 * Setup RSS
2755 */
2756 if (sc->rx_ring_inuse > 1) {
2757 uint8_t key[IX_NRSSRK * IX_RSSRK_SIZE];
2758 const struct if_ringmap *rm;
2759 int j, r, nreta, table_nent;
2760
2761 /*
2762 * NOTE:
2763 * When we reach here, RSS has already been disabled
2764 * in ix_stop(), so we could safely configure RSS key
2765 * and redirect table.
2766 */
2767
2768 /*
2769 * Configure RSS key
2770 */
2771 toeplitz_get_key(key, sizeof(key));
2772 for (i = 0; i < IX_NRSSRK; ++i) {
2773 uint32_t rssrk;
2774
2775 rssrk = IX_RSSRK_VAL(key, i);
2776 IX_RSS_DPRINTF(sc, 1, "rssrk%d 0x%08x\n",
2777 i, rssrk);
2778
2779 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rssrk);
2780 }
2781
2782 /*
2783 * Configure RSS redirect table.
2784 */
2785
2786 /* Table size will differ based on MAC */
2787 switch (hw->mac.type) {
2788 case ixgbe_mac_X550:
2789 case ixgbe_mac_X550EM_x:
2790 case ixgbe_mac_X550EM_a:
2791 nreta = IX_NRETA_X550;
2792 break;
2793 default:
2794 nreta = IX_NRETA;
2795 break;
2796 }
2797
2798 table_nent = nreta * IX_RETA_SIZE;
2799 KASSERT(table_nent <= IX_RDRTABLE_SIZE,
2800 ("invalid RETA count %d", nreta));
2801 if (polling)
2802 rm = sc->rx_rmap;
2803 else
2804 rm = sc->rx_rmap_intr;
2805 if_ringmap_rdrtable(rm, sc->rdr_table, table_nent);
2806
2807 r = 0;
2808 for (j = 0; j < nreta; ++j) {
2809 uint32_t reta = 0;
2810
2811 for (i = 0; i < IX_RETA_SIZE; ++i) {
2812 uint32_t q;
2813
2814 q = sc->rdr_table[r];
2815 KASSERT(q < sc->rx_ring_inuse,
2816 ("invalid RX ring index %d", q));
2817 reta |= q << (8 * i);
2818 ++r;
2819 }
2820 IX_RSS_DPRINTF(sc, 1, "reta 0x%08x\n", reta);
2821 if (j < IX_NRETA) {
2822 IXGBE_WRITE_REG(hw, IXGBE_RETA(j), reta);
2823 } else {
2824 IXGBE_WRITE_REG(hw, IXGBE_ERETA(j - IX_NRETA),
2825 reta);
2826 }
2827 }
2828
2829 /*
2830 * Enable multiple receive queues.
2831 * Enable IPv4 RSS standard hash functions.
2832 */
2833 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
2834 IXGBE_MRQC_RSSEN |
2835 IXGBE_MRQC_RSS_FIELD_IPV4 |
2836 IXGBE_MRQC_RSS_FIELD_IPV4_TCP);
2837
2838 /*
2839 * NOTE:
2840 * PCSD must be enabled to enable multiple
2841 * receive queues.
2842 */
2843 rxcsum |= IXGBE_RXCSUM_PCSD;
2844 }
2845
2846 if (ifp->if_capenable & IFCAP_RXCSUM)
2847 rxcsum |= IXGBE_RXCSUM_PCSD;
2848
2849 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2850 }
2851
2852 static __inline void
2853 ix_rx_refresh(struct ix_rx_ring *rxr, int i)
2854 {
2855 if (--i < 0)
2856 i = rxr->rx_ndesc - 1;
2857 IXGBE_WRITE_REG(&rxr->rx_sc->hw, IXGBE_RDT(rxr->rx_idx), i);
2858 }
2859
2860 static __inline void
2861 ix_rxcsum(uint32_t staterr, struct mbuf *mp, uint32_t ptype)
2862 {
2863 if ((ptype &
2864 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_IPV4_EX)) == 0) {
2865 /* Not IPv4 */
2866 return;
2867 }
2868
2869 if ((staterr & (IXGBE_RXD_STAT_IPCS | IXGBE_RXDADV_ERR_IPE)) ==
2870 IXGBE_RXD_STAT_IPCS)
2871 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
2872
2873 if ((ptype &
2874 (IXGBE_RXDADV_PKTTYPE_TCP | IXGBE_RXDADV_PKTTYPE_UDP)) == 0) {
2875 /*
2876 * - Neither TCP nor UDP
2877 * - IPv4 fragment
2878 */
2879 return;
2880 }
2881
2882 if ((staterr & (IXGBE_RXD_STAT_L4CS | IXGBE_RXDADV_ERR_TCPE)) ==
2883 IXGBE_RXD_STAT_L4CS) {
2884 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
2885 CSUM_FRAG_NOT_CHECKED;
2886 mp->m_pkthdr.csum_data = htons(0xffff);
2887 }
2888 }
2889
2890 static __inline struct pktinfo *
2891 ix_rssinfo(struct mbuf *m, struct pktinfo *pi,
2892 uint32_t hash, uint32_t hashtype, uint32_t ptype)
2893 {
2894 switch (hashtype) {
2895 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP:
2896 pi->pi_netisr = NETISR_IP;
2897 pi->pi_flags = 0;
2898 pi->pi_l3proto = IPPROTO_TCP;
2899 break;
2900
2901 case IXGBE_RXDADV_RSSTYPE_IPV4:
2902 if ((ptype & IXGBE_RXDADV_PKTTYPE_UDP) == 0) {
2903 /* Not UDP or is fragment */
2904 return NULL;
2905 }
2906 pi->pi_netisr = NETISR_IP;
2907 pi->pi_flags = 0;
2908 pi->pi_l3proto = IPPROTO_UDP;
2909 break;
2910
2911 default:
2912 return NULL;
2913 }
2914
2915 m_sethash(m, toeplitz_hash(hash));
2916 return pi;
2917 }
2918
2919 static __inline void
2920 ix_setup_rxdesc(union ixgbe_adv_rx_desc *rxd, const struct ix_rx_buf *rxbuf)
2921 {
2922 rxd->read.pkt_addr = htole64(rxbuf->paddr);
2923 rxd->wb.upper.status_error = 0;
2924 }
2925
2926 static void
2927 ix_rx_discard(struct ix_rx_ring *rxr, int i, boolean_t eop)
2928 {
2929 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i];
2930
2931 /*
2932 * XXX discard may not be correct
2933 */
2934 if (eop) {
2935 IFNET_STAT_INC(&rxr->rx_sc->arpcom.ac_if, ierrors, 1);
2936 rxr->rx_flags &= ~IX_RXRING_FLAG_DISC;
2937 } else {
2938 rxr->rx_flags |= IX_RXRING_FLAG_DISC;
2939 }
2940 if (rxbuf->fmp != NULL) {
2941 m_freem(rxbuf->fmp);
2942 rxbuf->fmp = NULL;
2943 rxbuf->lmp = NULL;
2944 }
2945 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf);
2946 }
2947
2948 static void
2949 ix_rxeof(struct ix_rx_ring *rxr, int count)
2950 {
2951 struct ifnet *ifp = &rxr->rx_sc->arpcom.ac_if;
2952 int i, nsegs = 0, cpuid = mycpuid;
2953
2954 i = rxr->rx_next_check;
2955 while (count != 0) {
2956 struct ix_rx_buf *rxbuf, *nbuf = NULL;
2957 union ixgbe_adv_rx_desc *cur;
2958 struct mbuf *sendmp = NULL, *mp;
2959 struct pktinfo *pi = NULL, pi0;
2960 uint32_t rsc = 0, ptype, staterr, hash, hashtype;
2961 uint16_t len;
2962 boolean_t eop;
2963
2964 cur = &rxr->rx_base[i];
2965 staterr = le32toh(cur->wb.upper.status_error);
2966
2967 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
2968 break;
2969 ++nsegs;
2970
2971 rxbuf = &rxr->rx_buf[i];
2972 mp = rxbuf->m_head;
2973
2974 len = le16toh(cur->wb.upper.length);
2975 ptype = le32toh(cur->wb.lower.lo_dword.data) &
2976 IXGBE_RXDADV_PKTTYPE_MASK;
2977 hash = le32toh(cur->wb.lower.hi_dword.rss);
2978 hashtype = le32toh(cur->wb.lower.lo_dword.data) &
2979 IXGBE_RXDADV_RSSTYPE_MASK;
2980
2981 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
2982 if (eop)
2983 --count;
2984
2985 /*
2986 * Make sure bad packets are discarded
2987 */
2988 if ((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) ||
2989 (rxr->rx_flags & IX_RXRING_FLAG_DISC)) {
2990 ix_rx_discard(rxr, i, eop);
2991 goto next_desc;
2992 }
2993
2994 bus_dmamap_sync(rxr->rx_tag, rxbuf->map, BUS_DMASYNC_POSTREAD);
2995 if (ix_newbuf(rxr, i, FALSE) != 0) {
2996 ix_rx_discard(rxr, i, eop);
2997 goto next_desc;
2998 }
2999
3000 /*
3001 * On 82599 which supports a hardware LRO, packets
3002 * need not be fragmented across sequential descriptors,
3003 * rather the next descriptor is indicated in bits
3004 * of the descriptor. This also means that we might
3005 * proceses more than one packet at a time, something
3006 * that has never been true before, it required
3007 * eliminating global chain pointers in favor of what
3008 * we are doing here.
3009 */
3010 if (!eop) {
3011 int nextp;
3012
3013 /*
3014 * Figure out the next descriptor
3015 * of this frame.
3016 */
3017 if (rxr->rx_flags & IX_RXRING_FLAG_LRO)
3018 rsc = ix_rsc_count(cur);
3019 if (rsc) { /* Get hardware index */
3020 nextp = ((staterr &
3021 IXGBE_RXDADV_NEXTP_MASK) >>
3022 IXGBE_RXDADV_NEXTP_SHIFT);
3023 } else { /* Just sequential */
3024 nextp = i + 1;
3025 if (nextp == rxr->rx_ndesc)
3026 nextp = 0;
3027 }
3028 nbuf = &rxr->rx_buf[nextp];
3029 prefetch(nbuf);
3030 }
3031 mp->m_len = len;
3032
3033 /*
3034 * Rather than using the fmp/lmp global pointers
3035 * we now keep the head of a packet chain in the
3036 * buffer struct and pass this along from one
3037 * descriptor to the next, until we get EOP.
3038 */
3039 if (rxbuf->fmp == NULL) {
3040 mp->m_pkthdr.len = len;
3041 rxbuf->fmp = mp;
3042 rxbuf->lmp = mp;
3043 } else {
3044 rxbuf->fmp->m_pkthdr.len += len;
3045 rxbuf->lmp->m_next = mp;
3046 rxbuf->lmp = mp;
3047 }
3048
3049 if (nbuf != NULL) {
3050 /*
3051 * Not the last fragment of this frame,
3052 * pass this fragment list on
3053 */
3054 nbuf->fmp = rxbuf->fmp;
3055 nbuf->lmp = rxbuf->lmp;
3056 } else {
3057 /*
3058 * Send this frame
3059 */
3060 sendmp = rxbuf->fmp;
3061
3062 sendmp->m_pkthdr.rcvif = ifp;
3063 IFNET_STAT_INC(ifp, ipackets, 1);
3064 #ifdef IX_RSS_DEBUG
3065 rxr->rx_pkts++;
3066 #endif
3067
3068 /* Process vlan info */
3069 if (staterr & IXGBE_RXD_STAT_VP) {
3070 sendmp->m_pkthdr.ether_vlantag =
3071 le16toh(cur->wb.upper.vlan);
3072 sendmp->m_flags |= M_VLANTAG;
3073 }
3074 if (ifp->if_capenable & IFCAP_RXCSUM)
3075 ix_rxcsum(staterr, sendmp, ptype);
3076 if (ifp->if_capenable & IFCAP_RSS) {
3077 pi = ix_rssinfo(sendmp, &pi0,
3078 hash, hashtype, ptype);
3079 }
3080 }
3081 rxbuf->fmp = NULL;
3082 rxbuf->lmp = NULL;
3083 next_desc:
3084 /* Advance our pointers to the next descriptor. */
3085 if (++i == rxr->rx_ndesc)
3086 i = 0;
3087
3088 if (sendmp != NULL)
3089 ifp->if_input(ifp, sendmp, pi, cpuid);
3090
3091 if (nsegs >= rxr->rx_wreg_nsegs) {
3092 ix_rx_refresh(rxr, i);
3093 nsegs = 0;
3094 }
3095 }
3096 rxr->rx_next_check = i;
3097
3098 if (nsegs > 0)
3099 ix_rx_refresh(rxr, i);
3100 }
3101
3102 static void
3103 ix_set_vlan(struct ix_softc *sc)
3104 {
3105 struct ixgbe_hw *hw = &sc->hw;
3106 uint32_t ctrl;
3107
3108 if ((sc->arpcom.ac_if.if_capenable & IFCAP_VLAN_HWTAGGING) == 0)
3109 return;
3110
3111 if (hw->mac.type == ixgbe_mac_82598EB) {
3112 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3113 ctrl |= IXGBE_VLNCTRL_VME;
3114 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
3115 } else {
3116 int i;
3117
3118 /*
3119 * On 82599 and later chips the VLAN enable is
3120 * per queue in RXDCTL
3121 */
3122 for (i = 0; i < sc->rx_ring_inuse; ++i) {
3123 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
3124 ctrl |= IXGBE_RXDCTL_VME;
3125 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
3126 }
3127 }
3128 }
3129
3130 static void
3131 ix_enable_intr(struct ix_softc *sc)
3132 {
3133 struct ixgbe_hw *hw = &sc->hw;
3134 uint32_t fwsm;
3135 int i;
3136
3137 for (i = 0; i < sc->intr_cnt; ++i)
3138 lwkt_serialize_handler_enable(sc->intr_data[i].intr_serialize);
3139
3140 sc->intr_mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3141
3142 switch (hw->mac.type) {
3143 case ixgbe_mac_82599EB:
3144 sc->intr_mask |= IXGBE_EIMS_ECC;
3145 /* Temperature sensor on some adapters */
3146 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0;
3147 /* SFP+ (RX_LOS_N & MOD_ABS_N) */
3148 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1;
3149 sc->intr_mask |= IXGBE_EIMS_GPI_SDP2;
3150 break;
3151
3152 case ixgbe_mac_X540:
3153 sc->intr_mask |= IXGBE_EIMS_ECC;
3154 /* Detect if Thermal Sensor is enabled */
3155 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3156 if (fwsm & IXGBE_FWSM_TS_ENABLED)
3157 sc->intr_mask |= IXGBE_EIMS_TS;
3158 break;
3159
3160 case ixgbe_mac_X550:
3161 sc->intr_mask |= IXGBE_EIMS_ECC;
3162 /* MAC thermal sensor is automatically enabled */
3163 sc->intr_mask |= IXGBE_EIMS_TS;
3164 break;
3165
3166 case ixgbe_mac_X550EM_a:
3167 case ixgbe_mac_X550EM_x:
3168 sc->intr_mask |= IXGBE_EIMS_ECC;
3169 /* Some devices use SDP0 for important information */
3170 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3171 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
3172 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
3173 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
3174 sc->intr_mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3175 if (hw->phy.type == ixgbe_phy_x550em_ext_t)
3176 sc->intr_mask |= IXGBE_EICR_GPI_SDP0_X540;
3177 break;
3178
3179 default:
3180 break;
3181 }
3182
3183 /* Enable Fan Failure detection */
3184 if (sc->caps & IX_CAP_DETECT_FANFAIL)
3185 sc->intr_mask |= IXGBE_EIMS_GPI_SDP1;
3186
3187 /* With MSI-X we use auto clear for RX and TX rings */
3188 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
3189 /*
3190 * There are no EIAC1/EIAC2 for newer chips; the related
3191 * bits for TX and RX rings > 16 are always auto clear.
3192 *
3193 * XXX which bits? There are _no_ documented EICR1 and
3194 * EICR2 at all; only EICR.
3195 */
3196 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE);
3197 } else {
3198 sc->intr_mask |= IX_TX_INTR_MASK | IX_RX0_INTR_MASK;
3199
3200 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS);
3201 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS)
3202 sc->intr_mask |= IX_RX1_INTR_MASK;
3203 }
3204
3205 IXGBE_WRITE_REG(hw, IXGBE_EIMS, sc->intr_mask);
3206
3207 /*
3208 * Enable RX and TX rings for MSI-X
3209 */
3210 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
3211 for (i = 0; i < sc->tx_ring_inuse; ++i) {
3212 const struct ix_tx_ring *txr = &sc->tx_rings[i];
3213
3214 if (txr->tx_intr_vec >= 0) {
3215 IXGBE_WRITE_REG(hw, txr->tx_eims,
3216 txr->tx_eims_val);
3217 }
3218 }
3219 for (i = 0; i < sc->rx_ring_inuse; ++i) {
3220 const struct ix_rx_ring *rxr = &sc->rx_rings[i];
3221
3222 KKASSERT(rxr->rx_intr_vec >= 0);
3223 IXGBE_WRITE_REG(hw, rxr->rx_eims, rxr->rx_eims_val);
3224 }
3225 }
3226
3227 IXGBE_WRITE_FLUSH(hw);
3228 }
3229
3230 static void
3231 ix_disable_intr(struct ix_softc *sc)
3232 {
3233 int i;
3234
3235 if (sc->intr_type == PCI_INTR_TYPE_MSIX)
3236 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);
3237
3238 if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3239 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0);
3240 } else {
3241 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000);
3242 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0);
3243 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0);
3244 }
3245 IXGBE_WRITE_FLUSH(&sc->hw);
3246
3247 for (i = 0; i < sc->intr_cnt; ++i)
3248 lwkt_serialize_handler_disable(sc->intr_data[i].intr_serialize);
3249 }
3250
3251 static void
3252 ix_slot_info(struct ix_softc *sc)
3253 {
3254 device_t dev = sc->dev;
3255 struct ixgbe_hw *hw = &sc->hw;
3256 uint32_t offset;
3257 uint16_t link;
3258 boolean_t bus_info_valid = TRUE;
3259
3260 /* Some devices are behind an internal bridge */
3261 switch (hw->device_id) {
3262 case IXGBE_DEV_ID_82599_SFP_SF_QP:
3263 case IXGBE_DEV_ID_82599_QSFP_SF_QP:
3264 goto get_parent_info;
3265 default:
3266 break;
3267 }
3268
3269 ixgbe_get_bus_info(hw);
3270
3271 /*
3272 * Some devices don't use PCI-E, but there is no need
3273 * to display "Unknown" for bus speed and width.
3274 */
3275 switch (hw->mac.type) {
3276 case ixgbe_mac_X550EM_x:
3277 case ixgbe_mac_X550EM_a:
3278 return;
3279 default:
3280 goto display;
3281 }
3282
3283 get_parent_info:
3284 /*
3285 * For the Quad port adapter we need to parse back up
3286 * the PCI tree to find the speed of the expansion slot
3287 * into which this adapter is plugged. A bit more work.
3288 */
3289 dev = device_get_parent(device_get_parent(dev));
3290 #ifdef IXGBE_DEBUG
3291 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev),
3292 pci_get_slot(dev), pci_get_function(dev));
3293 #endif
3294 dev = device_get_parent(device_get_parent(dev));
3295 #ifdef IXGBE_DEBUG
3296 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev),
3297 pci_get_slot(dev), pci_get_function(dev));
3298 #endif
3299 /* Now get the PCI Express Capabilities offset */
3300 offset = pci_get_pciecap_ptr(dev);
3301 if (offset == 0) {
3302 /*
3303 * Hmm...can't get PCI-Express capabilities.
3304 * Falling back to default method.
3305 */
3306 bus_info_valid = FALSE;
3307 ixgbe_get_bus_info(hw);
3308 goto display;
3309 }
3310 /* ...and read the Link Status Register */
3311 link = pci_read_config(dev, offset + PCIER_LINKSTAT, 2);
3312 ixgbe_set_pci_config_data_generic(hw, link);
3313
3314 display:
3315 device_printf(dev, "PCI Express Bus: Speed %s %s\n",
3316 hw->bus.speed == ixgbe_bus_speed_8000 ? "8.0GT/s" :
3317 hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
3318 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : "Unknown",
3319 hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
3320 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
3321 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : "Unknown");
3322
3323 if (bus_info_valid) {
3324 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP &&
3325 hw->bus.width <= ixgbe_bus_width_pcie_x4 &&
3326 hw->bus.speed == ixgbe_bus_speed_2500) {
3327 device_printf(dev, "PCI-Express bandwidth available "
3328 "for this card is not sufficient for optimal "
3329 "performance.\n");
3330 device_printf(dev, "For optimal performance a "
3331 "x8 PCIE, or x4 PCIE Gen2 slot is required.\n");
3332 }
3333 if (hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP &&
3334 hw->bus.width <= ixgbe_bus_width_pcie_x8 &&
3335 hw->bus.speed < ixgbe_bus_speed_8000) {
3336 device_printf(dev, "PCI-Express bandwidth available "
3337 "for this card is not sufficient for optimal "
3338 "performance.\n");
3339 device_printf(dev, "For optimal performance a "
3340 "x8 PCIE Gen3 slot is required.\n");
3341 }
3342 } else {
3343 device_printf(dev, "Unable to determine slot speed/width. "
3344 "The speed/width reported are that of the internal "
3345 "switch.\n");
3346 }
3347 }
3348
3349 /*
3350 * TODO comment is incorrect
3351 *
3352 * Setup the correct IVAR register for a particular MSIX interrupt
3353 * - entry is the register array entry
3354 * - vector is the MSIX vector for this queue
3355 * - type is RX/TX/MISC
3356 */
3357 static void
3358 ix_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector,
3359 int8_t type)
3360 {
3361 struct ixgbe_hw *hw = &sc->hw;
3362 uint32_t ivar, index;
3363
3364 vector |= IXGBE_IVAR_ALLOC_VAL;
3365
3366 switch (hw->mac.type) {
3367 case ixgbe_mac_82598EB:
3368 if (type == -1)
3369 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3370 else
3371 entry += (type * 64);
3372 index = (entry >> 2) & 0x1F;
3373 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3374 ivar &= ~(0xFF << (8 * (entry & 0x3)));
3375 ivar |= (vector << (8 * (entry & 0x3)));
3376 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
3377 break;
3378
3379 case ixgbe_mac_82599EB:
3380 case ixgbe_mac_X540:
3381 case ixgbe_mac_X550:
3382 case ixgbe_mac_X550EM_a:
3383 case ixgbe_mac_X550EM_x:
3384 if (type == -1) { /* MISC IVAR */
3385 index = (entry & 1) * 8;
3386 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3387 ivar &= ~(0xFF << index);
3388 ivar |= (vector << index);
3389 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3390 } else { /* RX/TX IVARS */
3391 index = (16 * (entry & 1)) + (8 * type);
3392 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3393 ivar &= ~(0xFF << index);
3394 ivar |= (vector << index);
3395 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3396 }
3397 /* FALL THROUGH */
3398 default:
3399 break;
3400 }
3401 }
3402
3403 static boolean_t
3404 ix_sfp_probe(struct ix_softc *sc)
3405 {
3406 struct ixgbe_hw *hw = &sc->hw;
3407
3408 if (hw->phy.type == ixgbe_phy_nl &&
3409 hw->phy.sfp_type == ixgbe_sfp_type_not_present) {
3410 int32_t ret;
3411
3412 ret = hw->phy.ops.identify_sfp(hw);
3413 if (ret)
3414 return FALSE;
3415
3416 ret = hw->phy.ops.reset(hw);
3417 sc->sfp_probe = FALSE;
3418 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3419 if_printf(&sc->arpcom.ac_if,
3420 "Unsupported SFP+ module detected! "
3421 "Reload driver with supported module.\n");
3422 return FALSE;
3423 }
3424 if_printf(&sc->arpcom.ac_if, "SFP+ module detected!\n");
3425
3426 /* We now have supported optics */
3427 return TRUE;
3428 }
3429 return FALSE;
3430 }
3431
3432 static void
3433 ix_handle_link(struct ix_softc *sc)
3434 {
3435 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);
3436 ix_update_link_status(sc);
3437 }
3438
3439 /*
3440 * Handling SFP module
3441 */
3442 static void
3443 ix_handle_mod(struct ix_softc *sc)
3444 {
3445 struct ixgbe_hw *hw = &sc->hw;
3446 uint32_t err;
3447
3448 if (sc->hw.need_crosstalk_fix) {
3449 uint32_t cage_full = 0;
3450
3451 switch (hw->mac.type) {
3452 case ixgbe_mac_82599EB:
3453 cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3454 IXGBE_ESDP_SDP2;
3455 break;
3456
3457 case ixgbe_mac_X550EM_x:
3458 case ixgbe_mac_X550EM_a:
3459 cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3460 IXGBE_ESDP_SDP0;
3461 break;
3462
3463 default:
3464 break;
3465 }
3466
3467 if (!cage_full)
3468 return;
3469 }
3470
3471 err = hw->phy.ops.identify_sfp(hw);
3472 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3473 if_printf(&sc->arpcom.ac_if,
3474 "Unsupported SFP+ module type was detected.\n");
3475 return;
3476 }
3477
3478 if (hw->mac.type == ixgbe_mac_82598EB)
3479 err = hw->phy.ops.reset(hw);
3480 else
3481 err = hw->mac.ops.setup_sfp(hw);
3482 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3483 if_printf(&sc->arpcom.ac_if,
3484 "Setup failure - unsupported SFP+ module type.\n");
3485 return;
3486 }
3487 ix_handle_msf(sc);
3488 }
3489
3490 /*
3491 * Handling MSF (multispeed fiber)
3492 */
3493 static void
3494 ix_handle_msf(struct ix_softc *sc)
3495 {
3496 struct ixgbe_hw *hw = &sc->hw;
3497 uint32_t autoneg;
3498
3499 sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
3500 ix_init_media(sc);
3501
3502 if (sc->advspeed != IXGBE_LINK_SPEED_UNKNOWN)
3503 autoneg = sc->advspeed;
3504 else
3505 autoneg = hw->phy.autoneg_advertised;
3506 if (!autoneg && hw->mac.ops.get_link_capabilities != NULL) {
3507 bool negotiate;
3508
3509 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
3510 }
3511 if (hw->mac.ops.setup_link != NULL)
3512 hw->mac.ops.setup_link(hw, autoneg, TRUE);
3513 }
3514
3515 static void
3516 ix_handle_phy(struct ix_softc *sc)
3517 {
3518 struct ixgbe_hw *hw = &sc->hw;
3519 int error;
3520
3521 error = hw->phy.ops.handle_lasi(hw);
3522 if (error == IXGBE_ERR_OVERTEMP) {
3523 if_printf(&sc->arpcom.ac_if,
3524 "CRITICAL: EXTERNAL PHY OVER TEMP!! "
3525 "PHY will downshift to lower power state!\n");
3526 } else if (error) {
3527 if_printf(&sc->arpcom.ac_if,
3528 "Error handling LASI interrupt: %d\n", error);
3529 }
3530 }
3531
3532 static void
3533 ix_update_stats(struct ix_softc *sc)
3534 {
3535 struct ifnet *ifp = &sc->arpcom.ac_if;
3536 struct ixgbe_hw *hw = &sc->hw;
3537 struct ixgbe_hw_stats *stats = &sc->stats;
3538 uint32_t missed_rx = 0, bprc, lxon, lxoff, total;
3539 uint64_t total_missed_rx = 0;
3540 int i;
3541
3542 stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
3543 stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
3544 stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
3545 stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
3546 stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0));
3547
3548 for (i = 0; i < 16; i++) {
3549 stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
3550 stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
3551 stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
3552 }
3553 stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
3554 stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
3555 stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
3556
3557 /* Hardware workaround, gprc counts missed packets */
3558 stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
3559 stats->gprc -= missed_rx;
3560
3561 if (hw->mac.type != ixgbe_mac_82598EB) {
3562 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
3563 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
3564 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
3565 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
3566 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
3567 ((uint64_t)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
3568 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
3569 stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
3570 } else {
3571 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
3572 stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
3573 /* 82598 only has a counter in the high register */
3574 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
3575 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
3576 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
3577 }
3578
3579 /*
3580 * Workaround: mprc hardware is incorrectly counting
3581 * broadcasts, so for now we subtract those.
3582 */
3583 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
3584 stats->bprc += bprc;
3585 stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
3586 if (hw->mac.type == ixgbe_mac_82598EB)
3587 stats->mprc -= bprc;
3588
3589 stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
3590 stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
3591 stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
3592 stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
3593 stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
3594 stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
3595
3596 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
3597 stats->lxontxc += lxon;
3598 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
3599 stats->lxofftxc += lxoff;
3600 total = lxon + lxoff;
3601
3602 stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
3603 stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
3604 stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
3605 stats->gptc -= total;
3606 stats->mptc -= total;
3607 stats->ptc64 -= total;
3608 stats->gotc -= total * ETHER_MIN_LEN;
3609
3610 stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
3611 stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
3612 stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
3613 stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
3614 stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
3615 stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
3616 stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
3617 stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
3618 stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
3619 stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
3620 stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
3621 stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
3622 stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
3623 stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
3624 stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
3625 stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
3626 stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
3627 stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
3628 /* Only read FCOE on 82599 */
3629 if (hw->mac.type != ixgbe_mac_82598EB) {
3630 stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
3631 stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
3632 stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
3633 stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
3634 stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
3635 }
3636
3637 /* Rx Errors */
3638 IFNET_STAT_SET(ifp, iqdrops, total_missed_rx);
3639 IFNET_STAT_SET(ifp, ierrors, sc->stats.crcerrs + sc->stats.rlec);
3640 }
3641
3642 #if 0
3643 /*
3644 * Add sysctl variables, one per statistic, to the system.
3645 */
3646 static void
3647 ix_add_hw_stats(struct ix_softc *sc)
3648 {
3649
3650 device_t dev = sc->dev;
3651
3652 struct ix_tx_ring *txr = sc->tx_rings;
3653 struct ix_rx_ring *rxr = sc->rx_rings;
3654
3655 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3656 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3657 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3658 struct ixgbe_hw_stats *stats = &sc->stats;
3659
3660 struct sysctl_oid *stat_node, *queue_node;
3661 struct sysctl_oid_list *stat_list, *queue_list;
3662
3663 #define QUEUE_NAME_LEN 32
3664 char namebuf[QUEUE_NAME_LEN];
3665
3666 /* MAC stats get the own sub node */
3667
3668 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
3669 CTLFLAG_RD, NULL, "MAC Statistics");
3670 stat_list = SYSCTL_CHILDREN(stat_node);
3671
3672 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
3673 CTLFLAG_RD, &stats->crcerrs,
3674 "CRC Errors");
3675 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
3676 CTLFLAG_RD, &stats->illerrc,
3677 "Illegal Byte Errors");
3678 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
3679 CTLFLAG_RD, &stats->errbc,
3680 "Byte Errors");
3681 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
3682 CTLFLAG_RD, &stats->mspdc,
3683 "MAC Short Packets Discarded");
3684 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
3685 CTLFLAG_RD, &stats->mlfc,
3686 "MAC Local Faults");
3687 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
3688 CTLFLAG_RD, &stats->mrfc,
3689 "MAC Remote Faults");
3690 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
3691 CTLFLAG_RD, &stats->rlec,
3692 "Receive Length Errors");
3693
3694 /* Flow Control stats */
3695 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
3696 CTLFLAG_RD, &stats->lxontxc,
3697 "Link XON Transmitted");
3698 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
3699 CTLFLAG_RD, &stats->lxonrxc,
3700 "Link XON Received");
3701 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
3702 CTLFLAG_RD, &stats->lxofftxc,
3703 "Link XOFF Transmitted");
3704 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
3705 CTLFLAG_RD, &stats->lxoffrxc,
3706 "Link XOFF Received");
3707
3708 /* Packet Reception Stats */
3709 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
3710 CTLFLAG_RD, &stats->tor,
3711 "Total Octets Received");
3712 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
3713 CTLFLAG_RD, &stats->gorc,
3714 "Good Octets Received");
3715 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
3716 CTLFLAG_RD, &stats->tpr,
3717 "Total Packets Received");
3718 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
3719 CTLFLAG_RD, &stats->gprc,
3720 "Good Packets Received");
3721 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
3722 CTLFLAG_RD, &stats->mprc,
3723 "Multicast Packets Received");
3724 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
3725 CTLFLAG_RD, &stats->bprc,
3726 "Broadcast Packets Received");
3727 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
3728 CTLFLAG_RD, &stats->prc64,
3729 "64 byte frames received ");
3730 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
3731 CTLFLAG_RD, &stats->prc127,
3732 "65-127 byte frames received");
3733 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
3734 CTLFLAG_RD, &stats->prc255,
3735 "128-255 byte frames received");
3736 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
3737 CTLFLAG_RD, &stats->prc511,
3738 "256-511 byte frames received");
3739 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
3740 CTLFLAG_RD, &stats->prc1023,
3741 "512-1023 byte frames received");
3742 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
3743 CTLFLAG_RD, &stats->prc1522,
3744 "1023-1522 byte frames received");
3745 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
3746 CTLFLAG_RD, &stats->ruc,
3747 "Receive Undersized");
3748 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
3749 CTLFLAG_RD, &stats->rfc,
3750 "Fragmented Packets Received ");
3751 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
3752 CTLFLAG_RD, &stats->roc,
3753 "Oversized Packets Received");
3754 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
3755 CTLFLAG_RD, &stats->rjc,
3756 "Received Jabber");
3757 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
3758 CTLFLAG_RD, &stats->mngprc,
3759 "Management Packets Received");
3760 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
3761 CTLFLAG_RD, &stats->mngptc,
3762 "Management Packets Dropped");
3763 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
3764 CTLFLAG_RD, &stats->xec,
3765 "Checksum Errors");
3766
3767 /* Packet Transmission Stats */
3768 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
3769 CTLFLAG_RD, &stats->gotc,
3770 "Good Octets Transmitted");
3771 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
3772 CTLFLAG_RD, &stats->tpt,
3773 "Total Packets Transmitted");
3774 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
3775 CTLFLAG_RD, &stats->gptc,
3776 "Good Packets Transmitted");
3777 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
3778 CTLFLAG_RD, &stats->bptc,
3779 "Broadcast Packets Transmitted");
3780 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
3781 CTLFLAG_RD, &stats->mptc,
3782 "Multicast Packets Transmitted");
3783 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
3784 CTLFLAG_RD, &stats->mngptc,
3785 "Management Packets Transmitted");
3786 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
3787 CTLFLAG_RD, &stats->ptc64,
3788 "64 byte frames transmitted ");
3789 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
3790 CTLFLAG_RD, &stats->ptc127,
3791 "65-127 byte frames transmitted");
3792 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
3793 CTLFLAG_RD, &stats->ptc255,
3794 "128-255 byte frames transmitted");
3795 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
3796 CTLFLAG_RD, &stats->ptc511,
3797 "256-511 byte frames transmitted");
3798 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
3799 CTLFLAG_RD, &stats->ptc1023,
3800 "512-1023 byte frames transmitted");
3801 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
3802 CTLFLAG_RD, &stats->ptc1522,
3803 "1024-1522 byte frames transmitted");
3804 }
3805 #endif
3806
3807 /*
3808 * Enable the hardware to drop packets when the buffer is full.
3809 * This is useful when multiple RX rings are used, so that no
3810 * single RX ring being full stalls the entire RX engine. We
3811 * only enable this when multiple RX rings are used and when
3812 * flow control is disabled.
3813 */
3814 static void
3815 ix_enable_rx_drop(struct ix_softc *sc)
3816 {
3817 struct ixgbe_hw *hw = &sc->hw;
3818 int i;
3819
3820 if (bootverbose) {
3821 if_printf(&sc->arpcom.ac_if,
3822 "flow control %s, enable RX drop\n",
3823 ix_fc2str(sc->hw.fc.current_mode));
3824 }
3825
3826 for (i = 0; i < sc->rx_ring_inuse; ++i) {
3827 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
3828
3829 srrctl |= IXGBE_SRRCTL_DROP_EN;
3830 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
3831 }
3832 }
3833
3834 static void
3835 ix_disable_rx_drop(struct ix_softc *sc)
3836 {
3837 struct ixgbe_hw *hw = &sc->hw;
3838 int i;
3839
3840 if (bootverbose) {
3841 if_printf(&sc->arpcom.ac_if,
3842 "flow control %s, disable RX drop\n",
3843 ix_fc2str(sc->hw.fc.current_mode));
3844 }
3845
3846 for (i = 0; i < sc->rx_ring_inuse; ++i) {
3847 uint32_t srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
3848
3849 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
3850 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
3851 }
3852 }
3853
3854 static void
3855 ix_setup_serialize(struct ix_softc *sc)
3856 {
3857 int i = 0, j;
3858
3859 /* Main + RX + TX */
3860 sc->nserialize = 1 + sc->rx_ring_cnt + sc->tx_ring_cnt;
3861 sc->serializes =
3862 kmalloc(sc->nserialize * sizeof(struct lwkt_serialize *),
3863 M_DEVBUF, M_WAITOK | M_ZERO);
3864
3865 /*
3866 * Setup serializes
3867 *
3868 * NOTE: Order is critical
3869 */
3870
3871 KKASSERT(i < sc->nserialize);
3872 sc->serializes[i++] = &sc->main_serialize;
3873
3874 for (j = 0; j < sc->rx_ring_cnt; ++j) {
3875 KKASSERT(i < sc->nserialize);
3876 sc->serializes[i++] = &sc->rx_rings[j].rx_serialize;
3877 }
3878
3879 for (j = 0; j < sc->tx_ring_cnt; ++j) {
3880 KKASSERT(i < sc->nserialize);
3881 sc->serializes[i++] = &sc->tx_rings[j].tx_serialize;
3882 }
3883
3884 KKASSERT(i == sc->nserialize);
3885 }
3886
3887 static int
3888 ix_alloc_intr(struct ix_softc *sc)
3889 {
3890 struct ix_intr_data *intr;
3891 struct ix_tx_ring *txr;
3892 u_int intr_flags;
3893 int i;
3894
3895 ix_alloc_msix(sc);
3896 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
3897 ix_set_ring_inuse(sc, FALSE);
3898 goto done;
3899 }
3900
3901 /*
3902 * Reset some settings changed by ix_alloc_msix().
3903 */
3904 if (sc->rx_rmap_intr != NULL) {
3905 if_ringmap_free(sc->rx_rmap_intr);
3906 sc->rx_rmap_intr = NULL;
3907 }
3908 if (sc->tx_rmap_intr != NULL) {
3909 if_ringmap_free(sc->tx_rmap_intr);
3910 sc->tx_rmap_intr = NULL;
3911 }
3912 if (sc->intr_data != NULL) {
3913 kfree(sc->intr_data, M_DEVBUF);
3914 sc->intr_data = NULL;
3915 }
3916 for (i = 0; i < sc->tx_ring_cnt; ++i) {
3917 txr = &sc->tx_rings[i];
3918 txr->tx_intr_vec = -1;
3919 txr->tx_intr_cpuid = -1;
3920 }
3921 for (i = 0; i < sc->rx_ring_cnt; ++i) {
3922 struct ix_rx_ring *rxr = &sc->rx_rings[i];
3923
3924 rxr->rx_intr_vec = -1;
3925 rxr->rx_txr = NULL;
3926 }
3927
3928 sc->intr_cnt = 1;
3929 sc->intr_data = kmalloc(sizeof(struct ix_intr_data), M_DEVBUF,
3930 M_WAITOK | M_ZERO);
3931 intr = &sc->intr_data[0];
3932
3933 /*
3934 * Allocate MSI/legacy interrupt resource
3935 */
3936 if (sc->caps & IX_CAP_LEGACY_INTR) {
3937 sc->intr_type = pci_alloc_1intr(sc->dev, ix_msi_enable,
3938 &intr->intr_rid, &intr_flags);
3939 } else {
3940 int cpu;
3941
3942 /*
3943 * Only MSI is supported.
3944 */
3945 cpu = device_getenv_int(sc->dev, "msi.cpu", -1);
3946 if (cpu >= ncpus)
3947 cpu = ncpus - 1;
3948
3949 if (pci_alloc_msi(sc->dev, &intr->intr_rid, 1, cpu) == 0) {
3950 sc->intr_type = PCI_INTR_TYPE_MSI;
3951 intr_flags = RF_ACTIVE;
3952 } else {
3953 sc->intr_type = PCI_INTR_TYPE_LEGACY;
3954 device_printf(sc->dev, "Unable to allocate MSI\n");
3955 return ENXIO;
3956 }
3957 }
3958
3959 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
3960 &intr->intr_rid, intr_flags);
3961 if (intr->intr_res == NULL) {
3962 device_printf(sc->dev, "Unable to allocate bus resource: "
3963 "interrupt\n");
3964 return ENXIO;
3965 }
3966
3967 intr->intr_serialize = &sc->main_serialize;
3968 intr->intr_cpuid = rman_get_cpuid(intr->intr_res);
3969 if (sc->hw.mac.type == ixgbe_mac_82598EB)
3970 intr->intr_func = ix_intr_82598;
3971 else
3972 intr->intr_func = ix_intr;
3973 intr->intr_funcarg = sc;
3974 intr->intr_rate = IX_INTR_RATE;
3975 intr->intr_use = IX_INTR_USE_RXTX;
3976
3977 sc->tx_rings[0].tx_intr_vec = IX_TX_INTR_VEC;
3978 sc->tx_rings[0].tx_intr_cpuid = intr->intr_cpuid;
3979
3980 sc->rx_rings[0].rx_intr_vec = IX_RX0_INTR_VEC;
3981
3982 ix_set_ring_inuse(sc, FALSE);
3983
3984 KKASSERT(sc->rx_ring_inuse <= IX_MIN_RXRING_RSS);
3985 if (sc->rx_ring_inuse == IX_MIN_RXRING_RSS) {
3986 sc->rx_rings[1].rx_intr_vec = IX_RX1_INTR_VEC;
3987
3988 /*
3989 * Allocate RX ring map for RSS setup.
3990 */
3991 sc->rx_rmap_intr = if_ringmap_alloc(sc->dev,
3992 IX_MIN_RXRING_RSS, IX_MIN_RXRING_RSS);
3993 KASSERT(if_ringmap_count(sc->rx_rmap_intr) ==
3994 sc->rx_ring_inuse, ("RX ring inuse mismatch"));
3995 }
3996 done:
3997 for (i = 0; i < sc->tx_ring_cnt; ++i) {
3998 txr = &sc->tx_rings[i];
3999 if (txr->tx_intr_cpuid < 0)
4000 txr->tx_intr_cpuid = 0;
4001 }
4002 return 0;
4003 }
4004
4005 static void
4006 ix_free_intr(struct ix_softc *sc)
4007 {
4008 if (sc->intr_data == NULL)
4009 return;
4010
4011 if (sc->intr_type != PCI_INTR_TYPE_MSIX) {
4012 struct ix_intr_data *intr = &sc->intr_data[0];
4013
4014 KKASSERT(sc->intr_cnt == 1);
4015 if (intr->intr_res != NULL) {
4016 bus_release_resource(sc->dev, SYS_RES_IRQ,
4017 intr->intr_rid, intr->intr_res);
4018 }
4019 if (sc->intr_type == PCI_INTR_TYPE_MSI)
4020 pci_release_msi(sc->dev);
4021
4022 kfree(sc->intr_data, M_DEVBUF);
4023 } else {
4024 ix_free_msix(sc, TRUE);
4025 }
4026 }
4027
4028 static void
4029 ix_set_ring_inuse(struct ix_softc *sc, boolean_t polling)
4030 {
4031 sc->rx_ring_inuse = ix_get_rxring_inuse(sc, polling);
4032 sc->tx_ring_inuse = ix_get_txring_inuse(sc, polling);
4033 if (bootverbose) {
4034 if_printf(&sc->arpcom.ac_if,
4035 "RX rings %d/%d, TX rings %d/%d\n",
4036 sc->rx_ring_inuse, sc->rx_ring_cnt,
4037 sc->tx_ring_inuse, sc->tx_ring_cnt);
4038 }
4039 }
4040
4041 static int
4042 ix_get_rxring_inuse(const struct ix_softc *sc, boolean_t polling)
4043 {
4044 if (!IX_ENABLE_HWRSS(sc))
4045 return 1;
4046
4047 if (polling)
4048 return sc->rx_ring_cnt;
4049 else if (sc->intr_type != PCI_INTR_TYPE_MSIX)
4050 return IX_MIN_RXRING_RSS;
4051 else
4052 return sc->rx_ring_msix;
4053 }
4054
4055 static int
4056 ix_get_txring_inuse(const struct ix_softc *sc, boolean_t polling)
4057 {
4058 if (!IX_ENABLE_HWTSS(sc))
4059 return 1;
4060
4061 if (polling)
4062 return sc->tx_ring_cnt;
4063 else if (sc->intr_type != PCI_INTR_TYPE_MSIX)
4064 return 1;
4065 else
4066 return sc->tx_ring_msix;
4067 }
4068
4069 static int
4070 ix_setup_intr(struct ix_softc *sc)
4071 {
4072 int i;
4073
4074 for (i = 0; i < sc->intr_cnt; ++i) {
4075 struct ix_intr_data *intr = &sc->intr_data[i];
4076 int error;
4077
4078 error = bus_setup_intr_descr(sc->dev, intr->intr_res,
4079 INTR_MPSAFE, intr->intr_func, intr->intr_funcarg,
4080 &intr->intr_hand, intr->intr_serialize, intr->intr_desc);
4081 if (error) {
4082 device_printf(sc->dev, "can't setup %dth intr\n", i);
4083 ix_teardown_intr(sc, i);
4084 return error;
4085 }
4086 }
4087 return 0;
4088 }
4089
4090 static void
4091 ix_teardown_intr(struct ix_softc *sc, int intr_cnt)
4092 {
4093 int i;
4094
4095 if (sc->intr_data == NULL)
4096 return;
4097
4098 for (i = 0; i < intr_cnt; ++i) {
4099 struct ix_intr_data *intr = &sc->intr_data[i];
4100
4101 bus_teardown_intr(sc->dev, intr->intr_res, intr->intr_hand);
4102 }
4103 }
4104
4105 static void
4106 ix_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
4107 {
4108 struct ix_softc *sc = ifp->if_softc;
4109
4110 ifnet_serialize_array_enter(sc->serializes, sc->nserialize, slz);
4111 }
4112
4113 static void
4114 ix_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
4115 {
4116 struct ix_softc *sc = ifp->if_softc;
4117
4118 ifnet_serialize_array_exit(sc->serializes, sc->nserialize, slz);
4119 }
4120
4121 static int
4122 ix_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
4123 {
4124 struct ix_softc *sc = ifp->if_softc;
4125
4126 return ifnet_serialize_array_try(sc->serializes, sc->nserialize, slz);
4127 }
4128
4129 static void
4130 ix_serialize_skipmain(struct ix_softc *sc)
4131 {
4132 lwkt_serialize_array_enter(sc->serializes, sc->nserialize, 1);
4133 }
4134
4135 static void
4136 ix_deserialize_skipmain(struct ix_softc *sc)
4137 {
4138 lwkt_serialize_array_exit(sc->serializes, sc->nserialize, 1);
4139 }
4140
4141 #ifdef INVARIANTS
4142
4143 static void
4144 ix_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
4145 boolean_t serialized)
4146 {
4147 struct ix_softc *sc = ifp->if_softc;
4148
4149 ifnet_serialize_array_assert(sc->serializes, sc->nserialize, slz,
4150 serialized);
4151 }
4152
4153 #endif /* INVARIANTS */
4154
4155 static void
4156 ix_free_rings(struct ix_softc *sc)
4157 {
4158 int i;
4159
4160 if (sc->tx_rings != NULL) {
4161 for (i = 0; i < sc->tx_ring_cnt; ++i) {
4162 struct ix_tx_ring *txr = &sc->tx_rings[i];
4163
4164 ix_destroy_tx_ring(txr, txr->tx_ndesc);
4165 }
4166 kfree(sc->tx_rings, M_DEVBUF);
4167 }
4168
4169 if (sc->rx_rings != NULL) {
4170 for (i =0; i < sc->rx_ring_cnt; ++i) {
4171 struct ix_rx_ring *rxr = &sc->rx_rings[i];
4172
4173 ix_destroy_rx_ring(rxr, rxr->rx_ndesc);
4174 }
4175 kfree(sc->rx_rings, M_DEVBUF);
4176 }
4177
4178 if (sc->parent_tag != NULL)
4179 bus_dma_tag_destroy(sc->parent_tag);
4180 }
4181
4182 static void
4183 ix_watchdog_reset(struct ix_softc *sc)
4184 {
4185 int i;
4186
4187 ASSERT_IFNET_SERIALIZED_ALL(&sc->arpcom.ac_if);
4188 ix_init(sc);
4189 for (i = 0; i < sc->tx_ring_inuse; ++i)
4190 ifsq_devstart_sched(sc->tx_rings[i].tx_ifsq);
4191 }
4192
4193 static void
4194 ix_sync_netisr(struct ix_softc *sc, int flags)
4195 {
4196 struct ifnet *ifp = &sc->arpcom.ac_if;
4197
4198 ifnet_serialize_all(ifp);
4199 if (ifp->if_flags & IFF_RUNNING) {
4200 ifp->if_flags &= ~(IFF_RUNNING | flags);
4201 } else {
4202 ifnet_deserialize_all(ifp);
4203 return;
4204 }
4205 ifnet_deserialize_all(ifp);
4206
4207 /* Make sure that polling stopped. */
4208 netmsg_service_sync();
4209 }
4210
4211 static void
4212 ix_watchdog_task(void *xsc, int pending __unused)
4213 {
4214 struct ix_softc *sc = xsc;
4215 struct ifnet *ifp = &sc->arpcom.ac_if;
4216
4217 ix_sync_netisr(sc, 0);
4218
4219 ifnet_serialize_all(ifp);
4220 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == IFF_UP)
4221 ix_watchdog_reset(sc);
4222 ifnet_deserialize_all(ifp);
4223 }
4224
4225 static void
4226 ix_watchdog(struct ifaltq_subque *ifsq)
4227 {
4228 struct ix_tx_ring *txr = ifsq_get_priv(ifsq);
4229 struct ifnet *ifp = ifsq_get_ifp(ifsq);
4230 struct ix_softc *sc = ifp->if_softc;
4231
4232 KKASSERT(txr->tx_ifsq == ifsq);
4233 ASSERT_IFNET_SERIALIZED_ALL(ifp);
4234
4235 /*
4236 * If the interface has been paused then don't do the watchdog check
4237 */
4238 if (IXGBE_READ_REG(&sc->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) {
4239 ifsq_watchdog_set_count(&txr->tx_watchdog, 5);
4240 return;
4241 }
4242
4243 if_printf(ifp, "Watchdog timeout -- resetting\n");
4244 if_printf(ifp, "Queue(%d) tdh = %d, hw tdt = %d\n", txr->tx_idx,
4245 IXGBE_READ_REG(&sc->hw, IXGBE_TDH(txr->tx_idx)),
4246 IXGBE_READ_REG(&sc->hw, IXGBE_TDT(txr->tx_idx)));
4247 if_printf(ifp, "TX(%d) desc avail = %d, next TX to Clean = %d\n",
4248 txr->tx_idx, txr->tx_avail, txr->tx_next_clean);
4249
4250 if ((ifp->if_flags & (IFF_IDIRECT | IFF_NPOLLING | IFF_RUNNING)) ==
4251 (IFF_IDIRECT | IFF_NPOLLING | IFF_RUNNING))
4252 taskqueue_enqueue(taskqueue_thread[0], &sc->wdog_task);
4253 else
4254 ix_watchdog_reset(sc);
4255 }
4256
4257 static void
4258 ix_free_tx_ring(struct ix_tx_ring *txr)
4259 {
4260 int i;
4261
4262 for (i = 0; i < txr->tx_ndesc; ++i) {
4263 struct ix_tx_buf *txbuf = &txr->tx_buf[i];
4264
4265 if (txbuf->m_head != NULL)
4266 ix_free_txbuf(txr, txbuf);
4267 }
4268 }
4269
4270 static void
4271 ix_free_rx_ring(struct ix_rx_ring *rxr)
4272 {
4273 int i;
4274
4275 for (i = 0; i < rxr->rx_ndesc; ++i) {
4276 struct ix_rx_buf *rxbuf = &rxr->rx_buf[i];
4277
4278 if (rxbuf->fmp != NULL) {
4279 m_freem(rxbuf->fmp);
4280 rxbuf->fmp = NULL;
4281 rxbuf->lmp = NULL;
4282 } else {
4283 KKASSERT(rxbuf->lmp == NULL);
4284 }
4285 if (rxbuf->m_head != NULL) {
4286 bus_dmamap_unload(rxr->rx_tag, rxbuf->map);
4287 m_freem(rxbuf->m_head);
4288 rxbuf->m_head = NULL;
4289 }
4290 }
4291 }
4292
4293 static int
4294 ix_newbuf(struct ix_rx_ring *rxr, int i, boolean_t wait)
4295 {
4296 struct mbuf *m;
4297 bus_dma_segment_t seg;
4298 bus_dmamap_t map;
4299 struct ix_rx_buf *rxbuf;
4300 int flags, error, nseg;
4301
4302 flags = M_NOWAIT;
4303 if (__predict_false(wait))
4304 flags = M_WAITOK;
4305
4306 m = m_getjcl(flags, MT_DATA, M_PKTHDR, rxr->rx_mbuf_sz);
4307 if (m == NULL) {
4308 if (wait) {
4309 if_printf(&rxr->rx_sc->arpcom.ac_if,
4310 "Unable to allocate RX mbuf\n");
4311 }
4312 return ENOBUFS;
4313 }
4314 m->m_len = m->m_pkthdr.len = rxr->rx_mbuf_sz;
4315
4316 error = bus_dmamap_load_mbuf_segment(rxr->rx_tag,
4317 rxr->rx_sparemap, m, &seg, 1, &nseg, BUS_DMA_NOWAIT);
4318 if (error) {
4319 m_freem(m);
4320 if (wait) {
4321 if_printf(&rxr->rx_sc->arpcom.ac_if,
4322 "Unable to load RX mbuf\n");
4323 }
4324 return error;
4325 }
4326
4327 rxbuf = &rxr->rx_buf[i];
4328 if (rxbuf->m_head != NULL)
4329 bus_dmamap_unload(rxr->rx_tag, rxbuf->map);
4330
4331 map = rxbuf->map;
4332 rxbuf->map = rxr->rx_sparemap;
4333 rxr->rx_sparemap = map;
4334
4335 rxbuf->m_head = m;
4336 rxbuf->paddr = seg.ds_addr;
4337
4338 ix_setup_rxdesc(&rxr->rx_base[i], rxbuf);
4339 return 0;
4340 }
4341
4342 static void
4343 ix_add_sysctl(struct ix_softc *sc)
4344 {
4345 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
4346 struct sysctl_oid *tree = device_get_sysctl_tree(sc->dev);
4347 char node[32];
4348 int i;
4349
4350 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4351 OID_AUTO, "rxr", CTLFLAG_RD, &sc->rx_ring_cnt, 0, "# of RX rings");
4352 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4353 OID_AUTO, "rxr_inuse", CTLFLAG_RD, &sc->rx_ring_inuse, 0,
4354 "# of RX rings used");
4355 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4356 OID_AUTO, "txr", CTLFLAG_RD, &sc->tx_ring_cnt, 0, "# of TX rings");
4357 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4358 OID_AUTO, "txr_inuse", CTLFLAG_RD, &sc->tx_ring_inuse, 0,
4359 "# of TX rings used");
4360 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4361 OID_AUTO, "rxd", CTLTYPE_INT | CTLFLAG_RD,
4362 sc, 0, ix_sysctl_rxd, "I",
4363 "# of RX descs");
4364 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4365 OID_AUTO, "txd", CTLTYPE_INT | CTLFLAG_RD,
4366 sc, 0, ix_sysctl_txd, "I",
4367 "# of TX descs");
4368 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4369 OID_AUTO, "tx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW,
4370 sc, 0, ix_sysctl_tx_wreg_nsegs, "I",
4371 "# of segments sent before write to hardware register");
4372 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4373 OID_AUTO, "rx_wreg_nsegs", CTLTYPE_INT | CTLFLAG_RW,
4374 sc, 0, ix_sysctl_rx_wreg_nsegs, "I",
4375 "# of received segments sent before write to hardware register");
4376 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4377 OID_AUTO, "tx_intr_nsegs", CTLTYPE_INT | CTLFLAG_RW,
4378 sc, 0, ix_sysctl_tx_intr_nsegs, "I",
4379 "# of segments per TX interrupt");
4380 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4381 OID_AUTO, "direct_input", CTLFLAG_RW, &sc->direct_input, 0,
4382 "Enable direct input");
4383 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
4384 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4385 OID_AUTO, "tx_msix_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD,
4386 sc->tx_rmap_intr, 0, if_ringmap_cpumap_sysctl, "I",
4387 "TX MSI-X CPU map");
4388 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4389 OID_AUTO, "rx_msix_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD,
4390 sc->rx_rmap_intr, 0, if_ringmap_cpumap_sysctl, "I",
4391 "RX MSI-X CPU map");
4392 }
4393 #ifdef IFPOLL_ENABLE
4394 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4395 OID_AUTO, "tx_poll_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD,
4396 sc->tx_rmap, 0, if_ringmap_cpumap_sysctl, "I",
4397 "TX polling CPU map");
4398 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4399 OID_AUTO, "rx_poll_cpumap", CTLTYPE_OPAQUE | CTLFLAG_RD,
4400 sc->rx_rmap, 0, if_ringmap_cpumap_sysctl, "I",
4401 "RX polling CPU map");
4402 #endif
4403
4404 #define IX_ADD_INTR_RATE_SYSCTL(sc, use, name) \
4405 do { \
4406 ix_add_intr_rate_sysctl(sc, IX_INTR_USE_##use, #name, \
4407 ix_sysctl_##name, #use " interrupt rate"); \
4408 } while (0)
4409
4410 IX_ADD_INTR_RATE_SYSCTL(sc, RXTX, rxtx_intr_rate);
4411 IX_ADD_INTR_RATE_SYSCTL(sc, RX, rx_intr_rate);
4412 IX_ADD_INTR_RATE_SYSCTL(sc, TX, tx_intr_rate);
4413 IX_ADD_INTR_RATE_SYSCTL(sc, STATUS, sts_intr_rate);
4414
4415 #undef IX_ADD_INTR_RATE_SYSCTL
4416
4417 #ifdef IX_RSS_DEBUG
4418 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4419 OID_AUTO, "rss_debug", CTLFLAG_RW, &sc->rss_debug, 0,
4420 "RSS debug level");
4421 for (i = 0; i < sc->rx_ring_cnt; ++i) {
4422 ksnprintf(node, sizeof(node), "rx%d_pkt", i);
4423 SYSCTL_ADD_ULONG(ctx,
4424 SYSCTL_CHILDREN(tree), OID_AUTO, node,
4425 CTLFLAG_RW, &sc->rx_rings[i].rx_pkts, "RXed packets");
4426 }
4427 #endif
4428 for (i = 0; i < sc->tx_ring_cnt; ++i) {
4429 struct ix_tx_ring *txr = &sc->tx_rings[i];
4430
4431 ksnprintf(node, sizeof(node), "tx%d_nmbuf", i);
4432 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, node,
4433 CTLTYPE_INT | CTLFLAG_RD, txr, 0, ix_sysctl_tx_nmbuf, "I",
4434 "# of pending TX mbufs");
4435
4436 ksnprintf(node, sizeof(node), "tx%d_gc", i);
4437 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, node,
4438 CTLFLAG_RW, &txr->tx_gc, "# of TX desc GC");
4439 }
4440
4441 #if 0
4442 ix_add_hw_stats(sc);
4443 #endif
4444
4445 }
4446
4447 static int
4448 ix_sysctl_tx_nmbuf(SYSCTL_HANDLER_ARGS)
4449 {
4450 struct ix_tx_ring *txr = (void *)arg1;
4451 int nmbuf;
4452
4453 nmbuf = txr->tx_nmbuf;
4454 return (sysctl_handle_int(oidp, &nmbuf, 0, req));
4455 }
4456
4457 static int
4458 ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS)
4459 {
4460 struct ix_softc *sc = (void *)arg1;
4461 struct ifnet *ifp = &sc->arpcom.ac_if;
4462 int error, nsegs, i;
4463
4464 nsegs = sc->tx_rings[0].tx_wreg_nsegs;
4465 error = sysctl_handle_int(oidp, &nsegs, 0, req);
4466 if (error || req->newptr == NULL)
4467 return error;
4468 if (nsegs < 0)
4469 return EINVAL;
4470
4471 ifnet_serialize_all(ifp);
4472 for (i = 0; i < sc->tx_ring_cnt; ++i)
4473 sc->tx_rings[i].tx_wreg_nsegs = nsegs;
4474 ifnet_deserialize_all(ifp);
4475
4476 return 0;
4477 }
4478
4479 static int
4480 ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS)
4481 {
4482 struct ix_softc *sc = (void *)arg1;
4483 struct ifnet *ifp = &sc->arpcom.ac_if;
4484 int error, nsegs, i;
4485
4486 nsegs = sc->rx_rings[0].rx_wreg_nsegs;
4487 error = sysctl_handle_int(oidp, &nsegs, 0, req);
4488 if (error || req->newptr == NULL)
4489 return error;
4490 if (nsegs < 0)
4491 return EINVAL;
4492
4493 ifnet_serialize_all(ifp);
4494 for (i = 0; i < sc->rx_ring_cnt; ++i)
4495 sc->rx_rings[i].rx_wreg_nsegs =nsegs;
4496 ifnet_deserialize_all(ifp);
4497
4498 return 0;
4499 }
4500
4501 static int
4502 ix_sysctl_txd(SYSCTL_HANDLER_ARGS)
4503 {
4504 struct ix_softc *sc = (void *)arg1;
4505 int txd;
4506
4507 txd = sc->tx_rings[0].tx_ndesc;
4508 return sysctl_handle_int(oidp, &txd, 0, req);
4509 }
4510
4511 static int
4512 ix_sysctl_rxd(SYSCTL_HANDLER_ARGS)
4513 {
4514 struct ix_softc *sc = (void *)arg1;
4515 int rxd;
4516
4517 rxd = sc->rx_rings[0].rx_ndesc;
4518 return sysctl_handle_int(oidp, &rxd, 0, req);
4519 }
4520
4521 static int
4522 ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS)
4523 {
4524 struct ix_softc *sc = (void *)arg1;
4525 struct ifnet *ifp = &sc->arpcom.ac_if;
4526 struct ix_tx_ring *txr = &sc->tx_rings[0];
4527 int error, nsegs;
4528
4529 nsegs = txr->tx_intr_nsegs;
4530 error = sysctl_handle_int(oidp, &nsegs, 0, req);
4531 if (error || req->newptr == NULL)
4532 return error;
4533 if (nsegs < 0)
4534 return EINVAL;
4535
4536 ifnet_serialize_all(ifp);
4537
4538 if (nsegs >= txr->tx_ndesc - IX_MAX_SCATTER - IX_TX_RESERVED) {
4539 error = EINVAL;
4540 } else {
4541 int i;
4542
4543 error = 0;
4544 for (i = 0; i < sc->tx_ring_cnt; ++i)
4545 sc->tx_rings[i].tx_intr_nsegs = nsegs;
4546 }
4547
4548 ifnet_deserialize_all(ifp);
4549
4550 return error;
4551 }
4552
4553 static void
4554 ix_set_eitr(struct ix_softc *sc, int idx, int rate)
4555 {
4556 uint32_t eitr, eitr_intvl;
4557
4558 eitr = IXGBE_READ_REG(&sc->hw, IXGBE_EITR(idx));
4559 eitr_intvl = 1000000000 / 256 / rate;
4560
4561 if (sc->hw.mac.type == ixgbe_mac_82598EB) {
4562 eitr &= ~IX_EITR_INTVL_MASK_82598;
4563 if (eitr_intvl == 0)
4564 eitr_intvl = 1;
4565 else if (eitr_intvl > IX_EITR_INTVL_MASK_82598)
4566 eitr_intvl = IX_EITR_INTVL_MASK_82598;
4567 } else {
4568 eitr &= ~IX_EITR_INTVL_MASK;
4569
4570 eitr_intvl &= ~IX_EITR_INTVL_RSVD_MASK;
4571 if (eitr_intvl == 0)
4572 eitr_intvl = IX_EITR_INTVL_MIN;
4573 else if (eitr_intvl > IX_EITR_INTVL_MAX)
4574 eitr_intvl = IX_EITR_INTVL_MAX;
4575 }
4576 eitr |= eitr_intvl;
4577
4578 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
4579 }
4580
4581 static int
4582 ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS)
4583 {
4584 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RXTX);
4585 }
4586
4587 static int
4588 ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS)
4589 {
4590 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_RX);
4591 }
4592
4593 static int
4594 ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS)
4595 {
4596 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_TX);
4597 }
4598
4599 static int
4600 ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS)
4601 {
4602 return ix_sysctl_intr_rate(oidp, arg1, arg2, req, IX_INTR_USE_STATUS);
4603 }
4604
4605 static int
4606 ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS, int use)
4607 {
4608 struct ix_softc *sc = (void *)arg1;
4609 struct ifnet *ifp = &sc->arpcom.ac_if;
4610 int error, rate, i;
4611
4612 rate = 0;
4613 for (i = 0; i < sc->intr_cnt; ++i) {
4614 if (sc->intr_data[i].intr_use == use) {
4615 rate = sc->intr_data[i].intr_rate;
4616 break;
4617 }
4618 }
4619
4620 error = sysctl_handle_int(oidp, &rate, 0, req);
4621 if (error || req->newptr == NULL)
4622 return error;
4623 if (rate <= 0)
4624 return EINVAL;
4625
4626 ifnet_serialize_all(ifp);
4627
4628 for (i = 0; i < sc->intr_cnt; ++i) {
4629 if (sc->intr_data[i].intr_use == use) {
4630 sc->intr_data[i].intr_rate = rate;
4631 if (ifp->if_flags & IFF_RUNNING)
4632 ix_set_eitr(sc, i, rate);
4633 }
4634 }
4635
4636 ifnet_deserialize_all(ifp);
4637
4638 return error;
4639 }
4640
4641 static void
4642 ix_add_intr_rate_sysctl(struct ix_softc *sc, int use,
4643 const char *name, int (*handler)(SYSCTL_HANDLER_ARGS), const char *desc)
4644 {
4645 int i;
4646
4647 for (i = 0; i < sc->intr_cnt; ++i) {
4648 if (sc->intr_data[i].intr_use == use) {
4649 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc->dev),
4650 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)),
4651 OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW,
4652 sc, 0, handler, "I", desc);
4653 break;
4654 }
4655 }
4656 }
4657
4658 static int
4659 ix_get_timer_cpuid(const struct ix_softc *sc, boolean_t polling)
4660 {
4661 if (polling || sc->intr_type == PCI_INTR_TYPE_MSIX)
4662 return 0; /* XXX fixed */
4663 else
4664 return rman_get_cpuid(sc->intr_data[0].intr_res);
4665 }
4666
4667 static void
4668 ix_alloc_msix(struct ix_softc *sc)
4669 {
4670 int msix_enable, msix_cnt, msix_ring, alloc_cnt;
4671 struct ix_intr_data *intr;
4672 int i, x, error;
4673 int ring_cnt, ring_cntmax;
4674 boolean_t setup = FALSE;
4675
4676 msix_enable = ix_msix_enable;
4677 /*
4678 * Don't enable MSI-X on 82598 by default, see:
4679 * 82598 specification update errata #38
4680 */
4681 if (sc->hw.mac.type == ixgbe_mac_82598EB)
4682 msix_enable = 0;
4683 msix_enable = device_getenv_int(sc->dev, "msix.enable", msix_enable);
4684 if (!msix_enable)
4685 return;
4686
4687 msix_cnt = pci_msix_count(sc->dev);
4688 #ifdef IX_MSIX_DEBUG
4689 msix_cnt = device_getenv_int(sc->dev, "msix.count", msix_cnt);
4690 #endif
4691 if (msix_cnt <= 1) {
4692 /* One MSI-X model does not make sense. */
4693 return;
4694 }
4695
4696 /*
4697 * Make sure that we don't break interrupt related registers
4698 * (EIMS, etc) limitation.
4699 */
4700 if (sc->hw.mac.type == ixgbe_mac_82598EB) {
4701 if (msix_cnt > IX_MAX_MSIX_82598)
4702 msix_cnt = IX_MAX_MSIX_82598;
4703 } else {
4704 if (msix_cnt > IX_MAX_MSIX)
4705 msix_cnt = IX_MAX_MSIX;
4706 }
4707 if (bootverbose)
4708 device_printf(sc->dev, "MSI-X count %d\n", msix_cnt);
4709 msix_ring = msix_cnt - 1; /* -1 for status */
4710
4711 /*
4712 * Configure # of RX/TX rings usable by MSI-X.
4713 */
4714 ix_get_rxring_cnt(sc, &ring_cnt, &ring_cntmax);
4715 if (ring_cntmax > msix_ring)
4716 ring_cntmax = msix_ring;
4717 sc->rx_rmap_intr = if_ringmap_alloc(sc->dev, ring_cnt, ring_cntmax);
4718
4719 ix_get_txring_cnt(sc, &ring_cnt, &ring_cntmax);
4720 if (ring_cntmax > msix_ring)
4721 ring_cntmax = msix_ring;
4722 sc->tx_rmap_intr = if_ringmap_alloc(sc->dev, ring_cnt, ring_cntmax);
4723
4724 if_ringmap_match(sc->dev, sc->rx_rmap_intr, sc->tx_rmap_intr);
4725 sc->rx_ring_msix = if_ringmap_count(sc->rx_rmap_intr);
4726 KASSERT(sc->rx_ring_msix <= sc->rx_ring_cnt,
4727 ("total RX ring count %d, MSI-X RX ring count %d",
4728 sc->rx_ring_cnt, sc->rx_ring_msix));
4729 sc->tx_ring_msix = if_ringmap_count(sc->tx_rmap_intr);
4730 KASSERT(sc->tx_ring_msix <= sc->tx_ring_cnt,
4731 ("total TX ring count %d, MSI-X TX ring count %d",
4732 sc->tx_ring_cnt, sc->tx_ring_msix));
4733
4734 /*
4735 * Aggregate TX/RX MSI-X
4736 */
4737 ring_cntmax = sc->rx_ring_msix;
4738 if (ring_cntmax < sc->tx_ring_msix)
4739 ring_cntmax = sc->tx_ring_msix;
4740 KASSERT(ring_cntmax <= msix_ring,
4741 ("invalid ring count max %d, MSI-X count for rings %d",
4742 ring_cntmax, msix_ring));
4743
4744 alloc_cnt = ring_cntmax + 1; /* +1 for status */
4745 if (bootverbose) {
4746 device_printf(sc->dev, "MSI-X alloc %d, "
4747 "RX ring %d, TX ring %d\n", alloc_cnt,
4748 sc->rx_ring_msix, sc->tx_ring_msix);
4749 }
4750
4751 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82598);
4752 sc->msix_mem_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
4753 &sc->msix_mem_rid, RF_ACTIVE);
4754 if (sc->msix_mem_res == NULL) {
4755 sc->msix_mem_rid = PCIR_BAR(IX_MSIX_BAR_82599);
4756 sc->msix_mem_res = bus_alloc_resource_any(sc->dev,
4757 SYS_RES_MEMORY, &sc->msix_mem_rid, RF_ACTIVE);
4758 if (sc->msix_mem_res == NULL) {
4759 device_printf(sc->dev, "Unable to map MSI-X table\n");
4760 return;
4761 }
4762 }
4763
4764 sc->intr_cnt = alloc_cnt;
4765 sc->intr_data = kmalloc(sizeof(struct ix_intr_data) * sc->intr_cnt,
4766 M_DEVBUF, M_WAITOK | M_ZERO);
4767 for (x = 0; x < sc->intr_cnt; ++x) {
4768 intr = &sc->intr_data[x];
4769 intr->intr_rid = -1;
4770 intr->intr_rate = IX_INTR_RATE;
4771 }
4772
4773 x = 0;
4774 for (i = 0; i < sc->rx_ring_msix; ++i) {
4775 struct ix_rx_ring *rxr = &sc->rx_rings[i];
4776 struct ix_tx_ring *txr = NULL;
4777 int cpuid, j;
4778
4779 KKASSERT(x < sc->intr_cnt);
4780 rxr->rx_intr_vec = x;
4781 ix_setup_msix_eims(sc, x,
4782 &rxr->rx_eims, &rxr->rx_eims_val);
4783
4784 cpuid = if_ringmap_cpumap(sc->rx_rmap_intr, i);
4785
4786 /*
4787 * Try finding TX ring to piggyback.
4788 */
4789 for (j = 0; j < sc->tx_ring_msix; ++j) {
4790 if (cpuid ==
4791 if_ringmap_cpumap(sc->tx_rmap_intr, j)) {
4792 txr = &sc->tx_rings[j];
4793 KKASSERT(txr->tx_intr_cpuid < 0);
4794 break;
4795 }
4796 }
4797 rxr->rx_txr = txr;
4798
4799 intr = &sc->intr_data[x++];
4800 intr->intr_serialize = &rxr->rx_serialize;
4801 if (txr != NULL) {
4802 ksnprintf(intr->intr_desc0,
4803 sizeof(intr->intr_desc0), "%s rx%dtx%d",
4804 device_get_nameunit(sc->dev), i, txr->tx_idx);
4805 intr->intr_use = IX_INTR_USE_RXTX;
4806 intr->intr_func = ix_msix_rxtx;
4807 } else {
4808 ksnprintf(intr->intr_desc0,
4809 sizeof(intr->intr_desc0), "%s rx%d",
4810 device_get_nameunit(sc->dev), i);
4811 intr->intr_rate = IX_MSIX_RX_RATE;
4812 intr->intr_use = IX_INTR_USE_RX;
4813 intr->intr_func = ix_msix_rx;
4814 }
4815 intr->intr_funcarg = rxr;
4816 intr->intr_cpuid = cpuid;
4817 KKASSERT(intr->intr_cpuid < netisr_ncpus);
4818 intr->intr_desc = intr->intr_desc0;
4819
4820 if (txr != NULL) {
4821 txr->tx_intr_cpuid = intr->intr_cpuid;
4822 /* NOTE: Leave TX ring's intr_vec negative. */
4823 }
4824 }
4825
4826 for (i = 0; i < sc->tx_ring_msix; ++i) {
4827 struct ix_tx_ring *txr = &sc->tx_rings[i];
4828
4829 if (txr->tx_intr_cpuid >= 0) {
4830 /* Piggybacked by RX ring. */
4831 continue;
4832 }
4833
4834 KKASSERT(x < sc->intr_cnt);
4835 txr->tx_intr_vec = x;
4836 ix_setup_msix_eims(sc, x, &txr->tx_eims, &txr->tx_eims_val);
4837
4838 intr = &sc->intr_data[x++];
4839 intr->intr_serialize = &txr->tx_serialize;
4840 intr->intr_rate = IX_MSIX_TX_RATE;
4841 intr->intr_use = IX_INTR_USE_TX;
4842 intr->intr_func = ix_msix_tx;
4843 intr->intr_funcarg = txr;
4844 intr->intr_cpuid = if_ringmap_cpumap(sc->tx_rmap_intr, i);
4845 KKASSERT(intr->intr_cpuid < netisr_ncpus);
4846 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s tx%d",
4847 device_get_nameunit(sc->dev), i);
4848 intr->intr_desc = intr->intr_desc0;
4849
4850 txr->tx_intr_cpuid = intr->intr_cpuid;
4851 }
4852
4853 /*
4854 * Status MSI-X
4855 */
4856 KKASSERT(x < sc->intr_cnt);
4857 sc->sts_msix_vec = x;
4858
4859 intr = &sc->intr_data[x++];
4860
4861 intr->intr_serialize = &sc->main_serialize;
4862 intr->intr_func = ix_msix_status;
4863 intr->intr_funcarg = sc;
4864 intr->intr_cpuid = 0;
4865 intr->intr_use = IX_INTR_USE_STATUS;
4866
4867 ksnprintf(intr->intr_desc0, sizeof(intr->intr_desc0), "%s sts",
4868 device_get_nameunit(sc->dev));
4869 intr->intr_desc = intr->intr_desc0;
4870
4871 KKASSERT(x == sc->intr_cnt);
4872
4873 error = pci_setup_msix(sc->dev);
4874 if (error) {
4875 device_printf(sc->dev, "Setup MSI-X failed\n");
4876 goto back;
4877 }
4878 setup = TRUE;
4879
4880 for (i = 0; i < sc->intr_cnt; ++i) {
4881 intr = &sc->intr_data[i];
4882
4883 error = pci_alloc_msix_vector(sc->dev, i, &intr->intr_rid,
4884 intr->intr_cpuid);
4885 if (error) {
4886 device_printf(sc->dev,
4887 "Unable to allocate MSI-X %d on cpu%d\n", i,
4888 intr->intr_cpuid);
4889 goto back;
4890 }
4891
4892 intr->intr_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
4893 &intr->intr_rid, RF_ACTIVE);
4894 if (intr->intr_res == NULL) {
4895 device_printf(sc->dev,
4896 "Unable to allocate MSI-X %d resource\n", i);
4897 error = ENOMEM;
4898 goto back;
4899 }
4900 }
4901
4902 pci_enable_msix(sc->dev);
4903 sc->intr_type = PCI_INTR_TYPE_MSIX;
4904 back:
4905 if (error)
4906 ix_free_msix(sc, setup);
4907 }
4908
4909 static void
4910 ix_free_msix(struct ix_softc *sc, boolean_t setup)
4911 {
4912 int i;
4913
4914 KKASSERT(sc->intr_cnt > 1);
4915
4916 for (i = 0; i < sc->intr_cnt; ++i) {
4917 struct ix_intr_data *intr = &sc->intr_data[i];
4918
4919 if (intr->intr_res != NULL) {
4920 bus_release_resource(sc->dev, SYS_RES_IRQ,
4921 intr->intr_rid, intr->intr_res);
4922 }
4923 if (intr->intr_rid >= 0)
4924 pci_release_msix_vector(sc->dev, intr->intr_rid);
4925 }
4926 if (setup)
4927 pci_teardown_msix(sc->dev);
4928
4929 sc->intr_cnt = 0;
4930 kfree(sc->intr_data, M_DEVBUF);
4931 sc->intr_data = NULL;
4932 }
4933
4934 static void
4935 ix_msix_rx(void *xrxr)
4936 {
4937 struct ix_rx_ring *rxr = xrxr;
4938
4939 ASSERT_SERIALIZED(&rxr->rx_serialize);
4940
4941 ix_rxeof(rxr, -1);
4942 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val);
4943 }
4944
4945 static void
4946 ix_msix_tx(void *xtxr)
4947 {
4948 struct ix_tx_ring *txr = xtxr;
4949
4950 ASSERT_SERIALIZED(&txr->tx_serialize);
4951
4952 ix_tx_intr(txr, *(txr->tx_hdr));
4953 IXGBE_WRITE_REG(&txr->tx_sc->hw, txr->tx_eims, txr->tx_eims_val);
4954 }
4955
4956 static void
4957 ix_msix_rxtx(void *xrxr)
4958 {
4959 struct ix_rx_ring *rxr = xrxr;
4960 struct ix_tx_ring *txr;
4961 int hdr;
4962
4963 ASSERT_SERIALIZED(&rxr->rx_serialize);
4964
4965 ix_rxeof(rxr, -1);
4966
4967 /*
4968 * NOTE:
4969 * Since tx_next_clean is only changed by ix_txeof(),
4970 * which is called only in interrupt handler, the
4971 * check w/o holding tx serializer is MPSAFE.
4972 */
4973 txr = rxr->rx_txr;
4974 hdr = *(txr->tx_hdr);
4975 if (hdr != txr->tx_next_clean) {
4976 lwkt_serialize_enter(&txr->tx_serialize);
4977 ix_tx_intr(txr, hdr);
4978 lwkt_serialize_exit(&txr->tx_serialize);
4979 }
4980
4981 IXGBE_WRITE_REG(&rxr->rx_sc->hw, rxr->rx_eims, rxr->rx_eims_val);
4982 }
4983
4984 static void
4985 ix_intr_status(struct ix_softc *sc, uint32_t eicr)
4986 {
4987 struct ixgbe_hw *hw = &sc->hw;
4988
4989 /* Link status change */
4990 if (eicr & IXGBE_EICR_LSC)
4991 ix_handle_link(sc);
4992
4993 if (hw->mac.type != ixgbe_mac_82598EB) {
4994 if (eicr & IXGBE_EICR_ECC)
4995 if_printf(&sc->arpcom.ac_if, "ECC ERROR!! REBOOT!!\n");
4996
4997 /* Check for over temp condition */
4998 if (sc->caps & IX_CAP_TEMP_SENSOR) {
4999 int32_t retval;
5000
5001 switch (sc->hw.mac.type) {
5002 case ixgbe_mac_X550EM_a:
5003 if ((eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) == 0)
5004 break;
5005 retval = hw->phy.ops.check_overtemp(hw);
5006 if (retval != IXGBE_ERR_OVERTEMP)
5007 break;
5008
5009 /* Disable more temp sensor interrupts. */
5010 IXGBE_WRITE_REG(hw, IXGBE_EIMC,
5011 IXGBE_EICR_GPI_SDP0_X550EM_a);
5012 if_printf(&sc->arpcom.ac_if, "CRITICAL: "
5013 "OVER TEMP!! PHY IS SHUT DOWN!! "
5014 "SHUTDOWN!!\n");
5015 break;
5016
5017 default:
5018 if ((eicr & IXGBE_EICR_TS) == 0)
5019 break;
5020 retval = hw->phy.ops.check_overtemp(hw);
5021 if (retval != IXGBE_ERR_OVERTEMP)
5022 break;
5023
5024 /* Disable more temp sensor interrupts. */
5025 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_TS);
5026 if_printf(&sc->arpcom.ac_if, "CRITICAL: "
5027 "OVER TEMP!! PHY IS SHUT DOWN!! "
5028 "SHUTDOWN!!\n");
5029 break;
5030 }
5031 }
5032 }
5033
5034 if (ix_is_sfp(hw)) {
5035 uint32_t eicr_mask;
5036
5037 /* Pluggable optics-related interrupt */
5038 if (hw->mac.type >= ixgbe_mac_X540)
5039 eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
5040 else
5041 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
5042
5043 if (eicr & eicr_mask)
5044 ix_handle_mod(sc);
5045
5046 if (hw->mac.type == ixgbe_mac_82599EB &&
5047 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)))
5048 ix_handle_msf(sc);
5049 }
5050
5051 /* Check for fan failure */
5052 if (sc->caps & IX_CAP_DETECT_FANFAIL)
5053 ix_detect_fanfail(sc, eicr, TRUE);
5054
5055 /* External PHY interrupt */
5056 if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
5057 (eicr & IXGBE_EICR_GPI_SDP0_X540))
5058 ix_handle_phy(sc);
5059 }
5060
5061 static void
5062 ix_msix_status(void *xsc)
5063 {
5064 struct ix_softc *sc = xsc;
5065 uint32_t eicr;
5066
5067 ASSERT_SERIALIZED(&sc->main_serialize);
5068
5069 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR);
5070 ix_intr_status(sc, eicr);
5071
5072 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, sc->intr_mask);
5073 }
5074
5075 static void
5076 ix_setup_msix_eims(const struct ix_softc *sc, int x,
5077 uint32_t *eims, uint32_t *eims_val)
5078 {
5079 if (x < 32) {
5080 if (sc->hw.mac.type == ixgbe_mac_82598EB) {
5081 KASSERT(x < IX_MAX_MSIX_82598,
5082 ("%s: invalid vector %d for 82598",
5083 device_get_nameunit(sc->dev), x));
5084 *eims = IXGBE_EIMS;
5085 } else {
5086 *eims = IXGBE_EIMS_EX(0);
5087 }
5088 *eims_val = 1 << x;
5089 } else {
5090 KASSERT(x < IX_MAX_MSIX, ("%s: invalid vector %d",
5091 device_get_nameunit(sc->dev), x));
5092 KASSERT(sc->hw.mac.type != ixgbe_mac_82598EB,
5093 ("%s: invalid vector %d for 82598",
5094 device_get_nameunit(sc->dev), x));
5095 *eims = IXGBE_EIMS_EX(1);
5096 *eims_val = 1 << (x - 32);
5097 }
5098 }
5099
5100 #ifdef IFPOLL_ENABLE
5101
5102 static void
5103 ix_npoll_status(struct ifnet *ifp)
5104 {
5105 struct ix_softc *sc = ifp->if_softc;
5106 uint32_t eicr;
5107
5108 ASSERT_SERIALIZED(&sc->main_serialize);
5109
5110 eicr = IXGBE_READ_REG(&sc->hw, IXGBE_EICR);
5111 ix_intr_status(sc, eicr);
5112 }
5113
5114 static void
5115 ix_npoll_tx(struct ifnet *ifp, void *arg, int cycle __unused)
5116 {
5117 struct ix_tx_ring *txr = arg;
5118
5119 ASSERT_SERIALIZED(&txr->tx_serialize);
5120
5121 ix_tx_intr(txr, *(txr->tx_hdr));
5122 ix_try_txgc(txr, 1);
5123 }
5124
5125 static void
5126 ix_npoll_rx(struct ifnet *ifp __unused, void *arg, int cycle)
5127 {
5128 struct ix_rx_ring *rxr = arg;
5129
5130 ASSERT_SERIALIZED(&rxr->rx_serialize);
5131 ix_rxeof(rxr, cycle);
5132 }
5133
5134 static void
5135 ix_npoll_rx_direct(struct ifnet *ifp __unused, void *arg, int cycle)
5136 {
5137 struct ix_rx_ring *rxr = arg;
5138
5139 ASSERT_NOT_SERIALIZED(&rxr->rx_serialize);
5140 ix_rxeof(rxr, cycle);
5141 }
5142
5143 static void
5144 ix_npoll(struct ifnet *ifp, struct ifpoll_info *info)
5145 {
5146 struct ix_softc *sc = ifp->if_softc;
5147 int i, txr_cnt, rxr_cnt, idirect;
5148
5149 ASSERT_IFNET_SERIALIZED_ALL(ifp);
5150
5151 idirect = sc->direct_input;
5152 cpu_ccfence();
5153
5154 if (info) {
5155 int cpu;
5156
5157 info->ifpi_status.status_func = ix_npoll_status;
5158 info->ifpi_status.serializer = &sc->main_serialize;
5159
5160 txr_cnt = ix_get_txring_inuse(sc, TRUE);
5161 for (i = 0; i < txr_cnt; ++i) {
5162 struct ix_tx_ring *txr = &sc->tx_rings[i];
5163
5164 cpu = if_ringmap_cpumap(sc->tx_rmap, i);
5165 KKASSERT(cpu < netisr_ncpus);
5166 info->ifpi_tx[cpu].poll_func = ix_npoll_tx;
5167 info->ifpi_tx[cpu].arg = txr;
5168 info->ifpi_tx[cpu].serializer = &txr->tx_serialize;
5169 ifsq_set_cpuid(txr->tx_ifsq, cpu);
5170 }
5171
5172 rxr_cnt = ix_get_rxring_inuse(sc, TRUE);
5173 for (i = 0; i < rxr_cnt; ++i) {
5174 struct ix_rx_ring *rxr = &sc->rx_rings[i];
5175
5176 cpu = if_ringmap_cpumap(sc->rx_rmap, i);
5177 KKASSERT(cpu < netisr_ncpus);
5178 info->ifpi_rx[cpu].arg = rxr;
5179 if (idirect) {
5180 info->ifpi_rx[cpu].poll_func =
5181 ix_npoll_rx_direct;
5182 info->ifpi_rx[cpu].serializer = NULL;
5183 } else {
5184 info->ifpi_rx[cpu].poll_func = ix_npoll_rx;
5185 info->ifpi_rx[cpu].serializer =
5186 &rxr->rx_serialize;
5187 }
5188 }
5189 if (idirect)
5190 ifp->if_flags |= IFF_IDIRECT;
5191 } else {
5192 ifp->if_flags &= ~IFF_IDIRECT;
5193 for (i = 0; i < sc->tx_ring_cnt; ++i) {
5194 struct ix_tx_ring *txr = &sc->tx_rings[i];
5195
5196 ifsq_set_cpuid(txr->tx_ifsq, txr->tx_intr_cpuid);
5197 }
5198 }
5199 if (ifp->if_flags & IFF_RUNNING)
5200 ix_init(sc);
5201 }
5202
5203 #endif /* IFPOLL_ENABLE */
5204
5205 static enum ixgbe_fc_mode
5206 ix_ifmedia2fc(int ifm)
5207 {
5208 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
5209
5210 switch (fc_opt) {
5211 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE):
5212 return ixgbe_fc_full;
5213
5214 case IFM_ETH_RXPAUSE:
5215 return ixgbe_fc_rx_pause;
5216
5217 case IFM_ETH_TXPAUSE:
5218 return ixgbe_fc_tx_pause;
5219
5220 default:
5221 return ixgbe_fc_none;
5222 }
5223 }
5224
5225 static const char *
5226 ix_ifmedia2str(int ifm)
5227 {
5228 int fc_opt = ifm & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
5229
5230 switch (fc_opt) {
5231 case (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE):
5232 return IFM_ETH_FC_FULL;
5233
5234 case IFM_ETH_RXPAUSE:
5235 return IFM_ETH_FC_RXPAUSE;
5236
5237 case IFM_ETH_TXPAUSE:
5238 return IFM_ETH_FC_TXPAUSE;
5239
5240 default:
5241 return IFM_ETH_FC_NONE;
5242 }
5243 }
5244
5245 static const char *
5246 ix_fc2str(enum ixgbe_fc_mode fc)
5247 {
5248 switch (fc) {
5249 case ixgbe_fc_full:
5250 return IFM_ETH_FC_FULL;
5251
5252 case ixgbe_fc_rx_pause:
5253 return IFM_ETH_FC_RXPAUSE;
5254
5255 case ixgbe_fc_tx_pause:
5256 return IFM_ETH_FC_TXPAUSE;
5257
5258 default:
5259 return IFM_ETH_FC_NONE;
5260 }
5261 }
5262
5263 static int
5264 ix_powerdown(struct ix_softc *sc)
5265 {
5266 struct ixgbe_hw *hw = &sc->hw;
5267 int error = 0;
5268
5269 /* Limit power management flow to X550EM baseT */
5270 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
5271 hw->phy.ops.enter_lplu) {
5272 /* Turn off support for APM wakeup. (Using ACPI instead) */
5273 IXGBE_WRITE_REG(hw, IXGBE_GRC,
5274 IXGBE_READ_REG(hw, IXGBE_GRC) & ~(uint32_t)2);
5275
5276 /*
5277 * Clear Wake Up Status register to prevent any previous wakeup
5278 * events from waking us up immediately after we suspend.
5279 */
5280 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
5281
5282 /*
5283 * Program the Wakeup Filter Control register with user filter
5284 * settings
5285 */
5286 IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc);
5287
5288 /* Enable wakeups and power management in Wakeup Control */
5289 IXGBE_WRITE_REG(hw, IXGBE_WUC,
5290 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN);
5291
5292 /* X550EM baseT adapters need a special LPLU flow */
5293 hw->phy.reset_disable = true;
5294 ix_stop(sc);
5295 error = hw->phy.ops.enter_lplu(hw);
5296 if (error) {
5297 if_printf(&sc->arpcom.ac_if,
5298 "Error entering LPLU: %d\n", error);
5299 }
5300 hw->phy.reset_disable = false;
5301 } else {
5302 /* Just stop for other adapters */
5303 ix_stop(sc);
5304 }
5305 return error;
5306 }
5307
5308 static void
5309 ix_config_flowctrl(struct ix_softc *sc)
5310 {
5311 struct ixgbe_hw *hw = &sc->hw;
5312 uint32_t rxpb, frame, size, tmp;
5313
5314 frame = sc->max_frame_size;
5315
5316 /* Calculate High Water */
5317 switch (hw->mac.type) {
5318 case ixgbe_mac_X540:
5319 case ixgbe_mac_X550:
5320 case ixgbe_mac_X550EM_a:
5321 case ixgbe_mac_X550EM_x:
5322 tmp = IXGBE_DV_X540(frame, frame);
5323 break;
5324 default:
5325 tmp = IXGBE_DV(frame, frame);
5326 break;
5327 }
5328 size = IXGBE_BT2KB(tmp);
5329 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
5330 hw->fc.high_water[0] = rxpb - size;
5331
5332 /* Now calculate Low Water */
5333 switch (hw->mac.type) {
5334 case ixgbe_mac_X540:
5335 case ixgbe_mac_X550:
5336 case ixgbe_mac_X550EM_a:
5337 case ixgbe_mac_X550EM_x:
5338 tmp = IXGBE_LOW_DV_X540(frame);
5339 break;
5340 default:
5341 tmp = IXGBE_LOW_DV(frame);
5342 break;
5343 }
5344 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
5345
5346 hw->fc.requested_mode = ix_ifmedia2fc(sc->ifm_media);
5347 if (sc->ifm_media & IFM_ETH_FORCEPAUSE)
5348 hw->fc.disable_fc_autoneg = TRUE;
5349 else
5350 hw->fc.disable_fc_autoneg = FALSE;
5351 hw->fc.pause_time = IX_FC_PAUSE;
5352 hw->fc.send_xon = TRUE;
5353 }
5354
5355 static void
5356 ix_config_dmac(struct ix_softc *sc)
5357 {
5358 struct ixgbe_hw *hw = &sc->hw;
5359 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;
5360
5361 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)
5362 return;
5363
5364 if ((dcfg->watchdog_timer ^ sc->dmac) ||
5365 (dcfg->link_speed ^ sc->link_speed)) {
5366 dcfg->watchdog_timer = sc->dmac;
5367 dcfg->fcoe_en = false;
5368 dcfg->link_speed = sc->link_speed;
5369 dcfg->num_tcs = 1;
5370
5371 if (bootverbose) {
5372 if_printf(&sc->arpcom.ac_if, "dmac settings: "
5373 "watchdog %d, link speed %d\n",
5374 dcfg->watchdog_timer, dcfg->link_speed);
5375 }
5376
5377 hw->mac.ops.dmac_config(hw);
5378 }
5379 }
5380
5381 static void
5382 ix_init_media(struct ix_softc *sc)
5383 {
5384 struct ixgbe_hw *hw = &sc->hw;
5385 uint32_t layer;
5386
5387 ifmedia_removeall(&sc->media);
5388
5389 layer = sc->phy_layer;
5390
5391 /*
5392 * Media types with matching DragonFlyBSD media defines
5393 */
5394 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
5395 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX,
5396 0, NULL);
5397 }
5398 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
5399 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
5400 0, NULL);
5401 }
5402 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) {
5403 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
5404 0, NULL);
5405 /* No half-duplex support */
5406 }
5407 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) {
5408 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX,
5409 0, NULL);
5410 /* No half-duplex support */
5411 }
5412
5413 if ((layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) ||
5414 (layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)) {
5415 ifmedia_add_nodup(&sc->media,
5416 IFM_ETHER | IFM_10G_TWINAX | IFM_FDX, 0, NULL);
5417 }
5418
5419 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
5420 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LR | IFM_FDX,
5421 0, NULL);
5422 if (hw->phy.multispeed_fiber) {
5423 ifmedia_add_nodup(&sc->media,
5424 IFM_ETHER | IFM_1000_LX | IFM_FDX, 0, NULL);
5425 }
5426 }
5427 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) {
5428 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_LRM | IFM_FDX,
5429 0, NULL);
5430 if (hw->phy.multispeed_fiber) {
5431 ifmedia_add_nodup(&sc->media,
5432 IFM_ETHER | IFM_1000_LX | IFM_FDX, 0, NULL);
5433 }
5434 }
5435
5436 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
5437 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX,
5438 0, NULL);
5439 if (hw->phy.multispeed_fiber) {
5440 ifmedia_add_nodup(&sc->media,
5441 IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL);
5442 }
5443 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
5444 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
5445 0, NULL);
5446 }
5447
5448 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) {
5449 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX,
5450 0, NULL);
5451 }
5452
5453 /*
5454 * XXX Other (no matching DragonFlyBSD media type):
5455 * To workaround this, we'll assign these completely
5456 * inappropriate media types.
5457 */
5458 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
5459 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKR\n");
5460 if_printf(&sc->arpcom.ac_if, "10GbaseKR mapped to 10GbaseSR\n");
5461 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_SR | IFM_FDX,
5462 0, NULL);
5463 }
5464 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
5465 if_printf(&sc->arpcom.ac_if, "Media supported: 10GbaseKX4\n");
5466 if_printf(&sc->arpcom.ac_if,
5467 "10GbaseKX4 mapped to 10GbaseCX4\n");
5468 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_10G_CX4 | IFM_FDX,
5469 0, NULL);
5470 }
5471 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
5472 if_printf(&sc->arpcom.ac_if, "Media supported: 1000baseKX\n");
5473 if_printf(&sc->arpcom.ac_if,
5474 "1000baseKX mapped to 1000baseCX\n");
5475 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_1000_CX | IFM_FDX,
5476 0, NULL);
5477 }
5478 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) {
5479 if_printf(&sc->arpcom.ac_if, "Media supported: 2500baseKX\n");
5480 if_printf(&sc->arpcom.ac_if,
5481 "2500baseKX mapped to 2500baseSX\n");
5482 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_2500_SX | IFM_FDX,
5483 0, NULL);
5484 }
5485 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) {
5486 if_printf(&sc->arpcom.ac_if,
5487 "Media supported: 1000baseBX, ignored\n");
5488 }
5489
5490 /* XXX we probably don't need this */
5491 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
5492 ifmedia_add_nodup(&sc->media,
5493 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
5494 }
5495
5496 ifmedia_add_nodup(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
5497
5498 if (ifmedia_tryset(&sc->media, sc->ifm_media)) {
5499 int flowctrl = (sc->ifm_media & IFM_ETH_FCMASK);
5500
5501 sc->advspeed = IXGBE_LINK_SPEED_UNKNOWN;
5502 sc->ifm_media = IX_IFM_DEFAULT | flowctrl;
5503 ifmedia_set(&sc->media, sc->ifm_media);
5504 }
5505 }
5506
5507 static void
5508 ix_setup_caps(struct ix_softc *sc)
5509 {
5510
5511 sc->caps |= IX_CAP_LEGACY_INTR;
5512
5513 switch (sc->hw.mac.type) {
5514 case ixgbe_mac_82598EB:
5515 if (sc->hw.device_id == IXGBE_DEV_ID_82598AT)
5516 sc->caps |= IX_CAP_DETECT_FANFAIL;
5517 break;
5518
5519 case ixgbe_mac_X550:
5520 sc->caps |= IX_CAP_TEMP_SENSOR | IX_CAP_FW_RECOVERY;
5521 break;
5522
5523 case ixgbe_mac_X550EM_x:
5524 if (sc->hw.device_id == IXGBE_DEV_ID_X550EM_X_KR)
5525 sc->caps |= IX_CAP_EEE;
5526 sc->caps |= IX_CAP_FW_RECOVERY;
5527 break;
5528
5529 case ixgbe_mac_X550EM_a:
5530 sc->caps &= ~IX_CAP_LEGACY_INTR;
5531 if (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T ||
5532 sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)
5533 sc->caps |= IX_CAP_TEMP_SENSOR | IX_CAP_EEE;
5534 sc->caps |= IX_CAP_FW_RECOVERY;
5535 break;
5536
5537 case ixgbe_mac_82599EB:
5538 if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)
5539 sc->caps &= ~IX_CAP_LEGACY_INTR;
5540 break;
5541
5542 default:
5543 break;
5544 }
5545 }
5546
5547 static void
5548 ix_detect_fanfail(struct ix_softc *sc, uint32_t reg, boolean_t intr)
5549 {
5550 uint32_t mask;
5551
5552 mask = intr ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) : IXGBE_ESDP_SDP1;
5553 if (reg & mask) {
5554 if_printf(&sc->arpcom.ac_if,
5555 "CRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
5556 }
5557 }
5558
5559 static void
5560 ix_config_gpie(struct ix_softc *sc)
5561 {
5562 struct ixgbe_hw *hw = &sc->hw;
5563 uint32_t gpie;
5564
5565 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
5566
5567 if (sc->intr_type == PCI_INTR_TYPE_MSIX) {
5568 /* Enable Enhanced MSI-X mode */
5569 gpie |= IXGBE_GPIE_MSIX_MODE |
5570 IXGBE_GPIE_EIAME |
5571 IXGBE_GPIE_PBA_SUPPORT |
5572 IXGBE_GPIE_OCD;
5573 }
5574
5575 /* Fan Failure Interrupt */
5576 if (sc->caps & IX_CAP_DETECT_FANFAIL)
5577 gpie |= IXGBE_SDP1_GPIEN;
5578
5579 /* Thermal Sensor Interrupt */
5580 if (sc->caps & IX_CAP_TEMP_SENSOR)
5581 gpie |= IXGBE_SDP0_GPIEN_X540;
5582
5583 /* Link detection */
5584 switch (hw->mac.type) {
5585 case ixgbe_mac_82599EB:
5586 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN;
5587 break;
5588
5589 case ixgbe_mac_X550EM_x:
5590 case ixgbe_mac_X550EM_a:
5591 gpie |= IXGBE_SDP0_GPIEN_X540;
5592 break;
5593
5594 default:
5595 break;
5596 }
5597
5598 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
5599 }
5600
5601 static void
5602 ix_fw_timer(void *xsc)
5603 {
5604 struct ix_softc *sc = xsc;
5605 struct ifnet *ifp = &sc->arpcom.ac_if;
5606
5607 lwkt_serialize_enter(&sc->main_serialize);
5608
5609 if (ixgbe_fw_recovery_mode(&sc->hw)) {
5610 if ((sc->flags & IX_FLAG_FW_RECOVERY) == 0) {
5611 sc->flags |= IX_FLAG_FW_RECOVERY;
5612 if (ifp->if_flags & IFF_RUNNING) {
5613 if_printf(ifp,
5614 "fw recovery mode entered, stop\n");
5615 ix_serialize_skipmain(sc);
5616 ix_stop(sc);
5617 ix_deserialize_skipmain(sc);
5618 } else {
5619 if_printf(ifp, "fw recovery mode entered\n");
5620 }
5621 }
5622 } else {
5623 if (sc->flags & IX_FLAG_FW_RECOVERY) {
5624 sc->flags &= ~IX_FLAG_FW_RECOVERY;
5625 if (ifp->if_flags & IFF_UP) {
5626 if_printf(ifp,
5627 "fw recovery mode exited, reinit\n");
5628 ix_serialize_skipmain(sc);
5629 ix_init(sc);
5630 ix_deserialize_skipmain(sc);
5631 } else {
5632 if_printf(ifp, "fw recovery mode exited\n");
5633 }
5634 }
5635 }
5636
5637 callout_reset_bycpu(&sc->fw_timer, hz, ix_fw_timer, sc,
5638 ix_get_timer_cpuid(sc,
5639 (ifp->if_flags & IFF_NPOLLING) ? TRUE : FALSE));
5640
5641 lwkt_serialize_exit(&sc->main_serialize);
5642 }
DragonFly BSD