2 * Copyright (c) 2001-2013, Intel Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the Intel Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
32 #include "opt_ifpoll.h"
35 #include <sys/param.h>
37 #include <sys/endian.h>
38 #include <sys/interrupt.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
44 #include <sys/serialize.h>
45 #include <sys/serialize2.h>
46 #include <sys/socket.h>
47 #include <sys/sockio.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
52 #include <net/ethernet.h>
54 #include <net/if_arp.h>
55 #include <net/if_dl.h>
56 #include <net/if_media.h>
57 #include <net/ifq_var.h>
58 #include <net/toeplitz.h>
59 #include <net/toeplitz2.h>
60 #include <net/vlan/if_vlan_var.h>
61 #include <net/vlan/if_vlan_ether.h>
62 #include <net/if_poll.h>
64 #include <netinet/in_systm.h>
65 #include <netinet/in.h>
66 #include <netinet/ip.h>
68 #include <bus/pci/pcivar.h>
69 #include <bus/pci/pcireg.h>
71 #include <dev/netif/ix/ixgbe_api.h>
72 #include <dev/netif/ix/if_ix.h>
75 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) \
77 if (sc->rss_debug >= lvl) \
78 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
80 #else /* !IX_RSS_DEBUG */
81 #define IX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0)
82 #endif /* IX_RSS_DEBUG */
84 #define IX_NAME "Intel(R) PRO/10GbE "
85 #define IX_DEVICE(id) \
86 { IXGBE_VENDOR_ID, IXGBE_DEV_ID_##id, IX_NAME #id }
87 #define IX_DEVICE_NULL { 0, 0, NULL }
89 static struct ix_device
{
94 IX_DEVICE(82598AF_DUAL_PORT
),
95 IX_DEVICE(82598AF_SINGLE_PORT
),
96 IX_DEVICE(82598EB_CX4
),
100 IX_DEVICE(82598_DA_DUAL_PORT
),
101 IX_DEVICE(82598_CX4_DUAL_PORT
),
102 IX_DEVICE(82598EB_XF_LR
),
103 IX_DEVICE(82598_SR_DUAL_PORT_EM
),
104 IX_DEVICE(82598EB_SFP_LOM
),
105 IX_DEVICE(82599_KX4
),
106 IX_DEVICE(82599_KX4_MEZZ
),
107 IX_DEVICE(82599_SFP
),
108 IX_DEVICE(82599_XAUI_LOM
),
109 IX_DEVICE(82599_CX4
),
110 IX_DEVICE(82599_T3_LOM
),
111 IX_DEVICE(82599_COMBO_BACKPLANE
),
112 IX_DEVICE(82599_BACKPLANE_FCOE
),
113 IX_DEVICE(82599_SFP_SF2
),
114 IX_DEVICE(82599_SFP_FCOE
),
115 IX_DEVICE(82599EN_SFP
),
116 IX_DEVICE(82599_SFP_SF_QP
),
119 /* required last entry */
123 static int ix_probe(device_t
);
124 static int ix_attach(device_t
);
125 static int ix_detach(device_t
);
126 static int ix_shutdown(device_t
);
128 static void ix_serialize(struct ifnet
*, enum ifnet_serialize
);
129 static void ix_deserialize(struct ifnet
*, enum ifnet_serialize
);
130 static int ix_tryserialize(struct ifnet
*, enum ifnet_serialize
);
132 static void ix_serialize_assert(struct ifnet
*, enum ifnet_serialize
,
135 static void ix_start(struct ifnet
*, struct ifaltq_subque
*);
136 static void ix_watchdog(struct ifaltq_subque
*);
137 static int ix_ioctl(struct ifnet
*, u_long
, caddr_t
, struct ucred
*);
138 static void ix_init(void *);
139 static void ix_stop(struct ix_softc
*);
140 static void ix_media_status(struct ifnet
*, struct ifmediareq
*);
141 static int ix_media_change(struct ifnet
*);
142 static void ix_timer(void *);
144 static void ix_npoll(struct ifnet
*, struct ifpoll_info
*);
145 static void ix_npoll_rx(struct ifnet
*, void *, int);
146 static void ix_npoll_tx(struct ifnet
*, void *, int);
147 static void ix_npoll_status(struct ifnet
*);
150 static void ix_add_sysctl(struct ix_softc
*);
151 static void ix_add_intr_rate_sysctl(struct ix_softc
*, int,
152 const char *, int (*)(SYSCTL_HANDLER_ARGS
), const char *);
153 static int ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS
);
154 static int ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS
);
155 static int ix_sysctl_txd(SYSCTL_HANDLER_ARGS
);
156 static int ix_sysctl_rxd(SYSCTL_HANDLER_ARGS
);
157 static int ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS
);
158 static int ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS
, int);
159 static int ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS
);
160 static int ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS
);
161 static int ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS
);
162 static int ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS
);
163 static int ix_sysctl_flowctrl(SYSCTL_HANDLER_ARGS
);
165 static int ix_sysctl_advspeed(SYSCTL_HANDLER_ARGS
);
168 static void ix_add_hw_stats(struct ix_softc
*);
171 static int ix_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS
);
172 static int ix_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS
);
175 static void ix_slot_info(struct ix_softc
*);
176 static int ix_alloc_rings(struct ix_softc
*);
177 static void ix_free_rings(struct ix_softc
*);
178 static void ix_setup_ifp(struct ix_softc
*);
179 static void ix_setup_serialize(struct ix_softc
*);
180 static void ix_set_ring_inuse(struct ix_softc
*, boolean_t
);
181 static void ix_set_timer_cpuid(struct ix_softc
*, boolean_t
);
182 static void ix_update_stats(struct ix_softc
*);
184 static void ix_set_promisc(struct ix_softc
*);
185 static void ix_set_multi(struct ix_softc
*);
186 static void ix_set_vlan(struct ix_softc
*);
187 static uint8_t *ix_mc_array_itr(struct ixgbe_hw
*, uint8_t **, uint32_t *);
189 static int ix_get_txring_inuse(const struct ix_softc
*, boolean_t
);
190 static void ix_init_tx_ring(struct ix_tx_ring
*);
191 static void ix_free_tx_ring(struct ix_tx_ring
*);
192 static int ix_create_tx_ring(struct ix_tx_ring
*);
193 static void ix_destroy_tx_ring(struct ix_tx_ring
*, int);
194 static void ix_init_tx_unit(struct ix_softc
*);
195 static int ix_encap(struct ix_tx_ring
*, struct mbuf
**,
197 static int ix_tx_ctx_setup(struct ix_tx_ring
*,
198 const struct mbuf
*, uint32_t *, uint32_t *);
199 static int ix_tso_ctx_setup(struct ix_tx_ring
*,
200 const struct mbuf
*, uint32_t *, uint32_t *);
201 static void ix_txeof(struct ix_tx_ring
*, int);
203 static int ix_get_rxring_inuse(const struct ix_softc
*, boolean_t
);
204 static int ix_init_rx_ring(struct ix_rx_ring
*);
205 static void ix_free_rx_ring(struct ix_rx_ring
*);
206 static int ix_create_rx_ring(struct ix_rx_ring
*);
207 static void ix_destroy_rx_ring(struct ix_rx_ring
*, int);
208 static void ix_init_rx_unit(struct ix_softc
*);
210 static void ix_setup_hw_rsc(struct ix_rx_ring
*);
212 static int ix_newbuf(struct ix_rx_ring
*, int, boolean_t
);
213 static void ix_rxeof(struct ix_rx_ring
*, int);
214 static void ix_rx_discard(struct ix_rx_ring
*, int, boolean_t
);
215 static void ix_enable_rx_drop(struct ix_softc
*);
216 static void ix_disable_rx_drop(struct ix_softc
*);
218 static void ix_alloc_msix(struct ix_softc
*);
219 static void ix_free_msix(struct ix_softc
*, boolean_t
);
220 static void ix_conf_rx_msix(struct ix_softc
*, int, int *, int);
221 static void ix_conf_tx_msix(struct ix_softc
*, int, int *, int);
222 static void ix_setup_msix_eims(const struct ix_softc
*, int,
223 uint32_t *, uint32_t *);
224 static int ix_alloc_intr(struct ix_softc
*);
225 static void ix_free_intr(struct ix_softc
*);
226 static int ix_setup_intr(struct ix_softc
*);
227 static void ix_teardown_intr(struct ix_softc
*, int);
228 static void ix_enable_intr(struct ix_softc
*);
229 static void ix_disable_intr(struct ix_softc
*);
230 static void ix_set_ivar(struct ix_softc
*, uint8_t, uint8_t, int8_t);
231 static void ix_set_eitr(struct ix_softc
*, int, int);
232 static void ix_intr_status(struct ix_softc
*, uint32_t);
233 static void ix_intr(void *);
234 static void ix_msix_rxtx(void *);
235 static void ix_msix_rx(void *);
236 static void ix_msix_tx(void *);
237 static void ix_msix_status(void *);
239 static void ix_config_link(struct ix_softc
*);
240 static boolean_t
ix_sfp_probe(struct ix_softc
*);
241 static boolean_t
ix_is_sfp(const struct ixgbe_hw
*);
242 static void ix_setup_optics(struct ix_softc
*);
243 static void ix_update_link_status(struct ix_softc
*);
244 static void ix_handle_link(struct ix_softc
*);
245 static void ix_handle_mod(struct ix_softc
*);
246 static void ix_handle_msf(struct ix_softc
*);
248 /* XXX Shared code structure requires this for the moment */
249 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw
*);
251 static device_method_t ix_methods
[] = {
252 /* Device interface */
253 DEVMETHOD(device_probe
, ix_probe
),
254 DEVMETHOD(device_attach
, ix_attach
),
255 DEVMETHOD(device_detach
, ix_detach
),
256 DEVMETHOD(device_shutdown
, ix_shutdown
),
260 static driver_t ix_driver
= {
263 sizeof(struct ix_softc
)
266 static devclass_t ix_devclass
;
268 DECLARE_DUMMY_MODULE(if_ix
);
269 DRIVER_MODULE(if_ix
, pci
, ix_driver
, ix_devclass
, NULL
, NULL
);
271 static int ix_msi_enable
= 1;
272 static int ix_msix_enable
= 1;
273 static int ix_msix_agg_rxtx
= 1;
274 static int ix_rxr
= 0;
275 static int ix_txr
= 0;
276 static int ix_txd
= IX_PERF_TXD
;
277 static int ix_rxd
= IX_PERF_RXD
;
278 static int ix_unsupported_sfp
= 0;
280 TUNABLE_INT("hw.ix.msi.enable", &ix_msi_enable
);
281 TUNABLE_INT("hw.ix.msix.enable", &ix_msix_enable
);
282 TUNABLE_INT("hw.ix.msix.agg_rxtx", &ix_msix_agg_rxtx
);
283 TUNABLE_INT("hw.ix.rxr", &ix_rxr
);
284 TUNABLE_INT("hw.ix.txr", &ix_txr
);
285 TUNABLE_INT("hw.ix.txd", &ix_txd
);
286 TUNABLE_INT("hw.ix.rxd", &ix_rxd
);
287 TUNABLE_INT("hw.ix.unsupported_sfp", &ix_unsupported_sfp
);
290 * Smart speed setting, default to on. This only works
291 * as a compile option right now as its during attach,
292 * set this to 'ixgbe_smart_speed_off' to disable.
294 static const enum ixgbe_smart_speed ix_smart_speed
=
295 ixgbe_smart_speed_on
;
298 ix_probe(device_t dev
)
300 const struct ix_device
*d
;
303 vid
= pci_get_vendor(dev
);
304 did
= pci_get_device(dev
);
306 for (d
= ix_devices
; d
->desc
!= NULL
; ++d
) {
307 if (vid
== d
->vid
&& did
== d
->did
) {
308 device_set_desc(dev
, d
->desc
);
316 ix_attach(device_t dev
)
318 struct ix_softc
*sc
= device_get_softc(dev
);
320 int error
, ring_cnt_max
;
324 int offset
, offset_def
;
327 sc
->dev
= sc
->osdep
.dev
= dev
;
330 if_initname(&sc
->arpcom
.ac_if
, device_get_name(dev
),
331 device_get_unit(dev
));
332 ifmedia_init(&sc
->media
, IFM_IMASK
,
333 ix_media_change
, ix_media_status
);
335 /* Save frame size */
336 sc
->max_frame_size
= ETHERMTU
+ ETHER_HDR_LEN
+ ETHER_CRC_LEN
;
338 callout_init_mp(&sc
->timer
);
339 lwkt_serialize_init(&sc
->main_serialize
);
342 * Save off the information about this board
344 hw
->vendor_id
= pci_get_vendor(dev
);
345 hw
->device_id
= pci_get_device(dev
);
346 hw
->revision_id
= pci_read_config(dev
, PCIR_REVID
, 1);
347 hw
->subsystem_vendor_id
= pci_read_config(dev
, PCIR_SUBVEND_0
, 2);
348 hw
->subsystem_device_id
= pci_read_config(dev
, PCIR_SUBDEV_0
, 2);
350 ixgbe_set_mac_type(hw
);
352 /* Pick up the 82599 and VF settings */
353 if (hw
->mac
.type
!= ixgbe_mac_82598EB
)
354 hw
->phy
.smart_speed
= ix_smart_speed
;
356 /* Enable bus mastering */
357 pci_enable_busmaster(dev
);
362 sc
->mem_rid
= PCIR_BAR(0);
363 sc
->mem_res
= bus_alloc_resource_any(dev
, SYS_RES_MEMORY
,
364 &sc
->mem_rid
, RF_ACTIVE
);
365 if (sc
->mem_res
== NULL
) {
366 device_printf(dev
, "Unable to allocate bus resource: memory\n");
371 sc
->osdep
.mem_bus_space_tag
= rman_get_bustag(sc
->mem_res
);
372 sc
->osdep
.mem_bus_space_handle
= rman_get_bushandle(sc
->mem_res
);
374 sc
->hw
.hw_addr
= (uint8_t *)&sc
->osdep
.mem_bus_space_handle
;
375 sc
->hw
.back
= &sc
->osdep
;
378 * Configure total supported RX/TX ring count
380 sc
->rx_ring_cnt
= device_getenv_int(dev
, "rxr", ix_rxr
);
381 sc
->rx_ring_cnt
= if_ring_count2(sc
->rx_ring_cnt
, IX_MAX_RXRING
);
382 sc
->rx_ring_inuse
= sc
->rx_ring_cnt
;
384 switch (hw
->mac
.type
) {
385 case ixgbe_mac_82598EB
:
386 ring_cnt_max
= IX_MAX_TXRING_82598
;
389 case ixgbe_mac_82599EB
:
390 ring_cnt_max
= IX_MAX_TXRING_82599
;
394 ring_cnt_max
= IX_MAX_TXRING_X540
;
401 sc
->tx_ring_cnt
= device_getenv_int(dev
, "txr", ix_txr
);
402 sc
->tx_ring_cnt
= if_ring_count2(sc
->tx_ring_cnt
, ring_cnt_max
);
403 sc
->tx_ring_inuse
= sc
->tx_ring_cnt
;
405 /* Allocate TX/RX rings */
406 error
= ix_alloc_rings(sc
);
412 * NPOLLING RX CPU offset
414 if (sc
->rx_ring_cnt
== ncpus2
) {
417 offset_def
= (sc
->rx_ring_cnt
* device_get_unit(dev
)) % ncpus2
;
418 offset
= device_getenv_int(dev
, "npoll.rxoff", offset_def
);
419 if (offset
>= ncpus2
||
420 offset
% sc
->rx_ring_cnt
!= 0) {
421 device_printf(dev
, "invalid npoll.rxoff %d, use %d\n",
426 sc
->rx_npoll_off
= offset
;
429 * NPOLLING TX CPU offset
431 if (sc
->tx_ring_cnt
== ncpus2
) {
434 offset_def
= (sc
->tx_ring_cnt
* device_get_unit(dev
)) % ncpus2
;
435 offset
= device_getenv_int(dev
, "npoll.txoff", offset_def
);
436 if (offset
>= ncpus2
||
437 offset
% sc
->tx_ring_cnt
!= 0) {
438 device_printf(dev
, "invalid npoll.txoff %d, use %d\n",
443 sc
->tx_npoll_off
= offset
;
446 /* Allocate interrupt */
447 error
= ix_alloc_intr(sc
);
451 /* Setup serializes */
452 ix_setup_serialize(sc
);
454 /* Allocate multicast array memory. */
455 sc
->mta
= kmalloc(IXGBE_ETH_LENGTH_OF_ADDRESS
* IX_MAX_MCASTADDR
,
458 /* Initialize the shared code */
459 hw
->allow_unsupported_sfp
= ix_unsupported_sfp
;
460 error
= ixgbe_init_shared_code(hw
);
461 if (error
== IXGBE_ERR_SFP_NOT_PRESENT
) {
463 * No optics in this port; ask timer routine
464 * to probe for later insertion.
466 sc
->sfp_probe
= TRUE
;
468 } else if (error
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
469 device_printf(dev
, "Unsupported SFP+ module detected!\n");
473 device_printf(dev
, "Unable to initialize the shared code\n");
478 /* Make sure we have a good EEPROM before we read from it */
479 if (ixgbe_validate_eeprom_checksum(&sc
->hw
, &csum
) < 0) {
480 device_printf(dev
, "The EEPROM Checksum Is Not Valid\n");
485 error
= ixgbe_init_hw(hw
);
486 if (error
== IXGBE_ERR_EEPROM_VERSION
) {
487 device_printf(dev
, "Pre-production device detected\n");
488 } else if (error
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
489 device_printf(dev
, "Unsupported SFP+ Module\n");
492 } else if (error
== IXGBE_ERR_SFP_NOT_PRESENT
) {
493 device_printf(dev
, "No SFP+ Module found\n");
496 /* Detect and set physical type */
499 /* Setup OS specific network interface */
502 /* Add sysctl tree */
505 error
= ix_setup_intr(sc
);
507 ether_ifdetach(&sc
->arpcom
.ac_if
);
511 /* Initialize statistics */
515 * Check PCIE slot type/speed/width
519 /* Set an initial default flow control value */
520 sc
->fc
= ixgbe_fc_full
;
522 /* Let hardware know driver is loaded */
523 ctrl_ext
= IXGBE_READ_REG(hw
, IXGBE_CTRL_EXT
);
524 ctrl_ext
|= IXGBE_CTRL_EXT_DRV_LOAD
;
525 IXGBE_WRITE_REG(hw
, IXGBE_CTRL_EXT
, ctrl_ext
);
534 ix_detach(device_t dev
)
536 struct ix_softc
*sc
= device_get_softc(dev
);
538 if (device_is_attached(dev
)) {
539 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
542 ifnet_serialize_all(ifp
);
545 ix_teardown_intr(sc
, sc
->intr_cnt
);
547 ifnet_deserialize_all(ifp
);
549 callout_terminate(&sc
->timer
);
552 /* Let hardware know driver is unloading */
553 ctrl_ext
= IXGBE_READ_REG(&sc
->hw
, IXGBE_CTRL_EXT
);
554 ctrl_ext
&= ~IXGBE_CTRL_EXT_DRV_LOAD
;
555 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_CTRL_EXT
, ctrl_ext
);
558 ifmedia_removeall(&sc
->media
);
559 bus_generic_detach(dev
);
563 if (sc
->msix_mem_res
!= NULL
) {
564 bus_release_resource(dev
, SYS_RES_MEMORY
, sc
->msix_mem_rid
,
567 if (sc
->mem_res
!= NULL
) {
568 bus_release_resource(dev
, SYS_RES_MEMORY
, sc
->mem_rid
,
575 kfree(sc
->mta
, M_DEVBUF
);
576 if (sc
->serializes
!= NULL
)
577 kfree(sc
->serializes
, M_DEVBUF
);
583 ix_shutdown(device_t dev
)
585 struct ix_softc
*sc
= device_get_softc(dev
);
586 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
588 ifnet_serialize_all(ifp
);
590 ifnet_deserialize_all(ifp
);
596 ix_start(struct ifnet
*ifp
, struct ifaltq_subque
*ifsq
)
598 struct ix_softc
*sc
= ifp
->if_softc
;
599 struct ix_tx_ring
*txr
= ifsq_get_priv(ifsq
);
603 KKASSERT(txr
->tx_ifsq
== ifsq
);
604 ASSERT_SERIALIZED(&txr
->tx_serialize
);
606 if ((ifp
->if_flags
& IFF_RUNNING
) == 0 || ifsq_is_oactive(ifsq
))
609 if (!sc
->link_active
|| (txr
->tx_flags
& IX_TXFLAG_ENABLED
) == 0) {
614 while (!ifsq_is_empty(ifsq
)) {
617 if (txr
->tx_avail
<= IX_MAX_SCATTER
+ IX_TX_RESERVED
) {
618 ifsq_set_oactive(ifsq
);
619 txr
->tx_watchdog
.wd_timer
= 5;
623 m_head
= ifsq_dequeue(ifsq
);
627 if (ix_encap(txr
, &m_head
, &nsegs
, &idx
)) {
628 IFNET_STAT_INC(ifp
, oerrors
, 1);
633 * TX interrupt are aggressively aggregated, so increasing
634 * opackets at TX interrupt time will make the opackets
635 * statistics vastly inaccurate; we do the opackets increment
638 IFNET_STAT_INC(ifp
, opackets
, 1);
640 if (nsegs
>= txr
->tx_wreg_nsegs
) {
641 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_TDT(txr
->tx_idx
), idx
);
646 ETHER_BPF_MTAP(ifp
, m_head
);
649 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_TDT(txr
->tx_idx
), idx
);
653 ix_ioctl(struct ifnet
*ifp
, u_long command
, caddr_t data
, struct ucred
*cr
)
655 struct ix_softc
*sc
= ifp
->if_softc
;
656 struct ifreq
*ifr
= (struct ifreq
*) data
;
657 int error
= 0, mask
, reinit
;
659 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
663 if (ifr
->ifr_mtu
> IX_MAX_FRAME_SIZE
- ETHER_HDR_LEN
) {
666 ifp
->if_mtu
= ifr
->ifr_mtu
;
668 ifp
->if_mtu
+ ETHER_HDR_LEN
+ ETHER_CRC_LEN
;
674 if (ifp
->if_flags
& IFF_UP
) {
675 if (ifp
->if_flags
& IFF_RUNNING
) {
676 if ((ifp
->if_flags
^ sc
->if_flags
) &
677 (IFF_PROMISC
| IFF_ALLMULTI
))
682 } else if (ifp
->if_flags
& IFF_RUNNING
) {
685 sc
->if_flags
= ifp
->if_flags
;
690 if (ifp
->if_flags
& IFF_RUNNING
) {
694 if ((ifp
->if_flags
& IFF_NPOLLING
) == 0)
702 error
= ifmedia_ioctl(ifp
, ifr
, &sc
->media
, command
);
707 mask
= ifr
->ifr_reqcap
^ ifp
->if_capenable
;
708 if (mask
& IFCAP_RXCSUM
) {
709 ifp
->if_capenable
^= IFCAP_RXCSUM
;
712 if (mask
& IFCAP_VLAN_HWTAGGING
) {
713 ifp
->if_capenable
^= IFCAP_VLAN_HWTAGGING
;
716 if (mask
& IFCAP_TXCSUM
) {
717 ifp
->if_capenable
^= IFCAP_TXCSUM
;
718 if (ifp
->if_capenable
& IFCAP_TXCSUM
)
719 ifp
->if_hwassist
|= CSUM_OFFLOAD
;
721 ifp
->if_hwassist
&= ~CSUM_OFFLOAD
;
723 if (mask
& IFCAP_TSO
) {
724 ifp
->if_capenable
^= IFCAP_TSO
;
725 if (ifp
->if_capenable
& IFCAP_TSO
)
726 ifp
->if_hwassist
|= CSUM_TSO
;
728 ifp
->if_hwassist
&= ~CSUM_TSO
;
730 if (mask
& IFCAP_RSS
)
731 ifp
->if_capenable
^= IFCAP_RSS
;
732 if (reinit
&& (ifp
->if_flags
& IFF_RUNNING
))
739 struct ixgbe_i2c_req i2c
;
740 error
= copyin(ifr
->ifr_data
, &i2c
, sizeof(i2c
));
743 if ((i2c
.dev_addr
!= 0xA0) || (i2c
.dev_addr
!= 0xA2)){
747 hw
->phy
.ops
.read_i2c_byte(hw
, i2c
.offset
,
748 i2c
.dev_addr
, i2c
.data
);
749 error
= copyout(&i2c
, ifr
->ifr_data
, sizeof(i2c
));
755 error
= ether_ioctl(ifp
, command
, data
);
761 #define IXGBE_MHADD_MFS_SHIFT 16
766 struct ix_softc
*sc
= xsc
;
767 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
768 struct ixgbe_hw
*hw
= &sc
->hw
;
769 uint32_t rxpb
, frame
, size
, tmp
;
770 uint32_t gpie
, rxctrl
;
774 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
780 if (ifp
->if_flags
& IFF_NPOLLING
)
784 /* Configure # of used RX/TX rings */
785 ix_set_ring_inuse(sc
, polling
);
786 ifq_set_subq_mask(&ifp
->if_snd
, sc
->tx_ring_inuse
- 1);
788 /* Get the latest mac address, User can use a LAA */
789 bcopy(IF_LLADDR(ifp
), hw
->mac
.addr
, IXGBE_ETH_LENGTH_OF_ADDRESS
);
790 ixgbe_set_rar(hw
, 0, hw
->mac
.addr
, 0, 1);
791 hw
->addr_ctrl
.rar_used_count
= 1;
793 /* Prepare transmit descriptors and buffers */
794 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
)
795 ix_init_tx_ring(&sc
->tx_rings
[i
]);
800 /* Setup Multicast table */
803 /* Prepare receive descriptors and buffers */
804 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
805 error
= ix_init_rx_ring(&sc
->rx_rings
[i
]);
807 if_printf(ifp
, "Could not initialize RX ring%d\n", i
);
813 /* Configure RX settings */
816 gpie
= IXGBE_READ_REG(hw
, IXGBE_GPIE
);
818 /* Enable Fan Failure Interrupt */
819 gpie
|= IXGBE_SDP1_GPIEN
;
821 /* Add for Module detection */
822 if (hw
->mac
.type
== ixgbe_mac_82599EB
)
823 gpie
|= IXGBE_SDP2_GPIEN
;
825 /* Thermal Failure Detection */
826 if (hw
->mac
.type
== ixgbe_mac_X540
)
827 gpie
|= IXGBE_SDP0_GPIEN
;
829 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
) {
830 /* Enable Enhanced MSIX mode */
831 gpie
|= IXGBE_GPIE_MSIX_MODE
;
832 gpie
|= IXGBE_GPIE_EIAME
| IXGBE_GPIE_PBA_SUPPORT
|
835 IXGBE_WRITE_REG(hw
, IXGBE_GPIE
, gpie
);
838 if (ifp
->if_mtu
> ETHERMTU
) {
841 mhadd
= IXGBE_READ_REG(hw
, IXGBE_MHADD
);
842 mhadd
&= ~IXGBE_MHADD_MFS_MASK
;
843 mhadd
|= sc
->max_frame_size
<< IXGBE_MHADD_MFS_SHIFT
;
844 IXGBE_WRITE_REG(hw
, IXGBE_MHADD
, mhadd
);
850 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
853 txdctl
= IXGBE_READ_REG(hw
, IXGBE_TXDCTL(i
));
854 txdctl
|= IXGBE_TXDCTL_ENABLE
;
857 * Set WTHRESH to 0, since TX head write-back is used
859 txdctl
&= ~(0x7f << 16);
862 * When the internal queue falls below PTHRESH (32),
863 * start prefetching as long as there are at least
864 * HTHRESH (1) buffers ready. The values are taken
865 * from the Intel linux driver 3.8.21.
866 * Prefetching enables tx line rate even with 1 queue.
868 txdctl
|= (32 << 0) | (1 << 8);
869 IXGBE_WRITE_REG(hw
, IXGBE_TXDCTL(i
), txdctl
);
875 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
879 rxdctl
= IXGBE_READ_REG(hw
, IXGBE_RXDCTL(i
));
880 if (hw
->mac
.type
== ixgbe_mac_82598EB
) {
889 rxdctl
|= IXGBE_RXDCTL_ENABLE
;
890 IXGBE_WRITE_REG(hw
, IXGBE_RXDCTL(i
), rxdctl
);
891 for (k
= 0; k
< 10; ++k
) {
892 if (IXGBE_READ_REG(hw
, IXGBE_RXDCTL(i
)) &
899 IXGBE_WRITE_REG(hw
, IXGBE_RDT(i
),
900 sc
->rx_rings
[0].rx_ndesc
- 1);
903 /* Set up VLAN support and filter */
906 /* Enable Receive engine */
907 rxctrl
= IXGBE_READ_REG(hw
, IXGBE_RXCTRL
);
908 if (hw
->mac
.type
== ixgbe_mac_82598EB
)
909 rxctrl
|= IXGBE_RXCTRL_DMBYPS
;
910 rxctrl
|= IXGBE_RXCTRL_RXEN
;
911 ixgbe_enable_rx_dma(hw
, rxctrl
);
913 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
914 const struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
916 if (txr
->tx_intr_vec
>= 0) {
917 ix_set_ivar(sc
, i
, txr
->tx_intr_vec
, 1);
920 * Unconfigured TX interrupt vector could only
923 KASSERT(sc
->intr_type
== PCI_INTR_TYPE_MSIX
,
924 ("TX intr vector is not set"));
925 KASSERT(i
< sc
->rx_ring_inuse
,
926 ("invalid TX ring %d, no piggyback RX ring", i
));
927 KASSERT(sc
->rx_rings
[i
].rx_txr
== txr
,
928 ("RX ring %d piggybacked TX ring mismatch", i
));
930 if_printf(ifp
, "IVAR skips TX ring %d\n", i
);
933 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
934 const struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
936 KKASSERT(rxr
->rx_intr_vec
>= 0);
937 ix_set_ivar(sc
, i
, rxr
->rx_intr_vec
, 0);
938 if (rxr
->rx_txr
!= NULL
) {
940 * Piggyback the TX ring interrupt onto the RX
941 * ring interrupt vector.
943 KASSERT(rxr
->rx_txr
->tx_intr_vec
< 0,
944 ("piggybacked TX ring configured intr vector"));
945 KASSERT(rxr
->rx_txr
->tx_idx
== i
,
946 ("RX ring %d piggybacked TX ring %u",
947 i
, rxr
->rx_txr
->tx_idx
));
948 ix_set_ivar(sc
, i
, rxr
->rx_intr_vec
, 1);
950 if_printf(ifp
, "IVAR RX ring %d piggybacks "
951 "TX ring %u\n", i
, rxr
->rx_txr
->tx_idx
);
955 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
) {
956 /* Set up status MSI-X vector; it is using fixed entry 1 */
957 ix_set_ivar(sc
, 1, sc
->sts_msix_vec
, -1);
959 /* Set up auto-mask for TX and RX rings */
960 if (hw
->mac
.type
== ixgbe_mac_82598EB
) {
961 IXGBE_WRITE_REG(hw
, IXGBE_EIAM
, IXGBE_EIMS_RTX_QUEUE
);
963 IXGBE_WRITE_REG(hw
, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
964 IXGBE_WRITE_REG(hw
, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
967 IXGBE_WRITE_REG(hw
, IXGBE_EIAM
, IXGBE_EIMS_RTX_QUEUE
);
969 for (i
= 0; i
< sc
->intr_cnt
; ++i
)
970 ix_set_eitr(sc
, i
, sc
->intr_data
[i
].intr_rate
);
973 * Check on any SFP devices that need to be kick-started
975 if (hw
->phy
.type
== ixgbe_phy_none
) {
976 error
= hw
->phy
.ops
.identify(hw
);
977 if (error
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
979 "Unsupported SFP+ module type was detected.\n");
985 /* Config/Enable Link */
989 * Hardware Packet Buffer & Flow Control setup
991 frame
= sc
->max_frame_size
;
993 /* Calculate High Water */
994 if (hw
->mac
.type
== ixgbe_mac_X540
)
995 tmp
= IXGBE_DV_X540(frame
, frame
);
997 tmp
= IXGBE_DV(frame
, frame
);
998 size
= IXGBE_BT2KB(tmp
);
999 rxpb
= IXGBE_READ_REG(hw
, IXGBE_RXPBSIZE(0)) >> 10;
1000 hw
->fc
.high_water
[0] = rxpb
- size
;
1002 /* Now calculate Low Water */
1003 if (hw
->mac
.type
== ixgbe_mac_X540
)
1004 tmp
= IXGBE_LOW_DV_X540(frame
);
1006 tmp
= IXGBE_LOW_DV(frame
);
1007 hw
->fc
.low_water
[0] = IXGBE_BT2KB(tmp
);
1009 hw
->fc
.requested_mode
= sc
->fc
;
1010 hw
->fc
.pause_time
= IX_FC_PAUSE
;
1011 hw
->fc
.send_xon
= TRUE
;
1013 /* Initialize the FC settings */
1017 * Only enable interrupts if we are not polling, make sure
1018 * they are off otherwise.
1021 ix_disable_intr(sc
);
1025 ifp
->if_flags
|= IFF_RUNNING
;
1026 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
1027 ifsq_clr_oactive(sc
->tx_rings
[i
].tx_ifsq
);
1028 ifsq_watchdog_start(&sc
->tx_rings
[i
].tx_watchdog
);
1031 ix_set_timer_cpuid(sc
, polling
);
1032 callout_reset_bycpu(&sc
->timer
, hz
, ix_timer
, sc
, sc
->timer_cpuid
);
1038 struct ix_softc
*sc
= xsc
;
1039 struct ixgbe_hw
*hw
= &sc
->hw
;
1042 ASSERT_SERIALIZED(&sc
->main_serialize
);
1044 eicr
= IXGBE_READ_REG(hw
, IXGBE_EICR
);
1046 IXGBE_WRITE_REG(hw
, IXGBE_EIMS
, sc
->intr_mask
);
1050 if (eicr
& IX_RX0_INTR_MASK
) {
1051 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[0];
1053 lwkt_serialize_enter(&rxr
->rx_serialize
);
1055 lwkt_serialize_exit(&rxr
->rx_serialize
);
1057 if (eicr
& IX_RX1_INTR_MASK
) {
1058 struct ix_rx_ring
*rxr
;
1060 KKASSERT(sc
->rx_ring_inuse
== IX_MIN_RXRING_RSS
);
1061 rxr
= &sc
->rx_rings
[1];
1063 lwkt_serialize_enter(&rxr
->rx_serialize
);
1065 lwkt_serialize_exit(&rxr
->rx_serialize
);
1068 if (eicr
& IX_TX_INTR_MASK
) {
1069 struct ix_tx_ring
*txr
= &sc
->tx_rings
[0];
1071 lwkt_serialize_enter(&txr
->tx_serialize
);
1072 ix_txeof(txr
, *(txr
->tx_hdr
));
1073 if (!ifsq_is_empty(txr
->tx_ifsq
))
1074 ifsq_devstart(txr
->tx_ifsq
);
1075 lwkt_serialize_exit(&txr
->tx_serialize
);
1078 if (__predict_false(eicr
& IX_EICR_STATUS
))
1079 ix_intr_status(sc
, eicr
);
1081 IXGBE_WRITE_REG(hw
, IXGBE_EIMS
, sc
->intr_mask
);
1085 ix_media_status(struct ifnet
*ifp
, struct ifmediareq
*ifmr
)
1087 struct ix_softc
*sc
= ifp
->if_softc
;
1089 ix_update_link_status(sc
);
1091 ifmr
->ifm_status
= IFM_AVALID
;
1092 ifmr
->ifm_active
= IFM_ETHER
;
1094 if (!sc
->link_active
)
1097 ifmr
->ifm_status
|= IFM_ACTIVE
;
1099 switch (sc
->link_speed
) {
1100 case IXGBE_LINK_SPEED_100_FULL
:
1101 ifmr
->ifm_active
|= IFM_100_TX
| IFM_FDX
;
1103 case IXGBE_LINK_SPEED_1GB_FULL
:
1104 ifmr
->ifm_active
|= IFM_1000_SX
| IFM_FDX
;
1106 case IXGBE_LINK_SPEED_10GB_FULL
:
1107 ifmr
->ifm_active
|= sc
->optics
| IFM_FDX
;
1113 ix_media_change(struct ifnet
*ifp
)
1115 struct ix_softc
*sc
= ifp
->if_softc
;
1116 struct ifmedia
*ifm
= &sc
->media
;
1118 if (IFM_TYPE(ifm
->ifm_media
) != IFM_ETHER
)
1121 switch (IFM_SUBTYPE(ifm
->ifm_media
)) {
1123 sc
->hw
.phy
.autoneg_advertised
=
1124 IXGBE_LINK_SPEED_100_FULL
|
1125 IXGBE_LINK_SPEED_1GB_FULL
|
1126 IXGBE_LINK_SPEED_10GB_FULL
;
1129 if_printf(ifp
, "Only auto media type\n");
1136 ix_tso_pullup(struct mbuf
**mp
)
1138 int hoff
, iphlen
, thoff
;
1142 KASSERT(M_WRITABLE(m
), ("TSO mbuf not writable"));
1144 iphlen
= m
->m_pkthdr
.csum_iphlen
;
1145 thoff
= m
->m_pkthdr
.csum_thlen
;
1146 hoff
= m
->m_pkthdr
.csum_lhlen
;
1148 KASSERT(iphlen
> 0, ("invalid ip hlen"));
1149 KASSERT(thoff
> 0, ("invalid tcp hlen"));
1150 KASSERT(hoff
> 0, ("invalid ether hlen"));
1152 if (__predict_false(m
->m_len
< hoff
+ iphlen
+ thoff
)) {
1153 m
= m_pullup(m
, hoff
+ iphlen
+ thoff
);
1164 ix_encap(struct ix_tx_ring
*txr
, struct mbuf
**m_headp
,
1165 uint16_t *segs_used
, int *idx
)
1167 uint32_t olinfo_status
= 0, cmd_type_len
, cmd_rs
= 0;
1168 int i
, j
, error
, nsegs
, first
, maxsegs
;
1169 struct mbuf
*m_head
= *m_headp
;
1170 bus_dma_segment_t segs
[IX_MAX_SCATTER
];
1172 struct ix_tx_buf
*txbuf
;
1173 union ixgbe_adv_tx_desc
*txd
= NULL
;
1175 if (m_head
->m_pkthdr
.csum_flags
& CSUM_TSO
) {
1176 error
= ix_tso_pullup(m_headp
);
1177 if (__predict_false(error
))
1182 /* Basic descriptor defines */
1183 cmd_type_len
= (IXGBE_ADVTXD_DTYP_DATA
|
1184 IXGBE_ADVTXD_DCMD_IFCS
| IXGBE_ADVTXD_DCMD_DEXT
);
1186 if (m_head
->m_flags
& M_VLANTAG
)
1187 cmd_type_len
|= IXGBE_ADVTXD_DCMD_VLE
;
1190 * Important to capture the first descriptor
1191 * used because it will contain the index of
1192 * the one we tell the hardware to report back
1194 first
= txr
->tx_next_avail
;
1195 txbuf
= &txr
->tx_buf
[first
];
1199 * Map the packet for DMA.
1201 maxsegs
= txr
->tx_avail
- IX_TX_RESERVED
;
1202 if (maxsegs
> IX_MAX_SCATTER
)
1203 maxsegs
= IX_MAX_SCATTER
;
1205 error
= bus_dmamap_load_mbuf_defrag(txr
->tx_tag
, map
, m_headp
,
1206 segs
, maxsegs
, &nsegs
, BUS_DMA_NOWAIT
);
1207 if (__predict_false(error
)) {
1212 bus_dmamap_sync(txr
->tx_tag
, map
, BUS_DMASYNC_PREWRITE
);
1217 * Set up the appropriate offload context if requested,
1218 * this may consume one TX descriptor.
1220 if (ix_tx_ctx_setup(txr
, m_head
, &cmd_type_len
, &olinfo_status
)) {
1225 *segs_used
+= nsegs
;
1226 txr
->tx_nsegs
+= nsegs
;
1227 if (txr
->tx_nsegs
>= txr
->tx_intr_nsegs
) {
1229 * Report Status (RS) is turned on every intr_nsegs
1230 * descriptors (roughly).
1233 cmd_rs
= IXGBE_TXD_CMD_RS
;
1236 i
= txr
->tx_next_avail
;
1237 for (j
= 0; j
< nsegs
; j
++) {
1241 txbuf
= &txr
->tx_buf
[i
];
1242 txd
= &txr
->tx_base
[i
];
1243 seglen
= segs
[j
].ds_len
;
1244 segaddr
= htole64(segs
[j
].ds_addr
);
1246 txd
->read
.buffer_addr
= segaddr
;
1247 txd
->read
.cmd_type_len
= htole32(IXGBE_TXD_CMD_IFCS
|
1248 cmd_type_len
|seglen
);
1249 txd
->read
.olinfo_status
= htole32(olinfo_status
);
1251 if (++i
== txr
->tx_ndesc
)
1254 txd
->read
.cmd_type_len
|= htole32(IXGBE_TXD_CMD_EOP
| cmd_rs
);
1256 txr
->tx_avail
-= nsegs
;
1257 txr
->tx_next_avail
= i
;
1259 txbuf
->m_head
= m_head
;
1260 txr
->tx_buf
[first
].map
= txbuf
->map
;
1264 * Defer TDT updating, until enough descrptors are setup
1272 ix_set_promisc(struct ix_softc
*sc
)
1274 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1278 reg_rctl
= IXGBE_READ_REG(&sc
->hw
, IXGBE_FCTRL
);
1279 reg_rctl
&= ~IXGBE_FCTRL_UPE
;
1280 if (ifp
->if_flags
& IFF_ALLMULTI
) {
1281 mcnt
= IX_MAX_MCASTADDR
;
1283 struct ifmultiaddr
*ifma
;
1285 TAILQ_FOREACH(ifma
, &ifp
->if_multiaddrs
, ifma_link
) {
1286 if (ifma
->ifma_addr
->sa_family
!= AF_LINK
)
1288 if (mcnt
== IX_MAX_MCASTADDR
)
1293 if (mcnt
< IX_MAX_MCASTADDR
)
1294 reg_rctl
&= ~IXGBE_FCTRL_MPE
;
1295 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_FCTRL
, reg_rctl
);
1297 if (ifp
->if_flags
& IFF_PROMISC
) {
1298 reg_rctl
|= IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
;
1299 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_FCTRL
, reg_rctl
);
1300 } else if (ifp
->if_flags
& IFF_ALLMULTI
) {
1301 reg_rctl
|= IXGBE_FCTRL_MPE
;
1302 reg_rctl
&= ~IXGBE_FCTRL_UPE
;
1303 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_FCTRL
, reg_rctl
);
1308 ix_set_multi(struct ix_softc
*sc
)
1310 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1311 struct ifmultiaddr
*ifma
;
1317 bzero(mta
, IXGBE_ETH_LENGTH_OF_ADDRESS
* IX_MAX_MCASTADDR
);
1319 TAILQ_FOREACH(ifma
, &ifp
->if_multiaddrs
, ifma_link
) {
1320 if (ifma
->ifma_addr
->sa_family
!= AF_LINK
)
1322 if (mcnt
== IX_MAX_MCASTADDR
)
1324 bcopy(LLADDR((struct sockaddr_dl
*)ifma
->ifma_addr
),
1325 &mta
[mcnt
* IXGBE_ETH_LENGTH_OF_ADDRESS
],
1326 IXGBE_ETH_LENGTH_OF_ADDRESS
);
1330 fctrl
= IXGBE_READ_REG(&sc
->hw
, IXGBE_FCTRL
);
1331 fctrl
|= (IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
);
1332 if (ifp
->if_flags
& IFF_PROMISC
) {
1333 fctrl
|= IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
;
1334 } else if (mcnt
>= IX_MAX_MCASTADDR
|| (ifp
->if_flags
& IFF_ALLMULTI
)) {
1335 fctrl
|= IXGBE_FCTRL_MPE
;
1336 fctrl
&= ~IXGBE_FCTRL_UPE
;
1338 fctrl
&= ~(IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
);
1340 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_FCTRL
, fctrl
);
1342 if (mcnt
< IX_MAX_MCASTADDR
) {
1343 ixgbe_update_mc_addr_list(&sc
->hw
,
1344 mta
, mcnt
, ix_mc_array_itr
, TRUE
);
1349 * This is an iterator function now needed by the multicast
1350 * shared code. It simply feeds the shared code routine the
1351 * addresses in the array of ix_set_multi() one by one.
1354 ix_mc_array_itr(struct ixgbe_hw
*hw
, uint8_t **update_ptr
, uint32_t *vmdq
)
1356 uint8_t *addr
= *update_ptr
;
1360 newptr
= addr
+ IXGBE_ETH_LENGTH_OF_ADDRESS
;
1361 *update_ptr
= newptr
;
1368 struct ix_softc
*sc
= arg
;
1370 lwkt_serialize_enter(&sc
->main_serialize
);
1372 if ((sc
->arpcom
.ac_if
.if_flags
& IFF_RUNNING
) == 0) {
1373 lwkt_serialize_exit(&sc
->main_serialize
);
1377 /* Check for pluggable optics */
1378 if (sc
->sfp_probe
) {
1379 if (!ix_sfp_probe(sc
))
1380 goto done
; /* Nothing to do */
1383 ix_update_link_status(sc
);
1384 ix_update_stats(sc
);
1387 callout_reset_bycpu(&sc
->timer
, hz
, ix_timer
, sc
, sc
->timer_cpuid
);
1388 lwkt_serialize_exit(&sc
->main_serialize
);
1392 ix_update_link_status(struct ix_softc
*sc
)
1394 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1397 if (sc
->link_active
== FALSE
) {
1399 if_printf(ifp
, "Link is up %d Gbps %s\n",
1400 sc
->link_speed
== 128 ? 10 : 1,
1403 sc
->link_active
= TRUE
;
1405 /* Update any Flow Control changes */
1406 ixgbe_fc_enable(&sc
->hw
);
1408 ifp
->if_link_state
= LINK_STATE_UP
;
1409 if_link_state_change(ifp
);
1411 } else { /* Link down */
1412 if (sc
->link_active
== TRUE
) {
1414 if_printf(ifp
, "Link is Down\n");
1415 ifp
->if_link_state
= LINK_STATE_DOWN
;
1416 if_link_state_change(ifp
);
1418 sc
->link_active
= FALSE
;
1424 ix_stop(struct ix_softc
*sc
)
1426 struct ixgbe_hw
*hw
= &sc
->hw
;
1427 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1430 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1432 ix_disable_intr(sc
);
1433 callout_stop(&sc
->timer
);
1435 ifp
->if_flags
&= ~IFF_RUNNING
;
1436 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
1437 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
1439 ifsq_clr_oactive(txr
->tx_ifsq
);
1440 ifsq_watchdog_stop(&txr
->tx_watchdog
);
1441 txr
->tx_flags
&= ~IX_TXFLAG_ENABLED
;
1445 hw
->adapter_stopped
= FALSE
;
1446 ixgbe_stop_adapter(hw
);
1447 if (hw
->mac
.type
== ixgbe_mac_82599EB
)
1448 ixgbe_stop_mac_link_on_d3_82599(hw
);
1449 /* Turn off the laser - noop with no optics */
1450 ixgbe_disable_tx_laser(hw
);
1452 /* Update the stack */
1453 sc
->link_up
= FALSE
;
1454 ix_update_link_status(sc
);
1456 /* Reprogram the RAR[0] in case user changed it. */
1457 ixgbe_set_rar(hw
, 0, hw
->mac
.addr
, 0, IXGBE_RAH_AV
);
1459 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
1460 ix_free_tx_ring(&sc
->tx_rings
[i
]);
1462 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
)
1463 ix_free_rx_ring(&sc
->rx_rings
[i
]);
1467 ix_setup_optics(struct ix_softc
*sc
)
1469 struct ixgbe_hw
*hw
= &sc
->hw
;
1472 layer
= ixgbe_get_supported_physical_layer(hw
);
1474 if (layer
& IXGBE_PHYSICAL_LAYER_10GBASE_T
) {
1475 sc
->optics
= IFM_10G_T
;
1479 if (layer
& IXGBE_PHYSICAL_LAYER_1000BASE_T
) {
1480 sc
->optics
= IFM_1000_T
;
1484 if (layer
& IXGBE_PHYSICAL_LAYER_1000BASE_SX
) {
1485 sc
->optics
= IFM_1000_SX
;
1489 if (layer
& (IXGBE_PHYSICAL_LAYER_10GBASE_LR
|
1490 IXGBE_PHYSICAL_LAYER_10GBASE_LRM
)) {
1491 sc
->optics
= IFM_10G_LR
;
1495 if (layer
& IXGBE_PHYSICAL_LAYER_10GBASE_SR
) {
1496 sc
->optics
= IFM_10G_SR
;
1500 if (layer
& IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU
) {
1501 sc
->optics
= IFM_10G_TWINAX
;
1505 if (layer
& (IXGBE_PHYSICAL_LAYER_10GBASE_KX4
|
1506 IXGBE_PHYSICAL_LAYER_10GBASE_CX4
)) {
1507 sc
->optics
= IFM_10G_CX4
;
1511 /* If we get here just set the default */
1512 sc
->optics
= IFM_ETHER
| IFM_AUTO
;
1516 ix_setup_ifp(struct ix_softc
*sc
)
1518 struct ixgbe_hw
*hw
= &sc
->hw
;
1519 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1522 ifp
->if_baudrate
= IF_Gbps(10UL);
1525 ifp
->if_flags
= IFF_BROADCAST
| IFF_SIMPLEX
| IFF_MULTICAST
;
1526 ifp
->if_init
= ix_init
;
1527 ifp
->if_ioctl
= ix_ioctl
;
1528 ifp
->if_start
= ix_start
;
1529 ifp
->if_serialize
= ix_serialize
;
1530 ifp
->if_deserialize
= ix_deserialize
;
1531 ifp
->if_tryserialize
= ix_tryserialize
;
1533 ifp
->if_serialize_assert
= ix_serialize_assert
;
1535 #ifdef IFPOLL_ENABLE
1536 ifp
->if_npoll
= ix_npoll
;
1539 /* Increase TSO burst length */
1540 ifp
->if_tsolen
= (8 * ETHERMTU
);
1542 ifq_set_maxlen(&ifp
->if_snd
, sc
->tx_rings
[0].tx_ndesc
- 2);
1543 ifq_set_ready(&ifp
->if_snd
);
1544 ifq_set_subq_cnt(&ifp
->if_snd
, sc
->tx_ring_cnt
);
1546 ifp
->if_mapsubq
= ifq_mapsubq_mask
;
1547 ifq_set_subq_mask(&ifp
->if_snd
, 0);
1549 ether_ifattach(ifp
, hw
->mac
.addr
, NULL
);
1551 ifp
->if_capabilities
=
1552 IFCAP_HWCSUM
| IFCAP_TSO
| IFCAP_VLAN_HWTAGGING
| IFCAP_VLAN_MTU
;
1553 if (IX_ENABLE_HWRSS(sc
))
1554 ifp
->if_capabilities
|= IFCAP_RSS
;
1555 ifp
->if_capenable
= ifp
->if_capabilities
;
1556 ifp
->if_hwassist
= CSUM_OFFLOAD
| CSUM_TSO
;
1559 * Tell the upper layer(s) we support long frames.
1561 ifp
->if_data
.ifi_hdrlen
= sizeof(struct ether_vlan_header
);
1563 /* Setup TX rings and subqueues */
1564 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
1565 struct ifaltq_subque
*ifsq
= ifq_get_subq(&ifp
->if_snd
, i
);
1566 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
1568 ifsq_set_cpuid(ifsq
, txr
->tx_intr_cpuid
);
1569 ifsq_set_priv(ifsq
, txr
);
1570 ifsq_set_hw_serialize(ifsq
, &txr
->tx_serialize
);
1571 txr
->tx_ifsq
= ifsq
;
1573 ifsq_watchdog_init(&txr
->tx_watchdog
, ifsq
, ix_watchdog
);
1577 * Specify the media types supported by this adapter and register
1578 * callbacks to update media and link information
1580 ifmedia_add(&sc
->media
, IFM_ETHER
| sc
->optics
, 0, NULL
);
1581 ifmedia_set(&sc
->media
, IFM_ETHER
| sc
->optics
);
1582 if (hw
->device_id
== IXGBE_DEV_ID_82598AT
) {
1583 ifmedia_add(&sc
->media
,
1584 IFM_ETHER
| IFM_1000_T
| IFM_FDX
, 0, NULL
);
1585 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_1000_T
, 0, NULL
);
1587 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_AUTO
, 0, NULL
);
1588 ifmedia_set(&sc
->media
, IFM_ETHER
| IFM_AUTO
);
1592 ix_is_sfp(const struct ixgbe_hw
*hw
)
1594 switch (hw
->phy
.type
) {
1595 case ixgbe_phy_sfp_avago
:
1596 case ixgbe_phy_sfp_ftl
:
1597 case ixgbe_phy_sfp_intel
:
1598 case ixgbe_phy_sfp_unknown
:
1599 case ixgbe_phy_sfp_passive_tyco
:
1600 case ixgbe_phy_sfp_passive_unknown
:
1608 ix_config_link(struct ix_softc
*sc
)
1610 struct ixgbe_hw
*hw
= &sc
->hw
;
1613 sfp
= ix_is_sfp(hw
);
1615 if (hw
->phy
.multispeed_fiber
) {
1616 hw
->mac
.ops
.setup_sfp(hw
);
1617 ixgbe_enable_tx_laser(hw
);
1623 uint32_t autoneg
, err
= 0;
1625 if (hw
->mac
.ops
.check_link
!= NULL
) {
1626 err
= ixgbe_check_link(hw
, &sc
->link_speed
,
1627 &sc
->link_up
, FALSE
);
1632 autoneg
= hw
->phy
.autoneg_advertised
;
1633 if (!autoneg
&& hw
->mac
.ops
.get_link_capabilities
!= NULL
) {
1636 err
= hw
->mac
.ops
.get_link_capabilities(hw
,
1637 &autoneg
, &negotiate
);
1642 if (hw
->mac
.ops
.setup_link
!= NULL
) {
1643 err
= hw
->mac
.ops
.setup_link(hw
,
1644 autoneg
, sc
->link_up
);
1652 ix_alloc_rings(struct ix_softc
*sc
)
1657 * Create top level busdma tag
1659 error
= bus_dma_tag_create(NULL
, 1, 0,
1660 BUS_SPACE_MAXADDR
, BUS_SPACE_MAXADDR
, NULL
, NULL
,
1661 BUS_SPACE_MAXSIZE_32BIT
, 0, BUS_SPACE_MAXSIZE_32BIT
, 0,
1664 device_printf(sc
->dev
, "could not create top level DMA tag\n");
1669 * Allocate TX descriptor rings and buffers
1671 sc
->tx_rings
= kmalloc_cachealign(
1672 sizeof(struct ix_tx_ring
) * sc
->tx_ring_cnt
,
1673 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1674 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
1675 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
1679 txr
->tx_intr_vec
= -1;
1680 lwkt_serialize_init(&txr
->tx_serialize
);
1682 error
= ix_create_tx_ring(txr
);
1688 * Allocate RX descriptor rings and buffers
1690 sc
->rx_rings
= kmalloc_cachealign(
1691 sizeof(struct ix_rx_ring
) * sc
->rx_ring_cnt
,
1692 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1693 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
1694 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
1698 rxr
->rx_intr_vec
= -1;
1699 lwkt_serialize_init(&rxr
->rx_serialize
);
1701 error
= ix_create_rx_ring(rxr
);
1710 ix_create_tx_ring(struct ix_tx_ring
*txr
)
1712 int error
, i
, tsize
, ntxd
;
1715 * Validate number of transmit descriptors. It must not exceed
1716 * hardware maximum, and must be multiple of IX_DBA_ALIGN.
1718 ntxd
= device_getenv_int(txr
->tx_sc
->dev
, "txd", ix_txd
);
1719 if (((ntxd
* sizeof(union ixgbe_adv_tx_desc
)) % IX_DBA_ALIGN
) != 0 ||
1720 ntxd
< IX_MIN_TXD
|| ntxd
> IX_MAX_TXD
) {
1721 device_printf(txr
->tx_sc
->dev
,
1722 "Using %d TX descriptors instead of %d!\n",
1724 txr
->tx_ndesc
= IX_DEF_TXD
;
1726 txr
->tx_ndesc
= ntxd
;
1730 * Allocate TX head write-back buffer
1732 txr
->tx_hdr
= bus_dmamem_coherent_any(txr
->tx_sc
->parent_tag
,
1733 __VM_CACHELINE_SIZE
, __VM_CACHELINE_SIZE
, BUS_DMA_WAITOK
,
1734 &txr
->tx_hdr_dtag
, &txr
->tx_hdr_map
, &txr
->tx_hdr_paddr
);
1735 if (txr
->tx_hdr
== NULL
) {
1736 device_printf(txr
->tx_sc
->dev
,
1737 "Unable to allocate TX head write-back buffer\n");
1742 * Allocate TX descriptor ring
1744 tsize
= roundup2(txr
->tx_ndesc
* sizeof(union ixgbe_adv_tx_desc
),
1746 txr
->tx_base
= bus_dmamem_coherent_any(txr
->tx_sc
->parent_tag
,
1747 IX_DBA_ALIGN
, tsize
, BUS_DMA_WAITOK
| BUS_DMA_ZERO
,
1748 &txr
->tx_base_dtag
, &txr
->tx_base_map
, &txr
->tx_base_paddr
);
1749 if (txr
->tx_base
== NULL
) {
1750 device_printf(txr
->tx_sc
->dev
,
1751 "Unable to allocate TX Descriptor memory\n");
1755 tsize
= __VM_CACHELINE_ALIGN(sizeof(struct ix_tx_buf
) * txr
->tx_ndesc
);
1756 txr
->tx_buf
= kmalloc_cachealign(tsize
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
1759 * Create DMA tag for TX buffers
1761 error
= bus_dma_tag_create(txr
->tx_sc
->parent_tag
,
1762 1, 0, /* alignment, bounds */
1763 BUS_SPACE_MAXADDR
, /* lowaddr */
1764 BUS_SPACE_MAXADDR
, /* highaddr */
1765 NULL
, NULL
, /* filter, filterarg */
1766 IX_TSO_SIZE
, /* maxsize */
1767 IX_MAX_SCATTER
, /* nsegments */
1768 PAGE_SIZE
, /* maxsegsize */
1769 BUS_DMA_WAITOK
| BUS_DMA_ALLOCNOW
|
1770 BUS_DMA_ONEBPAGE
, /* flags */
1773 device_printf(txr
->tx_sc
->dev
,
1774 "Unable to allocate TX DMA tag\n");
1775 kfree(txr
->tx_buf
, M_DEVBUF
);
1781 * Create DMA maps for TX buffers
1783 for (i
= 0; i
< txr
->tx_ndesc
; ++i
) {
1784 struct ix_tx_buf
*txbuf
= &txr
->tx_buf
[i
];
1786 error
= bus_dmamap_create(txr
->tx_tag
,
1787 BUS_DMA_WAITOK
| BUS_DMA_ONEBPAGE
, &txbuf
->map
);
1789 device_printf(txr
->tx_sc
->dev
,
1790 "Unable to create TX DMA map\n");
1791 ix_destroy_tx_ring(txr
, i
);
1797 * Initialize various watermark
1799 txr
->tx_wreg_nsegs
= IX_DEF_TXWREG_NSEGS
;
1800 txr
->tx_intr_nsegs
= txr
->tx_ndesc
/ 16;
1806 ix_destroy_tx_ring(struct ix_tx_ring
*txr
, int ndesc
)
1810 if (txr
->tx_hdr
!= NULL
) {
1811 bus_dmamap_unload(txr
->tx_hdr_dtag
, txr
->tx_hdr_map
);
1812 bus_dmamem_free(txr
->tx_hdr_dtag
,
1813 __DEVOLATILE(void *, txr
->tx_hdr
), txr
->tx_hdr_map
);
1814 bus_dma_tag_destroy(txr
->tx_hdr_dtag
);
1818 if (txr
->tx_base
!= NULL
) {
1819 bus_dmamap_unload(txr
->tx_base_dtag
, txr
->tx_base_map
);
1820 bus_dmamem_free(txr
->tx_base_dtag
, txr
->tx_base
,
1822 bus_dma_tag_destroy(txr
->tx_base_dtag
);
1823 txr
->tx_base
= NULL
;
1826 if (txr
->tx_buf
== NULL
)
1829 for (i
= 0; i
< ndesc
; ++i
) {
1830 struct ix_tx_buf
*txbuf
= &txr
->tx_buf
[i
];
1832 KKASSERT(txbuf
->m_head
== NULL
);
1833 bus_dmamap_destroy(txr
->tx_tag
, txbuf
->map
);
1835 bus_dma_tag_destroy(txr
->tx_tag
);
1837 kfree(txr
->tx_buf
, M_DEVBUF
);
1842 ix_init_tx_ring(struct ix_tx_ring
*txr
)
1844 /* Clear the old ring contents */
1845 bzero(txr
->tx_base
, sizeof(union ixgbe_adv_tx_desc
) * txr
->tx_ndesc
);
1847 /* Clear TX head write-back buffer */
1851 txr
->tx_next_avail
= 0;
1852 txr
->tx_next_clean
= 0;
1855 /* Set number of descriptors available */
1856 txr
->tx_avail
= txr
->tx_ndesc
;
1858 /* Enable this TX ring */
1859 txr
->tx_flags
|= IX_TXFLAG_ENABLED
;
1863 ix_init_tx_unit(struct ix_softc
*sc
)
1865 struct ixgbe_hw
*hw
= &sc
->hw
;
1869 * Setup the Base and Length of the Tx Descriptor Ring
1871 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
1872 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
1873 uint64_t tdba
= txr
->tx_base_paddr
;
1874 uint64_t hdr_paddr
= txr
->tx_hdr_paddr
;
1877 IXGBE_WRITE_REG(hw
, IXGBE_TDBAL(i
), (uint32_t)tdba
);
1878 IXGBE_WRITE_REG(hw
, IXGBE_TDBAH(i
), (uint32_t)(tdba
>> 32));
1879 IXGBE_WRITE_REG(hw
, IXGBE_TDLEN(i
),
1880 txr
->tx_ndesc
* sizeof(union ixgbe_adv_tx_desc
));
1882 /* Setup the HW Tx Head and Tail descriptor pointers */
1883 IXGBE_WRITE_REG(hw
, IXGBE_TDH(i
), 0);
1884 IXGBE_WRITE_REG(hw
, IXGBE_TDT(i
), 0);
1886 /* Disable TX head write-back relax ordering */
1887 switch (hw
->mac
.type
) {
1888 case ixgbe_mac_82598EB
:
1889 txctrl
= IXGBE_READ_REG(hw
, IXGBE_DCA_TXCTRL(i
));
1891 case ixgbe_mac_82599EB
:
1892 case ixgbe_mac_X540
:
1894 txctrl
= IXGBE_READ_REG(hw
, IXGBE_DCA_TXCTRL_82599(i
));
1897 txctrl
&= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN
;
1898 switch (hw
->mac
.type
) {
1899 case ixgbe_mac_82598EB
:
1900 IXGBE_WRITE_REG(hw
, IXGBE_DCA_TXCTRL(i
), txctrl
);
1902 case ixgbe_mac_82599EB
:
1903 case ixgbe_mac_X540
:
1905 IXGBE_WRITE_REG(hw
, IXGBE_DCA_TXCTRL_82599(i
), txctrl
);
1909 /* Enable TX head write-back */
1910 IXGBE_WRITE_REG(hw
, IXGBE_TDWBAH(i
),
1911 (uint32_t)(hdr_paddr
>> 32));
1912 IXGBE_WRITE_REG(hw
, IXGBE_TDWBAL(i
),
1913 ((uint32_t)hdr_paddr
) | IXGBE_TDWBAL_HEAD_WB_ENABLE
);
1916 if (hw
->mac
.type
!= ixgbe_mac_82598EB
) {
1917 uint32_t dmatxctl
, rttdcs
;
1919 dmatxctl
= IXGBE_READ_REG(hw
, IXGBE_DMATXCTL
);
1920 dmatxctl
|= IXGBE_DMATXCTL_TE
;
1921 IXGBE_WRITE_REG(hw
, IXGBE_DMATXCTL
, dmatxctl
);
1923 /* Disable arbiter to set MTQC */
1924 rttdcs
= IXGBE_READ_REG(hw
, IXGBE_RTTDCS
);
1925 rttdcs
|= IXGBE_RTTDCS_ARBDIS
;
1926 IXGBE_WRITE_REG(hw
, IXGBE_RTTDCS
, rttdcs
);
1928 IXGBE_WRITE_REG(hw
, IXGBE_MTQC
, IXGBE_MTQC_64Q_1PB
);
1930 /* Reenable aribter */
1931 rttdcs
&= ~IXGBE_RTTDCS_ARBDIS
;
1932 IXGBE_WRITE_REG(hw
, IXGBE_RTTDCS
, rttdcs
);
1937 ix_tx_ctx_setup(struct ix_tx_ring
*txr
, const struct mbuf
*mp
,
1938 uint32_t *cmd_type_len
, uint32_t *olinfo_status
)
1940 struct ixgbe_adv_tx_context_desc
*TXD
;
1941 uint32_t vlan_macip_lens
= 0, type_tucmd_mlhl
= 0;
1942 int ehdrlen
, ip_hlen
= 0, ctxd
;
1943 boolean_t offload
= TRUE
;
1945 /* First check if TSO is to be used */
1946 if (mp
->m_pkthdr
.csum_flags
& CSUM_TSO
) {
1947 return ix_tso_ctx_setup(txr
, mp
,
1948 cmd_type_len
, olinfo_status
);
1951 if ((mp
->m_pkthdr
.csum_flags
& CSUM_OFFLOAD
) == 0)
1954 /* Indicate the whole packet as payload when not doing TSO */
1955 *olinfo_status
|= mp
->m_pkthdr
.len
<< IXGBE_ADVTXD_PAYLEN_SHIFT
;
1958 * In advanced descriptors the vlan tag must be placed into the
1959 * context descriptor. Hence we need to make one even if not
1960 * doing checksum offloads.
1962 if (mp
->m_flags
& M_VLANTAG
) {
1963 vlan_macip_lens
|= htole16(mp
->m_pkthdr
.ether_vlantag
) <<
1964 IXGBE_ADVTXD_VLAN_SHIFT
;
1965 } else if (!offload
) {
1966 /* No TX descriptor is consumed */
1970 /* Set the ether header length */
1971 ehdrlen
= mp
->m_pkthdr
.csum_lhlen
;
1972 KASSERT(ehdrlen
> 0, ("invalid ether hlen"));
1973 vlan_macip_lens
|= ehdrlen
<< IXGBE_ADVTXD_MACLEN_SHIFT
;
1975 if (mp
->m_pkthdr
.csum_flags
& CSUM_IP
) {
1976 *olinfo_status
|= IXGBE_TXD_POPTS_IXSM
<< 8;
1977 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_IPV4
;
1978 ip_hlen
= mp
->m_pkthdr
.csum_iphlen
;
1979 KASSERT(ip_hlen
> 0, ("invalid ip hlen"));
1981 vlan_macip_lens
|= ip_hlen
;
1983 type_tucmd_mlhl
|= IXGBE_ADVTXD_DCMD_DEXT
| IXGBE_ADVTXD_DTYP_CTXT
;
1984 if (mp
->m_pkthdr
.csum_flags
& CSUM_TCP
)
1985 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_L4T_TCP
;
1986 else if (mp
->m_pkthdr
.csum_flags
& CSUM_UDP
)
1987 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_L4T_UDP
;
1989 if (mp
->m_pkthdr
.csum_flags
& (CSUM_TCP
| CSUM_UDP
))
1990 *olinfo_status
|= IXGBE_TXD_POPTS_TXSM
<< 8;
1992 /* Now ready a context descriptor */
1993 ctxd
= txr
->tx_next_avail
;
1994 TXD
= (struct ixgbe_adv_tx_context_desc
*)&txr
->tx_base
[ctxd
];
1996 /* Now copy bits into descriptor */
1997 TXD
->vlan_macip_lens
= htole32(vlan_macip_lens
);
1998 TXD
->type_tucmd_mlhl
= htole32(type_tucmd_mlhl
);
1999 TXD
->seqnum_seed
= htole32(0);
2000 TXD
->mss_l4len_idx
= htole32(0);
2002 /* We've consumed the first desc, adjust counters */
2003 if (++ctxd
== txr
->tx_ndesc
)
2005 txr
->tx_next_avail
= ctxd
;
2008 /* One TX descriptor is consumed */
2013 ix_tso_ctx_setup(struct ix_tx_ring
*txr
, const struct mbuf
*mp
,
2014 uint32_t *cmd_type_len
, uint32_t *olinfo_status
)
2016 struct ixgbe_adv_tx_context_desc
*TXD
;
2017 uint32_t vlan_macip_lens
= 0, type_tucmd_mlhl
= 0;
2018 uint32_t mss_l4len_idx
= 0, paylen
;
2019 int ctxd
, ehdrlen
, ip_hlen
, tcp_hlen
;
2021 ehdrlen
= mp
->m_pkthdr
.csum_lhlen
;
2022 KASSERT(ehdrlen
> 0, ("invalid ether hlen"));
2024 ip_hlen
= mp
->m_pkthdr
.csum_iphlen
;
2025 KASSERT(ip_hlen
> 0, ("invalid ip hlen"));
2027 tcp_hlen
= mp
->m_pkthdr
.csum_thlen
;
2028 KASSERT(tcp_hlen
> 0, ("invalid tcp hlen"));
2030 ctxd
= txr
->tx_next_avail
;
2031 TXD
= (struct ixgbe_adv_tx_context_desc
*) &txr
->tx_base
[ctxd
];
2033 if (mp
->m_flags
& M_VLANTAG
) {
2034 vlan_macip_lens
|= htole16(mp
->m_pkthdr
.ether_vlantag
) <<
2035 IXGBE_ADVTXD_VLAN_SHIFT
;
2037 vlan_macip_lens
|= ehdrlen
<< IXGBE_ADVTXD_MACLEN_SHIFT
;
2038 vlan_macip_lens
|= ip_hlen
;
2039 TXD
->vlan_macip_lens
= htole32(vlan_macip_lens
);
2041 /* ADV DTYPE TUCMD */
2042 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_IPV4
;
2043 type_tucmd_mlhl
|= IXGBE_ADVTXD_DCMD_DEXT
| IXGBE_ADVTXD_DTYP_CTXT
;
2044 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_L4T_TCP
;
2045 TXD
->type_tucmd_mlhl
= htole32(type_tucmd_mlhl
);
2048 mss_l4len_idx
|= (mp
->m_pkthdr
.tso_segsz
<< IXGBE_ADVTXD_MSS_SHIFT
);
2049 mss_l4len_idx
|= (tcp_hlen
<< IXGBE_ADVTXD_L4LEN_SHIFT
);
2050 TXD
->mss_l4len_idx
= htole32(mss_l4len_idx
);
2052 TXD
->seqnum_seed
= htole32(0);
2054 if (++ctxd
== txr
->tx_ndesc
)
2058 txr
->tx_next_avail
= ctxd
;
2060 *cmd_type_len
|= IXGBE_ADVTXD_DCMD_TSE
;
2062 /* This is used in the transmit desc in encap */
2063 paylen
= mp
->m_pkthdr
.len
- ehdrlen
- ip_hlen
- tcp_hlen
;
2065 *olinfo_status
|= IXGBE_TXD_POPTS_IXSM
<< 8;
2066 *olinfo_status
|= IXGBE_TXD_POPTS_TXSM
<< 8;
2067 *olinfo_status
|= paylen
<< IXGBE_ADVTXD_PAYLEN_SHIFT
;
2069 /* One TX descriptor is consumed */
2074 ix_txeof(struct ix_tx_ring
*txr
, int hdr
)
2078 if (txr
->tx_avail
== txr
->tx_ndesc
)
2081 first
= txr
->tx_next_clean
;
2085 avail
= txr
->tx_avail
;
2086 while (first
!= hdr
) {
2087 struct ix_tx_buf
*txbuf
= &txr
->tx_buf
[first
];
2090 if (txbuf
->m_head
) {
2091 bus_dmamap_unload(txr
->tx_tag
, txbuf
->map
);
2092 m_freem(txbuf
->m_head
);
2093 txbuf
->m_head
= NULL
;
2095 if (++first
== txr
->tx_ndesc
)
2098 txr
->tx_next_clean
= first
;
2099 txr
->tx_avail
= avail
;
2101 if (txr
->tx_avail
> IX_MAX_SCATTER
+ IX_TX_RESERVED
) {
2102 ifsq_clr_oactive(txr
->tx_ifsq
);
2103 txr
->tx_watchdog
.wd_timer
= 0;
2108 ix_create_rx_ring(struct ix_rx_ring
*rxr
)
2110 int i
, rsize
, error
, nrxd
;
2113 * Validate number of receive descriptors. It must not exceed
2114 * hardware maximum, and must be multiple of IX_DBA_ALIGN.
2116 nrxd
= device_getenv_int(rxr
->rx_sc
->dev
, "rxd", ix_rxd
);
2117 if (((nrxd
* sizeof(union ixgbe_adv_rx_desc
)) % IX_DBA_ALIGN
) != 0 ||
2118 nrxd
< IX_MIN_RXD
|| nrxd
> IX_MAX_RXD
) {
2119 device_printf(rxr
->rx_sc
->dev
,
2120 "Using %d RX descriptors instead of %d!\n",
2122 rxr
->rx_ndesc
= IX_DEF_RXD
;
2124 rxr
->rx_ndesc
= nrxd
;
2128 * Allocate RX descriptor ring
2130 rsize
= roundup2(rxr
->rx_ndesc
* sizeof(union ixgbe_adv_rx_desc
),
2132 rxr
->rx_base
= bus_dmamem_coherent_any(rxr
->rx_sc
->parent_tag
,
2133 IX_DBA_ALIGN
, rsize
, BUS_DMA_WAITOK
| BUS_DMA_ZERO
,
2134 &rxr
->rx_base_dtag
, &rxr
->rx_base_map
, &rxr
->rx_base_paddr
);
2135 if (rxr
->rx_base
== NULL
) {
2136 device_printf(rxr
->rx_sc
->dev
,
2137 "Unable to allocate TX Descriptor memory\n");
2141 rsize
= __VM_CACHELINE_ALIGN(sizeof(struct ix_rx_buf
) * rxr
->rx_ndesc
);
2142 rxr
->rx_buf
= kmalloc_cachealign(rsize
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
2145 * Create DMA tag for RX buffers
2147 error
= bus_dma_tag_create(rxr
->rx_sc
->parent_tag
,
2148 1, 0, /* alignment, bounds */
2149 BUS_SPACE_MAXADDR
, /* lowaddr */
2150 BUS_SPACE_MAXADDR
, /* highaddr */
2151 NULL
, NULL
, /* filter, filterarg */
2152 PAGE_SIZE
, /* maxsize */
2154 PAGE_SIZE
, /* maxsegsize */
2155 BUS_DMA_WAITOK
| BUS_DMA_ALLOCNOW
, /* flags */
2158 device_printf(rxr
->rx_sc
->dev
,
2159 "Unable to create RX DMA tag\n");
2160 kfree(rxr
->rx_buf
, M_DEVBUF
);
2166 * Create spare DMA map for RX buffers
2168 error
= bus_dmamap_create(rxr
->rx_tag
, BUS_DMA_WAITOK
,
2171 device_printf(rxr
->rx_sc
->dev
,
2172 "Unable to create spare RX DMA map\n");
2173 bus_dma_tag_destroy(rxr
->rx_tag
);
2174 kfree(rxr
->rx_buf
, M_DEVBUF
);
2180 * Create DMA maps for RX buffers
2182 for (i
= 0; i
< rxr
->rx_ndesc
; ++i
) {
2183 struct ix_rx_buf
*rxbuf
= &rxr
->rx_buf
[i
];
2185 error
= bus_dmamap_create(rxr
->rx_tag
,
2186 BUS_DMA_WAITOK
, &rxbuf
->map
);
2188 device_printf(rxr
->rx_sc
->dev
,
2189 "Unable to create RX dma map\n");
2190 ix_destroy_rx_ring(rxr
, i
);
2196 * Initialize various watermark
2198 rxr
->rx_wreg_nsegs
= IX_DEF_RXWREG_NSEGS
;
2204 ix_destroy_rx_ring(struct ix_rx_ring
*rxr
, int ndesc
)
2208 if (rxr
->rx_base
!= NULL
) {
2209 bus_dmamap_unload(rxr
->rx_base_dtag
, rxr
->rx_base_map
);
2210 bus_dmamem_free(rxr
->rx_base_dtag
, rxr
->rx_base
,
2212 bus_dma_tag_destroy(rxr
->rx_base_dtag
);
2213 rxr
->rx_base
= NULL
;
2216 if (rxr
->rx_buf
== NULL
)
2219 for (i
= 0; i
< ndesc
; ++i
) {
2220 struct ix_rx_buf
*rxbuf
= &rxr
->rx_buf
[i
];
2222 KKASSERT(rxbuf
->m_head
== NULL
);
2223 bus_dmamap_destroy(rxr
->rx_tag
, rxbuf
->map
);
2225 bus_dmamap_destroy(rxr
->rx_tag
, rxr
->rx_sparemap
);
2226 bus_dma_tag_destroy(rxr
->rx_tag
);
2228 kfree(rxr
->rx_buf
, M_DEVBUF
);
2233 ** Used to detect a descriptor that has
2234 ** been merged by Hardware RSC.
2236 static __inline
uint32_t
2237 ix_rsc_count(union ixgbe_adv_rx_desc
*rx
)
2239 return (le32toh(rx
->wb
.lower
.lo_dword
.data
) &
2240 IXGBE_RXDADV_RSCCNT_MASK
) >> IXGBE_RXDADV_RSCCNT_SHIFT
;
2244 /*********************************************************************
2246 * Initialize Hardware RSC (LRO) feature on 82599
2247 * for an RX ring, this is toggled by the LRO capability
2248 * even though it is transparent to the stack.
2250 * NOTE: since this HW feature only works with IPV4 and
2251 * our testing has shown soft LRO to be as effective
2252 * I have decided to disable this by default.
2254 **********************************************************************/
2256 ix_setup_hw_rsc(struct ix_rx_ring
*rxr
)
2258 struct ix_softc
*sc
= rxr
->rx_sc
;
2259 struct ixgbe_hw
*hw
= &sc
->hw
;
2260 uint32_t rscctrl
, rdrxctl
;
2263 /* If turning LRO/RSC off we need to disable it */
2264 if ((sc
->arpcom
.ac_if
.if_capenable
& IFCAP_LRO
) == 0) {
2265 rscctrl
= IXGBE_READ_REG(hw
, IXGBE_RSCCTL(rxr
->me
));
2266 rscctrl
&= ~IXGBE_RSCCTL_RSCEN
;
2271 rdrxctl
= IXGBE_READ_REG(hw
, IXGBE_RDRXCTL
);
2272 rdrxctl
&= ~IXGBE_RDRXCTL_RSCFRSTSIZE
;
2273 rdrxctl
|= IXGBE_RDRXCTL_CRCSTRIP
;
2274 rdrxctl
|= IXGBE_RDRXCTL_RSCACKC
;
2275 IXGBE_WRITE_REG(hw
, IXGBE_RDRXCTL
, rdrxctl
);
2277 rscctrl
= IXGBE_READ_REG(hw
, IXGBE_RSCCTL(rxr
->me
));
2278 rscctrl
|= IXGBE_RSCCTL_RSCEN
;
2280 ** Limit the total number of descriptors that
2281 ** can be combined, so it does not exceed 64K
2283 if (rxr
->mbuf_sz
== MCLBYTES
)
2284 rscctrl
|= IXGBE_RSCCTL_MAXDESC_16
;
2285 else if (rxr
->mbuf_sz
== MJUMPAGESIZE
)
2286 rscctrl
|= IXGBE_RSCCTL_MAXDESC_8
;
2287 else if (rxr
->mbuf_sz
== MJUM9BYTES
)
2288 rscctrl
|= IXGBE_RSCCTL_MAXDESC_4
;
2289 else /* Using 16K cluster */
2290 rscctrl
|= IXGBE_RSCCTL_MAXDESC_1
;
2292 IXGBE_WRITE_REG(hw
, IXGBE_RSCCTL(rxr
->me
), rscctrl
);
2294 /* Enable TCP header recognition */
2295 IXGBE_WRITE_REG(hw
, IXGBE_PSRTYPE(0),
2296 (IXGBE_READ_REG(hw
, IXGBE_PSRTYPE(0)) |
2297 IXGBE_PSRTYPE_TCPHDR
));
2299 /* Disable RSC for ACK packets */
2300 IXGBE_WRITE_REG(hw
, IXGBE_RSCDBU
,
2301 (IXGBE_RSCDBU_RSCACKDIS
| IXGBE_READ_REG(hw
, IXGBE_RSCDBU
)));
2308 ix_init_rx_ring(struct ix_rx_ring
*rxr
)
2312 /* Clear the ring contents */
2313 bzero(rxr
->rx_base
, rxr
->rx_ndesc
* sizeof(union ixgbe_adv_rx_desc
));
2315 /* XXX we need JUMPAGESIZE for RSC too */
2316 if (rxr
->rx_sc
->max_frame_size
<= MCLBYTES
)
2317 rxr
->rx_mbuf_sz
= MCLBYTES
;
2319 rxr
->rx_mbuf_sz
= MJUMPAGESIZE
;
2321 /* Now replenish the mbufs */
2322 for (i
= 0; i
< rxr
->rx_ndesc
; ++i
) {
2325 error
= ix_newbuf(rxr
, i
, TRUE
);
2330 /* Setup our descriptor indices */
2331 rxr
->rx_next_check
= 0;
2332 rxr
->rx_flags
&= ~IX_RXRING_FLAG_DISC
;
2336 ** Now set up the LRO interface:
2338 if (ixgbe_rsc_enable
)
2339 ix_setup_hw_rsc(rxr
);
2345 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2347 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
2350 ix_init_rx_unit(struct ix_softc
*sc
)
2352 struct ixgbe_hw
*hw
= &sc
->hw
;
2353 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
2354 uint32_t bufsz
, rxctrl
, fctrl
, rxcsum
, hlreg
;
2358 * Make sure receives are disabled while setting up the descriptor ring
2360 rxctrl
= IXGBE_READ_REG(hw
, IXGBE_RXCTRL
);
2361 IXGBE_WRITE_REG(hw
, IXGBE_RXCTRL
, rxctrl
& ~IXGBE_RXCTRL_RXEN
);
2363 /* Enable broadcasts */
2364 fctrl
= IXGBE_READ_REG(hw
, IXGBE_FCTRL
);
2365 fctrl
|= IXGBE_FCTRL_BAM
;
2366 fctrl
|= IXGBE_FCTRL_DPF
;
2367 fctrl
|= IXGBE_FCTRL_PMCF
;
2368 IXGBE_WRITE_REG(hw
, IXGBE_FCTRL
, fctrl
);
2370 /* Set for Jumbo Frames? */
2371 hlreg
= IXGBE_READ_REG(hw
, IXGBE_HLREG0
);
2372 if (ifp
->if_mtu
> ETHERMTU
)
2373 hlreg
|= IXGBE_HLREG0_JUMBOEN
;
2375 hlreg
&= ~IXGBE_HLREG0_JUMBOEN
;
2376 IXGBE_WRITE_REG(hw
, IXGBE_HLREG0
, hlreg
);
2378 KKASSERT(sc
->rx_rings
[0].rx_mbuf_sz
>= MCLBYTES
);
2379 bufsz
= (sc
->rx_rings
[0].rx_mbuf_sz
+ BSIZEPKT_ROUNDUP
) >>
2380 IXGBE_SRRCTL_BSIZEPKT_SHIFT
;
2382 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
2383 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
2384 uint64_t rdba
= rxr
->rx_base_paddr
;
2387 /* Setup the Base and Length of the Rx Descriptor Ring */
2388 IXGBE_WRITE_REG(hw
, IXGBE_RDBAL(i
), (uint32_t)rdba
);
2389 IXGBE_WRITE_REG(hw
, IXGBE_RDBAH(i
), (uint32_t)(rdba
>> 32));
2390 IXGBE_WRITE_REG(hw
, IXGBE_RDLEN(i
),
2391 rxr
->rx_ndesc
* sizeof(union ixgbe_adv_rx_desc
));
2394 * Set up the SRRCTL register
2396 srrctl
= IXGBE_READ_REG(hw
, IXGBE_SRRCTL(i
));
2398 srrctl
&= ~IXGBE_SRRCTL_BSIZEHDR_MASK
;
2399 srrctl
&= ~IXGBE_SRRCTL_BSIZEPKT_MASK
;
2401 srrctl
|= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF
;
2402 if (sc
->rx_ring_inuse
> 1) {
2403 /* See the commend near ix_enable_rx_drop() */
2405 case ixgbe_fc_rx_pause
:
2406 case ixgbe_fc_tx_pause
:
2408 srrctl
&= ~IXGBE_SRRCTL_DROP_EN
;
2409 if (i
== 0 && bootverbose
) {
2410 if_printf(ifp
, "flow control %d, "
2411 "disable RX drop\n", sc
->fc
);
2416 srrctl
|= IXGBE_SRRCTL_DROP_EN
;
2417 if (i
== 0 && bootverbose
) {
2418 if_printf(ifp
, "flow control %d, "
2419 "enable RX drop\n", sc
->fc
);
2427 IXGBE_WRITE_REG(hw
, IXGBE_SRRCTL(i
), srrctl
);
2429 /* Setup the HW Rx Head and Tail Descriptor Pointers */
2430 IXGBE_WRITE_REG(hw
, IXGBE_RDH(i
), 0);
2431 IXGBE_WRITE_REG(hw
, IXGBE_RDT(i
), 0);
2434 if (sc
->hw
.mac
.type
!= ixgbe_mac_82598EB
)
2435 IXGBE_WRITE_REG(hw
, IXGBE_PSRTYPE(0), 0);
2437 rxcsum
= IXGBE_READ_REG(hw
, IXGBE_RXCSUM
);
2442 if (IX_ENABLE_HWRSS(sc
)) {
2443 uint8_t key
[IX_NRSSRK
* IX_RSSRK_SIZE
];
2448 * When we reach here, RSS has already been disabled
2449 * in ix_stop(), so we could safely configure RSS key
2450 * and redirect table.
2456 toeplitz_get_key(key
, sizeof(key
));
2457 for (i
= 0; i
< IX_NRSSRK
; ++i
) {
2460 rssrk
= IX_RSSRK_VAL(key
, i
);
2461 IX_RSS_DPRINTF(sc
, 1, "rssrk%d 0x%08x\n",
2464 IXGBE_WRITE_REG(hw
, IXGBE_RSSRK(i
), rssrk
);
2468 * Configure RSS redirect table in following fashion:
2469 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
2472 for (j
= 0; j
< IX_NRETA
; ++j
) {
2475 for (i
= 0; i
< IX_RETA_SIZE
; ++i
) {
2478 q
= r
% sc
->rx_ring_inuse
;
2479 reta
|= q
<< (8 * i
);
2482 IX_RSS_DPRINTF(sc
, 1, "reta 0x%08x\n", reta
);
2483 IXGBE_WRITE_REG(hw
, IXGBE_RETA(j
), reta
);
2487 * Enable multiple receive queues.
2488 * Enable IPv4 RSS standard hash functions.
2490 IXGBE_WRITE_REG(hw
, IXGBE_MRQC
,
2492 IXGBE_MRQC_RSS_FIELD_IPV4
|
2493 IXGBE_MRQC_RSS_FIELD_IPV4_TCP
);
2497 * PCSD must be enabled to enable multiple
2500 rxcsum
|= IXGBE_RXCSUM_PCSD
;
2503 if (ifp
->if_capenable
& IFCAP_RXCSUM
)
2504 rxcsum
|= IXGBE_RXCSUM_PCSD
;
2506 IXGBE_WRITE_REG(hw
, IXGBE_RXCSUM
, rxcsum
);
2509 static __inline
void
2510 ix_rx_refresh(struct ix_rx_ring
*rxr
, int i
)
2513 i
= rxr
->rx_ndesc
- 1;
2514 IXGBE_WRITE_REG(&rxr
->rx_sc
->hw
, IXGBE_RDT(rxr
->rx_idx
), i
);
2517 static __inline
void
2518 ix_rxcsum(uint32_t staterr
, struct mbuf
*mp
, uint32_t ptype
)
2521 (IXGBE_RXDADV_PKTTYPE_IPV4
| IXGBE_RXDADV_PKTTYPE_IPV4_EX
)) == 0) {
2526 if ((staterr
& (IXGBE_RXD_STAT_IPCS
| IXGBE_RXDADV_ERR_IPE
)) ==
2527 IXGBE_RXD_STAT_IPCS
)
2528 mp
->m_pkthdr
.csum_flags
|= CSUM_IP_CHECKED
| CSUM_IP_VALID
;
2531 (IXGBE_RXDADV_PKTTYPE_TCP
| IXGBE_RXDADV_PKTTYPE_UDP
)) == 0) {
2533 * - Neither TCP nor UDP
2539 if ((staterr
& (IXGBE_RXD_STAT_L4CS
| IXGBE_RXDADV_ERR_TCPE
)) ==
2540 IXGBE_RXD_STAT_L4CS
) {
2541 mp
->m_pkthdr
.csum_flags
|= CSUM_DATA_VALID
| CSUM_PSEUDO_HDR
|
2542 CSUM_FRAG_NOT_CHECKED
;
2543 mp
->m_pkthdr
.csum_data
= htons(0xffff);
2547 static __inline
struct pktinfo
*
2548 ix_rssinfo(struct mbuf
*m
, struct pktinfo
*pi
,
2549 uint32_t hash
, uint32_t hashtype
, uint32_t ptype
)
2552 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP
:
2553 pi
->pi_netisr
= NETISR_IP
;
2555 pi
->pi_l3proto
= IPPROTO_TCP
;
2558 case IXGBE_RXDADV_RSSTYPE_IPV4
:
2559 if ((ptype
& IXGBE_RXDADV_PKTTYPE_UDP
) == 0) {
2560 /* Not UDP or is fragment */
2563 pi
->pi_netisr
= NETISR_IP
;
2565 pi
->pi_l3proto
= IPPROTO_UDP
;
2572 m
->m_flags
|= M_HASH
;
2573 m
->m_pkthdr
.hash
= toeplitz_hash(hash
);
2577 static __inline
void
2578 ix_setup_rxdesc(union ixgbe_adv_rx_desc
*rxd
, const struct ix_rx_buf
*rxbuf
)
2580 rxd
->read
.pkt_addr
= htole64(rxbuf
->paddr
);
2581 rxd
->wb
.upper
.status_error
= 0;
2585 ix_rx_discard(struct ix_rx_ring
*rxr
, int i
, boolean_t eop
)
2587 struct ix_rx_buf
*rxbuf
= &rxr
->rx_buf
[i
];
2590 * XXX discard may not be correct
2593 IFNET_STAT_INC(&rxr
->rx_sc
->arpcom
.ac_if
, ierrors
, 1);
2594 rxr
->rx_flags
&= ~IX_RXRING_FLAG_DISC
;
2596 rxr
->rx_flags
|= IX_RXRING_FLAG_DISC
;
2598 if (rxbuf
->fmp
!= NULL
) {
2599 m_freem(rxbuf
->fmp
);
2603 ix_setup_rxdesc(&rxr
->rx_base
[i
], rxbuf
);
2607 ix_rxeof(struct ix_rx_ring
*rxr
, int count
)
2609 struct ifnet
*ifp
= &rxr
->rx_sc
->arpcom
.ac_if
;
2610 int i
, nsegs
= 0, cpuid
= mycpuid
;
2612 i
= rxr
->rx_next_check
;
2613 while (count
!= 0) {
2614 struct ix_rx_buf
*rxbuf
, *nbuf
= NULL
;
2615 union ixgbe_adv_rx_desc
*cur
;
2616 struct mbuf
*sendmp
= NULL
, *mp
;
2617 struct pktinfo
*pi
= NULL
, pi0
;
2618 uint32_t rsc
= 0, ptype
, staterr
, hash
, hashtype
;
2622 cur
= &rxr
->rx_base
[i
];
2623 staterr
= le32toh(cur
->wb
.upper
.status_error
);
2625 if ((staterr
& IXGBE_RXD_STAT_DD
) == 0)
2629 rxbuf
= &rxr
->rx_buf
[i
];
2632 len
= le16toh(cur
->wb
.upper
.length
);
2633 ptype
= le32toh(cur
->wb
.lower
.lo_dword
.data
) &
2634 IXGBE_RXDADV_PKTTYPE_MASK
;
2635 hash
= le32toh(cur
->wb
.lower
.hi_dword
.rss
);
2636 hashtype
= le32toh(cur
->wb
.lower
.lo_dword
.data
) &
2637 IXGBE_RXDADV_RSSTYPE_MASK
;
2639 eop
= ((staterr
& IXGBE_RXD_STAT_EOP
) != 0);
2644 * Make sure bad packets are discarded
2646 if ((staterr
& IXGBE_RXDADV_ERR_FRAME_ERR_MASK
) ||
2647 (rxr
->rx_flags
& IX_RXRING_FLAG_DISC
)) {
2648 ix_rx_discard(rxr
, i
, eop
);
2652 bus_dmamap_sync(rxr
->rx_tag
, rxbuf
->map
, BUS_DMASYNC_POSTREAD
);
2653 if (ix_newbuf(rxr
, i
, FALSE
) != 0) {
2654 ix_rx_discard(rxr
, i
, eop
);
2659 * On 82599 which supports a hardware LRO, packets
2660 * need not be fragmented across sequential descriptors,
2661 * rather the next descriptor is indicated in bits
2662 * of the descriptor. This also means that we might
2663 * proceses more than one packet at a time, something
2664 * that has never been true before, it required
2665 * eliminating global chain pointers in favor of what
2666 * we are doing here.
2672 * Figure out the next descriptor
2675 if (rxr
->rx_flags
& IX_RXRING_FLAG_LRO
)
2676 rsc
= ix_rsc_count(cur
);
2677 if (rsc
) { /* Get hardware index */
2679 IXGBE_RXDADV_NEXTP_MASK
) >>
2680 IXGBE_RXDADV_NEXTP_SHIFT
);
2681 } else { /* Just sequential */
2683 if (nextp
== rxr
->rx_ndesc
)
2686 nbuf
= &rxr
->rx_buf
[nextp
];
2692 * Rather than using the fmp/lmp global pointers
2693 * we now keep the head of a packet chain in the
2694 * buffer struct and pass this along from one
2695 * descriptor to the next, until we get EOP.
2697 if (rxbuf
->fmp
== NULL
) {
2698 mp
->m_pkthdr
.len
= len
;
2702 rxbuf
->fmp
->m_pkthdr
.len
+= len
;
2703 rxbuf
->lmp
->m_next
= mp
;
2709 * Not the last fragment of this frame,
2710 * pass this fragment list on
2712 nbuf
->fmp
= rxbuf
->fmp
;
2713 nbuf
->lmp
= rxbuf
->lmp
;
2718 sendmp
= rxbuf
->fmp
;
2720 sendmp
->m_pkthdr
.rcvif
= ifp
;
2721 IFNET_STAT_INC(ifp
, ipackets
, 1);
2726 /* Process vlan info */
2727 if (staterr
& IXGBE_RXD_STAT_VP
) {
2728 sendmp
->m_pkthdr
.ether_vlantag
=
2729 le16toh(cur
->wb
.upper
.vlan
);
2730 sendmp
->m_flags
|= M_VLANTAG
;
2732 if (ifp
->if_capenable
& IFCAP_RXCSUM
)
2733 ix_rxcsum(staterr
, sendmp
, ptype
);
2734 if (ifp
->if_capenable
& IFCAP_RSS
) {
2735 pi
= ix_rssinfo(sendmp
, &pi0
,
2736 hash
, hashtype
, ptype
);
2742 /* Advance our pointers to the next descriptor. */
2743 if (++i
== rxr
->rx_ndesc
)
2747 ifp
->if_input(ifp
, sendmp
, pi
, cpuid
);
2749 if (nsegs
>= rxr
->rx_wreg_nsegs
) {
2750 ix_rx_refresh(rxr
, i
);
2754 rxr
->rx_next_check
= i
;
2757 ix_rx_refresh(rxr
, i
);
2761 ix_set_vlan(struct ix_softc
*sc
)
2763 struct ixgbe_hw
*hw
= &sc
->hw
;
2766 if (hw
->mac
.type
== ixgbe_mac_82598EB
) {
2767 ctrl
= IXGBE_READ_REG(hw
, IXGBE_VLNCTRL
);
2768 ctrl
|= IXGBE_VLNCTRL_VME
;
2769 IXGBE_WRITE_REG(hw
, IXGBE_VLNCTRL
, ctrl
);
2774 * On 82599 and later chips the VLAN enable is
2775 * per queue in RXDCTL
2777 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
2778 ctrl
= IXGBE_READ_REG(hw
, IXGBE_RXDCTL(i
));
2779 ctrl
|= IXGBE_RXDCTL_VME
;
2780 IXGBE_WRITE_REG(hw
, IXGBE_RXDCTL(i
), ctrl
);
2786 ix_enable_intr(struct ix_softc
*sc
)
2788 struct ixgbe_hw
*hw
= &sc
->hw
;
2792 for (i
= 0; i
< sc
->intr_cnt
; ++i
)
2793 lwkt_serialize_handler_enable(sc
->intr_data
[i
].intr_serialize
);
2795 sc
->intr_mask
= (IXGBE_EIMS_ENABLE_MASK
& ~IXGBE_EIMS_RTX_QUEUE
);
2797 /* Enable Fan Failure detection */
2798 if (hw
->device_id
== IXGBE_DEV_ID_82598AT
)
2799 sc
->intr_mask
|= IXGBE_EIMS_GPI_SDP1
;
2801 switch (sc
->hw
.mac
.type
) {
2802 case ixgbe_mac_82599EB
:
2803 sc
->intr_mask
|= IXGBE_EIMS_ECC
;
2804 sc
->intr_mask
|= IXGBE_EIMS_GPI_SDP0
;
2805 sc
->intr_mask
|= IXGBE_EIMS_GPI_SDP1
;
2806 sc
->intr_mask
|= IXGBE_EIMS_GPI_SDP2
;
2809 case ixgbe_mac_X540
:
2810 sc
->intr_mask
|= IXGBE_EIMS_ECC
;
2811 /* Detect if Thermal Sensor is enabled */
2812 fwsm
= IXGBE_READ_REG(hw
, IXGBE_FWSM
);
2813 if (fwsm
& IXGBE_FWSM_TS_ENABLED
)
2814 sc
->intr_mask
|= IXGBE_EIMS_TS
;
2820 /* With MSI-X we use auto clear for RX and TX rings */
2821 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
) {
2823 * There are no EIAC1/EIAC2 for newer chips; the related
2824 * bits for TX and RX rings > 16 are always auto clear.
2826 * XXX which bits? There are _no_ documented EICR1 and
2827 * EICR2 at all; only EICR.
2829 IXGBE_WRITE_REG(hw
, IXGBE_EIAC
, IXGBE_EIMS_RTX_QUEUE
);
2831 sc
->intr_mask
|= IX_TX_INTR_MASK
| IX_RX0_INTR_MASK
;
2833 KKASSERT(sc
->rx_ring_inuse
<= IX_MIN_RXRING_RSS
);
2834 if (sc
->rx_ring_inuse
== IX_MIN_RXRING_RSS
)
2835 sc
->intr_mask
|= IX_RX1_INTR_MASK
;
2838 IXGBE_WRITE_REG(hw
, IXGBE_EIMS
, sc
->intr_mask
);
2841 * Enable RX and TX rings for MSI-X
2843 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
) {
2844 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
2845 const struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
2847 if (txr
->tx_intr_vec
>= 0) {
2848 IXGBE_WRITE_REG(hw
, txr
->tx_eims
,
2852 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
2853 const struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
2855 KKASSERT(rxr
->rx_intr_vec
>= 0);
2856 IXGBE_WRITE_REG(hw
, rxr
->rx_eims
, rxr
->rx_eims_val
);
2860 IXGBE_WRITE_FLUSH(hw
);
2864 ix_disable_intr(struct ix_softc
*sc
)
2868 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
)
2869 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIAC
, 0);
2871 if (sc
->hw
.mac
.type
== ixgbe_mac_82598EB
) {
2872 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIMC
, ~0);
2874 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIMC
, 0xFFFF0000);
2875 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIMC_EX(0), ~0);
2876 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIMC_EX(1), ~0);
2878 IXGBE_WRITE_FLUSH(&sc
->hw
);
2880 for (i
= 0; i
< sc
->intr_cnt
; ++i
)
2881 lwkt_serialize_handler_disable(sc
->intr_data
[i
].intr_serialize
);
2885 ixgbe_read_pci_cfg(struct ixgbe_hw
*hw
, uint32_t reg
)
2887 return pci_read_config(((struct ixgbe_osdep
*)hw
->back
)->dev
,
2892 ixgbe_write_pci_cfg(struct ixgbe_hw
*hw
, uint32_t reg
, uint16_t value
)
2894 pci_write_config(((struct ixgbe_osdep
*)hw
->back
)->dev
,
2899 ix_slot_info(struct ix_softc
*sc
)
2901 struct ixgbe_hw
*hw
= &sc
->hw
;
2902 device_t dev
= sc
->dev
;
2903 struct ixgbe_mac_info
*mac
= &hw
->mac
;
2907 /* For most devices simply call the shared code routine */
2908 if (hw
->device_id
!= IXGBE_DEV_ID_82599_SFP_SF_QP
) {
2909 ixgbe_get_bus_info(hw
);
2914 * For the Quad port adapter we need to parse back
2915 * up the PCI tree to find the speed of the expansion
2916 * slot into which this adapter is plugged. A bit more work.
2918 dev
= device_get_parent(device_get_parent(dev
));
2920 device_printf(dev
, "parent pcib = %x,%x,%x\n",
2921 pci_get_bus(dev
), pci_get_slot(dev
), pci_get_function(dev
));
2923 dev
= device_get_parent(device_get_parent(dev
));
2925 device_printf(dev
, "slot pcib = %x,%x,%x\n",
2926 pci_get_bus(dev
), pci_get_slot(dev
), pci_get_function(dev
));
2928 /* Now get the PCI Express Capabilities offset */
2929 offset
= pci_get_pciecap_ptr(dev
);
2930 /* ...and read the Link Status Register */
2931 link
= pci_read_config(dev
, offset
+ PCIER_LINKSTAT
, 2);
2932 switch (link
& IXGBE_PCI_LINK_WIDTH
) {
2933 case IXGBE_PCI_LINK_WIDTH_1
:
2934 hw
->bus
.width
= ixgbe_bus_width_pcie_x1
;
2936 case IXGBE_PCI_LINK_WIDTH_2
:
2937 hw
->bus
.width
= ixgbe_bus_width_pcie_x2
;
2939 case IXGBE_PCI_LINK_WIDTH_4
:
2940 hw
->bus
.width
= ixgbe_bus_width_pcie_x4
;
2942 case IXGBE_PCI_LINK_WIDTH_8
:
2943 hw
->bus
.width
= ixgbe_bus_width_pcie_x8
;
2946 hw
->bus
.width
= ixgbe_bus_width_unknown
;
2950 switch (link
& IXGBE_PCI_LINK_SPEED
) {
2951 case IXGBE_PCI_LINK_SPEED_2500
:
2952 hw
->bus
.speed
= ixgbe_bus_speed_2500
;
2954 case IXGBE_PCI_LINK_SPEED_5000
:
2955 hw
->bus
.speed
= ixgbe_bus_speed_5000
;
2957 case IXGBE_PCI_LINK_SPEED_8000
:
2958 hw
->bus
.speed
= ixgbe_bus_speed_8000
;
2961 hw
->bus
.speed
= ixgbe_bus_speed_unknown
;
2965 mac
->ops
.set_lan_id(hw
);
2968 device_printf(dev
, "PCI Express Bus: Speed %s %s\n",
2969 hw
->bus
.speed
== ixgbe_bus_speed_8000
? "8.0GT/s" :
2970 hw
->bus
.speed
== ixgbe_bus_speed_5000
? "5.0GT/s" :
2971 hw
->bus
.speed
== ixgbe_bus_speed_2500
? "2.5GT/s" : "Unknown",
2972 hw
->bus
.width
== ixgbe_bus_width_pcie_x8
? "Width x8" :
2973 hw
->bus
.width
== ixgbe_bus_width_pcie_x4
? "Width x4" :
2974 hw
->bus
.width
== ixgbe_bus_width_pcie_x1
? "Width x1" : "Unknown");
2976 if (hw
->device_id
!= IXGBE_DEV_ID_82599_SFP_SF_QP
&&
2977 hw
->bus
.width
<= ixgbe_bus_width_pcie_x4
&&
2978 hw
->bus
.speed
== ixgbe_bus_speed_2500
) {
2979 device_printf(dev
, "For optimal performance a x8 "
2980 "PCIE, or x4 PCIE Gen2 slot is required.\n");
2981 } else if (hw
->device_id
== IXGBE_DEV_ID_82599_SFP_SF_QP
&&
2982 hw
->bus
.width
<= ixgbe_bus_width_pcie_x8
&&
2983 hw
->bus
.speed
< ixgbe_bus_speed_8000
) {
2984 device_printf(dev
, "For optimal performance a x8 "
2985 "PCIE Gen3 slot is required.\n");
2990 * TODO comment is incorrect
2992 * Setup the correct IVAR register for a particular MSIX interrupt
2993 * - entry is the register array entry
2994 * - vector is the MSIX vector for this queue
2995 * - type is RX/TX/MISC
2998 ix_set_ivar(struct ix_softc
*sc
, uint8_t entry
, uint8_t vector
,
3001 struct ixgbe_hw
*hw
= &sc
->hw
;
3002 uint32_t ivar
, index
;
3004 vector
|= IXGBE_IVAR_ALLOC_VAL
;
3006 switch (hw
->mac
.type
) {
3007 case ixgbe_mac_82598EB
:
3009 entry
= IXGBE_IVAR_OTHER_CAUSES_INDEX
;
3011 entry
+= (type
* 64);
3012 index
= (entry
>> 2) & 0x1F;
3013 ivar
= IXGBE_READ_REG(hw
, IXGBE_IVAR(index
));
3014 ivar
&= ~(0xFF << (8 * (entry
& 0x3)));
3015 ivar
|= (vector
<< (8 * (entry
& 0x3)));
3016 IXGBE_WRITE_REG(hw
, IXGBE_IVAR(index
), ivar
);
3019 case ixgbe_mac_82599EB
:
3020 case ixgbe_mac_X540
:
3021 if (type
== -1) { /* MISC IVAR */
3022 index
= (entry
& 1) * 8;
3023 ivar
= IXGBE_READ_REG(hw
, IXGBE_IVAR_MISC
);
3024 ivar
&= ~(0xFF << index
);
3025 ivar
|= (vector
<< index
);
3026 IXGBE_WRITE_REG(hw
, IXGBE_IVAR_MISC
, ivar
);
3027 } else { /* RX/TX IVARS */
3028 index
= (16 * (entry
& 1)) + (8 * type
);
3029 ivar
= IXGBE_READ_REG(hw
, IXGBE_IVAR(entry
>> 1));
3030 ivar
&= ~(0xFF << index
);
3031 ivar
|= (vector
<< index
);
3032 IXGBE_WRITE_REG(hw
, IXGBE_IVAR(entry
>> 1), ivar
);
3041 ix_sfp_probe(struct ix_softc
*sc
)
3043 struct ixgbe_hw
*hw
= &sc
->hw
;
3045 if (hw
->phy
.type
== ixgbe_phy_nl
&&
3046 hw
->phy
.sfp_type
== ixgbe_sfp_type_not_present
) {
3049 ret
= hw
->phy
.ops
.identify_sfp(hw
);
3053 ret
= hw
->phy
.ops
.reset(hw
);
3054 if (ret
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
3055 if_printf(&sc
->arpcom
.ac_if
,
3056 "Unsupported SFP+ module detected! "
3057 "Reload driver with supported module.\n");
3058 sc
->sfp_probe
= FALSE
;
3061 if_printf(&sc
->arpcom
.ac_if
, "SFP+ module detected!\n");
3063 /* We now have supported optics */
3064 sc
->sfp_probe
= FALSE
;
3065 /* Set the optics type so system reports correctly */
3066 ix_setup_optics(sc
);
3074 ix_handle_link(struct ix_softc
*sc
)
3076 ixgbe_check_link(&sc
->hw
, &sc
->link_speed
, &sc
->link_up
, 0);
3077 ix_update_link_status(sc
);
3081 * Handling SFP module
3084 ix_handle_mod(struct ix_softc
*sc
)
3086 struct ixgbe_hw
*hw
= &sc
->hw
;
3089 err
= hw
->phy
.ops
.identify_sfp(hw
);
3090 if (err
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
3091 if_printf(&sc
->arpcom
.ac_if
,
3092 "Unsupported SFP+ module type was detected.\n");
3095 err
= hw
->mac
.ops
.setup_sfp(hw
);
3096 if (err
== IXGBE_ERR_SFP_NOT_SUPPORTED
) {
3097 if_printf(&sc
->arpcom
.ac_if
,
3098 "Setup failure - unsupported SFP+ module type.\n");
3105 * Handling MSF (multispeed fiber)
3108 ix_handle_msf(struct ix_softc
*sc
)
3110 struct ixgbe_hw
*hw
= &sc
->hw
;
3113 autoneg
= hw
->phy
.autoneg_advertised
;
3114 if (!autoneg
&& hw
->mac
.ops
.get_link_capabilities
!= NULL
) {
3117 hw
->mac
.ops
.get_link_capabilities(hw
, &autoneg
, &negotiate
);
3119 if (hw
->mac
.ops
.setup_link
!= NULL
)
3120 hw
->mac
.ops
.setup_link(hw
, autoneg
, TRUE
);
3124 ix_update_stats(struct ix_softc
*sc
)
3126 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3127 struct ixgbe_hw
*hw
= &sc
->hw
;
3128 uint32_t missed_rx
= 0, bprc
, lxon
, lxoff
, total
;
3129 uint64_t total_missed_rx
= 0;
3132 sc
->stats
.crcerrs
+= IXGBE_READ_REG(hw
, IXGBE_CRCERRS
);
3133 sc
->stats
.illerrc
+= IXGBE_READ_REG(hw
, IXGBE_ILLERRC
);
3134 sc
->stats
.errbc
+= IXGBE_READ_REG(hw
, IXGBE_ERRBC
);
3135 sc
->stats
.mspdc
+= IXGBE_READ_REG(hw
, IXGBE_MSPDC
);
3138 * Note: These are for the 8 possible traffic classes, which
3139 * in current implementation is unused, therefore only 0 should
3142 for (i
= 0; i
< 8; i
++) {
3145 mp
= IXGBE_READ_REG(hw
, IXGBE_MPC(i
));
3146 /* missed_rx tallies misses for the gprc workaround */
3148 /* global total per queue */
3149 sc
->stats
.mpc
[i
] += mp
;
3151 /* Running comprehensive total for stats display */
3152 total_missed_rx
+= sc
->stats
.mpc
[i
];
3154 if (hw
->mac
.type
== ixgbe_mac_82598EB
) {
3155 sc
->stats
.rnbc
[i
] += IXGBE_READ_REG(hw
, IXGBE_RNBC(i
));
3156 sc
->stats
.qbtc
[i
] += IXGBE_READ_REG(hw
, IXGBE_QBTC(i
));
3157 sc
->stats
.qbrc
[i
] += IXGBE_READ_REG(hw
, IXGBE_QBRC(i
));
3158 sc
->stats
.pxonrxc
[i
] +=
3159 IXGBE_READ_REG(hw
, IXGBE_PXONRXC(i
));
3161 sc
->stats
.pxonrxc
[i
] +=
3162 IXGBE_READ_REG(hw
, IXGBE_PXONRXCNT(i
));
3164 sc
->stats
.pxontxc
[i
] += IXGBE_READ_REG(hw
, IXGBE_PXONTXC(i
));
3165 sc
->stats
.pxofftxc
[i
] += IXGBE_READ_REG(hw
, IXGBE_PXOFFTXC(i
));
3166 sc
->stats
.pxoffrxc
[i
] += IXGBE_READ_REG(hw
, IXGBE_PXOFFRXC(i
));
3167 sc
->stats
.pxon2offc
[i
] +=
3168 IXGBE_READ_REG(hw
, IXGBE_PXON2OFFCNT(i
));
3170 for (i
= 0; i
< 16; i
++) {
3171 sc
->stats
.qprc
[i
] += IXGBE_READ_REG(hw
, IXGBE_QPRC(i
));
3172 sc
->stats
.qptc
[i
] += IXGBE_READ_REG(hw
, IXGBE_QPTC(i
));
3173 sc
->stats
.qprdc
[i
] += IXGBE_READ_REG(hw
, IXGBE_QPRDC(i
));
3175 sc
->stats
.mlfc
+= IXGBE_READ_REG(hw
, IXGBE_MLFC
);
3176 sc
->stats
.mrfc
+= IXGBE_READ_REG(hw
, IXGBE_MRFC
);
3177 sc
->stats
.rlec
+= IXGBE_READ_REG(hw
, IXGBE_RLEC
);
3179 /* Hardware workaround, gprc counts missed packets */
3180 sc
->stats
.gprc
+= IXGBE_READ_REG(hw
, IXGBE_GPRC
);
3181 sc
->stats
.gprc
-= missed_rx
;
3183 if (hw
->mac
.type
!= ixgbe_mac_82598EB
) {
3184 sc
->stats
.gorc
+= IXGBE_READ_REG(hw
, IXGBE_GORCL
) +
3185 ((uint64_t)IXGBE_READ_REG(hw
, IXGBE_GORCH
) << 32);
3186 sc
->stats
.gotc
+= IXGBE_READ_REG(hw
, IXGBE_GOTCL
) +
3187 ((uint64_t)IXGBE_READ_REG(hw
, IXGBE_GOTCH
) << 32);
3188 sc
->stats
.tor
+= IXGBE_READ_REG(hw
, IXGBE_TORL
) +
3189 ((uint64_t)IXGBE_READ_REG(hw
, IXGBE_TORH
) << 32);
3190 sc
->stats
.lxonrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXONRXCNT
);
3191 sc
->stats
.lxoffrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXOFFRXCNT
);
3193 sc
->stats
.lxonrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXONRXC
);
3194 sc
->stats
.lxoffrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXOFFRXC
);
3195 /* 82598 only has a counter in the high register */
3196 sc
->stats
.gorc
+= IXGBE_READ_REG(hw
, IXGBE_GORCH
);
3197 sc
->stats
.gotc
+= IXGBE_READ_REG(hw
, IXGBE_GOTCH
);
3198 sc
->stats
.tor
+= IXGBE_READ_REG(hw
, IXGBE_TORH
);
3202 * Workaround: mprc hardware is incorrectly counting
3203 * broadcasts, so for now we subtract those.
3205 bprc
= IXGBE_READ_REG(hw
, IXGBE_BPRC
);
3206 sc
->stats
.bprc
+= bprc
;
3207 sc
->stats
.mprc
+= IXGBE_READ_REG(hw
, IXGBE_MPRC
);
3208 if (hw
->mac
.type
== ixgbe_mac_82598EB
)
3209 sc
->stats
.mprc
-= bprc
;
3211 sc
->stats
.prc64
+= IXGBE_READ_REG(hw
, IXGBE_PRC64
);
3212 sc
->stats
.prc127
+= IXGBE_READ_REG(hw
, IXGBE_PRC127
);
3213 sc
->stats
.prc255
+= IXGBE_READ_REG(hw
, IXGBE_PRC255
);
3214 sc
->stats
.prc511
+= IXGBE_READ_REG(hw
, IXGBE_PRC511
);
3215 sc
->stats
.prc1023
+= IXGBE_READ_REG(hw
, IXGBE_PRC1023
);
3216 sc
->stats
.prc1522
+= IXGBE_READ_REG(hw
, IXGBE_PRC1522
);
3218 lxon
= IXGBE_READ_REG(hw
, IXGBE_LXONTXC
);
3219 sc
->stats
.lxontxc
+= lxon
;
3220 lxoff
= IXGBE_READ_REG(hw
, IXGBE_LXOFFTXC
);
3221 sc
->stats
.lxofftxc
+= lxoff
;
3222 total
= lxon
+ lxoff
;
3224 sc
->stats
.gptc
+= IXGBE_READ_REG(hw
, IXGBE_GPTC
);
3225 sc
->stats
.mptc
+= IXGBE_READ_REG(hw
, IXGBE_MPTC
);
3226 sc
->stats
.ptc64
+= IXGBE_READ_REG(hw
, IXGBE_PTC64
);
3227 sc
->stats
.gptc
-= total
;
3228 sc
->stats
.mptc
-= total
;
3229 sc
->stats
.ptc64
-= total
;
3230 sc
->stats
.gotc
-= total
* ETHER_MIN_LEN
;
3232 sc
->stats
.ruc
+= IXGBE_READ_REG(hw
, IXGBE_RUC
);
3233 sc
->stats
.rfc
+= IXGBE_READ_REG(hw
, IXGBE_RFC
);
3234 sc
->stats
.roc
+= IXGBE_READ_REG(hw
, IXGBE_ROC
);
3235 sc
->stats
.rjc
+= IXGBE_READ_REG(hw
, IXGBE_RJC
);
3236 sc
->stats
.mngprc
+= IXGBE_READ_REG(hw
, IXGBE_MNGPRC
);
3237 sc
->stats
.mngpdc
+= IXGBE_READ_REG(hw
, IXGBE_MNGPDC
);
3238 sc
->stats
.mngptc
+= IXGBE_READ_REG(hw
, IXGBE_MNGPTC
);
3239 sc
->stats
.tpr
+= IXGBE_READ_REG(hw
, IXGBE_TPR
);
3240 sc
->stats
.tpt
+= IXGBE_READ_REG(hw
, IXGBE_TPT
);
3241 sc
->stats
.ptc127
+= IXGBE_READ_REG(hw
, IXGBE_PTC127
);
3242 sc
->stats
.ptc255
+= IXGBE_READ_REG(hw
, IXGBE_PTC255
);
3243 sc
->stats
.ptc511
+= IXGBE_READ_REG(hw
, IXGBE_PTC511
);
3244 sc
->stats
.ptc1023
+= IXGBE_READ_REG(hw
, IXGBE_PTC1023
);
3245 sc
->stats
.ptc1522
+= IXGBE_READ_REG(hw
, IXGBE_PTC1522
);
3246 sc
->stats
.bptc
+= IXGBE_READ_REG(hw
, IXGBE_BPTC
);
3247 sc
->stats
.xec
+= IXGBE_READ_REG(hw
, IXGBE_XEC
);
3248 sc
->stats
.fccrc
+= IXGBE_READ_REG(hw
, IXGBE_FCCRC
);
3249 sc
->stats
.fclast
+= IXGBE_READ_REG(hw
, IXGBE_FCLAST
);
3250 /* Only read FCOE on 82599 */
3251 if (hw
->mac
.type
!= ixgbe_mac_82598EB
) {
3252 sc
->stats
.fcoerpdc
+= IXGBE_READ_REG(hw
, IXGBE_FCOERPDC
);
3253 sc
->stats
.fcoeprc
+= IXGBE_READ_REG(hw
, IXGBE_FCOEPRC
);
3254 sc
->stats
.fcoeptc
+= IXGBE_READ_REG(hw
, IXGBE_FCOEPTC
);
3255 sc
->stats
.fcoedwrc
+= IXGBE_READ_REG(hw
, IXGBE_FCOEDWRC
);
3256 sc
->stats
.fcoedwtc
+= IXGBE_READ_REG(hw
, IXGBE_FCOEDWTC
);
3260 IFNET_STAT_SET(ifp
, iqdrops
, total_missed_rx
);
3261 IFNET_STAT_SET(ifp
, ierrors
, sc
->stats
.crcerrs
+ sc
->stats
.rlec
);
3266 * Add sysctl variables, one per statistic, to the system.
3269 ix_add_hw_stats(struct ix_softc
*sc
)
3272 device_t dev
= sc
->dev
;
3274 struct ix_tx_ring
*txr
= sc
->tx_rings
;
3275 struct ix_rx_ring
*rxr
= sc
->rx_rings
;
3277 struct sysctl_ctx_list
*ctx
= device_get_sysctl_ctx(dev
);
3278 struct sysctl_oid
*tree
= device_get_sysctl_tree(dev
);
3279 struct sysctl_oid_list
*child
= SYSCTL_CHILDREN(tree
);
3280 struct ixgbe_hw_stats
*stats
= &sc
->stats
;
3282 struct sysctl_oid
*stat_node
, *queue_node
;
3283 struct sysctl_oid_list
*stat_list
, *queue_list
;
3285 #define QUEUE_NAME_LEN 32
3286 char namebuf
[QUEUE_NAME_LEN
];
3288 /* MAC stats get the own sub node */
3290 stat_node
= SYSCTL_ADD_NODE(ctx
, child
, OID_AUTO
, "mac_stats",
3291 CTLFLAG_RD
, NULL
, "MAC Statistics");
3292 stat_list
= SYSCTL_CHILDREN(stat_node
);
3294 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "crc_errs",
3295 CTLFLAG_RD
, &stats
->crcerrs
,
3297 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "ill_errs",
3298 CTLFLAG_RD
, &stats
->illerrc
,
3299 "Illegal Byte Errors");
3300 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "byte_errs",
3301 CTLFLAG_RD
, &stats
->errbc
,
3303 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "short_discards",
3304 CTLFLAG_RD
, &stats
->mspdc
,
3305 "MAC Short Packets Discarded");
3306 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "local_faults",
3307 CTLFLAG_RD
, &stats
->mlfc
,
3308 "MAC Local Faults");
3309 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "remote_faults",
3310 CTLFLAG_RD
, &stats
->mrfc
,
3311 "MAC Remote Faults");
3312 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rec_len_errs",
3313 CTLFLAG_RD
, &stats
->rlec
,
3314 "Receive Length Errors");
3316 /* Flow Control stats */
3317 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "xon_txd",
3318 CTLFLAG_RD
, &stats
->lxontxc
,
3319 "Link XON Transmitted");
3320 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "xon_recvd",
3321 CTLFLAG_RD
, &stats
->lxonrxc
,
3322 "Link XON Received");
3323 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "xoff_txd",
3324 CTLFLAG_RD
, &stats
->lxofftxc
,
3325 "Link XOFF Transmitted");
3326 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "xoff_recvd",
3327 CTLFLAG_RD
, &stats
->lxoffrxc
,
3328 "Link XOFF Received");
3330 /* Packet Reception Stats */
3331 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "total_octets_rcvd",
3332 CTLFLAG_RD
, &stats
->tor
,
3333 "Total Octets Received");
3334 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "good_octets_rcvd",
3335 CTLFLAG_RD
, &stats
->gorc
,
3336 "Good Octets Received");
3337 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "total_pkts_rcvd",
3338 CTLFLAG_RD
, &stats
->tpr
,
3339 "Total Packets Received");
3340 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "good_pkts_rcvd",
3341 CTLFLAG_RD
, &stats
->gprc
,
3342 "Good Packets Received");
3343 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "mcast_pkts_rcvd",
3344 CTLFLAG_RD
, &stats
->mprc
,
3345 "Multicast Packets Received");
3346 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "bcast_pkts_rcvd",
3347 CTLFLAG_RD
, &stats
->bprc
,
3348 "Broadcast Packets Received");
3349 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_64",
3350 CTLFLAG_RD
, &stats
->prc64
,
3351 "64 byte frames received ");
3352 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_65_127",
3353 CTLFLAG_RD
, &stats
->prc127
,
3354 "65-127 byte frames received");
3355 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_128_255",
3356 CTLFLAG_RD
, &stats
->prc255
,
3357 "128-255 byte frames received");
3358 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_256_511",
3359 CTLFLAG_RD
, &stats
->prc511
,
3360 "256-511 byte frames received");
3361 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_512_1023",
3362 CTLFLAG_RD
, &stats
->prc1023
,
3363 "512-1023 byte frames received");
3364 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "rx_frames_1024_1522",
3365 CTLFLAG_RD
, &stats
->prc1522
,
3366 "1023-1522 byte frames received");
3367 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "recv_undersized",
3368 CTLFLAG_RD
, &stats
->ruc
,
3369 "Receive Undersized");
3370 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "recv_fragmented",
3371 CTLFLAG_RD
, &stats
->rfc
,
3372 "Fragmented Packets Received ");
3373 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "recv_oversized",
3374 CTLFLAG_RD
, &stats
->roc
,
3375 "Oversized Packets Received");
3376 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "recv_jabberd",
3377 CTLFLAG_RD
, &stats
->rjc
,
3379 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "management_pkts_rcvd",
3380 CTLFLAG_RD
, &stats
->mngprc
,
3381 "Management Packets Received");
3382 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "management_pkts_drpd",
3383 CTLFLAG_RD
, &stats
->mngptc
,
3384 "Management Packets Dropped");
3385 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "checksum_errs",
3386 CTLFLAG_RD
, &stats
->xec
,
3389 /* Packet Transmission Stats */
3390 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "good_octets_txd",
3391 CTLFLAG_RD
, &stats
->gotc
,
3392 "Good Octets Transmitted");
3393 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "total_pkts_txd",
3394 CTLFLAG_RD
, &stats
->tpt
,
3395 "Total Packets Transmitted");
3396 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "good_pkts_txd",
3397 CTLFLAG_RD
, &stats
->gptc
,
3398 "Good Packets Transmitted");
3399 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "bcast_pkts_txd",
3400 CTLFLAG_RD
, &stats
->bptc
,
3401 "Broadcast Packets Transmitted");
3402 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "mcast_pkts_txd",
3403 CTLFLAG_RD
, &stats
->mptc
,
3404 "Multicast Packets Transmitted");
3405 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "management_pkts_txd",
3406 CTLFLAG_RD
, &stats
->mngptc
,
3407 "Management Packets Transmitted");
3408 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_64",
3409 CTLFLAG_RD
, &stats
->ptc64
,
3410 "64 byte frames transmitted ");
3411 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_65_127",
3412 CTLFLAG_RD
, &stats
->ptc127
,
3413 "65-127 byte frames transmitted");
3414 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_128_255",
3415 CTLFLAG_RD
, &stats
->ptc255
,
3416 "128-255 byte frames transmitted");
3417 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_256_511",
3418 CTLFLAG_RD
, &stats
->ptc511
,
3419 "256-511 byte frames transmitted");
3420 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_512_1023",
3421 CTLFLAG_RD
, &stats
->ptc1023
,
3422 "512-1023 byte frames transmitted");
3423 SYSCTL_ADD_UQUAD(ctx
, stat_list
, OID_AUTO
, "tx_frames_1024_1522",
3424 CTLFLAG_RD
, &stats
->ptc1522
,
3425 "1024-1522 byte frames transmitted");
3430 * Enable the hardware to drop packets when the buffer is full.
3431 * This is useful when multiple RX rings are used, so that no
3432 * single RX ring being full stalls the entire RX engine. We
3433 * only enable this when multiple RX rings are used and when
3434 * flow control is disabled.
3437 ix_enable_rx_drop(struct ix_softc
*sc
)
3439 struct ixgbe_hw
*hw
= &sc
->hw
;
3443 if_printf(&sc
->arpcom
.ac_if
,
3444 "flow control %d, enable RX drop\n", sc
->fc
);
3447 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
3448 uint32_t srrctl
= IXGBE_READ_REG(hw
, IXGBE_SRRCTL(i
));
3450 srrctl
|= IXGBE_SRRCTL_DROP_EN
;
3451 IXGBE_WRITE_REG(hw
, IXGBE_SRRCTL(i
), srrctl
);
3456 ix_disable_rx_drop(struct ix_softc
*sc
)
3458 struct ixgbe_hw
*hw
= &sc
->hw
;
3462 if_printf(&sc
->arpcom
.ac_if
,
3463 "flow control %d, disable RX drop\n", sc
->fc
);
3466 for (i
= 0; i
< sc
->rx_ring_inuse
; ++i
) {
3467 uint32_t srrctl
= IXGBE_READ_REG(hw
, IXGBE_SRRCTL(i
));
3469 srrctl
&= ~IXGBE_SRRCTL_DROP_EN
;
3470 IXGBE_WRITE_REG(hw
, IXGBE_SRRCTL(i
), srrctl
);
3475 ix_sysctl_flowctrl(SYSCTL_HANDLER_ARGS
)
3477 struct ix_softc
*sc
= (struct ix_softc
*)arg1
;
3478 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3482 error
= sysctl_handle_int(oidp
, &fc
, 0, req
);
3483 if (error
|| req
->newptr
== NULL
)
3487 case ixgbe_fc_rx_pause
:
3488 case ixgbe_fc_tx_pause
:
3496 ifnet_serialize_all(ifp
);
3498 /* Don't bother if it's not changed */
3503 /* Don't do anything, if the interface is not up yet */
3504 if ((ifp
->if_flags
& IFF_RUNNING
) == 0)
3507 if (sc
->rx_ring_inuse
> 1) {
3509 case ixgbe_fc_rx_pause
:
3510 case ixgbe_fc_tx_pause
:
3512 ix_disable_rx_drop(sc
);
3516 ix_enable_rx_drop(sc
);
3520 panic("leading fc check mismatch");
3524 sc
->hw
.fc
.requested_mode
= sc
->fc
;
3525 /* Don't autoneg if forcing a value */
3526 sc
->hw
.fc
.disable_fc_autoneg
= TRUE
;
3527 ixgbe_fc_enable(&sc
->hw
);
3530 ifnet_deserialize_all(ifp
);
3535 /* XXX not working properly w/ 82599 connected w/ DAC */
3536 /* XXX only work after the interface is up */
3538 ix_sysctl_advspeed(SYSCTL_HANDLER_ARGS
)
3540 struct ix_softc
*sc
= (struct ix_softc
*)arg1
;
3541 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3542 struct ixgbe_hw
*hw
= &sc
->hw
;
3543 ixgbe_link_speed speed
;
3544 int error
, advspeed
;
3546 advspeed
= sc
->advspeed
;
3547 error
= sysctl_handle_int(oidp
, &advspeed
, 0, req
);
3548 if (error
|| req
->newptr
== NULL
)
3551 if (!(hw
->phy
.media_type
== ixgbe_media_type_copper
||
3552 hw
->phy
.multispeed_fiber
))
3554 if (hw
->mac
.ops
.setup_link
== NULL
)
3559 speed
= IXGBE_LINK_SPEED_UNKNOWN
;
3563 speed
= IXGBE_LINK_SPEED_1GB_FULL
;
3567 speed
= IXGBE_LINK_SPEED_100_FULL
;
3570 case 3: /* 1Gb/10Gb */
3571 speed
= IXGBE_LINK_SPEED_1GB_FULL
|
3572 IXGBE_LINK_SPEED_10GB_FULL
;
3579 ifnet_serialize_all(ifp
);
3581 if (sc
->advspeed
== advspeed
) /* no change */
3584 if ((speed
& IXGBE_LINK_SPEED_100_FULL
) &&
3585 hw
->mac
.type
!= ixgbe_mac_X540
) {
3590 sc
->advspeed
= advspeed
;
3592 if ((ifp
->if_flags
& IFF_RUNNING
) == 0)
3595 if (speed
== IXGBE_LINK_SPEED_UNKNOWN
) {
3598 hw
->mac
.autotry_restart
= TRUE
;
3599 hw
->mac
.ops
.setup_link(hw
, speed
, sc
->link_up
);
3603 ifnet_deserialize_all(ifp
);
3609 ix_setup_serialize(struct ix_softc
*sc
)
3613 /* Main + RX + TX */
3614 sc
->nserialize
= 1 + sc
->rx_ring_cnt
+ sc
->tx_ring_cnt
;
3616 kmalloc(sc
->nserialize
* sizeof(struct lwkt_serialize
*),
3617 M_DEVBUF
, M_WAITOK
| M_ZERO
);
3622 * NOTE: Order is critical
3625 KKASSERT(i
< sc
->nserialize
);
3626 sc
->serializes
[i
++] = &sc
->main_serialize
;
3628 for (j
= 0; j
< sc
->rx_ring_cnt
; ++j
) {
3629 KKASSERT(i
< sc
->nserialize
);
3630 sc
->serializes
[i
++] = &sc
->rx_rings
[j
].rx_serialize
;
3633 for (j
= 0; j
< sc
->tx_ring_cnt
; ++j
) {
3634 KKASSERT(i
< sc
->nserialize
);
3635 sc
->serializes
[i
++] = &sc
->tx_rings
[j
].tx_serialize
;
3638 KKASSERT(i
== sc
->nserialize
);
3642 ix_alloc_intr(struct ix_softc
*sc
)
3644 struct ix_intr_data
*intr
;
3648 if (sc
->intr_type
== PCI_INTR_TYPE_MSIX
) {
3649 ix_set_ring_inuse(sc
, FALSE
);
3653 if (sc
->intr_data
!= NULL
)
3654 kfree(sc
->intr_data
, M_DEVBUF
);
3657 sc
->intr_data
= kmalloc(sizeof(struct ix_intr_data
), M_DEVBUF
,
3659 intr
= &sc
->intr_data
[0];
3662 * Allocate MSI/legacy interrupt resource
3664 sc
->intr_type
= pci_alloc_1intr(sc
->dev
, ix_msi_enable
,
3665 &intr
->intr_rid
, &intr_flags
);
3667 intr
->intr_res
= bus_alloc_resource_any(sc
->dev
, SYS_RES_IRQ
,
3668 &intr
->intr_rid
, intr_flags
);
3669 if (intr
->intr_res
== NULL
) {
3670 device_printf(sc
->dev
, "Unable to allocate bus resource: "
3675 intr
->intr_serialize
= &sc
->main_serialize
;
3676 intr
->intr_cpuid
= rman_get_cpuid(intr
->intr_res
);
3677 intr
->intr_func
= ix_intr
;
3678 intr
->intr_funcarg
= sc
;
3679 intr
->intr_rate
= IX_INTR_RATE
;
3680 intr
->intr_use
= IX_INTR_USE_RXTX
;
3682 sc
->tx_rings
[0].tx_intr_cpuid
= intr
->intr_cpuid
;
3683 sc
->tx_rings
[0].tx_intr_vec
= IX_TX_INTR_VEC
;
3685 sc
->rx_rings
[0].rx_intr_vec
= IX_RX0_INTR_VEC
;
3687 ix_set_ring_inuse(sc
, FALSE
);
3689 KKASSERT(sc
->rx_ring_inuse
<= IX_MIN_RXRING_RSS
);
3690 if (sc
->rx_ring_inuse
== IX_MIN_RXRING_RSS
)
3691 sc
->rx_rings
[1].rx_intr_vec
= IX_RX1_INTR_VEC
;
3697 ix_free_intr(struct ix_softc
*sc
)
3699 if (sc
->intr_data
== NULL
)
3702 if (sc
->intr_type
!= PCI_INTR_TYPE_MSIX
) {
3703 struct ix_intr_data
*intr
= &sc
->intr_data
[0];
3705 KKASSERT(sc
->intr_cnt
== 1);
3706 if (intr
->intr_res
!= NULL
) {
3707 bus_release_resource(sc
->dev
, SYS_RES_IRQ
,
3708 intr
->intr_rid
, intr
->intr_res
);
3710 if (sc
->intr_type
== PCI_INTR_TYPE_MSI
)
3711 pci_release_msi(sc
->dev
);
3713 kfree(sc
->intr_data
, M_DEVBUF
);
3715 ix_free_msix(sc
, TRUE
);
3720 ix_set_ring_inuse(struct ix_softc
*sc
, boolean_t polling
)
3722 sc
->rx_ring_inuse
= ix_get_rxring_inuse(sc
, polling
);
3723 sc
->tx_ring_inuse
= ix_get_txring_inuse(sc
, polling
);
3725 if_printf(&sc
->arpcom
.ac_if
,
3726 "RX rings %d/%d, TX rings %d/%d\n",
3727 sc
->rx_ring_inuse
, sc
->rx_ring_cnt
,
3728 sc
->tx_ring_inuse
, sc
->tx_ring_cnt
);
3733 ix_get_rxring_inuse(const struct ix_softc
*sc
, boolean_t polling
)
3735 if (!IX_ENABLE_HWRSS(sc
))
3739 return sc
->rx_ring_cnt
;
3740 else if (sc
->intr_type
!= PCI_INTR_TYPE_MSIX
)
3741 return IX_MIN_RXRING_RSS
;
3743 return sc
->rx_ring_msix
;
3747 ix_get_txring_inuse(const struct ix_softc
*sc
, boolean_t polling
)
3749 if (!IX_ENABLE_HWTSS(sc
))
3753 return sc
->tx_ring_cnt
;
3754 else if (sc
->intr_type
!= PCI_INTR_TYPE_MSIX
)
3757 return sc
->tx_ring_msix
;
3761 ix_setup_intr(struct ix_softc
*sc
)
3765 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
3766 struct ix_intr_data
*intr
= &sc
->intr_data
[i
];
3769 error
= bus_setup_intr_descr(sc
->dev
, intr
->intr_res
,
3770 INTR_MPSAFE
, intr
->intr_func
, intr
->intr_funcarg
,
3771 &intr
->intr_hand
, intr
->intr_serialize
, intr
->intr_desc
);
3773 device_printf(sc
->dev
, "can't setup %dth intr\n", i
);
3774 ix_teardown_intr(sc
, i
);
3782 ix_teardown_intr(struct ix_softc
*sc
, int intr_cnt
)
3786 if (sc
->intr_data
== NULL
)
3789 for (i
= 0; i
< intr_cnt
; ++i
) {
3790 struct ix_intr_data
*intr
= &sc
->intr_data
[i
];
3792 bus_teardown_intr(sc
->dev
, intr
->intr_res
, intr
->intr_hand
);
3797 ix_serialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
3799 struct ix_softc
*sc
= ifp
->if_softc
;
3801 ifnet_serialize_array_enter(sc
->serializes
, sc
->nserialize
, slz
);
3805 ix_deserialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
3807 struct ix_softc
*sc
= ifp
->if_softc
;
3809 ifnet_serialize_array_exit(sc
->serializes
, sc
->nserialize
, slz
);
3813 ix_tryserialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
3815 struct ix_softc
*sc
= ifp
->if_softc
;
3817 return ifnet_serialize_array_try(sc
->serializes
, sc
->nserialize
, slz
);
3823 ix_serialize_assert(struct ifnet
*ifp
, enum ifnet_serialize slz
,
3824 boolean_t serialized
)
3826 struct ix_softc
*sc
= ifp
->if_softc
;
3828 ifnet_serialize_array_assert(sc
->serializes
, sc
->nserialize
, slz
,
3832 #endif /* INVARIANTS */
3835 ix_free_rings(struct ix_softc
*sc
)
3839 if (sc
->tx_rings
!= NULL
) {
3840 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3841 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
3843 ix_destroy_tx_ring(txr
, txr
->tx_ndesc
);
3845 kfree(sc
->tx_rings
, M_DEVBUF
);
3848 if (sc
->rx_rings
!= NULL
) {
3849 for (i
=0; i
< sc
->rx_ring_cnt
; ++i
) {
3850 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
3852 ix_destroy_rx_ring(rxr
, rxr
->rx_ndesc
);
3854 kfree(sc
->rx_rings
, M_DEVBUF
);
3857 if (sc
->parent_tag
!= NULL
)
3858 bus_dma_tag_destroy(sc
->parent_tag
);
3862 ix_watchdog(struct ifaltq_subque
*ifsq
)
3864 struct ix_tx_ring
*txr
= ifsq_get_priv(ifsq
);
3865 struct ifnet
*ifp
= ifsq_get_ifp(ifsq
);
3866 struct ix_softc
*sc
= ifp
->if_softc
;
3869 KKASSERT(txr
->tx_ifsq
== ifsq
);
3870 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
3873 * If the interface has been paused then don't do the watchdog check
3875 if (IXGBE_READ_REG(&sc
->hw
, IXGBE_TFCS
) & IXGBE_TFCS_TXOFF
) {
3876 txr
->tx_watchdog
.wd_timer
= 5;
3880 if_printf(ifp
, "Watchdog timeout -- resetting\n");
3881 if_printf(ifp
, "Queue(%d) tdh = %d, hw tdt = %d\n", txr
->tx_idx
,
3882 IXGBE_READ_REG(&sc
->hw
, IXGBE_TDH(txr
->tx_idx
)),
3883 IXGBE_READ_REG(&sc
->hw
, IXGBE_TDT(txr
->tx_idx
)));
3884 if_printf(ifp
, "TX(%d) desc avail = %d, next TX to Clean = %d\n",
3885 txr
->tx_idx
, txr
->tx_avail
, txr
->tx_next_clean
);
3888 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
)
3889 ifsq_devstart_sched(sc
->tx_rings
[i
].tx_ifsq
);
3893 ix_free_tx_ring(struct ix_tx_ring
*txr
)
3897 for (i
= 0; i
< txr
->tx_ndesc
; ++i
) {
3898 struct ix_tx_buf
*txbuf
= &txr
->tx_buf
[i
];
3900 if (txbuf
->m_head
!= NULL
) {
3901 bus_dmamap_unload(txr
->tx_tag
, txbuf
->map
);
3902 m_freem(txbuf
->m_head
);
3903 txbuf
->m_head
= NULL
;
3909 ix_free_rx_ring(struct ix_rx_ring
*rxr
)
3913 for (i
= 0; i
< rxr
->rx_ndesc
; ++i
) {
3914 struct ix_rx_buf
*rxbuf
= &rxr
->rx_buf
[i
];
3916 if (rxbuf
->fmp
!= NULL
) {
3917 m_freem(rxbuf
->fmp
);
3921 KKASSERT(rxbuf
->lmp
== NULL
);
3923 if (rxbuf
->m_head
!= NULL
) {
3924 bus_dmamap_unload(rxr
->rx_tag
, rxbuf
->map
);
3925 m_freem(rxbuf
->m_head
);
3926 rxbuf
->m_head
= NULL
;
3932 ix_newbuf(struct ix_rx_ring
*rxr
, int i
, boolean_t wait
)
3935 bus_dma_segment_t seg
;
3937 struct ix_rx_buf
*rxbuf
;
3938 int flags
, error
, nseg
;
3941 if (__predict_false(wait
))
3944 m
= m_getjcl(flags
, MT_DATA
, M_PKTHDR
, rxr
->rx_mbuf_sz
);
3947 if_printf(&rxr
->rx_sc
->arpcom
.ac_if
,
3948 "Unable to allocate RX mbuf\n");
3952 m
->m_len
= m
->m_pkthdr
.len
= rxr
->rx_mbuf_sz
;
3954 error
= bus_dmamap_load_mbuf_segment(rxr
->rx_tag
,
3955 rxr
->rx_sparemap
, m
, &seg
, 1, &nseg
, BUS_DMA_NOWAIT
);
3959 if_printf(&rxr
->rx_sc
->arpcom
.ac_if
,
3960 "Unable to load RX mbuf\n");
3965 rxbuf
= &rxr
->rx_buf
[i
];
3966 if (rxbuf
->m_head
!= NULL
)
3967 bus_dmamap_unload(rxr
->rx_tag
, rxbuf
->map
);
3970 rxbuf
->map
= rxr
->rx_sparemap
;
3971 rxr
->rx_sparemap
= map
;
3974 rxbuf
->paddr
= seg
.ds_addr
;
3976 ix_setup_rxdesc(&rxr
->rx_base
[i
], rxbuf
);
3981 ix_add_sysctl(struct ix_softc
*sc
)
3983 struct sysctl_ctx_list
*ctx
= device_get_sysctl_ctx(sc
->dev
);
3984 struct sysctl_oid
*tree
= device_get_sysctl_tree(sc
->dev
);
3990 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3991 OID_AUTO
, "rxr", CTLFLAG_RD
, &sc
->rx_ring_cnt
, 0, "# of RX rings");
3992 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3993 OID_AUTO
, "rxr_inuse", CTLFLAG_RD
, &sc
->rx_ring_inuse
, 0,
3994 "# of RX rings used");
3995 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3996 OID_AUTO
, "txr", CTLFLAG_RD
, &sc
->tx_ring_cnt
, 0, "# of TX rings");
3997 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3998 OID_AUTO
, "txr_inuse", CTLFLAG_RD
, &sc
->tx_ring_inuse
, 0,
3999 "# of TX rings used");
4000 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4001 OID_AUTO
, "rxd", CTLTYPE_INT
| CTLFLAG_RD
,
4002 sc
, 0, ix_sysctl_rxd
, "I",
4004 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4005 OID_AUTO
, "txd", CTLTYPE_INT
| CTLFLAG_RD
,
4006 sc
, 0, ix_sysctl_txd
, "I",
4008 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4009 OID_AUTO
, "tx_wreg_nsegs", CTLTYPE_INT
| CTLFLAG_RW
,
4010 sc
, 0, ix_sysctl_tx_wreg_nsegs
, "I",
4011 "# of segments sent before write to hardware register");
4012 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4013 OID_AUTO
, "rx_wreg_nsegs", CTLTYPE_INT
| CTLFLAG_RW
,
4014 sc
, 0, ix_sysctl_rx_wreg_nsegs
, "I",
4015 "# of received segments sent before write to hardware register");
4016 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4017 OID_AUTO
, "tx_intr_nsegs", CTLTYPE_INT
| CTLFLAG_RW
,
4018 sc
, 0, ix_sysctl_tx_intr_nsegs
, "I",
4019 "# of segments per TX interrupt");
4021 #ifdef IFPOLL_ENABLE
4022 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4023 OID_AUTO
, "npoll_rxoff", CTLTYPE_INT
|CTLFLAG_RW
,
4024 sc
, 0, ix_sysctl_npoll_rxoff
, "I", "NPOLLING RX cpu offset");
4025 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4026 OID_AUTO
, "npoll_txoff", CTLTYPE_INT
|CTLFLAG_RW
,
4027 sc
, 0, ix_sysctl_npoll_txoff
, "I", "NPOLLING TX cpu offset");
4030 #define IX_ADD_INTR_RATE_SYSCTL(sc, use, name) \
4032 ix_add_intr_rate_sysctl(sc, IX_INTR_USE_##use, #name, \
4033 ix_sysctl_##name, #use " interrupt rate"); \
4036 IX_ADD_INTR_RATE_SYSCTL(sc
, RXTX
, rxtx_intr_rate
);
4037 IX_ADD_INTR_RATE_SYSCTL(sc
, RX
, rx_intr_rate
);
4038 IX_ADD_INTR_RATE_SYSCTL(sc
, TX
, tx_intr_rate
);
4039 IX_ADD_INTR_RATE_SYSCTL(sc
, STATUS
, sts_intr_rate
);
4041 #undef IX_ADD_INTR_RATE_SYSCTL
4044 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
4045 OID_AUTO
, "rss_debug", CTLFLAG_RW
, &sc
->rss_debug
, 0,
4047 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
4048 ksnprintf(node
, sizeof(node
), "rx%d_pkt", i
);
4049 SYSCTL_ADD_ULONG(ctx
,
4050 SYSCTL_CHILDREN(tree
), OID_AUTO
, node
,
4051 CTLFLAG_RW
, &sc
->rx_rings
[i
].rx_pkts
, "RXed packets");
4055 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4056 OID_AUTO
, "flowctrl", CTLTYPE_INT
| CTLFLAG_RW
,
4057 sc
, 0, ix_sysctl_flowctrl
, "I",
4058 "flow control, 0 - off, 1 - rx pause, 2 - tx pause, 3 - full");
4062 * Allow a kind of speed control by forcing the autoneg
4063 * advertised speed list to only a certain value, this
4064 * supports 1G on 82599 devices, and 100Mb on X540.
4066 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
4067 OID_AUTO
, "advspeed", CTLTYPE_INT
| CTLFLAG_RW
,
4068 sc
, 0, ix_sysctl_advspeed
, "I",
4069 "advertised link speed, "
4070 "0 - auto, 1 - 1Gb, 2 - 100Mb, 3 - 1Gb/10Gb");
4074 ix_add_hw_stats(sc
);
4080 ix_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS
)
4082 struct ix_softc
*sc
= (void *)arg1
;
4083 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4084 int error
, nsegs
, i
;
4086 nsegs
= sc
->tx_rings
[0].tx_wreg_nsegs
;
4087 error
= sysctl_handle_int(oidp
, &nsegs
, 0, req
);
4088 if (error
|| req
->newptr
== NULL
)
4093 ifnet_serialize_all(ifp
);
4094 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
4095 sc
->tx_rings
[i
].tx_wreg_nsegs
= nsegs
;
4096 ifnet_deserialize_all(ifp
);
4102 ix_sysctl_rx_wreg_nsegs(SYSCTL_HANDLER_ARGS
)
4104 struct ix_softc
*sc
= (void *)arg1
;
4105 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4106 int error
, nsegs
, i
;
4108 nsegs
= sc
->rx_rings
[0].rx_wreg_nsegs
;
4109 error
= sysctl_handle_int(oidp
, &nsegs
, 0, req
);
4110 if (error
|| req
->newptr
== NULL
)
4115 ifnet_serialize_all(ifp
);
4116 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
)
4117 sc
->rx_rings
[i
].rx_wreg_nsegs
=nsegs
;
4118 ifnet_deserialize_all(ifp
);
4124 ix_sysctl_txd(SYSCTL_HANDLER_ARGS
)
4126 struct ix_softc
*sc
= (void *)arg1
;
4129 txd
= sc
->tx_rings
[0].tx_ndesc
;
4130 return sysctl_handle_int(oidp
, &txd
, 0, req
);
4134 ix_sysctl_rxd(SYSCTL_HANDLER_ARGS
)
4136 struct ix_softc
*sc
= (void *)arg1
;
4139 rxd
= sc
->rx_rings
[0].rx_ndesc
;
4140 return sysctl_handle_int(oidp
, &rxd
, 0, req
);
4144 ix_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS
)
4146 struct ix_softc
*sc
= (void *)arg1
;
4147 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4148 struct ix_tx_ring
*txr
= &sc
->tx_rings
[0];
4151 nsegs
= txr
->tx_intr_nsegs
;
4152 error
= sysctl_handle_int(oidp
, &nsegs
, 0, req
);
4153 if (error
|| req
->newptr
== NULL
)
4158 ifnet_serialize_all(ifp
);
4160 if (nsegs
>= txr
->tx_ndesc
- IX_MAX_SCATTER
- IX_TX_RESERVED
) {
4166 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
4167 sc
->tx_rings
[i
].tx_intr_nsegs
= nsegs
;
4170 ifnet_deserialize_all(ifp
);
4176 ix_set_eitr(struct ix_softc
*sc
, int idx
, int rate
)
4178 uint32_t eitr
, eitr_intvl
;
4180 eitr
= IXGBE_READ_REG(&sc
->hw
, IXGBE_EITR(idx
));
4181 eitr_intvl
= 1000000000 / 256 / rate
;
4183 if (sc
->hw
.mac
.type
== ixgbe_mac_82598EB
) {
4184 eitr
&= ~IX_EITR_INTVL_MASK_82598
;
4185 if (eitr_intvl
== 0)
4187 else if (eitr_intvl
> IX_EITR_INTVL_MASK_82598
)
4188 eitr_intvl
= IX_EITR_INTVL_MASK_82598
;
4190 eitr
&= ~IX_EITR_INTVL_MASK
;
4192 eitr_intvl
&= ~IX_EITR_INTVL_RSVD_MASK
;
4193 if (eitr_intvl
== 0)
4194 eitr_intvl
= IX_EITR_INTVL_MIN
;
4195 else if (eitr_intvl
> IX_EITR_INTVL_MAX
)
4196 eitr_intvl
= IX_EITR_INTVL_MAX
;
4200 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EITR(idx
), eitr
);
4204 ix_sysctl_rxtx_intr_rate(SYSCTL_HANDLER_ARGS
)
4206 return ix_sysctl_intr_rate(oidp
, arg1
, arg2
, req
, IX_INTR_USE_RXTX
);
4210 ix_sysctl_rx_intr_rate(SYSCTL_HANDLER_ARGS
)
4212 return ix_sysctl_intr_rate(oidp
, arg1
, arg2
, req
, IX_INTR_USE_RX
);
4216 ix_sysctl_tx_intr_rate(SYSCTL_HANDLER_ARGS
)
4218 return ix_sysctl_intr_rate(oidp
, arg1
, arg2
, req
, IX_INTR_USE_TX
);
4222 ix_sysctl_sts_intr_rate(SYSCTL_HANDLER_ARGS
)
4224 return ix_sysctl_intr_rate(oidp
, arg1
, arg2
, req
, IX_INTR_USE_STATUS
);
4228 ix_sysctl_intr_rate(SYSCTL_HANDLER_ARGS
, int use
)
4230 struct ix_softc
*sc
= (void *)arg1
;
4231 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4235 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
4236 if (sc
->intr_data
[i
].intr_use
== use
) {
4237 rate
= sc
->intr_data
[i
].intr_rate
;
4242 error
= sysctl_handle_int(oidp
, &rate
, 0, req
);
4243 if (error
|| req
->newptr
== NULL
)
4248 ifnet_serialize_all(ifp
);
4250 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
4251 if (sc
->intr_data
[i
].intr_use
== use
) {
4252 sc
->intr_data
[i
].intr_rate
= rate
;
4253 if (ifp
->if_flags
& IFF_RUNNING
)
4254 ix_set_eitr(sc
, i
, rate
);
4258 ifnet_deserialize_all(ifp
);
4264 ix_add_intr_rate_sysctl(struct ix_softc
*sc
, int use
,
4265 const char *name
, int (*handler
)(SYSCTL_HANDLER_ARGS
), const char *desc
)
4269 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
4270 if (sc
->intr_data
[i
].intr_use
== use
) {
4271 SYSCTL_ADD_PROC(device_get_sysctl_ctx(sc
->dev
),
4272 SYSCTL_CHILDREN(device_get_sysctl_tree(sc
->dev
)),
4273 OID_AUTO
, name
, CTLTYPE_INT
| CTLFLAG_RW
,
4274 sc
, 0, handler
, "I", desc
);
4281 ix_set_timer_cpuid(struct ix_softc
*sc
, boolean_t polling
)
4283 if (polling
|| sc
->intr_type
== PCI_INTR_TYPE_MSIX
)
4284 sc
->timer_cpuid
= 0; /* XXX fixed */
4286 sc
->timer_cpuid
= rman_get_cpuid(sc
->intr_data
[0].intr_res
);
4290 ix_alloc_msix(struct ix_softc
*sc
)
4292 int msix_enable
, msix_cnt
, msix_cnt2
, alloc_cnt
;
4293 struct ix_intr_data
*intr
;
4295 int offset
, offset_def
, agg_rxtx
, ring_max
;
4296 boolean_t aggregate
, setup
= FALSE
;
4298 msix_enable
= ix_msix_enable
;
4300 * Don't enable MSI-X on 82598 by default, see:
4301 * 82598 specification update errata #38
4303 if (sc
->hw
.mac
.type
== ixgbe_mac_82598EB
)
4305 msix_enable
= device_getenv_int(sc
->dev
, "msix.enable", msix_enable
);
4309 msix_cnt
= pci_msix_count(sc
->dev
);
4310 #ifdef IX_MSIX_DEBUG
4311 msix_cnt
= device_getenv_int(sc
->dev
, "msix.count", msix_cnt
);
4313 if (msix_cnt
<= 1) {
4314 /* One MSI-X model does not make sense */
4319 while ((1 << (i
+ 1)) <= msix_cnt
)
4324 device_printf(sc
->dev
, "MSI-X count %d/%d\n",
4325 msix_cnt2
, msix_cnt
);
4328 KKASSERT(msix_cnt
>= msix_cnt2
);
4329 if (msix_cnt
== msix_cnt2
) {
4330 /* We need at least one MSI-X for link status */
4332 if (msix_cnt2
<= 1) {
4333 /* One MSI-X for RX/TX does not make sense */
4334 device_printf(sc
->dev
, "not enough MSI-X for TX/RX, "
4335 "MSI-X count %d/%d\n", msix_cnt2
, msix_cnt
);
4338 KKASSERT(msix_cnt
> msix_cnt2
);
4341 device_printf(sc
->dev
, "MSI-X count eq fixup %d/%d\n",
4342 msix_cnt2
, msix_cnt
);
4347 * Make sure that we don't break interrupt related registers
4348 * (EIMS, etc) limitation.
4350 * NOTE: msix_cnt > msix_cnt2, when we reach here
4352 if (sc
->hw
.mac
.type
== ixgbe_mac_82598EB
) {
4353 if (msix_cnt2
> IX_MAX_MSIX_82598
)
4354 msix_cnt2
= IX_MAX_MSIX_82598
;
4356 if (msix_cnt2
> IX_MAX_MSIX
)
4357 msix_cnt2
= IX_MAX_MSIX
;
4359 msix_cnt
= msix_cnt2
+ 1; /* +1 for status */
4362 device_printf(sc
->dev
, "MSI-X count max fixup %d/%d\n",
4363 msix_cnt2
, msix_cnt
);
4366 sc
->rx_ring_msix
= sc
->rx_ring_cnt
;
4367 if (sc
->rx_ring_msix
> msix_cnt2
)
4368 sc
->rx_ring_msix
= msix_cnt2
;
4370 sc
->tx_ring_msix
= sc
->tx_ring_cnt
;
4371 if (sc
->tx_ring_msix
> msix_cnt2
)
4372 sc
->tx_ring_msix
= msix_cnt2
;
4374 ring_max
= sc
->rx_ring_msix
;
4375 if (ring_max
< sc
->tx_ring_msix
)
4376 ring_max
= sc
->tx_ring_msix
;
4378 /* Allow user to force independent RX/TX MSI-X handling */
4379 agg_rxtx
= device_getenv_int(sc
->dev
, "msix.agg_rxtx",
4382 if (!agg_rxtx
&& msix_cnt
>= sc
->tx_ring_msix
+ sc
->rx_ring_msix
+ 1) {
4384 * Independent TX/RX MSI-X
4388 device_printf(sc
->dev
, "independent TX/RX MSI-X\n");
4389 alloc_cnt
= sc
->tx_ring_msix
+ sc
->rx_ring_msix
;
4392 * Aggregate TX/RX MSI-X
4396 device_printf(sc
->dev
, "aggregate TX/RX MSI-X\n");
4397 alloc_cnt
= msix_cnt2
;
4398 if (alloc_cnt
> ring_max
)
4399 alloc_cnt
= ring_max
;
4400 KKASSERT(alloc_cnt
>= sc
->rx_ring_msix
&&
4401 alloc_cnt
>= sc
->tx_ring_msix
);
4403 ++alloc_cnt
; /* For status */
4406 device_printf(sc
->dev
, "MSI-X alloc %d, "
4407 "RX ring %d, TX ring %d\n", alloc_cnt
,
4408 sc
->rx_ring_msix
, sc
->tx_ring_msix
);
4411 sc
->msix_mem_rid
= PCIR_BAR(IX_MSIX_BAR_82598
);
4412 sc
->msix_mem_res
= bus_alloc_resource_any(sc
->dev
, SYS_RES_MEMORY
,
4413 &sc
->msix_mem_rid
, RF_ACTIVE
);
4414 if (sc
->msix_mem_res
== NULL
) {
4415 sc
->msix_mem_rid
= PCIR_BAR(IX_MSIX_BAR_82599
);
4416 sc
->msix_mem_res
= bus_alloc_resource_any(sc
->dev
,
4417 SYS_RES_MEMORY
, &sc
->msix_mem_rid
, RF_ACTIVE
);
4418 if (sc
->msix_mem_res
== NULL
) {
4419 device_printf(sc
->dev
, "Unable to map MSI-X table\n");
4424 sc
->intr_cnt
= alloc_cnt
;
4425 sc
->intr_data
= kmalloc(sizeof(struct ix_intr_data
) * sc
->intr_cnt
,
4426 M_DEVBUF
, M_WAITOK
| M_ZERO
);
4427 for (x
= 0; x
< sc
->intr_cnt
; ++x
) {
4428 intr
= &sc
->intr_data
[x
];
4429 intr
->intr_rid
= -1;
4430 intr
->intr_rate
= IX_INTR_RATE
;
4438 if (sc
->rx_ring_msix
== ncpus2
) {
4441 offset_def
= (sc
->rx_ring_msix
*
4442 device_get_unit(sc
->dev
)) % ncpus2
;
4444 offset
= device_getenv_int(sc
->dev
,
4445 "msix.rxoff", offset_def
);
4446 if (offset
>= ncpus2
||
4447 offset
% sc
->rx_ring_msix
!= 0) {
4448 device_printf(sc
->dev
,
4449 "invalid msix.rxoff %d, use %d\n",
4450 offset
, offset_def
);
4451 offset
= offset_def
;
4454 ix_conf_rx_msix(sc
, 0, &x
, offset
);
4459 if (sc
->tx_ring_msix
== ncpus2
) {
4462 offset_def
= (sc
->tx_ring_msix
*
4463 device_get_unit(sc
->dev
)) % ncpus2
;
4465 offset
= device_getenv_int(sc
->dev
,
4466 "msix.txoff", offset_def
);
4467 if (offset
>= ncpus2
||
4468 offset
% sc
->tx_ring_msix
!= 0) {
4469 device_printf(sc
->dev
,
4470 "invalid msix.txoff %d, use %d\n",
4471 offset
, offset_def
);
4472 offset
= offset_def
;
4475 ix_conf_tx_msix(sc
, 0, &x
, offset
);
4479 ring_agg
= sc
->rx_ring_msix
;
4480 if (ring_agg
> sc
->tx_ring_msix
)
4481 ring_agg
= sc
->tx_ring_msix
;
4483 if (ring_max
== ncpus2
) {
4486 offset_def
= (ring_max
* device_get_unit(sc
->dev
)) %
4489 offset
= device_getenv_int(sc
->dev
, "msix.off",
4491 if (offset
>= ncpus2
|| offset
% ring_max
!= 0) {
4492 device_printf(sc
->dev
,
4493 "invalid msix.off %d, use %d\n",
4494 offset
, offset_def
);
4495 offset
= offset_def
;
4499 for (i
= 0; i
< ring_agg
; ++i
) {
4500 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
4501 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
4503 KKASSERT(x
< sc
->intr_cnt
);
4504 rxr
->rx_intr_vec
= x
;
4505 ix_setup_msix_eims(sc
, x
,
4506 &rxr
->rx_eims
, &rxr
->rx_eims_val
);
4508 /* NOTE: Leave TX ring's intr_vec negative */
4510 intr
= &sc
->intr_data
[x
++];
4512 intr
->intr_serialize
= &rxr
->rx_serialize
;
4513 intr
->intr_func
= ix_msix_rxtx
;
4514 intr
->intr_funcarg
= rxr
;
4515 intr
->intr_use
= IX_INTR_USE_RXTX
;
4517 intr
->intr_cpuid
= i
+ offset
;
4518 KKASSERT(intr
->intr_cpuid
< ncpus2
);
4519 txr
->tx_intr_cpuid
= intr
->intr_cpuid
;
4521 ksnprintf(intr
->intr_desc0
, sizeof(intr
->intr_desc0
),
4522 "%s rxtx%d", device_get_nameunit(sc
->dev
), i
);
4523 intr
->intr_desc
= intr
->intr_desc0
;
4526 if (ring_agg
!= ring_max
) {
4527 if (ring_max
== sc
->tx_ring_msix
)
4528 ix_conf_tx_msix(sc
, i
, &x
, offset
);
4530 ix_conf_rx_msix(sc
, i
, &x
, offset
);
4537 KKASSERT(x
< sc
->intr_cnt
);
4538 sc
->sts_msix_vec
= x
;
4540 intr
= &sc
->intr_data
[x
++];
4542 intr
->intr_serialize
= &sc
->main_serialize
;
4543 intr
->intr_func
= ix_msix_status
;
4544 intr
->intr_funcarg
= sc
;
4545 intr
->intr_cpuid
= 0;
4546 intr
->intr_use
= IX_INTR_USE_STATUS
;
4548 ksnprintf(intr
->intr_desc0
, sizeof(intr
->intr_desc0
), "%s sts",
4549 device_get_nameunit(sc
->dev
));
4550 intr
->intr_desc
= intr
->intr_desc0
;
4552 KKASSERT(x
== sc
->intr_cnt
);
4554 error
= pci_setup_msix(sc
->dev
);
4556 device_printf(sc
->dev
, "Setup MSI-X failed\n");
4561 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
4562 intr
= &sc
->intr_data
[i
];
4564 error
= pci_alloc_msix_vector(sc
->dev
, i
, &intr
->intr_rid
,
4567 device_printf(sc
->dev
,
4568 "Unable to allocate MSI-X %d on cpu%d\n", i
,
4573 intr
->intr_res
= bus_alloc_resource_any(sc
->dev
, SYS_RES_IRQ
,
4574 &intr
->intr_rid
, RF_ACTIVE
);
4575 if (intr
->intr_res
== NULL
) {
4576 device_printf(sc
->dev
,
4577 "Unable to allocate MSI-X %d resource\n", i
);
4583 pci_enable_msix(sc
->dev
);
4584 sc
->intr_type
= PCI_INTR_TYPE_MSIX
;
4587 ix_free_msix(sc
, setup
);
4591 ix_free_msix(struct ix_softc
*sc
, boolean_t setup
)
4595 KKASSERT(sc
->intr_cnt
> 1);
4597 for (i
= 0; i
< sc
->intr_cnt
; ++i
) {
4598 struct ix_intr_data
*intr
= &sc
->intr_data
[i
];
4600 if (intr
->intr_res
!= NULL
) {
4601 bus_release_resource(sc
->dev
, SYS_RES_IRQ
,
4602 intr
->intr_rid
, intr
->intr_res
);
4604 if (intr
->intr_rid
>= 0)
4605 pci_release_msix_vector(sc
->dev
, intr
->intr_rid
);
4608 pci_teardown_msix(sc
->dev
);
4611 kfree(sc
->intr_data
, M_DEVBUF
);
4612 sc
->intr_data
= NULL
;
4616 ix_conf_rx_msix(struct ix_softc
*sc
, int i
, int *x0
, int offset
)
4620 for (; i
< sc
->rx_ring_msix
; ++i
) {
4621 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
4622 struct ix_intr_data
*intr
;
4624 KKASSERT(x
< sc
->intr_cnt
);
4625 rxr
->rx_intr_vec
= x
;
4626 ix_setup_msix_eims(sc
, x
, &rxr
->rx_eims
, &rxr
->rx_eims_val
);
4628 intr
= &sc
->intr_data
[x
++];
4630 intr
->intr_serialize
= &rxr
->rx_serialize
;
4631 intr
->intr_func
= ix_msix_rx
;
4632 intr
->intr_funcarg
= rxr
;
4633 intr
->intr_rate
= IX_MSIX_RX_RATE
;
4634 intr
->intr_use
= IX_INTR_USE_RX
;
4636 intr
->intr_cpuid
= i
+ offset
;
4637 KKASSERT(intr
->intr_cpuid
< ncpus2
);
4639 ksnprintf(intr
->intr_desc0
, sizeof(intr
->intr_desc0
), "%s rx%d",
4640 device_get_nameunit(sc
->dev
), i
);
4641 intr
->intr_desc
= intr
->intr_desc0
;
4647 ix_conf_tx_msix(struct ix_softc
*sc
, int i
, int *x0
, int offset
)
4651 for (; i
< sc
->tx_ring_msix
; ++i
) {
4652 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
4653 struct ix_intr_data
*intr
;
4655 KKASSERT(x
< sc
->intr_cnt
);
4656 txr
->tx_intr_vec
= x
;
4657 ix_setup_msix_eims(sc
, x
, &txr
->tx_eims
, &txr
->tx_eims_val
);
4659 intr
= &sc
->intr_data
[x
++];
4661 intr
->intr_serialize
= &txr
->tx_serialize
;
4662 intr
->intr_func
= ix_msix_tx
;
4663 intr
->intr_funcarg
= txr
;
4664 intr
->intr_rate
= IX_MSIX_TX_RATE
;
4665 intr
->intr_use
= IX_INTR_USE_TX
;
4667 intr
->intr_cpuid
= i
+ offset
;
4668 KKASSERT(intr
->intr_cpuid
< ncpus2
);
4669 txr
->tx_intr_cpuid
= intr
->intr_cpuid
;
4671 ksnprintf(intr
->intr_desc0
, sizeof(intr
->intr_desc0
), "%s tx%d",
4672 device_get_nameunit(sc
->dev
), i
);
4673 intr
->intr_desc
= intr
->intr_desc0
;
4679 ix_msix_rx(void *xrxr
)
4681 struct ix_rx_ring
*rxr
= xrxr
;
4683 ASSERT_SERIALIZED(&rxr
->rx_serialize
);
4686 IXGBE_WRITE_REG(&rxr
->rx_sc
->hw
, rxr
->rx_eims
, rxr
->rx_eims_val
);
4690 ix_msix_tx(void *xtxr
)
4692 struct ix_tx_ring
*txr
= xtxr
;
4694 ASSERT_SERIALIZED(&txr
->tx_serialize
);
4696 ix_txeof(txr
, *(txr
->tx_hdr
));
4697 if (!ifsq_is_empty(txr
->tx_ifsq
))
4698 ifsq_devstart(txr
->tx_ifsq
);
4699 IXGBE_WRITE_REG(&txr
->tx_sc
->hw
, txr
->tx_eims
, txr
->tx_eims_val
);
4703 ix_msix_rxtx(void *xrxr
)
4705 struct ix_rx_ring
*rxr
= xrxr
;
4706 struct ix_tx_ring
*txr
;
4709 ASSERT_SERIALIZED(&rxr
->rx_serialize
);
4715 * Since tx_next_clean is only changed by ix_txeof(),
4716 * which is called only in interrupt handler, the
4717 * check w/o holding tx serializer is MPSAFE.
4720 hdr
= *(txr
->tx_hdr
);
4721 if (hdr
!= txr
->tx_next_clean
) {
4722 lwkt_serialize_enter(&txr
->tx_serialize
);
4724 if (!ifsq_is_empty(txr
->tx_ifsq
))
4725 ifsq_devstart(txr
->tx_ifsq
);
4726 lwkt_serialize_exit(&txr
->tx_serialize
);
4729 IXGBE_WRITE_REG(&rxr
->rx_sc
->hw
, rxr
->rx_eims
, rxr
->rx_eims_val
);
4733 ix_intr_status(struct ix_softc
*sc
, uint32_t eicr
)
4735 struct ixgbe_hw
*hw
= &sc
->hw
;
4737 /* Link status change */
4738 if (eicr
& IXGBE_EICR_LSC
)
4741 if (hw
->mac
.type
!= ixgbe_mac_82598EB
) {
4742 if (eicr
& IXGBE_EICR_ECC
)
4743 if_printf(&sc
->arpcom
.ac_if
, "ECC ERROR!! Reboot!!\n");
4744 else if (eicr
& IXGBE_EICR_GPI_SDP1
)
4746 else if (eicr
& IXGBE_EICR_GPI_SDP2
)
4750 /* Check for fan failure */
4751 if (hw
->device_id
== IXGBE_DEV_ID_82598AT
&&
4752 (eicr
& IXGBE_EICR_GPI_SDP1
))
4753 if_printf(&sc
->arpcom
.ac_if
, "FAN FAILURE!! Replace!!\n");
4755 /* Check for over temp condition */
4756 if (hw
->mac
.type
== ixgbe_mac_X540
&& (eicr
& IXGBE_EICR_TS
)) {
4757 if_printf(&sc
->arpcom
.ac_if
, "OVER TEMP!! "
4758 "PHY IS SHUT DOWN!! Reboot\n");
4763 ix_msix_status(void *xsc
)
4765 struct ix_softc
*sc
= xsc
;
4768 ASSERT_SERIALIZED(&sc
->main_serialize
);
4770 eicr
= IXGBE_READ_REG(&sc
->hw
, IXGBE_EICR
);
4771 ix_intr_status(sc
, eicr
);
4773 IXGBE_WRITE_REG(&sc
->hw
, IXGBE_EIMS
, sc
->intr_mask
);
4777 ix_setup_msix_eims(const struct ix_softc
*sc
, int x
,
4778 uint32_t *eims
, uint32_t *eims_val
)
4781 if (sc
->hw
.mac
.type
== ixgbe_mac_82598EB
) {
4782 KASSERT(x
< IX_MAX_MSIX_82598
,
4783 ("%s: invalid vector %d for 82598",
4784 device_get_nameunit(sc
->dev
), x
));
4787 *eims
= IXGBE_EIMS_EX(0);
4791 KASSERT(x
< IX_MAX_MSIX
, ("%s: invalid vector %d",
4792 device_get_nameunit(sc
->dev
), x
));
4793 KASSERT(sc
->hw
.mac
.type
!= ixgbe_mac_82598EB
,
4794 ("%s: invalid vector %d for 82598",
4795 device_get_nameunit(sc
->dev
), x
));
4796 *eims
= IXGBE_EIMS_EX(1);
4797 *eims_val
= 1 << (x
- 32);
4801 #ifdef IFPOLL_ENABLE
4804 ix_npoll_status(struct ifnet
*ifp
)
4806 struct ix_softc
*sc
= ifp
->if_softc
;
4809 ASSERT_SERIALIZED(&sc
->main_serialize
);
4811 eicr
= IXGBE_READ_REG(&sc
->hw
, IXGBE_EICR
);
4812 ix_intr_status(sc
, eicr
);
4816 ix_npoll_tx(struct ifnet
*ifp
, void *arg
, int cycle __unused
)
4818 struct ix_tx_ring
*txr
= arg
;
4820 ASSERT_SERIALIZED(&txr
->tx_serialize
);
4822 ix_txeof(txr
, *(txr
->tx_hdr
));
4823 if (!ifsq_is_empty(txr
->tx_ifsq
))
4824 ifsq_devstart(txr
->tx_ifsq
);
4828 ix_npoll_rx(struct ifnet
*ifp __unused
, void *arg
, int cycle
)
4830 struct ix_rx_ring
*rxr
= arg
;
4832 ASSERT_SERIALIZED(&rxr
->rx_serialize
);
4834 ix_rxeof(rxr
, cycle
);
4838 ix_npoll(struct ifnet
*ifp
, struct ifpoll_info
*info
)
4840 struct ix_softc
*sc
= ifp
->if_softc
;
4841 int i
, txr_cnt
, rxr_cnt
;
4843 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
4848 info
->ifpi_status
.status_func
= ix_npoll_status
;
4849 info
->ifpi_status
.serializer
= &sc
->main_serialize
;
4851 txr_cnt
= ix_get_txring_inuse(sc
, TRUE
);
4852 off
= sc
->tx_npoll_off
;
4853 for (i
= 0; i
< txr_cnt
; ++i
) {
4854 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
4857 KKASSERT(idx
< ncpus2
);
4858 info
->ifpi_tx
[idx
].poll_func
= ix_npoll_tx
;
4859 info
->ifpi_tx
[idx
].arg
= txr
;
4860 info
->ifpi_tx
[idx
].serializer
= &txr
->tx_serialize
;
4861 ifsq_set_cpuid(txr
->tx_ifsq
, idx
);
4864 rxr_cnt
= ix_get_rxring_inuse(sc
, TRUE
);
4865 off
= sc
->rx_npoll_off
;
4866 for (i
= 0; i
< rxr_cnt
; ++i
) {
4867 struct ix_rx_ring
*rxr
= &sc
->rx_rings
[i
];
4870 KKASSERT(idx
< ncpus2
);
4871 info
->ifpi_rx
[idx
].poll_func
= ix_npoll_rx
;
4872 info
->ifpi_rx
[idx
].arg
= rxr
;
4873 info
->ifpi_rx
[idx
].serializer
= &rxr
->rx_serialize
;
4876 if (ifp
->if_flags
& IFF_RUNNING
) {
4877 if (rxr_cnt
== sc
->rx_ring_inuse
&&
4878 txr_cnt
== sc
->tx_ring_inuse
) {
4879 ix_set_timer_cpuid(sc
, TRUE
);
4880 ix_disable_intr(sc
);
4886 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
4887 struct ix_tx_ring
*txr
= &sc
->tx_rings
[i
];
4889 ifsq_set_cpuid(txr
->tx_ifsq
, txr
->tx_intr_cpuid
);
4892 if (ifp
->if_flags
& IFF_RUNNING
) {
4893 txr_cnt
= ix_get_txring_inuse(sc
, FALSE
);
4894 rxr_cnt
= ix_get_rxring_inuse(sc
, FALSE
);
4896 if (rxr_cnt
== sc
->rx_ring_inuse
&&
4897 txr_cnt
== sc
->tx_ring_inuse
) {
4898 ix_set_timer_cpuid(sc
, FALSE
);
4908 ix_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS
)
4910 struct ix_softc
*sc
= (void *)arg1
;
4911 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4914 off
= sc
->rx_npoll_off
;
4915 error
= sysctl_handle_int(oidp
, &off
, 0, req
);
4916 if (error
|| req
->newptr
== NULL
)
4921 ifnet_serialize_all(ifp
);
4922 if (off
>= ncpus2
|| off
% sc
->rx_ring_cnt
!= 0) {
4926 sc
->rx_npoll_off
= off
;
4928 ifnet_deserialize_all(ifp
);
4934 ix_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS
)
4936 struct ix_softc
*sc
= (void *)arg1
;
4937 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
4940 off
= sc
->tx_npoll_off
;
4941 error
= sysctl_handle_int(oidp
, &off
, 0, req
);
4942 if (error
|| req
->newptr
== NULL
)
4947 ifnet_serialize_all(ifp
);
4948 if (off
>= ncpus2
|| off
% sc
->tx_ring_cnt
!= 0) {
4952 sc
->tx_npoll_off
= off
;
4954 ifnet_deserialize_all(ifp
);
4959 #endif /* IFPOLL_ENABLE */