2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved.
4 * Copyright (c) 2001-2008, Intel Corporation
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the name of the Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
34 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
36 * This code is derived from software contributed to The DragonFly Project
37 * by Matthew Dillon <dillon@backplane.com>
39 * Redistribution and use in source and binary forms, with or without
40 * modification, are permitted provided that the following conditions
43 * 1. Redistributions of source code must retain the above copyright
44 * notice, this list of conditions and the following disclaimer.
45 * 2. Redistributions in binary form must reproduce the above copyright
46 * notice, this list of conditions and the following disclaimer in
47 * the documentation and/or other materials provided with the
49 * 3. Neither the name of The DragonFly Project nor the names of its
50 * contributors may be used to endorse or promote products derived
51 * from this software without specific, prior written permission.
53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
67 #include "opt_ifpoll.h"
70 #include <sys/param.h>
72 #include <sys/endian.h>
73 #include <sys/interrupt.h>
74 #include <sys/kernel.h>
76 #include <sys/malloc.h>
80 #include <sys/serialize.h>
81 #include <sys/serialize2.h>
82 #include <sys/socket.h>
83 #include <sys/sockio.h>
84 #include <sys/sysctl.h>
85 #include <sys/systm.h>
88 #include <net/ethernet.h>
90 #include <net/if_arp.h>
91 #include <net/if_dl.h>
92 #include <net/if_media.h>
93 #include <net/ifq_var.h>
94 #include <net/toeplitz.h>
95 #include <net/toeplitz2.h>
96 #include <net/vlan/if_vlan_var.h>
97 #include <net/vlan/if_vlan_ether.h>
98 #include <net/if_poll.h>
100 #include <netinet/in_systm.h>
101 #include <netinet/in.h>
102 #include <netinet/ip.h>
103 #include <netinet/tcp.h>
104 #include <netinet/udp.h>
106 #include <bus/pci/pcivar.h>
107 #include <bus/pci/pcireg.h>
109 #include <dev/netif/ig_hal/e1000_api.h>
110 #include <dev/netif/ig_hal/e1000_82571.h>
111 #include <dev/netif/ig_hal/e1000_dragonfly.h>
112 #include <dev/netif/emx/if_emx.h>
117 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) \
119 if (sc->rss_debug >= lvl) \
120 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
122 #else /* !EMX_RSS_DEBUG */
123 #define EMX_RSS_DPRINTF(sc, lvl, fmt, ...) ((void)0)
124 #endif /* EMX_RSS_DEBUG */
126 #define EMX_NAME "Intel(R) PRO/1000 "
128 #define EMX_DEVICE(id) \
129 { EMX_VENDOR_ID, E1000_DEV_ID_##id, EMX_NAME #id }
130 #define EMX_DEVICE_NULL { 0, 0, NULL }
132 static const struct emx_device
{
137 EMX_DEVICE(82571EB_COPPER
),
138 EMX_DEVICE(82571EB_FIBER
),
139 EMX_DEVICE(82571EB_SERDES
),
140 EMX_DEVICE(82571EB_SERDES_DUAL
),
141 EMX_DEVICE(82571EB_SERDES_QUAD
),
142 EMX_DEVICE(82571EB_QUAD_COPPER
),
143 EMX_DEVICE(82571EB_QUAD_COPPER_BP
),
144 EMX_DEVICE(82571EB_QUAD_COPPER_LP
),
145 EMX_DEVICE(82571EB_QUAD_FIBER
),
146 EMX_DEVICE(82571PT_QUAD_COPPER
),
148 EMX_DEVICE(82572EI_COPPER
),
149 EMX_DEVICE(82572EI_FIBER
),
150 EMX_DEVICE(82572EI_SERDES
),
154 EMX_DEVICE(82573E_IAMT
),
157 EMX_DEVICE(80003ES2LAN_COPPER_SPT
),
158 EMX_DEVICE(80003ES2LAN_SERDES_SPT
),
159 EMX_DEVICE(80003ES2LAN_COPPER_DPT
),
160 EMX_DEVICE(80003ES2LAN_SERDES_DPT
),
165 EMX_DEVICE(PCH_LPT_I217_LM
),
166 EMX_DEVICE(PCH_LPT_I217_V
),
167 EMX_DEVICE(PCH_LPTLP_I218_LM
),
168 EMX_DEVICE(PCH_LPTLP_I218_V
),
169 EMX_DEVICE(PCH_I218_LM2
),
170 EMX_DEVICE(PCH_I218_V2
),
171 EMX_DEVICE(PCH_I218_LM3
),
172 EMX_DEVICE(PCH_I218_V3
),
173 EMX_DEVICE(PCH_SPT_I219_LM
),
174 EMX_DEVICE(PCH_SPT_I219_V
),
175 EMX_DEVICE(PCH_SPT_I219_LM2
),
176 EMX_DEVICE(PCH_SPT_I219_V2
),
178 /* required last entry */
182 static int emx_probe(device_t
);
183 static int emx_attach(device_t
);
184 static int emx_detach(device_t
);
185 static int emx_shutdown(device_t
);
186 static int emx_suspend(device_t
);
187 static int emx_resume(device_t
);
189 static void emx_init(void *);
190 static void emx_stop(struct emx_softc
*);
191 static int emx_ioctl(struct ifnet
*, u_long
, caddr_t
, struct ucred
*);
192 static void emx_start(struct ifnet
*, struct ifaltq_subque
*);
194 static void emx_npoll(struct ifnet
*, struct ifpoll_info
*);
195 static void emx_npoll_status(struct ifnet
*);
196 static void emx_npoll_tx(struct ifnet
*, void *, int);
197 static void emx_npoll_rx(struct ifnet
*, void *, int);
199 static void emx_watchdog(struct ifaltq_subque
*);
200 static void emx_media_status(struct ifnet
*, struct ifmediareq
*);
201 static int emx_media_change(struct ifnet
*);
202 static void emx_timer(void *);
203 static void emx_serialize(struct ifnet
*, enum ifnet_serialize
);
204 static void emx_deserialize(struct ifnet
*, enum ifnet_serialize
);
205 static int emx_tryserialize(struct ifnet
*, enum ifnet_serialize
);
207 static void emx_serialize_assert(struct ifnet
*, enum ifnet_serialize
,
211 static void emx_intr(void *);
212 static void emx_intr_mask(void *);
213 static void emx_intr_body(struct emx_softc
*, boolean_t
);
214 static void emx_rxeof(struct emx_rxdata
*, int);
215 static void emx_txeof(struct emx_txdata
*);
216 static void emx_tx_collect(struct emx_txdata
*);
217 static void emx_tx_purge(struct emx_softc
*);
218 static void emx_enable_intr(struct emx_softc
*);
219 static void emx_disable_intr(struct emx_softc
*);
221 static int emx_dma_alloc(struct emx_softc
*);
222 static void emx_dma_free(struct emx_softc
*);
223 static void emx_init_tx_ring(struct emx_txdata
*);
224 static int emx_init_rx_ring(struct emx_rxdata
*);
225 static void emx_free_tx_ring(struct emx_txdata
*);
226 static void emx_free_rx_ring(struct emx_rxdata
*);
227 static int emx_create_tx_ring(struct emx_txdata
*);
228 static int emx_create_rx_ring(struct emx_rxdata
*);
229 static void emx_destroy_tx_ring(struct emx_txdata
*, int);
230 static void emx_destroy_rx_ring(struct emx_rxdata
*, int);
231 static int emx_newbuf(struct emx_rxdata
*, int, int);
232 static int emx_encap(struct emx_txdata
*, struct mbuf
**, int *, int *);
233 static int emx_txcsum(struct emx_txdata
*, struct mbuf
*,
234 uint32_t *, uint32_t *);
235 static int emx_tso_pullup(struct emx_txdata
*, struct mbuf
**);
236 static int emx_tso_setup(struct emx_txdata
*, struct mbuf
*,
237 uint32_t *, uint32_t *);
238 static int emx_get_txring_inuse(const struct emx_softc
*, boolean_t
);
240 static int emx_is_valid_eaddr(const uint8_t *);
241 static int emx_reset(struct emx_softc
*);
242 static void emx_setup_ifp(struct emx_softc
*);
243 static void emx_init_tx_unit(struct emx_softc
*);
244 static void emx_init_rx_unit(struct emx_softc
*);
245 static void emx_update_stats(struct emx_softc
*);
246 static void emx_set_promisc(struct emx_softc
*);
247 static void emx_disable_promisc(struct emx_softc
*);
248 static void emx_set_multi(struct emx_softc
*);
249 static void emx_update_link_status(struct emx_softc
*);
250 static void emx_smartspeed(struct emx_softc
*);
251 static void emx_set_itr(struct emx_softc
*, uint32_t);
252 static void emx_disable_aspm(struct emx_softc
*);
254 static void emx_print_debug_info(struct emx_softc
*);
255 static void emx_print_nvm_info(struct emx_softc
*);
256 static void emx_print_hw_stats(struct emx_softc
*);
258 static int emx_sysctl_stats(SYSCTL_HANDLER_ARGS
);
259 static int emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS
);
260 static int emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS
);
261 static int emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS
);
262 static int emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS
);
264 static int emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS
);
265 static int emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS
);
267 static void emx_add_sysctl(struct emx_softc
*);
269 static void emx_serialize_skipmain(struct emx_softc
*);
270 static void emx_deserialize_skipmain(struct emx_softc
*);
272 /* Management and WOL Support */
273 static void emx_get_mgmt(struct emx_softc
*);
274 static void emx_rel_mgmt(struct emx_softc
*);
275 static void emx_get_hw_control(struct emx_softc
*);
276 static void emx_rel_hw_control(struct emx_softc
*);
277 static void emx_enable_wol(device_t
);
279 static device_method_t emx_methods
[] = {
280 /* Device interface */
281 DEVMETHOD(device_probe
, emx_probe
),
282 DEVMETHOD(device_attach
, emx_attach
),
283 DEVMETHOD(device_detach
, emx_detach
),
284 DEVMETHOD(device_shutdown
, emx_shutdown
),
285 DEVMETHOD(device_suspend
, emx_suspend
),
286 DEVMETHOD(device_resume
, emx_resume
),
290 static driver_t emx_driver
= {
293 sizeof(struct emx_softc
),
296 static devclass_t emx_devclass
;
298 DECLARE_DUMMY_MODULE(if_emx
);
299 MODULE_DEPEND(emx
, ig_hal
, 1, 1, 1);
300 DRIVER_MODULE(if_emx
, pci
, emx_driver
, emx_devclass
, NULL
, NULL
);
305 static int emx_int_throttle_ceil
= EMX_DEFAULT_ITR
;
306 static int emx_rxd
= EMX_DEFAULT_RXD
;
307 static int emx_txd
= EMX_DEFAULT_TXD
;
308 static int emx_smart_pwr_down
= 0;
309 static int emx_rxr
= 0;
310 static int emx_txr
= 1;
312 /* Controls whether promiscuous also shows bad packets */
313 static int emx_debug_sbp
= 0;
315 static int emx_82573_workaround
= 1;
316 static int emx_msi_enable
= 1;
318 static char emx_flowctrl
[IFM_ETH_FC_STRLEN
] = IFM_ETH_FC_RXPAUSE
;
320 TUNABLE_INT("hw.emx.int_throttle_ceil", &emx_int_throttle_ceil
);
321 TUNABLE_INT("hw.emx.rxd", &emx_rxd
);
322 TUNABLE_INT("hw.emx.rxr", &emx_rxr
);
323 TUNABLE_INT("hw.emx.txd", &emx_txd
);
324 TUNABLE_INT("hw.emx.txr", &emx_txr
);
325 TUNABLE_INT("hw.emx.smart_pwr_down", &emx_smart_pwr_down
);
326 TUNABLE_INT("hw.emx.sbp", &emx_debug_sbp
);
327 TUNABLE_INT("hw.emx.82573_workaround", &emx_82573_workaround
);
328 TUNABLE_INT("hw.emx.msi.enable", &emx_msi_enable
);
329 TUNABLE_STR("hw.emx.flow_ctrl", emx_flowctrl
, sizeof(emx_flowctrl
));
331 /* Global used in WOL setup with multiport cards */
332 static int emx_global_quad_port_a
= 0;
334 /* Set this to one to display debug statistics */
335 static int emx_display_debug_stats
= 0;
337 #if !defined(KTR_IF_EMX)
338 #define KTR_IF_EMX KTR_ALL
340 KTR_INFO_MASTER(if_emx
);
341 KTR_INFO(KTR_IF_EMX
, if_emx
, intr_beg
, 0, "intr begin");
342 KTR_INFO(KTR_IF_EMX
, if_emx
, intr_end
, 1, "intr end");
343 KTR_INFO(KTR_IF_EMX
, if_emx
, pkt_receive
, 4, "rx packet");
344 KTR_INFO(KTR_IF_EMX
, if_emx
, pkt_txqueue
, 5, "tx packet");
345 KTR_INFO(KTR_IF_EMX
, if_emx
, pkt_txclean
, 6, "tx clean");
346 #define logif(name) KTR_LOG(if_emx_ ## name)
349 emx_setup_rxdesc(emx_rxdesc_t
*rxd
, const struct emx_rxbuf
*rxbuf
)
351 rxd
->rxd_bufaddr
= htole64(rxbuf
->paddr
);
352 /* DD bit must be cleared */
353 rxd
->rxd_staterr
= 0;
357 emx_rxcsum(uint32_t staterr
, struct mbuf
*mp
)
359 /* Ignore Checksum bit is set */
360 if (staterr
& E1000_RXD_STAT_IXSM
)
363 if ((staterr
& (E1000_RXD_STAT_IPCS
| E1000_RXDEXT_STATERR_IPE
)) ==
365 mp
->m_pkthdr
.csum_flags
|= CSUM_IP_CHECKED
| CSUM_IP_VALID
;
367 if ((staterr
& (E1000_RXD_STAT_TCPCS
| E1000_RXDEXT_STATERR_TCPE
)) ==
368 E1000_RXD_STAT_TCPCS
) {
369 mp
->m_pkthdr
.csum_flags
|= CSUM_DATA_VALID
|
371 CSUM_FRAG_NOT_CHECKED
;
372 mp
->m_pkthdr
.csum_data
= htons(0xffff);
376 static __inline
struct pktinfo
*
377 emx_rssinfo(struct mbuf
*m
, struct pktinfo
*pi
,
378 uint32_t mrq
, uint32_t hash
, uint32_t staterr
)
380 switch (mrq
& EMX_RXDMRQ_RSSTYPE_MASK
) {
381 case EMX_RXDMRQ_IPV4_TCP
:
382 pi
->pi_netisr
= NETISR_IP
;
384 pi
->pi_l3proto
= IPPROTO_TCP
;
387 case EMX_RXDMRQ_IPV6_TCP
:
388 pi
->pi_netisr
= NETISR_IPV6
;
390 pi
->pi_l3proto
= IPPROTO_TCP
;
393 case EMX_RXDMRQ_IPV4
:
394 if (staterr
& E1000_RXD_STAT_IXSM
)
398 (E1000_RXD_STAT_TCPCS
| E1000_RXDEXT_STATERR_TCPE
)) ==
399 E1000_RXD_STAT_TCPCS
) {
400 pi
->pi_netisr
= NETISR_IP
;
402 pi
->pi_l3proto
= IPPROTO_UDP
;
410 m_sethash(m
, toeplitz_hash(hash
));
415 emx_probe(device_t dev
)
417 const struct emx_device
*d
;
420 vid
= pci_get_vendor(dev
);
421 did
= pci_get_device(dev
);
423 for (d
= emx_devices
; d
->desc
!= NULL
; ++d
) {
424 if (vid
== d
->vid
&& did
== d
->did
) {
425 device_set_desc(dev
, d
->desc
);
426 device_set_async_attach(dev
, TRUE
);
434 emx_attach(device_t dev
)
436 struct emx_softc
*sc
= device_get_softc(dev
);
437 int error
= 0, i
, throttle
, msi_enable
, tx_ring_max
;
439 uint16_t eeprom_data
, device_id
, apme_mask
;
440 driver_intr_t
*intr_func
;
441 char flowctrl
[IFM_ETH_FC_STRLEN
];
443 int offset
, offset_def
;
449 for (i
= 0; i
< EMX_NRX_RING
; ++i
) {
450 sc
->rx_data
[i
].sc
= sc
;
451 sc
->rx_data
[i
].idx
= i
;
457 for (i
= 0; i
< EMX_NTX_RING
; ++i
) {
458 sc
->tx_data
[i
].sc
= sc
;
459 sc
->tx_data
[i
].idx
= i
;
463 * Initialize serializers
465 lwkt_serialize_init(&sc
->main_serialize
);
466 for (i
= 0; i
< EMX_NTX_RING
; ++i
)
467 lwkt_serialize_init(&sc
->tx_data
[i
].tx_serialize
);
468 for (i
= 0; i
< EMX_NRX_RING
; ++i
)
469 lwkt_serialize_init(&sc
->rx_data
[i
].rx_serialize
);
472 * Initialize serializer array
476 KKASSERT(i
< EMX_NSERIALIZE
);
477 sc
->serializes
[i
++] = &sc
->main_serialize
;
479 KKASSERT(i
< EMX_NSERIALIZE
);
480 sc
->serializes
[i
++] = &sc
->tx_data
[0].tx_serialize
;
481 KKASSERT(i
< EMX_NSERIALIZE
);
482 sc
->serializes
[i
++] = &sc
->tx_data
[1].tx_serialize
;
484 KKASSERT(i
< EMX_NSERIALIZE
);
485 sc
->serializes
[i
++] = &sc
->rx_data
[0].rx_serialize
;
486 KKASSERT(i
< EMX_NSERIALIZE
);
487 sc
->serializes
[i
++] = &sc
->rx_data
[1].rx_serialize
;
489 KKASSERT(i
== EMX_NSERIALIZE
);
491 ifmedia_init(&sc
->media
, IFM_IMASK
| IFM_ETH_FCMASK
,
492 emx_media_change
, emx_media_status
);
493 callout_init_mp(&sc
->timer
);
495 sc
->dev
= sc
->osdep
.dev
= dev
;
498 * Determine hardware and mac type
500 sc
->hw
.vendor_id
= pci_get_vendor(dev
);
501 sc
->hw
.device_id
= pci_get_device(dev
);
502 sc
->hw
.revision_id
= pci_get_revid(dev
);
503 sc
->hw
.subsystem_vendor_id
= pci_get_subvendor(dev
);
504 sc
->hw
.subsystem_device_id
= pci_get_subdevice(dev
);
506 if (e1000_set_mac_type(&sc
->hw
))
509 /* Enable bus mastering */
510 pci_enable_busmaster(dev
);
515 sc
->memory_rid
= EMX_BAR_MEM
;
516 sc
->memory
= bus_alloc_resource_any(dev
, SYS_RES_MEMORY
,
517 &sc
->memory_rid
, RF_ACTIVE
);
518 if (sc
->memory
== NULL
) {
519 device_printf(dev
, "Unable to allocate bus resource: memory\n");
523 sc
->osdep
.mem_bus_space_tag
= rman_get_bustag(sc
->memory
);
524 sc
->osdep
.mem_bus_space_handle
= rman_get_bushandle(sc
->memory
);
526 /* XXX This is quite goofy, it is not actually used */
527 sc
->hw
.hw_addr
= (uint8_t *)&sc
->osdep
.mem_bus_space_handle
;
530 * Don't enable MSI-X on 82574, see:
531 * 82574 specification update errata #15
533 * Don't enable MSI on 82571/82572, see:
534 * 82571/82572 specification update errata #63
536 msi_enable
= emx_msi_enable
;
538 (sc
->hw
.mac
.type
== e1000_82571
||
539 sc
->hw
.mac
.type
== e1000_82572
))
545 sc
->intr_type
= pci_alloc_1intr(dev
, msi_enable
,
546 &sc
->intr_rid
, &intr_flags
);
548 if (sc
->intr_type
== PCI_INTR_TYPE_LEGACY
) {
551 unshared
= device_getenv_int(dev
, "irq.unshared", 0);
553 sc
->flags
|= EMX_FLAG_SHARED_INTR
;
555 device_printf(dev
, "IRQ shared\n");
557 intr_flags
&= ~RF_SHAREABLE
;
559 device_printf(dev
, "IRQ unshared\n");
563 sc
->intr_res
= bus_alloc_resource_any(dev
, SYS_RES_IRQ
, &sc
->intr_rid
,
565 if (sc
->intr_res
== NULL
) {
566 device_printf(dev
, "Unable to allocate bus resource: "
572 /* Save PCI command register for Shared Code */
573 sc
->hw
.bus
.pci_cmd_word
= pci_read_config(dev
, PCIR_COMMAND
, 2);
574 sc
->hw
.back
= &sc
->osdep
;
577 * For I217/I218, we need to map the flash memory and this
578 * must happen after the MAC is identified.
580 if (sc
->hw
.mac
.type
== e1000_pch_lpt
) {
581 sc
->flash_rid
= EMX_BAR_FLASH
;
583 sc
->flash
= bus_alloc_resource_any(dev
, SYS_RES_MEMORY
,
584 &sc
->flash_rid
, RF_ACTIVE
);
585 if (sc
->flash
== NULL
) {
586 device_printf(dev
, "Mapping of Flash failed\n");
590 sc
->osdep
.flash_bus_space_tag
= rman_get_bustag(sc
->flash
);
591 sc
->osdep
.flash_bus_space_handle
=
592 rman_get_bushandle(sc
->flash
);
595 * This is used in the shared code
596 * XXX this goof is actually not used.
598 sc
->hw
.flash_address
= (uint8_t *)sc
->flash
;
601 /* Do Shared Code initialization */
602 if (e1000_setup_init_funcs(&sc
->hw
, TRUE
)) {
603 device_printf(dev
, "Setup of Shared code failed\n");
607 e1000_get_bus_info(&sc
->hw
);
609 sc
->hw
.mac
.autoneg
= EMX_DO_AUTO_NEG
;
610 sc
->hw
.phy
.autoneg_wait_to_complete
= FALSE
;
611 sc
->hw
.phy
.autoneg_advertised
= EMX_AUTONEG_ADV_DEFAULT
;
614 * Interrupt throttle rate
616 throttle
= device_getenv_int(dev
, "int_throttle_ceil",
617 emx_int_throttle_ceil
);
619 sc
->int_throttle_ceil
= 0;
622 throttle
= EMX_DEFAULT_ITR
;
624 /* Recalculate the tunable value to get the exact frequency. */
625 throttle
= 1000000000 / 256 / throttle
;
627 /* Upper 16bits of ITR is reserved and should be zero */
628 if (throttle
& 0xffff0000)
629 throttle
= 1000000000 / 256 / EMX_DEFAULT_ITR
;
631 sc
->int_throttle_ceil
= 1000000000 / 256 / throttle
;
634 e1000_init_script_state_82541(&sc
->hw
, TRUE
);
635 e1000_set_tbi_compatibility_82543(&sc
->hw
, TRUE
);
638 if (sc
->hw
.phy
.media_type
== e1000_media_type_copper
) {
639 sc
->hw
.phy
.mdix
= EMX_AUTO_ALL_MODES
;
640 sc
->hw
.phy
.disable_polarity_correction
= FALSE
;
641 sc
->hw
.phy
.ms_type
= EMX_MASTER_SLAVE
;
644 /* Set the frame limits assuming standard ethernet sized frames. */
645 sc
->hw
.mac
.max_frame_size
= ETHERMTU
+ ETHER_HDR_LEN
+ ETHER_CRC_LEN
;
647 /* This controls when hardware reports transmit completion status. */
648 sc
->hw
.mac
.report_tx_early
= 1;
650 /* Calculate # of RX rings */
651 sc
->rx_ring_cnt
= device_getenv_int(dev
, "rxr", emx_rxr
);
652 sc
->rx_ring_cnt
= if_ring_count2(sc
->rx_ring_cnt
, EMX_NRX_RING
);
655 * Calculate # of TX rings
658 * I217/I218 claims to have 2 TX queues
661 * Don't enable multiple TX queues on 82574; it always gives
662 * watchdog timeout on TX queue0, when multiple TCP streams are
663 * received. It was originally suspected that the hardware TX
664 * checksum offloading caused this watchdog timeout, since only
665 * TCP ACKs are sent during TCP receiving tests. However, even
666 * if the hardware TX checksum offloading is disable, TX queue0
667 * still will give watchdog.
670 if (sc
->hw
.mac
.type
== e1000_82571
||
671 sc
->hw
.mac
.type
== e1000_82572
||
672 sc
->hw
.mac
.type
== e1000_80003es2lan
||
673 sc
->hw
.mac
.type
== e1000_pch_lpt
||
674 sc
->hw
.mac
.type
== e1000_pch_spt
||
675 sc
->hw
.mac
.type
== e1000_82574
)
676 tx_ring_max
= EMX_NTX_RING
;
677 sc
->tx_ring_cnt
= device_getenv_int(dev
, "txr", emx_txr
);
678 sc
->tx_ring_cnt
= if_ring_count2(sc
->tx_ring_cnt
, tx_ring_max
);
680 /* Allocate RX/TX rings' busdma(9) stuffs */
681 error
= emx_dma_alloc(sc
);
685 /* Allocate multicast array memory. */
686 sc
->mta
= kmalloc(ETH_ADDR_LEN
* EMX_MCAST_ADDR_MAX
,
689 /* Indicate SOL/IDER usage */
690 if (e1000_check_reset_block(&sc
->hw
)) {
692 "PHY reset is blocked due to SOL/IDER session.\n");
695 /* Disable EEE on I217/I218 */
696 sc
->hw
.dev_spec
.ich8lan
.eee_disable
= 1;
699 * Start from a known state, this is important in reading the
700 * nvm and mac from that.
702 e1000_reset_hw(&sc
->hw
);
704 /* Make sure we have a good EEPROM before we read from it */
705 if (e1000_validate_nvm_checksum(&sc
->hw
) < 0) {
707 * Some PCI-E parts fail the first check due to
708 * the link being in sleep state, call it again,
709 * if it fails a second time its a real issue.
711 if (e1000_validate_nvm_checksum(&sc
->hw
) < 0) {
713 "The EEPROM Checksum Is Not Valid\n");
719 /* Copy the permanent MAC address out of the EEPROM */
720 if (e1000_read_mac_addr(&sc
->hw
) < 0) {
721 device_printf(dev
, "EEPROM read error while reading MAC"
726 if (!emx_is_valid_eaddr(sc
->hw
.mac
.addr
)) {
727 device_printf(dev
, "Invalid MAC address\n");
732 /* Disable ULP support */
733 e1000_disable_ulp_lpt_lp(&sc
->hw
, TRUE
);
735 /* Determine if we have to control management hardware */
736 if (e1000_enable_mng_pass_thru(&sc
->hw
))
737 sc
->flags
|= EMX_FLAG_HAS_MGMT
;
742 apme_mask
= EMX_EEPROM_APME
;
744 switch (sc
->hw
.mac
.type
) {
746 sc
->flags
|= EMX_FLAG_HAS_AMT
;
751 case e1000_80003es2lan
:
752 if (sc
->hw
.bus
.func
== 1) {
753 e1000_read_nvm(&sc
->hw
,
754 NVM_INIT_CONTROL3_PORT_B
, 1, &eeprom_data
);
756 e1000_read_nvm(&sc
->hw
,
757 NVM_INIT_CONTROL3_PORT_A
, 1, &eeprom_data
);
762 e1000_read_nvm(&sc
->hw
,
763 NVM_INIT_CONTROL3_PORT_A
, 1, &eeprom_data
);
766 if (eeprom_data
& apme_mask
)
767 sc
->wol
= E1000_WUFC_MAG
| E1000_WUFC_MC
;
770 * We have the eeprom settings, now apply the special cases
771 * where the eeprom may be wrong or the board won't support
772 * wake on lan on a particular port
774 device_id
= pci_get_device(dev
);
776 case E1000_DEV_ID_82571EB_FIBER
:
778 * Wake events only supported on port A for dual fiber
779 * regardless of eeprom setting
781 if (E1000_READ_REG(&sc
->hw
, E1000_STATUS
) &
786 case E1000_DEV_ID_82571EB_QUAD_COPPER
:
787 case E1000_DEV_ID_82571EB_QUAD_FIBER
:
788 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP
:
789 /* if quad port sc, disable WoL on all but port A */
790 if (emx_global_quad_port_a
!= 0)
792 /* Reset for multiple quad port adapters */
793 if (++emx_global_quad_port_a
== 4)
794 emx_global_quad_port_a
= 0;
798 /* XXX disable wol */
803 * NPOLLING RX CPU offset
805 if (sc
->rx_ring_cnt
== ncpus2
) {
808 offset_def
= (sc
->rx_ring_cnt
* device_get_unit(dev
)) % ncpus2
;
809 offset
= device_getenv_int(dev
, "npoll.rxoff", offset_def
);
810 if (offset
>= ncpus2
||
811 offset
% sc
->rx_ring_cnt
!= 0) {
812 device_printf(dev
, "invalid npoll.rxoff %d, use %d\n",
817 sc
->rx_npoll_off
= offset
;
820 * NPOLLING TX CPU offset
822 if (sc
->tx_ring_cnt
== ncpus2
) {
825 offset_def
= (sc
->tx_ring_cnt
* device_get_unit(dev
)) % ncpus2
;
826 offset
= device_getenv_int(dev
, "npoll.txoff", offset_def
);
827 if (offset
>= ncpus2
||
828 offset
% sc
->tx_ring_cnt
!= 0) {
829 device_printf(dev
, "invalid npoll.txoff %d, use %d\n",
834 sc
->tx_npoll_off
= offset
;
836 sc
->tx_ring_inuse
= emx_get_txring_inuse(sc
, FALSE
);
838 /* Setup flow control. */
839 device_getenv_string(dev
, "flow_ctrl", flowctrl
, sizeof(flowctrl
),
841 sc
->ifm_flowctrl
= ifmedia_str2ethfc(flowctrl
);
843 /* Setup OS specific network interface */
846 /* Add sysctl tree, must after em_setup_ifp() */
849 /* Reset the hardware */
850 error
= emx_reset(sc
);
853 * Some 82573 parts fail the first reset, call it again,
854 * if it fails a second time its a real issue.
856 error
= emx_reset(sc
);
858 device_printf(dev
, "Unable to reset the hardware\n");
859 ether_ifdetach(&sc
->arpcom
.ac_if
);
864 /* Initialize statistics */
865 emx_update_stats(sc
);
867 sc
->hw
.mac
.get_link_status
= 1;
868 emx_update_link_status(sc
);
870 /* Non-AMT based hardware can now take control from firmware */
871 if ((sc
->flags
& (EMX_FLAG_HAS_MGMT
| EMX_FLAG_HAS_AMT
)) ==
873 emx_get_hw_control(sc
);
876 * Missing Interrupt Following ICR read:
878 * 82571/82572 specification update errata #76
879 * 82573 specification update errata #31
880 * 82574 specification update errata #12
882 intr_func
= emx_intr
;
883 if ((sc
->flags
& EMX_FLAG_SHARED_INTR
) &&
884 (sc
->hw
.mac
.type
== e1000_82571
||
885 sc
->hw
.mac
.type
== e1000_82572
||
886 sc
->hw
.mac
.type
== e1000_82573
||
887 sc
->hw
.mac
.type
== e1000_82574
))
888 intr_func
= emx_intr_mask
;
890 error
= bus_setup_intr(dev
, sc
->intr_res
, INTR_MPSAFE
, intr_func
, sc
,
891 &sc
->intr_tag
, &sc
->main_serialize
);
893 device_printf(dev
, "Failed to register interrupt handler");
894 ether_ifdetach(&sc
->arpcom
.ac_if
);
904 emx_detach(device_t dev
)
906 struct emx_softc
*sc
= device_get_softc(dev
);
908 if (device_is_attached(dev
)) {
909 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
911 ifnet_serialize_all(ifp
);
915 e1000_phy_hw_reset(&sc
->hw
);
918 emx_rel_hw_control(sc
);
921 E1000_WRITE_REG(&sc
->hw
, E1000_WUC
, E1000_WUC_PME_EN
);
922 E1000_WRITE_REG(&sc
->hw
, E1000_WUFC
, sc
->wol
);
926 bus_teardown_intr(dev
, sc
->intr_res
, sc
->intr_tag
);
928 ifnet_deserialize_all(ifp
);
931 } else if (sc
->memory
!= NULL
) {
932 emx_rel_hw_control(sc
);
935 ifmedia_removeall(&sc
->media
);
936 bus_generic_detach(dev
);
938 if (sc
->intr_res
!= NULL
) {
939 bus_release_resource(dev
, SYS_RES_IRQ
, sc
->intr_rid
,
943 if (sc
->intr_type
== PCI_INTR_TYPE_MSI
)
944 pci_release_msi(dev
);
946 if (sc
->memory
!= NULL
) {
947 bus_release_resource(dev
, SYS_RES_MEMORY
, sc
->memory_rid
,
951 if (sc
->flash
!= NULL
) {
952 bus_release_resource(dev
, SYS_RES_MEMORY
, sc
->flash_rid
,
959 kfree(sc
->mta
, M_DEVBUF
);
965 emx_shutdown(device_t dev
)
967 return emx_suspend(dev
);
971 emx_suspend(device_t dev
)
973 struct emx_softc
*sc
= device_get_softc(dev
);
974 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
976 ifnet_serialize_all(ifp
);
981 emx_rel_hw_control(sc
);
984 E1000_WRITE_REG(&sc
->hw
, E1000_WUC
, E1000_WUC_PME_EN
);
985 E1000_WRITE_REG(&sc
->hw
, E1000_WUFC
, sc
->wol
);
989 ifnet_deserialize_all(ifp
);
991 return bus_generic_suspend(dev
);
995 emx_resume(device_t dev
)
997 struct emx_softc
*sc
= device_get_softc(dev
);
998 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1001 ifnet_serialize_all(ifp
);
1005 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
)
1006 ifsq_devstart_sched(sc
->tx_data
[i
].ifsq
);
1008 ifnet_deserialize_all(ifp
);
1010 return bus_generic_resume(dev
);
1014 emx_start(struct ifnet
*ifp
, struct ifaltq_subque
*ifsq
)
1016 struct emx_softc
*sc
= ifp
->if_softc
;
1017 struct emx_txdata
*tdata
= ifsq_get_priv(ifsq
);
1018 struct mbuf
*m_head
;
1019 int idx
= -1, nsegs
= 0;
1021 KKASSERT(tdata
->ifsq
== ifsq
);
1022 ASSERT_SERIALIZED(&tdata
->tx_serialize
);
1024 if ((ifp
->if_flags
& IFF_RUNNING
) == 0 || ifsq_is_oactive(ifsq
))
1027 if (!sc
->link_active
|| (tdata
->tx_flags
& EMX_TXFLAG_ENABLED
) == 0) {
1032 while (!ifsq_is_empty(ifsq
)) {
1033 /* Now do we at least have a minimal? */
1034 if (EMX_IS_OACTIVE(tdata
)) {
1035 emx_tx_collect(tdata
);
1036 if (EMX_IS_OACTIVE(tdata
)) {
1037 ifsq_set_oactive(ifsq
);
1043 m_head
= ifsq_dequeue(ifsq
);
1047 if (emx_encap(tdata
, &m_head
, &nsegs
, &idx
)) {
1048 IFNET_STAT_INC(ifp
, oerrors
, 1);
1049 emx_tx_collect(tdata
);
1054 * TX interrupt are aggressively aggregated, so increasing
1055 * opackets at TX interrupt time will make the opackets
1056 * statistics vastly inaccurate; we do the opackets increment
1059 IFNET_STAT_INC(ifp
, opackets
, 1);
1061 if (nsegs
>= tdata
->tx_wreg_nsegs
) {
1062 E1000_WRITE_REG(&sc
->hw
, E1000_TDT(tdata
->idx
), idx
);
1067 /* Send a copy of the frame to the BPF listener */
1068 ETHER_BPF_MTAP(ifp
, m_head
);
1070 /* Set timeout in case hardware has problems transmitting. */
1071 tdata
->tx_watchdog
.wd_timer
= EMX_TX_TIMEOUT
;
1074 E1000_WRITE_REG(&sc
->hw
, E1000_TDT(tdata
->idx
), idx
);
1078 emx_ioctl(struct ifnet
*ifp
, u_long command
, caddr_t data
, struct ucred
*cr
)
1080 struct emx_softc
*sc
= ifp
->if_softc
;
1081 struct ifreq
*ifr
= (struct ifreq
*)data
;
1082 uint16_t eeprom_data
= 0;
1083 int max_frame_size
, mask
, reinit
;
1086 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1090 switch (sc
->hw
.mac
.type
) {
1093 * 82573 only supports jumbo frames
1094 * if ASPM is disabled.
1096 e1000_read_nvm(&sc
->hw
, NVM_INIT_3GIO_3
, 1,
1098 if (eeprom_data
& NVM_WORD1A_ASPM_MASK
) {
1099 max_frame_size
= ETHER_MAX_LEN
;
1104 /* Limit Jumbo Frame size */
1110 case e1000_80003es2lan
:
1111 max_frame_size
= 9234;
1115 max_frame_size
= MAX_JUMBO_FRAME_SIZE
;
1118 if (ifr
->ifr_mtu
> max_frame_size
- ETHER_HDR_LEN
-
1124 ifp
->if_mtu
= ifr
->ifr_mtu
;
1125 sc
->hw
.mac
.max_frame_size
= ifp
->if_mtu
+ ETHER_HDR_LEN
+
1128 if (ifp
->if_flags
& IFF_RUNNING
)
1133 if (ifp
->if_flags
& IFF_UP
) {
1134 if ((ifp
->if_flags
& IFF_RUNNING
)) {
1135 if ((ifp
->if_flags
^ sc
->if_flags
) &
1136 (IFF_PROMISC
| IFF_ALLMULTI
)) {
1137 emx_disable_promisc(sc
);
1138 emx_set_promisc(sc
);
1143 } else if (ifp
->if_flags
& IFF_RUNNING
) {
1146 sc
->if_flags
= ifp
->if_flags
;
1151 if (ifp
->if_flags
& IFF_RUNNING
) {
1152 emx_disable_intr(sc
);
1154 #ifdef IFPOLL_ENABLE
1155 if (!(ifp
->if_flags
& IFF_NPOLLING
))
1157 emx_enable_intr(sc
);
1162 /* Check SOL/IDER usage */
1163 if (e1000_check_reset_block(&sc
->hw
)) {
1164 device_printf(sc
->dev
, "Media change is"
1165 " blocked due to SOL/IDER session.\n");
1171 error
= ifmedia_ioctl(ifp
, ifr
, &sc
->media
, command
);
1176 mask
= ifr
->ifr_reqcap
^ ifp
->if_capenable
;
1177 if (mask
& IFCAP_RXCSUM
) {
1178 ifp
->if_capenable
^= IFCAP_RXCSUM
;
1181 if (mask
& IFCAP_VLAN_HWTAGGING
) {
1182 ifp
->if_capenable
^= IFCAP_VLAN_HWTAGGING
;
1185 if (mask
& IFCAP_TXCSUM
) {
1186 ifp
->if_capenable
^= IFCAP_TXCSUM
;
1187 if (ifp
->if_capenable
& IFCAP_TXCSUM
)
1188 ifp
->if_hwassist
|= EMX_CSUM_FEATURES
;
1190 ifp
->if_hwassist
&= ~EMX_CSUM_FEATURES
;
1192 if (mask
& IFCAP_TSO
) {
1193 ifp
->if_capenable
^= IFCAP_TSO
;
1194 if (ifp
->if_capenable
& IFCAP_TSO
)
1195 ifp
->if_hwassist
|= CSUM_TSO
;
1197 ifp
->if_hwassist
&= ~CSUM_TSO
;
1199 if (mask
& IFCAP_RSS
)
1200 ifp
->if_capenable
^= IFCAP_RSS
;
1201 if (reinit
&& (ifp
->if_flags
& IFF_RUNNING
))
1206 error
= ether_ioctl(ifp
, command
, data
);
1213 emx_watchdog(struct ifaltq_subque
*ifsq
)
1215 struct emx_txdata
*tdata
= ifsq_get_priv(ifsq
);
1216 struct ifnet
*ifp
= ifsq_get_ifp(ifsq
);
1217 struct emx_softc
*sc
= ifp
->if_softc
;
1220 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1223 * The timer is set to 5 every time start queues a packet.
1224 * Then txeof keeps resetting it as long as it cleans at
1225 * least one descriptor.
1226 * Finally, anytime all descriptors are clean the timer is
1230 if (E1000_READ_REG(&sc
->hw
, E1000_TDT(tdata
->idx
)) ==
1231 E1000_READ_REG(&sc
->hw
, E1000_TDH(tdata
->idx
))) {
1233 * If we reach here, all TX jobs are completed and
1234 * the TX engine should have been idled for some time.
1235 * We don't need to call ifsq_devstart_sched() here.
1237 ifsq_clr_oactive(ifsq
);
1238 tdata
->tx_watchdog
.wd_timer
= 0;
1243 * If we are in this routine because of pause frames, then
1244 * don't reset the hardware.
1246 if (E1000_READ_REG(&sc
->hw
, E1000_STATUS
) & E1000_STATUS_TXOFF
) {
1247 tdata
->tx_watchdog
.wd_timer
= EMX_TX_TIMEOUT
;
1251 if_printf(ifp
, "TX %d watchdog timeout -- resetting\n", tdata
->idx
);
1253 IFNET_STAT_INC(ifp
, oerrors
, 1);
1256 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
)
1257 ifsq_devstart_sched(sc
->tx_data
[i
].ifsq
);
1263 struct emx_softc
*sc
= xsc
;
1264 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1265 device_t dev
= sc
->dev
;
1269 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1273 /* Get the latest mac address, User can use a LAA */
1274 bcopy(IF_LLADDR(ifp
), sc
->hw
.mac
.addr
, ETHER_ADDR_LEN
);
1276 /* Put the address into the Receive Address Array */
1277 e1000_rar_set(&sc
->hw
, sc
->hw
.mac
.addr
, 0);
1280 * With the 82571 sc, RAR[0] may be overwritten
1281 * when the other port is reset, we make a duplicate
1282 * in RAR[14] for that eventuality, this assures
1283 * the interface continues to function.
1285 if (sc
->hw
.mac
.type
== e1000_82571
) {
1286 e1000_set_laa_state_82571(&sc
->hw
, TRUE
);
1287 e1000_rar_set(&sc
->hw
, sc
->hw
.mac
.addr
,
1288 E1000_RAR_ENTRIES
- 1);
1291 /* Initialize the hardware */
1292 if (emx_reset(sc
)) {
1293 device_printf(dev
, "Unable to reset the hardware\n");
1294 /* XXX emx_stop()? */
1297 emx_update_link_status(sc
);
1299 /* Setup VLAN support, basic and offload if available */
1300 E1000_WRITE_REG(&sc
->hw
, E1000_VET
, ETHERTYPE_VLAN
);
1302 if (ifp
->if_capenable
& IFCAP_VLAN_HWTAGGING
) {
1305 ctrl
= E1000_READ_REG(&sc
->hw
, E1000_CTRL
);
1306 ctrl
|= E1000_CTRL_VME
;
1307 E1000_WRITE_REG(&sc
->hw
, E1000_CTRL
, ctrl
);
1310 /* Configure for OS presence */
1314 #ifdef IFPOLL_ENABLE
1315 if (ifp
->if_flags
& IFF_NPOLLING
)
1318 sc
->tx_ring_inuse
= emx_get_txring_inuse(sc
, polling
);
1319 ifq_set_subq_mask(&ifp
->if_snd
, sc
->tx_ring_inuse
- 1);
1321 /* Prepare transmit descriptors and buffers */
1322 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
)
1323 emx_init_tx_ring(&sc
->tx_data
[i
]);
1324 emx_init_tx_unit(sc
);
1326 /* Setup Multicast table */
1329 /* Prepare receive descriptors and buffers */
1330 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
1331 if (emx_init_rx_ring(&sc
->rx_data
[i
])) {
1333 "Could not setup receive structures\n");
1338 emx_init_rx_unit(sc
);
1340 /* Don't lose promiscuous settings */
1341 emx_set_promisc(sc
);
1343 ifp
->if_flags
|= IFF_RUNNING
;
1344 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
1345 ifsq_clr_oactive(sc
->tx_data
[i
].ifsq
);
1346 ifsq_watchdog_start(&sc
->tx_data
[i
].tx_watchdog
);
1349 callout_reset(&sc
->timer
, hz
, emx_timer
, sc
);
1350 e1000_clear_hw_cntrs_base_generic(&sc
->hw
);
1352 /* MSI/X configuration for 82574 */
1353 if (sc
->hw
.mac
.type
== e1000_82574
) {
1356 tmp
= E1000_READ_REG(&sc
->hw
, E1000_CTRL_EXT
);
1357 tmp
|= E1000_CTRL_EXT_PBA_CLR
;
1358 E1000_WRITE_REG(&sc
->hw
, E1000_CTRL_EXT
, tmp
);
1361 * Set the IVAR - interrupt vector routing.
1362 * Each nibble represents a vector, high bit
1363 * is enable, other 3 bits are the MSIX table
1364 * entry, we map RXQ0 to 0, TXQ0 to 1, and
1365 * Link (other) to 2, hence the magic number.
1367 E1000_WRITE_REG(&sc
->hw
, E1000_IVAR
, 0x800A0908);
1371 * Only enable interrupts if we are not polling, make sure
1372 * they are off otherwise.
1375 emx_disable_intr(sc
);
1377 emx_enable_intr(sc
);
1379 /* AMT based hardware can now take control from firmware */
1380 if ((sc
->flags
& (EMX_FLAG_HAS_MGMT
| EMX_FLAG_HAS_AMT
)) ==
1381 (EMX_FLAG_HAS_MGMT
| EMX_FLAG_HAS_AMT
))
1382 emx_get_hw_control(sc
);
1388 emx_intr_body(xsc
, TRUE
);
1392 emx_intr_body(struct emx_softc
*sc
, boolean_t chk_asserted
)
1394 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1398 ASSERT_SERIALIZED(&sc
->main_serialize
);
1400 reg_icr
= E1000_READ_REG(&sc
->hw
, E1000_ICR
);
1402 if (chk_asserted
&& (reg_icr
& E1000_ICR_INT_ASSERTED
) == 0) {
1408 * XXX: some laptops trigger several spurious interrupts
1409 * on emx(4) when in the resume cycle. The ICR register
1410 * reports all-ones value in this case. Processing such
1411 * interrupts would lead to a freeze. I don't know why.
1413 if (reg_icr
== 0xffffffff) {
1418 if (ifp
->if_flags
& IFF_RUNNING
) {
1420 (E1000_ICR_RXT0
| E1000_ICR_RXDMT0
| E1000_ICR_RXO
)) {
1423 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
1424 lwkt_serialize_enter(
1425 &sc
->rx_data
[i
].rx_serialize
);
1426 emx_rxeof(&sc
->rx_data
[i
], -1);
1427 lwkt_serialize_exit(
1428 &sc
->rx_data
[i
].rx_serialize
);
1431 if (reg_icr
& E1000_ICR_TXDW
) {
1432 struct emx_txdata
*tdata
= &sc
->tx_data
[0];
1434 lwkt_serialize_enter(&tdata
->tx_serialize
);
1436 if (!ifsq_is_empty(tdata
->ifsq
))
1437 ifsq_devstart(tdata
->ifsq
);
1438 lwkt_serialize_exit(&tdata
->tx_serialize
);
1442 /* Link status change */
1443 if (reg_icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
1444 emx_serialize_skipmain(sc
);
1446 callout_stop(&sc
->timer
);
1447 sc
->hw
.mac
.get_link_status
= 1;
1448 emx_update_link_status(sc
);
1450 /* Deal with TX cruft when link lost */
1453 callout_reset(&sc
->timer
, hz
, emx_timer
, sc
);
1455 emx_deserialize_skipmain(sc
);
1458 if (reg_icr
& E1000_ICR_RXO
)
1465 emx_intr_mask(void *xsc
)
1467 struct emx_softc
*sc
= xsc
;
1469 E1000_WRITE_REG(&sc
->hw
, E1000_IMC
, 0xffffffff);
1472 * ICR.INT_ASSERTED bit will never be set if IMS is 0,
1473 * so don't check it.
1475 emx_intr_body(sc
, FALSE
);
1476 E1000_WRITE_REG(&sc
->hw
, E1000_IMS
, IMS_ENABLE_MASK
);
1480 emx_media_status(struct ifnet
*ifp
, struct ifmediareq
*ifmr
)
1482 struct emx_softc
*sc
= ifp
->if_softc
;
1484 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1486 emx_update_link_status(sc
);
1488 ifmr
->ifm_status
= IFM_AVALID
;
1489 ifmr
->ifm_active
= IFM_ETHER
;
1491 if (!sc
->link_active
) {
1492 if (sc
->hw
.mac
.autoneg
)
1493 ifmr
->ifm_active
|= IFM_NONE
;
1495 ifmr
->ifm_active
|= sc
->media
.ifm_media
;
1499 ifmr
->ifm_status
|= IFM_ACTIVE
;
1500 if (sc
->ifm_flowctrl
& IFM_ETH_FORCEPAUSE
)
1501 ifmr
->ifm_active
|= sc
->ifm_flowctrl
;
1503 if (sc
->hw
.phy
.media_type
== e1000_media_type_fiber
||
1504 sc
->hw
.phy
.media_type
== e1000_media_type_internal_serdes
) {
1505 ifmr
->ifm_active
|= IFM_1000_SX
| IFM_FDX
;
1507 switch (sc
->link_speed
) {
1509 ifmr
->ifm_active
|= IFM_10_T
;
1512 ifmr
->ifm_active
|= IFM_100_TX
;
1516 ifmr
->ifm_active
|= IFM_1000_T
;
1519 if (sc
->link_duplex
== FULL_DUPLEX
)
1520 ifmr
->ifm_active
|= IFM_FDX
;
1522 ifmr
->ifm_active
|= IFM_HDX
;
1524 if (ifmr
->ifm_active
& IFM_FDX
)
1525 ifmr
->ifm_active
|= e1000_fc2ifmedia(sc
->hw
.fc
.current_mode
);
1529 emx_media_change(struct ifnet
*ifp
)
1531 struct emx_softc
*sc
= ifp
->if_softc
;
1532 struct ifmedia
*ifm
= &sc
->media
;
1534 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1536 if (IFM_TYPE(ifm
->ifm_media
) != IFM_ETHER
)
1539 switch (IFM_SUBTYPE(ifm
->ifm_media
)) {
1541 sc
->hw
.mac
.autoneg
= EMX_DO_AUTO_NEG
;
1542 sc
->hw
.phy
.autoneg_advertised
= EMX_AUTONEG_ADV_DEFAULT
;
1547 sc
->hw
.mac
.autoneg
= EMX_DO_AUTO_NEG
;
1548 sc
->hw
.phy
.autoneg_advertised
= ADVERTISE_1000_FULL
;
1552 if (IFM_OPTIONS(ifm
->ifm_media
) & IFM_FDX
) {
1553 sc
->hw
.mac
.forced_speed_duplex
= ADVERTISE_100_FULL
;
1555 if (IFM_OPTIONS(ifm
->ifm_media
) &
1556 (IFM_ETH_RXPAUSE
| IFM_ETH_TXPAUSE
)) {
1558 if_printf(ifp
, "Flow control is not "
1559 "allowed for half-duplex\n");
1563 sc
->hw
.mac
.forced_speed_duplex
= ADVERTISE_100_HALF
;
1565 sc
->hw
.mac
.autoneg
= FALSE
;
1566 sc
->hw
.phy
.autoneg_advertised
= 0;
1570 if (IFM_OPTIONS(ifm
->ifm_media
) & IFM_FDX
) {
1571 sc
->hw
.mac
.forced_speed_duplex
= ADVERTISE_10_FULL
;
1573 if (IFM_OPTIONS(ifm
->ifm_media
) &
1574 (IFM_ETH_RXPAUSE
| IFM_ETH_TXPAUSE
)) {
1576 if_printf(ifp
, "Flow control is not "
1577 "allowed for half-duplex\n");
1581 sc
->hw
.mac
.forced_speed_duplex
= ADVERTISE_10_HALF
;
1583 sc
->hw
.mac
.autoneg
= FALSE
;
1584 sc
->hw
.phy
.autoneg_advertised
= 0;
1589 if_printf(ifp
, "Unsupported media type %d\n",
1590 IFM_SUBTYPE(ifm
->ifm_media
));
1594 sc
->ifm_flowctrl
= ifm
->ifm_media
& IFM_ETH_FCMASK
;
1596 if (ifp
->if_flags
& IFF_RUNNING
)
1603 emx_encap(struct emx_txdata
*tdata
, struct mbuf
**m_headp
,
1604 int *segs_used
, int *idx
)
1606 bus_dma_segment_t segs
[EMX_MAX_SCATTER
];
1608 struct emx_txbuf
*tx_buffer
, *tx_buffer_mapped
;
1609 struct e1000_tx_desc
*ctxd
= NULL
;
1610 struct mbuf
*m_head
= *m_headp
;
1611 uint32_t txd_upper
, txd_lower
, cmd
= 0;
1612 int maxsegs
, nsegs
, i
, j
, first
, last
= 0, error
;
1614 if (m_head
->m_pkthdr
.csum_flags
& CSUM_TSO
) {
1615 error
= emx_tso_pullup(tdata
, m_headp
);
1621 txd_upper
= txd_lower
= 0;
1624 * Capture the first descriptor index, this descriptor
1625 * will have the index of the EOP which is the only one
1626 * that now gets a DONE bit writeback.
1628 first
= tdata
->next_avail_tx_desc
;
1629 tx_buffer
= &tdata
->tx_buf
[first
];
1630 tx_buffer_mapped
= tx_buffer
;
1631 map
= tx_buffer
->map
;
1633 maxsegs
= tdata
->num_tx_desc_avail
- EMX_TX_RESERVED
;
1634 KASSERT(maxsegs
>= tdata
->spare_tx_desc
, ("not enough spare TX desc"));
1635 if (maxsegs
> EMX_MAX_SCATTER
)
1636 maxsegs
= EMX_MAX_SCATTER
;
1638 error
= bus_dmamap_load_mbuf_defrag(tdata
->txtag
, map
, m_headp
,
1639 segs
, maxsegs
, &nsegs
, BUS_DMA_NOWAIT
);
1645 bus_dmamap_sync(tdata
->txtag
, map
, BUS_DMASYNC_PREWRITE
);
1648 tdata
->tx_nsegs
+= nsegs
;
1649 *segs_used
+= nsegs
;
1651 if (m_head
->m_pkthdr
.csum_flags
& CSUM_TSO
) {
1652 /* TSO will consume one TX desc */
1653 i
= emx_tso_setup(tdata
, m_head
, &txd_upper
, &txd_lower
);
1654 tdata
->tx_nsegs
+= i
;
1656 } else if (m_head
->m_pkthdr
.csum_flags
& EMX_CSUM_FEATURES
) {
1657 /* TX csum offloading will consume one TX desc */
1658 i
= emx_txcsum(tdata
, m_head
, &txd_upper
, &txd_lower
);
1659 tdata
->tx_nsegs
+= i
;
1663 /* Handle VLAN tag */
1664 if (m_head
->m_flags
& M_VLANTAG
) {
1665 /* Set the vlan id. */
1666 txd_upper
|= (htole16(m_head
->m_pkthdr
.ether_vlantag
) << 16);
1667 /* Tell hardware to add tag */
1668 txd_lower
|= htole32(E1000_TXD_CMD_VLE
);
1671 i
= tdata
->next_avail_tx_desc
;
1673 /* Set up our transmit descriptors */
1674 for (j
= 0; j
< nsegs
; j
++) {
1675 tx_buffer
= &tdata
->tx_buf
[i
];
1676 ctxd
= &tdata
->tx_desc_base
[i
];
1678 ctxd
->buffer_addr
= htole64(segs
[j
].ds_addr
);
1679 ctxd
->lower
.data
= htole32(E1000_TXD_CMD_IFCS
|
1680 txd_lower
| segs
[j
].ds_len
);
1681 ctxd
->upper
.data
= htole32(txd_upper
);
1684 if (++i
== tdata
->num_tx_desc
)
1688 tdata
->next_avail_tx_desc
= i
;
1690 KKASSERT(tdata
->num_tx_desc_avail
> nsegs
);
1691 tdata
->num_tx_desc_avail
-= nsegs
;
1693 tx_buffer
->m_head
= m_head
;
1694 tx_buffer_mapped
->map
= tx_buffer
->map
;
1695 tx_buffer
->map
= map
;
1697 if (tdata
->tx_nsegs
>= tdata
->tx_intr_nsegs
) {
1698 tdata
->tx_nsegs
= 0;
1701 * Report Status (RS) is turned on
1702 * every tx_intr_nsegs descriptors.
1704 cmd
= E1000_TXD_CMD_RS
;
1707 * Keep track of the descriptor, which will
1708 * be written back by hardware.
1710 tdata
->tx_dd
[tdata
->tx_dd_tail
] = last
;
1711 EMX_INC_TXDD_IDX(tdata
->tx_dd_tail
);
1712 KKASSERT(tdata
->tx_dd_tail
!= tdata
->tx_dd_head
);
1716 * Last Descriptor of Packet needs End Of Packet (EOP)
1718 ctxd
->lower
.data
|= htole32(E1000_TXD_CMD_EOP
| cmd
);
1721 * Defer TDT updating, until enough descriptors are setup
1725 #ifdef EMX_TSS_DEBUG
1733 emx_set_promisc(struct emx_softc
*sc
)
1735 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1738 reg_rctl
= E1000_READ_REG(&sc
->hw
, E1000_RCTL
);
1740 if (ifp
->if_flags
& IFF_PROMISC
) {
1741 reg_rctl
|= (E1000_RCTL_UPE
| E1000_RCTL_MPE
);
1742 /* Turn this on if you want to see bad packets */
1744 reg_rctl
|= E1000_RCTL_SBP
;
1745 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, reg_rctl
);
1746 } else if (ifp
->if_flags
& IFF_ALLMULTI
) {
1747 reg_rctl
|= E1000_RCTL_MPE
;
1748 reg_rctl
&= ~E1000_RCTL_UPE
;
1749 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, reg_rctl
);
1754 emx_disable_promisc(struct emx_softc
*sc
)
1758 reg_rctl
= E1000_READ_REG(&sc
->hw
, E1000_RCTL
);
1760 reg_rctl
&= ~E1000_RCTL_UPE
;
1761 reg_rctl
&= ~E1000_RCTL_MPE
;
1762 reg_rctl
&= ~E1000_RCTL_SBP
;
1763 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, reg_rctl
);
1767 emx_set_multi(struct emx_softc
*sc
)
1769 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1770 struct ifmultiaddr
*ifma
;
1771 uint32_t reg_rctl
= 0;
1776 bzero(mta
, ETH_ADDR_LEN
* EMX_MCAST_ADDR_MAX
);
1778 TAILQ_FOREACH(ifma
, &ifp
->if_multiaddrs
, ifma_link
) {
1779 if (ifma
->ifma_addr
->sa_family
!= AF_LINK
)
1782 if (mcnt
== EMX_MCAST_ADDR_MAX
)
1785 bcopy(LLADDR((struct sockaddr_dl
*)ifma
->ifma_addr
),
1786 &mta
[mcnt
* ETHER_ADDR_LEN
], ETHER_ADDR_LEN
);
1790 if (mcnt
>= EMX_MCAST_ADDR_MAX
) {
1791 reg_rctl
= E1000_READ_REG(&sc
->hw
, E1000_RCTL
);
1792 reg_rctl
|= E1000_RCTL_MPE
;
1793 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, reg_rctl
);
1795 e1000_update_mc_addr_list(&sc
->hw
, mta
, mcnt
);
1800 * This routine checks for link status and updates statistics.
1803 emx_timer(void *xsc
)
1805 struct emx_softc
*sc
= xsc
;
1806 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1808 lwkt_serialize_enter(&sc
->main_serialize
);
1810 emx_update_link_status(sc
);
1811 emx_update_stats(sc
);
1813 /* Reset LAA into RAR[0] on 82571 */
1814 if (e1000_get_laa_state_82571(&sc
->hw
) == TRUE
)
1815 e1000_rar_set(&sc
->hw
, sc
->hw
.mac
.addr
, 0);
1817 if (emx_display_debug_stats
&& (ifp
->if_flags
& IFF_RUNNING
))
1818 emx_print_hw_stats(sc
);
1822 callout_reset(&sc
->timer
, hz
, emx_timer
, sc
);
1824 lwkt_serialize_exit(&sc
->main_serialize
);
1828 emx_update_link_status(struct emx_softc
*sc
)
1830 struct e1000_hw
*hw
= &sc
->hw
;
1831 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1832 device_t dev
= sc
->dev
;
1833 uint32_t link_check
= 0;
1835 /* Get the cached link value or read phy for real */
1836 switch (hw
->phy
.media_type
) {
1837 case e1000_media_type_copper
:
1838 if (hw
->mac
.get_link_status
) {
1839 /* Do the work to read phy */
1840 e1000_check_for_link(hw
);
1841 link_check
= !hw
->mac
.get_link_status
;
1842 if (link_check
) /* ESB2 fix */
1843 e1000_cfg_on_link_up(hw
);
1849 case e1000_media_type_fiber
:
1850 e1000_check_for_link(hw
);
1851 link_check
= E1000_READ_REG(hw
, E1000_STATUS
) & E1000_STATUS_LU
;
1854 case e1000_media_type_internal_serdes
:
1855 e1000_check_for_link(hw
);
1856 link_check
= sc
->hw
.mac
.serdes_has_link
;
1859 case e1000_media_type_unknown
:
1864 /* Now check for a transition */
1865 if (link_check
&& sc
->link_active
== 0) {
1866 e1000_get_speed_and_duplex(hw
, &sc
->link_speed
,
1870 * Check if we should enable/disable SPEED_MODE bit on
1873 if (sc
->link_speed
!= SPEED_1000
&&
1874 (hw
->mac
.type
== e1000_82571
||
1875 hw
->mac
.type
== e1000_82572
)) {
1878 tarc0
= E1000_READ_REG(hw
, E1000_TARC(0));
1879 tarc0
&= ~EMX_TARC_SPEED_MODE
;
1880 E1000_WRITE_REG(hw
, E1000_TARC(0), tarc0
);
1883 char flowctrl
[IFM_ETH_FC_STRLEN
];
1885 e1000_fc2str(hw
->fc
.current_mode
, flowctrl
,
1887 device_printf(dev
, "Link is up %d Mbps %s, "
1888 "Flow control: %s\n",
1890 (sc
->link_duplex
== FULL_DUPLEX
) ?
1891 "Full Duplex" : "Half Duplex",
1894 if (sc
->ifm_flowctrl
& IFM_ETH_FORCEPAUSE
)
1895 e1000_force_flowctrl(hw
, sc
->ifm_flowctrl
);
1896 sc
->link_active
= 1;
1898 ifp
->if_baudrate
= sc
->link_speed
* 1000000;
1899 ifp
->if_link_state
= LINK_STATE_UP
;
1900 if_link_state_change(ifp
);
1901 } else if (!link_check
&& sc
->link_active
== 1) {
1902 ifp
->if_baudrate
= sc
->link_speed
= 0;
1903 sc
->link_duplex
= 0;
1905 device_printf(dev
, "Link is Down\n");
1906 sc
->link_active
= 0;
1907 ifp
->if_link_state
= LINK_STATE_DOWN
;
1908 if_link_state_change(ifp
);
1913 emx_stop(struct emx_softc
*sc
)
1915 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
1918 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
1920 emx_disable_intr(sc
);
1922 callout_stop(&sc
->timer
);
1924 ifp
->if_flags
&= ~IFF_RUNNING
;
1925 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
1926 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
1928 ifsq_clr_oactive(tdata
->ifsq
);
1929 ifsq_watchdog_stop(&tdata
->tx_watchdog
);
1930 tdata
->tx_flags
&= ~EMX_TXFLAG_ENABLED
;
1934 * Disable multiple receive queues.
1937 * We should disable multiple receive queues before
1938 * resetting the hardware.
1940 E1000_WRITE_REG(&sc
->hw
, E1000_MRQC
, 0);
1942 e1000_reset_hw(&sc
->hw
);
1943 E1000_WRITE_REG(&sc
->hw
, E1000_WUC
, 0);
1945 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
1946 emx_free_tx_ring(&sc
->tx_data
[i
]);
1947 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
)
1948 emx_free_rx_ring(&sc
->rx_data
[i
]);
1952 emx_reset(struct emx_softc
*sc
)
1954 device_t dev
= sc
->dev
;
1955 uint16_t rx_buffer_size
;
1958 /* Set up smart power down as default off on newer adapters. */
1959 if (!emx_smart_pwr_down
&&
1960 (sc
->hw
.mac
.type
== e1000_82571
||
1961 sc
->hw
.mac
.type
== e1000_82572
)) {
1962 uint16_t phy_tmp
= 0;
1964 /* Speed up time to link by disabling smart power down. */
1965 e1000_read_phy_reg(&sc
->hw
,
1966 IGP02E1000_PHY_POWER_MGMT
, &phy_tmp
);
1967 phy_tmp
&= ~IGP02E1000_PM_SPD
;
1968 e1000_write_phy_reg(&sc
->hw
,
1969 IGP02E1000_PHY_POWER_MGMT
, phy_tmp
);
1973 * Packet Buffer Allocation (PBA)
1974 * Writing PBA sets the receive portion of the buffer
1975 * the remainder is used for the transmit buffer.
1977 switch (sc
->hw
.mac
.type
) {
1978 /* Total Packet Buffer on these is 48K */
1981 case e1000_80003es2lan
:
1982 pba
= E1000_PBA_32K
; /* 32K for Rx, 16K for Tx */
1985 case e1000_82573
: /* 82573: Total Packet Buffer is 32K */
1986 pba
= E1000_PBA_12K
; /* 12K for Rx, 20K for Tx */
1990 pba
= E1000_PBA_20K
; /* 20K for Rx, 20K for Tx */
1995 pba
= E1000_PBA_26K
;
1999 /* Devices before 82547 had a Packet Buffer of 64K. */
2000 if (sc
->hw
.mac
.max_frame_size
> 8192)
2001 pba
= E1000_PBA_40K
; /* 40K for Rx, 24K for Tx */
2003 pba
= E1000_PBA_48K
; /* 48K for Rx, 16K for Tx */
2005 E1000_WRITE_REG(&sc
->hw
, E1000_PBA
, pba
);
2008 * These parameters control the automatic generation (Tx) and
2009 * response (Rx) to Ethernet PAUSE frames.
2010 * - High water mark should allow for at least two frames to be
2011 * received after sending an XOFF.
2012 * - Low water mark works best when it is very near the high water mark.
2013 * This allows the receiver to restart by sending XON when it has
2014 * drained a bit. Here we use an arbitary value of 1500 which will
2015 * restart after one full frame is pulled from the buffer. There
2016 * could be several smaller frames in the buffer and if so they will
2017 * not trigger the XON until their total number reduces the buffer
2019 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2021 rx_buffer_size
= (E1000_READ_REG(&sc
->hw
, E1000_PBA
) & 0xffff) << 10;
2023 sc
->hw
.fc
.high_water
= rx_buffer_size
-
2024 roundup2(sc
->hw
.mac
.max_frame_size
, 1024);
2025 sc
->hw
.fc
.low_water
= sc
->hw
.fc
.high_water
- 1500;
2027 sc
->hw
.fc
.pause_time
= EMX_FC_PAUSE_TIME
;
2028 sc
->hw
.fc
.send_xon
= TRUE
;
2029 sc
->hw
.fc
.requested_mode
= e1000_ifmedia2fc(sc
->ifm_flowctrl
);
2032 * Device specific overrides/settings
2034 if (sc
->hw
.mac
.type
== e1000_pch_lpt
||
2035 sc
->hw
.mac
.type
== e1000_pch_spt
) {
2036 sc
->hw
.fc
.high_water
= 0x5C20;
2037 sc
->hw
.fc
.low_water
= 0x5048;
2038 sc
->hw
.fc
.pause_time
= 0x0650;
2039 sc
->hw
.fc
.refresh_time
= 0x0400;
2040 /* Jumbos need adjusted PBA */
2041 if (sc
->arpcom
.ac_if
.if_mtu
> ETHERMTU
)
2042 E1000_WRITE_REG(&sc
->hw
, E1000_PBA
, 12);
2044 E1000_WRITE_REG(&sc
->hw
, E1000_PBA
, 26);
2045 } else if (sc
->hw
.mac
.type
== e1000_80003es2lan
) {
2046 sc
->hw
.fc
.pause_time
= 0xFFFF;
2049 /* Issue a global reset */
2050 e1000_reset_hw(&sc
->hw
);
2051 E1000_WRITE_REG(&sc
->hw
, E1000_WUC
, 0);
2052 emx_disable_aspm(sc
);
2054 if (e1000_init_hw(&sc
->hw
) < 0) {
2055 device_printf(dev
, "Hardware Initialization Failed\n");
2059 E1000_WRITE_REG(&sc
->hw
, E1000_VET
, ETHERTYPE_VLAN
);
2060 e1000_get_phy_info(&sc
->hw
);
2061 e1000_check_for_link(&sc
->hw
);
2067 emx_setup_ifp(struct emx_softc
*sc
)
2069 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
2072 if_initname(ifp
, device_get_name(sc
->dev
),
2073 device_get_unit(sc
->dev
));
2075 ifp
->if_flags
= IFF_BROADCAST
| IFF_SIMPLEX
| IFF_MULTICAST
;
2076 ifp
->if_init
= emx_init
;
2077 ifp
->if_ioctl
= emx_ioctl
;
2078 ifp
->if_start
= emx_start
;
2079 #ifdef IFPOLL_ENABLE
2080 ifp
->if_npoll
= emx_npoll
;
2082 ifp
->if_serialize
= emx_serialize
;
2083 ifp
->if_deserialize
= emx_deserialize
;
2084 ifp
->if_tryserialize
= emx_tryserialize
;
2086 ifp
->if_serialize_assert
= emx_serialize_assert
;
2089 ifp
->if_nmbclusters
= sc
->rx_ring_cnt
* sc
->rx_data
[0].num_rx_desc
;
2091 ifq_set_maxlen(&ifp
->if_snd
, sc
->tx_data
[0].num_tx_desc
- 1);
2092 ifq_set_ready(&ifp
->if_snd
);
2093 ifq_set_subq_cnt(&ifp
->if_snd
, sc
->tx_ring_cnt
);
2095 ifp
->if_mapsubq
= ifq_mapsubq_mask
;
2096 ifq_set_subq_mask(&ifp
->if_snd
, 0);
2098 ether_ifattach(ifp
, sc
->hw
.mac
.addr
, NULL
);
2100 ifp
->if_capabilities
= IFCAP_HWCSUM
|
2101 IFCAP_VLAN_HWTAGGING
|
2104 if (sc
->rx_ring_cnt
> 1)
2105 ifp
->if_capabilities
|= IFCAP_RSS
;
2106 ifp
->if_capenable
= ifp
->if_capabilities
;
2107 ifp
->if_hwassist
= EMX_CSUM_FEATURES
| CSUM_TSO
;
2110 * Tell the upper layer(s) we support long frames.
2112 ifp
->if_data
.ifi_hdrlen
= sizeof(struct ether_vlan_header
);
2114 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
2115 struct ifaltq_subque
*ifsq
= ifq_get_subq(&ifp
->if_snd
, i
);
2116 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
2118 ifsq_set_cpuid(ifsq
, rman_get_cpuid(sc
->intr_res
));
2119 ifsq_set_priv(ifsq
, tdata
);
2120 ifsq_set_hw_serialize(ifsq
, &tdata
->tx_serialize
);
2123 ifsq_watchdog_init(&tdata
->tx_watchdog
, ifsq
, emx_watchdog
);
2127 * Specify the media types supported by this sc and register
2128 * callbacks to update media and link information
2130 if (sc
->hw
.phy
.media_type
== e1000_media_type_fiber
||
2131 sc
->hw
.phy
.media_type
== e1000_media_type_internal_serdes
) {
2132 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_1000_SX
| IFM_FDX
,
2135 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_10_T
, 0, NULL
);
2136 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_10_T
| IFM_FDX
,
2138 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_100_TX
, 0, NULL
);
2139 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_100_TX
| IFM_FDX
,
2141 if (sc
->hw
.phy
.type
!= e1000_phy_ife
) {
2142 ifmedia_add(&sc
->media
,
2143 IFM_ETHER
| IFM_1000_T
| IFM_FDX
, 0, NULL
);
2146 ifmedia_add(&sc
->media
, IFM_ETHER
| IFM_AUTO
, 0, NULL
);
2147 ifmedia_set(&sc
->media
, IFM_ETHER
| IFM_AUTO
| sc
->ifm_flowctrl
);
2151 * Workaround for SmartSpeed on 82541 and 82547 controllers
2154 emx_smartspeed(struct emx_softc
*sc
)
2158 if (sc
->link_active
|| sc
->hw
.phy
.type
!= e1000_phy_igp
||
2159 sc
->hw
.mac
.autoneg
== 0 ||
2160 (sc
->hw
.phy
.autoneg_advertised
& ADVERTISE_1000_FULL
) == 0)
2163 if (sc
->smartspeed
== 0) {
2165 * If Master/Slave config fault is asserted twice,
2166 * we assume back-to-back
2168 e1000_read_phy_reg(&sc
->hw
, PHY_1000T_STATUS
, &phy_tmp
);
2169 if (!(phy_tmp
& SR_1000T_MS_CONFIG_FAULT
))
2171 e1000_read_phy_reg(&sc
->hw
, PHY_1000T_STATUS
, &phy_tmp
);
2172 if (phy_tmp
& SR_1000T_MS_CONFIG_FAULT
) {
2173 e1000_read_phy_reg(&sc
->hw
,
2174 PHY_1000T_CTRL
, &phy_tmp
);
2175 if (phy_tmp
& CR_1000T_MS_ENABLE
) {
2176 phy_tmp
&= ~CR_1000T_MS_ENABLE
;
2177 e1000_write_phy_reg(&sc
->hw
,
2178 PHY_1000T_CTRL
, phy_tmp
);
2180 if (sc
->hw
.mac
.autoneg
&&
2181 !e1000_phy_setup_autoneg(&sc
->hw
) &&
2182 !e1000_read_phy_reg(&sc
->hw
,
2183 PHY_CONTROL
, &phy_tmp
)) {
2184 phy_tmp
|= MII_CR_AUTO_NEG_EN
|
2185 MII_CR_RESTART_AUTO_NEG
;
2186 e1000_write_phy_reg(&sc
->hw
,
2187 PHY_CONTROL
, phy_tmp
);
2192 } else if (sc
->smartspeed
== EMX_SMARTSPEED_DOWNSHIFT
) {
2193 /* If still no link, perhaps using 2/3 pair cable */
2194 e1000_read_phy_reg(&sc
->hw
, PHY_1000T_CTRL
, &phy_tmp
);
2195 phy_tmp
|= CR_1000T_MS_ENABLE
;
2196 e1000_write_phy_reg(&sc
->hw
, PHY_1000T_CTRL
, phy_tmp
);
2197 if (sc
->hw
.mac
.autoneg
&&
2198 !e1000_phy_setup_autoneg(&sc
->hw
) &&
2199 !e1000_read_phy_reg(&sc
->hw
, PHY_CONTROL
, &phy_tmp
)) {
2200 phy_tmp
|= MII_CR_AUTO_NEG_EN
| MII_CR_RESTART_AUTO_NEG
;
2201 e1000_write_phy_reg(&sc
->hw
, PHY_CONTROL
, phy_tmp
);
2205 /* Restart process after EMX_SMARTSPEED_MAX iterations */
2206 if (sc
->smartspeed
++ == EMX_SMARTSPEED_MAX
)
2211 emx_create_tx_ring(struct emx_txdata
*tdata
)
2213 device_t dev
= tdata
->sc
->dev
;
2214 struct emx_txbuf
*tx_buffer
;
2215 int error
, i
, tsize
, ntxd
;
2218 * Validate number of transmit descriptors. It must not exceed
2219 * hardware maximum, and must be multiple of E1000_DBA_ALIGN.
2221 ntxd
= device_getenv_int(dev
, "txd", emx_txd
);
2222 if ((ntxd
* sizeof(struct e1000_tx_desc
)) % EMX_DBA_ALIGN
!= 0 ||
2223 ntxd
> EMX_MAX_TXD
|| ntxd
< EMX_MIN_TXD
) {
2224 device_printf(dev
, "Using %d TX descriptors instead of %d!\n",
2225 EMX_DEFAULT_TXD
, ntxd
);
2226 tdata
->num_tx_desc
= EMX_DEFAULT_TXD
;
2228 tdata
->num_tx_desc
= ntxd
;
2232 * Allocate Transmit Descriptor ring
2234 tsize
= roundup2(tdata
->num_tx_desc
* sizeof(struct e1000_tx_desc
),
2236 tdata
->tx_desc_base
= bus_dmamem_coherent_any(tdata
->sc
->parent_dtag
,
2237 EMX_DBA_ALIGN
, tsize
, BUS_DMA_WAITOK
,
2238 &tdata
->tx_desc_dtag
, &tdata
->tx_desc_dmap
,
2239 &tdata
->tx_desc_paddr
);
2240 if (tdata
->tx_desc_base
== NULL
) {
2241 device_printf(dev
, "Unable to allocate tx_desc memory\n");
2245 tsize
= __VM_CACHELINE_ALIGN(
2246 sizeof(struct emx_txbuf
) * tdata
->num_tx_desc
);
2247 tdata
->tx_buf
= kmalloc_cachealign(tsize
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
2250 * Create DMA tags for tx buffers
2252 error
= bus_dma_tag_create(tdata
->sc
->parent_dtag
, /* parent */
2253 1, 0, /* alignment, bounds */
2254 BUS_SPACE_MAXADDR
, /* lowaddr */
2255 BUS_SPACE_MAXADDR
, /* highaddr */
2256 NULL
, NULL
, /* filter, filterarg */
2257 EMX_TSO_SIZE
, /* maxsize */
2258 EMX_MAX_SCATTER
, /* nsegments */
2259 EMX_MAX_SEGSIZE
, /* maxsegsize */
2260 BUS_DMA_WAITOK
| BUS_DMA_ALLOCNOW
|
2261 BUS_DMA_ONEBPAGE
, /* flags */
2264 device_printf(dev
, "Unable to allocate TX DMA tag\n");
2265 kfree(tdata
->tx_buf
, M_DEVBUF
);
2266 tdata
->tx_buf
= NULL
;
2271 * Create DMA maps for tx buffers
2273 for (i
= 0; i
< tdata
->num_tx_desc
; i
++) {
2274 tx_buffer
= &tdata
->tx_buf
[i
];
2276 error
= bus_dmamap_create(tdata
->txtag
,
2277 BUS_DMA_WAITOK
| BUS_DMA_ONEBPAGE
,
2280 device_printf(dev
, "Unable to create TX DMA map\n");
2281 emx_destroy_tx_ring(tdata
, i
);
2287 * Setup TX parameters
2289 tdata
->spare_tx_desc
= EMX_TX_SPARE
;
2290 tdata
->tx_wreg_nsegs
= EMX_DEFAULT_TXWREG
;
2293 * Keep following relationship between spare_tx_desc, oact_tx_desc
2294 * and tx_intr_nsegs:
2295 * (spare_tx_desc + EMX_TX_RESERVED) <=
2296 * oact_tx_desc <= EMX_TX_OACTIVE_MAX <= tx_intr_nsegs
2298 tdata
->oact_tx_desc
= tdata
->num_tx_desc
/ 8;
2299 if (tdata
->oact_tx_desc
> EMX_TX_OACTIVE_MAX
)
2300 tdata
->oact_tx_desc
= EMX_TX_OACTIVE_MAX
;
2301 if (tdata
->oact_tx_desc
< tdata
->spare_tx_desc
+ EMX_TX_RESERVED
)
2302 tdata
->oact_tx_desc
= tdata
->spare_tx_desc
+ EMX_TX_RESERVED
;
2304 tdata
->tx_intr_nsegs
= tdata
->num_tx_desc
/ 16;
2305 if (tdata
->tx_intr_nsegs
< tdata
->oact_tx_desc
)
2306 tdata
->tx_intr_nsegs
= tdata
->oact_tx_desc
;
2309 * Pullup extra 4bytes into the first data segment for TSO, see:
2310 * 82571/82572 specification update errata #7
2312 * Same applies to I217 (and maybe I218 and I219).
2315 * 4bytes instead of 2bytes, which are mentioned in the errata,
2316 * are pulled; mainly to keep rest of the data properly aligned.
2318 if (tdata
->sc
->hw
.mac
.type
== e1000_82571
||
2319 tdata
->sc
->hw
.mac
.type
== e1000_82572
||
2320 tdata
->sc
->hw
.mac
.type
== e1000_pch_lpt
||
2321 tdata
->sc
->hw
.mac
.type
== e1000_pch_spt
)
2322 tdata
->tx_flags
|= EMX_TXFLAG_TSO_PULLEX
;
2328 emx_init_tx_ring(struct emx_txdata
*tdata
)
2330 /* Clear the old ring contents */
2331 bzero(tdata
->tx_desc_base
,
2332 sizeof(struct e1000_tx_desc
) * tdata
->num_tx_desc
);
2335 tdata
->next_avail_tx_desc
= 0;
2336 tdata
->next_tx_to_clean
= 0;
2337 tdata
->num_tx_desc_avail
= tdata
->num_tx_desc
;
2339 tdata
->tx_flags
|= EMX_TXFLAG_ENABLED
;
2340 if (tdata
->sc
->tx_ring_inuse
> 1) {
2341 tdata
->tx_flags
|= EMX_TXFLAG_FORCECTX
;
2343 if_printf(&tdata
->sc
->arpcom
.ac_if
,
2344 "TX %d force ctx setup\n", tdata
->idx
);
2350 emx_init_tx_unit(struct emx_softc
*sc
)
2352 uint32_t tctl
, tarc
, tipg
= 0, txdctl
;
2355 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
2356 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
2359 /* Setup the Base and Length of the Tx Descriptor Ring */
2360 bus_addr
= tdata
->tx_desc_paddr
;
2361 E1000_WRITE_REG(&sc
->hw
, E1000_TDLEN(i
),
2362 tdata
->num_tx_desc
* sizeof(struct e1000_tx_desc
));
2363 E1000_WRITE_REG(&sc
->hw
, E1000_TDBAH(i
),
2364 (uint32_t)(bus_addr
>> 32));
2365 E1000_WRITE_REG(&sc
->hw
, E1000_TDBAL(i
),
2366 (uint32_t)bus_addr
);
2367 /* Setup the HW Tx Head and Tail descriptor pointers */
2368 E1000_WRITE_REG(&sc
->hw
, E1000_TDT(i
), 0);
2369 E1000_WRITE_REG(&sc
->hw
, E1000_TDH(i
), 0);
2372 /* Set the default values for the Tx Inter Packet Gap timer */
2373 switch (sc
->hw
.mac
.type
) {
2374 case e1000_80003es2lan
:
2375 tipg
= DEFAULT_82543_TIPG_IPGR1
;
2376 tipg
|= DEFAULT_80003ES2LAN_TIPG_IPGR2
<<
2377 E1000_TIPG_IPGR2_SHIFT
;
2381 if (sc
->hw
.phy
.media_type
== e1000_media_type_fiber
||
2382 sc
->hw
.phy
.media_type
== e1000_media_type_internal_serdes
)
2383 tipg
= DEFAULT_82543_TIPG_IPGT_FIBER
;
2385 tipg
= DEFAULT_82543_TIPG_IPGT_COPPER
;
2386 tipg
|= DEFAULT_82543_TIPG_IPGR1
<< E1000_TIPG_IPGR1_SHIFT
;
2387 tipg
|= DEFAULT_82543_TIPG_IPGR2
<< E1000_TIPG_IPGR2_SHIFT
;
2391 E1000_WRITE_REG(&sc
->hw
, E1000_TIPG
, tipg
);
2393 /* NOTE: 0 is not allowed for TIDV */
2394 E1000_WRITE_REG(&sc
->hw
, E1000_TIDV
, 1);
2395 E1000_WRITE_REG(&sc
->hw
, E1000_TADV
, 0);
2398 * Errata workaround (obtained from Linux). This is necessary
2399 * to make multiple TX queues work on 82574.
2400 * XXX can't find it in any published errata though.
2402 txdctl
= E1000_READ_REG(&sc
->hw
, E1000_TXDCTL(0));
2403 E1000_WRITE_REG(&sc
->hw
, E1000_TXDCTL(1), txdctl
);
2405 if (sc
->hw
.mac
.type
== e1000_82571
||
2406 sc
->hw
.mac
.type
== e1000_82572
) {
2407 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(0));
2408 tarc
|= EMX_TARC_SPEED_MODE
;
2409 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(0), tarc
);
2410 } else if (sc
->hw
.mac
.type
== e1000_80003es2lan
) {
2411 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(0));
2413 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(0), tarc
);
2414 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(1));
2416 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(1), tarc
);
2419 /* Program the Transmit Control Register */
2420 tctl
= E1000_READ_REG(&sc
->hw
, E1000_TCTL
);
2421 tctl
&= ~E1000_TCTL_CT
;
2422 tctl
|= E1000_TCTL_PSP
| E1000_TCTL_RTLC
| E1000_TCTL_EN
|
2423 (E1000_COLLISION_THRESHOLD
<< E1000_CT_SHIFT
);
2424 tctl
|= E1000_TCTL_MULR
;
2426 /* This write will effectively turn on the transmit unit. */
2427 E1000_WRITE_REG(&sc
->hw
, E1000_TCTL
, tctl
);
2429 if (sc
->hw
.mac
.type
== e1000_82571
||
2430 sc
->hw
.mac
.type
== e1000_82572
||
2431 sc
->hw
.mac
.type
== e1000_80003es2lan
) {
2432 /* Bit 28 of TARC1 must be cleared when MULR is enabled */
2433 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(1));
2435 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(1), tarc
);
2438 if (sc
->tx_ring_inuse
> 1) {
2439 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(0));
2440 tarc
&= ~EMX_TARC_COUNT_MASK
;
2442 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(0), tarc
);
2444 tarc
= E1000_READ_REG(&sc
->hw
, E1000_TARC(1));
2445 tarc
&= ~EMX_TARC_COUNT_MASK
;
2447 E1000_WRITE_REG(&sc
->hw
, E1000_TARC(1), tarc
);
2452 emx_destroy_tx_ring(struct emx_txdata
*tdata
, int ndesc
)
2454 struct emx_txbuf
*tx_buffer
;
2457 /* Free Transmit Descriptor ring */
2458 if (tdata
->tx_desc_base
) {
2459 bus_dmamap_unload(tdata
->tx_desc_dtag
, tdata
->tx_desc_dmap
);
2460 bus_dmamem_free(tdata
->tx_desc_dtag
, tdata
->tx_desc_base
,
2461 tdata
->tx_desc_dmap
);
2462 bus_dma_tag_destroy(tdata
->tx_desc_dtag
);
2464 tdata
->tx_desc_base
= NULL
;
2467 if (tdata
->tx_buf
== NULL
)
2470 for (i
= 0; i
< ndesc
; i
++) {
2471 tx_buffer
= &tdata
->tx_buf
[i
];
2473 KKASSERT(tx_buffer
->m_head
== NULL
);
2474 bus_dmamap_destroy(tdata
->txtag
, tx_buffer
->map
);
2476 bus_dma_tag_destroy(tdata
->txtag
);
2478 kfree(tdata
->tx_buf
, M_DEVBUF
);
2479 tdata
->tx_buf
= NULL
;
2483 * The offload context needs to be set when we transfer the first
2484 * packet of a particular protocol (TCP/UDP). This routine has been
2485 * enhanced to deal with inserted VLAN headers.
2487 * If the new packet's ether header length, ip header length and
2488 * csum offloading type are same as the previous packet, we should
2489 * avoid allocating a new csum context descriptor; mainly to take
2490 * advantage of the pipeline effect of the TX data read request.
2492 * This function returns number of TX descrptors allocated for
2496 emx_txcsum(struct emx_txdata
*tdata
, struct mbuf
*mp
,
2497 uint32_t *txd_upper
, uint32_t *txd_lower
)
2499 struct e1000_context_desc
*TXD
;
2500 int curr_txd
, ehdrlen
, csum_flags
;
2501 uint32_t cmd
, hdr_len
, ip_hlen
;
2503 csum_flags
= mp
->m_pkthdr
.csum_flags
& EMX_CSUM_FEATURES
;
2504 ip_hlen
= mp
->m_pkthdr
.csum_iphlen
;
2505 ehdrlen
= mp
->m_pkthdr
.csum_lhlen
;
2507 if ((tdata
->tx_flags
& EMX_TXFLAG_FORCECTX
) == 0 &&
2508 tdata
->csum_lhlen
== ehdrlen
&& tdata
->csum_iphlen
== ip_hlen
&&
2509 tdata
->csum_flags
== csum_flags
) {
2511 * Same csum offload context as the previous packets;
2514 *txd_upper
= tdata
->csum_txd_upper
;
2515 *txd_lower
= tdata
->csum_txd_lower
;
2520 * Setup a new csum offload context.
2523 curr_txd
= tdata
->next_avail_tx_desc
;
2524 TXD
= (struct e1000_context_desc
*)&tdata
->tx_desc_base
[curr_txd
];
2528 /* Setup of IP header checksum. */
2529 if (csum_flags
& CSUM_IP
) {
2531 * Start offset for header checksum calculation.
2532 * End offset for header checksum calculation.
2533 * Offset of place to put the checksum.
2535 TXD
->lower_setup
.ip_fields
.ipcss
= ehdrlen
;
2536 TXD
->lower_setup
.ip_fields
.ipcse
=
2537 htole16(ehdrlen
+ ip_hlen
- 1);
2538 TXD
->lower_setup
.ip_fields
.ipcso
=
2539 ehdrlen
+ offsetof(struct ip
, ip_sum
);
2540 cmd
|= E1000_TXD_CMD_IP
;
2541 *txd_upper
|= E1000_TXD_POPTS_IXSM
<< 8;
2543 hdr_len
= ehdrlen
+ ip_hlen
;
2545 if (csum_flags
& CSUM_TCP
) {
2547 * Start offset for payload checksum calculation.
2548 * End offset for payload checksum calculation.
2549 * Offset of place to put the checksum.
2551 TXD
->upper_setup
.tcp_fields
.tucss
= hdr_len
;
2552 TXD
->upper_setup
.tcp_fields
.tucse
= htole16(0);
2553 TXD
->upper_setup
.tcp_fields
.tucso
=
2554 hdr_len
+ offsetof(struct tcphdr
, th_sum
);
2555 cmd
|= E1000_TXD_CMD_TCP
;
2556 *txd_upper
|= E1000_TXD_POPTS_TXSM
<< 8;
2557 } else if (csum_flags
& CSUM_UDP
) {
2559 * Start offset for header checksum calculation.
2560 * End offset for header checksum calculation.
2561 * Offset of place to put the checksum.
2563 TXD
->upper_setup
.tcp_fields
.tucss
= hdr_len
;
2564 TXD
->upper_setup
.tcp_fields
.tucse
= htole16(0);
2565 TXD
->upper_setup
.tcp_fields
.tucso
=
2566 hdr_len
+ offsetof(struct udphdr
, uh_sum
);
2567 *txd_upper
|= E1000_TXD_POPTS_TXSM
<< 8;
2570 *txd_lower
= E1000_TXD_CMD_DEXT
| /* Extended descr type */
2571 E1000_TXD_DTYP_D
; /* Data descr */
2573 /* Save the information for this csum offloading context */
2574 tdata
->csum_lhlen
= ehdrlen
;
2575 tdata
->csum_iphlen
= ip_hlen
;
2576 tdata
->csum_flags
= csum_flags
;
2577 tdata
->csum_txd_upper
= *txd_upper
;
2578 tdata
->csum_txd_lower
= *txd_lower
;
2580 TXD
->tcp_seg_setup
.data
= htole32(0);
2581 TXD
->cmd_and_length
=
2582 htole32(E1000_TXD_CMD_IFCS
| E1000_TXD_CMD_DEXT
| cmd
);
2584 if (++curr_txd
== tdata
->num_tx_desc
)
2587 KKASSERT(tdata
->num_tx_desc_avail
> 0);
2588 tdata
->num_tx_desc_avail
--;
2590 tdata
->next_avail_tx_desc
= curr_txd
;
2595 emx_txeof(struct emx_txdata
*tdata
)
2597 struct emx_txbuf
*tx_buffer
;
2598 int first
, num_avail
;
2600 if (tdata
->tx_dd_head
== tdata
->tx_dd_tail
)
2603 if (tdata
->num_tx_desc_avail
== tdata
->num_tx_desc
)
2606 num_avail
= tdata
->num_tx_desc_avail
;
2607 first
= tdata
->next_tx_to_clean
;
2609 while (tdata
->tx_dd_head
!= tdata
->tx_dd_tail
) {
2610 int dd_idx
= tdata
->tx_dd
[tdata
->tx_dd_head
];
2611 struct e1000_tx_desc
*tx_desc
;
2613 tx_desc
= &tdata
->tx_desc_base
[dd_idx
];
2614 if (tx_desc
->upper
.fields
.status
& E1000_TXD_STAT_DD
) {
2615 EMX_INC_TXDD_IDX(tdata
->tx_dd_head
);
2617 if (++dd_idx
== tdata
->num_tx_desc
)
2620 while (first
!= dd_idx
) {
2625 tx_buffer
= &tdata
->tx_buf
[first
];
2626 if (tx_buffer
->m_head
) {
2627 bus_dmamap_unload(tdata
->txtag
,
2629 m_freem(tx_buffer
->m_head
);
2630 tx_buffer
->m_head
= NULL
;
2633 if (++first
== tdata
->num_tx_desc
)
2640 tdata
->next_tx_to_clean
= first
;
2641 tdata
->num_tx_desc_avail
= num_avail
;
2643 if (tdata
->tx_dd_head
== tdata
->tx_dd_tail
) {
2644 tdata
->tx_dd_head
= 0;
2645 tdata
->tx_dd_tail
= 0;
2648 if (!EMX_IS_OACTIVE(tdata
)) {
2649 ifsq_clr_oactive(tdata
->ifsq
);
2651 /* All clean, turn off the timer */
2652 if (tdata
->num_tx_desc_avail
== tdata
->num_tx_desc
)
2653 tdata
->tx_watchdog
.wd_timer
= 0;
2658 emx_tx_collect(struct emx_txdata
*tdata
)
2660 struct emx_txbuf
*tx_buffer
;
2661 int tdh
, first
, num_avail
, dd_idx
= -1;
2663 if (tdata
->num_tx_desc_avail
== tdata
->num_tx_desc
)
2666 tdh
= E1000_READ_REG(&tdata
->sc
->hw
, E1000_TDH(tdata
->idx
));
2667 if (tdh
== tdata
->next_tx_to_clean
)
2670 if (tdata
->tx_dd_head
!= tdata
->tx_dd_tail
)
2671 dd_idx
= tdata
->tx_dd
[tdata
->tx_dd_head
];
2673 num_avail
= tdata
->num_tx_desc_avail
;
2674 first
= tdata
->next_tx_to_clean
;
2676 while (first
!= tdh
) {
2681 tx_buffer
= &tdata
->tx_buf
[first
];
2682 if (tx_buffer
->m_head
) {
2683 bus_dmamap_unload(tdata
->txtag
,
2685 m_freem(tx_buffer
->m_head
);
2686 tx_buffer
->m_head
= NULL
;
2689 if (first
== dd_idx
) {
2690 EMX_INC_TXDD_IDX(tdata
->tx_dd_head
);
2691 if (tdata
->tx_dd_head
== tdata
->tx_dd_tail
) {
2692 tdata
->tx_dd_head
= 0;
2693 tdata
->tx_dd_tail
= 0;
2696 dd_idx
= tdata
->tx_dd
[tdata
->tx_dd_head
];
2700 if (++first
== tdata
->num_tx_desc
)
2703 tdata
->next_tx_to_clean
= first
;
2704 tdata
->num_tx_desc_avail
= num_avail
;
2706 if (!EMX_IS_OACTIVE(tdata
)) {
2707 ifsq_clr_oactive(tdata
->ifsq
);
2709 /* All clean, turn off the timer */
2710 if (tdata
->num_tx_desc_avail
== tdata
->num_tx_desc
)
2711 tdata
->tx_watchdog
.wd_timer
= 0;
2716 * When Link is lost sometimes there is work still in the TX ring
2717 * which will result in a watchdog, rather than allow that do an
2718 * attempted cleanup and then reinit here. Note that this has been
2719 * seens mostly with fiber adapters.
2722 emx_tx_purge(struct emx_softc
*sc
)
2726 if (sc
->link_active
)
2729 for (i
= 0; i
< sc
->tx_ring_inuse
; ++i
) {
2730 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
2732 if (tdata
->tx_watchdog
.wd_timer
) {
2733 emx_tx_collect(tdata
);
2734 if (tdata
->tx_watchdog
.wd_timer
) {
2735 if_printf(&sc
->arpcom
.ac_if
,
2736 "Link lost, TX pending, reinit\n");
2745 emx_newbuf(struct emx_rxdata
*rdata
, int i
, int init
)
2748 bus_dma_segment_t seg
;
2750 struct emx_rxbuf
*rx_buffer
;
2753 m
= m_getcl(init
? M_WAITOK
: M_NOWAIT
, MT_DATA
, M_PKTHDR
);
2756 if_printf(&rdata
->sc
->arpcom
.ac_if
,
2757 "Unable to allocate RX mbuf\n");
2761 m
->m_len
= m
->m_pkthdr
.len
= MCLBYTES
;
2763 if (rdata
->sc
->hw
.mac
.max_frame_size
<= MCLBYTES
- ETHER_ALIGN
)
2764 m_adj(m
, ETHER_ALIGN
);
2766 error
= bus_dmamap_load_mbuf_segment(rdata
->rxtag
,
2767 rdata
->rx_sparemap
, m
,
2768 &seg
, 1, &nseg
, BUS_DMA_NOWAIT
);
2772 if_printf(&rdata
->sc
->arpcom
.ac_if
,
2773 "Unable to load RX mbuf\n");
2778 rx_buffer
= &rdata
->rx_buf
[i
];
2779 if (rx_buffer
->m_head
!= NULL
)
2780 bus_dmamap_unload(rdata
->rxtag
, rx_buffer
->map
);
2782 map
= rx_buffer
->map
;
2783 rx_buffer
->map
= rdata
->rx_sparemap
;
2784 rdata
->rx_sparemap
= map
;
2786 rx_buffer
->m_head
= m
;
2787 rx_buffer
->paddr
= seg
.ds_addr
;
2789 emx_setup_rxdesc(&rdata
->rx_desc
[i
], rx_buffer
);
2794 emx_create_rx_ring(struct emx_rxdata
*rdata
)
2796 device_t dev
= rdata
->sc
->dev
;
2797 struct emx_rxbuf
*rx_buffer
;
2798 int i
, error
, rsize
, nrxd
;
2801 * Validate number of receive descriptors. It must not exceed
2802 * hardware maximum, and must be multiple of E1000_DBA_ALIGN.
2804 nrxd
= device_getenv_int(dev
, "rxd", emx_rxd
);
2805 if ((nrxd
* sizeof(emx_rxdesc_t
)) % EMX_DBA_ALIGN
!= 0 ||
2806 nrxd
> EMX_MAX_RXD
|| nrxd
< EMX_MIN_RXD
) {
2807 device_printf(dev
, "Using %d RX descriptors instead of %d!\n",
2808 EMX_DEFAULT_RXD
, nrxd
);
2809 rdata
->num_rx_desc
= EMX_DEFAULT_RXD
;
2811 rdata
->num_rx_desc
= nrxd
;
2815 * Allocate Receive Descriptor ring
2817 rsize
= roundup2(rdata
->num_rx_desc
* sizeof(emx_rxdesc_t
),
2819 rdata
->rx_desc
= bus_dmamem_coherent_any(rdata
->sc
->parent_dtag
,
2820 EMX_DBA_ALIGN
, rsize
, BUS_DMA_WAITOK
,
2821 &rdata
->rx_desc_dtag
, &rdata
->rx_desc_dmap
,
2822 &rdata
->rx_desc_paddr
);
2823 if (rdata
->rx_desc
== NULL
) {
2824 device_printf(dev
, "Unable to allocate rx_desc memory\n");
2828 rsize
= __VM_CACHELINE_ALIGN(
2829 sizeof(struct emx_rxbuf
) * rdata
->num_rx_desc
);
2830 rdata
->rx_buf
= kmalloc_cachealign(rsize
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
2833 * Create DMA tag for rx buffers
2835 error
= bus_dma_tag_create(rdata
->sc
->parent_dtag
, /* parent */
2836 1, 0, /* alignment, bounds */
2837 BUS_SPACE_MAXADDR
, /* lowaddr */
2838 BUS_SPACE_MAXADDR
, /* highaddr */
2839 NULL
, NULL
, /* filter, filterarg */
2840 MCLBYTES
, /* maxsize */
2842 MCLBYTES
, /* maxsegsize */
2843 BUS_DMA_WAITOK
| BUS_DMA_ALLOCNOW
, /* flags */
2846 device_printf(dev
, "Unable to allocate RX DMA tag\n");
2847 kfree(rdata
->rx_buf
, M_DEVBUF
);
2848 rdata
->rx_buf
= NULL
;
2853 * Create spare DMA map for rx buffers
2855 error
= bus_dmamap_create(rdata
->rxtag
, BUS_DMA_WAITOK
,
2856 &rdata
->rx_sparemap
);
2858 device_printf(dev
, "Unable to create spare RX DMA map\n");
2859 bus_dma_tag_destroy(rdata
->rxtag
);
2860 kfree(rdata
->rx_buf
, M_DEVBUF
);
2861 rdata
->rx_buf
= NULL
;
2866 * Create DMA maps for rx buffers
2868 for (i
= 0; i
< rdata
->num_rx_desc
; i
++) {
2869 rx_buffer
= &rdata
->rx_buf
[i
];
2871 error
= bus_dmamap_create(rdata
->rxtag
, BUS_DMA_WAITOK
,
2874 device_printf(dev
, "Unable to create RX DMA map\n");
2875 emx_destroy_rx_ring(rdata
, i
);
2883 emx_free_rx_ring(struct emx_rxdata
*rdata
)
2887 for (i
= 0; i
< rdata
->num_rx_desc
; i
++) {
2888 struct emx_rxbuf
*rx_buffer
= &rdata
->rx_buf
[i
];
2890 if (rx_buffer
->m_head
!= NULL
) {
2891 bus_dmamap_unload(rdata
->rxtag
, rx_buffer
->map
);
2892 m_freem(rx_buffer
->m_head
);
2893 rx_buffer
->m_head
= NULL
;
2897 if (rdata
->fmp
!= NULL
)
2898 m_freem(rdata
->fmp
);
2904 emx_free_tx_ring(struct emx_txdata
*tdata
)
2908 for (i
= 0; i
< tdata
->num_tx_desc
; i
++) {
2909 struct emx_txbuf
*tx_buffer
= &tdata
->tx_buf
[i
];
2911 if (tx_buffer
->m_head
!= NULL
) {
2912 bus_dmamap_unload(tdata
->txtag
, tx_buffer
->map
);
2913 m_freem(tx_buffer
->m_head
);
2914 tx_buffer
->m_head
= NULL
;
2918 tdata
->tx_flags
&= ~EMX_TXFLAG_FORCECTX
;
2920 tdata
->csum_flags
= 0;
2921 tdata
->csum_lhlen
= 0;
2922 tdata
->csum_iphlen
= 0;
2923 tdata
->csum_thlen
= 0;
2924 tdata
->csum_mss
= 0;
2925 tdata
->csum_pktlen
= 0;
2927 tdata
->tx_dd_head
= 0;
2928 tdata
->tx_dd_tail
= 0;
2929 tdata
->tx_nsegs
= 0;
2933 emx_init_rx_ring(struct emx_rxdata
*rdata
)
2937 /* Reset descriptor ring */
2938 bzero(rdata
->rx_desc
, sizeof(emx_rxdesc_t
) * rdata
->num_rx_desc
);
2940 /* Allocate new ones. */
2941 for (i
= 0; i
< rdata
->num_rx_desc
; i
++) {
2942 error
= emx_newbuf(rdata
, i
, 1);
2947 /* Setup our descriptor pointers */
2948 rdata
->next_rx_desc_to_check
= 0;
2954 emx_init_rx_unit(struct emx_softc
*sc
)
2956 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
2958 uint32_t rctl
, itr
, rfctl
;
2962 * Make sure receives are disabled while setting
2963 * up the descriptor ring
2965 rctl
= E1000_READ_REG(&sc
->hw
, E1000_RCTL
);
2966 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, rctl
& ~E1000_RCTL_EN
);
2969 * Set the interrupt throttling rate. Value is calculated
2970 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns)
2972 if (sc
->int_throttle_ceil
)
2973 itr
= 1000000000 / 256 / sc
->int_throttle_ceil
;
2976 emx_set_itr(sc
, itr
);
2978 /* Use extended RX descriptor */
2979 rfctl
= E1000_RFCTL_EXTEN
;
2981 /* Disable accelerated ackknowledge */
2982 if (sc
->hw
.mac
.type
== e1000_82574
)
2983 rfctl
|= E1000_RFCTL_ACK_DIS
;
2985 E1000_WRITE_REG(&sc
->hw
, E1000_RFCTL
, rfctl
);
2988 * Receive Checksum Offload for TCP and UDP
2990 * Checksum offloading is also enabled if multiple receive
2991 * queue is to be supported, since we need it to figure out
2994 if ((ifp
->if_capenable
& IFCAP_RXCSUM
) ||
2995 sc
->rx_ring_cnt
> 1) {
2998 rxcsum
= E1000_READ_REG(&sc
->hw
, E1000_RXCSUM
);
3002 * PCSD must be enabled to enable multiple
3005 rxcsum
|= E1000_RXCSUM_IPOFL
| E1000_RXCSUM_TUOFL
|
3007 E1000_WRITE_REG(&sc
->hw
, E1000_RXCSUM
, rxcsum
);
3011 * Configure multiple receive queue (RSS)
3013 if (sc
->rx_ring_cnt
> 1) {
3014 uint8_t key
[EMX_NRSSRK
* EMX_RSSRK_SIZE
];
3017 KASSERT(sc
->rx_ring_cnt
== EMX_NRX_RING
,
3018 ("invalid number of RX ring (%d)", sc
->rx_ring_cnt
));
3022 * When we reach here, RSS has already been disabled
3023 * in emx_stop(), so we could safely configure RSS key
3024 * and redirect table.
3030 toeplitz_get_key(key
, sizeof(key
));
3031 for (i
= 0; i
< EMX_NRSSRK
; ++i
) {
3034 rssrk
= EMX_RSSRK_VAL(key
, i
);
3035 EMX_RSS_DPRINTF(sc
, 1, "rssrk%d 0x%08x\n", i
, rssrk
);
3037 E1000_WRITE_REG(&sc
->hw
, E1000_RSSRK(i
), rssrk
);
3041 * Configure RSS redirect table in following fashion:
3042 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
3045 for (i
= 0; i
< EMX_RETA_SIZE
; ++i
) {
3048 q
= (i
% sc
->rx_ring_cnt
) << EMX_RETA_RINGIDX_SHIFT
;
3049 reta
|= q
<< (8 * i
);
3051 EMX_RSS_DPRINTF(sc
, 1, "reta 0x%08x\n", reta
);
3053 for (i
= 0; i
< EMX_NRETA
; ++i
)
3054 E1000_WRITE_REG(&sc
->hw
, E1000_RETA(i
), reta
);
3057 * Enable multiple receive queues.
3058 * Enable IPv4 RSS standard hash functions.
3059 * Disable RSS interrupt.
3061 E1000_WRITE_REG(&sc
->hw
, E1000_MRQC
,
3062 E1000_MRQC_ENABLE_RSS_2Q
|
3063 E1000_MRQC_RSS_FIELD_IPV4_TCP
|
3064 E1000_MRQC_RSS_FIELD_IPV4
);
3068 * XXX TEMPORARY WORKAROUND: on some systems with 82573
3069 * long latencies are observed, like Lenovo X60. This
3070 * change eliminates the problem, but since having positive
3071 * values in RDTR is a known source of problems on other
3072 * platforms another solution is being sought.
3074 if (emx_82573_workaround
&& sc
->hw
.mac
.type
== e1000_82573
) {
3075 E1000_WRITE_REG(&sc
->hw
, E1000_RADV
, EMX_RADV_82573
);
3076 E1000_WRITE_REG(&sc
->hw
, E1000_RDTR
, EMX_RDTR_82573
);
3079 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
3080 struct emx_rxdata
*rdata
= &sc
->rx_data
[i
];
3083 * Setup the Base and Length of the Rx Descriptor Ring
3085 bus_addr
= rdata
->rx_desc_paddr
;
3086 E1000_WRITE_REG(&sc
->hw
, E1000_RDLEN(i
),
3087 rdata
->num_rx_desc
* sizeof(emx_rxdesc_t
));
3088 E1000_WRITE_REG(&sc
->hw
, E1000_RDBAH(i
),
3089 (uint32_t)(bus_addr
>> 32));
3090 E1000_WRITE_REG(&sc
->hw
, E1000_RDBAL(i
),
3091 (uint32_t)bus_addr
);
3094 * Setup the HW Rx Head and Tail Descriptor Pointers
3096 E1000_WRITE_REG(&sc
->hw
, E1000_RDH(i
), 0);
3097 E1000_WRITE_REG(&sc
->hw
, E1000_RDT(i
),
3098 sc
->rx_data
[i
].num_rx_desc
- 1);
3101 if (sc
->hw
.mac
.type
>= e1000_pch2lan
) {
3102 if (ifp
->if_mtu
> ETHERMTU
)
3103 e1000_lv_jumbo_workaround_ich8lan(&sc
->hw
, TRUE
);
3105 e1000_lv_jumbo_workaround_ich8lan(&sc
->hw
, FALSE
);
3108 /* Setup the Receive Control Register */
3109 rctl
&= ~(3 << E1000_RCTL_MO_SHIFT
);
3110 rctl
|= E1000_RCTL_EN
| E1000_RCTL_BAM
| E1000_RCTL_LBM_NO
|
3111 E1000_RCTL_RDMTS_HALF
| E1000_RCTL_SECRC
|
3112 (sc
->hw
.mac
.mc_filter_type
<< E1000_RCTL_MO_SHIFT
);
3114 /* Make sure VLAN Filters are off */
3115 rctl
&= ~E1000_RCTL_VFE
;
3117 /* Don't store bad paket */
3118 rctl
&= ~E1000_RCTL_SBP
;
3121 rctl
|= E1000_RCTL_SZ_2048
;
3123 if (ifp
->if_mtu
> ETHERMTU
)
3124 rctl
|= E1000_RCTL_LPE
;
3126 rctl
&= ~E1000_RCTL_LPE
;
3128 /* Enable Receives */
3129 E1000_WRITE_REG(&sc
->hw
, E1000_RCTL
, rctl
);
3133 emx_destroy_rx_ring(struct emx_rxdata
*rdata
, int ndesc
)
3135 struct emx_rxbuf
*rx_buffer
;
3138 /* Free Receive Descriptor ring */
3139 if (rdata
->rx_desc
) {
3140 bus_dmamap_unload(rdata
->rx_desc_dtag
, rdata
->rx_desc_dmap
);
3141 bus_dmamem_free(rdata
->rx_desc_dtag
, rdata
->rx_desc
,
3142 rdata
->rx_desc_dmap
);
3143 bus_dma_tag_destroy(rdata
->rx_desc_dtag
);
3145 rdata
->rx_desc
= NULL
;
3148 if (rdata
->rx_buf
== NULL
)
3151 for (i
= 0; i
< ndesc
; i
++) {
3152 rx_buffer
= &rdata
->rx_buf
[i
];
3154 KKASSERT(rx_buffer
->m_head
== NULL
);
3155 bus_dmamap_destroy(rdata
->rxtag
, rx_buffer
->map
);
3157 bus_dmamap_destroy(rdata
->rxtag
, rdata
->rx_sparemap
);
3158 bus_dma_tag_destroy(rdata
->rxtag
);
3160 kfree(rdata
->rx_buf
, M_DEVBUF
);
3161 rdata
->rx_buf
= NULL
;
3165 emx_rxeof(struct emx_rxdata
*rdata
, int count
)
3167 struct ifnet
*ifp
= &rdata
->sc
->arpcom
.ac_if
;
3169 emx_rxdesc_t
*current_desc
;
3171 int i
, cpuid
= mycpuid
;
3173 i
= rdata
->next_rx_desc_to_check
;
3174 current_desc
= &rdata
->rx_desc
[i
];
3175 staterr
= le32toh(current_desc
->rxd_staterr
);
3177 if (!(staterr
& E1000_RXD_STAT_DD
))
3180 while ((staterr
& E1000_RXD_STAT_DD
) && count
!= 0) {
3181 struct pktinfo
*pi
= NULL
, pi0
;
3182 struct emx_rxbuf
*rx_buf
= &rdata
->rx_buf
[i
];
3183 struct mbuf
*m
= NULL
;
3188 mp
= rx_buf
->m_head
;
3191 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3192 * needs to access the last received byte in the mbuf.
3194 bus_dmamap_sync(rdata
->rxtag
, rx_buf
->map
,
3195 BUS_DMASYNC_POSTREAD
);
3197 len
= le16toh(current_desc
->rxd_length
);
3198 if (staterr
& E1000_RXD_STAT_EOP
) {
3205 if (!(staterr
& E1000_RXDEXT_ERR_FRAME_ERR_MASK
)) {
3207 uint32_t mrq
, rss_hash
;
3210 * Save several necessary information,
3211 * before emx_newbuf() destroy it.
3213 if ((staterr
& E1000_RXD_STAT_VP
) && eop
)
3214 vlan
= le16toh(current_desc
->rxd_vlan
);
3216 mrq
= le32toh(current_desc
->rxd_mrq
);
3217 rss_hash
= le32toh(current_desc
->rxd_rss
);
3219 EMX_RSS_DPRINTF(rdata
->sc
, 10,
3220 "ring%d, mrq 0x%08x, rss_hash 0x%08x\n",
3221 rdata
->idx
, mrq
, rss_hash
);
3223 if (emx_newbuf(rdata
, i
, 0) != 0) {
3224 IFNET_STAT_INC(ifp
, iqdrops
, 1);
3228 /* Assign correct length to the current fragment */
3231 if (rdata
->fmp
== NULL
) {
3232 mp
->m_pkthdr
.len
= len
;
3233 rdata
->fmp
= mp
; /* Store the first mbuf */
3237 * Chain mbuf's together
3239 rdata
->lmp
->m_next
= mp
;
3240 rdata
->lmp
= rdata
->lmp
->m_next
;
3241 rdata
->fmp
->m_pkthdr
.len
+= len
;
3245 rdata
->fmp
->m_pkthdr
.rcvif
= ifp
;
3246 IFNET_STAT_INC(ifp
, ipackets
, 1);
3248 if (ifp
->if_capenable
& IFCAP_RXCSUM
)
3249 emx_rxcsum(staterr
, rdata
->fmp
);
3251 if (staterr
& E1000_RXD_STAT_VP
) {
3252 rdata
->fmp
->m_pkthdr
.ether_vlantag
=
3254 rdata
->fmp
->m_flags
|= M_VLANTAG
;
3260 if (ifp
->if_capenable
& IFCAP_RSS
) {
3261 pi
= emx_rssinfo(m
, &pi0
, mrq
,
3264 #ifdef EMX_RSS_DEBUG
3269 IFNET_STAT_INC(ifp
, ierrors
, 1);
3271 emx_setup_rxdesc(current_desc
, rx_buf
);
3272 if (rdata
->fmp
!= NULL
) {
3273 m_freem(rdata
->fmp
);
3281 ifp
->if_input(ifp
, m
, pi
, cpuid
);
3283 /* Advance our pointers to the next descriptor. */
3284 if (++i
== rdata
->num_rx_desc
)
3287 current_desc
= &rdata
->rx_desc
[i
];
3288 staterr
= le32toh(current_desc
->rxd_staterr
);
3290 rdata
->next_rx_desc_to_check
= i
;
3292 /* Advance the E1000's Receive Queue "Tail Pointer". */
3294 i
= rdata
->num_rx_desc
- 1;
3295 E1000_WRITE_REG(&rdata
->sc
->hw
, E1000_RDT(rdata
->idx
), i
);
3299 emx_enable_intr(struct emx_softc
*sc
)
3301 uint32_t ims_mask
= IMS_ENABLE_MASK
;
3303 lwkt_serialize_handler_enable(&sc
->main_serialize
);
3306 if (sc
->hw
.mac
.type
== e1000_82574
) {
3307 E1000_WRITE_REG(hw
, EMX_EIAC
, EM_MSIX_MASK
);
3308 ims_mask
|= EM_MSIX_MASK
;
3311 E1000_WRITE_REG(&sc
->hw
, E1000_IMS
, ims_mask
);
3315 emx_disable_intr(struct emx_softc
*sc
)
3317 if (sc
->hw
.mac
.type
== e1000_82574
)
3318 E1000_WRITE_REG(&sc
->hw
, EMX_EIAC
, 0);
3319 E1000_WRITE_REG(&sc
->hw
, E1000_IMC
, 0xffffffff);
3321 lwkt_serialize_handler_disable(&sc
->main_serialize
);
3325 * Bit of a misnomer, what this really means is
3326 * to enable OS management of the system... aka
3327 * to disable special hardware management features
3330 emx_get_mgmt(struct emx_softc
*sc
)
3332 /* A shared code workaround */
3333 if (sc
->flags
& EMX_FLAG_HAS_MGMT
) {
3334 int manc2h
= E1000_READ_REG(&sc
->hw
, E1000_MANC2H
);
3335 int manc
= E1000_READ_REG(&sc
->hw
, E1000_MANC
);
3337 /* disable hardware interception of ARP */
3338 manc
&= ~(E1000_MANC_ARP_EN
);
3340 /* enable receiving management packets to the host */
3341 manc
|= E1000_MANC_EN_MNG2HOST
;
3342 #define E1000_MNG2HOST_PORT_623 (1 << 5)
3343 #define E1000_MNG2HOST_PORT_664 (1 << 6)
3344 manc2h
|= E1000_MNG2HOST_PORT_623
;
3345 manc2h
|= E1000_MNG2HOST_PORT_664
;
3346 E1000_WRITE_REG(&sc
->hw
, E1000_MANC2H
, manc2h
);
3348 E1000_WRITE_REG(&sc
->hw
, E1000_MANC
, manc
);
3353 * Give control back to hardware management
3354 * controller if there is one.
3357 emx_rel_mgmt(struct emx_softc
*sc
)
3359 if (sc
->flags
& EMX_FLAG_HAS_MGMT
) {
3360 int manc
= E1000_READ_REG(&sc
->hw
, E1000_MANC
);
3362 /* re-enable hardware interception of ARP */
3363 manc
|= E1000_MANC_ARP_EN
;
3364 manc
&= ~E1000_MANC_EN_MNG2HOST
;
3366 E1000_WRITE_REG(&sc
->hw
, E1000_MANC
, manc
);
3371 * emx_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3372 * For ASF and Pass Through versions of f/w this means that
3373 * the driver is loaded. For AMT version (only with 82573)
3374 * of the f/w this means that the network i/f is open.
3377 emx_get_hw_control(struct emx_softc
*sc
)
3379 /* Let firmware know the driver has taken over */
3380 if (sc
->hw
.mac
.type
== e1000_82573
) {
3383 swsm
= E1000_READ_REG(&sc
->hw
, E1000_SWSM
);
3384 E1000_WRITE_REG(&sc
->hw
, E1000_SWSM
,
3385 swsm
| E1000_SWSM_DRV_LOAD
);
3389 ctrl_ext
= E1000_READ_REG(&sc
->hw
, E1000_CTRL_EXT
);
3390 E1000_WRITE_REG(&sc
->hw
, E1000_CTRL_EXT
,
3391 ctrl_ext
| E1000_CTRL_EXT_DRV_LOAD
);
3393 sc
->flags
|= EMX_FLAG_HW_CTRL
;
3397 * emx_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3398 * For ASF and Pass Through versions of f/w this means that the
3399 * driver is no longer loaded. For AMT version (only with 82573)
3400 * of the f/w this means that the network i/f is closed.
3403 emx_rel_hw_control(struct emx_softc
*sc
)
3405 if ((sc
->flags
& EMX_FLAG_HW_CTRL
) == 0)
3407 sc
->flags
&= ~EMX_FLAG_HW_CTRL
;
3409 /* Let firmware taken over control of h/w */
3410 if (sc
->hw
.mac
.type
== e1000_82573
) {
3413 swsm
= E1000_READ_REG(&sc
->hw
, E1000_SWSM
);
3414 E1000_WRITE_REG(&sc
->hw
, E1000_SWSM
,
3415 swsm
& ~E1000_SWSM_DRV_LOAD
);
3419 ctrl_ext
= E1000_READ_REG(&sc
->hw
, E1000_CTRL_EXT
);
3420 E1000_WRITE_REG(&sc
->hw
, E1000_CTRL_EXT
,
3421 ctrl_ext
& ~E1000_CTRL_EXT_DRV_LOAD
);
3426 emx_is_valid_eaddr(const uint8_t *addr
)
3428 char zero_addr
[ETHER_ADDR_LEN
] = { 0, 0, 0, 0, 0, 0 };
3430 if ((addr
[0] & 1) || !bcmp(addr
, zero_addr
, ETHER_ADDR_LEN
))
3437 * Enable PCI Wake On Lan capability
3440 emx_enable_wol(device_t dev
)
3442 uint16_t cap
, status
;
3445 /* First find the capabilities pointer*/
3446 cap
= pci_read_config(dev
, PCIR_CAP_PTR
, 2);
3448 /* Read the PM Capabilities */
3449 id
= pci_read_config(dev
, cap
, 1);
3450 if (id
!= PCIY_PMG
) /* Something wrong */
3454 * OK, we have the power capabilities,
3455 * so now get the status register
3457 cap
+= PCIR_POWER_STATUS
;
3458 status
= pci_read_config(dev
, cap
, 2);
3459 status
|= PCIM_PSTAT_PME
| PCIM_PSTAT_PMEENABLE
;
3460 pci_write_config(dev
, cap
, status
, 2);
3464 emx_update_stats(struct emx_softc
*sc
)
3466 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3468 if (sc
->hw
.phy
.media_type
== e1000_media_type_copper
||
3469 (E1000_READ_REG(&sc
->hw
, E1000_STATUS
) & E1000_STATUS_LU
)) {
3470 sc
->stats
.symerrs
+= E1000_READ_REG(&sc
->hw
, E1000_SYMERRS
);
3471 sc
->stats
.sec
+= E1000_READ_REG(&sc
->hw
, E1000_SEC
);
3473 sc
->stats
.crcerrs
+= E1000_READ_REG(&sc
->hw
, E1000_CRCERRS
);
3474 sc
->stats
.mpc
+= E1000_READ_REG(&sc
->hw
, E1000_MPC
);
3475 sc
->stats
.scc
+= E1000_READ_REG(&sc
->hw
, E1000_SCC
);
3476 sc
->stats
.ecol
+= E1000_READ_REG(&sc
->hw
, E1000_ECOL
);
3478 sc
->stats
.mcc
+= E1000_READ_REG(&sc
->hw
, E1000_MCC
);
3479 sc
->stats
.latecol
+= E1000_READ_REG(&sc
->hw
, E1000_LATECOL
);
3480 sc
->stats
.colc
+= E1000_READ_REG(&sc
->hw
, E1000_COLC
);
3481 sc
->stats
.dc
+= E1000_READ_REG(&sc
->hw
, E1000_DC
);
3482 sc
->stats
.rlec
+= E1000_READ_REG(&sc
->hw
, E1000_RLEC
);
3483 sc
->stats
.xonrxc
+= E1000_READ_REG(&sc
->hw
, E1000_XONRXC
);
3484 sc
->stats
.xontxc
+= E1000_READ_REG(&sc
->hw
, E1000_XONTXC
);
3485 sc
->stats
.xoffrxc
+= E1000_READ_REG(&sc
->hw
, E1000_XOFFRXC
);
3486 sc
->stats
.xofftxc
+= E1000_READ_REG(&sc
->hw
, E1000_XOFFTXC
);
3487 sc
->stats
.fcruc
+= E1000_READ_REG(&sc
->hw
, E1000_FCRUC
);
3488 sc
->stats
.prc64
+= E1000_READ_REG(&sc
->hw
, E1000_PRC64
);
3489 sc
->stats
.prc127
+= E1000_READ_REG(&sc
->hw
, E1000_PRC127
);
3490 sc
->stats
.prc255
+= E1000_READ_REG(&sc
->hw
, E1000_PRC255
);
3491 sc
->stats
.prc511
+= E1000_READ_REG(&sc
->hw
, E1000_PRC511
);
3492 sc
->stats
.prc1023
+= E1000_READ_REG(&sc
->hw
, E1000_PRC1023
);
3493 sc
->stats
.prc1522
+= E1000_READ_REG(&sc
->hw
, E1000_PRC1522
);
3494 sc
->stats
.gprc
+= E1000_READ_REG(&sc
->hw
, E1000_GPRC
);
3495 sc
->stats
.bprc
+= E1000_READ_REG(&sc
->hw
, E1000_BPRC
);
3496 sc
->stats
.mprc
+= E1000_READ_REG(&sc
->hw
, E1000_MPRC
);
3497 sc
->stats
.gptc
+= E1000_READ_REG(&sc
->hw
, E1000_GPTC
);
3499 /* For the 64-bit byte counters the low dword must be read first. */
3500 /* Both registers clear on the read of the high dword */
3502 sc
->stats
.gorc
+= E1000_READ_REG(&sc
->hw
, E1000_GORCH
);
3503 sc
->stats
.gotc
+= E1000_READ_REG(&sc
->hw
, E1000_GOTCH
);
3505 sc
->stats
.rnbc
+= E1000_READ_REG(&sc
->hw
, E1000_RNBC
);
3506 sc
->stats
.ruc
+= E1000_READ_REG(&sc
->hw
, E1000_RUC
);
3507 sc
->stats
.rfc
+= E1000_READ_REG(&sc
->hw
, E1000_RFC
);
3508 sc
->stats
.roc
+= E1000_READ_REG(&sc
->hw
, E1000_ROC
);
3509 sc
->stats
.rjc
+= E1000_READ_REG(&sc
->hw
, E1000_RJC
);
3511 sc
->stats
.tor
+= E1000_READ_REG(&sc
->hw
, E1000_TORH
);
3512 sc
->stats
.tot
+= E1000_READ_REG(&sc
->hw
, E1000_TOTH
);
3514 sc
->stats
.tpr
+= E1000_READ_REG(&sc
->hw
, E1000_TPR
);
3515 sc
->stats
.tpt
+= E1000_READ_REG(&sc
->hw
, E1000_TPT
);
3516 sc
->stats
.ptc64
+= E1000_READ_REG(&sc
->hw
, E1000_PTC64
);
3517 sc
->stats
.ptc127
+= E1000_READ_REG(&sc
->hw
, E1000_PTC127
);
3518 sc
->stats
.ptc255
+= E1000_READ_REG(&sc
->hw
, E1000_PTC255
);
3519 sc
->stats
.ptc511
+= E1000_READ_REG(&sc
->hw
, E1000_PTC511
);
3520 sc
->stats
.ptc1023
+= E1000_READ_REG(&sc
->hw
, E1000_PTC1023
);
3521 sc
->stats
.ptc1522
+= E1000_READ_REG(&sc
->hw
, E1000_PTC1522
);
3522 sc
->stats
.mptc
+= E1000_READ_REG(&sc
->hw
, E1000_MPTC
);
3523 sc
->stats
.bptc
+= E1000_READ_REG(&sc
->hw
, E1000_BPTC
);
3525 sc
->stats
.algnerrc
+= E1000_READ_REG(&sc
->hw
, E1000_ALGNERRC
);
3526 sc
->stats
.rxerrc
+= E1000_READ_REG(&sc
->hw
, E1000_RXERRC
);
3527 sc
->stats
.tncrs
+= E1000_READ_REG(&sc
->hw
, E1000_TNCRS
);
3528 sc
->stats
.cexterr
+= E1000_READ_REG(&sc
->hw
, E1000_CEXTERR
);
3529 sc
->stats
.tsctc
+= E1000_READ_REG(&sc
->hw
, E1000_TSCTC
);
3530 sc
->stats
.tsctfc
+= E1000_READ_REG(&sc
->hw
, E1000_TSCTFC
);
3532 IFNET_STAT_SET(ifp
, collisions
, sc
->stats
.colc
);
3535 IFNET_STAT_SET(ifp
, ierrors
,
3536 sc
->stats
.rxerrc
+ sc
->stats
.crcerrs
+ sc
->stats
.algnerrc
+
3537 sc
->stats
.ruc
+ sc
->stats
.roc
+ sc
->stats
.mpc
+ sc
->stats
.cexterr
);
3540 IFNET_STAT_SET(ifp
, oerrors
, sc
->stats
.ecol
+ sc
->stats
.latecol
);
3544 emx_print_debug_info(struct emx_softc
*sc
)
3546 device_t dev
= sc
->dev
;
3547 uint8_t *hw_addr
= sc
->hw
.hw_addr
;
3550 device_printf(dev
, "Adapter hardware address = %p \n", hw_addr
);
3551 device_printf(dev
, "CTRL = 0x%x RCTL = 0x%x \n",
3552 E1000_READ_REG(&sc
->hw
, E1000_CTRL
),
3553 E1000_READ_REG(&sc
->hw
, E1000_RCTL
));
3554 device_printf(dev
, "Packet buffer = Tx=%dk Rx=%dk \n",
3555 ((E1000_READ_REG(&sc
->hw
, E1000_PBA
) & 0xffff0000) >> 16),\
3556 (E1000_READ_REG(&sc
->hw
, E1000_PBA
) & 0xffff) );
3557 device_printf(dev
, "Flow control watermarks high = %d low = %d\n",
3558 sc
->hw
.fc
.high_water
, sc
->hw
.fc
.low_water
);
3559 device_printf(dev
, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3560 E1000_READ_REG(&sc
->hw
, E1000_TIDV
),
3561 E1000_READ_REG(&sc
->hw
, E1000_TADV
));
3562 device_printf(dev
, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3563 E1000_READ_REG(&sc
->hw
, E1000_RDTR
),
3564 E1000_READ_REG(&sc
->hw
, E1000_RADV
));
3566 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3567 device_printf(dev
, "hw %d tdh = %d, hw tdt = %d\n", i
,
3568 E1000_READ_REG(&sc
->hw
, E1000_TDH(i
)),
3569 E1000_READ_REG(&sc
->hw
, E1000_TDT(i
)));
3571 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
3572 device_printf(dev
, "hw %d rdh = %d, hw rdt = %d\n", i
,
3573 E1000_READ_REG(&sc
->hw
, E1000_RDH(i
)),
3574 E1000_READ_REG(&sc
->hw
, E1000_RDT(i
)));
3577 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3578 device_printf(dev
, "TX %d Tx descriptors avail = %d\n", i
,
3579 sc
->tx_data
[i
].num_tx_desc_avail
);
3580 device_printf(dev
, "TX %d TSO segments = %lu\n", i
,
3581 sc
->tx_data
[i
].tso_segments
);
3582 device_printf(dev
, "TX %d TSO ctx reused = %lu\n", i
,
3583 sc
->tx_data
[i
].tso_ctx_reused
);
3588 emx_print_hw_stats(struct emx_softc
*sc
)
3590 device_t dev
= sc
->dev
;
3592 device_printf(dev
, "Excessive collisions = %lld\n",
3593 (long long)sc
->stats
.ecol
);
3594 #if (DEBUG_HW > 0) /* Dont output these errors normally */
3595 device_printf(dev
, "Symbol errors = %lld\n",
3596 (long long)sc
->stats
.symerrs
);
3598 device_printf(dev
, "Sequence errors = %lld\n",
3599 (long long)sc
->stats
.sec
);
3600 device_printf(dev
, "Defer count = %lld\n",
3601 (long long)sc
->stats
.dc
);
3602 device_printf(dev
, "Missed Packets = %lld\n",
3603 (long long)sc
->stats
.mpc
);
3604 device_printf(dev
, "Receive No Buffers = %lld\n",
3605 (long long)sc
->stats
.rnbc
);
3606 /* RLEC is inaccurate on some hardware, calculate our own. */
3607 device_printf(dev
, "Receive Length Errors = %lld\n",
3608 ((long long)sc
->stats
.roc
+ (long long)sc
->stats
.ruc
));
3609 device_printf(dev
, "Receive errors = %lld\n",
3610 (long long)sc
->stats
.rxerrc
);
3611 device_printf(dev
, "Crc errors = %lld\n",
3612 (long long)sc
->stats
.crcerrs
);
3613 device_printf(dev
, "Alignment errors = %lld\n",
3614 (long long)sc
->stats
.algnerrc
);
3615 device_printf(dev
, "Collision/Carrier extension errors = %lld\n",
3616 (long long)sc
->stats
.cexterr
);
3617 device_printf(dev
, "RX overruns = %ld\n", sc
->rx_overruns
);
3618 device_printf(dev
, "XON Rcvd = %lld\n",
3619 (long long)sc
->stats
.xonrxc
);
3620 device_printf(dev
, "XON Xmtd = %lld\n",
3621 (long long)sc
->stats
.xontxc
);
3622 device_printf(dev
, "XOFF Rcvd = %lld\n",
3623 (long long)sc
->stats
.xoffrxc
);
3624 device_printf(dev
, "XOFF Xmtd = %lld\n",
3625 (long long)sc
->stats
.xofftxc
);
3626 device_printf(dev
, "Good Packets Rcvd = %lld\n",
3627 (long long)sc
->stats
.gprc
);
3628 device_printf(dev
, "Good Packets Xmtd = %lld\n",
3629 (long long)sc
->stats
.gptc
);
3633 emx_print_nvm_info(struct emx_softc
*sc
)
3635 uint16_t eeprom_data
;
3638 /* Its a bit crude, but it gets the job done */
3639 kprintf("\nInterface EEPROM Dump:\n");
3640 kprintf("Offset\n0x0000 ");
3641 for (i
= 0, j
= 0; i
< 32; i
++, j
++) {
3642 if (j
== 8) { /* Make the offset block */
3644 kprintf("\n0x00%x0 ",row
);
3646 e1000_read_nvm(&sc
->hw
, i
, 1, &eeprom_data
);
3647 kprintf("%04x ", eeprom_data
);
3653 emx_sysctl_debug_info(SYSCTL_HANDLER_ARGS
)
3655 struct emx_softc
*sc
;
3660 error
= sysctl_handle_int(oidp
, &result
, 0, req
);
3661 if (error
|| !req
->newptr
)
3664 sc
= (struct emx_softc
*)arg1
;
3665 ifp
= &sc
->arpcom
.ac_if
;
3667 ifnet_serialize_all(ifp
);
3670 emx_print_debug_info(sc
);
3673 * This value will cause a hex dump of the
3674 * first 32 16-bit words of the EEPROM to
3678 emx_print_nvm_info(sc
);
3680 ifnet_deserialize_all(ifp
);
3686 emx_sysctl_stats(SYSCTL_HANDLER_ARGS
)
3691 error
= sysctl_handle_int(oidp
, &result
, 0, req
);
3692 if (error
|| !req
->newptr
)
3696 struct emx_softc
*sc
= (struct emx_softc
*)arg1
;
3697 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3699 ifnet_serialize_all(ifp
);
3700 emx_print_hw_stats(sc
);
3701 ifnet_deserialize_all(ifp
);
3707 emx_add_sysctl(struct emx_softc
*sc
)
3709 struct sysctl_ctx_list
*ctx
;
3710 struct sysctl_oid
*tree
;
3711 #if defined(EMX_RSS_DEBUG) || defined(EMX_TSS_DEBUG)
3716 ctx
= device_get_sysctl_ctx(sc
->dev
);
3717 tree
= device_get_sysctl_tree(sc
->dev
);
3718 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3719 OID_AUTO
, "debug", CTLTYPE_INT
|CTLFLAG_RW
, sc
, 0,
3720 emx_sysctl_debug_info
, "I", "Debug Information");
3722 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3723 OID_AUTO
, "stats", CTLTYPE_INT
|CTLFLAG_RW
, sc
, 0,
3724 emx_sysctl_stats
, "I", "Statistics");
3726 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3727 OID_AUTO
, "rxd", CTLFLAG_RD
, &sc
->rx_data
[0].num_rx_desc
, 0,
3729 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3730 OID_AUTO
, "txd", CTLFLAG_RD
, &sc
->tx_data
[0].num_tx_desc
, 0,
3733 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3734 OID_AUTO
, "int_throttle_ceil", CTLTYPE_INT
|CTLFLAG_RW
, sc
, 0,
3735 emx_sysctl_int_throttle
, "I", "interrupt throttling rate");
3736 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3737 OID_AUTO
, "tx_intr_nsegs", CTLTYPE_INT
|CTLFLAG_RW
, sc
, 0,
3738 emx_sysctl_tx_intr_nsegs
, "I", "# segments per TX interrupt");
3739 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3740 OID_AUTO
, "tx_wreg_nsegs", CTLTYPE_INT
|CTLFLAG_RW
, sc
, 0,
3741 emx_sysctl_tx_wreg_nsegs
, "I",
3742 "# segments sent before write to hardware register");
3744 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3745 OID_AUTO
, "rx_ring_cnt", CTLFLAG_RD
, &sc
->rx_ring_cnt
, 0,
3747 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3748 OID_AUTO
, "tx_ring_cnt", CTLFLAG_RD
, &sc
->tx_ring_cnt
, 0,
3750 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3751 OID_AUTO
, "tx_ring_inuse", CTLFLAG_RD
, &sc
->tx_ring_inuse
, 0,
3752 "# of TX rings used");
3754 #ifdef IFPOLL_ENABLE
3755 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3756 OID_AUTO
, "npoll_rxoff", CTLTYPE_INT
|CTLFLAG_RW
,
3757 sc
, 0, emx_sysctl_npoll_rxoff
, "I",
3758 "NPOLLING RX cpu offset");
3759 SYSCTL_ADD_PROC(ctx
, SYSCTL_CHILDREN(tree
),
3760 OID_AUTO
, "npoll_txoff", CTLTYPE_INT
|CTLFLAG_RW
,
3761 sc
, 0, emx_sysctl_npoll_txoff
, "I",
3762 "NPOLLING TX cpu offset");
3765 #ifdef EMX_RSS_DEBUG
3766 SYSCTL_ADD_INT(ctx
, SYSCTL_CHILDREN(tree
),
3767 OID_AUTO
, "rss_debug", CTLFLAG_RW
, &sc
->rss_debug
,
3768 0, "RSS debug level");
3769 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
3770 ksnprintf(pkt_desc
, sizeof(pkt_desc
), "rx%d_pkt", i
);
3771 SYSCTL_ADD_ULONG(ctx
, SYSCTL_CHILDREN(tree
), OID_AUTO
,
3772 pkt_desc
, CTLFLAG_RW
, &sc
->rx_data
[i
].rx_pkts
,
3776 #ifdef EMX_TSS_DEBUG
3777 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3778 ksnprintf(pkt_desc
, sizeof(pkt_desc
), "tx%d_pkt", i
);
3779 SYSCTL_ADD_ULONG(ctx
, SYSCTL_CHILDREN(tree
), OID_AUTO
,
3780 pkt_desc
, CTLFLAG_RW
, &sc
->tx_data
[i
].tx_pkts
,
3787 emx_sysctl_int_throttle(SYSCTL_HANDLER_ARGS
)
3789 struct emx_softc
*sc
= (void *)arg1
;
3790 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3791 int error
, throttle
;
3793 throttle
= sc
->int_throttle_ceil
;
3794 error
= sysctl_handle_int(oidp
, &throttle
, 0, req
);
3795 if (error
|| req
->newptr
== NULL
)
3797 if (throttle
< 0 || throttle
> 1000000000 / 256)
3802 * Set the interrupt throttling rate in 256ns increments,
3803 * recalculate sysctl value assignment to get exact frequency.
3805 throttle
= 1000000000 / 256 / throttle
;
3807 /* Upper 16bits of ITR is reserved and should be zero */
3808 if (throttle
& 0xffff0000)
3812 ifnet_serialize_all(ifp
);
3815 sc
->int_throttle_ceil
= 1000000000 / 256 / throttle
;
3817 sc
->int_throttle_ceil
= 0;
3819 if (ifp
->if_flags
& IFF_RUNNING
)
3820 emx_set_itr(sc
, throttle
);
3822 ifnet_deserialize_all(ifp
);
3825 if_printf(ifp
, "Interrupt moderation set to %d/sec\n",
3826 sc
->int_throttle_ceil
);
3832 emx_sysctl_tx_intr_nsegs(SYSCTL_HANDLER_ARGS
)
3834 struct emx_softc
*sc
= (void *)arg1
;
3835 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3836 struct emx_txdata
*tdata
= &sc
->tx_data
[0];
3839 segs
= tdata
->tx_intr_nsegs
;
3840 error
= sysctl_handle_int(oidp
, &segs
, 0, req
);
3841 if (error
|| req
->newptr
== NULL
)
3846 ifnet_serialize_all(ifp
);
3849 * Don't allow tx_intr_nsegs to become:
3850 * o Less the oact_tx_desc
3851 * o Too large that no TX desc will cause TX interrupt to
3852 * be generated (OACTIVE will never recover)
3853 * o Too small that will cause tx_dd[] overflow
3855 if (segs
< tdata
->oact_tx_desc
||
3856 segs
>= tdata
->num_tx_desc
- tdata
->oact_tx_desc
||
3857 segs
< tdata
->num_tx_desc
/ EMX_TXDD_SAFE
) {
3863 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
3864 sc
->tx_data
[i
].tx_intr_nsegs
= segs
;
3867 ifnet_deserialize_all(ifp
);
3873 emx_sysctl_tx_wreg_nsegs(SYSCTL_HANDLER_ARGS
)
3875 struct emx_softc
*sc
= (void *)arg1
;
3876 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3877 int error
, nsegs
, i
;
3879 nsegs
= sc
->tx_data
[0].tx_wreg_nsegs
;
3880 error
= sysctl_handle_int(oidp
, &nsegs
, 0, req
);
3881 if (error
|| req
->newptr
== NULL
)
3884 ifnet_serialize_all(ifp
);
3885 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
)
3886 sc
->tx_data
[i
].tx_wreg_nsegs
=nsegs
;
3887 ifnet_deserialize_all(ifp
);
3892 #ifdef IFPOLL_ENABLE
3895 emx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS
)
3897 struct emx_softc
*sc
= (void *)arg1
;
3898 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3901 off
= sc
->rx_npoll_off
;
3902 error
= sysctl_handle_int(oidp
, &off
, 0, req
);
3903 if (error
|| req
->newptr
== NULL
)
3908 ifnet_serialize_all(ifp
);
3909 if (off
>= ncpus2
|| off
% sc
->rx_ring_cnt
!= 0) {
3913 sc
->rx_npoll_off
= off
;
3915 ifnet_deserialize_all(ifp
);
3921 emx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS
)
3923 struct emx_softc
*sc
= (void *)arg1
;
3924 struct ifnet
*ifp
= &sc
->arpcom
.ac_if
;
3927 off
= sc
->tx_npoll_off
;
3928 error
= sysctl_handle_int(oidp
, &off
, 0, req
);
3929 if (error
|| req
->newptr
== NULL
)
3934 ifnet_serialize_all(ifp
);
3935 if (off
>= ncpus2
|| off
% sc
->tx_ring_cnt
!= 0) {
3939 sc
->tx_npoll_off
= off
;
3941 ifnet_deserialize_all(ifp
);
3946 #endif /* IFPOLL_ENABLE */
3949 emx_dma_alloc(struct emx_softc
*sc
)
3954 * Create top level busdma tag
3956 error
= bus_dma_tag_create(NULL
, 1, 0,
3957 BUS_SPACE_MAXADDR
, BUS_SPACE_MAXADDR
,
3959 BUS_SPACE_MAXSIZE_32BIT
, 0, BUS_SPACE_MAXSIZE_32BIT
,
3960 0, &sc
->parent_dtag
);
3962 device_printf(sc
->dev
, "could not create top level DMA tag\n");
3967 * Allocate transmit descriptors ring and buffers
3969 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3970 error
= emx_create_tx_ring(&sc
->tx_data
[i
]);
3972 device_printf(sc
->dev
,
3973 "Could not setup transmit structures\n");
3979 * Allocate receive descriptors ring and buffers
3981 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
3982 error
= emx_create_rx_ring(&sc
->rx_data
[i
]);
3984 device_printf(sc
->dev
,
3985 "Could not setup receive structures\n");
3993 emx_dma_free(struct emx_softc
*sc
)
3997 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
3998 emx_destroy_tx_ring(&sc
->tx_data
[i
],
3999 sc
->tx_data
[i
].num_tx_desc
);
4002 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
4003 emx_destroy_rx_ring(&sc
->rx_data
[i
],
4004 sc
->rx_data
[i
].num_rx_desc
);
4007 /* Free top level busdma tag */
4008 if (sc
->parent_dtag
!= NULL
)
4009 bus_dma_tag_destroy(sc
->parent_dtag
);
4013 emx_serialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
4015 struct emx_softc
*sc
= ifp
->if_softc
;
4017 ifnet_serialize_array_enter(sc
->serializes
, EMX_NSERIALIZE
, slz
);
4021 emx_deserialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
4023 struct emx_softc
*sc
= ifp
->if_softc
;
4025 ifnet_serialize_array_exit(sc
->serializes
, EMX_NSERIALIZE
, slz
);
4029 emx_tryserialize(struct ifnet
*ifp
, enum ifnet_serialize slz
)
4031 struct emx_softc
*sc
= ifp
->if_softc
;
4033 return ifnet_serialize_array_try(sc
->serializes
, EMX_NSERIALIZE
, slz
);
4037 emx_serialize_skipmain(struct emx_softc
*sc
)
4039 lwkt_serialize_array_enter(sc
->serializes
, EMX_NSERIALIZE
, 1);
4043 emx_deserialize_skipmain(struct emx_softc
*sc
)
4045 lwkt_serialize_array_exit(sc
->serializes
, EMX_NSERIALIZE
, 1);
4051 emx_serialize_assert(struct ifnet
*ifp
, enum ifnet_serialize slz
,
4052 boolean_t serialized
)
4054 struct emx_softc
*sc
= ifp
->if_softc
;
4056 ifnet_serialize_array_assert(sc
->serializes
, EMX_NSERIALIZE
,
4060 #endif /* INVARIANTS */
4062 #ifdef IFPOLL_ENABLE
4065 emx_npoll_status(struct ifnet
*ifp
)
4067 struct emx_softc
*sc
= ifp
->if_softc
;
4070 ASSERT_SERIALIZED(&sc
->main_serialize
);
4072 reg_icr
= E1000_READ_REG(&sc
->hw
, E1000_ICR
);
4073 if (reg_icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
4074 callout_stop(&sc
->timer
);
4075 sc
->hw
.mac
.get_link_status
= 1;
4076 emx_update_link_status(sc
);
4077 callout_reset(&sc
->timer
, hz
, emx_timer
, sc
);
4082 emx_npoll_tx(struct ifnet
*ifp
, void *arg
, int cycle __unused
)
4084 struct emx_txdata
*tdata
= arg
;
4086 ASSERT_SERIALIZED(&tdata
->tx_serialize
);
4089 if (!ifsq_is_empty(tdata
->ifsq
))
4090 ifsq_devstart(tdata
->ifsq
);
4094 emx_npoll_rx(struct ifnet
*ifp __unused
, void *arg
, int cycle
)
4096 struct emx_rxdata
*rdata
= arg
;
4098 ASSERT_SERIALIZED(&rdata
->rx_serialize
);
4100 emx_rxeof(rdata
, cycle
);
4104 emx_npoll(struct ifnet
*ifp
, struct ifpoll_info
*info
)
4106 struct emx_softc
*sc
= ifp
->if_softc
;
4109 ASSERT_IFNET_SERIALIZED_ALL(ifp
);
4114 info
->ifpi_status
.status_func
= emx_npoll_status
;
4115 info
->ifpi_status
.serializer
= &sc
->main_serialize
;
4117 txr_cnt
= emx_get_txring_inuse(sc
, TRUE
);
4118 off
= sc
->tx_npoll_off
;
4119 for (i
= 0; i
< txr_cnt
; ++i
) {
4120 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
4123 KKASSERT(idx
< ncpus2
);
4124 info
->ifpi_tx
[idx
].poll_func
= emx_npoll_tx
;
4125 info
->ifpi_tx
[idx
].arg
= tdata
;
4126 info
->ifpi_tx
[idx
].serializer
= &tdata
->tx_serialize
;
4127 ifsq_set_cpuid(tdata
->ifsq
, idx
);
4130 off
= sc
->rx_npoll_off
;
4131 for (i
= 0; i
< sc
->rx_ring_cnt
; ++i
) {
4132 struct emx_rxdata
*rdata
= &sc
->rx_data
[i
];
4135 KKASSERT(idx
< ncpus2
);
4136 info
->ifpi_rx
[idx
].poll_func
= emx_npoll_rx
;
4137 info
->ifpi_rx
[idx
].arg
= rdata
;
4138 info
->ifpi_rx
[idx
].serializer
= &rdata
->rx_serialize
;
4141 if (ifp
->if_flags
& IFF_RUNNING
) {
4142 if (txr_cnt
== sc
->tx_ring_inuse
)
4143 emx_disable_intr(sc
);
4148 for (i
= 0; i
< sc
->tx_ring_cnt
; ++i
) {
4149 struct emx_txdata
*tdata
= &sc
->tx_data
[i
];
4151 ifsq_set_cpuid(tdata
->ifsq
,
4152 rman_get_cpuid(sc
->intr_res
));
4155 if (ifp
->if_flags
& IFF_RUNNING
) {
4156 txr_cnt
= emx_get_txring_inuse(sc
, FALSE
);
4157 if (txr_cnt
== sc
->tx_ring_inuse
)
4158 emx_enable_intr(sc
);
4165 #endif /* IFPOLL_ENABLE */
4168 emx_set_itr(struct emx_softc
*sc
, uint32_t itr
)
4170 E1000_WRITE_REG(&sc
->hw
, E1000_ITR
, itr
);
4171 if (sc
->hw
.mac
.type
== e1000_82574
) {
4175 * When using MSIX interrupts we need to
4176 * throttle using the EITR register
4178 for (i
= 0; i
< 4; ++i
)
4179 E1000_WRITE_REG(&sc
->hw
, E1000_EITR_82574(i
), itr
);
4184 * Disable the L0s, 82574L Errata #20
4187 emx_disable_aspm(struct emx_softc
*sc
)
4189 uint16_t link_cap
, link_ctrl
, disable
;
4190 uint8_t pcie_ptr
, reg
;
4191 device_t dev
= sc
->dev
;
4193 switch (sc
->hw
.mac
.type
) {
4198 * 82573 specification update
4199 * errata #8 disable L0s
4200 * errata #41 disable L1
4202 * 82571/82572 specification update
4203 # errata #13 disable L1
4204 * errata #68 disable L0s
4206 disable
= PCIEM_LNKCTL_ASPM_L0S
| PCIEM_LNKCTL_ASPM_L1
;
4211 * 82574 specification update errata #20
4213 * There is no need to disable L1
4215 disable
= PCIEM_LNKCTL_ASPM_L0S
;
4222 pcie_ptr
= pci_get_pciecap_ptr(dev
);
4226 link_cap
= pci_read_config(dev
, pcie_ptr
+ PCIER_LINKCAP
, 2);
4227 if ((link_cap
& PCIEM_LNKCAP_ASPM_MASK
) == 0)
4231 if_printf(&sc
->arpcom
.ac_if
, "disable ASPM %#02x\n", disable
);
4233 reg
= pcie_ptr
+ PCIER_LINKCTRL
;
4234 link_ctrl
= pci_read_config(dev
, reg
, 2);
4235 link_ctrl
&= ~disable
;
4236 pci_write_config(dev
, reg
, link_ctrl
, 2);
4240 emx_tso_pullup(struct emx_txdata
*tdata
, struct mbuf
**mp
)
4242 int iphlen
, hoff
, thoff
, ex
= 0;
4247 KASSERT(M_WRITABLE(m
), ("TSO mbuf not writable"));
4249 iphlen
= m
->m_pkthdr
.csum_iphlen
;
4250 thoff
= m
->m_pkthdr
.csum_thlen
;
4251 hoff
= m
->m_pkthdr
.csum_lhlen
;
4253 KASSERT(iphlen
> 0, ("invalid ip hlen"));
4254 KASSERT(thoff
> 0, ("invalid tcp hlen"));
4255 KASSERT(hoff
> 0, ("invalid ether hlen"));
4257 if (tdata
->tx_flags
& EMX_TXFLAG_TSO_PULLEX
)
4260 if (m
->m_len
< hoff
+ iphlen
+ thoff
+ ex
) {
4261 m
= m_pullup(m
, hoff
+ iphlen
+ thoff
+ ex
);
4268 ip
= mtodoff(m
, struct ip
*, hoff
);
4275 emx_tso_setup(struct emx_txdata
*tdata
, struct mbuf
*mp
,
4276 uint32_t *txd_upper
, uint32_t *txd_lower
)
4278 struct e1000_context_desc
*TXD
;
4279 int hoff
, iphlen
, thoff
, hlen
;
4280 int mss
, pktlen
, curr_txd
;
4282 #ifdef EMX_TSO_DEBUG
4283 tdata
->tso_segments
++;
4286 iphlen
= mp
->m_pkthdr
.csum_iphlen
;
4287 thoff
= mp
->m_pkthdr
.csum_thlen
;
4288 hoff
= mp
->m_pkthdr
.csum_lhlen
;
4289 mss
= mp
->m_pkthdr
.tso_segsz
;
4290 pktlen
= mp
->m_pkthdr
.len
;
4292 if ((tdata
->tx_flags
& EMX_TXFLAG_FORCECTX
) == 0 &&
4293 tdata
->csum_flags
== CSUM_TSO
&&
4294 tdata
->csum_iphlen
== iphlen
&&
4295 tdata
->csum_lhlen
== hoff
&&
4296 tdata
->csum_thlen
== thoff
&&
4297 tdata
->csum_mss
== mss
&&
4298 tdata
->csum_pktlen
== pktlen
) {
4299 *txd_upper
= tdata
->csum_txd_upper
;
4300 *txd_lower
= tdata
->csum_txd_lower
;
4301 #ifdef EMX_TSO_DEBUG
4302 tdata
->tso_ctx_reused
++;
4306 hlen
= hoff
+ iphlen
+ thoff
;
4309 * Setup a new TSO context.
4312 curr_txd
= tdata
->next_avail_tx_desc
;
4313 TXD
= (struct e1000_context_desc
*)&tdata
->tx_desc_base
[curr_txd
];
4315 *txd_lower
= E1000_TXD_CMD_DEXT
| /* Extended descr type */
4316 E1000_TXD_DTYP_D
| /* Data descr type */
4317 E1000_TXD_CMD_TSE
; /* Do TSE on this packet */
4319 /* IP and/or TCP header checksum calculation and insertion. */
4320 *txd_upper
= (E1000_TXD_POPTS_IXSM
| E1000_TXD_POPTS_TXSM
) << 8;
4323 * Start offset for header checksum calculation.
4324 * End offset for header checksum calculation.
4325 * Offset of place put the checksum.
4327 TXD
->lower_setup
.ip_fields
.ipcss
= hoff
;
4328 TXD
->lower_setup
.ip_fields
.ipcse
= htole16(hoff
+ iphlen
- 1);
4329 TXD
->lower_setup
.ip_fields
.ipcso
= hoff
+ offsetof(struct ip
, ip_sum
);
4332 * Start offset for payload checksum calculation.
4333 * End offset for payload checksum calculation.
4334 * Offset of place to put the checksum.
4336 TXD
->upper_setup
.tcp_fields
.tucss
= hoff
+ iphlen
;
4337 TXD
->upper_setup
.tcp_fields
.tucse
= 0;
4338 TXD
->upper_setup
.tcp_fields
.tucso
=
4339 hoff
+ iphlen
+ offsetof(struct tcphdr
, th_sum
);
4342 * Payload size per packet w/o any headers.
4343 * Length of all headers up to payload.
4345 TXD
->tcp_seg_setup
.fields
.mss
= htole16(mss
);
4346 TXD
->tcp_seg_setup
.fields
.hdr_len
= hlen
;
4347 TXD
->cmd_and_length
= htole32(E1000_TXD_CMD_IFCS
|
4348 E1000_TXD_CMD_DEXT
| /* Extended descr */
4349 E1000_TXD_CMD_TSE
| /* TSE context */
4350 E1000_TXD_CMD_IP
| /* Do IP csum */
4351 E1000_TXD_CMD_TCP
| /* Do TCP checksum */
4352 (pktlen
- hlen
)); /* Total len */
4354 /* Save the information for this TSO context */
4355 tdata
->csum_flags
= CSUM_TSO
;
4356 tdata
->csum_lhlen
= hoff
;
4357 tdata
->csum_iphlen
= iphlen
;
4358 tdata
->csum_thlen
= thoff
;
4359 tdata
->csum_mss
= mss
;
4360 tdata
->csum_pktlen
= pktlen
;
4361 tdata
->csum_txd_upper
= *txd_upper
;
4362 tdata
->csum_txd_lower
= *txd_lower
;
4364 if (++curr_txd
== tdata
->num_tx_desc
)
4367 KKASSERT(tdata
->num_tx_desc_avail
> 0);
4368 tdata
->num_tx_desc_avail
--;
4370 tdata
->next_avail_tx_desc
= curr_txd
;
4375 emx_get_txring_inuse(const struct emx_softc
*sc
, boolean_t polling
)
4378 return sc
->tx_ring_cnt
;