From 74b989c3047109458504db6bc5b1caec29c43ff3 Mon Sep 17 00:00:00 2001 From: Robert Mustacchi Date: Thu, 16 Mar 2017 18:15:23 +0000 Subject: [PATCH] 8579 ixgbe infinite loops in rar and vmdq clearing Reviewed by: Jerry Jelinek Reviewed by: Michael Speer Approved by: Richard Lowe --- usr/src/uts/common/io/ixgbe/core/README.illumos | 14 +++++++++++ usr/src/uts/common/io/ixgbe/core/ixgbe_common.c | 10 ++++---- usr/src/uts/common/io/ixgbe/core/ixgbe_common.h | 1 - usr/src/uts/common/io/ixgbe/ixgbe_osdep.c | 31 +++++++++++++++++++++++-- usr/src/uts/common/io/ixgbe/ixgbe_osdep.h | 4 +++- usr/src/uts/common/io/ixgbe/ixgbe_stat.c | 5 +++- usr/src/uts/common/io/ixgbe/ixgbe_sw.h | 3 ++- 7 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 usr/src/uts/common/io/ixgbe/core/README.illumos diff --git a/usr/src/uts/common/io/ixgbe/core/README.illumos b/usr/src/uts/common/io/ixgbe/core/README.illumos new file mode 100644 index 0000000000..a09765894d --- /dev/null +++ b/usr/src/uts/common/io/ixgbe/core/README.illumos @@ -0,0 +1,14 @@ +This directory contains the 'common code' to the ixgbe driver. This code +comes from FreeBSD. + +The following are a subset of the current modifications that are being +made to this common code to facilitate illumos specific features: + +o IXGBE_REMOVED + +The IXGBE_REMOVED macro normally taks a pointer to the hardware address. +However, that does not allow us to go through and actually do a useful +check. Instead we modify all users of this macro to pass the struct +ixgbe_hw of the ixgbe_t. Importantly, we also remove the current +definition of IXGBE_REMOVED in ixbe_common.h and instead use the +definition in ixgbe_osdep.h. diff --git a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c index 79d460f2a4..eb98709d14 100644 --- a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c +++ b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.c @@ -3130,7 +3130,7 @@ s32 ixgbe_disable_pcie_master(struct ixgbe_hw *hw) /* Exit if master requests are blocked */ if (!(IXGBE_READ_REG(hw, IXGBE_STATUS) & IXGBE_STATUS_GIO) || - IXGBE_REMOVED(hw->hw_addr)) + IXGBE_REMOVED(hw)) goto out; /* Poll for master request bit to clear */ @@ -3162,7 +3162,7 @@ s32 ixgbe_disable_pcie_master(struct ixgbe_hw *hw) for (i = 0; i < poll; i++) { usec_delay(100); value = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_DEVICE_STATUS); - if (IXGBE_REMOVED(hw->hw_addr)) + if (IXGBE_REMOVED(hw)) goto out; if (!(value & IXGBE_PCI_DEVICE_STATUS_TRANSACTION_PENDING)) goto out; @@ -3590,7 +3590,7 @@ u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_get_pcie_msix_count_generic"); msix_count = IXGBE_READ_PCIE_WORD(hw, pcie_offset); - if (IXGBE_REMOVED(hw->hw_addr)) + if (IXGBE_REMOVED(hw)) msix_count = 0; msix_count &= IXGBE_PCIE_MSIX_TBL_SZ_MASK; @@ -3695,7 +3695,7 @@ s32 ixgbe_clear_vmdq_generic(struct ixgbe_hw *hw, u32 rar, u32 vmdq) mpsar_lo = IXGBE_READ_REG(hw, IXGBE_MPSAR_LO(rar)); mpsar_hi = IXGBE_READ_REG(hw, IXGBE_MPSAR_HI(rar)); - if (IXGBE_REMOVED(hw->hw_addr)) + if (IXGBE_REMOVED(hw)) goto done; if (!mpsar_lo && !mpsar_hi) @@ -4631,7 +4631,7 @@ void ixgbe_clear_tx_pending(struct ixgbe_hw *hw) for (i = 0; i < poll; i++) { usec_delay(100); value = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_DEVICE_STATUS); - if (IXGBE_REMOVED(hw->hw_addr)) + if (IXGBE_REMOVED(hw)) goto out; if (!(value & IXGBE_PCI_DEVICE_STATUS_TRANSACTION_PENDING)) goto out; diff --git a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h index e685f5b32a..069fc88c96 100644 --- a/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h +++ b/usr/src/uts/common/io/ixgbe/core/ixgbe_common.h @@ -41,7 +41,6 @@ IXGBE_WRITE_REG(hw, reg, (u32) value); \ IXGBE_WRITE_REG(hw, reg + 4, (u32) (value >> 32)); \ } while (0) -#define IXGBE_REMOVED(a) (0) #if !defined(NO_READ_PBA_RAW) || !defined(NO_WRITE_PBA_RAW) struct ixgbe_pba { u16 word[2]; diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c index ae820a58cf..fcb695abfd 100644 --- a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c +++ b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.c @@ -24,10 +24,10 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (c) 2017, Joyent, Inc. */ -#include "ixgbe_osdep.h" -#include "ixgbe_api.h" +#include "ixgbe_sw.h" uint16_t ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) @@ -40,3 +40,30 @@ ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint32_t val) { pci_config_put16(OS_DEP(hw)->cfg_handle, reg, val); } + +/* + * This is our last line of defense against a hardware device that has decided + * to somehow disappear without our knowledge of it. To try and deal with this, + * we'll read the status register and see if it returns all 1s, indicating an + * invalid read. Note the status register is defined to have bits in all current + * revisions that are hardwired to zero. + */ +boolean_t +ixgbe_removed(struct ixgbe_hw *hw) +{ + uint32_t val; + + val = IXGBE_READ_REG(hw, IXGBE_STATUS); + if (val == PCI_EINVAL32) { + ixgbe_t *ixgbe = OS_DEP(hw)->ixgbe; + + ixgbe_error(ixgbe, "failed to read status register: device " + "may be gone"); + if (ixgbe->ixgbe_ks != NULL) { + ixgbe_stat_t *s = ixgbe->ixgbe_ks->ks_data; + s->dev_gone.value.ui64++; + } + return (B_TRUE); + } + return (B_FALSE); +} diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h index 9526364b2f..0dc911bfd0 100644 --- a/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h +++ b/usr/src/uts/common/io/ixgbe/ixgbe_osdep.h @@ -27,7 +27,7 @@ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. */ /* - * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * Copyright (c) 2017, Joyent, Inc. * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. */ @@ -65,6 +65,7 @@ extern "C" { struct ixgbe_hw; uint16_t ixgbe_read_pci_cfg(struct ixgbe_hw *, uint32_t); void ixgbe_write_pci_cfg(struct ixgbe_hw *, uint32_t, uint32_t); +boolean_t ixgbe_removed(struct ixgbe_hw *); #define usec_delay(x) drv_usecwait(x) #define msec_delay(x) drv_usecwait(x * 1000) @@ -128,6 +129,7 @@ void ixgbe_write_pci_cfg(struct ixgbe_hw *, uint32_t, uint32_t); #define UNREFERENCED_4PARAMETER(_p, _q, _r, _s) _NOTE(ARGUNUSED(_p, _q,_r, _s)) +#define IXGBE_REMOVED(hw) ixgbe_removed(hw) typedef int8_t s8; typedef int16_t s16; diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_stat.c b/usr/src/uts/common/io/ixgbe/ixgbe_stat.c index 96b5aeb646..b5892b3c45 100644 --- a/usr/src/uts/common/io/ixgbe/ixgbe_stat.c +++ b/usr/src/uts/common/io/ixgbe/ixgbe_stat.c @@ -27,7 +27,7 @@ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012 Nexenta Systems, Inc. All rights reserved. * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. - * Copyright 2017 Joyent, Inc. + * Copyright (c) 2017, Joyent, Inc. */ #include "ixgbe_sw.h" @@ -466,6 +466,9 @@ ixgbe_init_stats(ixgbe_t *ixgbe) KSTAT_DATA_UINT64); kstat_named_init(&ixgbe_ks->lroc, "lro_pkt_count", KSTAT_DATA_UINT64); + + kstat_named_init(&ixgbe_ks->dev_gone, "device_gone", + KSTAT_DATA_UINT64); /* * Function to provide kernel stat update on demand */ diff --git a/usr/src/uts/common/io/ixgbe/ixgbe_sw.h b/usr/src/uts/common/io/ixgbe/ixgbe_sw.h index 986061bfad..6cd03d571d 100644 --- a/usr/src/uts/common/io/ixgbe/ixgbe_sw.h +++ b/usr/src/uts/common/io/ixgbe/ixgbe_sw.h @@ -27,7 +27,7 @@ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013 Saso Kiselkov. All rights reserved. * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. - * Copyright 2017 Joyent, Inc. + * Copyright (c) 2017, Joyent, Inc. */ #ifndef _IXGBE_SW_H @@ -819,6 +819,7 @@ typedef struct ixgbe_stat { kstat_named_t mptc; /* Multicast Packets Xmited Count */ kstat_named_t bptc; /* Broadcast Packets Xmited Count */ kstat_named_t lroc; /* LRO Packets Received Count */ + kstat_named_t dev_gone; /* Number of device gone events encountered */ } ixgbe_stat_t; /* -- 2.11.4.GIT