kernel/drivers/net/ixgbe/ixgbe_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
  24  */
  25
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Copyright 2017, Joyent, Inc.
  29  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  30  * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
  31  * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
  32  * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
  33  */
  34
  35 #include "ixgbe_sw.h"
  36
  37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
  38
  39 /*
  40  * Local function protoypes
  41  */
  42 static int ixgbe_register_mac(ixgbe_t *);
  43 static int ixgbe_identify_hardware(ixgbe_t *);
  44 static int ixgbe_regs_map(ixgbe_t *);
  45 static void ixgbe_init_properties(ixgbe_t *);
  46 static int ixgbe_init_driver_settings(ixgbe_t *);
  47 static void ixgbe_init_locks(ixgbe_t *);
  48 static void ixgbe_destroy_locks(ixgbe_t *);
  49 static int ixgbe_init(ixgbe_t *);
  50 static int ixgbe_chip_start(ixgbe_t *);
  51 static void ixgbe_chip_stop(ixgbe_t *);
  52 static int ixgbe_reset(ixgbe_t *);
  53 static void ixgbe_tx_clean(ixgbe_t *);
  54 static bool ixgbe_tx_drain(ixgbe_t *);
  55 static bool ixgbe_rx_drain(ixgbe_t *);
  56 static int ixgbe_alloc_rings(ixgbe_t *);
  57 static void ixgbe_free_rings(ixgbe_t *);
  58 static int ixgbe_alloc_rx_data(ixgbe_t *);
  59 static void ixgbe_free_rx_data(ixgbe_t *);
  60 static void ixgbe_setup_rings(ixgbe_t *);
  61 static void ixgbe_setup_rx(ixgbe_t *);
  62 static void ixgbe_setup_tx(ixgbe_t *);
  63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
  64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
  65 static void ixgbe_setup_rss(ixgbe_t *);
  66 static void ixgbe_setup_vmdq(ixgbe_t *);
  67 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
  68 static void ixgbe_setup_rss_table(ixgbe_t *);
  69 static void ixgbe_init_unicst(ixgbe_t *);
  70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
  71 static void ixgbe_setup_multicst(ixgbe_t *);
  72 static void ixgbe_get_hw_state(ixgbe_t *);
  73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
  74 static void ixgbe_get_conf(ixgbe_t *);
  75 static void ixgbe_init_params(ixgbe_t *);
  76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
  77 static void ixgbe_driver_link_check(ixgbe_t *);
  78 static void ixgbe_sfp_check(void *);
  79 static void ixgbe_overtemp_check(void *);
  80 static void ixgbe_phy_check(void *);
  81 static void ixgbe_link_timer(void *);
  82 static void ixgbe_local_timer(void *);
  83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
  84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
  85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
  86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
  87 static bool is_valid_mac_addr(uint8_t *);
  88 static bool ixgbe_stall_check(ixgbe_t *);
  89 static bool ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
  90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
  91 static bool ixgbe_find_mac_address(ixgbe_t *);
  92 static int ixgbe_alloc_intrs(ixgbe_t *);
  93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
  94 static int ixgbe_add_intr_handlers(ixgbe_t *);
  95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
  96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
  97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
  98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
  99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
 100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
 101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
 102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
 103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
 104 static void ixgbe_rem_intrs(ixgbe_t *);
 105 static int ixgbe_enable_intrs(ixgbe_t *);
 106 static int ixgbe_disable_intrs(ixgbe_t *);
 107 static uint_t ixgbe_intr_legacy(void *, void *);
 108 static uint_t ixgbe_intr_msi(void *, void *);
 109 static uint_t ixgbe_intr_msix(void *, void *);
 110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
 111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
 112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
 113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
 114 static int ixgbe_addmac(void *, const uint8_t *);
 115 static int ixgbe_remmac(void *, const uint8_t *);
 116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
 117
 118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
 119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
 120 static int ixgbe_resume(dev_info_t *);
 121 static int ixgbe_suspend(dev_info_t *);
 122 static int ixgbe_quiesce(dev_info_t *);
 123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
 124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
 125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
 126 static int ixgbe_intr_cb_register(ixgbe_t *);
 127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
 128
 129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
 130     const void *impl_data);
 131 static void ixgbe_fm_init(ixgbe_t *);
 132 static void ixgbe_fm_fini(ixgbe_t *);
 133
 134 char *ixgbe_priv_props[] = {
 135         "_tx_copy_thresh",
 136         "_tx_recycle_thresh",
 137         "_tx_overload_thresh",
 138         "_tx_resched_thresh",
 139         "_rx_copy_thresh",
 140         "_rx_limit_per_intr",
 141         "_intr_throttling",
 142         "_adv_pause_cap",
 143         "_adv_asym_pause_cap",
 144         NULL
 145 };
 146
 147 #define IXGBE_MAX_PRIV_PROPS \
 148         (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
 149
 150 static struct cb_ops ixgbe_cb_ops = {
 151         nulldev,                /* cb_open */
 152         nulldev,                /* cb_close */
 153         nodev,                  /* cb_strategy */
 154         nodev,                  /* cb_print */
 155         nodev,                  /* cb_dump */
 156         nodev,                  /* cb_read */
 157         nodev,                  /* cb_write */
 158         nodev,                  /* cb_ioctl */
 159         nodev,                  /* cb_devmap */
 160         nodev,                  /* cb_mmap */
 161         nodev,                  /* cb_segmap */
 162         nochpoll,               /* cb_chpoll */
 163         ddi_prop_op,            /* cb_prop_op */
 164         NULL,                   /* cb_stream */
 165         D_MP | D_HOTPLUG,       /* cb_flag */
 166         CB_REV,                 /* cb_rev */
 167         nodev,                  /* cb_aread */
 168         nodev                   /* cb_awrite */
 169 };
 170
 171 static struct dev_ops ixgbe_dev_ops = {
 172         DEVO_REV,               /* devo_rev */
 173         0,                      /* devo_refcnt */
 174         NULL,                   /* devo_getinfo */
 175         nulldev,                /* devo_identify */
 176         nulldev,                /* devo_probe */
 177         ixgbe_attach,           /* devo_attach */
 178         ixgbe_detach,           /* devo_detach */
 179         nodev,                  /* devo_reset */
 180         &ixgbe_cb_ops,          /* devo_cb_ops */
 181         NULL,                   /* devo_bus_ops */
 182         ddi_power,              /* devo_power */
 183         ixgbe_quiesce,          /* devo_quiesce */
 184 };
 185
 186 static struct modldrv ixgbe_modldrv = {
 187         &mod_driverops,         /* Type of module.  This one is a driver */
 188         ixgbe_ident,            /* Discription string */
 189         &ixgbe_dev_ops          /* driver ops */
 190 };
 191
 192 static struct modlinkage ixgbe_modlinkage = {
 193         MODREV_1, &ixgbe_modldrv, NULL
 194 };
 195
 196 /*
 197  * Access attributes for register mapping
 198  */
 199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
 200         DDI_DEVICE_ATTR_V1,
 201         DDI_STRUCTURE_LE_ACC,
 202         DDI_STRICTORDER_ACC,
 203         DDI_FLAGERR_ACC
 204 };
 205
 206 /*
 207  * Loopback property
 208  */
 209 static lb_property_t lb_normal = {
 210         normal, "normal", IXGBE_LB_NONE
 211 };
 212
 213 static lb_property_t lb_mac = {
 214         internal, "MAC", IXGBE_LB_INTERNAL_MAC
 215 };
 216
 217 static lb_property_t lb_external = {
 218         external, "External", IXGBE_LB_EXTERNAL
 219 };
 220
 221 #define IXGBE_M_CALLBACK_FLAGS \
 222         (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
 223
 224 static mac_callbacks_t ixgbe_m_callbacks = {
 225         IXGBE_M_CALLBACK_FLAGS,
 226         ixgbe_m_stat,
 227         ixgbe_m_start,
 228         ixgbe_m_stop,
 229         ixgbe_m_promisc,
 230         ixgbe_m_multicst,
 231         NULL,
 232         NULL,
 233         NULL,
 234         ixgbe_m_ioctl,
 235         ixgbe_m_getcapab,
 236         NULL,
 237         NULL,
 238         ixgbe_m_setprop,
 239         ixgbe_m_getprop,
 240         ixgbe_m_propinfo
 241 };
 242
 243 /*
 244  * Initialize capabilities of each supported adapter type
 245  */
 246 static adapter_info_t ixgbe_82598eb_cap = {
 247         64,             /* maximum number of rx queues */
 248         1,              /* minimum number of rx queues */
 249         64,             /* default number of rx queues */
 250         16,             /* maximum number of rx groups */
 251         1,              /* minimum number of rx groups */
 252         1,              /* default number of rx groups */
 253         32,             /* maximum number of tx queues */
 254         1,              /* minimum number of tx queues */
 255         8,              /* default number of tx queues */
 256         16366,          /* maximum MTU size */
 257         0xFFFF,         /* maximum interrupt throttle rate */
 258         0,              /* minimum interrupt throttle rate */
 259         200,            /* default interrupt throttle rate */
 260         18,             /* maximum total msix vectors */
 261         16,             /* maximum number of ring vectors */
 262         2,              /* maximum number of other vectors */
 263         IXGBE_EICR_LSC, /* "other" interrupt types handled */
 264         0,              /* "other" interrupt types enable mask */
 265         (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
 266         | IXGBE_FLAG_RSS_CAPABLE
 267         | IXGBE_FLAG_VMDQ_CAPABLE)
 268 };
 269
 270 static adapter_info_t ixgbe_82599eb_cap = {
 271         128,            /* maximum number of rx queues */
 272         1,              /* minimum number of rx queues */
 273         128,            /* default number of rx queues */
 274         64,             /* maximum number of rx groups */
 275         1,              /* minimum number of rx groups */
 276         1,              /* default number of rx groups */
 277         128,            /* maximum number of tx queues */
 278         1,              /* minimum number of tx queues */
 279         8,              /* default number of tx queues */
 280         15500,          /* maximum MTU size */
 281         0xFF8,          /* maximum interrupt throttle rate */
 282         0,              /* minimum interrupt throttle rate */
 283         200,            /* default interrupt throttle rate */
 284         64,             /* maximum total msix vectors */
 285         16,             /* maximum number of ring vectors */
 286         2,              /* maximum number of other vectors */
 287         (IXGBE_EICR_LSC
 288         | IXGBE_EICR_GPI_SDP1
 289         | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
 290
 291         (IXGBE_SDP1_GPIEN
 292         | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
 293
 294         (IXGBE_FLAG_DCA_CAPABLE
 295         | IXGBE_FLAG_RSS_CAPABLE
 296         | IXGBE_FLAG_VMDQ_CAPABLE
 297         | IXGBE_FLAG_RSC_CAPABLE
 298         | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
 299 };
 300
 301 static adapter_info_t ixgbe_X540_cap = {
 302         128,            /* maximum number of rx queues */
 303         1,              /* minimum number of rx queues */
 304         128,            /* default number of rx queues */
 305         64,             /* maximum number of rx groups */
 306         1,              /* minimum number of rx groups */
 307         1,              /* default number of rx groups */
 308         128,            /* maximum number of tx queues */
 309         1,              /* minimum number of tx queues */
 310         8,              /* default number of tx queues */
 311         15500,          /* maximum MTU size */
 312         0xFF8,          /* maximum interrupt throttle rate */
 313         0,              /* minimum interrupt throttle rate */
 314         200,            /* default interrupt throttle rate */
 315         64,             /* maximum total msix vectors */
 316         16,             /* maximum number of ring vectors */
 317         2,              /* maximum number of other vectors */
 318         (IXGBE_EICR_LSC
 319         | IXGBE_EICR_GPI_SDP1_X540
 320         | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
 321
 322         (IXGBE_SDP1_GPIEN_X540
 323         | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
 324
 325         (IXGBE_FLAG_DCA_CAPABLE
 326         | IXGBE_FLAG_RSS_CAPABLE
 327         | IXGBE_FLAG_VMDQ_CAPABLE
 328         | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
 329 };
 330
 331 static adapter_info_t ixgbe_X550_cap = {
 332         128,            /* maximum number of rx queues */
 333         1,              /* minimum number of rx queues */
 334         128,            /* default number of rx queues */
 335         64,             /* maximum number of rx groups */
 336         1,              /* minimum number of rx groups */
 337         1,              /* default number of rx groups */
 338         128,            /* maximum number of tx queues */
 339         1,              /* minimum number of tx queues */
 340         8,              /* default number of tx queues */
 341         15500,          /* maximum MTU size */
 342         0xFF8,          /* maximum interrupt throttle rate */
 343         0,              /* minimum interrupt throttle rate */
 344         0x200,          /* default interrupt throttle rate */
 345         64,             /* maximum total msix vectors */
 346         16,             /* maximum number of ring vectors */
 347         2,              /* maximum number of other vectors */
 348         IXGBE_EICR_LSC, /* "other" interrupt types handled */
 349         0,              /* "other" interrupt types enable mask */
 350         (IXGBE_FLAG_RSS_CAPABLE
 351         | IXGBE_FLAG_VMDQ_CAPABLE
 352         | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
 353 };
 354
 355 /*
 356  * Module Initialization Functions.
 357  */
 358
 359 int
 360 _init(void)
 361 {
 362         int status;
 363
 364         mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
 365
 366         status = mod_install(&ixgbe_modlinkage);
 367
 368         if (status != DDI_SUCCESS) {
 369                 mac_fini_ops(&ixgbe_dev_ops);
 370         }
 371
 372         return (status);
 373 }
 374
 375 int
 376 _fini(void)
 377 {
 378         int status;
 379
 380         status = mod_remove(&ixgbe_modlinkage);
 381
 382         if (status == DDI_SUCCESS) {
 383                 mac_fini_ops(&ixgbe_dev_ops);
 384         }
 385
 386         return (status);
 387 }
 388
 389 int
 390 _info(struct modinfo *modinfop)
 391 {
 392         int status;
 393
 394         status = mod_info(&ixgbe_modlinkage, modinfop);
 395
 396         return (status);
 397 }
 398
 399 /*
 400  * ixgbe_attach - Driver attach.
 401  *
 402  * This function is the device specific initialization entry
 403  * point. This entry point is required and must be written.
 404  * The DDI_ATTACH command must be provided in the attach entry
 405  * point. When attach() is called with cmd set to DDI_ATTACH,
 406  * all normal kernel services (such as kmem_alloc(9F)) are
 407  * available for use by the driver.
 408  *
 409  * The attach() function will be called once for each instance
 410  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
 411  * Until attach() succeeds, the only driver entry points which
 412  * may be called are open(9E) and getinfo(9E).
 413  */
 414 static int
 415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
 416 {
 417         ixgbe_t *ixgbe;
 418         struct ixgbe_osdep *osdep;
 419         struct ixgbe_hw *hw;
 420         int instance;
 421         char taskqname[32];
 422
 423         /*
 424          * Check the command and perform corresponding operations
 425          */
 426         switch (cmd) {
 427         default:
 428                 return (DDI_FAILURE);
 429
 430         case DDI_RESUME:
 431                 return (ixgbe_resume(devinfo));
 432
 433         case DDI_ATTACH:
 434                 break;
 435         }
 436
 437         /* Get the device instance */
 438         instance = ddi_get_instance(devinfo);
 439
 440         /* Allocate memory for the instance data structure */
 441         ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
 442
 443         ixgbe->dip = devinfo;
 444         ixgbe->instance = instance;
 445
 446         hw = &ixgbe->hw;
 447         osdep = &ixgbe->osdep;
 448         hw->back = osdep;
 449         osdep->ixgbe = ixgbe;
 450
 451         /* Attach the instance pointer to the dev_info data structure */
 452         ddi_set_driver_private(devinfo, ixgbe);
 453
 454         /*
 455          * Initialize for FMA support
 456          */
 457         ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
 458             0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
 459             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 460         ixgbe_fm_init(ixgbe);
 461         ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
 462
 463         /*
 464          * Map PCI config space registers
 465          */
 466         if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
 467                 ixgbe_error(ixgbe, "Failed to map PCI configurations");
 468                 goto attach_fail;
 469         }
 470         ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
 471
 472         /*
 473          * Identify the chipset family
 474          */
 475         if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
 476                 ixgbe_error(ixgbe, "Failed to identify hardware");
 477                 goto attach_fail;
 478         }
 479
 480         /*
 481          * Map device registers
 482          */
 483         if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
 484                 ixgbe_error(ixgbe, "Failed to map device registers");
 485                 goto attach_fail;
 486         }
 487         ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
 488
 489         /*
 490          * Initialize driver parameters
 491          */
 492         ixgbe_init_properties(ixgbe);
 493         ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
 494
 495         /*
 496          * Register interrupt callback
 497          */
 498         if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
 499                 ixgbe_error(ixgbe, "Failed to register interrupt callback");
 500                 goto attach_fail;
 501         }
 502
 503         /*
 504          * Allocate interrupts
 505          */
 506         if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
 507                 ixgbe_error(ixgbe, "Failed to allocate interrupts");
 508                 goto attach_fail;
 509         }
 510         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
 511
 512         /*
 513          * Allocate rx/tx rings based on the ring numbers.
 514          * The actual numbers of rx/tx rings are decided by the number of
 515          * allocated interrupt vectors, so we should allocate the rings after
 516          * interrupts are allocated.
 517          */
 518         if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
 519                 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
 520                 goto attach_fail;
 521         }
 522         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
 523
 524         /*
 525          * Map rings to interrupt vectors
 526          */
 527         if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
 528                 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
 529                 goto attach_fail;
 530         }
 531
 532         /*
 533          * Add interrupt handlers
 534          */
 535         if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
 536                 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
 537                 goto attach_fail;
 538         }
 539         ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
 540
 541         /*
 542          * Create a taskq for sfp-change
 543          */
 544         (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
 545         if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
 546             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 547                 ixgbe_error(ixgbe, "sfp_taskq create failed");
 548                 goto attach_fail;
 549         }
 550         ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
 551
 552         /*
 553          * Create a taskq for over-temp
 554          */
 555         (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
 556         if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
 557             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 558                 ixgbe_error(ixgbe, "overtemp_taskq create failed");
 559                 goto attach_fail;
 560         }
 561         ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
 562
 563         /*
 564          * Create a taskq for processing external PHY interrupts
 565          */
 566         (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
 567         if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
 568             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 569                 ixgbe_error(ixgbe, "phy_taskq create failed");
 570                 goto attach_fail;
 571         }
 572         ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
 573
 574         /*
 575          * Initialize driver parameters
 576          */
 577         if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
 578                 ixgbe_error(ixgbe, "Failed to initialize driver settings");
 579                 goto attach_fail;
 580         }
 581
 582         /*
 583          * Initialize mutexes for this device.
 584          * Do this before enabling the interrupt handler and
 585          * register the softint to avoid the condition where
 586          * interrupt handler can try using uninitialized mutex.
 587          */
 588         ixgbe_init_locks(ixgbe);
 589         ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
 590
 591         /*
 592          * Initialize chipset hardware
 593          */
 594         if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
 595                 ixgbe_error(ixgbe, "Failed to initialize adapter");
 596                 goto attach_fail;
 597         }
 598         ixgbe->link_check_complete = B_FALSE;
 599         ixgbe->link_check_hrtime = gethrtime() +
 600             (IXGBE_LINK_UP_TIME * 100000000ULL);
 601         ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
 602
 603         if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
 604                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
 605                 goto attach_fail;
 606         }
 607
 608         /*
 609          * Initialize adapter capabilities
 610          */
 611         ixgbe_init_params(ixgbe);
 612
 613         /*
 614          * Initialize statistics
 615          */
 616         if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
 617                 ixgbe_error(ixgbe, "Failed to initialize statistics");
 618                 goto attach_fail;
 619         }
 620         ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
 621
 622         /*
 623          * Register the driver to the MAC
 624          */
 625         if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
 626                 ixgbe_error(ixgbe, "Failed to register MAC");
 627                 goto attach_fail;
 628         }
 629         mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
 630         ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
 631
 632         ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
 633             IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
 634         if (ixgbe->periodic_id == 0) {
 635                 ixgbe_error(ixgbe, "Failed to add the link check timer");
 636                 goto attach_fail;
 637         }
 638         ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
 639
 640         /*
 641          * Now that mutex locks are initialized, and the chip is also
 642          * initialized, enable interrupts.
 643          */
 644         if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
 645                 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
 646                 goto attach_fail;
 647         }
 648         ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
 649
 650         ixgbe_log(ixgbe, "%s", ixgbe_ident);
 651         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
 652
 653         return (DDI_SUCCESS);
 654
 655 attach_fail:
 656         ixgbe_unconfigure(devinfo, ixgbe);
 657         return (DDI_FAILURE);
 658 }
 659
 660 /*
 661  * ixgbe_detach - Driver detach.
 662  *
 663  * The detach() function is the complement of the attach routine.
 664  * If cmd is set to DDI_DETACH, detach() is used to remove  the
 665  * state  associated  with  a  given  instance of a device node
 666  * prior to the removal of that instance from the system.
 667  *
 668  * The detach() function will be called once for each  instance
 669  * of the device for which there has been a successful attach()
 670  * once there are no longer  any  opens  on  the  device.
 671  *
 672  * Interrupts routine are disabled, All memory allocated by this
 673  * driver are freed.
 674  */
 675 static int
 676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
 677 {
 678         ixgbe_t *ixgbe;
 679
 680         /*
 681          * Check detach command
 682          */
 683         switch (cmd) {
 684         default:
 685                 return (DDI_FAILURE);
 686
 687         case DDI_SUSPEND:
 688                 return (ixgbe_suspend(devinfo));
 689
 690         case DDI_DETACH:
 691                 break;
 692         }
 693
 694         /*
 695          * Get the pointer to the driver private data structure
 696          */
 697         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
 698         if (ixgbe == NULL)
 699                 return (DDI_FAILURE);
 700
 701         /*
 702          * If the device is still running, it needs to be stopped first.
 703          * This check is necessary because under some specific circumstances,
 704          * the detach routine can be called without stopping the interface
 705          * first.
 706          */
 707         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
 708                 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
 709                 mutex_enter(&ixgbe->gen_lock);
 710                 ixgbe_stop(ixgbe, B_TRUE);
 711                 mutex_exit(&ixgbe->gen_lock);
 712                 /* Disable and stop the watchdog timer */
 713                 ixgbe_disable_watchdog_timer(ixgbe);
 714         }
 715
 716         /*
 717          * Check if there are still rx buffers held by the upper layer.
 718          * If so, fail the detach.
 719          */
 720         if (!ixgbe_rx_drain(ixgbe))
 721                 return (DDI_FAILURE);
 722
 723         /*
 724          * Do the remaining unconfigure routines
 725          */
 726         ixgbe_unconfigure(devinfo, ixgbe);
 727
 728         return (DDI_SUCCESS);
 729 }
 730
 731 /*
 732  * quiesce(9E) entry point.
 733  *
 734  * This function is called when the system is single-threaded at high
 735  * PIL with preemption disabled. Therefore, this function must not be
 736  * blocked.
 737  *
 738  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 739  * DDI_FAILURE indicates an error condition and should almost never happen.
 740  */
 741 static int
 742 ixgbe_quiesce(dev_info_t *devinfo)
 743 {
 744         ixgbe_t *ixgbe;
 745         struct ixgbe_hw *hw;
 746
 747         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
 748
 749         if (ixgbe == NULL)
 750                 return (DDI_FAILURE);
 751
 752         hw = &ixgbe->hw;
 753
 754         /*
 755          * Disable the adapter interrupts
 756          */
 757         ixgbe_disable_adapter_interrupts(ixgbe);
 758
 759         /*
 760          * Tell firmware driver is no longer in control
 761          */
 762         ixgbe_release_driver_control(hw);
 763
 764         /*
 765          * Reset the chipset
 766          */
 767         (void) ixgbe_reset_hw(hw);
 768
 769         /*
 770          * Reset PHY
 771          */
 772         (void) ixgbe_reset_phy(hw);
 773
 774         return (DDI_SUCCESS);
 775 }
 776
 777 static void
 778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
 779 {
 780         /*
 781          * Disable interrupt
 782          */
 783         if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
 784                 (void) ixgbe_disable_intrs(ixgbe);
 785         }
 786
 787         /*
 788          * remove the link check timer
 789          */
 790         if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
 791                 if (ixgbe->periodic_id != NULL) {
 792                         ddi_periodic_delete(ixgbe->periodic_id);
 793                         ixgbe->periodic_id = NULL;
 794                 }
 795         }
 796
 797         /*
 798          * Unregister MAC
 799          */
 800         if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
 801                 (void) mac_unregister(ixgbe->mac_hdl);
 802         }
 803
 804         /*
 805          * Free statistics
 806          */
 807         if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
 808                 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
 809         }
 810
 811         /*
 812          * Remove interrupt handlers
 813          */
 814         if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
 815                 ixgbe_rem_intr_handlers(ixgbe);
 816         }
 817
 818         /*
 819          * Remove taskq for sfp-status-change
 820          */
 821         if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
 822                 ddi_taskq_destroy(ixgbe->sfp_taskq);
 823         }
 824
 825         /*
 826          * Remove taskq for over-temp
 827          */
 828         if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
 829                 ddi_taskq_destroy(ixgbe->overtemp_taskq);
 830         }
 831
 832         /*
 833          * Remove taskq for external PHYs
 834          */
 835         if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
 836                 ddi_taskq_destroy(ixgbe->phy_taskq);
 837         }
 838
 839         /*
 840          * Remove interrupts
 841          */
 842         if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
 843                 ixgbe_rem_intrs(ixgbe);
 844         }
 845
 846         /*
 847          * Unregister interrupt callback handler
 848          */
 849         (void) ddi_cb_unregister(ixgbe->cb_hdl);
 850
 851         /*
 852          * Remove driver properties
 853          */
 854         if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
 855                 (void) ddi_prop_remove_all(devinfo);
 856         }
 857
 858         /*
 859          * Stop the chipset
 860          */
 861         if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
 862                 mutex_enter(&ixgbe->gen_lock);
 863                 ixgbe_chip_stop(ixgbe);
 864                 mutex_exit(&ixgbe->gen_lock);
 865         }
 866
 867         /*
 868          * Free register handle
 869          */
 870         if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
 871                 if (ixgbe->osdep.reg_handle != NULL)
 872                         ddi_regs_map_free(&ixgbe->osdep.reg_handle);
 873         }
 874
 875         /*
 876          * Free PCI config handle
 877          */
 878         if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
 879                 if (ixgbe->osdep.cfg_handle != NULL)
 880                         pci_config_teardown(&ixgbe->osdep.cfg_handle);
 881         }
 882
 883         /*
 884          * Free locks
 885          */
 886         if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
 887                 ixgbe_destroy_locks(ixgbe);
 888         }
 889
 890         /*
 891          * Free the rx/tx rings
 892          */
 893         if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
 894                 ixgbe_free_rings(ixgbe);
 895         }
 896
 897         /*
 898          * Unregister FMA capabilities
 899          */
 900         if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
 901                 ixgbe_fm_fini(ixgbe);
 902         }
 903
 904         /*
 905          * Free the driver data structure
 906          */
 907         kmem_free(ixgbe, sizeof (ixgbe_t));
 908
 909         ddi_set_driver_private(devinfo, NULL);
 910 }
 911
 912 /*
 913  * ixgbe_register_mac - Register the driver and its function pointers with
 914  * the GLD interface.
 915  */
 916 static int
 917 ixgbe_register_mac(ixgbe_t *ixgbe)
 918 {
 919         struct ixgbe_hw *hw = &ixgbe->hw;
 920         mac_register_t *mac;
 921         int status;
 922
 923         if ((mac = mac_alloc(MAC_VERSION)) == NULL)
 924                 return (IXGBE_FAILURE);
 925
 926         mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
 927         mac->m_driver = ixgbe;
 928         mac->m_dip = ixgbe->dip;
 929         mac->m_src_addr = hw->mac.addr;
 930         mac->m_callbacks = &ixgbe_m_callbacks;
 931         mac->m_min_sdu = 0;
 932         mac->m_max_sdu = ixgbe->default_mtu;
 933         mac->m_margin = VLAN_TAGSZ;
 934         mac->m_priv_props = ixgbe_priv_props;
 935         mac->m_v12n = MAC_VIRT_LEVEL1;
 936
 937         status = mac_register(mac, &ixgbe->mac_hdl);
 938
 939         mac_free(mac);
 940
 941         return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
 942 }
 943
 944 /*
 945  * ixgbe_identify_hardware - Identify the type of the chipset.
 946  */
 947 static int
 948 ixgbe_identify_hardware(ixgbe_t *ixgbe)
 949 {
 950         struct ixgbe_hw *hw = &ixgbe->hw;
 951         struct ixgbe_osdep *osdep = &ixgbe->osdep;
 952
 953         /*
 954          * Get the device id
 955          */
 956         hw->vendor_id =
 957             pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
 958         hw->device_id =
 959             pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
 960         hw->revision_id =
 961             pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
 962         hw->subsystem_device_id =
 963             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
 964         hw->subsystem_vendor_id =
 965             pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
 966
 967         /*
 968          * Set the mac type of the adapter based on the device id
 969          */
 970         if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
 971                 return (IXGBE_FAILURE);
 972         }
 973
 974         /*
 975          * Install adapter capabilities
 976          */
 977         switch (hw->mac.type) {
 978         case ixgbe_mac_82598EB:
 979                 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
 980                 ixgbe->capab = &ixgbe_82598eb_cap;
 981
 982                 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
 983                         ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
 984                         ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
 985                         ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
 986                 }
 987                 break;
 988
 989         case ixgbe_mac_82599EB:
 990                 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
 991                 ixgbe->capab = &ixgbe_82599eb_cap;
 992
 993                 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
 994                         ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
 995                         ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
 996                         ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
 997                 }
 998                 break;
 999
1000         case ixgbe_mac_X540:
1001                 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1002                 ixgbe->capab = &ixgbe_X540_cap;
1003                 /*
1004                  * For now, X540 is all set in its capab structure.
1005                  * As other X540 variants show up, things can change here.
1006                  */
1007                 break;
1008
1009         case ixgbe_mac_X550:
1010         case ixgbe_mac_X550EM_x:
1011                 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1012                 ixgbe->capab = &ixgbe_X550_cap;
1013
1014                 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
1015                         ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1016
1017                 /*
1018                  * Link detection on X552 SFP+ and X552/X557-AT
1019                  */
1020                 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1021                     hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1022                         ixgbe->capab->other_intr |=
1023                             IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1024                         ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1025                 }
1026                 break;
1027
1028         default:
1029                 IXGBE_DEBUGLOG_1(ixgbe,
1030                     "adapter not supported in ixgbe_identify_hardware(): %d\n",
1031                     hw->mac.type);
1032                 return (IXGBE_FAILURE);
1033         }
1034
1035         return (IXGBE_SUCCESS);
1036 }
1037
1038 /*
1039  * ixgbe_regs_map - Map the device registers.
1040  *
1041  */
1042 static int
1043 ixgbe_regs_map(ixgbe_t *ixgbe)
1044 {
1045         dev_info_t *devinfo = ixgbe->dip;
1046         struct ixgbe_hw *hw = &ixgbe->hw;
1047         struct ixgbe_osdep *osdep = &ixgbe->osdep;
1048         off_t mem_size;
1049
1050         /*
1051          * First get the size of device registers to be mapped.
1052          */
1053         if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1054             != DDI_SUCCESS) {
1055                 return (IXGBE_FAILURE);
1056         }
1057
1058         /*
1059          * Call ddi_regs_map_setup() to map registers
1060          */
1061         if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1062             (caddr_t *)&hw->hw_addr, 0,
1063             mem_size, &ixgbe_regs_acc_attr,
1064             &osdep->reg_handle)) != DDI_SUCCESS) {
1065                 return (IXGBE_FAILURE);
1066         }
1067
1068         return (IXGBE_SUCCESS);
1069 }
1070
1071 /*
1072  * ixgbe_init_properties - Initialize driver properties.
1073  */
1074 static void
1075 ixgbe_init_properties(ixgbe_t *ixgbe)
1076 {
1077         /*
1078          * Get conf file properties, including link settings
1079          * jumbo frames, ring number, descriptor number, etc.
1080          */
1081         ixgbe_get_conf(ixgbe);
1082 }
1083
1084 /*
1085  * ixgbe_init_driver_settings - Initialize driver settings.
1086  *
1087  * The settings include hardware function pointers, bus information,
1088  * rx/tx rings settings, link state, and any other parameters that
1089  * need to be setup during driver initialization.
1090  */
1091 static int
1092 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1093 {
1094         struct ixgbe_hw *hw = &ixgbe->hw;
1095         dev_info_t *devinfo = ixgbe->dip;
1096         ixgbe_rx_ring_t *rx_ring;
1097         ixgbe_rx_group_t *rx_group;
1098         ixgbe_tx_ring_t *tx_ring;
1099         uint32_t rx_size;
1100         uint32_t tx_size;
1101         uint32_t ring_per_group;
1102         int i;
1103
1104         /*
1105          * Initialize chipset specific hardware function pointers
1106          */
1107         if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1108                 return (IXGBE_FAILURE);
1109         }
1110
1111         /*
1112          * Get the system page size
1113          */
1114         ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1115
1116         /*
1117          * Set rx buffer size
1118          *
1119          * The IP header alignment room is counted in the calculation.
1120          * The rx buffer size is in unit of 1K that is required by the
1121          * chipset hardware.
1122          */
1123         rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1124         ixgbe->rx_buf_size = ((rx_size >> 10) +
1125             ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1126
1127         /*
1128          * Set tx buffer size
1129          */
1130         tx_size = ixgbe->max_frame_size;
1131         ixgbe->tx_buf_size = ((tx_size >> 10) +
1132             ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1133
1134         /*
1135          * Initialize rx/tx rings/groups parameters
1136          */
1137         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1138         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1139                 rx_ring = &ixgbe->rx_rings[i];
1140                 rx_ring->index = i;
1141                 rx_ring->ixgbe = ixgbe;
1142                 rx_ring->group_index = i / ring_per_group;
1143                 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1144         }
1145
1146         for (i = 0; i < ixgbe->num_rx_groups; i++) {
1147                 rx_group = &ixgbe->rx_groups[i];
1148                 rx_group->index = i;
1149                 rx_group->ixgbe = ixgbe;
1150         }
1151
1152         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1153                 tx_ring = &ixgbe->tx_rings[i];
1154                 tx_ring->index = i;
1155                 tx_ring->ixgbe = ixgbe;
1156                 if (ixgbe->tx_head_wb_enable)
1157                         tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1158                 else
1159                         tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1160
1161                 tx_ring->ring_size = ixgbe->tx_ring_size;
1162                 tx_ring->free_list_size = ixgbe->tx_ring_size +
1163                     (ixgbe->tx_ring_size >> 1);
1164         }
1165
1166         /*
1167          * Initialize values of interrupt throttling rate
1168          */
1169         for (i = 1; i < MAX_INTR_VECTOR; i++)
1170                 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1171
1172         /*
1173          * The initial link state should be "unknown"
1174          */
1175         ixgbe->link_state = LINK_STATE_UNKNOWN;
1176
1177         return (IXGBE_SUCCESS);
1178 }
1179
1180 /*
1181  * ixgbe_init_locks - Initialize locks.
1182  */
1183 static void
1184 ixgbe_init_locks(ixgbe_t *ixgbe)
1185 {
1186         ixgbe_rx_ring_t *rx_ring;
1187         ixgbe_tx_ring_t *tx_ring;
1188         int i;
1189
1190         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1191                 rx_ring = &ixgbe->rx_rings[i];
1192                 mutex_init(&rx_ring->rx_lock, NULL,
1193                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1194         }
1195
1196         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1197                 tx_ring = &ixgbe->tx_rings[i];
1198                 mutex_init(&tx_ring->tx_lock, NULL,
1199                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1200                 mutex_init(&tx_ring->recycle_lock, NULL,
1201                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1202                 mutex_init(&tx_ring->tcb_head_lock, NULL,
1203                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1204                 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1205                     MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1206         }
1207
1208         mutex_init(&ixgbe->gen_lock, NULL,
1209             MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1210
1211         mutex_init(&ixgbe->watchdog_lock, NULL,
1212             MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1213 }
1214
1215 /*
1216  * ixgbe_destroy_locks - Destroy locks.
1217  */
1218 static void
1219 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1220 {
1221         ixgbe_rx_ring_t *rx_ring;
1222         ixgbe_tx_ring_t *tx_ring;
1223         int i;
1224
1225         for (i = 0; i < ixgbe->num_rx_rings; i++) {
1226                 rx_ring = &ixgbe->rx_rings[i];
1227                 mutex_destroy(&rx_ring->rx_lock);
1228         }
1229
1230         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1231                 tx_ring = &ixgbe->tx_rings[i];
1232                 mutex_destroy(&tx_ring->tx_lock);
1233                 mutex_destroy(&tx_ring->recycle_lock);
1234                 mutex_destroy(&tx_ring->tcb_head_lock);
1235                 mutex_destroy(&tx_ring->tcb_tail_lock);
1236         }
1237
1238         mutex_destroy(&ixgbe->gen_lock);
1239         mutex_destroy(&ixgbe->watchdog_lock);
1240 }
1241
1242 static int
1243 ixgbe_resume(dev_info_t *devinfo)
1244 {
1245         ixgbe_t *ixgbe;
1246         int i;
1247
1248         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1249         if (ixgbe == NULL)
1250                 return (DDI_FAILURE);
1251
1252         mutex_enter(&ixgbe->gen_lock);
1253
1254         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1255                 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1256                         mutex_exit(&ixgbe->gen_lock);
1257                         return (DDI_FAILURE);
1258                 }
1259
1260                 /*
1261                  * Enable and start the watchdog timer
1262                  */
1263                 ixgbe_enable_watchdog_timer(ixgbe);
1264         }
1265
1266         atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1267
1268         if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1269                 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1270                         mac_tx_ring_update(ixgbe->mac_hdl,
1271                             ixgbe->tx_rings[i].ring_handle);
1272                 }
1273         }
1274
1275         mutex_exit(&ixgbe->gen_lock);
1276
1277         return (DDI_SUCCESS);
1278 }
1279
1280 static int
1281 ixgbe_suspend(dev_info_t *devinfo)
1282 {
1283         ixgbe_t *ixgbe;
1284
1285         ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1286         if (ixgbe == NULL)
1287                 return (DDI_FAILURE);
1288
1289         mutex_enter(&ixgbe->gen_lock);
1290
1291         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1292         if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1293                 mutex_exit(&ixgbe->gen_lock);
1294                 return (DDI_SUCCESS);
1295         }
1296         ixgbe_stop(ixgbe, B_FALSE);
1297
1298         mutex_exit(&ixgbe->gen_lock);
1299
1300         /*
1301          * Disable and stop the watchdog timer
1302          */
1303         ixgbe_disable_watchdog_timer(ixgbe);
1304
1305         return (DDI_SUCCESS);
1306 }
1307
1308 /*
1309  * ixgbe_init - Initialize the device.
1310  */
1311 static int
1312 ixgbe_init(ixgbe_t *ixgbe)
1313 {
1314         struct ixgbe_hw *hw = &ixgbe->hw;
1315         u8 pbanum[IXGBE_PBANUM_LENGTH];
1316         int rv;
1317
1318         mutex_enter(&ixgbe->gen_lock);
1319
1320         /*
1321          * Configure/Initialize hardware
1322          */
1323         rv = ixgbe_init_hw(hw);
1324         if (rv != IXGBE_SUCCESS) {
1325                 switch (rv) {
1326
1327                 /*
1328                  * The first three errors are not prohibitive to us progressing
1329                  * further, and are maily advisory in nature. In the case of a
1330                  * SFP module not being present or not deemed supported by the
1331                  * common code, we adivse the operator of this fact but carry on
1332                  * instead of failing hard, as SFPs can be inserted or replaced
1333                  * while the driver is running. In the case of a unknown error,
1334                  * we fail-hard, logging the reason and emitting a FMA event.
1335                  */
1336                 case IXGBE_ERR_EEPROM_VERSION:
1337                         ixgbe_error(ixgbe,
1338                             "This Intel 10Gb Ethernet device is pre-release and"
1339                             " contains outdated firmware. Please contact your"
1340                             " hardware vendor for a replacement.");
1341                         break;
1342                 case IXGBE_ERR_SFP_NOT_PRESENT:
1343                         ixgbe_error(ixgbe,
1344                             "No SFP+ module detected on this interface. Please "
1345                             "install a supported SFP+ module for this "
1346                             "interface to become operational.");
1347                         break;
1348                 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1349                         ixgbe_error(ixgbe,
1350                             "Unsupported SFP+ module detected. Please replace "
1351                             "it with a supported SFP+ module per Intel "
1352                             "documentation, or bypass this check with "
1353                             "allow_unsupported_sfp=1 in ixgbe.conf.");
1354                         break;
1355                 default:
1356                         ixgbe_error(ixgbe,
1357                             "Failed to initialize hardware. ixgbe_init_hw "
1358                             "returned %d", rv);
1359                         ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1360                         goto init_fail;
1361                 }
1362         }
1363
1364         /*
1365          * Need to init eeprom before validating the checksum.
1366          */
1367         if (ixgbe_init_eeprom_params(hw) < 0) {
1368                 ixgbe_error(ixgbe,
1369                     "Unable to intitialize the eeprom interface.");
1370                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1371                 goto init_fail;
1372         }
1373
1374         /*
1375          * NVM validation
1376          */
1377         if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1378                 /*
1379                  * Some PCI-E parts fail the first check due to
1380                  * the link being in sleep state.  Call it again,
1381                  * if it fails a second time it's a real issue.
1382                  */
1383                 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1384                         ixgbe_error(ixgbe,
1385                             "Invalid NVM checksum. Please contact "
1386                             "the vendor to update the NVM.");
1387                         ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1388                         goto init_fail;
1389                 }
1390         }
1391
1392         /*
1393          * Setup default flow control thresholds - enable/disable
1394          * & flow control type is controlled by ixgbe.conf
1395          */
1396         hw->fc.high_water[0] = DEFAULT_FCRTH;
1397         hw->fc.low_water[0] = DEFAULT_FCRTL;
1398         hw->fc.pause_time = DEFAULT_FCPAUSE;
1399         hw->fc.send_xon = B_TRUE;
1400
1401         /*
1402          * Initialize flow control
1403          */
1404         (void) ixgbe_start_hw(hw);
1405
1406         /*
1407          * Initialize link settings
1408          */
1409         (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1410
1411         /*
1412          * Initialize the chipset hardware
1413          */
1414         if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1415                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1416                 goto init_fail;
1417         }
1418
1419         /*
1420          * Read identifying information and place in devinfo.
1421          */
1422         pbanum[0] = '\0';
1423         (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1424         if (*pbanum != '\0') {
1425                 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1426                     "printed-board-assembly", (char *)pbanum);
1427         }
1428
1429         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1430                 goto init_fail;
1431         }
1432
1433         mutex_exit(&ixgbe->gen_lock);
1434         return (IXGBE_SUCCESS);
1435
1436 init_fail:
1437         /*
1438          * Reset PHY
1439          */
1440         (void) ixgbe_reset_phy(hw);
1441
1442         mutex_exit(&ixgbe->gen_lock);
1443         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1444         return (IXGBE_FAILURE);
1445 }
1446
1447 /*
1448  * ixgbe_chip_start - Initialize and start the chipset hardware.
1449  */
1450 static int
1451 ixgbe_chip_start(ixgbe_t *ixgbe)
1452 {
1453         struct ixgbe_hw *hw = &ixgbe->hw;
1454         int i;
1455
1456         ASSERT(mutex_owned(&ixgbe->gen_lock));
1457
1458         /*
1459          * Get the mac address
1460          * This function should handle SPARC case correctly.
1461          */
1462         if (!ixgbe_find_mac_address(ixgbe)) {
1463                 ixgbe_error(ixgbe, "Failed to get the mac address");
1464                 return (IXGBE_FAILURE);
1465         }
1466
1467         /*
1468          * Validate the mac address
1469          */
1470         (void) ixgbe_init_rx_addrs(hw);
1471         if (!is_valid_mac_addr(hw->mac.addr)) {
1472                 ixgbe_error(ixgbe, "Invalid mac address");
1473                 return (IXGBE_FAILURE);
1474         }
1475
1476         /*
1477          * Re-enable relaxed ordering for performance.  It is disabled
1478          * by default in the hardware init.
1479          */
1480         if (ixgbe->relax_order_enable == B_TRUE)
1481                 ixgbe_enable_relaxed_ordering(hw);
1482
1483         /*
1484          * Setup adapter interrupt vectors
1485          */
1486         ixgbe_setup_adapter_vector(ixgbe);
1487
1488         /*
1489          * Initialize unicast addresses.
1490          */
1491         ixgbe_init_unicst(ixgbe);
1492
1493         /*
1494          * Setup and initialize the mctable structures.
1495          */
1496         ixgbe_setup_multicst(ixgbe);
1497
1498         /*
1499          * Set interrupt throttling rate
1500          */
1501         for (i = 0; i < ixgbe->intr_cnt; i++) {
1502                 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1503         }
1504
1505         /*
1506          * Disable Wake-on-LAN
1507          */
1508         IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1509
1510         /*
1511          * Some adapters offer Energy Efficient Ethernet (EEE) support.
1512          * Due to issues with EEE in e1000g/igb, we disable this by default
1513          * as a precautionary measure.
1514          *
1515          * Currently, the only known adapter which supports EEE in the ixgbe
1516          * line is 8086,15AB (IXGBE_DEV_ID_X550EM_X_KR), and only after the
1517          * first revision of it, as well as any X550 with MAC type 6 (non-EM)
1518          */
1519         (void) ixgbe_setup_eee(hw, B_FALSE);
1520
1521         /*
1522          * Turn on any present SFP Tx laser
1523          */
1524         ixgbe_enable_tx_laser(hw);
1525
1526         /*
1527          * Power on the PHY
1528          */
1529         (void) ixgbe_set_phy_power(hw, B_TRUE);
1530
1531         /*
1532          * Save the state of the PHY
1533          */
1534         ixgbe_get_hw_state(ixgbe);
1535
1536         /*
1537          * Make sure driver has control
1538          */
1539         ixgbe_get_driver_control(hw);
1540
1541         return (IXGBE_SUCCESS);
1542 }
1543
1544 /*
1545  * ixgbe_chip_stop - Stop the chipset hardware
1546  */
1547 static void
1548 ixgbe_chip_stop(ixgbe_t *ixgbe)
1549 {
1550         struct ixgbe_hw *hw = &ixgbe->hw;
1551         int rv;
1552
1553         ASSERT(mutex_owned(&ixgbe->gen_lock));
1554
1555         /*
1556          * Stop interupt generation and disable Tx unit
1557          */
1558         hw->adapter_stopped = B_FALSE;
1559         (void) ixgbe_stop_adapter(hw);
1560
1561         /*
1562          * Reset the chipset
1563          */
1564         (void) ixgbe_reset_hw(hw);
1565
1566         /*
1567          * Reset PHY
1568          */
1569         (void) ixgbe_reset_phy(hw);
1570
1571         /*
1572          * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1573          * the PHY while doing so. Else, just power down the PHY.
1574          */
1575         if (hw->phy.ops.enter_lplu != NULL) {
1576                 hw->phy.reset_disable = B_TRUE;
1577                 rv = hw->phy.ops.enter_lplu(hw);
1578                 if (rv != IXGBE_SUCCESS)
1579                         ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1580                 hw->phy.reset_disable = B_FALSE;
1581         } else {
1582                 (void) ixgbe_set_phy_power(hw, B_FALSE);
1583         }
1584
1585         /*
1586          * Turn off any present SFP Tx laser
1587          * Expected for health and safety reasons
1588          */
1589         ixgbe_disable_tx_laser(hw);
1590
1591         /*
1592          * Tell firmware driver is no longer in control
1593          */
1594         ixgbe_release_driver_control(hw);
1595
1596 }
1597
1598 /*
1599  * ixgbe_reset - Reset the chipset and re-start the driver.
1600  *
1601  * It involves stopping and re-starting the chipset,
1602  * and re-configuring the rx/tx rings.
1603  */
1604 static int
1605 ixgbe_reset(ixgbe_t *ixgbe)
1606 {
1607         int i;
1608
1609         /*
1610          * Disable and stop the watchdog timer
1611          */
1612         ixgbe_disable_watchdog_timer(ixgbe);
1613
1614         mutex_enter(&ixgbe->gen_lock);
1615
1616         ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1617         atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1618
1619         ixgbe_stop(ixgbe, B_FALSE);
1620
1621         if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1622                 mutex_exit(&ixgbe->gen_lock);
1623                 return (IXGBE_FAILURE);
1624         }
1625
1626         /*
1627          * After resetting, need to recheck the link status.
1628          */
1629         ixgbe->link_check_complete = B_FALSE;
1630         ixgbe->link_check_hrtime = gethrtime() +
1631             (IXGBE_LINK_UP_TIME * 100000000ULL);
1632
1633         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1634
1635         if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1636                 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1637                         mac_tx_ring_update(ixgbe->mac_hdl,
1638                             ixgbe->tx_rings[i].ring_handle);
1639                 }
1640         }
1641
1642         mutex_exit(&ixgbe->gen_lock);
1643
1644         /*
1645          * Enable and start the watchdog timer
1646          */
1647         ixgbe_enable_watchdog_timer(ixgbe);
1648
1649         return (IXGBE_SUCCESS);
1650 }
1651
1652 /*
1653  * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1654  */
1655 static void
1656 ixgbe_tx_clean(ixgbe_t *ixgbe)
1657 {
1658         ixgbe_tx_ring_t *tx_ring;
1659         tx_control_block_t *tcb;
1660         link_list_t pending_list;
1661         uint32_t desc_num;
1662         int i, j;
1663
1664         LINK_LIST_INIT(&pending_list);
1665
1666         for (i = 0; i < ixgbe->num_tx_rings; i++) {
1667                 tx_ring = &ixgbe->tx_rings[i];
1668
1669                 mutex_enter(&tx_ring->recycle_lock);
1670
1671                 /*
1672                  * Clean the pending tx data - the pending packets in the
1673                  * work_list that have no chances to be transmitted again.
1674                  *
1675                  * We must ensure the chipset is stopped or the link is down
1676                  * before cleaning the transmit packets.
1677                  */
1678                 desc_num = 0;
1679                 for (j = 0; j < tx_ring->ring_size; j++) {
1680                         tcb = tx_ring->work_list[j];
1681                         if (tcb != NULL) {
1682                                 desc_num += tcb->desc_num;
1683
1684                                 tx_ring->work_list[j] = NULL;
1685
1686                                 ixgbe_free_tcb(tcb);
1687
1688                                 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1689                         }
1690                 }
1691
1692                 if (desc_num > 0) {
1693                         atomic_add_32(&tx_ring->tbd_free, desc_num);
1694                         ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1695
1696                         /*
1697                          * Reset the head and tail pointers of the tbd ring;
1698                          * Reset the writeback head if it's enable.
1699                          */
1700                         tx_ring->tbd_head = 0;
1701                         tx_ring->tbd_tail = 0;
1702                         if (ixgbe->tx_head_wb_enable)
1703                                 *tx_ring->tbd_head_wb = 0;
1704
1705                         IXGBE_WRITE_REG(&ixgbe->hw,
1706                             IXGBE_TDH(tx_ring->index), 0);
1707                         IXGBE_WRITE_REG(&ixgbe->hw,
1708                             IXGBE_TDT(tx_ring->index), 0);
1709                 }
1710
1711                 mutex_exit(&tx_ring->recycle_lock);
1712
1713                 /*
1714                  * Add the tx control blocks in the pending list to
1715                  * the free list.
1716                  */
1717                 ixgbe_put_free_list(tx_ring, &pending_list);
1718         }
1719 }
1720
1721 /*
1722  * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1723  * transmitted.
1724  */
1725 static bool
1726 ixgbe_tx_drain(ixgbe_t *ixgbe)
1727 {
1728         ixgbe_tx_ring_t *tx_ring;
1729         bool done;
1730         int i, j;
1731
1732         /*
1733          * Wait for a specific time to allow pending tx packets
1734          * to be transmitted.
1735          *
1736          * Check the counter tbd_free to see if transmission is done.
1737          * No lock protection is needed here.
1738          *
1739          * Return B_TRUE if all pending packets have been transmitted;
1740          * Otherwise return B_FALSE;
1741          */
1742         for (i = 0; i < TX_DRAIN_TIME; i++) {
1743
1744                 done = B_TRUE;
1745                 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1746                         tx_ring = &ixgbe->tx_rings[j];
1747                         done = done &&
1748                             (tx_ring->tbd_free == tx_ring->ring_size);
1749                 }
1750
1751                 if (done)
1752                         break;
1753
1754                 msec_delay(1);
1755         }
1756
1757         return (done);
1758 }
1759
1760 /*
1761  * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1762  */
1763 static bool
1764 ixgbe_rx_drain(ixgbe_t *ixgbe)
1765 {
1766         bool done = B_TRUE;
1767         int i;
1768
1769         /*
1770          * Polling the rx free list to check if those rx buffers held by
1771          * the upper layer are released.
1772          *
1773          * Check the counter rcb_free to see if all pending buffers are
1774          * released. No lock protection is needed here.
1775          *
1776          * Return B_TRUE if all pending buffers have been released;
1777          * Otherwise return B_FALSE;
1778          */
1779         for (i = 0; i < RX_DRAIN_TIME; i++) {
1780                 done = (ixgbe->rcb_pending == 0);
1781
1782                 if (done)
1783                         break;
1784
1785                 msec_delay(1);
1786         }
1787
1788         return (done);
1789 }
1790
1791 /*
1792  * ixgbe_start - Start the driver/chipset.
1793  */
1794 int
1795 ixgbe_start(ixgbe_t *ixgbe, bool alloc_buffer)
1796 {
1797         struct ixgbe_hw *hw = &ixgbe->hw;
1798         int i;
1799
1800         ASSERT(mutex_owned(&ixgbe->gen_lock));
1801
1802         if (alloc_buffer) {
1803                 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1804                         ixgbe_error(ixgbe,
1805                             "Failed to allocate software receive rings");
1806                         return (IXGBE_FAILURE);
1807                 }
1808
1809                 /* Allocate buffers for all the rx/tx rings */
1810                 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1811                         ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1812                         return (IXGBE_FAILURE);
1813                 }
1814
1815                 ixgbe->tx_ring_init = B_TRUE;
1816         } else {
1817                 ixgbe->tx_ring_init = B_FALSE;
1818         }
1819
1820         for (i = 0; i < ixgbe->num_rx_rings; i++)
1821                 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1822         for (i = 0; i < ixgbe->num_tx_rings; i++)
1823                 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1824
1825         /*
1826          * Start the chipset hardware
1827          */
1828         if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1829                 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1830                 goto start_failure;
1831         }
1832
1833         /*
1834          * Configure link now for X550
1835          *
1836          * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1837          * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1838          * the resting state of the link would be the maximum speed that
1839          * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1840          * so we never bothered with explicitly setting the link to 10Gb as it
1841          * would already be at that state on driver attach. With X550, we must
1842          * trigger a re-negotiation of the link in order to switch from a LPLU
1843          * 1Gb link to 10Gb (cable and link partner permitting.)
1844          */
1845         if (hw->mac.type == ixgbe_mac_X550 ||
1846             hw->mac.type == ixgbe_mac_X550EM_x) {
1847                 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1848                 ixgbe_get_hw_state(ixgbe);
1849         }
1850
1851         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1852                 goto start_failure;
1853         }
1854
1855         /*
1856          * Setup the rx/tx rings
1857          */
1858         ixgbe_setup_rings(ixgbe);
1859
1860         /*
1861          * ixgbe_start() will be called when resetting, however if reset
1862          * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1863          * before enabling the interrupts.
1864          */
1865         atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1866             | IXGBE_STALL| IXGBE_OVERTEMP));
1867
1868         /*
1869          * Enable adapter interrupts
1870          * The interrupts must be enabled after the driver state is START
1871          */
1872         ixgbe_enable_adapter_interrupts(ixgbe);
1873
1874         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1875                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1876         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1877                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1878
1879         return (IXGBE_SUCCESS);
1880
1881 start_failure:
1882         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1883                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1884         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1885                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1886
1887         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1888
1889         return (IXGBE_FAILURE);
1890 }
1891
1892 /*
1893  * ixgbe_stop - Stop the driver/chipset.
1894  */
1895 void
1896 ixgbe_stop(ixgbe_t *ixgbe, bool free_buffer)
1897 {
1898         int i;
1899
1900         ASSERT(mutex_owned(&ixgbe->gen_lock));
1901
1902         /*
1903          * Disable the adapter interrupts
1904          */
1905         ixgbe_disable_adapter_interrupts(ixgbe);
1906
1907         /*
1908          * Drain the pending tx packets
1909          */
1910         (void) ixgbe_tx_drain(ixgbe);
1911
1912         for (i = 0; i < ixgbe->num_rx_rings; i++)
1913                 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1914         for (i = 0; i < ixgbe->num_tx_rings; i++)
1915                 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1916
1917         /*
1918          * Stop the chipset hardware
1919          */
1920         ixgbe_chip_stop(ixgbe);
1921
1922         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1923                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1924         }
1925
1926         /*
1927          * Clean the pending tx data/resources
1928          */
1929         ixgbe_tx_clean(ixgbe);
1930
1931         for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1932                 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1933         for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1934                 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1935
1936         if (ixgbe->link_state == LINK_STATE_UP) {
1937                 ixgbe->link_state = LINK_STATE_UNKNOWN;
1938                 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1939         }
1940
1941         if (free_buffer) {
1942                 /*
1943                  * Release the DMA/memory resources of rx/tx rings
1944                  */
1945                 ixgbe_free_dma(ixgbe);
1946                 ixgbe_free_rx_data(ixgbe);
1947         }
1948 }
1949
1950 /*
1951  * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1952  */
1953 /* ARGSUSED */
1954 static int
1955 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1956     void *arg1, void *arg2)
1957 {
1958         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1959
1960         switch (cbaction) {
1961         /* IRM callback */
1962         int count;
1963         case DDI_CB_INTR_ADD:
1964         case DDI_CB_INTR_REMOVE:
1965                 count = (int)(uintptr_t)cbarg;
1966                 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1967                 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1968                     int, ixgbe->intr_cnt);
1969                 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1970                     DDI_SUCCESS) {
1971                         ixgbe_error(ixgbe,
1972                             "IRM CB: Failed to adjust interrupts");
1973                         goto cb_fail;
1974                 }
1975                 break;
1976         default:
1977                 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1978                     cbaction);
1979                 return (DDI_ENOTSUP);
1980         }
1981         return (DDI_SUCCESS);
1982 cb_fail:
1983         return (DDI_FAILURE);
1984 }
1985
1986 /*
1987  * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1988  */
1989 static int
1990 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1991 {
1992         int i, rc, actual;
1993
1994         if (count == 0)
1995                 return (DDI_SUCCESS);
1996
1997         if ((cbaction == DDI_CB_INTR_ADD &&
1998             ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1999             (cbaction == DDI_CB_INTR_REMOVE &&
2000             ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2001                 return (DDI_FAILURE);
2002
2003         if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2004                 return (DDI_FAILURE);
2005         }
2006
2007         for (i = 0; i < ixgbe->num_rx_rings; i++)
2008                 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2009         for (i = 0; i < ixgbe->num_tx_rings; i++)
2010                 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2011
2012         mutex_enter(&ixgbe->gen_lock);
2013         ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2014         ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2015         ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2016         mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2017
2018         ixgbe_stop(ixgbe, B_FALSE);
2019         /*
2020          * Disable interrupts
2021          */
2022         if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2023                 rc = ixgbe_disable_intrs(ixgbe);
2024                 ASSERT(rc == IXGBE_SUCCESS);
2025         }
2026         ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2027
2028         /*
2029          * Remove interrupt handlers
2030          */
2031         if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2032                 ixgbe_rem_intr_handlers(ixgbe);
2033         }
2034         ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2035
2036         /*
2037          * Clear vect_map
2038          */
2039         bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2040         switch (cbaction) {
2041         case DDI_CB_INTR_ADD:
2042                 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2043                     DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2044                     DDI_INTR_ALLOC_NORMAL);
2045                 if (rc != DDI_SUCCESS || actual != count) {
2046                         ixgbe_log(ixgbe, "Adjust interrupts failed."
2047                             "return: %d, irm cb size: %d, actual: %d",
2048                             rc, count, actual);
2049                         goto intr_adjust_fail;
2050                 }
2051                 ixgbe->intr_cnt += count;
2052                 break;
2053
2054         case DDI_CB_INTR_REMOVE:
2055                 for (i = ixgbe->intr_cnt - count;
2056                     i < ixgbe->intr_cnt; i ++) {
2057                         rc = ddi_intr_free(ixgbe->htable[i]);
2058                         ixgbe->htable[i] = NULL;
2059                         if (rc != DDI_SUCCESS) {
2060                                 ixgbe_log(ixgbe, "Adjust interrupts failed."
2061                                     "return: %d, irm cb size: %d, actual: %d",
2062                                     rc, count, actual);
2063                                 goto intr_adjust_fail;
2064                         }
2065                 }
2066                 ixgbe->intr_cnt -= count;
2067                 break;
2068         }
2069
2070         /*
2071          * Get priority for first vector, assume remaining are all the same
2072          */
2073         rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2074         if (rc != DDI_SUCCESS) {
2075                 ixgbe_log(ixgbe,
2076                     "Get interrupt priority failed: %d", rc);
2077                 goto intr_adjust_fail;
2078         }
2079         rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2080         if (rc != DDI_SUCCESS) {
2081                 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2082                 goto intr_adjust_fail;
2083         }
2084         ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2085
2086         /*
2087          * Map rings to interrupt vectors
2088          */
2089         if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2090                 ixgbe_error(ixgbe,
2091                     "IRM CB: Failed to map interrupts to vectors");
2092                 goto intr_adjust_fail;
2093         }
2094
2095         /*
2096          * Add interrupt handlers
2097          */
2098         if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2099                 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2100                 goto intr_adjust_fail;
2101         }
2102         ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2103
2104         /*
2105          * Now that mutex locks are initialized, and the chip is also
2106          * initialized, enable interrupts.
2107          */
2108         if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2109                 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2110                 goto intr_adjust_fail;
2111         }
2112         ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2113         if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2114                 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2115                 goto intr_adjust_fail;
2116         }
2117         ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2118         ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2119         ixgbe->ixgbe_state |= IXGBE_STARTED;
2120         mutex_exit(&ixgbe->gen_lock);
2121
2122         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2123                 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2124                     ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2125         }
2126         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2127                 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2128                     ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2129         }
2130
2131         /* Wakeup all Tx rings */
2132         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2133                 mac_tx_ring_update(ixgbe->mac_hdl,
2134                     ixgbe->tx_rings[i].ring_handle);
2135         }
2136
2137         IXGBE_DEBUGLOG_3(ixgbe,
2138             "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2139             ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2140         return (DDI_SUCCESS);
2141
2142 intr_adjust_fail:
2143         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2144         mutex_exit(&ixgbe->gen_lock);
2145         return (DDI_FAILURE);
2146 }
2147
2148 /*
2149  * ixgbe_intr_cb_register - Register interrupt callback function.
2150  */
2151 static int
2152 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2153 {
2154         if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2155             ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2156                 return (IXGBE_FAILURE);
2157         }
2158         IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2159         return (IXGBE_SUCCESS);
2160 }
2161
2162 /*
2163  * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2164  */
2165 static int
2166 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2167 {
2168         /*
2169          * Allocate memory space for rx rings
2170          */
2171         ixgbe->rx_rings = kmem_zalloc(
2172             sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2173             KM_NOSLEEP);
2174
2175         if (ixgbe->rx_rings == NULL) {
2176                 return (IXGBE_FAILURE);
2177         }
2178
2179         /*
2180          * Allocate memory space for tx rings
2181          */
2182         ixgbe->tx_rings = kmem_zalloc(
2183             sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2184             KM_NOSLEEP);
2185
2186         if (ixgbe->tx_rings == NULL) {
2187                 kmem_free(ixgbe->rx_rings,
2188                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2189                 ixgbe->rx_rings = NULL;
2190                 return (IXGBE_FAILURE);
2191         }
2192
2193         /*
2194          * Allocate memory space for rx ring groups
2195          */
2196         ixgbe->rx_groups = kmem_zalloc(
2197             sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2198             KM_NOSLEEP);
2199
2200         if (ixgbe->rx_groups == NULL) {
2201                 kmem_free(ixgbe->rx_rings,
2202                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2203                 kmem_free(ixgbe->tx_rings,
2204                     sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2205                 ixgbe->rx_rings = NULL;
2206                 ixgbe->tx_rings = NULL;
2207                 return (IXGBE_FAILURE);
2208         }
2209
2210         return (IXGBE_SUCCESS);
2211 }
2212
2213 /*
2214  * ixgbe_free_rings - Free the memory space of rx/tx rings.
2215  */
2216 static void
2217 ixgbe_free_rings(ixgbe_t *ixgbe)
2218 {
2219         if (ixgbe->rx_rings != NULL) {
2220                 kmem_free(ixgbe->rx_rings,
2221                     sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2222                 ixgbe->rx_rings = NULL;
2223         }
2224
2225         if (ixgbe->tx_rings != NULL) {
2226                 kmem_free(ixgbe->tx_rings,
2227                     sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2228                 ixgbe->tx_rings = NULL;
2229         }
2230
2231         if (ixgbe->rx_groups != NULL) {
2232                 kmem_free(ixgbe->rx_groups,
2233                     sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2234                 ixgbe->rx_groups = NULL;
2235         }
2236 }
2237
2238 static int
2239 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2240 {
2241         ixgbe_rx_ring_t *rx_ring;
2242         int i;
2243
2244         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2245                 rx_ring = &ixgbe->rx_rings[i];
2246                 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2247                         goto alloc_rx_rings_failure;
2248         }
2249         return (IXGBE_SUCCESS);
2250
2251 alloc_rx_rings_failure:
2252         ixgbe_free_rx_data(ixgbe);
2253         return (IXGBE_FAILURE);
2254 }
2255
2256 static void
2257 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2258 {
2259         ixgbe_rx_ring_t *rx_ring;
2260         ixgbe_rx_data_t *rx_data;
2261         int i;
2262
2263         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2264                 rx_ring = &ixgbe->rx_rings[i];
2265
2266                 mutex_enter(&ixgbe->rx_pending_lock);
2267                 rx_data = rx_ring->rx_data;
2268
2269                 if (rx_data != NULL) {
2270                         rx_data->flag |= IXGBE_RX_STOPPED;
2271
2272                         if (rx_data->rcb_pending == 0) {
2273                                 ixgbe_free_rx_ring_data(rx_data);
2274                                 rx_ring->rx_data = NULL;
2275                         }
2276                 }
2277
2278                 mutex_exit(&ixgbe->rx_pending_lock);
2279         }
2280 }
2281
2282 /*
2283  * ixgbe_setup_rings - Setup rx/tx rings.
2284  */
2285 static void
2286 ixgbe_setup_rings(ixgbe_t *ixgbe)
2287 {
2288         /*
2289          * Setup the rx/tx rings, including the following:
2290          *
2291          * 1. Setup the descriptor ring and the control block buffers;
2292          * 2. Initialize necessary registers for receive/transmit;
2293          * 3. Initialize software pointers/parameters for receive/transmit;
2294          */
2295         ixgbe_setup_rx(ixgbe);
2296
2297         ixgbe_setup_tx(ixgbe);
2298 }
2299
2300 static void
2301 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2302 {
2303         ixgbe_t *ixgbe = rx_ring->ixgbe;
2304         ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2305         struct ixgbe_hw *hw = &ixgbe->hw;
2306         rx_control_block_t *rcb;
2307         union ixgbe_adv_rx_desc *rbd;
2308         uint32_t size;
2309         uint32_t buf_low;
2310         uint32_t buf_high;
2311         uint32_t reg_val;
2312         int i;
2313
2314         ASSERT(mutex_owned(&rx_ring->rx_lock));
2315         ASSERT(mutex_owned(&ixgbe->gen_lock));
2316
2317         for (i = 0; i < ixgbe->rx_ring_size; i++) {
2318                 rcb = rx_data->work_list[i];
2319                 rbd = &rx_data->rbd_ring[i];
2320
2321                 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2322                 rbd->read.hdr_addr = (uintptr_t)NULL;
2323         }
2324
2325         /*
2326          * Initialize the length register
2327          */
2328         size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2329         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2330
2331         /*
2332          * Initialize the base address registers
2333          */
2334         buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2335         buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2336         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2337         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2338
2339         /*
2340          * Setup head & tail pointers
2341          */
2342         IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2343             rx_data->ring_size - 1);
2344         IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2345
2346         rx_data->rbd_next = 0;
2347         rx_data->lro_first = 0;
2348
2349         /*
2350          * Setup the Receive Descriptor Control Register (RXDCTL)
2351          * PTHRESH=32 descriptors (half the internal cache)
2352          * HTHRESH=0 descriptors (to minimize latency on fetch)
2353          * WTHRESH defaults to 1 (writeback each descriptor)
2354          */
2355         reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2356         reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2357
2358         /* Not a valid value for 82599, X540 or X550 */
2359         if (hw->mac.type == ixgbe_mac_82598EB) {
2360                 reg_val |= 0x0020;      /* pthresh */
2361         }
2362         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2363
2364         if (hw->mac.type == ixgbe_mac_82599EB ||
2365             hw->mac.type == ixgbe_mac_X540 ||
2366             hw->mac.type == ixgbe_mac_X550 ||
2367             hw->mac.type == ixgbe_mac_X550EM_x) {
2368                 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2369                 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2370                 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2371         }
2372
2373         /*
2374          * Setup the Split and Replication Receive Control Register.
2375          * Set the rx buffer size and the advanced descriptor type.
2376          */
2377         reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2378             IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2379         reg_val |= IXGBE_SRRCTL_DROP_EN;
2380         IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2381 }
2382
2383 static void
2384 ixgbe_setup_rx(ixgbe_t *ixgbe)
2385 {
2386         ixgbe_rx_ring_t *rx_ring;
2387         struct ixgbe_hw *hw = &ixgbe->hw;
2388         uint32_t reg_val;
2389         uint32_t ring_mapping;
2390         uint32_t i, index;
2391         uint32_t psrtype_rss_bit;
2392
2393         /*
2394          * Ensure that Rx is disabled while setting up
2395          * the Rx unit and Rx descriptor ring(s)
2396          */
2397         ixgbe_disable_rx(hw);
2398
2399         /* PSRTYPE must be configured for 82599 */
2400         if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2401             ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2402                 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2403                     IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2404                 reg_val |= IXGBE_PSRTYPE_L2HDR;
2405                 reg_val |= 0x80000000;
2406                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2407         } else {
2408                 if (ixgbe->num_rx_groups > 32) {
2409                         psrtype_rss_bit = 0x20000000;
2410                 } else {
2411                         psrtype_rss_bit = 0x40000000;
2412                 }
2413                 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2414                         reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2415                             IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2416                         reg_val |= IXGBE_PSRTYPE_L2HDR;
2417                         reg_val |= psrtype_rss_bit;
2418                         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2419                 }
2420         }
2421
2422         /*
2423          * Set filter control in FCTRL to determine types of packets are passed
2424          * up to the driver.
2425          * - Pass broadcast packets.
2426          * - Do not pass flow control pause frames (82598-specific)
2427          */
2428         reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2429         reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2430         if (hw->mac.type == ixgbe_mac_82598EB) {
2431                 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2432         }
2433         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2434
2435         /*
2436          * Hardware checksum settings
2437          */
2438         if (ixgbe->rx_hcksum_enable) {
2439                 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2440                 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2441                 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2442         }
2443
2444         /*
2445          * Setup VMDq and RSS for multiple receive queues
2446          */
2447         switch (ixgbe->classify_mode) {
2448         case IXGBE_CLASSIFY_RSS:
2449                 /*
2450                  * One group, only RSS is needed when more than
2451                  * one ring enabled.
2452                  */
2453                 ixgbe_setup_rss(ixgbe);
2454                 break;
2455
2456         case IXGBE_CLASSIFY_VMDQ:
2457                 /*
2458                  * Multiple groups, each group has one ring,
2459                  * only VMDq is needed.
2460                  */
2461                 ixgbe_setup_vmdq(ixgbe);
2462                 break;
2463
2464         case IXGBE_CLASSIFY_VMDQ_RSS:
2465                 /*
2466                  * Multiple groups and multiple rings, both
2467                  * VMDq and RSS are needed.
2468                  */
2469                 ixgbe_setup_vmdq_rss(ixgbe);
2470                 break;
2471
2472         default:
2473                 break;
2474         }
2475
2476         /*
2477          * Enable the receive unit.  This must be done after filter
2478          * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2479          * 82598 is the only adapter which defines this RXCTRL option.
2480          */
2481         reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2482         if (hw->mac.type == ixgbe_mac_82598EB)
2483                 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2484         reg_val |= IXGBE_RXCTRL_RXEN;
2485         (void) ixgbe_enable_rx_dma(hw, reg_val);
2486
2487         /*
2488          * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2489          */
2490         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2491                 rx_ring = &ixgbe->rx_rings[i];
2492                 ixgbe_setup_rx_ring(rx_ring);
2493         }
2494
2495         /*
2496          * Setup the per-ring statistics mapping.
2497          */
2498         ring_mapping = 0;
2499         for (i = 0; i < ixgbe->num_rx_rings; i++) {
2500                 index = ixgbe->rx_rings[i].hw_index;
2501                 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2502                 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2503                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2504         }
2505
2506         /*
2507          * The Max Frame Size in MHADD/MAXFRS will be internally increased
2508          * by four bytes if the packet has a VLAN field, so includes MTU,
2509          * ethernet header and frame check sequence.
2510          * Register is MAXFRS in 82599.
2511          */
2512         reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2513         reg_val &= ~IXGBE_MHADD_MFS_MASK;
2514         reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2515             + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2516         IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2517
2518         /*
2519          * Setup Jumbo Frame enable bit
2520          */
2521         reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2522         if (ixgbe->default_mtu > ETHERMTU)
2523                 reg_val |= IXGBE_HLREG0_JUMBOEN;
2524         else
2525                 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2526         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2527
2528         /*
2529          * Setup RSC for multiple receive queues.
2530          */
2531         if (ixgbe->lro_enable) {
2532                 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2533                         /*
2534                          * Make sure rx_buf_size * MAXDESC not greater
2535                          * than 65535.
2536                          * Intel recommends 4 for MAXDESC field value.
2537                          */
2538                         reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2539                         reg_val |= IXGBE_RSCCTL_RSCEN;
2540                         if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2541                                 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2542                         else
2543                                 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2544                         IXGBE_WRITE_REG(hw,  IXGBE_RSCCTL(i), reg_val);
2545                 }
2546
2547                 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2548                 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2549                 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2550
2551                 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2552                 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2553                 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2554                 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2555
2556                 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2557         }
2558 }
2559
2560 static void
2561 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2562 {
2563         ixgbe_t *ixgbe = tx_ring->ixgbe;
2564         struct ixgbe_hw *hw = &ixgbe->hw;
2565         uint32_t size;
2566         uint32_t buf_low;
2567         uint32_t buf_high;
2568         uint32_t reg_val;
2569
2570         ASSERT(mutex_owned(&tx_ring->tx_lock));
2571         ASSERT(mutex_owned(&ixgbe->gen_lock));
2572
2573         /*
2574          * Initialize the length register
2575          */
2576         size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2577         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2578
2579         /*
2580          * Initialize the base address registers
2581          */
2582         buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2583         buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2584         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2585         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2586
2587         /*
2588          * Setup head & tail pointers
2589          */
2590         IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2591         IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2592
2593         /*
2594          * Setup head write-back
2595          */
2596         if (ixgbe->tx_head_wb_enable) {
2597                 /*
2598                  * The memory of the head write-back is allocated using
2599                  * the extra tbd beyond the tail of the tbd ring.
2600                  */
2601                 tx_ring->tbd_head_wb = (uint32_t *)
2602                     ((uintptr_t)tx_ring->tbd_area.address + size);
2603                 *tx_ring->tbd_head_wb = 0;
2604
2605                 buf_low = (uint32_t)
2606                     (tx_ring->tbd_area.dma_address + size);
2607                 buf_high = (uint32_t)
2608                     ((tx_ring->tbd_area.dma_address + size) >> 32);
2609
2610                 /* Set the head write-back enable bit */
2611                 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2612
2613                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2614                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2615
2616                 /*
2617                  * Turn off relaxed ordering for head write back or it will
2618                  * cause problems with the tx recycling
2619                  */
2620
2621                 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2622                     IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2623                     IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2624                 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2625                 if (hw->mac.type == ixgbe_mac_82598EB) {
2626                         IXGBE_WRITE_REG(hw,
2627                             IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2628                 } else {
2629                         IXGBE_WRITE_REG(hw,
2630                             IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2631                 }
2632         } else {
2633                 tx_ring->tbd_head_wb = NULL;
2634         }
2635
2636         tx_ring->tbd_head = 0;
2637         tx_ring->tbd_tail = 0;
2638         tx_ring->tbd_free = tx_ring->ring_size;
2639
2640         if (ixgbe->tx_ring_init == B_TRUE) {
2641                 tx_ring->tcb_head = 0;
2642                 tx_ring->tcb_tail = 0;
2643                 tx_ring->tcb_free = tx_ring->free_list_size;
2644         }
2645
2646         /*
2647          * Initialize the s/w context structure
2648          */
2649         bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2650 }
2651
2652 static void
2653 ixgbe_setup_tx(ixgbe_t *ixgbe)
2654 {
2655         struct ixgbe_hw *hw = &ixgbe->hw;
2656         ixgbe_tx_ring_t *tx_ring;
2657         uint32_t reg_val;
2658         uint32_t ring_mapping;
2659         int i;
2660
2661         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2662                 tx_ring = &ixgbe->tx_rings[i];
2663                 ixgbe_setup_tx_ring(tx_ring);
2664         }
2665
2666         /*
2667          * Setup the per-ring statistics mapping.
2668          */
2669         ring_mapping = 0;
2670         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2671                 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2672                 if ((i & 0x3) == 0x3) {
2673                         switch (hw->mac.type) {
2674                         case ixgbe_mac_82598EB:
2675                                 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2676                                     ring_mapping);
2677                                 break;
2678
2679                         case ixgbe_mac_82599EB:
2680                         case ixgbe_mac_X540:
2681                         case ixgbe_mac_X550:
2682                         case ixgbe_mac_X550EM_x:
2683                                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2684                                     ring_mapping);
2685                                 break;
2686
2687                         default:
2688                                 break;
2689                         }
2690
2691                         ring_mapping = 0;
2692                 }
2693         }
2694         if (i & 0x3) {
2695                 switch (hw->mac.type) {
2696                 case ixgbe_mac_82598EB:
2697                         IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2698                         break;
2699
2700                 case ixgbe_mac_82599EB:
2701                 case ixgbe_mac_X540:
2702                 case ixgbe_mac_X550:
2703                 case ixgbe_mac_X550EM_x:
2704                         IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2705                         break;
2706
2707                 default:
2708                         break;
2709                 }
2710         }
2711
2712         /*
2713          * Enable CRC appending and TX padding (for short tx frames)
2714          */
2715         reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2716         reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2717         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2718
2719         /*
2720          * enable DMA for 82599, X540 and X550 parts
2721          */
2722         if (hw->mac.type == ixgbe_mac_82599EB ||
2723             hw->mac.type == ixgbe_mac_X540 ||
2724             hw->mac.type == ixgbe_mac_X550 ||
2725             hw->mac.type == ixgbe_mac_X550EM_x) {
2726                 /* DMATXCTL.TE must be set after all Tx config is complete */
2727                 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2728                 reg_val |= IXGBE_DMATXCTL_TE;
2729                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2730
2731                 /* Disable arbiter to set MTQC */
2732                 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2733                 reg_val |= IXGBE_RTTDCS_ARBDIS;
2734                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2735                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2736                 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2737                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2738         }
2739
2740         /*
2741          * Enabling tx queues ..
2742          * For 82599 must be done after DMATXCTL.TE is set
2743          */
2744         for (i = 0; i < ixgbe->num_tx_rings; i++) {
2745                 tx_ring = &ixgbe->tx_rings[i];
2746                 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2747                 reg_val |= IXGBE_TXDCTL_ENABLE;
2748                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2749         }
2750 }
2751
2752 /*
2753  * ixgbe_setup_rss - Setup receive-side scaling feature.
2754  */
2755 static void
2756 ixgbe_setup_rss(ixgbe_t *ixgbe)
2757 {
2758         struct ixgbe_hw *hw = &ixgbe->hw;
2759         uint32_t mrqc;
2760
2761         /*
2762          * Initialize RETA/ERETA table
2763          */
2764         ixgbe_setup_rss_table(ixgbe);
2765
2766         /*
2767          * Enable RSS & perform hash on these packet types
2768          */
2769         mrqc = IXGBE_MRQC_RSSEN |
2770             IXGBE_MRQC_RSS_FIELD_IPV4 |
2771             IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2772             IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2773             IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2774             IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2775             IXGBE_MRQC_RSS_FIELD_IPV6 |
2776             IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2777             IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2778             IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2779         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2780 }
2781
2782 /*
2783  * ixgbe_setup_vmdq - Setup MAC classification feature
2784  */
2785 static void
2786 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2787 {
2788         struct ixgbe_hw *hw = &ixgbe->hw;
2789         uint32_t vmdctl, i, vtctl;
2790
2791         /*
2792          * Setup the VMDq Control register, enable VMDq based on
2793          * packet destination MAC address:
2794          */
2795         switch (hw->mac.type) {
2796         case ixgbe_mac_82598EB:
2797                 /*
2798                  * VMDq Enable = 1;
2799                  * VMDq Filter = 0; MAC filtering
2800                  * Default VMDq output index = 0;
2801                  */
2802                 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2803                 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2804                 break;
2805
2806         case ixgbe_mac_82599EB:
2807         case ixgbe_mac_X540:
2808         case ixgbe_mac_X550:
2809         case ixgbe_mac_X550EM_x:
2810                 /*
2811                  * Enable VMDq-only.
2812                  */
2813                 vmdctl = IXGBE_MRQC_VMDQEN;
2814                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2815
2816                 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2817                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2818                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2819                 }
2820
2821                 /*
2822                  * Enable Virtualization and Replication.
2823                  */
2824                 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2825                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2826
2827                 /*
2828                  * Enable receiving packets to all VFs
2829                  */
2830                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2831                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2832                 break;
2833
2834         default:
2835                 break;
2836         }
2837 }
2838
2839 /*
2840  * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2841  */
2842 static void
2843 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2844 {
2845         struct ixgbe_hw *hw = &ixgbe->hw;
2846         uint32_t i, mrqc;
2847         uint32_t vtctl, vmdctl;
2848
2849         /*
2850          * Initialize RETA/ERETA table
2851          */
2852         ixgbe_setup_rss_table(ixgbe);
2853
2854         /*
2855          * Enable and setup RSS and VMDq
2856          */
2857         switch (hw->mac.type) {
2858         case ixgbe_mac_82598EB:
2859                 /*
2860                  * Enable RSS & Setup RSS Hash functions
2861                  */
2862                 mrqc = IXGBE_MRQC_RSSEN |
2863                     IXGBE_MRQC_RSS_FIELD_IPV4 |
2864                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2865                     IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2866                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2867                     IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2868                     IXGBE_MRQC_RSS_FIELD_IPV6 |
2869                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2870                     IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2871                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2872                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2873
2874                 /*
2875                  * Enable and Setup VMDq
2876                  * VMDq Filter = 0; MAC filtering
2877                  * Default VMDq output index = 0;
2878                  */
2879                 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2880                 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2881                 break;
2882
2883         case ixgbe_mac_82599EB:
2884         case ixgbe_mac_X540:
2885         case ixgbe_mac_X550:
2886         case ixgbe_mac_X550EM_x:
2887                 /*
2888                  * Enable RSS & Setup RSS Hash functions
2889                  */
2890                 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2891                     IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2892                     IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2893                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2894                     IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2895                     IXGBE_MRQC_RSS_FIELD_IPV6 |
2896                     IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2897                     IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2898                     IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2899
2900                 /*
2901                  * Enable VMDq+RSS.
2902                  */
2903                 if (ixgbe->num_rx_groups > 32)  {
2904                         mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2905                 } else {
2906                         mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2907                 }
2908
2909                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2910
2911                 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2912                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2913                         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2914                 }
2915                 break;
2916
2917         default:
2918                 break;
2919
2920         }
2921
2922         if (hw->mac.type == ixgbe_mac_82599EB ||
2923             hw->mac.type == ixgbe_mac_X540 ||
2924             hw->mac.type == ixgbe_mac_X550 ||
2925             hw->mac.type == ixgbe_mac_X550EM_x) {
2926                 /*
2927                  * Enable Virtualization and Replication.
2928                  */
2929                 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2930                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2931
2932                 /*
2933                  * Enable receiving packets to all VFs
2934                  */
2935                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2936                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2937         }
2938 }
2939
2940 /*
2941  * ixgbe_setup_rss_table - Setup RSS table
2942  */
2943 static void
2944 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
2945 {
2946         struct ixgbe_hw *hw = &ixgbe->hw;
2947         uint32_t i, j;
2948         uint32_t random;
2949         uint32_t reta;
2950         uint32_t ring_per_group;
2951         uint32_t ring;
2952         uint32_t table_size;
2953         uint32_t index_mult;
2954         uint32_t rxcsum;
2955
2956         /*
2957          * Set multiplier for RETA setup and table size based on MAC type.
2958          * RETA table sizes vary by model:
2959          *
2960          * 82598, 82599, X540: 128 table entries.
2961          * X550: 512 table entries.
2962          */
2963         index_mult = 0x1;
2964         table_size = 128;
2965         switch (ixgbe->hw.mac.type) {
2966         case ixgbe_mac_82598EB:
2967                 index_mult = 0x11;
2968                 break;
2969         case ixgbe_mac_X550:
2970         case ixgbe_mac_X550EM_x:
2971                 table_size = 512;
2972                 break;
2973         default:
2974                 break;
2975         }
2976
2977         /*
2978          * Fill out RSS redirection table. The configuation of the indices is
2979          * hardware-dependent.
2980          *
2981          *  82598: 8 bits wide containing two 4 bit RSS indices
2982          *  82599, X540: 8 bits wide containing one 4 bit RSS index
2983          *  X550: 8 bits wide containing one 6 bit RSS index
2984          */
2985         reta = 0;
2986         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2987
2988         for (i = 0, j = 0; i < table_size; i++, j++) {
2989                 if (j == ring_per_group) j = 0;
2990
2991                 /*
2992                  * The low 8 bits are for hash value (n+0);
2993                  * The next 8 bits are for hash value (n+1), etc.
2994                  */
2995                 ring = (j * index_mult);
2996                 reta = reta >> 8;
2997                 reta = reta | (((uint32_t)ring) << 24);
2998
2999                 if ((i & 3) == 3) {
3000                         /*
3001                          * The first 128 table entries are programmed into the
3002                          * RETA register, with any beyond that (eg; on X550)
3003                          * into ERETA.
3004                          */
3005                         if (i < 128)
3006                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3007                         else
3008                                 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3009                                     reta);
3010                         reta = 0;
3011                 }
3012         }
3013
3014         /*
3015          * Fill out hash function seeds with a random constant
3016          */
3017         for (i = 0; i < 10; i++) {
3018                 (void) random_get_pseudo_bytes((uint8_t *)&random,
3019                     sizeof (uint32_t));
3020                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3021         }
3022
3023         /*
3024          * Disable Packet Checksum to enable RSS for multiple receive queues.
3025          * It is an adapter hardware limitation that Packet Checksum is
3026          * mutually exclusive with RSS.
3027          */
3028         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3029         rxcsum |= IXGBE_RXCSUM_PCSD;
3030         rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3031         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3032 }
3033
3034 /*
3035  * ixgbe_init_unicst - Initialize the unicast addresses.
3036  */
3037 static void
3038 ixgbe_init_unicst(ixgbe_t *ixgbe)
3039 {
3040         struct ixgbe_hw *hw = &ixgbe->hw;
3041         uint8_t *mac_addr;
3042         int slot;
3043         /*
3044          * Here we should consider two situations:
3045          *
3046          * 1. Chipset is initialized at the first time,
3047          *    Clear all the multiple unicast addresses.
3048          *
3049          * 2. Chipset is reset
3050          *    Recover the multiple unicast addresses from the
3051          *    software data structure to the RAR registers.
3052          */
3053         if (!ixgbe->unicst_init) {
3054                 /*
3055                  * Initialize the multiple unicast addresses
3056                  */
3057                 ixgbe->unicst_total = hw->mac.num_rar_entries;
3058                 ixgbe->unicst_avail = ixgbe->unicst_total;
3059                 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3060                         mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3061                         bzero(mac_addr, ETHERADDRL);
3062                         (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3063                         ixgbe->unicst_addr[slot].mac.set = 0;
3064                 }
3065                 ixgbe->unicst_init = B_TRUE;
3066         } else {
3067                 /* Re-configure the RAR registers */
3068                 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3069                         mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3070                         if (ixgbe->unicst_addr[slot].mac.set == 1) {
3071                                 (void) ixgbe_set_rar(hw, slot, mac_addr,
3072                                     ixgbe->unicst_addr[slot].mac.group_index,
3073                                     IXGBE_RAH_AV);
3074                         } else {
3075                                 bzero(mac_addr, ETHERADDRL);
3076                                 (void) ixgbe_set_rar(hw, slot, mac_addr,
3077                                     0, 0);
3078                         }
3079                 }
3080         }
3081 }
3082
3083 /*
3084  * ixgbe_unicst_find - Find the slot for the specified unicast address
3085  */
3086 int
3087 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3088 {
3089         int slot;
3090
3091         ASSERT(mutex_owned(&ixgbe->gen_lock));
3092
3093         for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3094                 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3095                     mac_addr, ETHERADDRL) == 0)
3096                         return (slot);
3097         }
3098
3099         return (-1);
3100 }
3101
3102 /*
3103  * ixgbe_multicst_add - Add a multicst address.
3104  */
3105 int
3106 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3107 {
3108         ASSERT(mutex_owned(&ixgbe->gen_lock));
3109
3110         if ((multiaddr[0] & 01) == 0) {
3111                 return (EINVAL);
3112         }
3113
3114         if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3115                 return (ENOENT);
3116         }
3117
3118         bcopy(multiaddr,
3119             &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3120         ixgbe->mcast_count++;
3121
3122         /*
3123          * Update the multicast table in the hardware
3124          */
3125         ixgbe_setup_multicst(ixgbe);
3126
3127         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3128                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3129                 return (EIO);
3130         }
3131
3132         return (0);
3133 }
3134
3135 /*
3136  * ixgbe_multicst_remove - Remove a multicst address.
3137  */
3138 int
3139 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3140 {
3141         int i;
3142
3143         ASSERT(mutex_owned(&ixgbe->gen_lock));
3144
3145         for (i = 0; i < ixgbe->mcast_count; i++) {
3146                 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3147                     ETHERADDRL) == 0) {
3148                         for (i++; i < ixgbe->mcast_count; i++) {
3149                                 ixgbe->mcast_table[i - 1] =
3150                                     ixgbe->mcast_table[i];
3151                         }
3152                         ixgbe->mcast_count--;
3153                         break;
3154                 }
3155         }
3156
3157         /*
3158          * Update the multicast table in the hardware
3159          */
3160         ixgbe_setup_multicst(ixgbe);
3161
3162         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3163                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3164                 return (EIO);
3165         }
3166
3167         return (0);
3168 }
3169
3170 /*
3171  * ixgbe_setup_multicast - Setup multicast data structures.
3172  *
3173  * This routine initializes all of the multicast related structures
3174  * and save them in the hardware registers.
3175  */
3176 static void
3177 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3178 {
3179         uint8_t *mc_addr_list;
3180         uint32_t mc_addr_count;
3181         struct ixgbe_hw *hw = &ixgbe->hw;
3182
3183         ASSERT(mutex_owned(&ixgbe->gen_lock));
3184
3185         ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3186
3187         mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3188         mc_addr_count = ixgbe->mcast_count;
3189
3190         /*
3191          * Update the multicast addresses to the MTA registers
3192          */
3193         (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3194             ixgbe_mc_table_itr, TRUE);
3195 }
3196
3197 /*
3198  * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3199  *
3200  * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3201  * Different chipsets may have different allowed configuration of vmdq and rss.
3202  */
3203 static void
3204 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3205 {
3206         struct ixgbe_hw *hw = &ixgbe->hw;
3207         uint32_t ring_per_group;
3208
3209         switch (hw->mac.type) {
3210         case ixgbe_mac_82598EB:
3211                 /*
3212                  * 82598 supports the following combination:
3213                  * vmdq no. x rss no.
3214                  * [5..16]  x 1
3215                  * [1..4]   x [1..16]
3216                  * However 8 rss queue per pool (vmdq) is sufficient for
3217                  * most cases.
3218                  */
3219                 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3220                 if (ixgbe->num_rx_groups > 4) {
3221                         ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3222                 } else {
3223                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3224                             min(8, ring_per_group);
3225                 }
3226
3227                 break;
3228
3229         case ixgbe_mac_82599EB:
3230         case ixgbe_mac_X540:
3231         case ixgbe_mac_X550:
3232         case ixgbe_mac_X550EM_x:
3233                 /*
3234                  * 82599 supports the following combination:
3235                  * vmdq no. x rss no.
3236                  * [33..64] x [1..2]
3237                  * [2..32]  x [1..4]
3238                  * 1 x [1..16]
3239                  * However 8 rss queue per pool (vmdq) is sufficient for
3240                  * most cases.
3241                  *
3242                  * For now, treat X540 and X550 like the 82599.
3243                  */
3244                 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3245                 if (ixgbe->num_rx_groups == 1) {
3246                         ixgbe->num_rx_rings = min(8, ring_per_group);
3247                 } else if (ixgbe->num_rx_groups <= 32) {
3248                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3249                             min(4, ring_per_group);
3250                 } else if (ixgbe->num_rx_groups <= 64) {
3251                         ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3252                             min(2, ring_per_group);
3253                 }
3254                 break;
3255
3256         default:
3257                 break;
3258         }
3259
3260         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3261
3262         if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3263                 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3264         } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3265                 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3266         } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3267                 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3268         } else {
3269                 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3270         }
3271
3272         IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3273             ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3274 }
3275
3276 /*
3277  * ixgbe_get_conf - Get driver configurations set in driver.conf.
3278  *
3279  * This routine gets user-configured values out of the configuration
3280  * file ixgbe.conf.
3281  *
3282  * For each configurable value, there is a minimum, a maximum, and a
3283  * default.
3284  * If user does not configure a value, use the default.
3285  * If user configures below the minimum, use the minumum.
3286  * If user configures above the maximum, use the maxumum.
3287  */
3288 static void
3289 ixgbe_get_conf(ixgbe_t *ixgbe)
3290 {
3291         struct ixgbe_hw *hw = &ixgbe->hw;
3292         uint32_t flow_control;
3293
3294         /*
3295          * ixgbe driver supports the following user configurations:
3296          *
3297          * Jumbo frame configuration:
3298          *    default_mtu
3299          *
3300          * Ethernet flow control configuration:
3301          *    flow_control
3302          *
3303          * Multiple rings configurations:
3304          *    tx_queue_number
3305          *    tx_ring_size
3306          *    rx_queue_number
3307          *    rx_ring_size
3308          *
3309          * Call ixgbe_get_prop() to get the value for a specific
3310          * configuration parameter.
3311          */
3312
3313         /*
3314          * Jumbo frame configuration - max_frame_size controls host buffer
3315          * allocation, so includes MTU, ethernet header, vlan tag and
3316          * frame check sequence.
3317          */
3318         ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3319             MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3320
3321         ixgbe->max_frame_size = ixgbe->default_mtu +
3322             sizeof (struct ether_vlan_header) + ETHERFCSL;
3323
3324         /*
3325          * Ethernet flow control configuration
3326          */
3327         flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3328             ixgbe_fc_none, 3, ixgbe_fc_none);
3329         if (flow_control == 3)
3330                 flow_control = ixgbe_fc_default;
3331
3332         /*
3333          * fc.requested mode is what the user requests.  After autoneg,
3334          * fc.current_mode will be the flow_control mode that was negotiated.
3335          */
3336         hw->fc.requested_mode = flow_control;
3337
3338         /*
3339          * Multiple rings configurations
3340          */
3341         ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3342             ixgbe->capab->min_tx_que_num,
3343             ixgbe->capab->max_tx_que_num,
3344             ixgbe->capab->def_tx_que_num);
3345         ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3346             MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3347
3348         ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3349             ixgbe->capab->min_rx_que_num,
3350             ixgbe->capab->max_rx_que_num,
3351             ixgbe->capab->def_rx_que_num);
3352         ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3353             MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3354
3355         /*
3356          * Multiple groups configuration
3357          */
3358         ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3359             ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3360             ixgbe->capab->def_rx_grp_num);
3361
3362         ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3363             0, 1, DEFAULT_MR_ENABLE);
3364
3365         if (ixgbe->mr_enable == B_FALSE) {
3366                 ixgbe->num_tx_rings = 1;
3367                 ixgbe->num_rx_rings = 1;
3368                 ixgbe->num_rx_groups = 1;
3369                 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3370         } else {
3371                 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3372                     max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3373                 /*
3374                  * The combination of num_rx_rings and num_rx_groups
3375                  * may be not supported by h/w. We need to adjust
3376                  * them to appropriate values.
3377                  */
3378                 ixgbe_setup_vmdq_rss_conf(ixgbe);
3379         }
3380
3381         /*
3382          * Tunable used to force an interrupt type. The only use is
3383          * for testing of the lesser interrupt types.
3384          * 0 = don't force interrupt type
3385          * 1 = force interrupt type MSI-X
3386          * 2 = force interrupt type MSI
3387          * 3 = force interrupt type Legacy
3388          */
3389         ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3390             IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3391
3392         ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3393             0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3394         ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3395             0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3396         ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3397             0, 1, DEFAULT_LSO_ENABLE);
3398         ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3399             0, 1, DEFAULT_LRO_ENABLE);
3400         ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3401             0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3402         ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3403             PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3404
3405         /* Head Write Back not recommended for 82599, X540 and X550 */
3406         if (hw->mac.type == ixgbe_mac_82599EB ||
3407             hw->mac.type == ixgbe_mac_X540 ||
3408             hw->mac.type == ixgbe_mac_X550 ||
3409             hw->mac.type == ixgbe_mac_X550EM_x) {
3410                 ixgbe->tx_head_wb_enable = B_FALSE;
3411         }
3412
3413         /*
3414          * ixgbe LSO needs the tx h/w checksum support.
3415          * LSO will be disabled if tx h/w checksum is not
3416          * enabled.
3417          */
3418         if (ixgbe->tx_hcksum_enable == B_FALSE) {
3419                 ixgbe->lso_enable = B_FALSE;
3420         }
3421
3422         /*
3423          * ixgbe LRO needs the rx h/w checksum support.
3424          * LRO will be disabled if rx h/w checksum is not
3425          * enabled.
3426          */
3427         if (ixgbe->rx_hcksum_enable == B_FALSE) {
3428                 ixgbe->lro_enable = B_FALSE;
3429         }
3430
3431         /*
3432          * ixgbe LRO only supported by 82599, X540 and X550
3433          */
3434         if (hw->mac.type == ixgbe_mac_82598EB) {
3435                 ixgbe->lro_enable = B_FALSE;
3436         }
3437         ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3438             MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3439             DEFAULT_TX_COPY_THRESHOLD);
3440         ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3441             PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3442             MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3443         ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3444             PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3445             MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3446         ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3447             PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3448             MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3449
3450         ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3451             MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3452             DEFAULT_RX_COPY_THRESHOLD);
3453         ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3454             MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3455             DEFAULT_RX_LIMIT_PER_INTR);
3456
3457         ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3458             ixgbe->capab->min_intr_throttle,
3459             ixgbe->capab->max_intr_throttle,
3460             ixgbe->capab->def_intr_throttle);
3461         /*
3462          * 82599, X540 and X550 require the interrupt throttling rate is
3463          * a multiple of 8. This is enforced by the register definiton.
3464          */
3465         if (hw->mac.type == ixgbe_mac_82599EB ||
3466             hw->mac.type == ixgbe_mac_X540 ||
3467             hw->mac.type == ixgbe_mac_X550 ||
3468             hw->mac.type == ixgbe_mac_X550EM_x)
3469                 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3470
3471         hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3472             PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3473 }
3474
3475 static void
3476 ixgbe_init_params(ixgbe_t *ixgbe)
3477 {
3478         struct ixgbe_hw *hw = &ixgbe->hw;
3479         ixgbe_link_speed speeds_supported = 0;
3480         bool negotiate;
3481
3482         /*
3483          * Get a list of speeds the adapter supports. If the hw struct hasn't
3484          * been populated with this information yet, retrieve it from the
3485          * adapter and save it to our own variable.
3486          *
3487          * On certain adapters, such as ones which use SFPs, the contents of
3488          * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3489          * updated, so we must rely on calling ixgbe_get_link_capabilities()
3490          * in order to ascertain the speeds which we are capable of supporting,
3491          * and in the case of SFP-equipped adapters, which speed we are
3492          * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3493          * we'll go with a default list of speeds as a last resort.
3494          */
3495         speeds_supported = hw->phy.speeds_supported;
3496
3497         if (speeds_supported == 0) {
3498                 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3499                     &negotiate) != IXGBE_SUCCESS) {
3500                         if (hw->mac.type == ixgbe_mac_82598EB) {
3501                                 speeds_supported =
3502                                     IXGBE_LINK_SPEED_82598_AUTONEG;
3503                         } else {
3504                                 speeds_supported =
3505                                     IXGBE_LINK_SPEED_82599_AUTONEG;
3506                         }
3507                 }
3508         }
3509         ixgbe->speeds_supported = speeds_supported;
3510
3511         /*
3512          * By default, all supported speeds are enabled and advertised.
3513          */
3514         if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3515                 ixgbe->param_en_10000fdx_cap = 1;
3516                 ixgbe->param_adv_10000fdx_cap = 1;
3517         } else {
3518                 ixgbe->param_en_10000fdx_cap = 0;
3519                 ixgbe->param_adv_10000fdx_cap = 0;
3520         }
3521
3522         if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3523                 ixgbe->param_en_5000fdx_cap = 1;
3524                 ixgbe->param_adv_5000fdx_cap = 1;
3525         } else {
3526                 ixgbe->param_en_5000fdx_cap = 0;
3527                 ixgbe->param_adv_5000fdx_cap = 0;
3528         }
3529
3530         if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3531                 ixgbe->param_en_2500fdx_cap = 1;
3532                 ixgbe->param_adv_2500fdx_cap = 1;
3533         } else {
3534                 ixgbe->param_en_2500fdx_cap = 0;
3535                 ixgbe->param_adv_2500fdx_cap = 0;
3536         }
3537
3538         if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3539                 ixgbe->param_en_1000fdx_cap = 1;
3540                 ixgbe->param_adv_1000fdx_cap = 1;
3541         } else {
3542                 ixgbe->param_en_1000fdx_cap = 0;
3543                 ixgbe->param_adv_1000fdx_cap = 0;
3544         }
3545
3546         if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3547                 ixgbe->param_en_100fdx_cap = 1;
3548                 ixgbe->param_adv_100fdx_cap = 1;
3549         } else {
3550                 ixgbe->param_en_100fdx_cap = 0;
3551                 ixgbe->param_adv_100fdx_cap = 0;
3552         }
3553
3554         ixgbe->param_pause_cap = 1;
3555         ixgbe->param_asym_pause_cap = 1;
3556         ixgbe->param_rem_fault = 0;
3557
3558         ixgbe->param_adv_autoneg_cap = 1;
3559         ixgbe->param_adv_pause_cap = 1;
3560         ixgbe->param_adv_asym_pause_cap = 1;
3561         ixgbe->param_adv_rem_fault = 0;
3562
3563         ixgbe->param_lp_10000fdx_cap = 0;
3564         ixgbe->param_lp_5000fdx_cap = 0;
3565         ixgbe->param_lp_2500fdx_cap = 0;
3566         ixgbe->param_lp_1000fdx_cap = 0;
3567         ixgbe->param_lp_100fdx_cap = 0;
3568         ixgbe->param_lp_autoneg_cap = 0;
3569         ixgbe->param_lp_pause_cap = 0;
3570         ixgbe->param_lp_asym_pause_cap = 0;
3571         ixgbe->param_lp_rem_fault = 0;
3572 }
3573
3574 /*
3575  * ixgbe_get_prop - Get a property value out of the configuration file
3576  * ixgbe.conf.
3577  *
3578  * Caller provides the name of the property, a default value, a minimum
3579  * value, and a maximum value.
3580  *
3581  * Return configured value of the property, with default, minimum and
3582  * maximum properly applied.
3583  */
3584 static int
3585 ixgbe_get_prop(ixgbe_t *ixgbe,
3586     char *propname,     /* name of the property */
3587     int minval,         /* minimum acceptable value */
3588     int maxval,         /* maximim acceptable value */
3589     int defval)         /* default value */
3590 {
3591         int value;
3592
3593         /*
3594          * Call ddi_prop_get_int() to read the conf settings
3595          */
3596         value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3597             DDI_PROP_DONTPASS, propname, defval);
3598         if (value > maxval)
3599                 value = maxval;
3600
3601         if (value < minval)
3602                 value = minval;
3603
3604         return (value);
3605 }
3606
3607 /*
3608  * ixgbe_driver_setup_link - Using the link properties to setup the link.
3609  */
3610 int
3611 ixgbe_driver_setup_link(ixgbe_t *ixgbe, bool setup_hw)
3612 {
3613         struct ixgbe_hw *hw = &ixgbe->hw;
3614         ixgbe_link_speed advertised = 0;
3615
3616         /*
3617          * Assemble a list of enabled speeds to auto-negotiate with.
3618          */
3619         if (ixgbe->param_en_10000fdx_cap == 1)
3620                 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3621
3622         if (ixgbe->param_en_5000fdx_cap == 1)
3623                 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3624
3625         if (ixgbe->param_en_2500fdx_cap == 1)
3626                 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3627
3628         if (ixgbe->param_en_1000fdx_cap == 1)
3629                 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3630
3631         if (ixgbe->param_en_100fdx_cap == 1)
3632                 advertised |= IXGBE_LINK_SPEED_100_FULL;
3633
3634         /*
3635          * As a last resort, autoneg with a default list of speeds.
3636          */
3637         if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3638                 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3639                     "to autonegotiate with full capabilities.");
3640
3641                 if (hw->mac.type == ixgbe_mac_82598EB)
3642                         advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3643                 else
3644                         advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3645         }
3646
3647         if (setup_hw) {
3648                 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3649                     ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3650                         ixgbe_notice(ixgbe, "Setup link failed on this "
3651                             "device.");
3652                         return (IXGBE_FAILURE);
3653                 }
3654         }
3655
3656         return (IXGBE_SUCCESS);
3657 }
3658
3659 /*
3660  * ixgbe_driver_link_check - Link status processing.
3661  *
3662  * This function can be called in both kernel context and interrupt context
3663  */
3664 static void
3665 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3666 {
3667         struct ixgbe_hw *hw = &ixgbe->hw;
3668         ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3669         bool link_up = B_FALSE;
3670         bool link_changed = B_FALSE;
3671
3672         ASSERT(mutex_owned(&ixgbe->gen_lock));
3673
3674         (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3675         if (link_up) {
3676                 ixgbe->link_check_complete = B_TRUE;
3677
3678                 /* Link is up, enable flow control settings */
3679                 (void) ixgbe_fc_enable(hw);
3680
3681                 /*
3682                  * The Link is up, check whether it was marked as down earlier
3683                  */
3684                 if (ixgbe->link_state != LINK_STATE_UP) {
3685                         switch (speed) {
3686                         case IXGBE_LINK_SPEED_10GB_FULL:
3687                                 ixgbe->link_speed = SPEED_10GB;
3688                                 break;
3689                         case IXGBE_LINK_SPEED_5GB_FULL:
3690                                 ixgbe->link_speed = SPEED_5GB;
3691                                 break;
3692                         case IXGBE_LINK_SPEED_2_5GB_FULL:
3693                                 ixgbe->link_speed = SPEED_2_5GB;
3694                                 break;
3695                         case IXGBE_LINK_SPEED_1GB_FULL:
3696                                 ixgbe->link_speed = SPEED_1GB;
3697                                 break;
3698                         case IXGBE_LINK_SPEED_100_FULL:
3699                                 ixgbe->link_speed = SPEED_100;
3700                         }
3701                         ixgbe->link_duplex = LINK_DUPLEX_FULL;
3702                         ixgbe->link_state = LINK_STATE_UP;
3703                         link_changed = B_TRUE;
3704                 }
3705         } else {
3706                 if (ixgbe->link_check_complete == B_TRUE ||
3707                     (ixgbe->link_check_complete == B_FALSE &&
3708                     gethrtime() >= ixgbe->link_check_hrtime)) {
3709                         /*
3710                          * The link is really down
3711                          */
3712                         ixgbe->link_check_complete = B_TRUE;
3713
3714                         if (ixgbe->link_state != LINK_STATE_DOWN) {
3715                                 ixgbe->link_speed = 0;
3716                                 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3717                                 ixgbe->link_state = LINK_STATE_DOWN;
3718                                 link_changed = B_TRUE;
3719                         }
3720                 }
3721         }
3722
3723         /*
3724          * If we are in an interrupt context, need to re-enable the
3725          * interrupt, which was automasked
3726          */
3727         if (servicing_interrupt() != 0) {
3728                 ixgbe->eims |= IXGBE_EICR_LSC;
3729                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3730         }
3731
3732         if (link_changed) {
3733                 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3734         }
3735 }
3736
3737 /*
3738  * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3739  */
3740 static void
3741 ixgbe_sfp_check(void *arg)
3742 {
3743         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3744         uint32_t eicr = ixgbe->eicr;
3745         struct ixgbe_hw *hw = &ixgbe->hw;
3746
3747         mutex_enter(&ixgbe->gen_lock);
3748         if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3749                 /* clear the interrupt */
3750                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3751
3752                 /* if link up, do multispeed fiber setup */
3753                 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3754                     B_TRUE);
3755                 ixgbe_driver_link_check(ixgbe);
3756                 ixgbe_get_hw_state(ixgbe);
3757         } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3758                 /* clear the interrupt */
3759                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3760
3761                 /* if link up, do sfp module setup */
3762                 (void) hw->mac.ops.setup_sfp(hw);
3763
3764                 /* do multispeed fiber setup */
3765                 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3766                     B_TRUE);
3767                 ixgbe_driver_link_check(ixgbe);
3768                 ixgbe_get_hw_state(ixgbe);
3769         }
3770         mutex_exit(&ixgbe->gen_lock);
3771
3772         /*
3773          * We need to fully re-check the link later.
3774          */
3775         ixgbe->link_check_complete = B_FALSE;
3776         ixgbe->link_check_hrtime = gethrtime() +
3777             (IXGBE_LINK_UP_TIME * 100000000ULL);
3778 }
3779
3780 /*
3781  * ixgbe_overtemp_check - overtemp module processing done in taskq
3782  *
3783  * This routine will only be called on adapters with temperature sensor.
3784  * The indication of over-temperature can be either SDP0 interrupt or the link
3785  * status change interrupt.
3786  */
3787 static void
3788 ixgbe_overtemp_check(void *arg)
3789 {
3790         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3791         struct ixgbe_hw *hw = &ixgbe->hw;
3792         uint32_t eicr = ixgbe->eicr;
3793         ixgbe_link_speed speed;
3794         bool link_up;
3795
3796         mutex_enter(&ixgbe->gen_lock);
3797
3798         /* make sure we know current state of link */
3799         (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3800
3801         /* check over-temp condition */
3802         if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3803             (eicr & IXGBE_EICR_LSC)) {
3804                 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3805                         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3806
3807                         /*
3808                          * Disable the adapter interrupts
3809                          */
3810                         ixgbe_disable_adapter_interrupts(ixgbe);
3811
3812                         /*
3813                          * Disable Rx/Tx units
3814                          */
3815                         (void) ixgbe_stop_adapter(hw);
3816
3817                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3818                         ixgbe_error(ixgbe,
3819                             "Problem: Network adapter has been stopped "
3820                             "because it has overheated");
3821                         ixgbe_error(ixgbe,
3822                             "Action: Restart the computer. "
3823                             "If the problem persists, power off the system "
3824                             "and replace the adapter");
3825                 }
3826         }
3827
3828         /* write to clear the interrupt */
3829         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3830
3831         mutex_exit(&ixgbe->gen_lock);
3832 }
3833
3834 /*
3835  * ixgbe_phy_check - taskq to process interrupts from an external PHY
3836  *
3837  * This routine will only be called on adapters with external PHYs
3838  * (such as X550) that may be trying to raise our attention to some event.
3839  * Currently, this is limited to claiming PHY overtemperature and link status
3840  * change (LSC) events, however this may expand to include other things in
3841  * future adapters.
3842  */
3843 static void
3844 ixgbe_phy_check(void *arg)
3845 {
3846         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3847         struct ixgbe_hw *hw = &ixgbe->hw;
3848         int rv;
3849
3850         mutex_enter(&ixgbe->gen_lock);
3851
3852         /*
3853          * X550 baseT PHY overtemp and LSC events are handled here.
3854          *
3855          * If an overtemp event occurs, it will be reflected in the
3856          * return value of phy.ops.handle_lasi() and the common code will
3857          * automatically power off the baseT PHY. This is our cue to trigger
3858          * an FMA event.
3859          *
3860          * If a link status change event occurs, phy.ops.handle_lasi() will
3861          * automatically initiate a link setup between the integrated KR PHY
3862          * and the external X557 PHY to ensure that the link speed between
3863          * them matches the link speed of the baseT link.
3864          */
3865         rv = ixgbe_handle_lasi(hw);
3866
3867         if (rv == IXGBE_ERR_OVERTEMP) {
3868                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3869
3870                 /*
3871                  * Disable the adapter interrupts
3872                  */
3873                 ixgbe_disable_adapter_interrupts(ixgbe);
3874
3875                 /*
3876                  * Disable Rx/Tx units
3877                  */
3878                 (void) ixgbe_stop_adapter(hw);
3879
3880                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3881                 ixgbe_error(ixgbe,
3882                     "Problem: Network adapter has been stopped due to a "
3883                     "overtemperature event being detected.");
3884                 ixgbe_error(ixgbe,
3885                     "Action: Shut down or restart the computer. If the issue "
3886                     "persists, please take action in accordance with the "
3887                     "recommendations from your system vendor.");
3888         }
3889
3890         mutex_exit(&ixgbe->gen_lock);
3891 }
3892
3893 /*
3894  * ixgbe_link_timer - timer for link status detection
3895  */
3896 static void
3897 ixgbe_link_timer(void *arg)
3898 {
3899         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3900
3901         mutex_enter(&ixgbe->gen_lock);
3902         ixgbe_driver_link_check(ixgbe);
3903         mutex_exit(&ixgbe->gen_lock);
3904 }
3905
3906 /*
3907  * ixgbe_local_timer - Driver watchdog function.
3908  *
3909  * This function will handle the transmit stall check and other routines.
3910  */
3911 static void
3912 ixgbe_local_timer(void *arg)
3913 {
3914         ixgbe_t *ixgbe = (ixgbe_t *)arg;
3915
3916         if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3917                 goto out;
3918
3919         if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3920                 ixgbe->reset_count++;
3921                 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3922                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3923                 goto out;
3924         }
3925
3926         if (ixgbe_stall_check(ixgbe)) {
3927                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3928                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3929
3930                 ixgbe->reset_count++;
3931                 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3932                         ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3933         }
3934
3935 out:
3936         ixgbe_restart_watchdog_timer(ixgbe);
3937 }
3938
3939 /*
3940  * ixgbe_stall_check - Check for transmit stall.
3941  *
3942  * This function checks if the adapter is stalled (in transmit).
3943  *
3944  * It is called each time the watchdog timeout is invoked.
3945  * If the transmit descriptor reclaim continuously fails,
3946  * the watchdog value will increment by 1. If the watchdog
3947  * value exceeds the threshold, the ixgbe is assumed to
3948  * have stalled and need to be reset.
3949  */
3950 static bool
3951 ixgbe_stall_check(ixgbe_t *ixgbe)
3952 {
3953         ixgbe_tx_ring_t *tx_ring;
3954         bool result;
3955         int i;
3956
3957         if (ixgbe->link_state != LINK_STATE_UP)
3958                 return (B_FALSE);
3959
3960         /*
3961          * If any tx ring is stalled, we'll reset the chipset
3962          */
3963         result = B_FALSE;
3964         for (i = 0; i < ixgbe->num_tx_rings; i++) {
3965                 tx_ring = &ixgbe->tx_rings[i];
3966                 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3967                         tx_ring->tx_recycle(tx_ring);
3968                 }
3969
3970                 if (tx_ring->recycle_fail > 0)
3971                         tx_ring->stall_watchdog++;
3972                 else
3973                         tx_ring->stall_watchdog = 0;
3974
3975                 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3976                         result = B_TRUE;
3977                         break;
3978                 }
3979         }
3980
3981         if (result) {
3982                 tx_ring->stall_watchdog = 0;
3983                 tx_ring->recycle_fail = 0;
3984         }
3985
3986         return (result);
3987 }
3988
3989
3990 /*
3991  * is_valid_mac_addr - Check if the mac address is valid.
3992  */
3993 static bool
3994 is_valid_mac_addr(uint8_t *mac_addr)
3995 {
3996         const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3997         const uint8_t addr_test2[6] =
3998             { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3999
4000         if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4001             !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4002                 return (B_FALSE);
4003
4004         return (B_TRUE);
4005 }
4006
4007 static bool
4008 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4009 {
4010         _NOTE(ARGUNUSED(ixgbe));
4011
4012         return (B_TRUE);
4013 }
4014
4015 #pragma inline(ixgbe_arm_watchdog_timer)
4016 static void
4017 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4018 {
4019         /*
4020          * Fire a watchdog timer
4021          */
4022         ixgbe->watchdog_tid =
4023             timeout(ixgbe_local_timer,
4024             (void *)ixgbe, 1 * drv_usectohz(1000000));
4025
4026 }
4027
4028 /*
4029  * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4030  */
4031 void
4032 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4033 {
4034         mutex_enter(&ixgbe->watchdog_lock);
4035
4036         if (!ixgbe->watchdog_enable) {
4037                 ixgbe->watchdog_enable = B_TRUE;
4038                 ixgbe->watchdog_start = B_TRUE;
4039                 ixgbe_arm_watchdog_timer(ixgbe);
4040         }
4041
4042         mutex_exit(&ixgbe->watchdog_lock);
4043 }
4044
4045 /*
4046  * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4047  */
4048 void
4049 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4050 {
4051         timeout_id_t tid;
4052
4053         mutex_enter(&ixgbe->watchdog_lock);
4054
4055         ixgbe->watchdog_enable = B_FALSE;
4056         ixgbe->watchdog_start = B_FALSE;
4057         tid = ixgbe->watchdog_tid;
4058         ixgbe->watchdog_tid = 0;
4059
4060         mutex_exit(&ixgbe->watchdog_lock);
4061
4062         if (tid != 0)
4063                 (void) untimeout(tid);
4064 }
4065
4066 /*
4067  * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4068  */
4069 void
4070 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4071 {
4072         mutex_enter(&ixgbe->watchdog_lock);
4073
4074         if (ixgbe->watchdog_enable) {
4075                 if (!ixgbe->watchdog_start) {
4076                         ixgbe->watchdog_start = B_TRUE;
4077                         ixgbe_arm_watchdog_timer(ixgbe);
4078                 }
4079         }
4080
4081         mutex_exit(&ixgbe->watchdog_lock);
4082 }
4083
4084 /*
4085  * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4086  */
4087 static void
4088 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4089 {
4090         mutex_enter(&ixgbe->watchdog_lock);
4091
4092         if (ixgbe->watchdog_start)
4093                 ixgbe_arm_watchdog_timer(ixgbe);
4094
4095         mutex_exit(&ixgbe->watchdog_lock);
4096 }
4097
4098 /*
4099  * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4100  */
4101 void
4102 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4103 {
4104         timeout_id_t tid;
4105
4106         mutex_enter(&ixgbe->watchdog_lock);
4107
4108         ixgbe->watchdog_start = B_FALSE;
4109         tid = ixgbe->watchdog_tid;
4110         ixgbe->watchdog_tid = 0;
4111
4112         mutex_exit(&ixgbe->watchdog_lock);
4113
4114         if (tid != 0)
4115                 (void) untimeout(tid);
4116 }
4117
4118 /*
4119  * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4120  */
4121 static void
4122 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4123 {
4124         struct ixgbe_hw *hw = &ixgbe->hw;
4125
4126         /*
4127          * mask all interrupts off
4128          */
4129         IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4130
4131         /*
4132          * for MSI-X, also disable autoclear
4133          */
4134         if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4135                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4136         }
4137
4138         IXGBE_WRITE_FLUSH(hw);
4139 }
4140
4141 /*
4142  * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4143  */
4144 static void
4145 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4146 {
4147         struct ixgbe_hw *hw = &ixgbe->hw;
4148         uint32_t eiac, eiam;
4149         uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4150
4151         /* interrupt types to enable */
4152         ixgbe->eims = IXGBE_EIMS_ENABLE_MASK;   /* shared code default */
4153         ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER;   /* minus tcp timer */
4154         ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4155
4156         /* enable automask on "other" causes that this adapter can generate */
4157         eiam = ixgbe->capab->other_intr;
4158
4159         /*
4160          * msi-x mode
4161          */
4162         if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4163                 /* enable autoclear but not on bits 29:20 */
4164                 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4165
4166                 /* general purpose interrupt enable */
4167                 gpie |= (IXGBE_GPIE_MSIX_MODE
4168                     | IXGBE_GPIE_PBA_SUPPORT
4169                     | IXGBE_GPIE_OCD
4170                     | IXGBE_GPIE_EIAME);
4171         /*
4172          * non-msi-x mode
4173          */
4174         } else {
4175
4176                 /* disable autoclear, leave gpie at default */
4177                 eiac = 0;
4178
4179                 /*
4180                  * General purpose interrupt enable.
4181                  * For 82599, X540 and X550, extended interrupt
4182                  * automask enable only in MSI or MSI-X mode
4183                  */
4184                 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4185                     (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4186                         gpie |= IXGBE_GPIE_EIAME;
4187                 }
4188         }
4189
4190         /* Enable specific "other" interrupt types */
4191         switch (hw->mac.type) {
4192         case ixgbe_mac_82598EB:
4193                 gpie |= ixgbe->capab->other_gpie;
4194                 break;
4195
4196         case ixgbe_mac_82599EB:
4197         case ixgbe_mac_X540:
4198         case ixgbe_mac_X550:
4199         case ixgbe_mac_X550EM_x:
4200                 gpie |= ixgbe->capab->other_gpie;
4201
4202                 /* Enable RSC Delay 8us when LRO enabled  */
4203                 if (ixgbe->lro_enable) {
4204                         gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4205                 }
4206                 break;
4207
4208         default:
4209                 break;
4210         }
4211
4212         /* write to interrupt control registers */
4213         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4214         IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4215         IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4216         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4217         IXGBE_WRITE_FLUSH(hw);
4218 }
4219
4220 /*
4221  * ixgbe_loopback_ioctl - Loopback support.
4222  */
4223 enum ioc_reply
4224 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4225 {
4226         lb_info_sz_t *lbsp;
4227         lb_property_t *lbpp;
4228         uint32_t *lbmp;
4229         uint32_t size;
4230         uint32_t value;
4231
4232         if (mp->b_cont == NULL)
4233                 return (IOC_INVAL);
4234
4235         switch (iocp->ioc_cmd) {
4236         default:
4237                 return (IOC_INVAL);
4238
4239         case LB_GET_INFO_SIZE:
4240                 size = sizeof (lb_info_sz_t);
4241                 if (iocp->ioc_count != size)
4242                         return (IOC_INVAL);
4243
4244                 value = sizeof (lb_normal);
4245                 value += sizeof (lb_mac);
4246                 value += sizeof (lb_external);
4247
4248                 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4249                 *lbsp = value;
4250                 break;
4251
4252         case LB_GET_INFO:
4253                 value = sizeof (lb_normal);
4254                 value += sizeof (lb_mac);
4255                 value += sizeof (lb_external);
4256
4257                 size = value;
4258                 if (iocp->ioc_count != size)
4259                         return (IOC_INVAL);
4260
4261                 value = 0;
4262                 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4263
4264                 lbpp[value++] = lb_normal;
4265                 lbpp[value++] = lb_mac;
4266                 lbpp[value++] = lb_external;
4267                 break;
4268
4269         case LB_GET_MODE:
4270                 size = sizeof (uint32_t);
4271                 if (iocp->ioc_count != size)
4272                         return (IOC_INVAL);
4273
4274                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4275                 *lbmp = ixgbe->loopback_mode;
4276                 break;
4277
4278         case LB_SET_MODE:
4279                 size = 0;
4280                 if (iocp->ioc_count != sizeof (uint32_t))
4281                         return (IOC_INVAL);
4282
4283                 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4284                 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4285                         return (IOC_INVAL);
4286                 break;
4287         }
4288
4289         iocp->ioc_count = size;
4290         iocp->ioc_error = 0;
4291
4292         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4293                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4294                 return (IOC_INVAL);
4295         }
4296
4297         return (IOC_REPLY);
4298 }
4299
4300 /*
4301  * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4302  */
4303 static bool
4304 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4305 {
4306         if (mode == ixgbe->loopback_mode)
4307                 return (B_TRUE);
4308
4309         ixgbe->loopback_mode = mode;
4310
4311         if (mode == IXGBE_LB_NONE) {
4312                 /*
4313                  * Reset the chip
4314                  */
4315                 (void) ixgbe_reset(ixgbe);
4316                 return (B_TRUE);
4317         }
4318
4319         mutex_enter(&ixgbe->gen_lock);
4320
4321         switch (mode) {
4322         default:
4323                 mutex_exit(&ixgbe->gen_lock);
4324                 return (B_FALSE);
4325
4326         case IXGBE_LB_EXTERNAL:
4327                 break;
4328
4329         case IXGBE_LB_INTERNAL_MAC:
4330                 ixgbe_set_internal_mac_loopback(ixgbe);
4331                 break;
4332         }
4333
4334         mutex_exit(&ixgbe->gen_lock);
4335
4336         return (B_TRUE);
4337 }
4338
4339 /*
4340  * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4341  */
4342 static void
4343 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4344 {
4345         struct ixgbe_hw *hw;
4346         uint32_t reg;
4347         uint8_t atlas;
4348
4349         hw = &ixgbe->hw;
4350
4351         /*
4352          * Setup MAC loopback
4353          */
4354         reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4355         reg |= IXGBE_HLREG0_LPBK;
4356         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4357
4358         reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4359         reg &= ~IXGBE_AUTOC_LMS_MASK;
4360         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4361
4362         /*
4363          * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4364          */
4365         switch (hw->mac.type) {
4366         case ixgbe_mac_82598EB:
4367                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4368                     &atlas);
4369                 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4370                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4371                     atlas);
4372
4373                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4374                     &atlas);
4375                 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4376                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4377                     atlas);
4378
4379                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4380                     &atlas);
4381                 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4382                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4383                     atlas);
4384
4385                 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4386                     &atlas);
4387                 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4388                 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4389                     atlas);
4390                 break;
4391
4392         case ixgbe_mac_82599EB:
4393         case ixgbe_mac_X540:
4394         case ixgbe_mac_X550:
4395         case ixgbe_mac_X550EM_x:
4396                 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4397                 reg |= (IXGBE_AUTOC_FLU |
4398                     IXGBE_AUTOC_10G_KX4);
4399                 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4400
4401                 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4402                     B_FALSE);
4403                 break;
4404
4405         default:
4406                 break;
4407         }
4408 }
4409
4410 #pragma inline(ixgbe_intr_rx_work)
4411 /*
4412  * ixgbe_intr_rx_work - RX processing of ISR.
4413  */
4414 static void
4415 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4416 {
4417         mblk_t *mp;
4418
4419         mutex_enter(&rx_ring->rx_lock);
4420
4421         mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4422         mutex_exit(&rx_ring->rx_lock);
4423
4424         if (mp != NULL)
4425                 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4426                     rx_ring->ring_gen_num);
4427 }
4428
4429 #pragma inline(ixgbe_intr_tx_work)
4430 /*
4431  * ixgbe_intr_tx_work - TX processing of ISR.
4432  */
4433 static void
4434 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4435 {
4436         ixgbe_t *ixgbe = tx_ring->ixgbe;
4437
4438         /*
4439          * Recycle the tx descriptors
4440          */
4441         tx_ring->tx_recycle(tx_ring);
4442
4443         /*
4444          * Schedule the re-transmit
4445          */
4446         if (tx_ring->reschedule &&
4447             (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4448                 tx_ring->reschedule = B_FALSE;
4449                 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4450                     tx_ring->ring_handle);
4451                 tx_ring->stat_reschedule++;
4452         }
4453 }
4454
4455 #pragma inline(ixgbe_intr_other_work)
4456 /*
4457  * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4458  */
4459 static void
4460 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4461 {
4462         struct ixgbe_hw *hw = &ixgbe->hw;
4463
4464         ASSERT(mutex_owned(&ixgbe->gen_lock));
4465
4466         /*
4467          * handle link status change
4468          */
4469         if (eicr & IXGBE_EICR_LSC) {
4470                 ixgbe_driver_link_check(ixgbe);
4471                 ixgbe_get_hw_state(ixgbe);
4472         }
4473
4474         /*
4475          * check for fan failure on adapters with fans
4476          */
4477         if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4478             (eicr & IXGBE_EICR_GPI_SDP1)) {
4479                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4480
4481                 /*
4482                  * Disable the adapter interrupts
4483                  */
4484                 ixgbe_disable_adapter_interrupts(ixgbe);
4485
4486                 /*
4487                  * Disable Rx/Tx units
4488                  */
4489                 (void) ixgbe_stop_adapter(&ixgbe->hw);
4490
4491                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4492                 ixgbe_error(ixgbe,
4493                     "Problem: Network adapter has been stopped "
4494                     "because the fan has stopped.\n");
4495                 ixgbe_error(ixgbe,
4496                     "Action: Replace the adapter.\n");
4497
4498                 /* re-enable the interrupt, which was automasked */
4499                 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4500         }
4501
4502         /*
4503          * Do SFP check for adapters with hot-plug capability
4504          */
4505         if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4506             ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4507             (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4508                 ixgbe->eicr = eicr;
4509                 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4510                     ixgbe_sfp_check, (void *)ixgbe,
4511                     DDI_NOSLEEP)) != DDI_SUCCESS) {
4512                         ixgbe_log(ixgbe, "No memory available to dispatch "
4513                             "taskq for SFP check");
4514                 }
4515         }
4516
4517         /*
4518          * Do over-temperature check for adapters with temp sensor
4519          */
4520         if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4521             ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4522             (eicr & IXGBE_EICR_LSC))) {
4523                 ixgbe->eicr = eicr;
4524                 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4525                     ixgbe_overtemp_check, (void *)ixgbe,
4526                     DDI_NOSLEEP)) != DDI_SUCCESS) {
4527                         ixgbe_log(ixgbe, "No memory available to dispatch "
4528                             "taskq for overtemp check");
4529                 }
4530         }
4531
4532         /*
4533          * Process an external PHY interrupt
4534          */
4535         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4536             (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4537                 ixgbe->eicr = eicr;
4538                 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4539                     ixgbe_phy_check, (void *)ixgbe,
4540                     DDI_NOSLEEP)) != DDI_SUCCESS) {
4541                         ixgbe_log(ixgbe, "No memory available to dispatch "
4542                             "taskq for PHY check");
4543                 }
4544         }
4545 }
4546
4547 /*
4548  * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4549  */
4550 static uint_t
4551 ixgbe_intr_legacy(void *arg1, void *arg2)
4552 {
4553         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4554         struct ixgbe_hw *hw = &ixgbe->hw;
4555         ixgbe_tx_ring_t *tx_ring;
4556         ixgbe_rx_ring_t *rx_ring;
4557         uint32_t eicr;
4558         mblk_t *mp;
4559         bool tx_reschedule;
4560         uint_t result;
4561
4562         _NOTE(ARGUNUSED(arg2));
4563
4564         mutex_enter(&ixgbe->gen_lock);
4565         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4566                 mutex_exit(&ixgbe->gen_lock);
4567                 return (DDI_INTR_UNCLAIMED);
4568         }
4569
4570         mp = NULL;
4571         tx_reschedule = B_FALSE;
4572
4573         /*
4574          * Any bit set in eicr: claim this interrupt
4575          */
4576         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4577
4578         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4579                 mutex_exit(&ixgbe->gen_lock);
4580                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4581                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4582                 return (DDI_INTR_CLAIMED);
4583         }
4584
4585         if (eicr) {
4586                 /*
4587                  * For legacy interrupt, we have only one interrupt,
4588                  * so we have only one rx ring and one tx ring enabled.
4589                  */
4590                 ASSERT(ixgbe->num_rx_rings == 1);
4591                 ASSERT(ixgbe->num_tx_rings == 1);
4592
4593                 /*
4594                  * For legacy interrupt, rx rings[0] will use RTxQ[0].
4595                  */
4596                 if (eicr & 0x1) {
4597                         ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4598                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4599                         ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4600                         /*
4601                          * Clean the rx descriptors
4602                          */
4603                         rx_ring = &ixgbe->rx_rings[0];
4604                         mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4605                 }
4606
4607                 /*
4608                  * For legacy interrupt, tx rings[0] will use RTxQ[1].
4609                  */
4610                 if (eicr & 0x2) {
4611                         /*
4612                          * Recycle the tx descriptors
4613                          */
4614                         tx_ring = &ixgbe->tx_rings[0];
4615                         tx_ring->tx_recycle(tx_ring);
4616
4617                         /*
4618                          * Schedule the re-transmit
4619                          */
4620                         tx_reschedule = (tx_ring->reschedule &&
4621                             (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4622                 }
4623
4624                 /* any interrupt type other than tx/rx */
4625                 if (eicr & ixgbe->capab->other_intr) {
4626                         switch (hw->mac.type) {
4627                         case ixgbe_mac_82598EB:
4628                                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4629                                 break;
4630
4631                         case ixgbe_mac_82599EB:
4632                         case ixgbe_mac_X540:
4633                         case ixgbe_mac_X550:
4634                         case ixgbe_mac_X550EM_x:
4635                                 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4636                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4637                                 break;
4638
4639                         default:
4640                                 break;
4641                         }
4642                         ixgbe_intr_other_work(ixgbe, eicr);
4643                         ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4644                 }
4645
4646                 mutex_exit(&ixgbe->gen_lock);
4647
4648                 result = DDI_INTR_CLAIMED;
4649         } else {
4650                 mutex_exit(&ixgbe->gen_lock);
4651
4652                 /*
4653                  * No interrupt cause bits set: don't claim this interrupt.
4654                  */
4655                 result = DDI_INTR_UNCLAIMED;
4656         }
4657
4658         /* re-enable the interrupts which were automasked */
4659         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4660
4661         /*
4662          * Do the following work outside of the gen_lock
4663          */
4664         if (mp != NULL) {
4665                 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4666                     rx_ring->ring_gen_num);
4667         }
4668
4669         if (tx_reschedule)  {
4670                 tx_ring->reschedule = B_FALSE;
4671                 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4672                 tx_ring->stat_reschedule++;
4673         }
4674
4675         return (result);
4676 }
4677
4678 /*
4679  * ixgbe_intr_msi - Interrupt handler for MSI.
4680  */
4681 static uint_t
4682 ixgbe_intr_msi(void *arg1, void *arg2)
4683 {
4684         ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4685         struct ixgbe_hw *hw = &ixgbe->hw;
4686         uint32_t eicr;
4687
4688         _NOTE(ARGUNUSED(arg2));
4689
4690         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4691
4692         if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4693                 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4694                 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4695                 return (DDI_INTR_CLAIMED);
4696         }
4697
4698         /*
4699          * For MSI interrupt, we have only one vector,
4700          * so we have only one rx ring and one tx ring enabled.
4701          */
4702         ASSERT(ixgbe->num_rx_rings == 1);
4703         ASSERT(ixgbe->num_tx_rings == 1);
4704
4705         /*
4706          * For MSI interrupt, rx rings[0] will use RTxQ[0].
4707          */
4708         if (eicr & 0x1) {
4709                 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4710         }
4711
4712         /*
4713          * For MSI interrupt, tx rings[0] will use RTxQ[1].
4714          */
4715         if (eicr & 0x2) {
4716                 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4717         }
4718
4719         /* any interrupt type other than tx/rx */
4720         if (eicr & ixgbe->capab->other_intr) {
4721                 mutex_enter(&ixgbe->gen_lock);
4722                 switch (hw->mac.type) {
4723                 case ixgbe_mac_82598EB:
4724                         ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4725                         break;
4726
4727                 case ixgbe_mac_82599EB:
4728                 case ixgbe_mac_X540:
4729                 case ixgbe_mac_X550:
4730                 case ixgbe_mac_X550EM_x:
4731                         ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4732                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4733                         break;
4734
4735                 default:
4736                         break;
4737                 }
4738                 ixgbe_intr_other_work(ixgbe, eicr);
4739                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4740                 mutex_exit(&ixgbe->gen_lock);
4741         }
4742
4743         /* re-enable the interrupts which were automasked */
4744         IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4745
4746         return (DDI_INTR_CLAIMED);
4747 }
4748
4749 /*
4750  * ixgbe_intr_msix - Interrupt handler for MSI-X.
4751  */
4752 static uint_t
4753 ixgbe_intr_msix(void *arg1, void *arg2)
4754 {
4755         ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4756         ixgbe_t *ixgbe = vect->ixgbe;
4757         struct ixgbe_hw *hw = &ixgbe->hw;
4758         uint32_t eicr;
4759         int r_idx = 0;
4760
4761         _NOTE(ARGUNUSED(arg2));
4762
4763         /*
4764          * Clean each rx ring that has its bit set in the map
4765          */
4766         r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4767         while (r_idx >= 0) {
4768                 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4769                 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4770                     (ixgbe->num_rx_rings - 1));
4771         }
4772
4773         /*
4774          * Clean each tx ring that has its bit set in the map
4775          */
4776         r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4777         while (r_idx >= 0) {
4778                 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4779                 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4780                     (ixgbe->num_tx_rings - 1));
4781         }
4782
4783
4784         /*
4785          * Clean other interrupt (link change) that has its bit set in the map
4786          */
4787         if (BT_TEST(vect->other_map, 0) == 1) {
4788                 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4789
4790                 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4791                     DDI_FM_OK) {
4792                         ddi_fm_service_impact(ixgbe->dip,
4793                             DDI_SERVICE_DEGRADED);
4794                         atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4795                         return (DDI_INTR_CLAIMED);
4796                 }
4797
4798                 /*
4799                  * Check "other" cause bits: any interrupt type other than tx/rx
4800                  */
4801                 if (eicr & ixgbe->capab->other_intr) {
4802                         mutex_enter(&ixgbe->gen_lock);
4803                         switch (hw->mac.type) {
4804                         case ixgbe_mac_82598EB:
4805                                 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4806                                 ixgbe_intr_other_work(ixgbe, eicr);
4807                                 break;
4808
4809                         case ixgbe_mac_82599EB:
4810                         case ixgbe_mac_X540:
4811                         case ixgbe_mac_X550:
4812                         case ixgbe_mac_X550EM_x:
4813                                 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4814                                 ixgbe_intr_other_work(ixgbe, eicr);
4815                                 break;
4816
4817                         default:
4818                                 break;
4819                         }
4820                         mutex_exit(&ixgbe->gen_lock);
4821                 }
4822
4823                 /* re-enable the interrupts which were automasked */
4824                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4825         }
4826
4827         return (DDI_INTR_CLAIMED);
4828 }
4829
4830 /*
4831  * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4832  *
4833  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4834  * if not successful, try Legacy.
4835  * ixgbe->intr_force can be used to force sequence to start with
4836  * any of the 3 types.
4837  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4838  */
4839 static int
4840 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4841 {
4842         dev_info_t *devinfo;
4843         int intr_types;
4844         int rc;
4845
4846         devinfo = ixgbe->dip;
4847
4848         /*
4849          * Get supported interrupt types
4850          */
4851         rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4852
4853         if (rc != DDI_SUCCESS) {
4854                 ixgbe_log(ixgbe,
4855                     "Get supported interrupt types failed: %d", rc);
4856                 return (IXGBE_FAILURE);
4857         }
4858         IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4859
4860         ixgbe->intr_type = 0;
4861
4862         /*
4863          * Install MSI-X interrupts
4864          */
4865         if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4866             (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4867                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4868                 if (rc == IXGBE_SUCCESS)
4869                         return (IXGBE_SUCCESS);
4870
4871                 ixgbe_log(ixgbe,
4872                     "Allocate MSI-X failed, trying MSI interrupts...");
4873         }
4874
4875         /*
4876          * MSI-X not used, force rings and groups to 1
4877          */
4878         ixgbe->num_rx_rings = 1;
4879         ixgbe->num_rx_groups = 1;
4880         ixgbe->num_tx_rings = 1;
4881         ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4882         ixgbe_log(ixgbe,
4883             "MSI-X not used, force rings and groups number to 1");
4884
4885         /*
4886          * Install MSI interrupts
4887          */
4888         if ((intr_types & DDI_INTR_TYPE_MSI) &&
4889             (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4890                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4891                 if (rc == IXGBE_SUCCESS)
4892                         return (IXGBE_SUCCESS);
4893
4894                 ixgbe_log(ixgbe,
4895                     "Allocate MSI failed, trying Legacy interrupts...");
4896         }
4897
4898         /*
4899          * Install legacy interrupts
4900          */
4901         if (intr_types & DDI_INTR_TYPE_FIXED) {
4902                 /*
4903                  * Disallow legacy interrupts for X550. X550 has a silicon
4904                  * bug which prevents Shared Legacy interrupts from working.
4905                  * For details, please reference:
4906                  *
4907                  * Intel Ethernet Controller X550 Specification Update rev. 2.1
4908                  * May 2016, erratum 22: PCIe Interrupt Status Bit
4909                  */
4910                 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
4911                     ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
4912                     ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
4913                     ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf) {
4914                         ixgbe_log(ixgbe,
4915                             "Legacy interrupts are not supported on this "
4916                             "adapter. Please use MSI or MSI-X instead.");
4917                         return (IXGBE_FAILURE);
4918                 }
4919                 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4920                 if (rc == IXGBE_SUCCESS)
4921                         return (IXGBE_SUCCESS);
4922
4923                 ixgbe_log(ixgbe,
4924                     "Allocate Legacy interrupts failed");
4925         }
4926
4927         /*
4928          * If none of the 3 types succeeded, return failure
4929          */
4930         return (IXGBE_FAILURE);
4931 }
4932
4933 /*
4934  * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4935  *
4936  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4937  * if fewer than 2 handles are available, return failure.
4938  * Upon success, this maps the vectors to rx and tx rings for
4939  * interrupts.
4940  */
4941 static int
4942 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4943 {
4944         dev_info_t *devinfo;
4945         int request, count, actual;
4946         int minimum;
4947         int rc;
4948         uint32_t ring_per_group;
4949
4950         devinfo = ixgbe->dip;
4951
4952         switch (intr_type) {
4953         case DDI_INTR_TYPE_FIXED:
4954                 request = 1;    /* Request 1 legacy interrupt handle */
4955                 minimum = 1;
4956                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4957                 break;
4958
4959         case DDI_INTR_TYPE_MSI:
4960                 request = 1;    /* Request 1 MSI interrupt handle */
4961                 minimum = 1;
4962                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4963                 break;
4964
4965         case DDI_INTR_TYPE_MSIX:
4966                 /*
4967                  * Best number of vectors for the adapter is
4968                  * (# rx rings + # tx rings), however we will
4969                  * limit the request number.
4970                  */
4971                 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4972                 if (request > ixgbe->capab->max_ring_vect)
4973                         request = ixgbe->capab->max_ring_vect;
4974                 minimum = 1;
4975                 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4976                 break;
4977
4978         default:
4979                 ixgbe_log(ixgbe,
4980                     "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4981                     intr_type);
4982                 return (IXGBE_FAILURE);
4983         }
4984         IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d  minimum: %d",
4985             request, minimum);
4986
4987         /*
4988          * Get number of supported interrupts
4989          */
4990         rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4991         if ((rc != DDI_SUCCESS) || (count < minimum)) {
4992                 ixgbe_log(ixgbe,
4993                     "Get interrupt number failed. Return: %d, count: %d",
4994                     rc, count);
4995                 return (IXGBE_FAILURE);
4996         }
4997         IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4998
4999         actual = 0;
5000         ixgbe->intr_cnt = 0;
5001         ixgbe->intr_cnt_max = 0;
5002         ixgbe->intr_cnt_min = 0;
5003
5004         /*
5005          * Allocate an array of interrupt handles
5006          */
5007         ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5008         ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5009
5010         rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5011             request, &actual, DDI_INTR_ALLOC_NORMAL);
5012         if (rc != DDI_SUCCESS) {
5013                 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5014                     "return: %d, request: %d, actual: %d",
5015                     rc, request, actual);
5016                 goto alloc_handle_fail;
5017         }
5018         IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5019
5020         /*
5021          * upper/lower limit of interrupts
5022          */
5023         ixgbe->intr_cnt = actual;
5024         ixgbe->intr_cnt_max = request;
5025         ixgbe->intr_cnt_min = minimum;
5026
5027         /*
5028          * rss number per group should not exceed the rx interrupt number,
5029          * else need to adjust rx ring number.
5030          */
5031         ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5032         ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5033         if (actual < ring_per_group) {
5034                 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5035                 ixgbe_setup_vmdq_rss_conf(ixgbe);
5036         }
5037
5038         /*
5039          * Now we know the actual number of vectors.  Here we map the vector
5040          * to other, rx rings and tx ring.
5041          */
5042         if (actual < minimum) {
5043                 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5044                     actual);
5045                 goto alloc_handle_fail;
5046         }
5047
5048         /*
5049          * Get priority for first vector, assume remaining are all the same
5050          */
5051         rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5052         if (rc != DDI_SUCCESS) {
5053                 ixgbe_log(ixgbe,
5054                     "Get interrupt priority failed: %d", rc);
5055                 goto alloc_handle_fail;
5056         }
5057
5058         rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5059         if (rc != DDI_SUCCESS) {
5060                 ixgbe_log(ixgbe,
5061                     "Get interrupt cap failed: %d", rc);
5062                 goto alloc_handle_fail;
5063         }
5064
5065         ixgbe->intr_type = intr_type;
5066
5067         return (IXGBE_SUCCESS);
5068
5069 alloc_handle_fail:
5070         ixgbe_rem_intrs(ixgbe);
5071
5072         return (IXGBE_FAILURE);
5073 }
5074
5075 /*
5076  * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5077  *
5078  * Before adding the interrupt handlers, the interrupt vectors have
5079  * been allocated, and the rx/tx rings have also been allocated.
5080  */
5081 static int
5082 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5083 {
5084         int vector = 0;
5085         int rc;
5086
5087         switch (ixgbe->intr_type) {
5088         case DDI_INTR_TYPE_MSIX:
5089                 /*
5090                  * Add interrupt handler for all vectors
5091                  */
5092                 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5093                         /*
5094                          * install pointer to vect_map[vector]
5095                          */
5096                         rc = ddi_intr_add_handler(ixgbe->htable[vector],
5097                             (ddi_intr_handler_t *)ixgbe_intr_msix,
5098                             (void *)&ixgbe->vect_map[vector], NULL);
5099
5100                         if (rc != DDI_SUCCESS) {
5101                                 ixgbe_log(ixgbe,
5102                                     "Add interrupt handler failed. "
5103                                     "return: %d, vector: %d", rc, vector);
5104                                 for (vector--; vector >= 0; vector--) {
5105                                         (void) ddi_intr_remove_handler(
5106                                             ixgbe->htable[vector]);
5107                                 }
5108                                 return (IXGBE_FAILURE);
5109                         }
5110                 }
5111
5112                 break;
5113
5114         case DDI_INTR_TYPE_MSI:
5115                 /*
5116                  * Add interrupt handlers for the only vector
5117                  */
5118                 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5119                     (ddi_intr_handler_t *)ixgbe_intr_msi,
5120                     (void *)ixgbe, NULL);
5121
5122                 if (rc != DDI_SUCCESS) {
5123                         ixgbe_log(ixgbe,
5124                             "Add MSI interrupt handler failed: %d", rc);
5125                         return (IXGBE_FAILURE);
5126                 }
5127
5128                 break;
5129
5130         case DDI_INTR_TYPE_FIXED:
5131                 /*
5132                  * Add interrupt handlers for the only vector
5133                  */
5134                 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5135                     (ddi_intr_handler_t *)ixgbe_intr_legacy,
5136                     (void *)ixgbe, NULL);
5137
5138                 if (rc != DDI_SUCCESS) {
5139                         ixgbe_log(ixgbe,
5140                             "Add legacy interrupt handler failed: %d", rc);
5141                         return (IXGBE_FAILURE);
5142                 }
5143
5144                 break;
5145
5146         default:
5147                 return (IXGBE_FAILURE);
5148         }
5149
5150         return (IXGBE_SUCCESS);
5151 }
5152
5153 #pragma inline(ixgbe_map_rxring_to_vector)
5154 /*
5155  * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5156  */
5157 static void
5158 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5159 {
5160         /*
5161          * Set bit in map
5162          */
5163         BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5164
5165         /*
5166          * Count bits set
5167          */
5168         ixgbe->vect_map[v_idx].rxr_cnt++;
5169
5170         /*
5171          * Remember bit position
5172          */
5173         ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5174         ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5175 }
5176
5177 #pragma inline(ixgbe_map_txring_to_vector)
5178 /*
5179  * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5180  */
5181 static void
5182 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5183 {
5184         /*
5185          * Set bit in map
5186          */
5187         BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5188
5189         /*
5190          * Count bits set
5191          */
5192         ixgbe->vect_map[v_idx].txr_cnt++;
5193
5194         /*
5195          * Remember bit position
5196          */
5197         ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5198         ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5199 }
5200
5201 /*
5202  * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5203  * allocation register (IVAR).
5204  * cause:
5205  *   -1 : other cause
5206  *    0 : rx
5207  *    1 : tx
5208  */
5209 static void
5210 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5211     int8_t cause)
5212 {
5213         struct ixgbe_hw *hw = &ixgbe->hw;
5214         u32 ivar, index;
5215
5216         switch (hw->mac.type) {
5217         case ixgbe_mac_82598EB:
5218                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5219                 if (cause == -1) {
5220                         cause = 0;
5221                 }
5222                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5223                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5224                 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5225                 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5226                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5227                 break;
5228
5229         case ixgbe_mac_82599EB:
5230         case ixgbe_mac_X540:
5231         case ixgbe_mac_X550:
5232         case ixgbe_mac_X550EM_x:
5233                 if (cause == -1) {
5234                         /* other causes */
5235                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5236                         index = (intr_alloc_entry & 1) * 8;
5237                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5238                         ivar &= ~(0xFF << index);
5239                         ivar |= (msix_vector << index);
5240                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5241                 } else {
5242                         /* tx or rx causes */
5243                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5244                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5245                         ivar = IXGBE_READ_REG(hw,
5246                             IXGBE_IVAR(intr_alloc_entry >> 1));
5247                         ivar &= ~(0xFF << index);
5248                         ivar |= (msix_vector << index);
5249                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5250                             ivar);
5251                 }
5252                 break;
5253
5254         default:
5255                 break;
5256         }
5257 }
5258
5259 /*
5260  * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5261  * given interrupt vector allocation register (IVAR).
5262  * cause:
5263  *   -1 : other cause
5264  *    0 : rx
5265  *    1 : tx
5266  */
5267 static void
5268 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5269 {
5270         struct ixgbe_hw *hw = &ixgbe->hw;
5271         u32 ivar, index;
5272
5273         switch (hw->mac.type) {
5274         case ixgbe_mac_82598EB:
5275                 if (cause == -1) {
5276                         cause = 0;
5277                 }
5278                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5279                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5280                 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5281                     (intr_alloc_entry & 0x3)));
5282                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5283                 break;
5284
5285         case ixgbe_mac_82599EB:
5286         case ixgbe_mac_X540:
5287         case ixgbe_mac_X550:
5288         case ixgbe_mac_X550EM_x:
5289                 if (cause == -1) {
5290                         /* other causes */
5291                         index = (intr_alloc_entry & 1) * 8;
5292                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5293                         ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5294                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5295                 } else {
5296                         /* tx or rx causes */
5297                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5298                         ivar = IXGBE_READ_REG(hw,
5299                             IXGBE_IVAR(intr_alloc_entry >> 1));
5300                         ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5301                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5302                             ivar);
5303                 }
5304                 break;
5305
5306         default:
5307                 break;
5308         }
5309 }
5310
5311 /*
5312  * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5313  * given interrupt vector allocation register (IVAR).
5314  * cause:
5315  *   -1 : other cause
5316  *    0 : rx
5317  *    1 : tx
5318  */
5319 static void
5320 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5321 {
5322         struct ixgbe_hw *hw = &ixgbe->hw;
5323         u32 ivar, index;
5324
5325         switch (hw->mac.type) {
5326         case ixgbe_mac_82598EB:
5327                 if (cause == -1) {
5328                         cause = 0;
5329                 }
5330                 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5331                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5332                 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5333                     (intr_alloc_entry & 0x3)));
5334                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5335                 break;
5336
5337         case ixgbe_mac_82599EB:
5338         case ixgbe_mac_X540:
5339         case ixgbe_mac_X550:
5340         case ixgbe_mac_X550EM_x:
5341                 if (cause == -1) {
5342                         /* other causes */
5343                         index = (intr_alloc_entry & 1) * 8;
5344                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5345                         ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5346                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5347                 } else {
5348                         /* tx or rx causes */
5349                         index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5350                         ivar = IXGBE_READ_REG(hw,
5351                             IXGBE_IVAR(intr_alloc_entry >> 1));
5352                         ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5353                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5354                             ivar);
5355                 }
5356                 break;
5357
5358         default:
5359                 break;
5360         }
5361 }
5362
5363 /*
5364  * Convert the rx ring index driver maintained to the rx ring index
5365  * in h/w.
5366  */
5367 static uint32_t
5368 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5369 {
5370
5371         struct ixgbe_hw *hw = &ixgbe->hw;
5372         uint32_t rx_ring_per_group, hw_rx_index;
5373
5374         if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5375             ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5376                 return (sw_rx_index);
5377         } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5378                 switch (hw->mac.type) {
5379                 case ixgbe_mac_82598EB:
5380                         return (sw_rx_index);
5381
5382                 case ixgbe_mac_82599EB:
5383                 case ixgbe_mac_X540:
5384                 case ixgbe_mac_X550:
5385                 case ixgbe_mac_X550EM_x:
5386                         return (sw_rx_index * 2);
5387
5388                 default:
5389                         break;
5390                 }
5391         } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5392                 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5393
5394                 switch (hw->mac.type) {
5395                 case ixgbe_mac_82598EB:
5396                         hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5397                             16 + (sw_rx_index % rx_ring_per_group);
5398                         return (hw_rx_index);
5399
5400                 case ixgbe_mac_82599EB:
5401                 case ixgbe_mac_X540:
5402                 case ixgbe_mac_X550:
5403                 case ixgbe_mac_X550EM_x:
5404                         if (ixgbe->num_rx_groups > 32) {
5405                                 hw_rx_index = (sw_rx_index /
5406                                     rx_ring_per_group) * 2 +
5407                                     (sw_rx_index % rx_ring_per_group);
5408                         } else {
5409                                 hw_rx_index = (sw_rx_index /
5410                                     rx_ring_per_group) * 4 +
5411                                     (sw_rx_index % rx_ring_per_group);
5412                         }
5413                         return (hw_rx_index);
5414
5415                 default:
5416                         break;
5417                 }
5418         }
5419
5420         /*
5421          * Should never reach. Just to make compiler happy.
5422          */
5423         return (sw_rx_index);
5424 }
5425
5426 /*
5427  * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5428  *
5429  * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5430  * to vector[0 - (intr_cnt -1)].
5431  */
5432 static int
5433 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5434 {
5435         int i, vector = 0;
5436
5437         /* initialize vector map */
5438         bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5439         for (i = 0; i < ixgbe->intr_cnt; i++) {
5440                 ixgbe->vect_map[i].ixgbe = ixgbe;
5441         }
5442
5443         /*
5444          * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5445          * tx rings[0] on RTxQ[1].
5446          */
5447         if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5448                 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5449                 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5450                 return (IXGBE_SUCCESS);
5451         }
5452
5453         /*
5454          * Interrupts/vectors mapping for MSI-X
5455          */
5456
5457         /*
5458          * Map other interrupt to vector 0,
5459          * Set bit in map and count the bits set.
5460          */
5461         BT_SET(ixgbe->vect_map[vector].other_map, 0);
5462         ixgbe->vect_map[vector].other_cnt++;
5463
5464         /*
5465          * Map rx ring interrupts to vectors
5466          */
5467         for (i = 0; i < ixgbe->num_rx_rings; i++) {
5468                 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5469                 vector = (vector +1) % ixgbe->intr_cnt;
5470         }
5471
5472         /*
5473          * Map tx ring interrupts to vectors
5474          */
5475         for (i = 0; i < ixgbe->num_tx_rings; i++) {
5476                 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5477                 vector = (vector +1) % ixgbe->intr_cnt;
5478         }
5479
5480         return (IXGBE_SUCCESS);
5481 }
5482
5483 /*
5484  * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5485  *
5486  * This relies on ring/vector mapping already set up in the
5487  * vect_map[] structures
5488  */
5489 static void
5490 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5491 {
5492         struct ixgbe_hw *hw = &ixgbe->hw;
5493         ixgbe_intr_vector_t *vect;      /* vector bitmap */
5494         int r_idx;      /* ring index */
5495         int v_idx;      /* vector index */
5496         uint32_t hw_index;
5497
5498         /*
5499          * Clear any previous entries
5500          */
5501         switch (hw->mac.type) {
5502         case ixgbe_mac_82598EB:
5503                 for (v_idx = 0; v_idx < 25; v_idx++)
5504                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5505                 break;
5506
5507         case ixgbe_mac_82599EB:
5508         case ixgbe_mac_X540:
5509         case ixgbe_mac_X550:
5510         case ixgbe_mac_X550EM_x:
5511                 for (v_idx = 0; v_idx < 64; v_idx++)
5512                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5513                 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5514                 break;
5515
5516         default:
5517                 break;
5518         }
5519
5520         /*
5521          * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5522          * tx rings[0] will use RTxQ[1].
5523          */
5524         if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5525                 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5526                 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5527                 return;
5528         }
5529
5530         /*
5531          * For MSI-X interrupt, "Other" is always on vector[0].
5532          */
5533         ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5534
5535         /*
5536          * For each interrupt vector, populate the IVAR table
5537          */
5538         for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5539                 vect = &ixgbe->vect_map[v_idx];
5540
5541                 /*
5542                  * For each rx ring bit set
5543                  */
5544                 r_idx = bt_getlowbit(vect->rx_map, 0,
5545                     (ixgbe->num_rx_rings - 1));
5546
5547                 while (r_idx >= 0) {
5548                         hw_index = ixgbe->rx_rings[r_idx].hw_index;
5549                         ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5550                         r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5551                             (ixgbe->num_rx_rings - 1));
5552                 }
5553
5554                 /*
5555                  * For each tx ring bit set
5556                  */
5557                 r_idx = bt_getlowbit(vect->tx_map, 0,
5558                     (ixgbe->num_tx_rings - 1));
5559
5560                 while (r_idx >= 0) {
5561                         ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5562                         r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5563                             (ixgbe->num_tx_rings - 1));
5564                 }
5565         }
5566 }
5567
5568 /*
5569  * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5570  */
5571 static void
5572 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5573 {
5574         int i;
5575         int rc;
5576
5577         for (i = 0; i < ixgbe->intr_cnt; i++) {
5578                 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5579                 if (rc != DDI_SUCCESS) {
5580                         IXGBE_DEBUGLOG_1(ixgbe,
5581                             "Remove intr handler failed: %d", rc);
5582                 }
5583         }
5584 }
5585
5586 /*
5587  * ixgbe_rem_intrs - Remove the allocated interrupts.
5588  */
5589 static void
5590 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5591 {
5592         int i;
5593         int rc;
5594
5595         for (i = 0; i < ixgbe->intr_cnt; i++) {
5596                 rc = ddi_intr_free(ixgbe->htable[i]);
5597                 if (rc != DDI_SUCCESS) {
5598                         IXGBE_DEBUGLOG_1(ixgbe,
5599                             "Free intr failed: %d", rc);
5600                 }
5601         }
5602
5603         kmem_free(ixgbe->htable, ixgbe->intr_size);
5604         ixgbe->htable = NULL;
5605 }
5606
5607 /*
5608  * ixgbe_enable_intrs - Enable all the ddi interrupts.
5609  */
5610 static int
5611 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5612 {
5613         int i;
5614         int rc;
5615
5616         /*
5617          * Enable interrupts
5618          */
5619         if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5620                 /*
5621                  * Call ddi_intr_block_enable() for MSI
5622                  */
5623                 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5624                 if (rc != DDI_SUCCESS) {
5625                         ixgbe_log(ixgbe,
5626                             "Enable block intr failed: %d", rc);
5627                         return (IXGBE_FAILURE);
5628                 }
5629         } else {
5630                 /*
5631                  * Call ddi_intr_enable() for Legacy/MSI non block enable
5632                  */
5633                 for (i = 0; i < ixgbe->intr_cnt; i++) {
5634                         rc = ddi_intr_enable(ixgbe->htable[i]);
5635                         if (rc != DDI_SUCCESS) {
5636                                 ixgbe_log(ixgbe,
5637                                     "Enable intr failed: %d", rc);
5638                                 return (IXGBE_FAILURE);
5639                         }
5640                 }
5641         }
5642
5643         return (IXGBE_SUCCESS);
5644 }
5645
5646 /*
5647  * ixgbe_disable_intrs - Disable all the interrupts.
5648  */
5649 static int
5650 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5651 {
5652         int i;
5653         int rc;
5654
5655         /*
5656          * Disable all interrupts
5657          */
5658         if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5659                 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5660                 if (rc != DDI_SUCCESS) {
5661                         ixgbe_log(ixgbe,
5662                             "Disable block intr failed: %d", rc);
5663                         return (IXGBE_FAILURE);
5664                 }
5665         } else {
5666                 for (i = 0; i < ixgbe->intr_cnt; i++) {
5667                         rc = ddi_intr_disable(ixgbe->htable[i]);
5668                         if (rc != DDI_SUCCESS) {
5669                                 ixgbe_log(ixgbe,
5670                                     "Disable intr failed: %d", rc);
5671                                 return (IXGBE_FAILURE);
5672                         }
5673                 }
5674         }
5675
5676         return (IXGBE_SUCCESS);
5677 }
5678
5679 /*
5680  * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5681  */
5682 static void
5683 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5684 {
5685         struct ixgbe_hw *hw = &ixgbe->hw;
5686         ixgbe_link_speed speed = 0;
5687         bool link_up = B_FALSE;
5688         uint32_t pcs1g_anlp = 0;
5689
5690         ASSERT(mutex_owned(&ixgbe->gen_lock));
5691         ixgbe->param_lp_1000fdx_cap = 0;
5692         ixgbe->param_lp_100fdx_cap  = 0;
5693
5694         /* check for link, don't wait */
5695         (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5696
5697         /*
5698          * Update the observed Link Partner's capabilities. Not all adapters
5699          * can provide full information on the LP's capable speeds, so we
5700          * provide what we can.
5701          */
5702         if (link_up) {
5703                 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5704
5705                 ixgbe->param_lp_1000fdx_cap =
5706                     (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5707                 ixgbe->param_lp_100fdx_cap =
5708                     (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5709         }
5710
5711         /*
5712          * Update GLD's notion of the adapter's currently advertised speeds.
5713          * Since the common code doesn't always record the current autonegotiate
5714          * settings in the phy struct for all parts (specifically, adapters with
5715          * SFPs) we first test to see if it is 0, and if so, we fall back to
5716          * using the adapter's speed capabilities which we saved during instance
5717          * init in ixgbe_init_params().
5718          *
5719          * Adapters with SFPs will always be shown as advertising all of their
5720          * supported speeds, and adapters with baseT PHYs (where the phy struct
5721          * is maintained by the common code) will always have a factual view of
5722          * their currently-advertised speeds. In the case of SFPs, this is
5723          * acceptable as we default to advertising all speeds that the adapter
5724          * claims to support, and those properties are immutable; unlike on
5725          * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5726          */
5727         speed = hw->phy.autoneg_advertised;
5728         if (speed == 0)
5729                 speed = ixgbe->speeds_supported;
5730
5731         ixgbe->param_adv_10000fdx_cap =
5732             (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5733         ixgbe->param_adv_5000fdx_cap =
5734             (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5735         ixgbe->param_adv_2500fdx_cap =
5736             (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5737         ixgbe->param_adv_1000fdx_cap =
5738             (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5739         ixgbe->param_adv_100fdx_cap =
5740             (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5741 }
5742
5743 /*
5744  * ixgbe_get_driver_control - Notify that driver is in control of device.
5745  */
5746 static void
5747 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5748 {
5749         uint32_t ctrl_ext;
5750
5751         /*
5752          * Notify firmware that driver is in control of device
5753          */
5754         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5755         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5756         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5757 }
5758
5759 /*
5760  * ixgbe_release_driver_control - Notify that driver is no longer in control
5761  * of device.
5762  */
5763 static void
5764 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5765 {
5766         uint32_t ctrl_ext;
5767
5768         /*
5769          * Notify firmware that driver is no longer in control of device
5770          */
5771         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5772         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5773         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5774 }
5775
5776 /*
5777  * ixgbe_atomic_reserve - Atomic decrease operation.
5778  */
5779 int
5780 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5781 {
5782         uint32_t oldval;
5783         uint32_t newval;
5784
5785         /*
5786          * ATOMICALLY
5787          */
5788         do {
5789                 oldval = *count_p;
5790                 if (oldval < n)
5791                         return (-1);
5792                 newval = oldval - n;
5793         } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5794
5795         return (newval);
5796 }
5797
5798 /*
5799  * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5800  */
5801 static uint8_t *
5802 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5803 {
5804         uint8_t *addr = *upd_ptr;
5805         uint8_t *new_ptr;
5806
5807         _NOTE(ARGUNUSED(hw));
5808         _NOTE(ARGUNUSED(vmdq));
5809
5810         new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5811         *upd_ptr = new_ptr;
5812         return (addr);
5813 }
5814
5815 /*
5816  * FMA support
5817  */
5818 int
5819 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5820 {
5821         ddi_fm_error_t de;
5822
5823         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5824         ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5825         return (de.fme_status);
5826 }
5827
5828 int
5829 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5830 {
5831         ddi_fm_error_t de;
5832
5833         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5834         return (de.fme_status);
5835 }
5836
5837 /*
5838  * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5839  */
5840 static int
5841 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5842 {
5843         _NOTE(ARGUNUSED(impl_data));
5844         /*
5845          * as the driver can always deal with an error in any dma or
5846          * access handle, we can just return the fme_status value.
5847          */
5848         pci_ereport_post(dip, err, NULL);
5849         return (err->fme_status);
5850 }
5851
5852 static void
5853 ixgbe_fm_init(ixgbe_t *ixgbe)
5854 {
5855         ddi_iblock_cookie_t iblk;
5856         int fma_dma_flag;
5857
5858         /*
5859          * Only register with IO Fault Services if we have some capability
5860          */
5861         if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5862                 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5863         } else {
5864                 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5865         }
5866
5867         if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5868                 fma_dma_flag = 1;
5869         } else {
5870                 fma_dma_flag = 0;
5871         }
5872
5873         ixgbe_set_fma_flags(fma_dma_flag);
5874
5875         if (ixgbe->fm_capabilities) {
5876
5877                 /*
5878                  * Register capabilities with IO Fault Services
5879                  */
5880                 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5881
5882                 /*
5883                  * Initialize pci ereport capabilities if ereport capable
5884                  */
5885                 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5886                     DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5887                         pci_ereport_setup(ixgbe->dip);
5888
5889                 /*
5890                  * Register error callback if error callback capable
5891                  */
5892                 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5893                         ddi_fm_handler_register(ixgbe->dip,
5894                             ixgbe_fm_error_cb, (void*) ixgbe);
5895         }
5896 }
5897
5898 static void
5899 ixgbe_fm_fini(ixgbe_t *ixgbe)
5900 {
5901         /*
5902          * Only unregister FMA capabilities if they are registered
5903          */
5904         if (ixgbe->fm_capabilities) {
5905
5906                 /*
5907                  * Release any resources allocated by pci_ereport_setup()
5908                  */
5909                 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5910                     DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5911                         pci_ereport_teardown(ixgbe->dip);
5912
5913                 /*
5914                  * Un-register error callback if error callback capable
5915                  */
5916                 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5917                         ddi_fm_handler_unregister(ixgbe->dip);
5918
5919                 /*
5920                  * Unregister from IO Fault Service
5921                  */
5922                 ddi_fm_fini(ixgbe->dip);
5923         }
5924 }
5925
5926 void
5927 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5928 {
5929         uint64_t ena;
5930         char buf[FM_MAX_CLASS];
5931
5932         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5933         ena = fm_ena_generate(0, FM_ENA_FMT1);
5934         if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5935                 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5936                     FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5937         }
5938 }
5939
5940 static int
5941 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5942 {
5943         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5944
5945         mutex_enter(&rx_ring->rx_lock);
5946         rx_ring->ring_gen_num = mr_gen_num;
5947         mutex_exit(&rx_ring->rx_lock);
5948         return (0);
5949 }
5950
5951 /*
5952  * Get the global ring index by a ring index within a group.
5953  */
5954 static int
5955 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5956 {
5957         ixgbe_rx_ring_t *rx_ring;
5958         int i;
5959
5960         for (i = 0; i < ixgbe->num_rx_rings; i++) {
5961                 rx_ring = &ixgbe->rx_rings[i];
5962                 if (rx_ring->group_index == gindex)
5963                         rindex--;
5964                 if (rindex < 0)
5965                         return (i);
5966         }
5967
5968         return (-1);
5969 }
5970
5971 /*
5972  * Callback funtion for MAC layer to register all rings.
5973  */
5974 /* ARGSUSED */
5975 void
5976 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5977     const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5978 {
5979         ixgbe_t *ixgbe = (ixgbe_t *)arg;
5980         mac_intr_t *mintr = &infop->mri_intr;
5981
5982         switch (rtype) {
5983         case MAC_RING_TYPE_RX: {
5984                 /*
5985                  * 'index' is the ring index within the group.
5986                  * Need to get the global ring index by searching in groups.
5987                  */
5988                 int global_ring_index = ixgbe_get_rx_ring_index(
5989                     ixgbe, group_index, ring_index);
5990
5991                 ASSERT(global_ring_index >= 0);
5992
5993                 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5994                 rx_ring->ring_handle = rh;
5995
5996                 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5997                 infop->mri_start = ixgbe_ring_start;
5998                 infop->mri_stop = NULL;
5999                 infop->mri_poll = ixgbe_ring_rx_poll;
6000                 infop->mri_stat = ixgbe_rx_ring_stat;
6001
6002                 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6003                 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6004                 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6005                 if (ixgbe->intr_type &
6006                     (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6007                         mintr->mi_ddi_handle =
6008                             ixgbe->htable[rx_ring->intr_vector];
6009                 }
6010
6011                 break;
6012         }
6013         case MAC_RING_TYPE_TX: {
6014                 ASSERT(group_index == -1);
6015                 ASSERT(ring_index < ixgbe->num_tx_rings);
6016
6017                 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6018                 tx_ring->ring_handle = rh;
6019
6020                 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6021                 infop->mri_start = NULL;
6022                 infop->mri_stop = NULL;
6023                 infop->mri_tx = ixgbe_ring_tx;
6024                 infop->mri_stat = ixgbe_tx_ring_stat;
6025                 if (ixgbe->intr_type &
6026                     (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6027                         mintr->mi_ddi_handle =
6028                             ixgbe->htable[tx_ring->intr_vector];
6029                 }
6030                 break;
6031         }
6032         default:
6033                 break;
6034         }
6035 }
6036
6037 /*
6038  * Callback funtion for MAC layer to register all groups.
6039  */
6040 void
6041 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6042     mac_group_info_t *infop, mac_group_handle_t gh)
6043 {
6044         ixgbe_t *ixgbe = (ixgbe_t *)arg;
6045
6046         switch (rtype) {
6047         case MAC_RING_TYPE_RX: {
6048                 ixgbe_rx_group_t *rx_group;
6049
6050                 rx_group = &ixgbe->rx_groups[index];
6051                 rx_group->group_handle = gh;
6052
6053                 infop->mgi_driver = (mac_group_driver_t)rx_group;
6054                 infop->mgi_start = NULL;
6055                 infop->mgi_stop = NULL;
6056                 infop->mgi_addmac = ixgbe_addmac;
6057                 infop->mgi_remmac = ixgbe_remmac;
6058                 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6059
6060                 break;
6061         }
6062         case MAC_RING_TYPE_TX:
6063                 break;
6064         default:
6065                 break;
6066         }
6067 }
6068
6069 /*
6070  * Enable interrupt on the specificed rx ring.
6071  */
6072 int
6073 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6074 {
6075         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6076         ixgbe_t *ixgbe = rx_ring->ixgbe;
6077         int r_idx = rx_ring->index;
6078         int hw_r_idx = rx_ring->hw_index;
6079         int v_idx = rx_ring->intr_vector;
6080
6081         mutex_enter(&ixgbe->gen_lock);
6082         if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6083                 mutex_exit(&ixgbe->gen_lock);
6084                 /*
6085                  * Simply return 0.
6086                  * Interrupts are being adjusted. ixgbe_intr_adjust()
6087                  * will eventually re-enable the interrupt when it's
6088                  * done with the adjustment.
6089                  */
6090                 return (0);
6091         }
6092
6093         /*
6094          * To enable interrupt by setting the VAL bit of given interrupt
6095          * vector allocation register (IVAR).
6096          */
6097         ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6098
6099         BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6100
6101         /*
6102          * Trigger a Rx interrupt on this ring
6103          */
6104         IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6105         IXGBE_WRITE_FLUSH(&ixgbe->hw);
6106
6107         mutex_exit(&ixgbe->gen_lock);
6108
6109         return (0);
6110 }
6111
6112 /*
6113  * Disable interrupt on the specificed rx ring.
6114  */
6115 int
6116 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6117 {
6118         ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6119         ixgbe_t *ixgbe = rx_ring->ixgbe;
6120         int r_idx = rx_ring->index;
6121         int hw_r_idx = rx_ring->hw_index;
6122         int v_idx = rx_ring->intr_vector;
6123
6124         mutex_enter(&ixgbe->gen_lock);
6125         if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6126                 mutex_exit(&ixgbe->gen_lock);
6127                 /*
6128                  * Simply return 0.
6129                  * In the rare case where an interrupt is being
6130                  * disabled while interrupts are being adjusted,
6131                  * we don't fail the operation. No interrupts will
6132                  * be generated while they are adjusted, and
6133                  * ixgbe_intr_adjust() will cause the interrupts
6134                  * to be re-enabled once it completes. Note that
6135                  * in this case, packets may be delivered to the
6136                  * stack via interrupts before xgbe_rx_ring_intr_enable()
6137                  * is called again. This is acceptable since interrupt
6138                  * adjustment is infrequent, and the stack will be
6139                  * able to handle these packets.
6140                  */
6141                 return (0);
6142         }
6143
6144         /*
6145          * To disable interrupt by clearing the VAL bit of given interrupt
6146          * vector allocation register (IVAR).
6147          */
6148         ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6149
6150         BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6151
6152         mutex_exit(&ixgbe->gen_lock);
6153
6154         return (0);
6155 }
6156
6157 /*
6158  * Add a mac address.
6159  */
6160 static int
6161 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6162 {
6163         ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6164         ixgbe_t *ixgbe = rx_group->ixgbe;
6165         struct ixgbe_hw *hw = &ixgbe->hw;
6166         int slot, i;
6167
6168         mutex_enter(&ixgbe->gen_lock);
6169
6170         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6171                 mutex_exit(&ixgbe->gen_lock);
6172                 return (ECANCELED);
6173         }
6174
6175         if (ixgbe->unicst_avail == 0) {
6176                 /* no slots available */
6177                 mutex_exit(&ixgbe->gen_lock);
6178                 return (ENOSPC);
6179         }
6180
6181         /*
6182          * The first ixgbe->num_rx_groups slots are reserved for each respective
6183          * group. The rest slots are shared by all groups. While adding a
6184          * MAC address, reserved slots are firstly checked then the shared
6185          * slots are searched.
6186          */
6187         slot = -1;
6188         if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6189                 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6190                         if (ixgbe->unicst_addr[i].mac.set == 0) {
6191                                 slot = i;
6192                                 break;
6193                         }
6194                 }
6195         } else {
6196                 slot = rx_group->index;
6197         }
6198
6199         if (slot == -1) {
6200                 /* no slots available */
6201                 mutex_exit(&ixgbe->gen_lock);
6202                 return (ENOSPC);
6203         }
6204
6205         bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6206         (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6207             rx_group->index, IXGBE_RAH_AV);
6208         ixgbe->unicst_addr[slot].mac.set = 1;
6209         ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6210         ixgbe->unicst_avail--;
6211
6212         mutex_exit(&ixgbe->gen_lock);
6213
6214         return (0);
6215 }
6216
6217 /*
6218  * Remove a mac address.
6219  */
6220 static int
6221 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6222 {
6223         ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6224         ixgbe_t *ixgbe = rx_group->ixgbe;
6225         struct ixgbe_hw *hw = &ixgbe->hw;
6226         int slot;
6227
6228         mutex_enter(&ixgbe->gen_lock);
6229
6230         if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6231                 mutex_exit(&ixgbe->gen_lock);
6232                 return (ECANCELED);
6233         }
6234
6235         slot = ixgbe_unicst_find(ixgbe, mac_addr);
6236         if (slot == -1) {
6237                 mutex_exit(&ixgbe->gen_lock);
6238                 return (EINVAL);
6239         }
6240
6241         if (ixgbe->unicst_addr[slot].mac.set == 0) {
6242                 mutex_exit(&ixgbe->gen_lock);
6243                 return (EINVAL);
6244         }
6245
6246         bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6247         (void) ixgbe_clear_rar(hw, slot);
6248         ixgbe->unicst_addr[slot].mac.set = 0;
6249         ixgbe->unicst_avail++;
6250
6251         mutex_exit(&ixgbe->gen_lock);
6252
6253         return (0);
6254 }