From 15ada8fc6dd9f38f563acc69f9f22f88a48b752a Mon Sep 17 00:00:00 2001 From: Robert Mustacchi Date: Sun, 8 Jan 2017 22:03:45 -0800 Subject: [PATCH] 7751 mpt_sas sometimes times out sending SEP messages Reviewed by: Joshua M. Clulow Reviewed by: Dan McDonald Reviewed by: Hans Rosenfeld Approved by: Richard Lowe --- .../uts/common/io/scsi/adapters/mpt_sas/mptsas.c | 136 ++++++++++++++++++++- .../common/io/scsi/adapters/mpt_sas/mptsas_impl.c | 101 +++++++++++++-- .../common/sys/scsi/adapters/mpt_sas/mptsas_var.h | 28 +++-- 3 files changed, 242 insertions(+), 23 deletions(-) diff --git a/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c b/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c index 00f3543a36..7b08af001d 100644 --- a/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c +++ b/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c @@ -22,7 +22,7 @@ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2016 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2014, Joyent, Inc. All rights reserved. + * Copyright 2016 Joyent, Inc. * Copyright 2014 OmniTI Computer Consulting, Inc. All rights reserved. * Copyright (c) 2014, Tegile Systems Inc. All rights reserved. */ @@ -144,6 +144,8 @@ static void mptsas_config_space_fini(mptsas_t *mpt); static void mptsas_iport_register(mptsas_t *mpt); static int mptsas_smp_setup(mptsas_t *mpt); static void mptsas_smp_teardown(mptsas_t *mpt); +static int mptsas_enc_setup(mptsas_t *mpt); +static void mptsas_enc_teardown(mptsas_t *mpt); static int mptsas_cache_create(mptsas_t *mpt); static void mptsas_cache_destroy(mptsas_t *mpt); static int mptsas_alloc_request_frames(mptsas_t *mpt); @@ -155,6 +157,7 @@ static void mptsas_alloc_reply_args(mptsas_t *mpt); static int mptsas_alloc_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd); static void mptsas_free_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd); static int mptsas_init_chip(mptsas_t *mpt, int first_time); +static void mptsas_update_hashtab(mptsas_t *mpt); /* * SCSA function prototypes @@ -1080,6 +1083,7 @@ mptsas_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) char config_setup = 0; char hba_attach_setup = 0; char smp_attach_setup = 0; + char enc_attach_setup = 0; char mutex_init_done = 0; char event_taskq_create = 0; char dr_taskq_create = 0; @@ -1433,6 +1437,10 @@ mptsas_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) goto fail; smp_attach_setup++; + if (mptsas_enc_setup(mpt) == FALSE) + goto fail; + enc_attach_setup++; + if (mptsas_cache_create(mpt) == FALSE) goto fail; @@ -1576,6 +1584,9 @@ fail: if (smp_attach_setup) { mptsas_smp_teardown(mpt); } + if (enc_attach_setup) { + mptsas_enc_teardown(mpt); + } if (hba_attach_setup) { mptsas_hba_teardown(mpt); } @@ -2059,6 +2070,7 @@ mptsas_do_detach(dev_info_t *dip) cv_destroy(&mpt->m_extreq_sense_refcount_cv); mptsas_smp_teardown(mpt); + mptsas_enc_teardown(mpt); mptsas_hba_teardown(mpt); mptsas_config_space_fini(mpt); @@ -2309,6 +2321,42 @@ mptsas_smp_teardown(mptsas_t *mpt) } static int +mptsas_enc_setup(mptsas_t *mpt) +{ + list_create(&mpt->m_enclosures, sizeof (mptsas_enclosure_t), + offsetof(mptsas_enclosure_t, me_link)); + return (TRUE); +} + +static void +mptsas_enc_teardown(mptsas_t *mpt) +{ + mptsas_enclosure_t *mep; + + while ((mep = list_remove_head(&mpt->m_enclosures)) != NULL) { + kmem_free(mep, sizeof (mptsas_enclosure_t)); + } + list_destroy(&mpt->m_enclosures); +} + +static mptsas_enclosure_t * +mptsas_enc_lookup(mptsas_t *mpt, uint16_t hdl) +{ + mptsas_enclosure_t *mep; + + ASSERT(MUTEX_HELD(&mpt->m_mutex)); + + for (mep = list_head(&mpt->m_enclosures); mep != NULL; + mep = list_next(&mpt->m_enclosures, mep)) { + if (hdl == mep->me_enchdl) { + return (mep); + } + } + + return (NULL); +} + +static int mptsas_cache_create(mptsas_t *mpt) { int instance = mpt->m_instance; @@ -7720,18 +7768,28 @@ mptsas_handle_event(void *args) { pMpi2EventDataSasEnclDevStatusChange_t encstatus; uint8_t rc; + uint16_t enchdl; char string[80]; + mptsas_enclosure_t *mep; encstatus = (pMpi2EventDataSasEnclDevStatusChange_t) eventreply->EventData; rc = ddi_get8(mpt->m_acc_reply_frame_hdl, &encstatus->ReasonCode); + enchdl = ddi_get16(mpt->m_acc_reply_frame_hdl, + &encstatus->EnclosureHandle); + switch (rc) { case MPI2_EVENT_SAS_ENCL_RC_ADDED: (void) sprintf(string, "added"); break; case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING: + mep = mptsas_enc_lookup(mpt, enchdl); + if (mep != NULL) { + list_remove(&mpt->m_enclosures, mep); + kmem_free(mep, sizeof (*mep)); + } (void) sprintf(string, ", not responding"); break; default: @@ -7741,6 +7799,13 @@ mptsas_handle_event(void *args) "%x%s\n", mpt->m_instance, ddi_get16(mpt->m_acc_reply_frame_hdl, &encstatus->EnclosureHandle), string)); + + /* + * No matter what has happened, update all of our device state + * for enclosures, by retriggering an evaluation. + */ + mpt->m_done_traverse_enc = 0; + mptsas_update_hashtab(mpt); break; } @@ -14494,7 +14559,29 @@ mptsas_offline_missed_luns(dev_info_t *pdip, uint16_t *repluns, } } -void +/* + * If this enclosure doesn't exist in the enclosure list, add it. If it does, + * update it. + */ +static void +mptsas_enclosure_update(mptsas_t *mpt, mptsas_enclosure_t *mep) +{ + mptsas_enclosure_t *m; + + ASSERT(MUTEX_HELD(&mpt->m_mutex)); + m = mptsas_enc_lookup(mpt, mep->me_enchdl); + if (m != NULL) { + m->me_flags = mep->me_flags; + return; + } + + m = kmem_zalloc(sizeof (*m), KM_SLEEP); + m->me_enchdl = mep->me_enchdl; + m->me_flags = mep->me_flags; + list_insert_tail(&mpt->m_enclosures, m); +} + +static void mptsas_update_hashtab(struct mptsas *mpt) { uint32_t page_address; @@ -14509,7 +14596,7 @@ mptsas_update_hashtab(struct mptsas *mpt) (void) mptsas_get_raid_info(mpt); dev_handle = mpt->m_smp_devhdl; - for (; mpt->m_done_traverse_smp == 0; ) { + while (mpt->m_done_traverse_smp == 0) { page_address = (MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL & MPI2_SAS_EXPAND_PGAD_FORM_MASK) | (uint32_t)dev_handle; if (mptsas_get_sas_expander_page0(mpt, page_address, &smp_node) @@ -14521,16 +14608,34 @@ mptsas_update_hashtab(struct mptsas *mpt) } /* + * Loop over enclosures so we can understand what's there. + */ + dev_handle = MPTSAS_INVALID_DEVHDL; + while (mpt->m_done_traverse_enc == 0) { + mptsas_enclosure_t me; + + page_address = (MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE & + MPI2_SAS_ENCLOS_PGAD_FORM_MASK) | (uint32_t)dev_handle; + + if (mptsas_get_enclosure_page0(mpt, page_address, &me) != + DDI_SUCCESS) { + break; + } + dev_handle = me.me_enchdl; + mptsas_enclosure_update(mpt, &me); + } + + /* * Config target devices */ dev_handle = mpt->m_dev_handle; /* - * Do loop to get sas device page 0 by GetNextHandle till the + * Loop to get sas device page 0 by GetNextHandle till the * the last handle. If the sas device is a SATA/SSP target, * we try to config it. */ - for (; mpt->m_done_traverse_dev == 0; ) { + while (mpt->m_done_traverse_dev == 0) { ptgt = NULL; page_address = (MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE & @@ -14587,6 +14692,7 @@ mptsas_update_driver_data(struct mptsas *mpt) } mpt->m_done_traverse_dev = 0; mpt->m_done_traverse_smp = 0; + mpt->m_done_traverse_enc = 0; mpt->m_dev_handle = mpt->m_smp_devhdl = MPTSAS_INVALID_DEVHDL; mptsas_update_hashtab(mpt); } @@ -14617,7 +14723,8 @@ mptsas_config_all(dev_info_t *pdip) mutex_enter(&mpt->m_mutex); - if (!mpt->m_done_traverse_dev || !mpt->m_done_traverse_smp) { + if (!mpt->m_done_traverse_dev || !mpt->m_done_traverse_smp || + !mpt->m_done_traverse_enc) { mptsas_update_hashtab(mpt); } @@ -16494,6 +16601,8 @@ mptsas_send_sep(mptsas_t *mpt, mptsas_target_t *ptgt, Mpi2SepRequest_t req; Mpi2SepReply_t rep; int ret; + mptsas_enclosure_t *mep; + uint16_t enctype; ASSERT(mutex_owned(&mpt->m_mutex)); @@ -16515,6 +16624,21 @@ mptsas_send_sep(mptsas_t *mpt, mptsas_target_t *ptgt, return (ENOTTY); } + /* + * Look through the enclosures and make sure that this enclosure is + * something that is directly attached device. If we didn't find an + * enclosure for this device, don't send the ioctl. + */ + mep = mptsas_enc_lookup(mpt, ptgt->m_enclosure); + if (mep == NULL) + return (ENOTTY); + enctype = mep->me_flags & MPI2_SAS_ENCLS0_FLAGS_MNG_MASK; + if (enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_SES && + enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_SGPIO && + enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_GPIO) { + return (ENOTTY); + } + bzero(&req, sizeof (req)); bzero(&rep, sizeof (rep)); diff --git a/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas_impl.c b/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas_impl.c index ddf1cb2553..ddb6e169b6 100644 --- a/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas_impl.c +++ b/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas_impl.c @@ -24,6 +24,7 @@ * Copyright 2012 Nexenta Systems, Inc. All rights reserved. * Copyright 2014 OmniTI Computer Consulting, Inc. All rights reserved. * Copyright (c) 2014, Tegile Systems Inc. All rights reserved. + * Copyright 2016 Joyent, Inc. */ /* @@ -612,8 +613,8 @@ page_done: int mptsas_send_config_request_msg(mptsas_t *mpt, uint8_t action, uint8_t pagetype, - uint32_t pageaddress, uint8_t pagenumber, uint8_t pageversion, - uint8_t pagelength, uint32_t SGEflagslength, uint64_t SGEaddress) + uint32_t pageaddress, uint8_t pagenumber, uint8_t pageversion, + uint8_t pagelength, uint32_t SGEflagslength, uint64_t SGEaddress) { pMpi2ConfigRequest_t config; int send_numbytes; @@ -648,9 +649,9 @@ mptsas_send_config_request_msg(mptsas_t *mpt, uint8_t action, uint8_t pagetype, int mptsas_send_extended_config_request_msg(mptsas_t *mpt, uint8_t action, - uint8_t extpagetype, uint32_t pageaddress, uint8_t pagenumber, - uint8_t pageversion, uint16_t extpagelength, - uint32_t SGEflagslength, uint64_t SGEaddress) + uint8_t extpagetype, uint32_t pageaddress, uint8_t pagenumber, + uint8_t pageversion, uint16_t extpagelength, + uint32_t SGEflagslength, uint64_t SGEaddress) { pMpi2ConfigRequest_t config; int send_numbytes; @@ -717,7 +718,7 @@ mptsas_ioc_wait_for_doorbell(mptsas_t *mpt) int mptsas_send_handshake_msg(mptsas_t *mpt, caddr_t memp, int numbytes, - ddi_acc_handle_t accessp) + ddi_acc_handle_t accessp) { int i; @@ -769,7 +770,7 @@ mptsas_send_handshake_msg(mptsas_t *mpt, caddr_t memp, int numbytes, int mptsas_get_handshake_msg(mptsas_t *mpt, caddr_t memp, int numbytes, - ddi_acc_handle_t accessp) + ddi_acc_handle_t accessp) { int i, totalbytes, bytesleft; uint16_t val; @@ -1086,7 +1087,7 @@ mptsas_return_to_pool(mptsas_t *mpt, mptsas_cmd_t *cmd) */ int mptsas_ioc_task_management(mptsas_t *mpt, int task_type, uint16_t dev_handle, - int lun, uint8_t *reply, uint32_t reply_size, int mode) + int lun, uint8_t *reply, uint32_t reply_size, int mode) { /* * In order to avoid allocating variables on the stack, @@ -2798,3 +2799,87 @@ done: return (rval); } + +static int +mptsas_enclosurepage_0_cb(mptsas_t *mpt, caddr_t page_memp, + ddi_acc_handle_t accessp, uint16_t iocstatus, uint32_t iocloginfo, + va_list ap) +{ + uint32_t page_address; + pMpi2SasEnclosurePage0_t encpage, encout; + + if ((iocstatus != MPI2_IOCSTATUS_SUCCESS) && + (iocstatus != MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)) { + mptsas_log(mpt, CE_WARN, "mptsas_get_enclsourepage0 " + "header: IOCStatus=0x%x, IOCLogInfo=0x%x", + iocstatus, iocloginfo); + return (DDI_FAILURE); + } + + page_address = va_arg(ap, uint32_t); + encout = va_arg(ap, pMpi2SasEnclosurePage0_t); + encpage = (pMpi2SasEnclosurePage0_t)page_memp; + + /* + * The INVALID_PAGE status is normal if using GET_NEXT_HANDLE and there + * are no more pages. If everything is OK up to this point but the + * status is INVALID_PAGE, change rval to FAILURE and quit. Also, + * signal that enclosure traversal is complete. + */ + if (iocstatus == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE) { + if ((page_address & MPI2_SAS_DEVICE_PGAD_FORM_MASK) == + MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE) { + mpt->m_done_traverse_enc = 1; + } + return (DDI_FAILURE); + } + + encout->Header.PageVersion = ddi_get8(accessp, + &encpage->Header.PageVersion); + encout->Header.PageNumber = ddi_get8(accessp, + &encpage->Header.PageNumber); + encout->Header.PageType = ddi_get8(accessp, &encpage->Header.PageType); + encout->Header.ExtPageLength = ddi_get16(accessp, + &encpage->Header.ExtPageLength); + encout->Header.ExtPageType = ddi_get8(accessp, + &encpage->Header.ExtPageType); + + encout->EnclosureLogicalID.Low = ddi_get32(accessp, + &encpage->EnclosureLogicalID.Low); + encout->EnclosureLogicalID.High = ddi_get32(accessp, + &encpage->EnclosureLogicalID.High); + encout->Flags = ddi_get16(accessp, &encpage->Flags); + encout->EnclosureHandle = ddi_get16(accessp, &encpage->EnclosureHandle); + encout->NumSlots = ddi_get16(accessp, &encpage->NumSlots); + encout->StartSlot = ddi_get16(accessp, &encpage->StartSlot); + encout->EnclosureLevel = ddi_get8(accessp, &encpage->EnclosureLevel); + encout->SEPDevHandle = ddi_get16(accessp, &encpage->SEPDevHandle); + + return (DDI_SUCCESS); +} + +/* + * Request information about the SES enclosures. + */ +int +mptsas_get_enclosure_page0(mptsas_t *mpt, uint32_t page_address, + mptsas_enclosure_t *mep) +{ + int rval = DDI_SUCCESS; + Mpi2SasEnclosurePage0_t encpage; + + ASSERT(MUTEX_HELD(&mpt->m_mutex)); + + bzero(&encpage, sizeof (encpage)); + rval = mptsas_access_config_page(mpt, + MPI2_CONFIG_ACTION_PAGE_READ_CURRENT, + MPI2_CONFIG_EXTPAGETYPE_ENCLOSURE, 0, page_address, + mptsas_enclosurepage_0_cb, page_address, &encpage); + + if (rval == DDI_SUCCESS) { + mep->me_enchdl = encpage.EnclosureHandle; + mep->me_flags = encpage.Flags; + } + + return (rval); +} diff --git a/usr/src/uts/common/sys/scsi/adapters/mpt_sas/mptsas_var.h b/usr/src/uts/common/sys/scsi/adapters/mpt_sas/mptsas_var.h index 2465883c36..f15cfa131e 100644 --- a/usr/src/uts/common/sys/scsi/adapters/mpt_sas/mptsas_var.h +++ b/usr/src/uts/common/sys/scsi/adapters/mpt_sas/mptsas_var.h @@ -22,7 +22,7 @@ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2015 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2013, Joyent, Inc. All rights reserved. + * Copyright 2016 Joyent, Inc. * Copyright (c) 2014, Tegile Systems Inc. All rights reserved. */ @@ -250,6 +250,16 @@ typedef struct mptsas_smp { uint32_t m_pdevinfo; } mptsas_smp_t; +/* + * This represents a single enclosure. Targets point to an enclosure through + * their m_enclosure member. + */ +typedef struct mptsas_enclosure { + list_node_t me_link; + uint16_t me_enchdl; + uint16_t me_flags; +} mptsas_enclosure_t; + typedef struct mptsas_cache_frames { ddi_dma_handle_t m_dma_hdl; ddi_acc_handle_t m_acc_hdl; @@ -701,6 +711,7 @@ typedef struct mptsas { refhash_t *m_targets; refhash_t *m_smp_targets; + list_t m_enclosures; refhash_t *m_tmp_targets; m_raidconfig_t m_raidconfig[MPTSAS_MAX_RAIDCONFIGS]; @@ -900,6 +911,7 @@ typedef struct mptsas { int m_mpxio_enable; uint8_t m_done_traverse_dev; uint8_t m_done_traverse_smp; + uint8_t m_done_traverse_enc; int m_diag_action_in_progress; uint16_t m_dev_handle; uint16_t m_smp_devhdl; @@ -1361,16 +1373,14 @@ int mptsas_get_manufacture_page5(mptsas_t *mpt); int mptsas_get_sas_port_page0(mptsas_t *mpt, uint32_t page_address, uint64_t *sas_wwn, uint8_t *portwidth); int mptsas_get_bios_page3(mptsas_t *mpt, uint32_t *bios_version); -int -mptsas_get_sas_phy_page0(mptsas_t *mpt, uint32_t page_address, +int mptsas_get_sas_phy_page0(mptsas_t *mpt, uint32_t page_address, smhba_info_t *info); -int -mptsas_get_sas_phy_page1(mptsas_t *mpt, uint32_t page_address, +int mptsas_get_sas_phy_page1(mptsas_t *mpt, uint32_t page_address, smhba_info_t *info); -int -mptsas_get_manufacture_page0(mptsas_t *mpt); -void -mptsas_create_phy_stats(mptsas_t *mpt, char *iport, dev_info_t *dip); +int mptsas_get_manufacture_page0(mptsas_t *mpt); +int mptsas_get_enclosure_page0(mptsas_t *mpt, uint32_t page_address, + mptsas_enclosure_t *mpe); +void mptsas_create_phy_stats(mptsas_t *mpt, char *iport, dev_info_t *dip); void mptsas_destroy_phy_stats(mptsas_t *mpt); int mptsas_smhba_phy_init(mptsas_t *mpt); /* -- 2.11.4.GIT