8634 epoll fails to wake on certain edge-triggered conditions

[unleashed.git] / usr / src / uts / common / io / ib / clients / of / sol_uverbs / sol_uverbs_event.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2017 Joyent, Inc.
 */

/*
 * sol_uverbs_event.c
 *
 * OFED User Verbs Kernel Async Event funtions
 *
 */
#include <sys/file.h>
#include <sys/fcntl.h>
#include <sys/vfs.h>
#include <sys/errno.h>
#include <sys/cred.h>
#include <sys/uio.h>
#include <sys/semaphore.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

#include <sys/ib/ibtl/ibvti.h>
#include <sys/ib/clients/of/ofa_solaris.h>
#include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
#include <sys/ib/clients/of/ofed_kernel.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>

extern char     *sol_uverbs_dbg_str;

static void
uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t,
    llist_head_t *, uint32_t *);

/*
 * Function:
 *      sol_uverbs_event_file_close
 * Input:
 *      ufile   - Pointer to the event ufile
 *
 * Output:
 *      None
 * Returns:
 *      Zero on success, else error code.
 * Description:
 *      Called when all kernel references to the event file have been
 *      removed and the kernel (asynchronous) or user library (completion)
 *      have closed the file.
 */
void
sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile)
{
        if (!ufile) {
                SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                    "UFILE CLOSE: Ufile NULL\n");
                return;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "UFILE CLOSE: Is async? %s",
            ufile->is_async ? "yes" : "no");

        /*
         * Remove the user file from the user object table and
         * releases appropriate references.  The object resources
         * are freed when it is no longer referenced.
         *
         * If sol_ofs_uobj_remove() returns NULL then the obj was already
         * removed.
         */
        rw_enter(&(ufile->uobj.uo_lock), RW_WRITER);
        if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) {
                rw_exit(&(ufile->uobj.uo_lock));
                sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file);
        } else {
                rw_exit(&(ufile->uobj.uo_lock));
        }
}

/*
 * Function:
 *      sol_uverbs_event_file_read
 * Input:
 *      ufile   - The user file pointer of the event channel.
 *      uiop    - The user I/O pointer in which to place the event.
 *      cred    - Pointer to the callers credentials.
 * Output:
 *      uiop    - Upon success the caller's buffer has been updated with
 *                the event details.
 * Returns:
 *      Zero on success, else error code.
 *              EAGAIN  - No event available and caller is non-
 *                        blocking.
 *              ERESTART- A signal was received.
 *              EINVAL  - Caller parameter/user buffer invalid.
 *              EFAULT  - Failure copying data to user.
 * Description:
 *      Perfrom a blocking read to retrieve an event (asynchronous or
 *      completion) for the user file specified.  If an event is available it
 *      is immediately returned. If an event is not available, then the
 *      caller will block unless the open flags indicate it is a non-
 *      blocking open.  If the caller does block it does so interruptable
 *      and therefore will return upon an event or receipt of a signal.
 */
/* ARGSUSED */
int
sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop,
    cred_t *cred)
{
        int                     rc = 0;
        uverbs_event_t          *evt;
        llist_head_t            *entry;
        int                     eventsz;
        int                     ioflag;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), "
            "ufile = %p, is_async =%d, uio_resid=%d",
            ufile, ufile->is_async, uiop->uio_resid);

        ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY);

        mutex_enter(&ufile->lock);

        /*
         * If Event list not empty and CQ event notification is disabled
         * by sol_ucma, do not  return events. Either return EAGAIN (if
         * flag is O_NONBLOCK, or wait using cv_wait_sig().
         */
        if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE &&
            llist_empty(&ufile->event_list) != 0) {
                if (ioflag) {
                        mutex_exit(&ufile->lock);
                        SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                            "event_file_read - notify disabled, no block");
                        return (EAGAIN);
                }

                if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
                        mutex_exit(&ufile->lock);
                        SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                            "event_file_read - sig_wakeup");
                        return (ERESTART);
                }
        }

        while (llist_empty(&ufile->event_list)) {
                if (ioflag) {
                        mutex_exit(&ufile->lock);
                        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                            "event_file_read - no events, no block");
                        return (EAGAIN);
                }

                if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
                        mutex_exit(&ufile->lock);
                        SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                            "event_file_read - sig_wakeup");
                        return (ERESTART);
                }
        }

        entry = ufile->event_list.nxt;
        evt   = entry->ptr;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: "
            "Event entry found: entry:%p, event:%p, evt_list %p",
            entry, evt, &evt->ev_list);

        if (ufile->is_async) {
                eventsz = sizeof (struct ib_uverbs_async_event_desc);
        } else {
                eventsz = sizeof (struct ib_uverbs_comp_event_desc);
        }

        if (eventsz > uiop->uio_resid) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "event_file_read - Event too big");
                rc  = EINVAL;
                evt = NULL;
        } else {
                llist_del(ufile->event_list.nxt);
                if (evt->ev_counter) {
                        ++(*evt->ev_counter);
                        llist_del(&evt->ev_obj_list);
                }
        }

        mutex_exit(&ufile->lock);

        if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "EVENT FILE READ: Error writing ev");
                rc = EFAULT;
        }

        if (evt) {
                kmem_free(evt, sizeof (*evt));
        }

        return (rc);
}

/*
 * Function:
 *      sol_uverbs_event_file_poll
 * Input:
 *      ufile   - user file for desired completion channel event file
 *      events  - The events that may occur.
 *      anyyet  - A flag that is non-zero if any files in the set
 *                of descriptors has an event waiting.
 *      ct      - Pointer to the callers context.
 * Output:
 *      reventssp - A pointer updated to return a bitmask of events specified.
 *      phpp      - A pointer to a pollhead pointer, updated to reflect the
 *                  the event file's pollhead used for synchronization.
 * Returns:
 *      Zero on success, else error code.
 *              EINVAL    - Vnode does not point to valid event file.
 * Description:
 *      Support for event channel polling interface, allows use of completion
 *      channel in asynchronous type environment.  If events may be read
 *      without blocking indicate a POLLIN | POLLRDNORM event; otherwise if
 *      no other descriptors in the set have data waiting, set the pollhead
 *      pointer to our the associated completion event file's pollhead.
 */
int
sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events,
    int anyyet, short *reventsp, pollhead_t **phpp)
{
        short   revent = 0;

#ifdef DEBUG
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)",
            ufile, events);
#endif

        if (!ufile) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ",
                    "ufile %p", ufile);
                return (EINVAL);
        }

#ifdef DEBUG
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
            "ufile = %p, is_async =%d", ufile, ufile->is_async);
#endif

        mutex_enter(&ufile->lock);

        /*
         * If poll request and event is ready.
         */
        if ((events & (POLLIN | POLLRDNORM)) &&
            !llist_empty(&ufile->event_list)) {

                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
                    "Event entry available");

                revent |= POLLIN | POLLRDNORM;
        }

        /*
         * If we didn't get an event or are edge-triggered
         */
        if ((revent == 0 && !anyyet) || (events & POLLET)) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
                    "Event entry NOT available");

                *phpp = &ufile->poll_head;
        }

        mutex_exit(&ufile->lock);
        *reventsp = revent;
        return (0);
}

/*
 * Function:
 *      uverbs_alloc_event_file
 * Input:
 *      uctxt    - The Solaris User Verbs user context associated with the
 *                 event channel.
 *      is_async - Indicates the file is for asynchronous events if non-zero;
 *                 other wise it is for completion events.
 * Output:
 *      None.
 * Returns:
 *      New user verb event file object or NULL on error.
 * Description:
 *      Allocate an asynchronous or completion event file
 */
uverbs_ufile_uobj_t *
uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async)
{
        uverbs_ufile_uobj_t     *ufile;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)",
            uctxt, is_async);

        ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP);
        if (!ufile) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_event_file: mem alloc fail");
                return (NULL);
        }
        ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE;
        sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE);
        rw_enter(&ufile->uobj.uo_lock, RW_WRITER);

        if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) {
                /*
                 * The initialization routine set's the initial reference,
                 * we dereference the object here to clean it up.
                 */
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "ALLOC UFILE: Object add failed");
                rw_exit(&ufile->uobj.uo_lock);
                ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t);
                sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free);
                return (NULL);
        }

        ufile->is_async         = is_async ? 1 : 0;
        llist_head_init(&ufile->event_list, NULL);

        mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL);
        cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL);

        ufile->uctxt            = uctxt;
        ufile->uobj.uo_live     = 1;
        rw_exit(&ufile->uobj.uo_lock);
        return (ufile);
}

/*
 * Function:
 *      uverbs_release_event_file
 * Input:
 *      ufile    - Pointer to the ufile user object that is being freed.
 * Output:
 *      None.
 * Returns:
 *      None.
 * Description:
 *      Release/destroy event file resources before freeing memory.  This
 *      routine should only be used if the event file is successfully
 *      created.
 */
void
uverbs_release_event_file(sol_ofs_uobj_t *uobj)
{
        uverbs_ufile_uobj_t     *ufile = (uverbs_ufile_uobj_t *)uobj;
        uverbs_event_t          *evt;
        llist_head_t            *entry;
        llist_head_t            *list_tmp;

        if (!ufile) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "UFILE RELEASE: Ufile NULL\n");
                return;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "UFILE RELEASE: Event file=%p, is async = %s",
            ufile, ufile->is_async ? "yes" : "no");

        /*
         * Release any events still queued to the event file.
         */
        mutex_enter(&ufile->lock);

        entry = ufile->event_list.nxt;
        list_tmp = entry->nxt;
        while (entry != &ufile->event_list) {
                ASSERT(entry);
                evt = (uverbs_event_t *)entry->ptr;

                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "UFILE RELEASE: Deleting event %p on event file %p",
                    evt, ufile);

                llist_del(&evt->ev_list);
                kmem_free(evt, sizeof (*evt));
                entry = list_tmp;
                list_tmp = entry->nxt;
        }

        mutex_exit(&ufile->lock);

        cv_destroy(&ufile->poll_wait);
        mutex_destroy(&ufile->lock);
        sol_ofs_uobj_free(uobj);
}

/*
 * Function:
 *      uverbs_ibt_to_ofa_event_code
 * Input:
 *      code    - The OFA event code.
 * Output:
 *      The OFED event code.
 * Returns:
 *      Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if
 *      a valid translation does not exist.
 * Description:
 *      Map an IBT asynchronous event code to an OFED event code.
 */
enum ib_event_type
uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)
{
        enum ib_event_type    ofa_code;

        switch (code) {
        case IBT_EVENT_PATH_MIGRATED:
                ofa_code = IB_EVENT_PATH_MIG;
                break;

        case IBT_EVENT_SQD:
                ofa_code = IB_EVENT_SQ_DRAINED;
                break;

        case IBT_EVENT_COM_EST:
                ofa_code = IB_EVENT_COMM_EST;
                break;

        case IBT_ERROR_CATASTROPHIC_CHAN:
        case IBT_ERROR_LOCAL_CATASTROPHIC:
                ofa_code = IB_EVENT_QP_FATAL;
                break;

        case IBT_ERROR_INVALID_REQUEST_CHAN:
                ofa_code = IB_EVENT_QP_REQ_ERR;
                break;

        case IBT_ERROR_ACCESS_VIOLATION_CHAN:
                ofa_code = IB_EVENT_QP_ACCESS_ERR;
                break;

        case IBT_ERROR_PATH_MIGRATE_REQ:
                ofa_code = IB_EVENT_PATH_MIG_ERR;
                break;

        case IBT_ERROR_CQ:
                ofa_code = IB_EVENT_CQ_ERR;
                break;

        case IBT_EVENT_PORT_UP:
                ofa_code = IB_EVENT_PORT_ACTIVE;
                break;

        case IBT_ERROR_PORT_DOWN:
                ofa_code = IB_EVENT_PORT_ERR;
                break;

        case IBT_HCA_ATTACH_EVENT:
                ofa_code = IB_EVENT_CLIENT_REREGISTER;
                break;

        case IBT_EVENT_LIMIT_REACHED_SRQ:
                ofa_code = IB_EVENT_SRQ_LIMIT_REACHED;
                break;

        case IBT_ERROR_CATASTROPHIC_SRQ:
                ofa_code = IB_EVENT_SRQ_ERR;
                break;

        case IBT_EVENT_EMPTY_CHAN:
                ofa_code = IB_EVENT_QP_LAST_WQE_REACHED;
                break;

        /*
         * No mapping exists.
         */
        case IBT_ASYNC_OPAQUE1:
        case IBT_ASYNC_OPAQUE2:
        case IBT_ASYNC_OPAQUE3:
        case IBT_ASYNC_OPAQUE4:
        case IBT_HCA_DETACH_EVENT:
        default:
                ofa_code = -1;
        }

        return (ofa_code);
}

/*
 * Function:
 *      uverbs_async_qp_event_handler
 * Input:
 *      clnt_private    - The IBT attach client handle.
 *      hca_hdl         - The IBT hca handle associated with the notification.
 *      code            - The OFED event identifier.
 *      event           - The IBT event.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Handle QP affiliated asynchronous event noficiation.
 */
/* ARGSUSED */
void
uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    enum ib_event_type code, ibt_async_event_t *event)
{
        uverbs_uqp_uobj_t       *uqp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "async_qp_event_handler()");

        if (event->ev_chan_hdl == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "async_qp_event_handler: event handle NULL");
                return;
        }
        uqp = ibt_get_qp_private(event->ev_chan_hdl);
        ASSERT(uqp);
        if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) {
                SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                    "async_qp_event_handler: User QP context has been freed");
                return;
        }
        if (uqp->qp != event->ev_chan_hdl) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "async_qp_event_handler: QP handle mismatch");
                return;
        }

        uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle,
            code, &uqp->async_list, &uqp->async_events_reported);

}

/*
 * Function:
 *      uverbs_async_cq_event_handler
 * Input:
 *      clnt_private    - The IBT attach client handle.
 *      hca_hdl         - The IBT hca handle associated with the notification.
 *      code            - The OFED event identifier.
 *      event           - The IBT event.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Handle a CQ affiliated asynchronous event notification.
 */
/* ARGSUSED */
void
uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    enum ib_event_type code, ibt_async_event_t *event)
{
        uverbs_ucq_uobj_t       *ucq;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "ASYNC CQ EVENT HANDLER:");

        if (event->ev_cq_hdl == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "ASYNC CQ EVENT HANDLER: event handle is NULL");
                return;
        }

        ucq = ibt_get_cq_private(event->ev_cq_hdl);
        if (ucq->cq != event->ev_cq_hdl) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "ASYNC CQ EVENT HANDLER: CQ handle mismatch");
                return;
        }

        uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle,
            code, &ucq->async_list, &ucq->async_events_reported);
}

/*
 * Function:
 *      uverbs_async_srq_event_handler
 * Input:
 *      clnt_private    - The IBT attach client handle.
 *      hca_hdl         - The IBT hca handle associated with the notification.
 *      code            - The OFED event identifier.
 *      event           - The IBT event.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Handle a shared receive queue asynchronous event notification.
 */
/* ARGSUSED */
void
uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    enum ib_event_type code, ibt_async_event_t *event)
{
        uverbs_usrq_uobj_t      *usrq;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "ASYNC SRQ EVENT HANDLER:");

        if (event->ev_srq_hdl == NULL) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "ASYNC SRQ EVENT HANDLER: event handle is NULL");
                return;
        }

        usrq = ibt_get_srq_private(event->ev_srq_hdl);
        if (usrq->srq != event->ev_srq_hdl) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch");
                return;
        }

        uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle,
            code, &usrq->async_list, &usrq->async_events_reported);
}

/*
 * Function:
 *      uverbs_async_unaff_event_handler
 * Input:
 *      clnt_private    - The IBT attach client handle.
 *      hca_hdl         - The IBT hca handle associated with the notification.
 *      code            - The OFED event identifier.
 *      event           - The IBT event.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Handle an unaffiliated asynchronous event notification.
 */
/* ARGSUSED */
void
uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    enum ib_event_type code, ibt_async_event_t *event)
{
        sol_ofs_uobj_table_t    *uo_tbl = &uverbs_uctxt_uo_tbl;
        sol_ofs_uobj_blk_t      *blk;
        uverbs_uctxt_uobj_t     *uctxt;
        int                     i, j;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "ASYNC UNAFF EVENT HANDLER:");

        /*
         * Unaffiliated events are returned at the IBT client level.  We must
         * return the event to all user context allocated for the specific
         * HCA device specified.
         */
        rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER);

        for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) {
                blk = uo_tbl->uobj_tbl_uo_root[i];
                if (blk == NULL) {
                        continue;
                }
                for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) {
                        uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j];
                        if (uctxt == NULL) {
                                continue;
                        }
                        /*
                         * OK, check to see if this user context belongs
                         * to the idicated hca.
                         */
                        if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) {
                                uverbs_async_event_common(uctxt,
                                    event->ev_port, code, NULL, NULL);
                        }
                }
        }
        rw_exit(&uo_tbl->uobj_tbl_lock);
}

/*
 * Function:
 *      uverbs_async_event_handler
 * Input:
 *      clnt_private    - The IBT attach client handle.
 *      hca_hdl         - The IBT hca handle associated with the notification.
 *      code            - The OFED event identifier.
 *      event           - The IBT event.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Main IBT asynchronous event handler registered at ibt_attach.
 *      Convert to OFA event type and forward to the appropriate
 *      asynchronous handler.
 */
void
uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    ibt_async_code_t code, ibt_async_event_t *event)
{
        enum ib_event_type              ofa_type;
        sol_uverbs_ib_event_handler_t   *handler;
        llist_head_t                    *entry;
        sol_uverbs_hca_t                *hca;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "ASYNNC EVENT HANDLER: entry, code=%d", code);

        ofa_type = uverbs_ibt_to_ofa_event_code(code);

        if ((int)ofa_type < 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "ASYNC EVENT HANDLER:Event %d did not map to OFA "
                    "Event", code);
                return;
        }

        switch (ofa_type) {
                case IB_EVENT_QP_FATAL:
                case IB_EVENT_QP_REQ_ERR:
                case IB_EVENT_QP_ACCESS_ERR:
                case IB_EVENT_QP_LAST_WQE_REACHED:
                case IB_EVENT_SQ_DRAINED:
                case IB_EVENT_PATH_MIG:
                        /*
                         * These events are related with a QP
                         */
                        uverbs_async_qp_event_handler(clnt_private, hca_hdl,
                            ofa_type, event);
                        break;

                case IB_EVENT_CQ_ERR:
                        /*
                         * These events are related with a CQ
                         */
                        uverbs_async_cq_event_handler(clnt_private, hca_hdl,
                            ofa_type, event);
                        break;

                case IB_EVENT_SRQ_ERR:
                case IB_EVENT_SRQ_LIMIT_REACHED:
                        /*
                         * These events are related with a SRQ
                         */
                        uverbs_async_srq_event_handler(clnt_private, hca_hdl,
                            ofa_type, event);
                        break;


                case IB_EVENT_PORT_ERR:
                case IB_EVENT_PORT_ACTIVE:
                case IB_EVENT_LID_CHANGE:
                case IB_EVENT_PKEY_CHANGE:
                case IB_EVENT_SM_CHANGE:
                case IB_EVENT_CLIENT_REREGISTER:
                case IB_EVENT_DEVICE_FATAL:
                case IB_EVENT_PATH_MIG_ERR:
                        /*
                         * Unaffiliated asynchronous notifications.
                         */
                        uverbs_async_unaff_event_handler(clnt_private, hca_hdl,
                            ofa_type, event);
                        break;

                default:
                break;
        }

        /*
         * Give other kernel agents a notification.
         */
        hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl);
        if (hca) {
                mutex_enter(&hca->event_handler_lock);
                list_for_each(entry, &hca->event_handler_list) {
                        handler = (sol_uverbs_ib_event_handler_t *)entry->ptr;

                        ASSERT(handler != NULL);
                        handler->handler(handler, hca_hdl, code, event);
                }
                mutex_exit(&hca->event_handler_lock);
        }
}

/*
 * Function:
 *      uverbs_async_event_common
 * Input:
 *      uctxt           - Pointer to the user context associated with the
 *                      affiliated event.
 *      element         - The users handle to the associated object.
 *      event           - The event type.
 *      uobj_list       - The list to enqueue the asynchronous event.
 *      counter         - The counter to track the event delivery.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Create an asyncronous event and enqueue it on the specified list.
 *      Then wake callers that may  be blocked on the list.
 */
static void
uverbs_async_event_common(uverbs_uctxt_uobj_t  *uctxt, uint64_t element,
    uint32_t event, llist_head_t *obj_list, uint32_t *counter)
{
        uverbs_ufile_uobj_t     *ufile = uctxt->async_evfile;
        uverbs_event_t          *entry;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, "
            "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter);

        if (!ufile) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
                    "ufile %p", ufile);
                return;
        }

        mutex_enter(&ufile->lock);
        entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
        if (!entry) {
                mutex_exit(&ufile->lock);
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
                    "kmem_zalloc failed");
                return;
        }

        entry->ev_desc.async.element    = element;
        entry->ev_desc.async.event_type = event;
        entry->ev_counter               = counter;

        llist_head_init(&entry->ev_list, entry);
        llist_head_init(&entry->ev_obj_list, entry);

        llist_add_tail(&entry->ev_list, &ufile->event_list);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
            "adding ASYNC entry-ev_list=%p, entry %p",
            &entry->ev_list, entry);

        if (obj_list) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
                    "adding ASYNC entry-ev_obj_list=%p, entry=%p",
                    &entry->ev_obj_list, entry);
                llist_add_tail(&entry->ev_obj_list, obj_list);
        }

        mutex_exit(&ufile->lock);
        cv_signal(&ufile->poll_wait);
        pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
}

/*
 * Function:
 *      uverbs_release_ucq_channel
 * Input:
 *      uctxt   - A pointer to the callers user context.
 *      ufile - A pointer to the event file associated with a CQ.
 *      ucq     - A pointer to the user CQ object.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Release any completion and asynchronous events that may
 *      be queued to the specified completion channel/UCQ but not
 *      yet reaped.
 */
void
uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt,
    uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t   *ucq)
{
        uverbs_event_t  *evt;
        llist_head_t    *entry;
        llist_head_t    *list_tmp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p",
            uctxt, ufile, ucq);

        /*
         * Release completion events that have been queued on the CQ completion
         * eventlist.
         */
        if (ufile) {
                rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
                ufile->ufile_cq_cnt--;
                if (ufile->ufile_cq_cnt) {
                        rw_exit(&ufile->uobj.uo_lock);
                        uverbs_release_ucq_uevents(ufile, ucq);
                        return;
                }
                rw_exit(&ufile->uobj.uo_lock);
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "release_ucq_chan : comp_list %p, prv %p, nxt %p",
                    &ucq->comp_list, ucq->comp_list.prv,
                    ucq->comp_list.nxt);
                mutex_enter(&ufile->lock);

                entry = ucq->comp_list.nxt;
                list_tmp = entry->nxt;
                while (entry != &ucq->comp_list) {
                        ASSERT(entry);
                        evt = (uverbs_event_t *)entry->ptr;

                        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                            "RELEASE UCQ CHANNEL:Deleting event "
                            "on CQ comp list: %p", evt);
                        llist_del(&evt->ev_list);
                        llist_del(&evt->ev_obj_list);
                        kmem_free(evt, sizeof (*evt));
                        entry = list_tmp;
                        list_tmp = entry->nxt;
                }
                mutex_exit(&ufile->lock);

                uverbs_release_ucq_uevents(ufile, ucq);
        }

}

/*
 * Function:
 *      uverbs_release_ucq_uevents
 * Input:
 *      ufile   - A pointer to the asynchronous event file associated with a QP.
 *      ucq     - A pointer to the user CQ object.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Free any user asynchronous events that have been queued for the
 *      user CQ object specified.
 */
void
uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq)
{
        uverbs_event_t  *evt;
        llist_head_t    *entry;
        llist_head_t    *list_tmp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq);

        if (ufile) {
                mutex_enter(&ufile->lock);

                entry = ucq->async_list.nxt;
                list_tmp = entry->nxt;
                while (entry != &ucq->async_list) {
                        ASSERT(entry);
                        evt = (uverbs_event_t *)entry->ptr;
                        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                            "RELEASE UCQ EVENTS: Deleting event "
                            "on CQ async list: %p", evt);
                        llist_del(&evt->ev_list);
                        llist_del(&evt->ev_obj_list);
                        kmem_free(evt, sizeof (*evt));
                        entry = list_tmp;
                        list_tmp = entry->nxt;
                }
                mutex_exit(&ufile->lock);
        }
}

/*
 * Function:
 *      uverbs_release_uqp_uevents
 * Input:
 *      ufile   - A pointer to the asynchronous event file associated with a QP.
 *      uqp     - A pointer to the user QP object.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Free any user asynchronous events that have been queued for the
 *      user QP object specified.
 */
void
uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp)
{
        uverbs_event_t  *evt;
        llist_head_t    *entry;
        llist_head_t    *list_tmp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp);

        if (ufile) {
                mutex_enter(&ufile->lock);
                entry = uqp->async_list.nxt;
                list_tmp = entry->nxt;
                while (entry != &uqp->async_list) {
                        ASSERT(entry);
                        evt = (uverbs_event_t *)entry->ptr;
                        ASSERT(evt);

                        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                            "RELEASE UQP EVENTS: Deleting event "
                            "ON qp async list: %p", evt);
                        llist_del(&evt->ev_list);
                        llist_del(&evt->ev_obj_list);
                        kmem_free(evt, sizeof (*evt));
                        entry = list_tmp;
                        list_tmp = entry->nxt;
                }
                mutex_exit(&ufile->lock);
        }
}

/*
 * Function:
 *      uverbs_release_usrq_uevents
 * Input:
 *      ufile   - A pointer to the asynchronous event file associated with a
 *                SRQ.
 *      uqp     - A pointer to the user SRQ object.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Free any user asynchronous events that have been queued for the
 *      user SRQ object specified.
 */
void
uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile,
    uverbs_usrq_uobj_t *usrq)
{
        uverbs_event_t  *evt;
        llist_head_t    *entry;
        llist_head_t    *list_tmp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq);

        if (ufile) {
                mutex_enter(&ufile->lock);

                entry = usrq->async_list.nxt;
                list_tmp = entry->nxt;
                while (entry != &usrq->async_list) {
                        ASSERT(entry);
                        evt = (uverbs_event_t *)entry->ptr;
                        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                            "RELEASE SRQ EVENTS: Deleting event "
                            "on SRQ async list: %p", evt);
                        llist_del(&evt->ev_list);
                        llist_del(&evt->ev_obj_list);
                        kmem_free(evt, sizeof (*evt));
                        entry = list_tmp;
                        list_tmp = entry->nxt;
                }
                mutex_exit(&ufile->lock);
        }
}