uts: make emu10k non-verbose

[unleashed.git] / kernel / os / netstack.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2016, Joyent, Inc.  All rights reserved.
 */

#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/vm.h>
#include <sys/proc.h>
#include <sys/tuneable.h>
#include <sys/systm.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/sdt.h>
#include <sys/mutex.h>
#include <sys/bitmap.h>
#include <sys/atomic.h>
#include <sys/kobj.h>
#include <sys/disp.h>
#include <vm/seg_kmem.h>
#include <sys/zone.h>
#include <sys/netstack.h>

/*
 * What we use so that the zones framework can tell us about new zones,
 * which we use to create new stacks.
 */
static zone_key_t netstack_zone_key;

static int      netstack_initialized = 0;

/*
 * Track the registered netstacks.
 * The global lock protects
 * - ns_reg
 * - the list starting at netstack_head and following the netstack_next
 *   pointers.
 */
static kmutex_t netstack_g_lock;

/*
 * Registry of netstacks with their create/shutdown/destory functions.
 */
static struct netstack_registry ns_reg[NS_MAX];

/*
 * Global list of existing stacks.  We use this when a new zone with
 * an exclusive IP instance is created.
 *
 * Note that in some cases a netstack_t needs to stay around after the zone
 * has gone away. This is because there might be outstanding references
 * (from TCP TIME_WAIT connections, IPsec state, etc). The netstack_t data
 * structure and all the foo_stack_t's hanging off of it will be cleaned up
 * when the last reference to it is dropped.
 * However, the same zone might be rebooted. That is handled using the
 * assumption that the zones framework picks a new zoneid each time a zone
 * is (re)booted. We assert for that condition in netstack_zone_create().
 * Thus the old netstack_t can take its time for things to time out.
 */
static netstack_t *netstack_head;

/*
 * To support kstat_create_netstack() using kstat_zone_add we need
 * to track both
 *  - all zoneids that use the global/shared stack
 *  - all kstats that have been added for the shared stack
 */
struct shared_zone_list {
        struct shared_zone_list *sz_next;
        zoneid_t                sz_zoneid;
};

struct shared_kstat_list {
        struct shared_kstat_list *sk_next;
        kstat_t                  *sk_kstat;
};

static kmutex_t netstack_shared_lock;   /* protects the following two */
static struct shared_zone_list  *netstack_shared_zones;
static struct shared_kstat_list *netstack_shared_kstats;

static void     *netstack_zone_create(zoneid_t zoneid);
static void     netstack_zone_shutdown(zoneid_t zoneid, void *arg);
static void     netstack_zone_destroy(zoneid_t zoneid, void *arg);

static void     netstack_shared_zone_add(zoneid_t zoneid);
static void     netstack_shared_zone_remove(zoneid_t zoneid);
static void     netstack_shared_kstat_add(kstat_t *ks);
static void     netstack_shared_kstat_remove(kstat_t *ks);

typedef boolean_t applyfn_t(kmutex_t *, netstack_t *, int);

static void     apply_all_netstacks(int, applyfn_t *);
static void     apply_all_modules(netstack_t *, applyfn_t *);
static void     apply_all_modules_reverse(netstack_t *, applyfn_t *);
static boolean_t netstack_apply_create(kmutex_t *, netstack_t *, int);
static boolean_t netstack_apply_shutdown(kmutex_t *, netstack_t *, int);
static boolean_t netstack_apply_destroy(kmutex_t *, netstack_t *, int);
static boolean_t wait_for_zone_creator(netstack_t *, kmutex_t *);
static boolean_t wait_for_nms_inprogress(netstack_t *, nm_state_t *,
    kmutex_t *);

void
netstack_init(void)
{
        mutex_init(&netstack_g_lock, NULL, MUTEX_DEFAULT, NULL);
        mutex_init(&netstack_shared_lock, NULL, MUTEX_DEFAULT, NULL);

        netstack_initialized = 1;

        /*
         * We want to be informed each time a zone is created or
         * destroyed in the kernel, so we can maintain the
         * stack instance information.
         */
        zone_key_create(&netstack_zone_key, netstack_zone_create,
            netstack_zone_shutdown, netstack_zone_destroy);
}

/*
 * Register a new module with the framework.
 * This registers interest in changes to the set of netstacks.
 * The createfn and destroyfn are required, but the shutdownfn can be
 * NULL.
 * Note that due to the current zsd implementation, when the create
 * function is called the zone isn't fully present, thus functions
 * like zone_find_by_* will fail, hence the create function can not
 * use many zones kernel functions including zcmn_err().
 */
void
netstack_register(int moduleid,
    void *(*module_create)(netstackid_t, netstack_t *),
    void (*module_shutdown)(netstackid_t, void *),
    void (*module_destroy)(netstackid_t, void *))
{
        netstack_t *ns;

        ASSERT(netstack_initialized);
        ASSERT(moduleid >= 0 && moduleid < NS_MAX);
        ASSERT(module_create != NULL);

        /*
         * Make instances created after this point in time run the create
         * callback.
         */
        mutex_enter(&netstack_g_lock);
        ASSERT(ns_reg[moduleid].nr_create == NULL);
        ASSERT(ns_reg[moduleid].nr_flags == 0);
        ns_reg[moduleid].nr_create = module_create;
        ns_reg[moduleid].nr_shutdown = module_shutdown;
        ns_reg[moduleid].nr_destroy = module_destroy;
        ns_reg[moduleid].nr_flags = NRF_REGISTERED;

        /*
         * Determine the set of stacks that exist before we drop the lock.
         * Set NSS_CREATE_NEEDED for each of those.
         * netstacks which have been deleted will have NSS_CREATE_COMPLETED
         * set, but check NSF_CLOSING to be sure.
         */
        for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
                nm_state_t *nms = &ns->netstack_m_state[moduleid];

                mutex_enter(&ns->netstack_lock);
                if (!(ns->netstack_flags & NSF_CLOSING) &&
                    (nms->nms_flags & NSS_CREATE_ALL) == 0) {
                        nms->nms_flags |= NSS_CREATE_NEEDED;
                        DTRACE_PROBE2(netstack__create__needed,
                            netstack_t *, ns, int, moduleid);
                }
                mutex_exit(&ns->netstack_lock);
        }
        mutex_exit(&netstack_g_lock);

        /*
         * At this point in time a new instance can be created or an instance
         * can be destroyed, or some other module can register or unregister.
         * Make sure we either run all the create functions for this moduleid
         * or we wait for any other creators for this moduleid.
         */
        apply_all_netstacks(moduleid, netstack_apply_create);
}

void
netstack_unregister(int moduleid)
{
        netstack_t *ns;

        ASSERT(moduleid >= 0 && moduleid < NS_MAX);

        ASSERT(ns_reg[moduleid].nr_create != NULL);
        ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);

        mutex_enter(&netstack_g_lock);
        /*
         * Determine the set of stacks that exist before we drop the lock.
         * Set NSS_SHUTDOWN_NEEDED and NSS_DESTROY_NEEDED for each of those.
         * That ensures that when we return all the callbacks for existing
         * instances have completed. And since we set NRF_DYING no new
         * instances can use this module.
         */
        for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
                boolean_t created = B_FALSE;
                nm_state_t *nms = &ns->netstack_m_state[moduleid];

                mutex_enter(&ns->netstack_lock);

                /*
                 * We need to be careful here. We could actually have a netstack
                 * being created as we speak waiting for us to let go of this
                 * lock to proceed. It may have set NSS_CREATE_NEEDED, but not
                 * have gotten to the point of completing it yet. If
                 * NSS_CREATE_NEEDED, we can safely just remove it here and
                 * never create the module. However, if NSS_CREATE_INPROGRESS is
                 * set, we need to still flag this module for shutdown and
                 * deletion, just as though it had reached NSS_CREATE_COMPLETED.
                 *
                 * It is safe to do that because of two different guarantees
                 * that exist in the system. The first is that before we do a
                 * create, shutdown, or destroy, we ensure that nothing else is
                 * in progress in the system for this netstack and wait for it
                 * to complete. Secondly, because the zone is being created, we
                 * know that the following call to apply_all_netstack will block
                 * on the zone finishing its initialization.
                 */
                if (nms->nms_flags & NSS_CREATE_NEEDED)
                        nms->nms_flags &= ~NSS_CREATE_NEEDED;

                if (nms->nms_flags & NSS_CREATE_INPROGRESS ||
                    nms->nms_flags & NSS_CREATE_COMPLETED)
                        created = B_TRUE;

                if (ns_reg[moduleid].nr_shutdown != NULL && created &&
                    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
                    (nms->nms_flags & NSS_SHUTDOWN_ALL) == 0) {
                        nms->nms_flags |= NSS_SHUTDOWN_NEEDED;
                        DTRACE_PROBE2(netstack__shutdown__needed,
                            netstack_t *, ns, int, moduleid);
                }
                if ((ns_reg[moduleid].nr_flags & NRF_REGISTERED) &&
                    ns_reg[moduleid].nr_destroy != NULL && created &&
                    (nms->nms_flags & NSS_DESTROY_ALL) == 0) {
                        nms->nms_flags |= NSS_DESTROY_NEEDED;
                        DTRACE_PROBE2(netstack__destroy__needed,
                            netstack_t *, ns, int, moduleid);
                }
                mutex_exit(&ns->netstack_lock);
        }
        /*
         * Prevent any new netstack from calling the registered create
         * function, while keeping the function pointers in place until the
         * shutdown and destroy callbacks are complete.
         */
        ns_reg[moduleid].nr_flags |= NRF_DYING;
        mutex_exit(&netstack_g_lock);

        apply_all_netstacks(moduleid, netstack_apply_shutdown);
        apply_all_netstacks(moduleid, netstack_apply_destroy);

        /*
         * Clear the nms_flags so that we can handle this module
         * being loaded again.
         * Also remove the registered functions.
         */
        mutex_enter(&netstack_g_lock);
        ASSERT(ns_reg[moduleid].nr_flags & NRF_REGISTERED);
        ASSERT(ns_reg[moduleid].nr_flags & NRF_DYING);
        for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
                nm_state_t *nms = &ns->netstack_m_state[moduleid];

                mutex_enter(&ns->netstack_lock);
                if (nms->nms_flags & NSS_DESTROY_COMPLETED) {
                        nms->nms_flags = 0;
                        DTRACE_PROBE2(netstack__destroy__done,
                            netstack_t *, ns, int, moduleid);
                }
                mutex_exit(&ns->netstack_lock);
        }

        ns_reg[moduleid].nr_create = NULL;
        ns_reg[moduleid].nr_shutdown = NULL;
        ns_reg[moduleid].nr_destroy = NULL;
        ns_reg[moduleid].nr_flags = 0;
        mutex_exit(&netstack_g_lock);
}

/*
 * Lookup and/or allocate a netstack for this zone.
 */
static void *
netstack_zone_create(zoneid_t zoneid)
{
        netstackid_t stackid;
        netstack_t *ns;
        netstack_t **nsp;
        zone_t  *zone;
        int i;

        ASSERT(netstack_initialized);

        zone = zone_find_by_id_nolock(zoneid);
        ASSERT(zone != NULL);

        if (zone->zone_flags & ZF_NET_EXCL) {
                stackid = zoneid;
        } else {
                /* Look for the stack instance for the global */
                stackid = GLOBAL_NETSTACKID;
        }

        /* Allocate even if it isn't needed; simplifies locking */
        ns = (netstack_t *)kmem_zalloc(sizeof (netstack_t), KM_SLEEP);

        /* Look if there is a matching stack instance */
        mutex_enter(&netstack_g_lock);
        for (nsp = &netstack_head; *nsp != NULL;
            nsp = &((*nsp)->netstack_next)) {
                if ((*nsp)->netstack_stackid == stackid) {
                        /*
                         * Should never find a pre-existing exclusive stack
                         */
                        VERIFY(stackid == GLOBAL_NETSTACKID);
                        kmem_free(ns, sizeof (netstack_t));
                        ns = *nsp;
                        mutex_enter(&ns->netstack_lock);
                        ns->netstack_numzones++;
                        mutex_exit(&ns->netstack_lock);
                        mutex_exit(&netstack_g_lock);
                        DTRACE_PROBE1(netstack__inc__numzones,
                            netstack_t *, ns);
                        /* Record that we have a new shared stack zone */
                        netstack_shared_zone_add(zoneid);
                        zone->zone_netstack = ns;
                        return (ns);
                }
        }
        /* Not found */
        mutex_init(&ns->netstack_lock, NULL, MUTEX_DEFAULT, NULL);
        cv_init(&ns->netstack_cv, NULL, CV_DEFAULT, NULL);
        ns->netstack_stackid = zoneid;
        ns->netstack_numzones = 1;
        ns->netstack_refcnt = 1; /* Decremented by netstack_zone_destroy */
        ns->netstack_flags = NSF_UNINIT;
        *nsp = ns;
        zone->zone_netstack = ns;

        mutex_enter(&ns->netstack_lock);
        /*
         * Mark this netstack as having a CREATE running so
         * any netstack_register/netstack_unregister waits for
         * the existing create callbacks to complete in moduleid order
         */
        ns->netstack_flags |= NSF_ZONE_CREATE;

        /*
         * Determine the set of module create functions that need to be
         * called before we drop the lock.
         * Set NSS_CREATE_NEEDED for each of those.
         * Skip any with NRF_DYING set, since those are in the process of
         * going away, by checking for flags being exactly NRF_REGISTERED.
         */
        for (i = 0; i < NS_MAX; i++) {
                nm_state_t *nms = &ns->netstack_m_state[i];

                cv_init(&nms->nms_cv, NULL, CV_DEFAULT, NULL);

                if ((ns_reg[i].nr_flags == NRF_REGISTERED) &&
                    (nms->nms_flags & NSS_CREATE_ALL) == 0) {
                        nms->nms_flags |= NSS_CREATE_NEEDED;
                        DTRACE_PROBE2(netstack__create__needed,
                            netstack_t *, ns, int, i);
                }
        }
        mutex_exit(&ns->netstack_lock);
        mutex_exit(&netstack_g_lock);

        apply_all_modules(ns, netstack_apply_create);

        /* Tell any waiting netstack_register/netstack_unregister to proceed */
        mutex_enter(&ns->netstack_lock);
        ns->netstack_flags &= ~NSF_UNINIT;
        ASSERT(ns->netstack_flags & NSF_ZONE_CREATE);
        ns->netstack_flags &= ~NSF_ZONE_CREATE;
        cv_broadcast(&ns->netstack_cv);
        mutex_exit(&ns->netstack_lock);

        return (ns);
}

/* ARGSUSED */
static void
netstack_zone_shutdown(zoneid_t zoneid, void *arg)
{
        netstack_t *ns = (netstack_t *)arg;
        int i;

        ASSERT(arg != NULL);

        mutex_enter(&ns->netstack_lock);
        ASSERT(ns->netstack_numzones > 0);
        if (ns->netstack_numzones != 1) {
                /* Stack instance being used by other zone */
                mutex_exit(&ns->netstack_lock);
                ASSERT(ns->netstack_stackid == GLOBAL_NETSTACKID);
                return;
        }
        mutex_exit(&ns->netstack_lock);

        mutex_enter(&netstack_g_lock);
        mutex_enter(&ns->netstack_lock);
        /*
         * Mark this netstack as having a SHUTDOWN running so
         * any netstack_register/netstack_unregister waits for
         * the existing create callbacks to complete in moduleid order
         */
        ASSERT(!(ns->netstack_flags & NSF_ZONE_INPROGRESS));
        ns->netstack_flags |= NSF_ZONE_SHUTDOWN;

        /*
         * Determine the set of stacks that exist before we drop the lock.
         * Set NSS_SHUTDOWN_NEEDED for each of those.
         */
        for (i = 0; i < NS_MAX; i++) {
                nm_state_t *nms = &ns->netstack_m_state[i];

                if ((ns_reg[i].nr_flags & NRF_REGISTERED) &&
                    ns_reg[i].nr_shutdown != NULL &&
                    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
                    (nms->nms_flags & NSS_SHUTDOWN_ALL) == 0) {
                        nms->nms_flags |= NSS_SHUTDOWN_NEEDED;
                        DTRACE_PROBE2(netstack__shutdown__needed,
                            netstack_t *, ns, int, i);
                }
        }
        mutex_exit(&ns->netstack_lock);
        mutex_exit(&netstack_g_lock);

        /*
         * Call the shutdown function for all registered modules for this
         * netstack.
         */
        apply_all_modules_reverse(ns, netstack_apply_shutdown);

        /* Tell any waiting netstack_register/netstack_unregister to proceed */
        mutex_enter(&ns->netstack_lock);
        ASSERT(ns->netstack_flags & NSF_ZONE_SHUTDOWN);
        ns->netstack_flags &= ~NSF_ZONE_SHUTDOWN;
        cv_broadcast(&ns->netstack_cv);
        mutex_exit(&ns->netstack_lock);
}

/*
 * Common routine to release a zone.
 * If this was the last zone using the stack instance then prepare to
 * have the refcnt dropping to zero free the zone.
 */
/* ARGSUSED */
static void
netstack_zone_destroy(zoneid_t zoneid, void *arg)
{
        netstack_t *ns = (netstack_t *)arg;

        ASSERT(arg != NULL);

        mutex_enter(&ns->netstack_lock);
        ASSERT(ns->netstack_numzones > 0);
        ns->netstack_numzones--;
        if (ns->netstack_numzones != 0) {
                /* Stack instance being used by other zone */
                mutex_exit(&ns->netstack_lock);
                ASSERT(ns->netstack_stackid == GLOBAL_NETSTACKID);
                /* Record that we a shared stack zone has gone away */
                netstack_shared_zone_remove(zoneid);
                return;
        }
        /*
         * Set CLOSING so that netstack_find_by will not find it.
         */
        ns->netstack_flags |= NSF_CLOSING;
        mutex_exit(&ns->netstack_lock);
        DTRACE_PROBE1(netstack__dec__numzones, netstack_t *, ns);
        /* No other thread can call zone_destroy for this stack */

        /*
         * Decrease refcnt to account for the one in netstack_zone_init()
         */
        netstack_rele(ns);
}

/*
 * Called when the reference count drops to zero.
 * Call the destroy functions for each registered module.
 */
static void
netstack_stack_inactive(netstack_t *ns)
{
        int i;

        mutex_enter(&netstack_g_lock);
        mutex_enter(&ns->netstack_lock);
        /*
         * Mark this netstack as having a DESTROY running so
         * any netstack_register/netstack_unregister waits for
         * the existing destroy callbacks to complete in reverse moduleid order
         */
        ASSERT(!(ns->netstack_flags & NSF_ZONE_INPROGRESS));
        ns->netstack_flags |= NSF_ZONE_DESTROY;
        /*
         * If the shutdown callback wasn't called earlier (e.g., if this is
         * a netstack shared between multiple zones), then we schedule it now.
         *
         * Determine the set of stacks that exist before we drop the lock.
         * Set NSS_DESTROY_NEEDED for each of those. That
         * ensures that when we return all the callbacks for existing
         * instances have completed.
         */
        for (i = 0; i < NS_MAX; i++) {
                nm_state_t *nms = &ns->netstack_m_state[i];

                if ((ns_reg[i].nr_flags & NRF_REGISTERED) &&
                    ns_reg[i].nr_shutdown != NULL &&
                    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
                    (nms->nms_flags & NSS_SHUTDOWN_ALL) == 0) {
                        nms->nms_flags |= NSS_SHUTDOWN_NEEDED;
                        DTRACE_PROBE2(netstack__shutdown__needed,
                            netstack_t *, ns, int, i);
                }

                if ((ns_reg[i].nr_flags & NRF_REGISTERED) &&
                    ns_reg[i].nr_destroy != NULL &&
                    (nms->nms_flags & NSS_CREATE_COMPLETED) &&
                    (nms->nms_flags & NSS_DESTROY_ALL) == 0) {
                        nms->nms_flags |= NSS_DESTROY_NEEDED;
                        DTRACE_PROBE2(netstack__destroy__needed,
                            netstack_t *, ns, int, i);
                }
        }
        mutex_exit(&ns->netstack_lock);
        mutex_exit(&netstack_g_lock);

        /*
         * Call the shutdown and destroy functions for all registered modules
         * for this netstack.
         *
         * Since there are some ordering dependencies between the modules we
         * tear them down in the reverse order of what was used to create them.
         *
         * Since a netstack_t is never reused (when a zone is rebooted it gets
         * a new zoneid == netstackid i.e. a new netstack_t is allocated) we
         * leave nms_flags the way it is i.e. with NSS_DESTROY_COMPLETED set.
         * That is different than in the netstack_unregister() case.
         */
        apply_all_modules_reverse(ns, netstack_apply_shutdown);
        apply_all_modules_reverse(ns, netstack_apply_destroy);

        /* Tell any waiting netstack_register/netstack_unregister to proceed */
        mutex_enter(&ns->netstack_lock);
        ASSERT(ns->netstack_flags & NSF_ZONE_DESTROY);
        ns->netstack_flags &= ~NSF_ZONE_DESTROY;
        cv_broadcast(&ns->netstack_cv);
        mutex_exit(&ns->netstack_lock);
}

/*
 * Apply a function to all netstacks for a particular moduleid.
 *
 * If there is any zone activity (due to a zone being created, shutdown,
 * or destroyed) we wait for that to complete before we proceed. This ensures
 * that the moduleids are processed in order when a zone is created or
 * destroyed.
 *
 * The applyfn has to drop netstack_g_lock if it does some work.
 * In that case we don't follow netstack_next,
 * even if it is possible to do so without any hazards. This is
 * because we want the design to allow for the list of netstacks threaded
 * by netstack_next to change in any arbitrary way during the time the
 * lock was dropped.
 *
 * It is safe to restart the loop at netstack_head since the applyfn
 * changes netstack_m_state as it processes things, so a subsequent
 * pass through will have no effect in applyfn, hence the loop will terminate
 * in at worst O(N^2).
 */
static void
apply_all_netstacks(int moduleid, applyfn_t *applyfn)
{
        netstack_t *ns;

        mutex_enter(&netstack_g_lock);
        ns = netstack_head;
        while (ns != NULL) {
                if (wait_for_zone_creator(ns, &netstack_g_lock)) {
                        /* Lock dropped - restart at head */
                        ns = netstack_head;
                } else if ((applyfn)(&netstack_g_lock, ns, moduleid)) {
                        /* Lock dropped - restart at head */
                        ns = netstack_head;
                } else {
                        ns = ns->netstack_next;
                }
        }
        mutex_exit(&netstack_g_lock);
}

/*
 * Apply a function to all moduleids for a particular netstack.
 *
 * Since the netstack linkage doesn't matter in this case we can
 * ignore whether the function drops the lock.
 */
static void
apply_all_modules(netstack_t *ns, applyfn_t *applyfn)
{
        int i;

        mutex_enter(&netstack_g_lock);
        for (i = 0; i < NS_MAX; i++) {
                /*
                 * We don't care whether the lock was dropped
                 * since we are not iterating over netstack_head.
                 */
                (void) (applyfn)(&netstack_g_lock, ns, i);
        }
        mutex_exit(&netstack_g_lock);
}

/* Like the above but in reverse moduleid order */
static void
apply_all_modules_reverse(netstack_t *ns, applyfn_t *applyfn)
{
        int i;

        mutex_enter(&netstack_g_lock);
        for (i = NS_MAX-1; i >= 0; i--) {
                /*
                 * We don't care whether the lock was dropped
                 * since we are not iterating over netstack_head.
                 */
                (void) (applyfn)(&netstack_g_lock, ns, i);
        }
        mutex_exit(&netstack_g_lock);
}

/*
 * Call the create function for the ns and moduleid if CREATE_NEEDED
 * is set.
 * If some other thread gets here first and sets *_INPROGRESS, then
 * we wait for that thread to complete so that we can ensure that
 * all the callbacks are done when we've looped over all netstacks/moduleids.
 *
 * When we call the create function, we temporarily drop the netstack_lock
 * held by the caller, and return true to tell the caller it needs to
 * re-evalute the state.
 */
static boolean_t
netstack_apply_create(kmutex_t *lockp, netstack_t *ns, int moduleid)
{
        void *result;
        netstackid_t stackid;
        nm_state_t *nms = &ns->netstack_m_state[moduleid];
        boolean_t dropped = B_FALSE;

        ASSERT(MUTEX_HELD(lockp));
        mutex_enter(&ns->netstack_lock);

        if (wait_for_nms_inprogress(ns, nms, lockp))
                dropped = B_TRUE;

        if (nms->nms_flags & NSS_CREATE_NEEDED) {
                nms->nms_flags &= ~NSS_CREATE_NEEDED;
                nms->nms_flags |= NSS_CREATE_INPROGRESS;
                DTRACE_PROBE2(netstack__create__inprogress,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                mutex_exit(lockp);
                dropped = B_TRUE;

                ASSERT(ns_reg[moduleid].nr_create != NULL);
                stackid = ns->netstack_stackid;
                DTRACE_PROBE2(netstack__create__start,
                    netstackid_t, stackid,
                    netstack_t *, ns);
                result = (ns_reg[moduleid].nr_create)(stackid, ns);
                DTRACE_PROBE2(netstack__create__end,
                    void *, result, netstack_t *, ns);

                ASSERT(result != NULL);
                mutex_enter(lockp);
                mutex_enter(&ns->netstack_lock);
                ns->netstack_modules[moduleid] = result;
                nms->nms_flags &= ~NSS_CREATE_INPROGRESS;
                nms->nms_flags |= NSS_CREATE_COMPLETED;
                cv_broadcast(&nms->nms_cv);
                DTRACE_PROBE2(netstack__create__completed,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        } else {
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        }
}

/*
 * Call the shutdown function for the ns and moduleid if SHUTDOWN_NEEDED
 * is set.
 * If some other thread gets here first and sets *_INPROGRESS, then
 * we wait for that thread to complete so that we can ensure that
 * all the callbacks are done when we've looped over all netstacks/moduleids.
 *
 * When we call the shutdown function, we temporarily drop the netstack_lock
 * held by the caller, and return true to tell the caller it needs to
 * re-evalute the state.
 */
static boolean_t
netstack_apply_shutdown(kmutex_t *lockp, netstack_t *ns, int moduleid)
{
        netstackid_t stackid;
        void * netstack_module;
        nm_state_t *nms = &ns->netstack_m_state[moduleid];
        boolean_t dropped = B_FALSE;

        ASSERT(MUTEX_HELD(lockp));
        mutex_enter(&ns->netstack_lock);

        if (wait_for_nms_inprogress(ns, nms, lockp))
                dropped = B_TRUE;

        if (nms->nms_flags & NSS_SHUTDOWN_NEEDED) {
                nms->nms_flags &= ~NSS_SHUTDOWN_NEEDED;
                nms->nms_flags |= NSS_SHUTDOWN_INPROGRESS;
                DTRACE_PROBE2(netstack__shutdown__inprogress,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                mutex_exit(lockp);
                dropped = B_TRUE;

                ASSERT(ns_reg[moduleid].nr_shutdown != NULL);
                stackid = ns->netstack_stackid;
                netstack_module = ns->netstack_modules[moduleid];
                DTRACE_PROBE2(netstack__shutdown__start,
                    netstackid_t, stackid,
                    void *, netstack_module);
                (ns_reg[moduleid].nr_shutdown)(stackid, netstack_module);
                DTRACE_PROBE1(netstack__shutdown__end,
                    netstack_t *, ns);

                mutex_enter(lockp);
                mutex_enter(&ns->netstack_lock);
                nms->nms_flags &= ~NSS_SHUTDOWN_INPROGRESS;
                nms->nms_flags |= NSS_SHUTDOWN_COMPLETED;
                cv_broadcast(&nms->nms_cv);
                DTRACE_PROBE2(netstack__shutdown__completed,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        } else {
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        }
}

/*
 * Call the destroy function for the ns and moduleid if DESTROY_NEEDED
 * is set.
 * If some other thread gets here first and sets *_INPROGRESS, then
 * we wait for that thread to complete so that we can ensure that
 * all the callbacks are done when we've looped over all netstacks/moduleids.
 *
 * When we call the destroy function, we temporarily drop the netstack_lock
 * held by the caller, and return true to tell the caller it needs to
 * re-evalute the state.
 */
static boolean_t
netstack_apply_destroy(kmutex_t *lockp, netstack_t *ns, int moduleid)
{
        netstackid_t stackid;
        void * netstack_module;
        nm_state_t *nms = &ns->netstack_m_state[moduleid];
        boolean_t dropped = B_FALSE;

        ASSERT(MUTEX_HELD(lockp));
        mutex_enter(&ns->netstack_lock);

        if (wait_for_nms_inprogress(ns, nms, lockp))
                dropped = B_TRUE;

        if (nms->nms_flags & NSS_DESTROY_NEEDED) {
                nms->nms_flags &= ~NSS_DESTROY_NEEDED;
                nms->nms_flags |= NSS_DESTROY_INPROGRESS;
                DTRACE_PROBE2(netstack__destroy__inprogress,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                mutex_exit(lockp);
                dropped = B_TRUE;

                ASSERT(ns_reg[moduleid].nr_destroy != NULL);
                stackid = ns->netstack_stackid;
                netstack_module = ns->netstack_modules[moduleid];
                DTRACE_PROBE2(netstack__destroy__start,
                    netstackid_t, stackid,
                    void *, netstack_module);
                (ns_reg[moduleid].nr_destroy)(stackid, netstack_module);
                DTRACE_PROBE1(netstack__destroy__end,
                    netstack_t *, ns);

                mutex_enter(lockp);
                mutex_enter(&ns->netstack_lock);
                ns->netstack_modules[moduleid] = NULL;
                nms->nms_flags &= ~NSS_DESTROY_INPROGRESS;
                nms->nms_flags |= NSS_DESTROY_COMPLETED;
                cv_broadcast(&nms->nms_cv);
                DTRACE_PROBE2(netstack__destroy__completed,
                    netstack_t *, ns, int, moduleid);
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        } else {
                mutex_exit(&ns->netstack_lock);
                return (dropped);
        }
}

/*
 * If somebody  is creating the netstack (due to a new zone being created)
 * then we wait for them to complete. This ensures that any additional
 * netstack_register() doesn't cause the create functions to run out of
 * order.
 * Note that we do not need such a global wait in the case of the shutdown
 * and destroy callbacks, since in that case it is sufficient for both
 * threads to set NEEDED and wait for INPROGRESS to ensure ordering.
 * Returns true if lockp was temporarily dropped while waiting.
 */
static boolean_t
wait_for_zone_creator(netstack_t *ns, kmutex_t *lockp)
{
        boolean_t dropped = B_FALSE;

        mutex_enter(&ns->netstack_lock);
        while (ns->netstack_flags & NSF_ZONE_CREATE) {
                DTRACE_PROBE1(netstack__wait__zone__inprogress,
                    netstack_t *, ns);
                if (lockp != NULL) {
                        dropped = B_TRUE;
                        mutex_exit(lockp);
                }
                cv_wait(&ns->netstack_cv, &ns->netstack_lock);
                if (lockp != NULL) {
                        /* First drop netstack_lock to preserve order */
                        mutex_exit(&ns->netstack_lock);
                        mutex_enter(lockp);
                        mutex_enter(&ns->netstack_lock);
                }
        }
        mutex_exit(&ns->netstack_lock);
        return (dropped);
}

/*
 * Wait for any INPROGRESS flag to be cleared for the netstack/moduleid
 * combination.
 * Returns true if lockp was temporarily dropped while waiting.
 */
static boolean_t
wait_for_nms_inprogress(netstack_t *ns, nm_state_t *nms, kmutex_t *lockp)
{
        boolean_t dropped = B_FALSE;

        while (nms->nms_flags & NSS_ALL_INPROGRESS) {
                DTRACE_PROBE2(netstack__wait__nms__inprogress,
                    netstack_t *, ns, nm_state_t *, nms);
                if (lockp != NULL) {
                        dropped = B_TRUE;
                        mutex_exit(lockp);
                }
                cv_wait(&nms->nms_cv, &ns->netstack_lock);
                if (lockp != NULL) {
                        /* First drop netstack_lock to preserve order */
                        mutex_exit(&ns->netstack_lock);
                        mutex_enter(lockp);
                        mutex_enter(&ns->netstack_lock);
                }
        }
        return (dropped);
}

/*
 * Get the stack instance used in caller's zone.
 * Increases the reference count, caller must do a netstack_rele.
 * It can't be called after zone_destroy() has started.
 */
netstack_t *
netstack_get_current(void)
{
        netstack_t *ns;

        ns = curproc->p_zone->zone_netstack;
        ASSERT(ns != NULL);
        if (ns->netstack_flags & (NSF_UNINIT|NSF_CLOSING))
                return (NULL);

        netstack_hold(ns);

        return (ns);
}

/*
 * Find a stack instance given the cred.
 * This is used by the modules to potentially allow for a future when
 * something other than the zoneid is used to determine the stack.
 */
netstack_t *
netstack_find_by_cred(const cred_t *cr)
{
        zoneid_t zoneid = crgetzoneid(cr);

        /* Handle the case when cr_zone is NULL */
        if (zoneid == (zoneid_t)-1)
                zoneid = GLOBAL_ZONEID;

        /* For performance ... */
        if (curproc->p_zone->zone_id == zoneid)
                return (netstack_get_current());
        else
                return (netstack_find_by_zoneid(zoneid));
}

/*
 * Find a stack instance given the zoneid.
 * Increases the reference count if found; caller must do a
 * netstack_rele().
 *
 * If there is no exact match then assume the shared stack instance
 * matches.
 *
 * Skip the unitialized ones.
 */
netstack_t *
netstack_find_by_zoneid(zoneid_t zoneid)
{
        netstack_t *ns;
        zone_t *zone;

        zone = zone_find_by_id(zoneid);

        if (zone == NULL)
                return (NULL);

        ns = zone->zone_netstack;
        ASSERT(ns != NULL);
        if (ns->netstack_flags & (NSF_UNINIT|NSF_CLOSING))
                ns = NULL;
        else
                netstack_hold(ns);

        zone_rele(zone);
        return (ns);
}

/*
 * Find a stack instance given the zoneid. Can only be called from
 * the create callback. See the comments in zone_find_by_id_nolock why
 * that limitation exists.
 *
 * Increases the reference count if found; caller must do a
 * netstack_rele().
 *
 * If there is no exact match then assume the shared stack instance
 * matches.
 *
 * Skip the unitialized ones.
 */
netstack_t *
netstack_find_by_zoneid_nolock(zoneid_t zoneid)
{
        netstack_t *ns;
        zone_t *zone;

        zone = zone_find_by_id_nolock(zoneid);

        if (zone == NULL)
                return (NULL);

        ns = zone->zone_netstack;
        ASSERT(ns != NULL);

        if (ns->netstack_flags & (NSF_UNINIT|NSF_CLOSING))
                ns = NULL;
        else
                netstack_hold(ns);

        /* zone_find_by_id_nolock does not have a hold on the zone */
        return (ns);
}

/*
 * Find a stack instance given the stackid with exact match?
 * Increases the reference count if found; caller must do a
 * netstack_rele().
 *
 * Skip the unitialized ones.
 */
netstack_t *
netstack_find_by_stackid(netstackid_t stackid)
{
        netstack_t *ns;

        mutex_enter(&netstack_g_lock);
        for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
                mutex_enter(&ns->netstack_lock);
                if (ns->netstack_stackid == stackid &&
                    !(ns->netstack_flags & (NSF_UNINIT|NSF_CLOSING))) {
                        mutex_exit(&ns->netstack_lock);
                        netstack_hold(ns);
                        mutex_exit(&netstack_g_lock);
                        return (ns);
                }
                mutex_exit(&ns->netstack_lock);
        }
        mutex_exit(&netstack_g_lock);
        return (NULL);
}

boolean_t
netstack_inuse_by_stackid(netstackid_t stackid)
{
        netstack_t *ns;
        boolean_t rval = B_FALSE;

        mutex_enter(&netstack_g_lock);

        for (ns = netstack_head; ns != NULL; ns = ns->netstack_next) {
                if (ns->netstack_stackid == stackid) {
                        rval = B_TRUE;
                        break;
                }
        }

        mutex_exit(&netstack_g_lock);

        return (rval);
}

void
netstack_rele(netstack_t *ns)
{
        netstack_t **nsp;
        boolean_t found;
        int refcnt, numzones;
        int i;

        mutex_enter(&ns->netstack_lock);
        ASSERT(ns->netstack_refcnt > 0);
        ns->netstack_refcnt--;
        /*
         * As we drop the lock additional netstack_rele()s can come in
         * and decrement the refcnt to zero and free the netstack_t.
         * Store pointers in local variables and if we were not the last
         * then don't reference the netstack_t after that.
         */
        refcnt = ns->netstack_refcnt;
        numzones = ns->netstack_numzones;
        DTRACE_PROBE1(netstack__dec__ref, netstack_t *, ns);
        mutex_exit(&ns->netstack_lock);

        if (refcnt == 0 && numzones == 0) {
                /*
                 * Time to call the destroy functions and free up
                 * the structure
                 */
                netstack_stack_inactive(ns);

                /* Make sure nothing increased the references */
                ASSERT(ns->netstack_refcnt == 0);
                ASSERT(ns->netstack_numzones == 0);

                /* Finally remove from list of netstacks */
                mutex_enter(&netstack_g_lock);
                found = B_FALSE;
                for (nsp = &netstack_head; *nsp != NULL;
                    nsp = &(*nsp)->netstack_next) {
                        if (*nsp == ns) {
                                *nsp = ns->netstack_next;
                                ns->netstack_next = NULL;
                                found = B_TRUE;
                                break;
                        }
                }
                ASSERT(found);
                mutex_exit(&netstack_g_lock);

                /* Make sure nothing increased the references */
                ASSERT(ns->netstack_refcnt == 0);
                ASSERT(ns->netstack_numzones == 0);

                ASSERT(ns->netstack_flags & NSF_CLOSING);

                for (i = 0; i < NS_MAX; i++) {
                        nm_state_t *nms = &ns->netstack_m_state[i];

                        cv_destroy(&nms->nms_cv);
                }
                mutex_destroy(&ns->netstack_lock);
                cv_destroy(&ns->netstack_cv);
                kmem_free(ns, sizeof (*ns));
        }
}

void
netstack_hold(netstack_t *ns)
{
        mutex_enter(&ns->netstack_lock);
        ns->netstack_refcnt++;
        ASSERT(ns->netstack_refcnt > 0);
        mutex_exit(&ns->netstack_lock);
        DTRACE_PROBE1(netstack__inc__ref, netstack_t *, ns);
}

/*
 * To support kstat_create_netstack() using kstat_zone_add we need
 * to track both
 *  - all zoneids that use the global/shared stack
 *  - all kstats that have been added for the shared stack
 */
kstat_t *
kstat_create_netstack(char *ks_module, int ks_instance, char *ks_name,
    char *ks_class, uchar_t ks_type, uint_t ks_ndata, uchar_t ks_flags,
    netstackid_t ks_netstackid)
{
        kstat_t *ks;

        if (ks_netstackid == GLOBAL_NETSTACKID) {
                ks = kstat_create_zone(ks_module, ks_instance, ks_name,
                    ks_class, ks_type, ks_ndata, ks_flags, GLOBAL_ZONEID);
                if (ks != NULL)
                        netstack_shared_kstat_add(ks);
                return (ks);
        } else {
                zoneid_t zoneid = ks_netstackid;

                return (kstat_create_zone(ks_module, ks_instance, ks_name,
                    ks_class, ks_type, ks_ndata, ks_flags, zoneid));
        }
}

void
kstat_delete_netstack(kstat_t *ks, netstackid_t ks_netstackid)
{
        if (ks_netstackid == GLOBAL_NETSTACKID) {
                netstack_shared_kstat_remove(ks);
        }
        kstat_delete(ks);
}

static void
netstack_shared_zone_add(zoneid_t zoneid)
{
        struct shared_zone_list *sz;
        struct shared_kstat_list *sk;

        sz = kmem_zalloc(sizeof (*sz), KM_SLEEP);
        sz->sz_zoneid = zoneid;

        /* Insert in list */
        mutex_enter(&netstack_shared_lock);
        sz->sz_next = netstack_shared_zones;
        netstack_shared_zones = sz;

        /*
         * Perform kstat_zone_add for each existing shared stack kstat.
         * Note: Holds netstack_shared_lock lock across kstat_zone_add.
         */
        for (sk = netstack_shared_kstats; sk != NULL; sk = sk->sk_next) {
                kstat_zone_add(sk->sk_kstat, zoneid);
        }
        mutex_exit(&netstack_shared_lock);
}

static void
netstack_shared_zone_remove(zoneid_t zoneid)
{
        struct shared_zone_list **szp, *sz;
        struct shared_kstat_list *sk;

        /* Find in list */
        mutex_enter(&netstack_shared_lock);
        sz = NULL;
        for (szp = &netstack_shared_zones; *szp != NULL;
            szp = &((*szp)->sz_next)) {
                if ((*szp)->sz_zoneid == zoneid) {
                        sz = *szp;
                        break;
                }
        }
        /* We must find it */
        ASSERT(sz != NULL);
        *szp = sz->sz_next;
        sz->sz_next = NULL;

        /*
         * Perform kstat_zone_remove for each existing shared stack kstat.
         * Note: Holds netstack_shared_lock lock across kstat_zone_remove.
         */
        for (sk = netstack_shared_kstats; sk != NULL; sk = sk->sk_next) {
                kstat_zone_remove(sk->sk_kstat, zoneid);
        }
        mutex_exit(&netstack_shared_lock);

        kmem_free(sz, sizeof (*sz));
}

static void
netstack_shared_kstat_add(kstat_t *ks)
{
        struct shared_zone_list *sz;
        struct shared_kstat_list *sk;

        sk = kmem_zalloc(sizeof (*sk), KM_SLEEP);
        sk->sk_kstat = ks;

        /* Insert in list */
        mutex_enter(&netstack_shared_lock);
        sk->sk_next = netstack_shared_kstats;
        netstack_shared_kstats = sk;

        /*
         * Perform kstat_zone_add for each existing shared stack zone.
         * Note: Holds netstack_shared_lock lock across kstat_zone_add.
         */
        for (sz = netstack_shared_zones; sz != NULL; sz = sz->sz_next) {
                kstat_zone_add(ks, sz->sz_zoneid);
        }
        mutex_exit(&netstack_shared_lock);
}

static void
netstack_shared_kstat_remove(kstat_t *ks)
{
        struct shared_zone_list *sz;
        struct shared_kstat_list **skp, *sk;

        /* Find in list */
        mutex_enter(&netstack_shared_lock);
        sk = NULL;
        for (skp = &netstack_shared_kstats; *skp != NULL;
            skp = &((*skp)->sk_next)) {
                if ((*skp)->sk_kstat == ks) {
                        sk = *skp;
                        break;
                }
        }
        /* Must find it */
        ASSERT(sk != NULL);
        *skp = sk->sk_next;
        sk->sk_next = NULL;

        /*
         * Perform kstat_zone_remove for each existing shared stack kstat.
         * Note: Holds netstack_shared_lock lock across kstat_zone_remove.
         */
        for (sz = netstack_shared_zones; sz != NULL; sz = sz->sz_next) {
                kstat_zone_remove(ks, sz->sz_zoneid);
        }
        mutex_exit(&netstack_shared_lock);
        kmem_free(sk, sizeof (*sk));
}

/*
 * If a zoneid is part of the shared zone, return true
 */
static boolean_t
netstack_find_shared_zoneid(zoneid_t zoneid)
{
        struct shared_zone_list *sz;

        mutex_enter(&netstack_shared_lock);
        for (sz = netstack_shared_zones; sz != NULL; sz = sz->sz_next) {
                if (sz->sz_zoneid == zoneid) {
                        mutex_exit(&netstack_shared_lock);
                        return (B_TRUE);
                }
        }
        mutex_exit(&netstack_shared_lock);
        return (B_FALSE);
}

/*
 * Hide the fact that zoneids and netstackids are allocated from
 * the same space in the current implementation.
 * We currently do not check that the stackid/zoneids are valid, since there
 * is no need for that. But this should only be done for ids that are
 * valid.
 */
zoneid_t
netstackid_to_zoneid(netstackid_t stackid)
{
        return (stackid);
}

netstackid_t
zoneid_to_netstackid(zoneid_t zoneid)
{
        if (netstack_find_shared_zoneid(zoneid))
                return (GLOBAL_ZONEID);
        else
                return (zoneid);
}

zoneid_t
netstack_get_zoneid(netstack_t *ns)
{
        return (netstackid_to_zoneid(ns->netstack_stackid));
}

/*
 * Simplistic support for walking all the handles.
 * Example usage:
 *      netstack_handle_t nh;
 *      netstack_t *ns;
 *
 *      netstack_next_init(&nh);
 *      while ((ns = netstack_next(&nh)) != NULL) {
 *              do something;
 *              netstack_rele(ns);
 *      }
 *      netstack_next_fini(&nh);
 */
void
netstack_next_init(netstack_handle_t *handle)
{
        *handle = 0;
}

/* ARGSUSED */
void
netstack_next_fini(netstack_handle_t *handle)
{
}

netstack_t *
netstack_next(netstack_handle_t *handle)
{
        netstack_t *ns;
        int i, end;

        end = *handle;
        /* Walk skipping *handle number of instances */

        /* Look if there is a matching stack instance */
        mutex_enter(&netstack_g_lock);
        ns = netstack_head;
        for (i = 0; i < end; i++) {
                if (ns == NULL)
                        break;
                ns = ns->netstack_next;
        }
        /* skip those with that aren't really here */
        while (ns != NULL) {
                mutex_enter(&ns->netstack_lock);
                if ((ns->netstack_flags & (NSF_UNINIT|NSF_CLOSING)) == 0) {
                        mutex_exit(&ns->netstack_lock);
                        break;
                }
                mutex_exit(&ns->netstack_lock);
                end++;
                ns = ns->netstack_next;
        }
        if (ns != NULL) {
                *handle = end + 1;
                netstack_hold(ns);
        }
        mutex_exit(&netstack_g_lock);
        return (ns);
}