Unleashed v1.4

[unleashed.git] / kernel / os / pool.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/pool.h>
#include <sys/pool_impl.h>
#include <sys/pool_pset.h>
#include <sys/id_space.h>
#include <sys/mutex.h>
#include <sys/nvpair.h>
#include <sys/cpuvar.h>
#include <sys/errno.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/fss.h>
#include <sys/class.h>
#include <sys/exacct.h>
#include <sys/utsname.h>
#include <sys/procset.h>
#include <sys/atomic.h>
#include <sys/zone.h>
#include <sys/policy.h>
#include <sys/schedctl.h>
#include <sys/taskq.h>

/*
 * RESOURCE POOLS
 *
 * The resource pools facility brings together process-bindable resource into
 * a common abstraction called a pool. Processor sets and other entities can
 * be configured, grouped, and labelled such that workload components can be
 * associated with a subset of a system's total resources.
 *
 * When disabled, the pools facility is "invisible".  All processes belong
 * to the same pool (pool_default), and processor sets can be managed through
 * the old pset() system call.  When enabled, processor sets can only be
 * managed via the pools facility.  New pools can be created and associated
 * with processor sets.  Processes can be bound to pools which have non-empty
 * resource sets.
 *
 * Locking: pool_lock() protects global pools state and must be called
 * before modifying the configuration, or when taking a snapshot of the
 * configuration.  If pool_lock_intr() is used, the operation may be
 * interrupted by a signal or a request.
 *
 * To prevent processes from being rebound between pools while they are
 * the middle of an operation which affects resource set bindings, such
 * operations must be surrounded by calls to pool_barrier_enter() and
 * pool_barrier_exit().  This mechanism guarantees that such processes will
 * be stopped either at the beginning or at the end of the barrier so that
 * the rebind operation can atomically bind the process and its threads
 * to new resource sets, and then let process run again.
 *
 * Lock ordering with respect to other locks is as follows:
 *
 *      pool_lock() -> cpu_lock -> pidlock -> p_lock -> pool_barrier_lock
 *
 * Most static and global variables defined in this file are protected
 * by calling pool_lock().
 *
 * The operation that binds tasks and projects to pools is atomic.  That is,
 * either all processes in a given task or a project will be bound to a
 * new pool, or (in case of an error) they will be all left bound to the
 * old pool. Processes in a given task or a given project can only be bound to
 * different pools if they were rebound individually one by one as single
 * processes.  Threads or LWPs of the same process do not have pool bindings,
 * and are bound to the same resource sets associated with the resource pool
 * of that process.
 *
 * The following picture shows one possible pool configuration with three
 * pools and three processor sets.  Note that processor set "foo" is not
 * associated with any pools and therefore cannot have any processes
 * bound to it.  Two pools (default and foo) are associated with the
 * same processor set (default).  Also, note that processes in Task 2
 * are bound to different pools.
 *
 *
 *                                                             Processor Sets
 *                                                              +---------+
 *                     +--------------+========================>| default |
 *                    a|              |                         +---------+
 *                    s|              |                             ||
 *                    s|              |                         +---------+
 *                    o|              |                         |   foo   |
 *                    c|              |                         +---------+
 *                    i|              |                             ||
 *                    a|              |                         +---------+
 *                    t|              |                 +------>|   bar   |
 *                    e|              |                 |       +---------+
 *                    d|              |                 |
 *                     |              |                 |
 *             +---------+      +---------+      +---------+
 *     Pools   | default |======|   foo   |======|   bar   |
 *             +---------+      +---------+      +---------+
 *                 @  @            @              @ @   @
 *                b|  |            |              | |   |
 *                o|  |            |              | |   |
 *                u|  +-----+      |      +-------+ |   +---+
 *                n|        |      |      |         |       |
 *            ....d|........|......|......|.........|.......|....
 *            :    |   ::   |      |      |    ::   |       |   :
 *            :  +---+ :: +---+  +---+  +---+  :: +---+   +---+ :
 *  Processes :  | p | :: | p |  | p |  | p |  :: | p |...| p | :
 *            :  +---+ :: +---+  +---+  +---+  :: +---+   +---+ :
 *            :........::......................::...............:
 *              Task 1            Task 2              Task N
 *                 |                 |                  |
 *                 |                 |                  |
 *                 |  +-----------+  |             +-----------+
 *                 +--| Project 1 |--+             | Project N |
 *                    +-----------+                +-----------+
 *
 * This is just an illustration of relationships between processes, tasks,
 * projects, pools, and processor sets. New types of resource sets will be
 * added in the future.
 */

pool_t          *pool_default;  /* default pool which always exists */
int             pool_count;     /* number of pools created on this system */
int             pool_state;     /* pools state -- enabled/disabled */
void            *pool_buf;      /* pre-commit snapshot of the pools state */
size_t          pool_bufsz;     /* size of pool_buf */
static hrtime_t pool_pool_mod;  /* last modification time for pools */
static hrtime_t pool_sys_mod;   /* last modification time for system */
static nvlist_t *pool_sys_prop; /* system properties */
static id_space_t *pool_ids;    /* pool ID space */
static list_t   pool_list;      /* doubly-linked list of pools */
static kmutex_t         pool_mutex;             /* protects pool_busy_* */
static kcondvar_t       pool_busy_cv;           /* waiting for "pool_lock" */
static kthread_t        *pool_busy_thread;      /* thread holding "pool_lock" */
static kmutex_t         pool_barrier_lock;      /* synch. with pool_barrier_* */
static kcondvar_t       pool_barrier_cv;        /* synch. with pool_barrier_* */
static int              pool_barrier_count;     /* synch. with pool_barrier_* */
static list_t           pool_event_cb_list;     /* pool event callbacks */
static boolean_t        pool_event_cb_init = B_FALSE;
static kmutex_t         pool_event_cb_lock;
static taskq_t          *pool_event_cb_taskq = NULL;

void pool_event_dispatch(pool_event_t, poolid_t);

/*
 * Boot-time pool initialization.
 */
void
pool_init(void)
{
        pool_ids = id_space_create("pool_ids", POOL_DEFAULT + 1, POOL_MAXID);

        /*
         * Initialize default pool.
         */
        pool_default = kmem_zalloc(sizeof (pool_t), KM_SLEEP);
        pool_default->pool_id = POOL_DEFAULT;
        list_create(&pool_list, sizeof (pool_t), offsetof(pool_t, pool_link));
        list_insert_head(&pool_list, pool_default);

        /*
         * Initialize plugins for resource sets.
         */
        pool_pset_init();
        pool_count = 1;
        p0.p_pool = pool_default;
        global_zone->zone_pool = pool_default;
        pool_default->pool_ref = 1;
}

/*
 * Synchronization routines.
 *
 * pool_lock is only called from syscall-level routines (processor_bind(),
 * pset_*(), and /dev/pool ioctls).  The pool "lock" may be held for long
 * periods of time, including across sleeping operations, so we allow its
 * acquisition to be interruptible.
 *
 * The current thread that owns the "lock" is stored in the variable
 * pool_busy_thread, both to let pool_lock_held() work and to aid debugging.
 */
void
pool_lock(void)
{
        mutex_enter(&pool_mutex);
        ASSERT(!pool_lock_held());
        while (pool_busy_thread != NULL)
                cv_wait(&pool_busy_cv, &pool_mutex);
        pool_busy_thread = curthread;
        mutex_exit(&pool_mutex);
}

int
pool_lock_intr(void)
{
        mutex_enter(&pool_mutex);
        ASSERT(!pool_lock_held());
        while (pool_busy_thread != NULL) {
                if (cv_wait_sig(&pool_busy_cv, &pool_mutex) == 0) {
                        cv_signal(&pool_busy_cv);
                        mutex_exit(&pool_mutex);
                        return (1);
                }
        }
        pool_busy_thread = curthread;
        mutex_exit(&pool_mutex);
        return (0);
}

int
pool_lock_held(void)
{
        return (pool_busy_thread == curthread);
}

void
pool_unlock(void)
{
        mutex_enter(&pool_mutex);
        ASSERT(pool_lock_held());
        pool_busy_thread = NULL;
        cv_signal(&pool_busy_cv);
        mutex_exit(&pool_mutex);
}

/*
 * Routines allowing fork(), exec(), exit(), and lwp_create() to synchronize
 * with pool_do_bind().
 *
 * Calls to pool_barrier_enter() and pool_barrier_exit() must bracket all
 * operations which modify pool or pset associations.  They can be called
 * while the process is multi-threaded.  In the common case, when current
 * process is not being rebound (PBWAIT flag is not set), these functions
 * will be just incrementing and decrementing reference counts.
 */
void
pool_barrier_enter(void)
{
        proc_t *p = curproc;

        ASSERT(MUTEX_HELD(&p->p_lock));
        while (p->p_poolflag & PBWAIT)
                cv_wait(&p->p_poolcv, &p->p_lock);
        p->p_poolcnt++;
}

void
pool_barrier_exit(void)
{
        proc_t *p = curproc;

        ASSERT(MUTEX_HELD(&p->p_lock));
        ASSERT(p->p_poolcnt > 0);
        p->p_poolcnt--;
        if (p->p_poolflag & PBWAIT) {
                mutex_enter(&pool_barrier_lock);
                ASSERT(pool_barrier_count > 0);
                pool_barrier_count--;
                if (pool_barrier_count == 0)
                        cv_signal(&pool_barrier_cv);
                mutex_exit(&pool_barrier_lock);
                while (p->p_poolflag & PBWAIT)
                        cv_wait(&p->p_poolcv, &p->p_lock);
        }
}

/*
 * Enable pools facility.
 */
static int
pool_enable(void)
{
        int ret;

        ASSERT(pool_lock_held());
        ASSERT(pool_count == 1);

        ret = pool_pset_enable();
        if (ret != 0)
                return (ret);
        (void) nvlist_alloc(&pool_sys_prop, NV_UNIQUE_NAME, KM_SLEEP);
        (void) nvlist_add_string(pool_sys_prop, "system.name",
            "default");
        (void) nvlist_add_string(pool_sys_prop, "system.comment", "");
        (void) nvlist_add_int64(pool_sys_prop, "system.version", 1);
        (void) nvlist_add_byte(pool_sys_prop, "system.bind-default", 1);
        (void) nvlist_add_string(pool_sys_prop, "system.poold.objectives",
            "wt-load");

        (void) nvlist_alloc(&pool_default->pool_props,
            NV_UNIQUE_NAME, KM_SLEEP);
        (void) nvlist_add_string(pool_default->pool_props,
            "pool.name", "pool_default");
        (void) nvlist_add_string(pool_default->pool_props, "pool.comment", "");
        (void) nvlist_add_byte(pool_default->pool_props, "pool.default", 1);
        (void) nvlist_add_byte(pool_default->pool_props, "pool.active", 1);
        (void) nvlist_add_int64(pool_default->pool_props,
            "pool.importance", 1);
        (void) nvlist_add_int64(pool_default->pool_props, "pool.sys_id",
            pool_default->pool_id);

        pool_sys_mod = pool_pool_mod = gethrtime();

        return (ret);
}

/*
 * Disable pools facility.
 */
static int
pool_disable(void)
{
        int ret;

        ASSERT(pool_lock_held());

        if (pool_count > 1)     /* must destroy all pools first */
                return (EBUSY);

        ret = pool_pset_disable();
        if (ret != 0)
                return (ret);
        if (pool_sys_prop != NULL) {
                nvlist_free(pool_sys_prop);
                pool_sys_prop = NULL;
        }
        if (pool_default->pool_props != NULL) {
                nvlist_free(pool_default->pool_props);
                pool_default->pool_props = NULL;
        }
        return (0);
}

pool_t *
pool_lookup_pool_by_name(char *name)
{
        pool_t *pool = pool_default;
        char *p;

        ASSERT(pool_lock_held());
        for (pool = list_head(&pool_list); pool;
            pool = list_next(&pool_list, pool)) {
                if (nvlist_lookup_string(pool->pool_props,
                    "pool.name", &p) == 0 && strcmp(name, p) == 0)
                        return (pool);
        }
        return (NULL);
}

pool_t *
pool_lookup_pool_by_id(poolid_t poolid)
{
        pool_t *pool = pool_default;

        ASSERT(pool_lock_held());
        for (pool = list_head(&pool_list); pool;
            pool = list_next(&pool_list, pool)) {
                if (pool->pool_id == poolid)
                        return (pool);
        }
        return (NULL);
}

pool_t *
pool_lookup_pool_by_pset(int id)
{
        pool_t *pool = pool_default;
        psetid_t psetid = (psetid_t)id;

        ASSERT(pool_lock_held());
        for (pool = list_head(&pool_list); pool != NULL;
            pool = list_next(&pool_list, pool)) {
                if (pool->pool_pset->pset_id == psetid)
                        return (pool);
        }
        return (NULL);
}

/*
 * Create new pool, associate it with default resource sets, and give
 * it a temporary name.
 */
static int
pool_pool_create(poolid_t *poolid)
{
        pool_t *pool;
        char pool_name[40];

        ASSERT(pool_lock_held());

        pool = kmem_zalloc(sizeof (pool_t), KM_SLEEP);
        pool->pool_id = *poolid = id_alloc(pool_ids);
        pool->pool_pset = pool_pset_default;
        pool_pset_default->pset_npools++;
        list_insert_tail(&pool_list, pool);
        (void) nvlist_alloc(&pool->pool_props, NV_UNIQUE_NAME, KM_SLEEP);
        (void) nvlist_add_int64(pool->pool_props, "pool.sys_id", pool->pool_id);
        (void) nvlist_add_byte(pool->pool_props, "pool.default", 0);
        pool_pool_mod = gethrtime();
        (void) snprintf(pool_name, sizeof (pool_name), "pool_%lld",
            pool_pool_mod);
        (void) nvlist_add_string(pool->pool_props, "pool.name", pool_name);
        pool_count++;
        return (0);
}

struct destroy_zone_arg {
        pool_t *old;
        pool_t *new;
};

/*
 * Update pool pointers for zones that are currently bound to pool "old"
 * to be bound to pool "new".
 */
static int
pool_destroy_zone_cb(zone_t *zone, void *arg)
{
        struct destroy_zone_arg *dza = arg;

        ASSERT(pool_lock_held());
        ASSERT(MUTEX_HELD(&cpu_lock));

        if (zone_pool_get(zone) == dza->old)
                zone_pool_set(zone, dza->new);
        return (0);
}

/*
 * Destroy specified pool, and rebind all processes in it
 * to the default pool.
 */
static int
pool_pool_destroy(poolid_t poolid)
{
        pool_t *pool;
        int ret;

        ASSERT(pool_lock_held());

        if (poolid == POOL_DEFAULT)
                return (EINVAL);
        if ((pool = pool_lookup_pool_by_id(poolid)) == NULL)
                return (ESRCH);
        ret = pool_do_bind(pool_default, P_POOLID, poolid, POOL_BIND_ALL);
        if (ret == 0) {
                struct destroy_zone_arg dzarg;

                dzarg.old = pool;
                dzarg.new = pool_default;
                mutex_enter(&cpu_lock);
                ret = zone_walk(pool_destroy_zone_cb, &dzarg);
                mutex_exit(&cpu_lock);
                ASSERT(ret == 0);
                ASSERT(pool->pool_ref == 0);
                (void) nvlist_free(pool->pool_props);
                id_free(pool_ids, pool->pool_id);
                pool->pool_pset->pset_npools--;
                list_remove(&pool_list, pool);
                pool_count--;
                pool_pool_mod = gethrtime();
                kmem_free(pool, sizeof (pool_t));
        }
        return (ret);
}

/*
 * Create new pool or resource set.
 */
int
pool_create(int class, int subclass, id_t *id)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (class) {
        case PEC_POOL:
                ret = pool_pool_create((poolid_t *)id);
                break;
        case PEC_RES_COMP:
                switch (subclass) {
                case PREC_PSET:
                        ret = pool_pset_create((psetid_t *)id);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        case PEC_RES_AGG:
                ret = ENOTSUP;
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

/*
 * Destroy an existing pool or resource set.
 */
int
pool_destroy(int class, int subclass, id_t id)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (class) {
        case PEC_POOL:
                ret = pool_pool_destroy((poolid_t)id);
                break;
        case PEC_RES_COMP:
                switch (subclass) {
                case PREC_PSET:
                        ret = pool_pset_destroy((psetid_t)id);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        case PEC_RES_AGG:
                ret = ENOTSUP;
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

/*
 * Enable or disable pools.
 */
int
pool_status(int status)
{
        int ret = 0;

        ASSERT(pool_lock_held());

        if (pool_state == status)
                return (0);
        switch (status) {
        case POOL_ENABLED:
                ret = pool_enable();
                if (ret != 0)
                        return (ret);
                pool_state = POOL_ENABLED;
                pool_event_dispatch(POOL_E_ENABLE, 0);
                break;
        case POOL_DISABLED:
                ret = pool_disable();
                if (ret != 0)
                        return (ret);
                pool_state = POOL_DISABLED;
                pool_event_dispatch(POOL_E_DISABLE, 0);
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

/*
 * Associate pool with resource set.
 */
int
pool_assoc(poolid_t poolid, int idtype, id_t id)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (idtype) {
        case PREC_PSET:
                ret = pool_pset_assoc(poolid, (psetid_t)id);
                if (ret == 0)
                        pool_event_dispatch(POOL_E_CHANGE, poolid);
                break;
        default:
                ret = EINVAL;
        }
        if (ret == 0)
                pool_pool_mod = gethrtime();
        return (ret);
}

/*
 * Disassociate resource set from pool.
 */
int
pool_dissoc(poolid_t poolid, int idtype)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (idtype) {
        case PREC_PSET:
                ret = pool_pset_assoc(poolid, PS_NONE);
                if (ret == 0)
                        pool_event_dispatch(POOL_E_CHANGE, poolid);
                break;
        default:
                ret = EINVAL;
        }
        if (ret == 0)
                pool_pool_mod = gethrtime();
        return (ret);
}

/*
 * Transfer specified quantity of resources between resource sets.
 */
/*ARGSUSED*/
int
pool_transfer(int type, id_t src, id_t dst, uint64_t qty)
{
        int ret = EINVAL;

        return (ret);
}

static poolid_t
pool_lookup_id_by_pset(int id)
{
        pool_t *pool = pool_default;
        psetid_t psetid = (psetid_t)id;

        ASSERT(pool_lock_held());
        for (pool = list_head(&pool_list); pool != NULL;
            pool = list_next(&pool_list, pool)) {
                if (pool->pool_pset->pset_id == psetid)
                        return (pool->pool_id);
        }
        return (POOL_INVALID);
}

/*
 * Transfer resources specified by their IDs between resource sets.
 */
int
pool_xtransfer(int type, id_t src_pset, id_t dst_pset, uint_t size, id_t *ids)
{
        int ret;
        poolid_t src_pool, dst_pool;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (type) {
        case PREC_PSET:
                ret = pool_pset_xtransfer((psetid_t)src_pset,
                    (psetid_t)dst_pset, size, ids);
                if (ret == 0) {
                        if ((src_pool =  pool_lookup_id_by_pset(src_pset)) !=
                            POOL_INVALID)
                                pool_event_dispatch(POOL_E_CHANGE, src_pool);
                        if ((dst_pool =  pool_lookup_id_by_pset(dst_pset)) !=
                            POOL_INVALID)
                                pool_event_dispatch(POOL_E_CHANGE, dst_pool);
                }
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

/*
 * Bind processes to pools.
 */
int
pool_bind(poolid_t poolid, idtype_t idtype, id_t id)
{
        pool_t  *pool;

        ASSERT(pool_lock_held());

        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        if ((pool = pool_lookup_pool_by_id(poolid)) == NULL)
                return (ESRCH);

        switch (idtype) {
        case P_PID:
        case P_TASKID:
        case P_PROJID:
        case P_ZONEID:
                break;
        default:
                return (EINVAL);
        }
        return (pool_do_bind(pool, idtype, id, POOL_BIND_ALL));
}

/*
 * Query pool binding of the specifed process.
 */
int
pool_query_binding(idtype_t idtype, id_t id, id_t *poolid)
{
        proc_t *p;

        if (idtype != P_PID)
                return (ENOTSUP);
        if (id == P_MYID)
                id = curproc->p_pid;

        ASSERT(pool_lock_held());

        mutex_enter(&pidlock);
        if ((p = prfind((pid_t)id)) == NULL) {
                mutex_exit(&pidlock);
                return (ESRCH);
        }
        mutex_enter(&p->p_lock);
        /*
         * In local zones, lie about pool bindings of processes from
         * the global zone.
         */
        if (!INGLOBALZONE(curproc) && INGLOBALZONE(p)) {
                pool_t *pool;

                pool = zone_pool_get(curproc->p_zone);
                *poolid = pool->pool_id;
        } else {
                *poolid = p->p_pool->pool_id;
        }
        mutex_exit(&p->p_lock);
        mutex_exit(&pidlock);
        return (0);
}

static ea_object_t *
pool_system_pack(void)
{
        ea_object_t *eo_system;
        size_t bufsz = 0;
        char *buf = NULL;

        ASSERT(pool_lock_held());

        eo_system = ea_alloc_group(EXT_GROUP | EXC_LOCAL | EXD_GROUP_SYSTEM);
        (void) ea_attach_item(eo_system, &pool_sys_mod, sizeof (hrtime_t),
            EXC_LOCAL | EXD_SYSTEM_TSTAMP | EXT_UINT64);
        if (INGLOBALZONE(curproc))
                (void) ea_attach_item(eo_system, &pool_pool_mod,
                    sizeof (hrtime_t),
                    EXC_LOCAL | EXD_POOL_TSTAMP | EXT_UINT64);
        else
                (void) ea_attach_item(eo_system,
                    &curproc->p_zone->zone_pool_mod,
                    sizeof (hrtime_t),
                    EXC_LOCAL | EXD_POOL_TSTAMP | EXT_UINT64);
        (void) ea_attach_item(eo_system, &pool_pset_mod, sizeof (hrtime_t),
            EXC_LOCAL | EXD_PSET_TSTAMP | EXT_UINT64);
        (void) ea_attach_item(eo_system, &pool_cpu_mod, sizeof (hrtime_t),
            EXC_LOCAL | EXD_CPU_TSTAMP | EXT_UINT64);
        (void) nvlist_pack(pool_sys_prop, &buf, &bufsz, NV_ENCODE_NATIVE, 0);
        (void) ea_attach_item(eo_system, buf, bufsz,
            EXC_LOCAL | EXD_SYSTEM_PROP | EXT_RAW);
        kmem_free(buf, bufsz);
        return (eo_system);
}

/*
 * Pack information about pools and attach it to specified exacct group.
 */
static int
pool_pool_pack(ea_object_t *eo_system)
{
        ea_object_t *eo_pool;
        pool_t *pool;
        size_t bufsz;
        char *buf;
        pool_t *myzonepool;

        ASSERT(pool_lock_held());
        myzonepool = zone_pool_get(curproc->p_zone);
        for (pool = list_head(&pool_list); pool;
            pool = list_next(&pool_list, pool)) {
                if (!INGLOBALZONE(curproc) && myzonepool != pool)
                        continue;
                bufsz = 0;
                buf = NULL;
                eo_pool = ea_alloc_group(EXT_GROUP |
                    EXC_LOCAL | EXD_GROUP_POOL);
                (void) ea_attach_item(eo_pool, &pool->pool_id, sizeof (id_t),
                    EXC_LOCAL | EXD_POOL_POOLID | EXT_UINT32);
                (void) ea_attach_item(eo_pool, &pool->pool_pset->pset_id,
                    sizeof (id_t), EXC_LOCAL | EXD_POOL_PSETID | EXT_UINT32);
                (void) nvlist_pack(pool->pool_props, &buf, &bufsz,
                    NV_ENCODE_NATIVE, 0);
                (void) ea_attach_item(eo_pool, buf, bufsz,
                    EXC_LOCAL | EXD_POOL_PROP | EXT_RAW);
                kmem_free(buf, bufsz);
                (void) ea_attach_to_group(eo_system, eo_pool);
        }
        return (0);
}

/*
 * Pack the whole pool configuration in the specified buffer.
 */
int
pool_pack_conf(void *kbuf, size_t kbufsz, size_t *asize)
{
        ea_object_t *eo_system;
        size_t ksize;
        int ret = 0;

        ASSERT(pool_lock_held());

        eo_system = pool_system_pack();         /* 1. pack system */
        (void) pool_pool_pack(eo_system);       /* 2. pack all pools */
        (void) pool_pset_pack(eo_system);       /* 3. pack all psets */
        ksize = ea_pack_object(eo_system, NULL, 0);
        if (kbuf == NULL || kbufsz == 0)
                *asize = ksize;
        else if (ksize > kbufsz)
                ret = ENOMEM;
        else
                *asize = ea_pack_object(eo_system, kbuf, kbufsz);
        ea_free_object(eo_system, EUP_ALLOC);
        return (ret);
}

/*
 * Start/end the commit transaction.  If commit transaction is currently
 * in progress, then all POOL_QUERY ioctls will return pools configuration
 * at the beginning of transaction.
 */
int
pool_commit(int state)
{
        ea_object_t *eo_system;
        int ret = 0;

        ASSERT(pool_lock_held());

        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (state) {
        case 1:
                /*
                 * Beginning commit transation.
                 */
                if (pool_buf != NULL)           /* transaction in progress */
                        return (EBUSY);
                eo_system = pool_system_pack();         /* 1. pack system */
                (void) pool_pool_pack(eo_system);       /* 2. pack all pools */
                (void) pool_pset_pack(eo_system);       /* 3. pack all psets */
                pool_bufsz = ea_pack_object(eo_system, NULL, 0);
                pool_buf = kmem_alloc(pool_bufsz, KM_SLEEP);
                pool_bufsz = ea_pack_object(eo_system, pool_buf, pool_bufsz);
                ea_free_object(eo_system, EUP_ALLOC);
                break;
        case 0:
                /*
                 * Finishing commit transaction.
                 */
                if (pool_buf != NULL) {
                        kmem_free(pool_buf, pool_bufsz);
                        pool_buf = NULL;
                        pool_bufsz = 0;
                }
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

/*
 * Check is the specified property is special
 */
static pool_property_t *
pool_property_find(char *name, pool_property_t *list)
{
        pool_property_t *prop;

        for (prop = list; prop->pp_name != NULL; prop++)
                if (strcmp(prop->pp_name, name) == 0)
                        return (prop);
        return (NULL);
}

static pool_property_t pool_prop_sys[] = {
        { "system.name",                DATA_TYPE_STRING,       PP_RDWR },
        { "system.comment",             DATA_TYPE_STRING,       PP_RDWR },
        { "system.version",             DATA_TYPE_UINT64,       PP_READ },
        { "system.bind-default",        DATA_TYPE_BYTE,         PP_RDWR },
        { "system.allocate-method",     DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { "system.poold.log-level",     DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { "system.poold.log-location",  DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { "system.poold.monitor-interval",      DATA_TYPE_UINT64,
            PP_RDWR | PP_OPTIONAL },
        { "system.poold.history-file",  DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { "system.poold.objectives",    DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { NULL,                         0,                      0 }
};

static pool_property_t pool_prop_pool[] = {
        { "pool.sys_id",                DATA_TYPE_UINT64,       PP_READ },
        { "pool.name",                  DATA_TYPE_STRING,       PP_RDWR },
        { "pool.default",               DATA_TYPE_BYTE,         PP_READ },
        { "pool.active",                DATA_TYPE_BYTE,         PP_RDWR },
        { "pool.importance",            DATA_TYPE_INT64,        PP_RDWR },
        { "pool.comment",               DATA_TYPE_STRING,       PP_RDWR },
        { "pool.scheduler",             DATA_TYPE_STRING,
            PP_RDWR | PP_OPTIONAL },
        { NULL,                         0,                      0 }
};

/*
 * Common routine to put new property on the specified list
 */
int
pool_propput_common(nvlist_t *nvlist, nvpair_t *pair, pool_property_t *props)
{
        pool_property_t *prop;

        if ((prop = pool_property_find(nvpair_name(pair), props)) != NULL) {
                /*
                 * No read-only properties or properties with bad types
                 */
                if (!(prop->pp_perm & PP_WRITE) ||
                    prop->pp_type != nvpair_type(pair))
                        return (EINVAL);
        }
        return (nvlist_add_nvpair(nvlist, pair));
}

/*
 * Common routine to remove property from the given list
 */
int
pool_proprm_common(nvlist_t *nvlist, char *name, pool_property_t *props)
{
        pool_property_t *prop;

        if ((prop = pool_property_find(name, props)) != NULL) {
                if (!(prop->pp_perm & PP_OPTIONAL))
                        return (EINVAL);
        }
        return (nvlist_remove_all(nvlist, name));
}

static int
pool_system_propput(nvpair_t *pair)
{
        int ret;

        ASSERT(pool_lock_held());
        ret = pool_propput_common(pool_sys_prop, pair, pool_prop_sys);
        if (ret == 0)
                pool_sys_mod = gethrtime();
        return (ret);
}

static int
pool_system_proprm(char *name)
{
        int ret;

        ASSERT(pool_lock_held());
        ret = pool_proprm_common(pool_sys_prop, name, pool_prop_sys);
        if (ret == 0)
                pool_sys_mod = gethrtime();
        return (ret);
}

static int
pool_pool_propput(poolid_t poolid, nvpair_t *pair)
{
        pool_t *pool;
        int ret;

        ASSERT(pool_lock_held());
        if ((pool = pool_lookup_pool_by_id(poolid)) == NULL)
                return (ESRCH);
        ret = pool_propput_common(pool->pool_props, pair, pool_prop_pool);
        if (ret == 0)
                pool_pool_mod = gethrtime();
        return (ret);
}

static int
pool_pool_proprm(poolid_t poolid, char *name)
{
        int ret;
        pool_t *pool;

        ASSERT(pool_lock_held());
        if ((pool = pool_lookup_pool_by_id(poolid)) == NULL)
                return (ESRCH);
        ret = pool_proprm_common(pool->pool_props, name, pool_prop_pool);
        if (ret == 0)
                pool_pool_mod = gethrtime();
        return (ret);
}

int
pool_propput(int class, int subclass, id_t id, nvpair_t *pair)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (class) {
        case PEC_SYSTEM:
                ret = pool_system_propput(pair);
                break;
        case PEC_POOL:
                ret = pool_pool_propput((poolid_t)id, pair);
                break;
        case PEC_RES_COMP:
                switch (subclass) {
                case PREC_PSET:
                        ret = pool_pset_propput((psetid_t)id, pair);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        case PEC_RES_AGG:
                ret = ENOTSUP;
                break;
        case PEC_COMP:
                switch (subclass) {
                case PCEC_CPU:
                        ret = pool_cpu_propput((processorid_t)id, pair);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

int
pool_proprm(int class, int subclass, id_t id, char *name)
{
        int ret;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);
        switch (class) {
        case PEC_SYSTEM:
                ret = pool_system_proprm(name);
                break;
        case PEC_POOL:
                ret = pool_pool_proprm((poolid_t)id, name);
                break;
        case PEC_RES_COMP:
                switch (subclass) {
                case PREC_PSET:
                        ret = pool_pset_proprm((psetid_t)id, name);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        case PEC_RES_AGG:
                ret = ENOTSUP;
                break;
        case PEC_COMP:
                switch (subclass) {
                case PCEC_CPU:
                        ret = pool_cpu_proprm((processorid_t)id, name);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        default:
                ret = EINVAL;
        }
        return (ret);
}

int
pool_propget(char *name, int class, int subclass, id_t id, nvlist_t **nvlp)
{
        int ret;
        nvlist_t *nvl;

        ASSERT(pool_lock_held());
        if (pool_state == POOL_DISABLED)
                return (ENOTACTIVE);

        (void) nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP);

        switch (class) {
        case PEC_SYSTEM:
        case PEC_POOL:
                ret = EINVAL;
                break;
        case PEC_RES_COMP:
                switch (subclass) {
                case PREC_PSET:
                        ret = pool_pset_propget((psetid_t)id, name, nvl);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        case PEC_RES_AGG:
                ret = ENOTSUP;
                break;
        case PEC_COMP:
                switch (subclass) {
                case PCEC_CPU:
                        ret = pool_cpu_propget((processorid_t)id, name, nvl);
                        break;
                default:
                        ret = EINVAL;
                }
                break;
        default:
                ret = EINVAL;
        }
        if (ret == 0)
                *nvlp = nvl;
        else
                nvlist_free(nvl);
        return (ret);
}

/*
 * pool_bind_wake and pool_bind_wakeall are helper functions to undo PBWAITs
 * in case of failure in pool_do_bind().
 */
static void
pool_bind_wake(proc_t *p)
{
        ASSERT(pool_lock_held());

        mutex_enter(&p->p_lock);
        ASSERT(p->p_poolflag & PBWAIT);
        if (p->p_poolcnt > 0) {
                mutex_enter(&pool_barrier_lock);
                pool_barrier_count -= p->p_poolcnt;
                mutex_exit(&pool_barrier_lock);
        }
        p->p_poolflag &= ~PBWAIT;
        cv_signal(&p->p_poolcv);
        mutex_exit(&p->p_lock);
}

static void
pool_bind_wakeall(proc_t **procs)
{
        proc_t *p, **pp;

        ASSERT(pool_lock_held());
        for (pp = procs; (p = *pp) != NULL; pp++)
                pool_bind_wake(p);
}

/*
 * Return the scheduling class for this pool, or
 *      POOL_CLASS_UNSET if not set
 *      POOL_CLASS_INVAL if set to an invalid class ID.
 */
id_t
pool_get_class(pool_t *pool)
{
        char *name;
        id_t cid;

        ASSERT(pool_lock_held());

        if (nvlist_lookup_string(pool->pool_props, "pool.scheduler",
            &name) == 0) {
                if (getcidbyname(name, &cid) == 0)
                        return (cid);
                else
                        return (POOL_CLASS_INVAL);
        }
        return (POOL_CLASS_UNSET);
}

/*
 * Move process to the new scheduling class.
 */
static void
pool_change_class(proc_t *p, id_t cid)
{
        kthread_t *t;
        void *cldata;
        id_t oldcid;
        void **bufs;
        void **buf;
        int nlwp;
        int ret;
        int i;

        /*
         * Do not move kernel processes (such as zsched).
         */
        if (p->p_flag & SSYS)
                return;
        /*
         * This process is in the pool barrier, so it can't possibly be
         * adding new threads and we can use p_lwpcnt + p_zombcnt + 1
         * (for possible agent LWP which doesn't use pool barrier) as
         * our upper bound.
         */
        nlwp = p->p_lwpcnt + p->p_zombcnt + 1;

        /*
         * Pre-allocate scheduling class specific buffers before
         * grabbing p_lock.
         */
        bufs = kmem_zalloc(nlwp * sizeof (void *), KM_SLEEP);
        for (i = 0, buf = bufs; i < nlwp; i++, buf++) {
                ret = CL_ALLOC(buf, cid, KM_SLEEP);
                ASSERT(ret == 0);
        }

        /*
         * Move threads one by one to the new scheduling class.
         * This never fails because we have all the right
         * privileges here.
         */
        mutex_enter(&p->p_lock);
        ASSERT(p->p_poolflag & PBWAIT);
        buf = bufs;
        t = p->p_tlist;
        ASSERT(t != NULL);
        do {
                if (t->t_cid != cid) {
                        oldcid = t->t_cid;
                        cldata = t->t_cldata;
                        ret = CL_ENTERCLASS(t, cid, NULL, NULL, *buf);
                        ASSERT(ret == 0);
                        CL_EXITCLASS(oldcid, cldata);
                        schedctl_set_cidpri(t);
                        *buf++ = NULL;
                }
        } while ((t = t->t_forw) != p->p_tlist);
        mutex_exit(&p->p_lock);
        /*
         * Free unused scheduling class specific buffers.
         */
        for (i = 0, buf = bufs; i < nlwp; i++, buf++) {
                if (*buf != NULL) {
                        CL_FREE(cid, *buf);
                        *buf = NULL;
                }
        }
        kmem_free(bufs, nlwp * sizeof (void *));
}

void
pool_get_name(pool_t *pool, char **name)
{
        ASSERT(pool_lock_held());

        (void) nvlist_lookup_string(pool->pool_props, "pool.name", name);

        ASSERT(strlen(*name) != 0);
}


/*
 * The meat of the bind operation.  The steps in pool_do_bind are:
 *
 * 1) Set PBWAIT in the p_poolflag of any process of interest, and add all
 *    such processes to an array.  For any interesting process that has
 *    threads inside the pool barrier set, increment a counter by the
 *    count of such threads.  Once PBWAIT is set on a process, that process
 *    will not disappear.
 *
 * 2) Wait for the counter from step 2 to drop to zero.  Any process which
 *    calls pool_barrier_exit() and notices that PBWAIT has been set on it
 *    will decrement that counter before going to sleep, and the process
 *    calling pool_barrier_exit() which does the final decrement will wake us.
 *
 * 3) For each interesting process, perform a calculation on it to see if
 *    the bind will actually succeed.  This uses the following three
 *    resource-set-specific functions:
 *
 *    - int set_bind_start(procs, pool)
 *
 *      Determine whether the given array of processes can be bound to the
 *      resource set associated with the given pool.  If it can, take and hold
 *      any locks necessary to ensure that the operation will succeed, and
 *      make any necessary reservations in the target resource set.  If it
 *      can't, return failure with no reservations made and no new locks held.
 *
 *    - void set_bind_abort(procs, pool)
 *
 *      set_bind_start() has completed successfully, but another resource set's
 *      set_bind_start() has failed, and we haven't begun the bind yet.  Undo
 *      any reservations made and drop any locks acquired by our
 *      set_bind_start().
 *
 *    - void set_bind_finish(void)
 *
 *      The bind has completed successfully.  The processes have been released,
 *      and the reservation acquired in set_bind_start() has been depleted as
 *      the processes have finished their bindings.  Drop any locks acquired by
 *      set_bind_start().
 *
 * 4) If we've decided that we can proceed with the bind, iterate through
 *    the list of interesting processes, grab the necessary locks (which
 *    may differ per resource set), perform the bind, and ASSERT that it
 *    succeeds.  Once a process has been rebound, it can be awakened.
 *
 * The operations from step 4 must be kept in sync with anything which might
 * cause the bind operations (e.g., cpupart_bind_thread()) to fail, and
 * are thus located in the same source files as the associated bind operations.
 */
int
pool_do_bind(pool_t *pool, idtype_t idtype, id_t id, int flags)
{
        extern uint_t nproc;
        klwp_t *lwp = ttolwp(curthread);
        proc_t **pp, **procs;
        proc_t *prstart;
        int procs_count = 0;
        kproject_t *kpj;
        procset_t set;
        zone_t *zone;
        int procs_size;
        int rv = 0;
        proc_t *p;
        id_t cid = -1;

        ASSERT(pool_lock_held());

        if ((cid = pool_get_class(pool)) == POOL_CLASS_INVAL)
                return (EINVAL);

        if (idtype == P_ZONEID) {
                zone = zone_find_by_id(id);
                if (zone == NULL)
                        return (ESRCH);
                if (zone_status_get(zone) > ZONE_IS_RUNNING) {
                        zone_rele(zone);
                        return (EBUSY);
                }
        }

        if (idtype == P_PROJID) {
                kpj = project_hold_by_id(id, global_zone, PROJECT_HOLD_FIND);
                if (kpj == NULL)
                        return (ESRCH);
                mutex_enter(&kpj->kpj_poolbind);
        }

        if (idtype == P_PID) {
                /*
                 * Fast-path for a single process case.
                 */
                procs_size = 2; /* procs is NULL-terminated */
                procs = kmem_zalloc(procs_size * sizeof (proc_t *), KM_SLEEP);
                mutex_enter(&pidlock);
        } else {
                /*
                 * We will need enough slots for proc_t pointers for as many as
                 * twice the number of currently running processes (assuming
                 * that each one could be in fork() creating a new child).
                 */
                for (;;) {
                        procs_size = nproc * 2;
                        procs = kmem_zalloc(procs_size * sizeof (proc_t *),
                            KM_SLEEP);
                        mutex_enter(&pidlock);

                        if (nproc * 2 <= procs_size)
                                break;
                        /*
                         * If nproc has changed, try again.
                         */
                        mutex_exit(&pidlock);
                        kmem_free(procs, procs_size * sizeof (proc_t *));
                }
        }

        if (id == P_MYID)
                id = getmyid(idtype);
        setprocset(&set, POP_AND, idtype, id, P_ALL, 0);

        /*
         * Do a first scan, and select target processes.
         */
        if (idtype == P_PID)
                prstart = prfind(id);
        else
                prstart = practive;
        for (p = prstart, pp = procs; p != NULL; p = p->p_next) {
                mutex_enter(&p->p_lock);
                /*
                 * Skip processes that don't match our (id, idtype) set or
                 * on the way of becoming zombies.  Skip kernel processes
                 * from the global zone.
                 */
                if (procinset(p, &set) == 0 ||
                    p->p_poolflag & PEXITED ||
                    ((p->p_flag & SSYS) && INGLOBALZONE(p))) {
                        mutex_exit(&p->p_lock);
                        continue;
                }
                if (!INGLOBALZONE(p)) {
                        switch (idtype) {
                        case P_PID:
                        case P_TASKID:
                                /*
                                 * Can't bind processes or tasks
                                 * in local zones to pools.
                                 */
                                mutex_exit(&p->p_lock);
                                mutex_exit(&pidlock);
                                pool_bind_wakeall(procs);
                                rv = EINVAL;
                                goto out;
                        case P_PROJID:
                                /*
                                 * Only projects in the global
                                 * zone can be rebound.
                                 */
                                mutex_exit(&p->p_lock);
                                continue;
                        case P_POOLID:
                                /*
                                 * When rebinding pools, processes can be
                                 * in different zones.
                                 */
                                break;
                        }
                }

                p->p_poolflag |= PBWAIT;
                /*
                 * If some threads in this process are inside the pool
                 * barrier, add them to pool_barrier_count, as we have
                 * to wait for all of them to exit the barrier.
                 */
                if (p->p_poolcnt > 0) {
                        mutex_enter(&pool_barrier_lock);
                        pool_barrier_count += p->p_poolcnt;
                        mutex_exit(&pool_barrier_lock);
                }
                ASSERT(pp < &procs[procs_size]);
                *pp++ = p;
                procs_count++;
                mutex_exit(&p->p_lock);

                /*
                 * We just found our process, so if we're only rebinding a
                 * single process then get out of this loop.
                 */
                if (idtype == P_PID)
                        break;
        }
        *pp = NULL;     /* cap off the end of the array */
        mutex_exit(&pidlock);

        /*
         * Wait for relevant processes to stop before they try to enter the
         * barrier or at the exit from the barrier.  Make sure that we do
         * not get stopped here while we're holding pool_lock.  If we were
         * requested to stop, or got a signal then return EAGAIN to let the
         * library know that it needs to retry.
         */
        mutex_enter(&pool_barrier_lock);
        lwp->lwp_nostop++;
        while (pool_barrier_count > 0) {
                (void) cv_wait_sig(&pool_barrier_cv, &pool_barrier_lock);
                if (pool_barrier_count > 0) {
                        /*
                         * We either got a signal or were requested to
                         * stop by /proc.  Bail out with EAGAIN.  If we were
                         * requested to stop, we'll stop in post_syscall()
                         * on our way back to userland.
                         */
                        mutex_exit(&pool_barrier_lock);
                        pool_bind_wakeall(procs);
                        lwp->lwp_nostop--;
                        rv = EAGAIN;
                        goto out;
                }
        }
        lwp->lwp_nostop--;
        mutex_exit(&pool_barrier_lock);

        if (idtype == P_PID) {
                if ((p = *procs) == NULL)
                        goto skip;
                mutex_enter(&p->p_lock);
                /* Drop the process if it is exiting */
                if (p->p_poolflag & PEXITED) {
                        mutex_exit(&p->p_lock);
                        pool_bind_wake(p);
                        procs_count--;
                } else
                        mutex_exit(&p->p_lock);
                goto skip;
        }

        /*
         * Do another run, and drop processes that were inside the barrier
         * in exit(), but when they have dropped to pool_barrier_exit
         * they have become of no interest to us.  Pick up child processes that
         * were created by fork() but didn't exist during our first scan.
         * Their parents are now stopped at pool_barrier_exit in cfork().
         */
        mutex_enter(&pidlock);
        for (pp = procs; (p = *pp) != NULL; pp++) {
                mutex_enter(&p->p_lock);
                if (p->p_poolflag & PEXITED) {
                        ASSERT(p->p_lwpcnt == 0);
                        mutex_exit(&p->p_lock);
                        pool_bind_wake(p);
                        /* flip w/last non-NULL slot */
                        *pp = procs[procs_count - 1];
                        procs[procs_count - 1] = NULL;
                        procs_count--;
                        pp--;                   /* try this slot again */
                        continue;
                } else
                        mutex_exit(&p->p_lock);
                /*
                 * Look at the child and check if it should be rebound also.
                 * We're holding pidlock, so it is safe to reference p_child.
                 */
                if ((p = p->p_child) == NULL)
                        continue;

                mutex_enter(&p->p_lock);

                /*
                 * Skip system processes and make sure that the child is in
                 * the same task/project/pool/zone as the parent.
                 */
                if ((!INGLOBALZONE(p) && idtype != P_ZONEID &&
                    idtype != P_POOLID) || p->p_flag & SSYS) {
                        mutex_exit(&p->p_lock);
                        continue;
                }

                /*
                 * If the child process has been already created by fork(), has
                 * not exited, and has not been added to the list already,
                 * then add it now.  We will hit this process again (since we
                 * stick it at the end of the procs list) but it will ignored
                 * because it will have the PBWAIT flag set.
                 */
                if (procinset(p, &set) &&
                    !(p->p_poolflag & PEXITED) &&
                    !(p->p_poolflag & PBWAIT)) {
                        ASSERT(p->p_child == NULL); /* no child of a child */
                        procs[procs_count] = p;
                        procs[procs_count + 1] = NULL;
                        procs_count++;
                        p->p_poolflag |= PBWAIT;
                }
                mutex_exit(&p->p_lock);
        }
        mutex_exit(&pidlock);
skip:
        /*
         * If there's no processes to rebind then return ESRCH, unless
         * we're associating a pool with new resource set, destroying it,
         * or binding a zone to a pool.
         */
        if (procs_count == 0) {
                if (idtype == P_POOLID || idtype == P_ZONEID)
                        rv = 0;
                else
                        rv = ESRCH;
                goto out;
        }

#ifdef DEBUG
        /*
         * All processes in the array should have PBWAIT set, and none
         * should be in the critical section. Thus, although p_poolflag
         * and p_poolcnt are protected by p_lock, their ASSERTions below
         * should be stable without it. procinset(), however, ASSERTs that
         * the p_lock is held upon entry.
         */
        for (pp = procs; (p = *pp) != NULL; pp++) {
                int in_set;

                mutex_enter(&p->p_lock);
                in_set = procinset(p, &set);
                mutex_exit(&p->p_lock);

                ASSERT(in_set);
                ASSERT(p->p_poolflag & PBWAIT);
                ASSERT(p->p_poolcnt == 0);
        }
#endif

        /*
         * Do the check if processor set rebinding is going to succeed or not.
         */
        if ((flags & POOL_BIND_PSET) &&
            (rv = pset_bind_start(procs, pool)) != 0) {
                pool_bind_wakeall(procs);
                goto out;
        }

        /*
         * At this point, all bind operations should succeed.
         */
        for (pp = procs; (p = *pp) != NULL; pp++) {
                if (flags & POOL_BIND_PSET) {
                        psetid_t psetid = pool->pool_pset->pset_id;
                        void *zonebuf;
                        void *projbuf;

                        /*
                         * Pre-allocate one buffer for FSS (per-project
                         * buffer for a new pset) in case if this is the
                         * first thread from its current project getting
                         * bound to this processor set.
                         */
                        projbuf = fss_allocbuf(FSS_ONE_BUF, FSS_ALLOC_PROJ);
                        zonebuf = fss_allocbuf(FSS_ONE_BUF, FSS_ALLOC_ZONE);

                        mutex_enter(&pidlock);
                        mutex_enter(&p->p_lock);
                        pool_pset_bind(p, psetid, projbuf, zonebuf);
                        mutex_exit(&p->p_lock);
                        mutex_exit(&pidlock);
                        /*
                         * Free buffers pre-allocated above if it
                         * wasn't actually used.
                         */
                        fss_freebuf(projbuf, FSS_ALLOC_PROJ);
                        fss_freebuf(zonebuf, FSS_ALLOC_ZONE);
                }
                /*
                 * Now let's change the scheduling class of this
                 * process if our target pool has it defined.
                 */
                if (cid != POOL_CLASS_UNSET)
                        pool_change_class(p, cid);

                /*
                 * It is safe to reference p_pool here without holding
                 * p_lock because it cannot change underneath of us.
                 * We're holding pool_lock here, so nobody else can be
                 * moving this process between pools.  If process "p"
                 * would be exiting, we're guaranteed that it would be blocked
                 * at pool_barrier_enter() in exit().  Otherwise, it would've
                 * been skipped by one of our scans of the practive list
                 * as a process with PEXITED flag set.
                 */
                if (p->p_pool != pool) {
                        ASSERT(p->p_pool->pool_ref > 0);
                        atomic_dec_32(&p->p_pool->pool_ref);
                        p->p_pool = pool;
                        atomic_inc_32(&p->p_pool->pool_ref);
                }
                /*
                 * Okay, we've tortured this guy enough.
                 * Let this poor process go now.
                 */
                pool_bind_wake(p);
        }
        if (flags & POOL_BIND_PSET)
                pset_bind_finish();

out:    switch (idtype) {
        case P_PROJID:
                ASSERT(kpj != NULL);
                mutex_exit(&kpj->kpj_poolbind);
                project_rele(kpj);
                break;
        case P_ZONEID:
                if (rv == 0) {
                        mutex_enter(&cpu_lock);
                        zone_pool_set(zone, pool);
                        mutex_exit(&cpu_lock);
                }
                zone->zone_pool_mod = gethrtime();
                zone_rele(zone);
                break;
        }

        kmem_free(procs, procs_size * sizeof (proc_t *));
        ASSERT(pool_barrier_count == 0);
        return (rv);
}

void
pool_event_cb_register(pool_event_cb_t *cb)
{
        ASSERT(!pool_lock_held() || panicstr);
        ASSERT(cb->pec_func != NULL);

        mutex_enter(&pool_event_cb_lock);
        if (!pool_event_cb_init) {
                list_create(&pool_event_cb_list,  sizeof (pool_event_cb_t),
                    offsetof(pool_event_cb_t, pec_list));
                pool_event_cb_init = B_TRUE;
        }
        list_insert_tail(&pool_event_cb_list, cb);
        mutex_exit(&pool_event_cb_lock);
}

void
pool_event_cb_unregister(pool_event_cb_t *cb)
{
        ASSERT(!pool_lock_held() || panicstr);

        mutex_enter(&pool_event_cb_lock);
        list_remove(&pool_event_cb_list, cb);
        mutex_exit(&pool_event_cb_lock);
}

typedef struct {
        pool_event_t    tqd_what;
        poolid_t        tqd_id;
} pool_tqd_t;

void
pool_event_notify(void *arg)
{
        pool_tqd_t      *tqd = (pool_tqd_t *)arg;
        pool_event_cb_t *cb;

        ASSERT(!pool_lock_held() || panicstr);

        mutex_enter(&pool_event_cb_lock);
        for (cb = list_head(&pool_event_cb_list); cb != NULL;
            cb = list_next(&pool_event_cb_list, cb)) {
                cb->pec_func(tqd->tqd_what, tqd->tqd_id, cb->pec_arg);
        }
        mutex_exit(&pool_event_cb_lock);
        kmem_free(tqd, sizeof (*tqd));
}

void
pool_event_dispatch(pool_event_t what, poolid_t id)
{
        pool_tqd_t *tqd = NULL;

        ASSERT(pool_lock_held());

        if (pool_event_cb_taskq == NULL) {
                pool_event_cb_taskq = taskq_create("pool_event_cb_taskq", 1,
                    -1, 1, 1, TASKQ_PREPOPULATE);
        }

        tqd = kmem_alloc(sizeof (*tqd), KM_SLEEP);
        tqd->tqd_what = what;
        tqd->tqd_id = id;

        (void) taskq_dispatch(pool_event_cb_taskq, pool_event_notify, tqd,
            KM_SLEEP);
}