5285 pass in cpu_pause_func via pause_cpus

[unleashed.git] / usr / src / uts / common / disp / cmt.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/systm.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/thread.h>
#include <sys/cpuvar.h>
#include <sys/cpupart.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/kstat.h>
#include <sys/processor.h>
#include <sys/disp.h>
#include <sys/group.h>
#include <sys/pghw.h>
#include <sys/bitset.h>
#include <sys/lgrp.h>
#include <sys/cmt.h>
#include <sys/cpu_pm.h>

/*
 * CMT scheduler / dispatcher support
 *
 * This file implements CMT scheduler support using Processor Groups.
 * The CMT processor group class creates and maintains the CMT class
 * specific processor group pg_cmt_t.
 *
 * ---------------------------- <-- pg_cmt_t *
 * | pghw_t                   |
 * ----------------------------
 * | CMT class specific data  |
 * | - hierarchy linkage      |
 * | - CMT load balancing data|
 * | - active CPU group/bitset|
 * ----------------------------
 *
 * The scheduler/dispatcher leverages knowledge of the performance
 * relevant CMT sharing relationships existing between cpus to implement
 * optimized affinity, load balancing, and coalescence policies.
 *
 * Load balancing policy seeks to improve performance by minimizing
 * contention over shared processor resources / facilities, Affinity
 * policies seek to improve cache and TLB utilization. Coalescence
 * policies improve resource utilization and ultimately power efficiency.
 *
 * The CMT PGs created by this class are already arranged into a
 * hierarchy (which is done in the pghw layer). To implement the top-down
 * CMT load balancing algorithm, the CMT PGs additionally maintain
 * parent, child and sibling hierarchy relationships.
 * Parent PGs always contain a superset of their children(s) resources,
 * each PG can have at most one parent, and siblings are the group of PGs
 * sharing the same parent.
 *
 * On UMA based systems, the CMT load balancing algorithm begins by balancing
 * load across the group of top level PGs in the system hierarchy.
 * On NUMA systems, the CMT load balancing algorithm balances load across the
 * group of top level PGs in each leaf lgroup...but for root homed threads,
 * is willing to balance against all the top level PGs in the system.
 *
 * Groups of top level PGs are maintained to implement the above, one for each
 * leaf lgroup (containing the top level PGs in that lgroup), and one (for the
 * root lgroup) that contains all the top level PGs in the system.
 */
static cmt_lgrp_t       *cmt_lgrps = NULL;      /* cmt_lgrps list head */
static cmt_lgrp_t       *cpu0_lgrp = NULL;      /* boot CPU's initial lgrp */
                                                /* used for null_proc_lpa */
cmt_lgrp_t              *cmt_root = NULL;       /* Reference to root cmt pg */

static int              is_cpu0 = 1; /* true if this is boot CPU context */

/*
 * Array of hardware sharing relationships that are blacklisted.
 * CMT scheduling optimizations won't be performed for blacklisted sharing
 * relationships.
 */
static int              cmt_hw_blacklisted[PGHW_NUM_COMPONENTS];

/*
 * Set this to non-zero to disable CMT scheduling
 * This must be done via kmdb -d, as /etc/system will be too late
 */
int                     cmt_sched_disabled = 0;

/*
 * Status codes for CMT lineage validation
 * See pg_cmt_lineage_validate() below
 */
typedef enum cmt_lineage_validation {
        CMT_LINEAGE_VALID,
        CMT_LINEAGE_NON_CONCENTRIC,
        CMT_LINEAGE_PG_SPANS_LGRPS,
        CMT_LINEAGE_NON_PROMOTABLE,
        CMT_LINEAGE_REPAIRED,
        CMT_LINEAGE_UNRECOVERABLE
} cmt_lineage_validation_t;

/*
 * Status of the current lineage under construction.
 * One must be holding cpu_lock to change this.
 */
cmt_lineage_validation_t        cmt_lineage_status = CMT_LINEAGE_VALID;

/*
 * Power domain definitions (on x86) are defined by ACPI, and
 * therefore may be subject to BIOS bugs.
 */
#define PG_CMT_HW_SUSPECT(hw)   PGHW_IS_PM_DOMAIN(hw)

/*
 * Macro to test if PG is managed by the CMT PG class
 */
#define IS_CMT_PG(pg)   (((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id)

static pg_cid_t         pg_cmt_class_id;                /* PG class id */

static pg_t             *pg_cmt_alloc();
static void             pg_cmt_free(pg_t *);
static void             pg_cmt_cpu_init(cpu_t *, cpu_pg_t *);
static void             pg_cmt_cpu_fini(cpu_t *, cpu_pg_t *);
static void             pg_cmt_cpu_active(cpu_t *);
static void             pg_cmt_cpu_inactive(cpu_t *);
static void             pg_cmt_cpupart_in(cpu_t *, cpupart_t *);
static void             pg_cmt_cpupart_move(cpu_t *, cpupart_t *, cpupart_t *);
static char             *pg_cmt_policy_name(pg_t *);
static void             pg_cmt_hier_sort(pg_cmt_t **, int);
static pg_cmt_t         *pg_cmt_hier_rank(pg_cmt_t *, pg_cmt_t *);
static int              pg_cmt_cpu_belongs(pg_t *, cpu_t *);
static int              pg_cmt_hw(pghw_type_t);
static cmt_lgrp_t       *pg_cmt_find_lgrp(lgrp_handle_t);
static cmt_lgrp_t       *pg_cmt_lgrp_create(lgrp_handle_t);
static void             cmt_ev_thread_swtch(pg_t *, cpu_t *, hrtime_t,
                            kthread_t *, kthread_t *);
static void             cmt_ev_thread_swtch_pwr(pg_t *, cpu_t *, hrtime_t,
                            kthread_t *, kthread_t *);
static void             cmt_ev_thread_remain_pwr(pg_t *, cpu_t *, kthread_t *);
static cmt_lineage_validation_t pg_cmt_lineage_validate(pg_cmt_t **, int *,
                            cpu_pg_t *);

/*
 * CMT PG ops
 */
struct pg_ops pg_ops_cmt = {
        pg_cmt_alloc,
        pg_cmt_free,
        pg_cmt_cpu_init,
        pg_cmt_cpu_fini,
        pg_cmt_cpu_active,
        pg_cmt_cpu_inactive,
        pg_cmt_cpupart_in,
        NULL,                   /* cpupart_out */
        pg_cmt_cpupart_move,
        pg_cmt_cpu_belongs,
        pg_cmt_policy_name,
};

/*
 * Initialize the CMT PG class
 */
void
pg_cmt_class_init(void)
{
        if (cmt_sched_disabled)
                return;

        pg_cmt_class_id = pg_class_register("cmt", &pg_ops_cmt, PGR_PHYSICAL);
}

/*
 * Called to indicate a new CPU has started up so
 * that either t0 or the slave startup thread can
 * be accounted for.
 */
void
pg_cmt_cpu_startup(cpu_t *cp)
{
        pg_ev_thread_swtch(cp, gethrtime_unscaled(), cp->cpu_idle_thread,
            cp->cpu_thread);
}

/*
 * Return non-zero if thread can migrate between "from" and "to"
 * without a performance penalty
 */
int
pg_cmt_can_migrate(cpu_t *from, cpu_t *to)
{
        if (from->cpu_physid->cpu_cacheid ==
            to->cpu_physid->cpu_cacheid)
                return (1);
        return (0);
}

/*
 * CMT class specific PG allocation
 */
static pg_t *
pg_cmt_alloc(void)
{
        return (kmem_zalloc(sizeof (pg_cmt_t), KM_NOSLEEP));
}

/*
 * Class specific PG de-allocation
 */
static void
pg_cmt_free(pg_t *pg)
{
        ASSERT(pg != NULL);
        ASSERT(IS_CMT_PG(pg));

        kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t));
}

/*
 * Given a hardware sharing relationship, return which dispatcher
 * policies should be implemented to optimize performance and efficiency
 */
static pg_cmt_policy_t
pg_cmt_policy(pghw_type_t hw)
{
        pg_cmt_policy_t p;

        /*
         * Give the platform a chance to override the default
         */
        if ((p = pg_plat_cmt_policy(hw)) != CMT_NO_POLICY)
                return (p);

        switch (hw) {
        case PGHW_IPIPE:
        case PGHW_FPU:
        case PGHW_PROCNODE:
        case PGHW_CHIP:
                return (CMT_BALANCE);
        case PGHW_CACHE:
                return (CMT_AFFINITY | CMT_BALANCE);
        case PGHW_POW_ACTIVE:
        case PGHW_POW_IDLE:
                return (CMT_BALANCE);
        default:
                return (CMT_NO_POLICY);
        }
}

/*
 * Rank the importance of optimizing for the pg1 relationship vs.
 * the pg2 relationship.
 */
static pg_cmt_t *
pg_cmt_hier_rank(pg_cmt_t *pg1, pg_cmt_t *pg2)
{
        pghw_type_t hw1 = ((pghw_t *)pg1)->pghw_hw;
        pghw_type_t hw2 = ((pghw_t *)pg2)->pghw_hw;

        /*
         * A power domain is only important if CPUPM is enabled.
         */
        if (cpupm_get_policy() == CPUPM_POLICY_DISABLED) {
                if (PGHW_IS_PM_DOMAIN(hw1) && !PGHW_IS_PM_DOMAIN(hw2))
                        return (pg2);
                if (PGHW_IS_PM_DOMAIN(hw2) && !PGHW_IS_PM_DOMAIN(hw1))
                        return (pg1);
        }

        /*
         * Otherwise, ask the platform
         */
        if (pg_plat_hw_rank(hw1, hw2) == hw1)
                return (pg1);
        else
                return (pg2);
}

/*
 * Initialize CMT callbacks for the given PG
 */
static void
cmt_callback_init(pg_t *pg)
{
        /*
         * Stick with the default callbacks if there isn't going to be
         * any CMT thread placement optimizations implemented.
         */
        if (((pg_cmt_t *)pg)->cmt_policy == CMT_NO_POLICY)
                return;

        switch (((pghw_t *)pg)->pghw_hw) {
        case PGHW_POW_ACTIVE:
                pg->pg_cb.thread_swtch = cmt_ev_thread_swtch_pwr;
                pg->pg_cb.thread_remain = cmt_ev_thread_remain_pwr;
                break;
        default:
                pg->pg_cb.thread_swtch = cmt_ev_thread_swtch;

        }
}

/*
 * Promote PG above it's current parent.
 * This is only legal if PG has an equal or greater number of CPUs than its
 * parent.
 *
 * This routine operates on the CPU specific processor group data (for the CPUs
 * in the PG being promoted), and may be invoked from a context where one CPU's
 * PG data is under construction. In this case the argument "pgdata", if not
 * NULL, is a reference to the CPU's under-construction PG data.
 */
static void
cmt_hier_promote(pg_cmt_t *pg, cpu_pg_t *pgdata)
{
        pg_cmt_t        *parent;
        group_t         *children;
        cpu_t           *cpu;
        group_iter_t    iter;
        pg_cpu_itr_t    cpu_iter;
        int             r;
        int             err;
        int             nchildren;

        ASSERT(MUTEX_HELD(&cpu_lock));

        parent = pg->cmt_parent;
        if (parent == NULL) {
                /*
                 * Nothing to do
                 */
                return;
        }

        ASSERT(PG_NUM_CPUS((pg_t *)pg) >= PG_NUM_CPUS((pg_t *)parent));

        /*
         * We're changing around the hierarchy, which is actively traversed
         * by the dispatcher. Pause CPUS to ensure exclusivity.
         */
        pause_cpus(NULL, NULL);

        /*
         * If necessary, update the parent's sibling set, replacing parent
         * with PG.
         */
        if (parent->cmt_siblings) {
                if (group_remove(parent->cmt_siblings, parent, GRP_NORESIZE)
                    != -1) {
                        r = group_add(parent->cmt_siblings, pg, GRP_NORESIZE);
                        ASSERT(r != -1);
                }
        }

        /*
         * If the parent is at the top of the hierarchy, replace it's entry
         * in the root lgroup's group of top level PGs.
         */
        if (parent->cmt_parent == NULL &&
            parent->cmt_siblings != &cmt_root->cl_pgs) {
                if (group_remove(&cmt_root->cl_pgs, parent, GRP_NORESIZE)
                    != -1) {
                        r = group_add(&cmt_root->cl_pgs, pg, GRP_NORESIZE);
                        ASSERT(r != -1);
                }
        }

        /*
         * We assume (and therefore assert) that the PG being promoted is an
         * only child of it's parent. Update the parent's children set
         * replacing PG's entry with the parent (since the parent is becoming
         * the child). Then have PG and the parent swap children sets and
         * children counts.
         */
        ASSERT(GROUP_SIZE(parent->cmt_children) <= 1);
        if (group_remove(parent->cmt_children, pg, GRP_NORESIZE) != -1) {
                r = group_add(parent->cmt_children, parent, GRP_NORESIZE);
                ASSERT(r != -1);
        }

        children = pg->cmt_children;
        pg->cmt_children = parent->cmt_children;
        parent->cmt_children = children;

        nchildren = pg->cmt_nchildren;
        pg->cmt_nchildren = parent->cmt_nchildren;
        parent->cmt_nchildren = nchildren;

        /*
         * Update the sibling references for PG and it's parent
         */
        pg->cmt_siblings = parent->cmt_siblings;
        parent->cmt_siblings = pg->cmt_children;

        /*
         * Update any cached lineages in the per CPU pg data.
         */
        PG_CPU_ITR_INIT(pg, cpu_iter);
        while ((cpu = pg_cpu_next(&cpu_iter)) != NULL) {
                int             idx;
                int             sz;
                pg_cmt_t        *cpu_pg;
                cpu_pg_t        *pgd;   /* CPU's PG data */

                /*
                 * The CPU's whose lineage is under construction still
                 * references the bootstrap CPU PG data structure.
                 */
                if (pg_cpu_is_bootstrapped(cpu))
                        pgd = pgdata;
                else
                        pgd = cpu->cpu_pg;

                /*
                 * Iterate over the CPU's PGs updating the children
                 * of the PG being promoted, since they have a new parent.
                 */
                group_iter_init(&iter);
                while ((cpu_pg = group_iterate(&pgd->cmt_pgs, &iter)) != NULL) {
                        if (cpu_pg->cmt_parent == pg) {
                                cpu_pg->cmt_parent = parent;
                        }
                }

                /*
                 * Update the CMT load balancing lineage
                 */
                if ((idx = group_find(&pgd->cmt_pgs, (void *)pg)) == -1) {
                        /*
                         * Unless this is the CPU who's lineage is being
                         * constructed, the PG being promoted should be
                         * in the lineage.
                         */
                        ASSERT(pg_cpu_is_bootstrapped(cpu));
                        continue;
                }

                ASSERT(idx > 0);
                ASSERT(GROUP_ACCESS(&pgd->cmt_pgs, idx - 1) == parent);

                /*
                 * Have the child and the parent swap places in the CPU's
                 * lineage
                 */
                group_remove_at(&pgd->cmt_pgs, idx);
                group_remove_at(&pgd->cmt_pgs, idx - 1);
                err = group_add_at(&pgd->cmt_pgs, parent, idx);
                ASSERT(err == 0);
                err = group_add_at(&pgd->cmt_pgs, pg, idx - 1);
                ASSERT(err == 0);

                /*
                 * Ensure cmt_lineage references CPU's leaf PG.
                 * Since cmt_pgs is top-down ordered, the bottom is the last
                 * element.
                 */
                if ((sz = GROUP_SIZE(&pgd->cmt_pgs)) > 0)
                        pgd->cmt_lineage = GROUP_ACCESS(&pgd->cmt_pgs, sz - 1);
        }

        /*
         * Update the parent references for PG and it's parent
         */
        pg->cmt_parent = parent->cmt_parent;
        parent->cmt_parent = pg;

        start_cpus();
}

/*
 * CMT class callback for a new CPU entering the system
 *
 * This routine operates on the CPU specific processor group data (for the CPU
 * being initialized). The argument "pgdata" is a reference to the CPU's PG
 * data to be constructed.
 *
 * cp->cpu_pg is used by the dispatcher to access the CPU's PG data
 * references a "bootstrap" structure. pg_cmt_cpu_init() and the routines it
 * calls must be careful to operate only on the "pgdata" argument, and not
 * cp->cpu_pg.
 */
static void
pg_cmt_cpu_init(cpu_t *cp, cpu_pg_t *pgdata)
{
        pg_cmt_t        *pg;
        group_t         *cmt_pgs;
        int             levels, level;
        pghw_type_t     hw;
        pg_t            *pg_cache = NULL;
        pg_cmt_t        *cpu_cmt_hier[PGHW_NUM_COMPONENTS];
        lgrp_handle_t   lgrp_handle;
        cmt_lgrp_t      *lgrp;
        cmt_lineage_validation_t        lineage_status;

        ASSERT(MUTEX_HELD(&cpu_lock));
        ASSERT(pg_cpu_is_bootstrapped(cp));

        if (cmt_sched_disabled)
                return;

        /*
         * A new CPU is coming into the system.
         * Interrogate the platform to see if the CPU
         * has any performance or efficiency relevant
         * sharing relationships
         */
        cmt_pgs = &pgdata->cmt_pgs;
        pgdata->cmt_lineage = NULL;

        bzero(cpu_cmt_hier, sizeof (cpu_cmt_hier));
        levels = 0;
        for (hw = PGHW_START; hw < PGHW_NUM_COMPONENTS; hw++) {

                pg_cmt_policy_t policy;

                /*
                 * We're only interested in the hw sharing relationships
                 * for which we know how to optimize.
                 */
                policy = pg_cmt_policy(hw);
                if (policy == CMT_NO_POLICY ||
                    pg_plat_hw_shared(cp, hw) == 0)
                        continue;

                /*
                 * We will still create the PGs for hardware sharing
                 * relationships that have been blacklisted, but won't
                 * implement CMT thread placement optimizations against them.
                 */
                if (cmt_hw_blacklisted[hw] == 1)
                        policy = CMT_NO_POLICY;

                /*
                 * Find (or create) the PG associated with
                 * the hw sharing relationship in which cp
                 * belongs.
                 *
                 * Determine if a suitable PG already
                 * exists, or if one needs to be created.
                 */
                pg = (pg_cmt_t *)pghw_place_cpu(cp, hw);
                if (pg == NULL) {
                        /*
                         * Create a new one.
                         * Initialize the common...
                         */
                        pg = (pg_cmt_t *)pg_create(pg_cmt_class_id);

                        /* ... physical ... */
                        pghw_init((pghw_t *)pg, cp, hw);

                        /*
                         * ... and CMT specific portions of the
                         * structure.
                         */
                        pg->cmt_policy = policy;

                        /* CMT event callbacks */
                        cmt_callback_init((pg_t *)pg);

                        bitset_init(&pg->cmt_cpus_actv_set);
                        group_create(&pg->cmt_cpus_actv);
                } else {
                        ASSERT(IS_CMT_PG(pg));
                }

                ((pghw_t *)pg)->pghw_generation++;

                /* Add the CPU to the PG */
                pg_cpu_add((pg_t *)pg, cp, pgdata);

                /*
                 * Ensure capacity of the active CPU group/bitset
                 */
                group_expand(&pg->cmt_cpus_actv,
                    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));

                if (cp->cpu_seqid >=
                    bitset_capacity(&pg->cmt_cpus_actv_set)) {
                        bitset_resize(&pg->cmt_cpus_actv_set,
                            cp->cpu_seqid + 1);
                }

                /*
                 * Build a lineage of CMT PGs for load balancing / coalescence
                 */
                if (policy & (CMT_BALANCE | CMT_COALESCE)) {
                        cpu_cmt_hier[levels++] = pg;
                }

                /* Cache this for later */
                if (hw == PGHW_CACHE)
                        pg_cache = (pg_t *)pg;
        }

        group_expand(cmt_pgs, levels);

        if (cmt_root == NULL)
                cmt_root = pg_cmt_lgrp_create(lgrp_plat_root_hand());

        /*
         * Find the lgrp that encapsulates this CPU's CMT hierarchy
         */
        lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);
        if ((lgrp = pg_cmt_find_lgrp(lgrp_handle)) == NULL)
                lgrp = pg_cmt_lgrp_create(lgrp_handle);

        /*
         * Ascendingly sort the PGs in the lineage by number of CPUs
         */
        pg_cmt_hier_sort(cpu_cmt_hier, levels);

        /*
         * Examine the lineage and validate it.
         * This routine will also try to fix the lineage along with the
         * rest of the PG hierarchy should it detect an issue.
         *
         * If it returns anything other than VALID or REPAIRED, an
         * unrecoverable error has occurred, and we cannot proceed.
         */
        lineage_status = pg_cmt_lineage_validate(cpu_cmt_hier, &levels, pgdata);
        if ((lineage_status != CMT_LINEAGE_VALID) &&
            (lineage_status != CMT_LINEAGE_REPAIRED)) {
                /*
                 * In the case of an unrecoverable error where CMT scheduling
                 * has been disabled, assert that the under construction CPU's
                 * PG data has an empty CMT load balancing lineage.
                 */
                ASSERT((cmt_sched_disabled == 0) ||
                    (GROUP_SIZE(&(pgdata->cmt_pgs)) == 0));
                return;
        }

        /*
         * For existing PGs in the lineage, verify that the parent is
         * correct, as the generation in the lineage may have changed
         * as a result of the sorting. Start the traversal at the top
         * of the lineage, moving down.
         */
        for (level = levels - 1; level >= 0; ) {
                int reorg;

                reorg = 0;
                pg = cpu_cmt_hier[level];

                /*
                 * Promote PGs at an incorrect generation into place.
                 */
                while (pg->cmt_parent &&
                    pg->cmt_parent != cpu_cmt_hier[level + 1]) {
                        cmt_hier_promote(pg, pgdata);
                        reorg++;
                }
                if (reorg > 0)
                        level = levels - 1;
                else
                        level--;
        }

        /*
         * For each of the PGs in the CPU's lineage:
         *      - Add an entry in the CPU sorted CMT PG group
         *        which is used for top down CMT load balancing
         *      - Tie the PG into the CMT hierarchy by connecting
         *        it to it's parent and siblings.
         */
        for (level = 0; level < levels; level++) {
                uint_t          children;
                int             err;

                pg = cpu_cmt_hier[level];
                err = group_add_at(cmt_pgs, pg, levels - level - 1);
                ASSERT(err == 0);

                if (level == 0)
                        pgdata->cmt_lineage = (pg_t *)pg;

                if (pg->cmt_siblings != NULL) {
                        /* Already initialized */
                        ASSERT(pg->cmt_parent == NULL ||
                            pg->cmt_parent == cpu_cmt_hier[level + 1]);
                        ASSERT(pg->cmt_siblings == &lgrp->cl_pgs ||
                            ((pg->cmt_parent != NULL) &&
                            pg->cmt_siblings == pg->cmt_parent->cmt_children));
                        continue;
                }

                if ((level + 1) == levels) {
                        pg->cmt_parent = NULL;

                        pg->cmt_siblings = &lgrp->cl_pgs;
                        children = ++lgrp->cl_npgs;
                        if (cmt_root != lgrp)
                                cmt_root->cl_npgs++;
                } else {
                        pg->cmt_parent = cpu_cmt_hier[level + 1];

                        /*
                         * A good parent keeps track of their children.
                         * The parent's children group is also the PG's
                         * siblings.
                         */
                        if (pg->cmt_parent->cmt_children == NULL) {
                                pg->cmt_parent->cmt_children =
                                    kmem_zalloc(sizeof (group_t), KM_SLEEP);
                                group_create(pg->cmt_parent->cmt_children);
                        }
                        pg->cmt_siblings = pg->cmt_parent->cmt_children;
                        children = ++pg->cmt_parent->cmt_nchildren;
                }

                group_expand(pg->cmt_siblings, children);
                group_expand(&cmt_root->cl_pgs, cmt_root->cl_npgs);
        }

        /*
         * Cache the chip and core IDs in the cpu_t->cpu_physid structure
         * for fast lookups later.
         */
        if (cp->cpu_physid) {
                cp->cpu_physid->cpu_chipid =
                    pg_plat_hw_instance_id(cp, PGHW_CHIP);
                cp->cpu_physid->cpu_coreid = pg_plat_get_core_id(cp);

                /*
                 * If this cpu has a PG representing shared cache, then set
                 * cpu_cacheid to that PG's logical id
                 */
                if (pg_cache)
                        cp->cpu_physid->cpu_cacheid = pg_cache->pg_id;
        }

        /* CPU0 only initialization */
        if (is_cpu0) {
                is_cpu0 = 0;
                cpu0_lgrp = lgrp;
        }

}

/*
 * Class callback when a CPU is leaving the system (deletion)
 *
 * "pgdata" is a reference to the CPU's PG data to be deconstructed.
 *
 * cp->cpu_pg is used by the dispatcher to access the CPU's PG data
 * references a "bootstrap" structure across this function's invocation.
 * pg_cmt_cpu_fini() and the routines it calls must be careful to operate only
 * on the "pgdata" argument, and not cp->cpu_pg.
 */
static void
pg_cmt_cpu_fini(cpu_t *cp, cpu_pg_t *pgdata)
{
        group_iter_t    i;
        pg_cmt_t        *pg;
        group_t         *pgs, *cmt_pgs;
        lgrp_handle_t   lgrp_handle;
        cmt_lgrp_t      *lgrp;

        if (cmt_sched_disabled)
                return;

        ASSERT(pg_cpu_is_bootstrapped(cp));

        pgs = &pgdata->pgs;
        cmt_pgs = &pgdata->cmt_pgs;

        /*
         * Find the lgroup that encapsulates this CPU's CMT hierarchy
         */
        lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);

        lgrp = pg_cmt_find_lgrp(lgrp_handle);
        if (ncpus == 1 && lgrp != cpu0_lgrp) {
                /*
                 * One might wonder how we could be deconfiguring the
                 * only CPU in the system.
                 *
                 * On Starcat systems when null_proc_lpa is detected,
                 * the boot CPU (which is already configured into a leaf
                 * lgroup), is moved into the root lgroup. This is done by
                 * deconfiguring it from both lgroups and processor
                 * groups), and then later reconfiguring it back in.  This
                 * call to pg_cmt_cpu_fini() is part of that deconfiguration.
                 *
                 * This special case is detected by noting that the platform
                 * has changed the CPU's lgrp affiliation (since it now
                 * belongs in the root). In this case, use the cmt_lgrp_t
                 * cached for the boot CPU, since this is what needs to be
                 * torn down.
                 */
                lgrp = cpu0_lgrp;
        }

        ASSERT(lgrp != NULL);

        /*
         * First, clean up anything load balancing specific for each of
         * the CPU's PGs that participated in CMT load balancing
         */
        pg = (pg_cmt_t *)pgdata->cmt_lineage;
        while (pg != NULL) {

                ((pghw_t *)pg)->pghw_generation++;

                /*
                 * Remove the PG from the CPU's load balancing lineage
                 */
                (void) group_remove(cmt_pgs, pg, GRP_RESIZE);

                /*
                 * If it's about to become empty, destroy it's children
                 * group, and remove it's reference from it's siblings.
                 * This is done here (rather than below) to avoid removing
                 * our reference from a PG that we just eliminated.
                 */
                if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) {
                        if (pg->cmt_children != NULL)
                                group_destroy(pg->cmt_children);
                        if (pg->cmt_siblings != NULL) {
                                if (pg->cmt_siblings == &lgrp->cl_pgs)
                                        lgrp->cl_npgs--;
                                else
                                        pg->cmt_parent->cmt_nchildren--;
                        }
                }
                pg = pg->cmt_parent;
        }
        ASSERT(GROUP_SIZE(cmt_pgs) == 0);

        /*
         * Now that the load balancing lineage updates have happened,
         * remove the CPU from all it's PGs (destroying any that become
         * empty).
         */
        group_iter_init(&i);
        while ((pg = group_iterate(pgs, &i)) != NULL) {
                if (IS_CMT_PG(pg) == 0)
                        continue;

                pg_cpu_delete((pg_t *)pg, cp, pgdata);
                /*
                 * Deleting the CPU from the PG changes the CPU's
                 * PG group over which we are actively iterating
                 * Re-initialize the iteration
                 */
                group_iter_init(&i);

                if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) {

                        /*
                         * The PG has become zero sized, so destroy it.
                         */
                        group_destroy(&pg->cmt_cpus_actv);
                        bitset_fini(&pg->cmt_cpus_actv_set);
                        pghw_fini((pghw_t *)pg);

                        pg_destroy((pg_t *)pg);
                }
        }
}

/*
 * Class callback when a CPU is entering a cpu partition
 */
static void
pg_cmt_cpupart_in(cpu_t *cp, cpupart_t *pp)
{
        group_t         *pgs;
        pg_t            *pg;
        group_iter_t    i;

        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled)
                return;

        pgs = &cp->cpu_pg->pgs;

        /*
         * Ensure that the new partition's PG bitset
         * is large enough for all CMT PG's to which cp
         * belongs
         */
        group_iter_init(&i);
        while ((pg = group_iterate(pgs, &i)) != NULL) {
                if (IS_CMT_PG(pg) == 0)
                        continue;

                if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id)
                        bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1);
        }
}

/*
 * Class callback when a CPU is actually moving partitions
 */
static void
pg_cmt_cpupart_move(cpu_t *cp, cpupart_t *oldpp, cpupart_t *newpp)
{
        cpu_t           *cpp;
        group_t         *pgs;
        pg_t            *pg;
        group_iter_t    pg_iter;
        pg_cpu_itr_t    cpu_iter;
        boolean_t       found;

        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled)
                return;

        pgs = &cp->cpu_pg->pgs;
        group_iter_init(&pg_iter);

        /*
         * Iterate over the CPUs CMT PGs
         */
        while ((pg = group_iterate(pgs, &pg_iter)) != NULL) {

                if (IS_CMT_PG(pg) == 0)
                        continue;

                /*
                 * Add the PG to the bitset in the new partition.
                 */
                bitset_add(&newpp->cp_cmt_pgs, pg->pg_id);

                /*
                 * Remove the PG from the bitset in the old partition
                 * if the last of the PG's CPUs have left.
                 */
                found = B_FALSE;
                PG_CPU_ITR_INIT(pg, cpu_iter);
                while ((cpp = pg_cpu_next(&cpu_iter)) != NULL) {
                        if (cpp == cp)
                                continue;
                        if (CPU_ACTIVE(cpp) &&
                            cpp->cpu_part->cp_id == oldpp->cp_id) {
                                found = B_TRUE;
                                break;
                        }
                }
                if (!found)
                        bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id);
        }
}

/*
 * Class callback when a CPU becomes active (online)
 *
 * This is called in a context where CPUs are paused
 */
static void
pg_cmt_cpu_active(cpu_t *cp)
{
        int             err;
        group_iter_t    i;
        pg_cmt_t        *pg;
        group_t         *pgs;

        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled)
                return;

        pgs = &cp->cpu_pg->pgs;
        group_iter_init(&i);

        /*
         * Iterate over the CPU's PGs
         */
        while ((pg = group_iterate(pgs, &i)) != NULL) {

                if (IS_CMT_PG(pg) == 0)
                        continue;

                /*
                 * Move to the next generation since topology is changing
                 */
                ((pghw_t *)pg)->pghw_generation++;

                err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
                ASSERT(err == 0);

                /*
                 * If this is the first active CPU in the PG, and it
                 * represents a hardware sharing relationship over which
                 * CMT load balancing is performed, add it as a candidate
                 * for balancing with it's siblings.
                 */
                if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 &&
                    (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) {
                        err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE);
                        ASSERT(err == 0);

                        /*
                         * If this is a top level PG, add it as a balancing
                         * candidate when balancing within the root lgroup.
                         */
                        if (pg->cmt_parent == NULL &&
                            pg->cmt_siblings != &cmt_root->cl_pgs) {
                                err = group_add(&cmt_root->cl_pgs, pg,
                                    GRP_NORESIZE);
                                ASSERT(err == 0);
                        }
                }

                /*
                 * Notate the CPU in the PGs active CPU bitset.
                 * Also notate the PG as being active in it's associated
                 * partition
                 */
                bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
                bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id);
        }
}

/*
 * Class callback when a CPU goes inactive (offline)
 *
 * This is called in a context where CPUs are paused
 */
static void
pg_cmt_cpu_inactive(cpu_t *cp)
{
        int             err;
        group_t         *pgs;
        pg_cmt_t        *pg;
        cpu_t           *cpp;
        group_iter_t    i;
        pg_cpu_itr_t    cpu_itr;
        boolean_t       found;

        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled)
                return;

        pgs = &cp->cpu_pg->pgs;
        group_iter_init(&i);

        while ((pg = group_iterate(pgs, &i)) != NULL) {

                if (IS_CMT_PG(pg) == 0)
                        continue;

                /*
                 * Move to the next generation since topology is changing
                 */
                ((pghw_t *)pg)->pghw_generation++;

                /*
                 * Remove the CPU from the CMT PGs active CPU group
                 * bitmap
                 */
                err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
                ASSERT(err == 0);

                bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid);

                /*
                 * If there are no more active CPUs in this PG over which
                 * load was balanced, remove it as a balancing candidate.
                 */
                if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 &&
                    (pg->cmt_policy & (CMT_BALANCE | CMT_COALESCE))) {
                        err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
                        ASSERT(err == 0);

                        if (pg->cmt_parent == NULL &&
                            pg->cmt_siblings != &cmt_root->cl_pgs) {
                                err = group_remove(&cmt_root->cl_pgs, pg,
                                    GRP_NORESIZE);
                                ASSERT(err == 0);
                        }
                }

                /*
                 * Assert the number of active CPUs does not exceed
                 * the total number of CPUs in the PG
                 */
                ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <=
                    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));

                /*
                 * Update the PG bitset in the CPU's old partition
                 */
                found = B_FALSE;
                PG_CPU_ITR_INIT(pg, cpu_itr);
                while ((cpp = pg_cpu_next(&cpu_itr)) != NULL) {
                        if (cpp == cp)
                                continue;
                        if (CPU_ACTIVE(cpp) &&
                            cpp->cpu_part->cp_id == cp->cpu_part->cp_id) {
                                found = B_TRUE;
                                break;
                        }
                }
                if (!found) {
                        bitset_del(&cp->cpu_part->cp_cmt_pgs,
                            ((pg_t *)pg)->pg_id);
                }
        }
}

/*
 * Return non-zero if the CPU belongs in the given PG
 */
static int
pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp)
{
        cpu_t   *pg_cpu;

        pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0);

        ASSERT(pg_cpu != NULL);

        /*
         * The CPU belongs if, given the nature of the hardware sharing
         * relationship represented by the PG, the CPU has that
         * relationship with some other CPU already in the PG
         */
        if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw))
                return (1);

        return (0);
}

/*
 * Sort the CPUs CMT hierarchy, where "size" is the number of levels.
 */
static void
pg_cmt_hier_sort(pg_cmt_t **hier, int size)
{
        int             i, j, inc, sz;
        int             start, end;
        pg_t            *tmp;
        pg_t            **h = (pg_t **)hier;

        /*
         * First sort by number of CPUs
         */
        inc = size / 2;
        while (inc > 0) {
                for (i = inc; i < size; i++) {
                        j = i;
                        tmp = h[i];
                        while ((j >= inc) &&
                            (PG_NUM_CPUS(h[j - inc]) > PG_NUM_CPUS(tmp))) {
                                h[j] = h[j - inc];
                                j = j - inc;
                        }
                        h[j] = tmp;
                }
                if (inc == 2)
                        inc = 1;
                else
                        inc = (inc * 5) / 11;
        }

        /*
         * Break ties by asking the platform.
         * Determine if h[i] outranks h[i + 1] and if so, swap them.
         */
        for (start = 0; start < size; start++) {

                /*
                 * Find various contiguous sets of elements,
                 * in the array, with the same number of cpus
                 */
                end = start;
                sz = PG_NUM_CPUS(h[start]);
                while ((end < size) && (sz == PG_NUM_CPUS(h[end])))
                        end++;
                /*
                 * Sort each such set of the array by rank
                 */
                for (i = start + 1; i < end; i++) {
                        j = i - 1;
                        tmp = h[i];
                        while (j >= start &&
                            pg_cmt_hier_rank(hier[j],
                            (pg_cmt_t *)tmp) == hier[j]) {
                                h[j + 1] = h[j];
                                j--;
                        }
                        h[j + 1] = tmp;
                }
        }
}

/*
 * Return a cmt_lgrp_t * given an lgroup handle.
 */
static cmt_lgrp_t *
pg_cmt_find_lgrp(lgrp_handle_t hand)
{
        cmt_lgrp_t      *lgrp;

        ASSERT(MUTEX_HELD(&cpu_lock));

        lgrp = cmt_lgrps;
        while (lgrp != NULL) {
                if (lgrp->cl_hand == hand)
                        break;
                lgrp = lgrp->cl_next;
        }
        return (lgrp);
}

/*
 * Create a cmt_lgrp_t with the specified handle.
 */
static cmt_lgrp_t *
pg_cmt_lgrp_create(lgrp_handle_t hand)
{
        cmt_lgrp_t      *lgrp;

        ASSERT(MUTEX_HELD(&cpu_lock));

        lgrp = kmem_zalloc(sizeof (cmt_lgrp_t), KM_SLEEP);

        lgrp->cl_hand = hand;
        lgrp->cl_npgs = 0;
        lgrp->cl_next = cmt_lgrps;
        cmt_lgrps = lgrp;
        group_create(&lgrp->cl_pgs);

        return (lgrp);
}

/*
 * Interfaces to enable and disable power aware dispatching
 * The caller must be holding cpu_lock.
 *
 * Return 0 on success and -1 on failure.
 */
int
cmt_pad_enable(pghw_type_t type)
{
        group_t         *hwset;
        group_iter_t    iter;
        pg_cmt_t        *pg;

        ASSERT(PGHW_IS_PM_DOMAIN(type));
        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled == 1)
                return (-1);

        if ((hwset = pghw_set_lookup(type)) == NULL ||
            cmt_hw_blacklisted[type]) {
                /*
                 * Unable to find any instances of the specified type
                 * of power domain, or the power domains have been blacklisted.
                 */
                return (-1);
        }

        /*
         * Iterate over the power domains, setting the default dispatcher
         * policy for power/performance optimization.
         *
         * Simply setting the policy isn't enough in the case where the power
         * domain is an only child of another PG. Because the dispatcher walks
         * the PG hierarchy in a top down fashion, the higher up PG's policy
         * will dominate. So promote the power domain above it's parent if both
         * PG and it's parent have the same CPUs to ensure it's policy
         * dominates.
         */
        group_iter_init(&iter);
        while ((pg = group_iterate(hwset, &iter)) != NULL) {
                /*
                 * If the power domain is an only child to a parent
                 * not implementing the same policy, promote the child
                 * above the parent to activate the policy.
                 */
                pg->cmt_policy = pg_cmt_policy(((pghw_t *)pg)->pghw_hw);
                while ((pg->cmt_parent != NULL) &&
                    (pg->cmt_parent->cmt_policy != pg->cmt_policy) &&
                    (PG_NUM_CPUS((pg_t *)pg) ==
                    PG_NUM_CPUS((pg_t *)pg->cmt_parent))) {
                        cmt_hier_promote(pg, NULL);
                }
        }

        return (0);
}

int
cmt_pad_disable(pghw_type_t type)
{
        group_t         *hwset;
        group_iter_t    iter;
        pg_cmt_t        *pg;
        pg_cmt_t        *child;

        ASSERT(PGHW_IS_PM_DOMAIN(type));
        ASSERT(MUTEX_HELD(&cpu_lock));

        if (cmt_sched_disabled == 1)
                return (-1);

        if ((hwset = pghw_set_lookup(type)) == NULL) {
                /*
                 * Unable to find any instances of the specified type of
                 * power domain.
                 */
                return (-1);
        }
        /*
         * Iterate over the power domains, setting the default dispatcher
         * policy for performance optimization (load balancing).
         */
        group_iter_init(&iter);
        while ((pg = group_iterate(hwset, &iter)) != NULL) {

                /*
                 * If the power domain has an only child that implements
                 * policy other than load balancing, promote the child
                 * above the power domain to ensure it's policy dominates.
                 */
                if (pg->cmt_children != NULL &&
                    GROUP_SIZE(pg->cmt_children) == 1) {
                        child = GROUP_ACCESS(pg->cmt_children, 0);
                        if ((child->cmt_policy & CMT_BALANCE) == 0) {
                                cmt_hier_promote(child, NULL);
                        }
                }
                pg->cmt_policy = CMT_BALANCE;
        }
        return (0);
}

/* ARGSUSED */
static void
cmt_ev_thread_swtch(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old,
                    kthread_t *new)
{
        pg_cmt_t        *cmt_pg = (pg_cmt_t *)pg;

        if (old == cp->cpu_idle_thread) {
                atomic_inc_32(&cmt_pg->cmt_utilization);
        } else if (new == cp->cpu_idle_thread) {
                atomic_dec_32(&cmt_pg->cmt_utilization);
        }
}

/*
 * Macro to test whether a thread is currently runnable on a CPU in a PG.
 */
#define THREAD_RUNNABLE_IN_PG(t, pg)                                    \
        ((t)->t_state == TS_RUN &&                                      \
            (t)->t_disp_queue->disp_cpu &&                              \
            bitset_in_set(&(pg)->cmt_cpus_actv_set,                     \
            (t)->t_disp_queue->disp_cpu->cpu_seqid))

static void
cmt_ev_thread_swtch_pwr(pg_t *pg, cpu_t *cp, hrtime_t now, kthread_t *old,
    kthread_t *new)
{
        pg_cmt_t        *cmt = (pg_cmt_t *)pg;
        cpupm_domain_t  *dom;
        uint32_t        u;

        if (old == cp->cpu_idle_thread) {
                ASSERT(new != cp->cpu_idle_thread);
                u = atomic_inc_32_nv(&cmt->cmt_utilization);
                if (u == 1) {
                        /*
                         * Notify the CPU power manager that the domain
                         * is non-idle.
                         */
                        dom = (cpupm_domain_t *)cmt->cmt_pg.pghw_handle;
                        cpupm_utilization_event(cp, now, dom,
                            CPUPM_DOM_BUSY_FROM_IDLE);
                }
        } else if (new == cp->cpu_idle_thread) {
                ASSERT(old != cp->cpu_idle_thread);
                u = atomic_dec_32_nv(&cmt->cmt_utilization);
                if (u == 0) {
                        /*
                         * The domain is idle, notify the CPU power
                         * manager.
                         *
                         * Avoid notifying if the thread is simply migrating
                         * between CPUs in the domain.
                         */
                        if (!THREAD_RUNNABLE_IN_PG(old, cmt)) {
                                dom = (cpupm_domain_t *)cmt->cmt_pg.pghw_handle;
                                cpupm_utilization_event(cp, now, dom,
                                    CPUPM_DOM_IDLE_FROM_BUSY);
                        }
                }
        }
}

/* ARGSUSED */
static void
cmt_ev_thread_remain_pwr(pg_t *pg, cpu_t *cp, kthread_t *t)
{
        pg_cmt_t        *cmt = (pg_cmt_t *)pg;
        cpupm_domain_t  *dom;

        dom = (cpupm_domain_t *)cmt->cmt_pg.pghw_handle;
        cpupm_utilization_event(cp, (hrtime_t)0, dom, CPUPM_DOM_REMAIN_BUSY);
}

/*
 * Return the name of the CMT scheduling policy
 * being implemented across this PG
 */
static char *
pg_cmt_policy_name(pg_t *pg)
{
        pg_cmt_policy_t policy;

        policy = ((pg_cmt_t *)pg)->cmt_policy;

        if (policy & CMT_AFFINITY) {
                if (policy & CMT_BALANCE)
                        return ("Load Balancing & Affinity");
                else if (policy & CMT_COALESCE)
                        return ("Load Coalescence & Affinity");
                else
                        return ("Affinity");
        } else {
                if (policy & CMT_BALANCE)
                        return ("Load Balancing");
                else if (policy & CMT_COALESCE)
                        return ("Load Coalescence");
                else
                        return ("None");
        }
}

/*
 * Prune PG, and all other instances of PG's hardware sharing relationship
 * from the CMT PG hierarchy.
 *
 * This routine operates on the CPU specific processor group data (for the CPUs
 * in the PG being pruned), and may be invoked from a context where one CPU's
 * PG data is under construction. In this case the argument "pgdata", if not
 * NULL, is a reference to the CPU's under-construction PG data.
 */
static int
pg_cmt_prune(pg_cmt_t *pg_bad, pg_cmt_t **lineage, int *sz, cpu_pg_t *pgdata)
{
        group_t         *hwset, *children;
        int             i, j, r, size = *sz;
        group_iter_t    hw_iter, child_iter;
        pg_cpu_itr_t    cpu_iter;
        pg_cmt_t        *pg, *child;
        cpu_t           *cpu;
        int             cap_needed;
        pghw_type_t     hw;

        ASSERT(MUTEX_HELD(&cpu_lock));

        /*
         * Inform pghw layer that this PG is pruned.
         */
        pghw_cmt_fini((pghw_t *)pg_bad);

        hw = ((pghw_t *)pg_bad)->pghw_hw;

        if (hw == PGHW_POW_ACTIVE) {
                cmn_err(CE_NOTE, "!Active CPUPM domain groups look suspect. "
                    "Event Based CPUPM Unavailable");
        } else if (hw == PGHW_POW_IDLE) {
                cmn_err(CE_NOTE, "!Idle CPUPM domain groups look suspect. "
                    "Dispatcher assisted CPUPM disabled.");
        }

        /*
         * Find and eliminate the PG from the lineage.
         */
        for (i = 0; i < size; i++) {
                if (lineage[i] == pg_bad) {
                        for (j = i; j < size - 1; j++)
                                lineage[j] = lineage[j + 1];
                        *sz = size - 1;
                        break;
                }
        }

        /*
         * We'll prune all instances of the hardware sharing relationship
         * represented by pg. But before we do that (and pause CPUs) we need
         * to ensure the hierarchy's groups are properly sized.
         */
        hwset = pghw_set_lookup(hw);

        /*
         * Blacklist the hardware so future processor groups of this type won't
         * participate in CMT thread placement.
         *
         * XXX
         * For heterogeneous system configurations, this might be overkill.
         * We may only need to blacklist the illegal PGs, and other instances
         * of this hardware sharing relationship may be ok.
         */
        cmt_hw_blacklisted[hw] = 1;

        /*
         * For each of the PGs being pruned, ensure sufficient capacity in
         * the siblings set for the PG's children
         */
        group_iter_init(&hw_iter);
        while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {
                /*
                 * PG is being pruned, but if it is bringing up more than
                 * one child, ask for more capacity in the siblings group.
                 */
                cap_needed = 0;
                if (pg->cmt_children &&
                    GROUP_SIZE(pg->cmt_children) > 1) {
                        cap_needed = GROUP_SIZE(pg->cmt_children) - 1;

                        group_expand(pg->cmt_siblings,
                            GROUP_SIZE(pg->cmt_siblings) + cap_needed);

                        /*
                         * If this is a top level group, also ensure the
                         * capacity in the root lgrp level CMT grouping.
                         */
                        if (pg->cmt_parent == NULL &&
                            pg->cmt_siblings != &cmt_root->cl_pgs) {
                                group_expand(&cmt_root->cl_pgs,
                                    GROUP_SIZE(&cmt_root->cl_pgs) + cap_needed);
                                cmt_root->cl_npgs += cap_needed;
                        }
                }
        }

        /*
         * We're operating on the PG hierarchy. Pause CPUs to ensure
         * exclusivity with respect to the dispatcher.
         */
        pause_cpus(NULL, NULL);

        /*
         * Prune all PG instances of the hardware sharing relationship
         * represented by pg.
         */
        group_iter_init(&hw_iter);
        while ((pg = group_iterate(hwset, &hw_iter)) != NULL) {

                /*
                 * Remove PG from it's group of siblings, if it's there.
                 */
                if (pg->cmt_siblings) {
                        (void) group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
                }
                if (pg->cmt_parent == NULL &&
                    pg->cmt_siblings != &cmt_root->cl_pgs) {
                        (void) group_remove(&cmt_root->cl_pgs, pg,
                            GRP_NORESIZE);
                }

                /*
                 * Indicate that no CMT policy will be implemented across
                 * this PG.
                 */
                pg->cmt_policy = CMT_NO_POLICY;

                /*
                 * Move PG's children from it's children set to it's parent's
                 * children set. Note that the parent's children set, and PG's
                 * siblings set are the same thing.
                 *
                 * Because we are iterating over the same group that we are
                 * operating on (removing the children), first add all of PG's
                 * children to the parent's children set, and once we are done
                 * iterating, empty PG's children set.
                 */
                if (pg->cmt_children != NULL) {
                        children = pg->cmt_children;

                        group_iter_init(&child_iter);
                        while ((child = group_iterate(children, &child_iter))
                            != NULL) {
                                if (pg->cmt_siblings != NULL) {
                                        r = group_add(pg->cmt_siblings, child,
                                            GRP_NORESIZE);
                                        ASSERT(r == 0);

                                        if (pg->cmt_parent == NULL &&
                                            pg->cmt_siblings !=
                                            &cmt_root->cl_pgs) {
                                                r = group_add(&cmt_root->cl_pgs,
                                                    child, GRP_NORESIZE);
                                                ASSERT(r == 0);
                                        }
                                }
                        }
                        group_empty(pg->cmt_children);
                }

                /*
                 * Reset the callbacks to the defaults
                 */
                pg_callback_set_defaults((pg_t *)pg);

                /*
                 * Update all the CPU lineages in each of PG's CPUs
                 */
                PG_CPU_ITR_INIT(pg, cpu_iter);
                while ((cpu = pg_cpu_next(&cpu_iter)) != NULL) {
                        pg_cmt_t        *cpu_pg;
                        group_iter_t    liter;  /* Iterator for the lineage */
                        cpu_pg_t        *cpd;   /* CPU's PG data */

                        /*
                         * The CPU's lineage is under construction still
                         * references the bootstrap CPU PG data structure.
                         */
                        if (pg_cpu_is_bootstrapped(cpu))
                                cpd = pgdata;
                        else
                                cpd = cpu->cpu_pg;

                        /*
                         * Iterate over the CPU's PGs updating the children
                         * of the PG being promoted, since they have a new
                         * parent and siblings set.
                         */
                        group_iter_init(&liter);
                        while ((cpu_pg = group_iterate(&cpd->pgs,
                            &liter)) != NULL) {
                                if (cpu_pg->cmt_parent == pg) {
                                        cpu_pg->cmt_parent = pg->cmt_parent;
                                        cpu_pg->cmt_siblings = pg->cmt_siblings;
                                }
                        }

                        /*
                         * Update the CPU's lineages
                         *
                         * Remove the PG from the CPU's group used for CMT
                         * scheduling.
                         */
                        (void) group_remove(&cpd->cmt_pgs, pg, GRP_NORESIZE);
                }
        }
        start_cpus();
        return (0);
}

/*
 * Disable CMT scheduling
 */
static void
pg_cmt_disable(void)
{
        cpu_t           *cpu;

        ASSERT(MUTEX_HELD(&cpu_lock));

        pause_cpus(NULL, NULL);
        cpu = cpu_list;

        do {
                if (cpu->cpu_pg)
                        group_empty(&cpu->cpu_pg->cmt_pgs);
        } while ((cpu = cpu->cpu_next) != cpu_list);

        cmt_sched_disabled = 1;
        start_cpus();
        cmn_err(CE_NOTE, "!CMT thread placement optimizations unavailable");
}

/*
 * CMT lineage validation
 *
 * This routine is invoked by pg_cmt_cpu_init() to validate the integrity
 * of the PGs in a CPU's lineage. This is necessary because it's possible that
 * some groupings (power domain groupings in particular) may be defined by
 * sources that are buggy (e.g. BIOS bugs). In such cases, it may not be
 * possible to integrate those groupings into the CMT PG hierarchy, if doing
 * so would violate the subset invariant of the hierarchy, which says that
 * a PG must be subset of its parent (if it has one).
 *
 * pg_cmt_lineage_validate()'s purpose is to detect grouping definitions that
 * would result in a violation of this invariant. If a violation is found,
 * and the PG is of a grouping type who's definition is known to originate from
 * suspect sources (BIOS), then pg_cmt_prune() will be invoked to prune the
 * PG (and all other instances PG's sharing relationship type) from the CMT
 * hierarchy. Further, future instances of that sharing relationship type won't
 * be added. If the grouping definition doesn't originate from suspect
 * sources, then pg_cmt_disable() will be invoked to log an error, and disable
 * CMT scheduling altogether.
 *
 * This routine is invoked after the CPU has been added to the PGs in which
 * it belongs, but before those PGs have been added to (or had their place
 * adjusted in) the CMT PG hierarchy.
 *
 * The first argument is the CPUs PG lineage (essentially an array of PGs in
 * which the CPU belongs) that has already been sorted in ascending order
 * by CPU count. Some of the PGs in the CPUs lineage may already have other
 * CPUs in them, and have already been integrated into the CMT hierarchy.
 *
 * The addition of this new CPU to these pre-existing PGs means that those
 * PGs may need to be promoted up in the hierarchy to satisfy the subset
 * invariant. In additon to testing the subset invariant for the lineage,
 * this routine also verifies that the addition of the new CPU to the
 * existing PGs wouldn't cause the subset invariant to be violated in
 * the exiting lineages.
 *
 * This routine will normally return one of the following:
 * CMT_LINEAGE_VALID - There were no problems detected with the lineage.
 * CMT_LINEAGE_REPAIRED - Problems were detected, but repaired via pruning.
 *
 * Otherwise, this routine will return a value indicating which error it
 * was unable to recover from (and set cmt_lineage_status along the way).
 *
 * This routine operates on the CPU specific processor group data (for the CPU
 * whose lineage is being validated), which is under-construction.
 * "pgdata" is a reference to the CPU's under-construction PG data.
 * This routine must be careful to operate only on "pgdata", and not cp->cpu_pg.
 */
static cmt_lineage_validation_t
pg_cmt_lineage_validate(pg_cmt_t **lineage, int *sz, cpu_pg_t *pgdata)
{
        int             i, j, size;
        pg_cmt_t        *pg, *pg_next, *pg_bad, *pg_tmp, *parent;
        cpu_t           *cp;
        pg_cpu_itr_t    cpu_iter;
        lgrp_handle_t   lgrp;

        ASSERT(MUTEX_HELD(&cpu_lock));

revalidate:
        size = *sz;
        pg_bad = NULL;
        lgrp = LGRP_NULL_HANDLE;
        for (i = 0; i < size; i++) {

                pg = lineage[i];
                if (i < size - 1)
                        pg_next = lineage[i + 1];
                else
                        pg_next = NULL;

                /*
                 * We assume that the lineage has already been sorted
                 * by the number of CPUs. In fact, we depend on it.
                 */
                ASSERT(pg_next == NULL ||
                    (PG_NUM_CPUS((pg_t *)pg) <= PG_NUM_CPUS((pg_t *)pg_next)));

                /*
                 * The CPUs PG lineage was passed as the first argument to
                 * this routine and contains the sorted list of the CPU's
                 * PGs. Ultimately, the ordering of the PGs in that list, and
                 * the ordering as traversed by the cmt_parent list must be
                 * the same. PG promotion will be used as the mechanism to
                 * achieve this, but first we need to look for cases where
                 * promotion will be necessary, and validate that will be
                 * possible without violating the subset invarient described
                 * above.
                 *
                 * Since the PG topology is in the middle of being changed, we
                 * need to check whether the PG's existing parent (if any) is
                 * part of this CPU's lineage (and therefore should contain
                 * the new CPU). If not, it means that the addition of the
                 * new CPU should have made this PG have more CPUs than its
                 * parent (and other ancestors not in the same lineage) and
                 * will need to be promoted into place.
                 *
                 * We need to verify all of this to defend against a buggy
                 * BIOS giving bad power domain CPU groupings. Sigh.
                 */
                parent = pg->cmt_parent;
                while (parent != NULL) {
                        /*
                         * Determine if the parent/ancestor is in this lineage
                         */
                        pg_tmp = NULL;
                        for (j = 0; (j < size) && (pg_tmp != parent); j++) {
                                pg_tmp = lineage[j];
                        }
                        if (pg_tmp == parent) {
                                /*
                                 * It's in the lineage. The concentricity
                                 * checks will handle the rest.
                                 */
                                break;
                        }
                        /*
                         * If it is not in the lineage, PG will eventually
                         * need to be promoted above it. Verify the ancestor
                         * is a proper subset. There is still an error if
                         * the ancestor has the same number of CPUs as PG,
                         * since that would imply it should be in the lineage,
                         * and we already know it isn't.
                         */
                        if (PG_NUM_CPUS((pg_t *)parent) >=
                            PG_NUM_CPUS((pg_t *)pg)) {
                                /*
                                 * Not a proper subset if the parent/ancestor
                                 * has the same or more CPUs than PG.
                                 */
                                cmt_lineage_status = CMT_LINEAGE_NON_PROMOTABLE;
                                goto handle_error;
                        }
                        parent = parent->cmt_parent;
                }

                /*
                 * Walk each of the CPUs in the PGs group and perform
                 * consistency checks along the way.
                 */
                PG_CPU_ITR_INIT((pg_t *)pg, cpu_iter);
                while ((cp = pg_cpu_next(&cpu_iter)) != NULL) {
                        /*
                         * Verify that there aren't any CPUs contained in PG
                         * that the next PG in the lineage (which is larger
                         * or same size) doesn't also contain.
                         */
                        if (pg_next != NULL &&
                            pg_cpu_find((pg_t *)pg_next, cp) == B_FALSE) {
                                cmt_lineage_status = CMT_LINEAGE_NON_CONCENTRIC;
                                goto handle_error;
                        }

                        /*
                         * Verify that all the CPUs in the PG are in the same
                         * lgroup.
                         */
                        if (lgrp == LGRP_NULL_HANDLE) {
                                lgrp = lgrp_plat_cpu_to_hand(cp->cpu_id);
                        } else if (lgrp_plat_cpu_to_hand(cp->cpu_id) != lgrp) {
                                cmt_lineage_status = CMT_LINEAGE_PG_SPANS_LGRPS;
                                goto handle_error;
                        }
                }
        }

handle_error:
        /*
         * Some of these validation errors can result when the CPU grouping
         * information is derived from buggy sources (for example, incorrect
         * ACPI tables on x86 systems).
         *
         * We'll try to recover in such cases by pruning out the illegal
         * groupings from the PG hierarchy, which means that we won't optimize
         * for those levels, but we will for the remaining ones.
         */
        switch (cmt_lineage_status) {
        case CMT_LINEAGE_VALID:
        case CMT_LINEAGE_REPAIRED:
                break;
        case CMT_LINEAGE_PG_SPANS_LGRPS:
                /*
                 * We've detected a PG whose CPUs span lgroups.
                 *
                 * This isn't supported, as the dispatcher isn't allowed to
                 * to do CMT thread placement across lgroups, as this would
                 * conflict with policies implementing MPO thread affinity.
                 *
                 * If the PG is of a sharing relationship type known to
                 * legitimately span lgroups, specify that no CMT thread
                 * placement policy should be implemented, and prune the PG
                 * from the existing CMT PG hierarchy.
                 *
                 * Otherwise, fall though to the case below for handling.
                 */
                if (((pghw_t *)pg)->pghw_hw == PGHW_CHIP) {
                        if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) {
                                cmt_lineage_status = CMT_LINEAGE_REPAIRED;
                                goto revalidate;
                        }
                }
                /*LINTED*/
        case CMT_LINEAGE_NON_PROMOTABLE:
                /*
                 * We've detected a PG that already exists in another CPU's
                 * lineage that cannot cannot legally be promoted into place
                 * without breaking the invariants of the hierarchy.
                 */
                if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) {
                        if (pg_cmt_prune(pg, lineage, sz, pgdata) == 0) {
                                cmt_lineage_status = CMT_LINEAGE_REPAIRED;
                                goto revalidate;
                        }
                }
                /*
                 * Something went wrong trying to prune out the bad level.
                 * Disable CMT scheduling altogether.
                 */
                pg_cmt_disable();
                break;
        case CMT_LINEAGE_NON_CONCENTRIC:
                /*
                 * We've detected a non-concentric PG lineage, which means that
                 * there's a PG in the lineage that has CPUs that the next PG
                 * over in the lineage (which is the same size or larger)
                 * doesn't have.
                 *
                 * In this case, we examine the two PGs to see if either
                 * grouping is defined by potentially buggy sources.
                 *
                 * If one has less CPUs than the other, and contains CPUs
                 * not found in the parent, and it is an untrusted enumeration,
                 * then prune it. If both have the same number of CPUs, then
                 * prune the one that is untrusted.
                 *
                 * This process repeats until we have a concentric lineage,
                 * or we would have to prune out level derived from what we
                 * thought was a reliable source, in which case CMT scheduling
                 * is disabled altogether.
                 */
                if ((PG_NUM_CPUS((pg_t *)pg) < PG_NUM_CPUS((pg_t *)pg_next)) &&
                    (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw))) {
                        pg_bad = pg;
                } else if (PG_NUM_CPUS((pg_t *)pg) ==
                    PG_NUM_CPUS((pg_t *)pg_next)) {
                        if (PG_CMT_HW_SUSPECT(((pghw_t *)pg_next)->pghw_hw)) {
                                pg_bad = pg_next;
                        } else if (PG_CMT_HW_SUSPECT(((pghw_t *)pg)->pghw_hw)) {
                                pg_bad = pg;
                        }
                }
                if (pg_bad) {
                        if (pg_cmt_prune(pg_bad, lineage, sz, pgdata) == 0) {
                                cmt_lineage_status = CMT_LINEAGE_REPAIRED;
                                goto revalidate;
                        }
                }
                /*
                 * Something went wrong trying to identify and/or prune out
                 * the bad level. Disable CMT scheduling altogether.
                 */
                pg_cmt_disable();
                break;
        default:
                /*
                 * If we're here, we've encountered a validation error for
                 * which we don't know how to recover. In this case, disable
                 * CMT scheduling altogether.
                 */
                cmt_lineage_status = CMT_LINEAGE_UNRECOVERABLE;
                pg_cmt_disable();
        }
        return (cmt_lineage_status);
}