kernel/sched: convert to C99 bool

[unleashed.git] / kernel / sched / fss / fss.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2013, Joyent, Inc. All rights reserved.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/cred.h>
#include <sys/proc.h>
#include <sys/strsubr.h>
#include <sys/priocntl.h>
#include <sys/class.h>
#include <sys/disp.h>
#include <sys/procset.h>
#include <sys/debug.h>
#include <sys/kmem.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/schedctl.h>
#include <sys/vmsystm.h>
#include <sys/atomic.h>
#include <sys/project.h>
#include <sys/modctl.h>
#include <sys/fss.h>
#include <sys/fsspriocntl.h>
#include <sys/cpupart.h>
#include <sys/zone.h>
#include <vm/rm.h>
#include <vm/seg_kmem.h>
#include <sys/tnf_probe.h>
#include <sys/policy.h>
#include <sys/sdt.h>
#include <sys/cpucaps.h>

/*
 * The fair share scheduling class ensures that collections of processes
 * (zones and projects) each get their configured share of CPU.  This is in
 * contrast to the TS class which considers individual processes.
 *
 * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
 * projects using the project.cpu-shares rctl.  By default the value is 1
 * and it can range from 0 - 64k.  A value of 0 means that processes in the
 * collection will only get CPU resources when there are no other processes
 * that need CPU. The cpu-share is used as one of the inputs to calculate a
 * thread's "user-mode" priority (umdpri) for the scheduler.  The umdpri falls
 * in the range 0-59.  FSS calculates other, internal, priorities which are not
 * visible outside of the FSS class.
 *
 * The FSS class should approximate TS behavior when there are excess CPU
 * resources.  When there is a backlog of runnable processes, then the share
 * is used as input into the runnable process's priority calculation, where
 * the final umdpri is used by the scheduler to determine when the process runs.
 *
 * Projects in a zone compete with each other for CPU time, receiving CPU
 * allocation within a zone proportional to the project's share; at a higher
 * level zones compete with each other, receiving allocation in a pset
 * proportional to the zone's share.
 *
 * The FSS priority calculation consists of several parts.
 *
 * 1) Once per second the fss_update function runs. The first thing it does is
 *    call fss_decay_usage. This function does three things.
 *
 * a) fss_decay_usage first decays the maxfsspri value for the pset.  This
 *    value is used in the per-process priority calculation described in step
 *    (2b).  The maxfsspri is decayed using the following formula:
 *
 *                      maxfsspri * fss_nice_decay[NZERO])
 *        maxfsspri =  ------------------------------------
 *                            FSS_DECAY_BASE
 *
 *
 *     - NZERO is the default process priority (i.e. 20)
 *
 *    The fss_nice_decay array is a fixed set of values used to adjust the
 *    decay rate of processes based on their nice value.  Entries in this
 *    array are initialized in fss_init using the following formula:
 *
 *                        (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
 *       FSS_DECAY_MIN + -------------------------------------
 *                               FSS_NICE_RANGE - 1
 *
 *     - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
 *     - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
 *     - FSS_NICE_RANGE is 40 (range is 0 - 39)
 *
 * b) The second thing fss_decay_usage does is update each project's "usage"
 *    for the last second and then recalculates the project's "share usage".
 *
 *    The usage value is the recent CPU usage for all of the threads in the
 *    project. It is decayed and updated this way:
 *
 *                  (usage * FSS_DECAY_USG)
 *        usage =  ------------------------- + ticks;
 *                       FSS_DECAY_BASE
 *
 *     - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
 *     - FSS_DECAY_USG is 96 - approximates 75% (96/128)
 *     - ticks is updated whenever a process in this project is running
 *       when the scheduler's tick processing fires. This is not a simple
 *       counter, the values are based on the entries in the fss_nice_tick
 *       array (see section 3 below). ticks is then reset to 0 so it can track
 *       the next seconds worth of nice-adjusted time for the project.
 *
 * c) The third thing fss_decay_usage does is update each project's "share
 *    usage" (shusage). This is the normalized usage value for the project and
 *    is calculated this way:
 *
 *                pset_shares^2    zone_int_shares^2
 *        usage * ------------- * ------------------
 *                kpj_shares^2     zone_ext_shares^2
 *
 *    - usage - see (1b) for more details
 *    - pset_shares is the total of all *active* zone shares in the pset (by
 *      default there is only one pset)
 *    - kpj_shares is the individual project's share (project.cpu-shares rctl)
 *    - zone_int_shares is the sum of shares of all active projects within the
 *      zone (the zone-internal total)
 *    - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
 *
 *    The shusage is used in step (2b) to calculate the thread's new internal
 *    priority. A larger shusage value leads to a lower priority.
 *
 * 2) The fss_update function then calls fss_update_list to update the priority
 *    of all threads. This does two things.
 *
 * a) First the thread's internal priority is decayed using the following
 *    formula:
 *
 *                  fsspri * fss_nice_decay[nice_value])
 *        fsspri =  ------------------------------------
 *                            FSS_DECAY_BASE
 *
 *     - FSS_DECAY_BASE is 128 as described above
 *
 * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
 *    to update the user-mode priority (umdpri) of the runnable thread.
 *    Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
 *    are not updated at this time. The updated user-mode priority can cause
 *    threads to change their position in the run queue.
 *
 *    The process's new internal fsspri is calculated using the following
 *    formula. All runnable threads in the project will use the same shusage
 *    and nrunnable values in their calculation.
 *
 *        fsspri += shusage * nrunnable * ticks
 *
 *     - shusage is the project's share usage, calculated in (1c)
 *     - nrunnable is the number of runnable threads in the project
 *     - ticks is the number of ticks this thread ran since the last fss_newpri
 *       invocation.
 *
 *    Finally the process's new user-mode priority is calculated using the
 *    following formula:
 *
 *                              (fsspri * umdprirange)
 *        umdpri = maxumdpri - ------------------------
 *                                    maxfsspri
 *
 *     - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
 *     - umdprirange is maxumdpri - 1 (i.e. 58)
 *     - maxfsspri is the largest fsspri seen so far, as we're iterating all
 *       runnable processes
 *
 *    Thus, a higher internal priority (fsspri) leads to a lower user-mode
 *    priority which means the thread runs less. The fsspri is higher when
 *    the project's normalized share usage is higher, when the project has
 *    more runnable threads, or when the thread has accumulated more run-time.
 *
 *    This code has various checks to ensure the resulting umdpri is in the
 *    range 1-59.  See fss_newpri for more details.
 *
 * To reiterate, the above processing is performed once per second to recompute
 * the runnable thread user-mode priorities.
 *
 * 3) The final major component in the priority calculation is the tick
 *    processing which occurs on a thread that is running when the clock
 *    calls fss_tick.
 *
 *    A thread can run continuously in user-land (compute-bound) for the
 *    fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
 *    The fss_quantum defaults to 11 (i.e. 11 ticks).
 *
 *    Once the quantum has been consumed, the thread will call fss_newpri to
 *    recompute its umdpri priority, as described above in (2b). Threads that
 *    were T_ONPROC at the one second interval when runnable thread priorities
 *    were recalculated will have their umdpri priority recalculated when their
 *    quanta expires.
 *
 *    To ensure that runnable threads within a project see the expected
 *    round-robin behavior, there is a special case in fss_newpri for a thread
 *    that has run for its quanta within the one second update interval.  See
 *    the handling for the quanta_up parameter within fss_newpri.
 *
 *    Also of interest, the fss_tick code increments the project's tick value
 *    using the fss_nice_tick array entry for the thread's nice value. The idea
 *    behind the fss_nice_tick array is that the cost of a tick is lower at
 *    positive nice values (so that it doesn't increase the project's usage
 *    as much as normal) with a 50% drop at the maximum level and a 50%
 *    increase at the minimum level. See (1b). The fss_nice_tick array is
 *    initialized in fss_init using the following formula:
 *
 *         FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
 *        --------------------------------------------------
 *                          FSS_NICE_RANGE
 *
 *     - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
 *
 * FSS Data Structures:
 *
 *                 fsszone
 *                  -----           -----
 *  -----          |     |         |     |
 * |     |-------->|     |<------->|     |<---->...
 * |     |          -----           -----
 * |     |          ^    ^            ^
 * |     |---       |     \            \
 *  -----    |      |      \            \
 * fsspset   |      |       \            \
 *           |      |        \            \
 *           |    -----       -----       -----
 *            -->|     |<--->|     |<--->|     |
 *               |     |     |     |     |     |
 *                -----       -----       -----
 *               fssproj
 *
 * That is, fsspsets contain a list of fsszone's that are currently active in
 * the pset, and a list of fssproj's, corresponding to projects with runnable
 * threads on the pset.  fssproj's in turn point to the fsszone which they
 * are a member of.
 *
 * An fssproj_t is removed when there are no threads in it.
 *
 * An fsszone_t is removed when there are no projects with threads in it.
 */

static pri_t fss_init(id_t, int, classfuncs_t **);

static struct sclass fss = {
        "FSS",
        fss_init,
        0
};

extern struct mod_ops mod_schedops;

/*
 * Module linkage information for the kernel.
 */
static struct modlsched modlsched = {
        &mod_schedops, "fair share scheduling class", &fss
};

static struct modlinkage modlinkage = {
        MODREV_1, (void *)&modlsched, NULL
};

#define FSS_MAXUPRI     60

/*
 * The fssproc_t structures are kept in an array of circular doubly linked
 * lists.  A hash on the thread pointer is used to determine which list each
 * thread should be placed in.  Each list has a dummy "head" which is never
 * removed, so the list is never empty.  fss_update traverses these lists to
 * update the priorities of threads that have been waiting on the run queue.
 */
#define FSS_LISTS               16 /* number of lists, must be power of 2 */
#define FSS_LIST_HASH(t)        (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
#define FSS_LIST_NEXT(i)        (((i) + 1) & (FSS_LISTS - 1))

#define FSS_LIST_INSERT(fssproc)                                \
{                                                               \
        int index = FSS_LIST_HASH(fssproc->fss_tp);             \
        kmutex_t *lockp = &fss_listlock[index];                 \
        fssproc_t *headp = &fss_listhead[index];                \
        mutex_enter(lockp);                                     \
        fssproc->fss_next = headp->fss_next;                    \
        fssproc->fss_prev = headp;                              \
        headp->fss_next->fss_prev = fssproc;                    \
        headp->fss_next = fssproc;                              \
        mutex_exit(lockp);                                      \
}

#define FSS_LIST_DELETE(fssproc)                                \
{                                                               \
        int index = FSS_LIST_HASH(fssproc->fss_tp);             \
        kmutex_t *lockp = &fss_listlock[index];                 \
        mutex_enter(lockp);                                     \
        fssproc->fss_prev->fss_next = fssproc->fss_next;        \
        fssproc->fss_next->fss_prev = fssproc->fss_prev;        \
        mutex_exit(lockp);                                      \
}

#define FSS_TICK_COST   1000    /* tick cost for threads with nice level = 0 */

/*
 * Decay rate percentages are based on n/128 rather than n/100 so  that
 * calculations can avoid having to do an integer divide by 100 (divide
 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
 *
 * FSS_DECAY_MIN        =  83/128 ~= 65%
 * FSS_DECAY_MAX        = 108/128 ~= 85%
 * FSS_DECAY_USG        =  96/128 ~= 75%
 */
#define FSS_DECAY_MIN   83      /* fsspri decay pct for threads w/ nice -20 */
#define FSS_DECAY_MAX   108     /* fsspri decay pct for threads w/ nice +19 */
#define FSS_DECAY_USG   96      /* fssusage decay pct for projects */
#define FSS_DECAY_BASE  128     /* base for decay percentages above */

#define FSS_NICE_MIN    0
#define FSS_NICE_MAX    (2 * NZERO - 1)
#define FSS_NICE_RANGE  (FSS_NICE_MAX - FSS_NICE_MIN + 1)

static int      fss_nice_tick[FSS_NICE_RANGE];
static int      fss_nice_decay[FSS_NICE_RANGE];

static pri_t    fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
static pri_t    fss_maxumdpri; /* maximum user mode fss priority */
static pri_t    fss_maxglobpri; /* maximum global priority used by fss class */
static pri_t    fss_minglobpri; /* minimum global priority */

static fssproc_t fss_listhead[FSS_LISTS];
static kmutex_t fss_listlock[FSS_LISTS];

static fsspset_t *fsspsets;
static kmutex_t fsspsets_lock;  /* protects fsspsets */

static id_t     fss_cid;

static int      fss_quantum = 11;

static void     fss_newpri(fssproc_t *, bool);
static void     fss_update(void *);
static int      fss_update_list(int);
static void     fss_change_priority(kthread_t *, fssproc_t *);

static int      fss_admin(caddr_t, cred_t *);
static int      fss_getclinfo(void *);
static int      fss_parmsin(void *);
static int      fss_parmsout(void *, pc_vaparms_t *);
static int      fss_vaparmsin(void *, pc_vaparms_t *);
static int      fss_vaparmsout(void *, pc_vaparms_t *);
static int      fss_getclpri(pcpri_t *);
static int      fss_alloc(void **, int);
static void     fss_free(void *);

static int      fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
static void     fss_exitclass(void *);
static int      fss_canexit(kthread_t *, cred_t *);
static int      fss_fork(kthread_t *, kthread_t *, void *);
static void     fss_forkret(kthread_t *, kthread_t *);
static void     fss_parmsget(kthread_t *, void *);
static int      fss_parmsset(kthread_t *, void *, id_t, cred_t *);
static void     fss_stop(kthread_t *, int, int);
static void     fss_exit(kthread_t *);
static void     fss_active(kthread_t *);
static void     fss_inactive(kthread_t *);
static void     fss_trapret(kthread_t *);
static void     fss_preempt(kthread_t *);
static void     fss_setrun(kthread_t *);
static void     fss_sleep(kthread_t *);
static void     fss_tick(kthread_t *);
static void     fss_wakeup(kthread_t *);
static int      fss_donice(kthread_t *, cred_t *, int, int *);
static int      fss_doprio(kthread_t *, cred_t *, int, int *);
static pri_t    fss_globpri(kthread_t *);
static void     fss_yield(kthread_t *);
static void     fss_nullsys();

static struct classfuncs fss_classfuncs = {
        /* class functions */
        fss_admin,
        fss_getclinfo,
        fss_parmsin,
        fss_parmsout,
        fss_vaparmsin,
        fss_vaparmsout,
        fss_getclpri,
        fss_alloc,
        fss_free,

        /* thread functions */
        fss_enterclass,
        fss_exitclass,
        fss_canexit,
        fss_fork,
        fss_forkret,
        fss_parmsget,
        fss_parmsset,
        fss_stop,
        fss_exit,
        fss_active,
        fss_inactive,
        fss_trapret,
        fss_preempt,
        fss_setrun,
        fss_sleep,
        fss_tick,
        fss_wakeup,
        fss_donice,
        fss_globpri,
        fss_nullsys,    /* set_process_group */
        fss_yield,
        fss_doprio,
};

int
_init()
{
        return (mod_install(&modlinkage));
}

int
_fini()
{
        return (EBUSY);
}

int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

/*ARGSUSED*/
static int
fss_project_walker(kproject_t *kpj, void *buf)
{
        return (0);
}

void *
fss_allocbuf(int op, int type)
{
        fssbuf_t *fssbuf;
        void **fsslist;
        int cnt;
        int i;
        size_t size;

        ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
        ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
        ASSERT(MUTEX_HELD(&cpu_lock));

        fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
        switch (op) {
        case FSS_NPSET_BUF:
                cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
                break;
        case FSS_NPROJ_BUF:
                cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
                break;
        case FSS_ONE_BUF:
                cnt = 1;
                break;
        }

        switch (type) {
        case FSS_ALLOC_PROJ:
                size = sizeof (fssproj_t);
                break;
        case FSS_ALLOC_ZONE:
                size = sizeof (fsszone_t);
                break;
        }
        fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
        fssbuf->fssb_size = cnt;
        fssbuf->fssb_list = fsslist;
        for (i = 0; i < cnt; i++)
                fsslist[i] = kmem_zalloc(size, KM_SLEEP);
        return (fssbuf);
}

void
fss_freebuf(fssbuf_t *fssbuf, int type)
{
        void **fsslist;
        int i;
        size_t size;

        ASSERT(fssbuf != NULL);
        ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
        fsslist = fssbuf->fssb_list;

        switch (type) {
        case FSS_ALLOC_PROJ:
                size = sizeof (fssproj_t);
                break;
        case FSS_ALLOC_ZONE:
                size = sizeof (fsszone_t);
                break;
        }

        for (i = 0; i < fssbuf->fssb_size; i++) {
                if (fsslist[i] != NULL)
                        kmem_free(fsslist[i], size);
        }
        kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
        kmem_free(fssbuf, sizeof (fssbuf_t));
}

static fsspset_t *
fss_find_fsspset(cpupart_t *cpupart)
{
        int i;
        fsspset_t *fsspset = NULL;
        int found = 0;

        ASSERT(cpupart != NULL);
        ASSERT(MUTEX_HELD(&fsspsets_lock));

        /*
         * Search for the cpupart pointer in the array of fsspsets.
         */
        for (i = 0; i < max_ncpus; i++) {
                fsspset = &fsspsets[i];
                if (fsspset->fssps_cpupart == cpupart) {
                        ASSERT(fsspset->fssps_nproj > 0);
                        found = 1;
                        break;
                }
        }
        if (found == 0) {
                /*
                 * If we didn't find anything, then use the first
                 * available slot in the fsspsets array.
                 */
                for (i = 0; i < max_ncpus; i++) {
                        fsspset = &fsspsets[i];
                        if (fsspset->fssps_cpupart == NULL) {
                                ASSERT(fsspset->fssps_nproj == 0);
                                found = 1;
                                break;
                        }
                }
                fsspset->fssps_cpupart = cpupart;
        }
        ASSERT(found == 1);
        return (fsspset);
}

static void
fss_del_fsspset(fsspset_t *fsspset)
{
        ASSERT(MUTEX_HELD(&fsspsets_lock));
        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
        ASSERT(fsspset->fssps_nproj == 0);
        ASSERT(fsspset->fssps_list == NULL);
        ASSERT(fsspset->fssps_zones == NULL);
        fsspset->fssps_cpupart = NULL;
        fsspset->fssps_maxfsspri = 0;
        fsspset->fssps_shares = 0;
}

/*
 * The following routine returns a pointer to the fsszone structure which
 * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
 */
static fsszone_t *
fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
{
        fsszone_t *fsszone;

        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));

        if (fsspset->fssps_list != NULL) {
                /*
                 * There are projects/zones active on this cpu partition
                 * already.  Try to find our zone among them.
                 */
                fsszone = fsspset->fssps_zones;
                do {
                        if (fsszone->fssz_zone == zone) {
                                return (fsszone);
                        }
                        fsszone = fsszone->fssz_next;
                } while (fsszone != fsspset->fssps_zones);
        }
        return (NULL);
}

/*
 * The following routine links new fsszone structure into doubly linked list of
 * zones active on the specified cpu partition.
 */
static void
fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
{
        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));

        fsszone->fssz_zone = zone;
        fsszone->fssz_rshares = zone->zone_shares;

        if (fsspset->fssps_zones == NULL) {
                /*
                 * This will be the first fsszone for this fsspset
                 */
                fsszone->fssz_next = fsszone->fssz_prev = fsszone;
                fsspset->fssps_zones = fsszone;
        } else {
                /*
                 * Insert this fsszone to the doubly linked list.
                 */
                fsszone_t *fssz_head = fsspset->fssps_zones;

                fsszone->fssz_next = fssz_head;
                fsszone->fssz_prev = fssz_head->fssz_prev;
                fssz_head->fssz_prev->fssz_next = fsszone;
                fssz_head->fssz_prev = fsszone;
                fsspset->fssps_zones = fsszone;
        }
}

/*
 * The following routine removes a single fsszone structure from the doubly
 * linked list of zones active on the specified cpu partition.  Note that
 * global fsspsets_lock must be held in case this fsszone structure is the last
 * on the above mentioned list.  Also note that the fsszone structure is not
 * freed here, it is the responsibility of the caller to call kmem_free for it.
 */
static void
fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
{
        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
        ASSERT(fsszone->fssz_nproj == 0);
        ASSERT(fsszone->fssz_shares == 0);
        ASSERT(fsszone->fssz_runnable == 0);

        if (fsszone->fssz_next != fsszone) {
                /*
                 * This is not the last zone in the list.
                 */
                fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
                fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
                if (fsspset->fssps_zones == fsszone)
                        fsspset->fssps_zones = fsszone->fssz_next;
        } else {
                /*
                 * This was the last zone active in this cpu partition.
                 */
                fsspset->fssps_zones = NULL;
        }
}

/*
 * The following routine returns a pointer to the fssproj structure
 * which belongs to project kpj and cpu partition fsspset, if such structure
 * exists.
 */
static fssproj_t *
fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
{
        fssproj_t *fssproj;

        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));

        if (fsspset->fssps_list != NULL) {
                /*
                 * There are projects running on this cpu partition already.
                 * Try to find our project among them.
                 */
                fssproj = fsspset->fssps_list;
                do {
                        if (fssproj->fssp_proj == kpj) {
                                ASSERT(fssproj->fssp_pset == fsspset);
                                return (fssproj);
                        }
                        fssproj = fssproj->fssp_next;
                } while (fssproj != fsspset->fssps_list);
        }
        return (NULL);
}

/*
 * The following routine links new fssproj structure into doubly linked list
 * of projects running on the specified cpu partition.
 */
static void
fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
    fssproj_t *fssproj)
{
        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));

        fssproj->fssp_pset = fsspset;
        fssproj->fssp_proj = kpj;
        fssproj->fssp_shares = kpj->kpj_shares;

        fsspset->fssps_nproj++;

        if (fsspset->fssps_list == NULL) {
                /*
                 * This will be the first fssproj for this fsspset
                 */
                fssproj->fssp_next = fssproj->fssp_prev = fssproj;
                fsspset->fssps_list = fssproj;
        } else {
                /*
                 * Insert this fssproj to the doubly linked list.
                 */
                fssproj_t *fssp_head = fsspset->fssps_list;

                fssproj->fssp_next = fssp_head;
                fssproj->fssp_prev = fssp_head->fssp_prev;
                fssp_head->fssp_prev->fssp_next = fssproj;
                fssp_head->fssp_prev = fssproj;
                fsspset->fssps_list = fssproj;
        }
        fssproj->fssp_fsszone = fsszone;
        fsszone->fssz_nproj++;
        ASSERT(fsszone->fssz_nproj != 0);
}

/*
 * The following routine removes a single fssproj structure from the doubly
 * linked list of projects running on the specified cpu partition.  Note that
 * global fsspsets_lock must be held in case if this fssproj structure is the
 * last on the above mentioned list.  Also note that the fssproj structure is
 * not freed here, it is the responsibility of the caller to call kmem_free
 * for it.
 */
static void
fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
{
        fsszone_t *fsszone;

        ASSERT(MUTEX_HELD(&fsspsets_lock));
        ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
        ASSERT(fssproj->fssp_runnable == 0);

        fsspset->fssps_nproj--;

        fsszone = fssproj->fssp_fsszone;
        fsszone->fssz_nproj--;

        if (fssproj->fssp_next != fssproj) {
                /*
                 * This is not the last part in the list.
                 */
                fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
                fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
                if (fsspset->fssps_list == fssproj)
                        fsspset->fssps_list = fssproj->fssp_next;
                if (fsszone->fssz_nproj == 0)
                        fss_remove_fsszone(fsspset, fsszone);
        } else {
                /*
                 * This was the last project part running
                 * at this cpu partition.
                 */
                fsspset->fssps_list = NULL;
                ASSERT(fsspset->fssps_nproj == 0);
                ASSERT(fsszone->fssz_nproj == 0);
                fss_remove_fsszone(fsspset, fsszone);
                fss_del_fsspset(fsspset);
        }
}

static void
fss_inactive(kthread_t *t)
{
        fssproc_t *fssproc;
        fssproj_t *fssproj;
        fsspset_t *fsspset;
        fsszone_t *fsszone;

        ASSERT(THREAD_LOCK_HELD(t));
        fssproc = FSSPROC(t);
        fssproj = FSSPROC2FSSPROJ(fssproc);
        if (fssproj == NULL)    /* if this thread already exited */
                return;
        fsspset = FSSPROJ2FSSPSET(fssproj);
        fsszone = fssproj->fssp_fsszone;
        disp_lock_enter_high(&fsspset->fssps_displock);
        ASSERT(fssproj->fssp_runnable > 0);
        if (--fssproj->fssp_runnable == 0) {
                fsszone->fssz_shares -= fssproj->fssp_shares;
                if (--fsszone->fssz_runnable == 0)
                        fsspset->fssps_shares -= fsszone->fssz_rshares;
        }
        ASSERT(fssproc->fss_runnable == 1);
        fssproc->fss_runnable = 0;
        disp_lock_exit_high(&fsspset->fssps_displock);
}

static void
fss_active(kthread_t *t)
{
        fssproc_t *fssproc;
        fssproj_t *fssproj;
        fsspset_t *fsspset;
        fsszone_t *fsszone;

        ASSERT(THREAD_LOCK_HELD(t));
        fssproc = FSSPROC(t);
        fssproj = FSSPROC2FSSPROJ(fssproc);
        if (fssproj == NULL)    /* if this thread already exited */
                return;
        fsspset = FSSPROJ2FSSPSET(fssproj);
        fsszone = fssproj->fssp_fsszone;
        disp_lock_enter_high(&fsspset->fssps_displock);
        if (++fssproj->fssp_runnable == 1) {
                fsszone->fssz_shares += fssproj->fssp_shares;
                if (++fsszone->fssz_runnable == 1)
                        fsspset->fssps_shares += fsszone->fssz_rshares;
        }
        ASSERT(fssproc->fss_runnable == 0);
        fssproc->fss_runnable = 1;
        disp_lock_exit_high(&fsspset->fssps_displock);
}

/*
 * Fair share scheduler initialization. Called by dispinit() at boot time.
 * We can ignore clparmsz argument since we know that the smallest possible
 * parameter buffer is big enough for us.
 */
/*ARGSUSED*/
static pri_t
fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
{
        int i;

        ASSERT(MUTEX_HELD(&cpu_lock));

        fss_cid = cid;
        fss_maxumdpri = minclsyspri - 1;
        fss_maxglobpri = minclsyspri;
        fss_minglobpri = 0;
        fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);

        /*
         * Initialize the fssproc hash table.
         */
        for (i = 0; i < FSS_LISTS; i++)
                fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
                    &fss_listhead[i];

        *clfuncspp = &fss_classfuncs;

        /*
         * Fill in fss_nice_tick and fss_nice_decay arrays:
         * The cost of a tick is lower at positive nice values (so that it
         * will not increase its project's usage as much as normal) with 50%
         * drop at the maximum level and 50% increase at the minimum level.
         * The fsspri decay is slower at positive nice values.  fsspri values
         * of processes with negative nice levels must decay faster to receive
         * time slices more frequently than normal.
         */
        for (i = 0; i < FSS_NICE_RANGE; i++) {
                fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
                    - i)) / FSS_NICE_RANGE;
                fss_nice_decay[i] = FSS_DECAY_MIN +
                    ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
                    (FSS_NICE_RANGE - 1);
        }

        return (fss_maxglobpri);
}

/*
 * Calculate the new fss_umdpri based on the usage, the normalized share usage
 * and the number of active threads.  Reset the tick counter for this thread.
 *
 * When calculating the new priority using the standard formula we can hit
 * a scenario where we don't have good round-robin behavior.  This would be
 * most commonly seen when there is a zone with lots of runnable threads.
 * In the bad scenario we will see the following behavior when using the
 * standard formula and these conditions:
 *
 *      - there are multiple runnable threads in the zone (project)
 *      - the fssps_maxfsspri is a very large value
 *      - (we also know all of these threads will use the project's
 *          fssp_shusage)
 *
 * Under these conditions, a thread with a low fss_fsspri value is chosen
 * to run and the thread gets a high fss_umdpri.  This thread can run for
 * its full quanta (fss_timeleft) at which time fss_newpri is called to
 * calculate the thread's new priority.
 *
 * In this case, because the newly calculated fsspri value is much smaller
 * (orders of magnitude) than the fssps_maxfsspri value, if we used the
 * standard formula the thread will still get a high fss_umdpri value and
 * will run again for another quanta, even though there are other runnable
 * threads in the project.
 *
 * For a thread that is runnable for a long time, the thread can continue
 * to run for many quanta (totaling many seconds) before the thread's fsspri
 * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
 * down to 1.  This behavior also keeps the fssps_maxfsspr at a high value,
 * so that the next runnable thread might repeat this cycle.
 *
 * This leads to the case where we don't have round-robin behavior at quanta
 * granularity, but instead, runnable threads within the project only run
 * at several second intervals.
 *
 * To prevent this scenario from occuring, when a thread has consumed its
 * quanta and there are multiple runnable threads in the project, we
 * immediately cause the thread to hit fssps_maxfsspri so that it gets
 * reset back to 1 and another runnable thread in the project can run.
 */
static void
fss_newpri(fssproc_t *fssproc, bool quanta_up)
{
        kthread_t *tp;
        fssproj_t *fssproj;
        fsspset_t *fsspset;
        fsszone_t *fsszone;
        fsspri_t fsspri, maxfsspri;
        uint32_t n_runnable;
        pri_t invpri;
        uint32_t ticks;

        tp = fssproc->fss_tp;
        ASSERT(tp != NULL);

        if (tp->t_cid != fss_cid)
                return;

        ASSERT(THREAD_LOCK_HELD(tp));

        fssproj = FSSPROC2FSSPROJ(fssproc);
        fsszone = FSSPROJ2FSSZONE(fssproj);
        if (fssproj == NULL)
                /*
                 * No need to change priority of exited threads.
                 */
                return;

        fsspset = FSSPROJ2FSSPSET(fssproj);
        disp_lock_enter_high(&fsspset->fssps_displock);

        ticks = fssproc->fss_ticks;
        fssproc->fss_ticks = 0;

        if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
                /*
                 * Special case: threads with no shares.
                 */
                fssproc->fss_umdpri = fss_minglobpri;
                disp_lock_exit_high(&fsspset->fssps_displock);
                return;
        }

        maxfsspri = fsspset->fssps_maxfsspri;
        n_runnable = fssproj->fssp_runnable;

        if (quanta_up && n_runnable > 1) {
                fsspri = maxfsspri;
        } else {
                /*
                 * fsspri += fssp_shusage * nrunnable * ticks
                 * If all three values are non-0, this typically calculates to
                 * a large number (sometimes > 1M, sometimes > 100B) due to
                 * fssp_shusage which can be > 1T.
                 */
                fsspri = fssproc->fss_fsspri;
                fsspri += fssproj->fssp_shusage * n_runnable * ticks;
        }

        fssproc->fss_fsspri = fsspri;

        /*
         * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
         * If the previous calculation resulted in 0 (e.g. was 0 and added 0
         * because ticks == 0), then instead of 0, we use the largest priority,
         * which is still small in comparison to the large numbers we typically
         * see.
         */
        if (fsspri < fss_maxumdpri)
                fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */

        /*
         * The general priority formula:
         *
         *                      (fsspri * umdprirange)
         *   pri = maxumdpri - ------------------------
         *                              maxfsspri
         *
         * If this thread's fsspri is greater than the previous largest
         * fsspri, then record it as the new high and priority for this
         * thread will be one (the lowest priority assigned to a thread
         * that has non-zero shares). Because of this check, maxfsspri can
         * change as this function is called via the
         * fss_update -> fss_update_list -> fss_newpri code path to update
         * all runnable threads. See the code in fss_update for how we
         * mitigate this issue.
         *
         * Note that this formula cannot produce out of bounds priority
         * values (0-59); if it is changed, additional checks may need to be
         * added.
         */
        if (fsspri >= maxfsspri) {
                fsspset->fssps_maxfsspri = fsspri;
                disp_lock_exit_high(&fsspset->fssps_displock);
                fssproc->fss_umdpri = 1;
        } else {
                disp_lock_exit_high(&fsspset->fssps_displock);
                invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
                fssproc->fss_umdpri = fss_maxumdpri - invpri;
        }
}

/*
 * Decays usages of all running projects, resets their tick counters and
 * calcluates the projects normalized share usage. Called once per second from
 * fss_update().
 */
static void
fss_decay_usage()
{
        uint32_t zone_ext_shares, zone_int_shares;
        uint32_t kpj_shares, pset_shares;
        fsspset_t *fsspset;
        fssproj_t *fssproj;
        fsszone_t *fsszone;
        fsspri_t maxfsspri;
        int psetid;
        struct zone *zp;

        mutex_enter(&fsspsets_lock);
        /*
         * Go through all active processor sets and decay usages of projects
         * running on them.
         */
        for (psetid = 0; psetid < max_ncpus; psetid++) {
                fsspset = &fsspsets[psetid];
                mutex_enter(&fsspset->fssps_lock);

                fsspset->fssps_gen++;

                if (fsspset->fssps_cpupart == NULL ||
                    (fssproj = fsspset->fssps_list) == NULL) {
                        mutex_exit(&fsspset->fssps_lock);
                        continue;
                }

                /*
                 * Decay maxfsspri for this cpu partition with the
                 * fastest possible decay rate.
                 */
                disp_lock_enter(&fsspset->fssps_displock);

                pset_shares = fsspset->fssps_shares;

                maxfsspri = (fsspset->fssps_maxfsspri *
                    fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
                if (maxfsspri < fss_maxumdpri)
                        maxfsspri = fss_maxumdpri;
                fsspset->fssps_maxfsspri = maxfsspri;

                do {
                        fsszone = fssproj->fssp_fsszone;
                        zp = fsszone->fssz_zone;

                        /*
                         * Reset zone's FSS stats if they are from a
                         * previous cycle.
                         */
                        if (fsspset->fssps_gen != zp->zone_fss_gen) {
                                zp->zone_fss_gen = fsspset->fssps_gen;
                                zp->zone_run_ticks = 0;
                        }

                        /*
                         * Decay project usage, then add in this cycle's
                         * nice tick value.
                         */
                        fssproj->fssp_usage =
                            (fssproj->fssp_usage * FSS_DECAY_USG) /
                            FSS_DECAY_BASE +
                            fssproj->fssp_ticks;

                        fssproj->fssp_ticks = 0;
                        zp->zone_run_ticks += fssproj->fssp_tick_cnt;
                        fssproj->fssp_tick_cnt = 0;

                        /*
                         * Readjust the project's number of shares if it has
                         * changed since we checked it last time.
                         */
                        kpj_shares = fssproj->fssp_proj->kpj_shares;
                        if (fssproj->fssp_shares != kpj_shares) {
                                if (fssproj->fssp_runnable != 0) {
                                        fsszone->fssz_shares -=
                                            fssproj->fssp_shares;
                                        fsszone->fssz_shares += kpj_shares;
                                }
                                fssproj->fssp_shares = kpj_shares;
                        }

                        /*
                         * Readjust the zone's number of shares if it
                         * has changed since we checked it last time.
                         */
                        zone_ext_shares = zp->zone_shares;
                        if (fsszone->fssz_rshares != zone_ext_shares) {
                                if (fsszone->fssz_runnable != 0) {
                                        fsspset->fssps_shares -=
                                            fsszone->fssz_rshares;
                                        fsspset->fssps_shares +=
                                            zone_ext_shares;
                                        pset_shares = fsspset->fssps_shares;
                                }
                                fsszone->fssz_rshares = zone_ext_shares;
                        }
                        zone_int_shares = fsszone->fssz_shares;

                        /*
                         * If anything is runnable in the project, track the
                         * overall project share percent for monitoring useage.
                         */
                        if (fssproj->fssp_runnable > 0) {
                                uint32_t zone_shr_pct;
                                uint32_t int_shr_pct;

                                /*
                                 * Times 1000 to get tenths of a percent
                                 *
                                 *                zone_ext_shares
                                 * zone_shr_pct = ---------------
                                 *                pset_shares
                                 *
                                 *                kpj_shares
                                 * int_shr_pct =  ---------------
                                 *                zone_int_shares
                                 */
                                if (pset_shares == 0 || zone_int_shares == 0) {
                                        fssproj->fssp_shr_pct = 0;
                                } else {
                                        zone_shr_pct =
                                            (zone_ext_shares * 1000) /
                                            pset_shares;
                                        int_shr_pct = (kpj_shares * 1000) /
                                            zone_int_shares;
                                        fssproj->fssp_shr_pct =
                                            (zone_shr_pct * int_shr_pct) /
                                            1000;
                                }
                        } else {
                                DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
                                    fssproj);
                        }

                        /*
                         * Calculate fssp_shusage value to be used
                         * for fsspri increments for the next second.
                         */
                        if (kpj_shares == 0 || zone_ext_shares == 0) {
                                fssproj->fssp_shusage = 0;
                        } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
                                uint32_t zone_shr_pct;

                                /*
                                 * Project 0 in the global zone has 50%
                                 * of its zone. See calculation above for
                                 * the zone's share percent.
                                 */
                                if (pset_shares == 0)
                                        zone_shr_pct = 1000;
                                else
                                        zone_shr_pct =
                                            (zone_ext_shares * 1000) /
                                            pset_shares;

                                fssproj->fssp_shr_pct = zone_shr_pct / 2;

                                fssproj->fssp_shusage = (fssproj->fssp_usage *
                                    zone_int_shares * zone_int_shares) /
                                    (zone_ext_shares * zone_ext_shares);
                        } else {
                                /*
                                 * Thread's priority is based on its project's
                                 * normalized usage (shusage) value which gets
                                 * calculated this way:
                                 *
                                 *         pset_shares^2    zone_int_shares^2
                                 * usage * ------------- * ------------------
                                 *         kpj_shares^2     zone_ext_shares^2
                                 *
                                 * Where zone_int_shares is the sum of shares
                                 * of all active projects within the zone (and
                                 * the pset), and zone_ext_shares is the number
                                 * of zone shares (ie, zone.cpu-shares).
                                 *
                                 * If there is only one zone active on the pset
                                 * the above reduces to:
                                 *
                                 *                      zone_int_shares^2
                                 * shusage = usage * ---------------------
                                 *                      kpj_shares^2
                                 *
                                 * If there's only one project active in the
                                 * zone this formula reduces to:
                                 *
                                 *                      pset_shares^2
                                 * shusage = usage * ----------------------
                                 *                      zone_ext_shares^2
                                 *
                                 * shusage is one input to calculating fss_pri
                                 * in fss_newpri(). Larger values tend toward
                                 * lower priorities for processes in the proj.
                                 */
                                fssproj->fssp_shusage = fssproj->fssp_usage *
                                    pset_shares * zone_int_shares;
                                fssproj->fssp_shusage /=
                                    kpj_shares * zone_ext_shares;
                                fssproj->fssp_shusage *=
                                    pset_shares * zone_int_shares;
                                fssproj->fssp_shusage /=
                                    kpj_shares * zone_ext_shares;
                        }
                        fssproj = fssproj->fssp_next;
                } while (fssproj != fsspset->fssps_list);

                disp_lock_exit(&fsspset->fssps_displock);
                mutex_exit(&fsspset->fssps_lock);
        }
        mutex_exit(&fsspsets_lock);
}

static void
fss_change_priority(kthread_t *t, fssproc_t *fssproc)
{
        pri_t new_pri;

        ASSERT(THREAD_LOCK_HELD(t));
        new_pri = fssproc->fss_umdpri;
        ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);

        t->t_cpri = fssproc->fss_upri;
        fssproc->fss_flags &= ~FSSRESTORE;
        if (t == curthread || t->t_state == TS_ONPROC) {
                /*
                 * curthread is always onproc
                 */
                cpu_t *cp = t->t_disp_queue->disp_cpu;
                THREAD_CHANGE_PRI(t, new_pri);
                if (t == cp->cpu_dispthread)
                        cp->cpu_dispatch_pri = DISP_PRIO(t);
                if (DISP_MUST_SURRENDER(t)) {
                        fssproc->fss_flags |= FSSBACKQ;
                        cpu_surrender(t);
                } else {
                        fssproc->fss_timeleft = fss_quantum;
                }
        } else {
                /*
                 * When the priority of a thread is changed, it may be
                 * necessary to adjust its position on a sleep queue or
                 * dispatch queue.  The function thread_change_pri accomplishes
                 * this.
                 */
                if (thread_change_pri(t, new_pri, 0)) {
                        /*
                         * The thread was on a run queue.
                         */
                        fssproc->fss_timeleft = fss_quantum;
                } else {
                        fssproc->fss_flags |= FSSBACKQ;
                }
        }
}

/*
 * Update priorities of all fair-sharing threads that are currently runnable
 * at a user mode priority based on the number of shares and current usage.
 * Called once per second via timeout which we reset here.
 *
 * There are several lists of fair-sharing threads broken up by a hash on the
 * thread pointer.  Each list has its own lock.  This avoids blocking all
 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
 * fss_update traverses each list in turn.
 *
 * Each time we're run (once/second) we may start at the next list and iterate
 * through all of the lists. By starting with a different list, we mitigate any
 * effects we would see updating the fssps_maxfsspri value in fss_newpri.
 */
static void
fss_update(void *arg)
{
        int i;
        int new_marker = -1;
        static int fss_update_marker;

        /*
         * Decay and update usages for all projects.
         */
        fss_decay_usage();

        /*
         * Start with the fss_update_marker list, then do the rest.
         */
        i = fss_update_marker;

        /*
         * Go around all threads, set new priorities and decay
         * per-thread CPU usages.
         */
        do {
                /*
                 * If this is the first list after the current marker to have
                 * threads with priority updates, advance the marker to this
                 * list for the next time fss_update runs.
                 */
                if (fss_update_list(i) &&
                    new_marker == -1 && i != fss_update_marker)
                        new_marker = i;
        } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);

        /*
         * Advance marker for the next fss_update call
         */
        if (new_marker != -1)
                fss_update_marker = new_marker;

        (void) timeout(fss_update, arg, hz);
}

/*
 * Updates priority for a list of threads.  Returns 1 if the priority of one
 * of the threads was actually updated, 0 if none were for various reasons
 * (thread is no longer in the FSS class, is not runnable, has the preemption
 * control no-preempt bit set, etc.)
 */
static int
fss_update_list(int i)
{
        fssproc_t *fssproc;
        fssproj_t *fssproj;
        fsspri_t fsspri;
        pri_t fss_umdpri;
        kthread_t *t;
        int updated = 0;

        mutex_enter(&fss_listlock[i]);
        for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
            fssproc = fssproc->fss_next) {
                t = fssproc->fss_tp;
                /*
                 * Lock the thread and verify the state.
                 */
                thread_lock(t);
                /*
                 * Skip the thread if it is no longer in the FSS class or
                 * is running with kernel mode priority.
                 */
                if (t->t_cid != fss_cid)
                        goto next;
                if ((fssproc->fss_flags & FSSKPRI) != 0)
                        goto next;

                fssproj = FSSPROC2FSSPROJ(fssproc);
                if (fssproj == NULL)
                        goto next;

                if (fssproj->fssp_shares != 0) {
                        /*
                         * Decay fsspri value.
                         */
                        fsspri = fssproc->fss_fsspri;
                        fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
                            FSS_DECAY_BASE;
                        fssproc->fss_fsspri = fsspri;
                }

                if (t->t_schedctl && schedctl_get_nopreempt(t))
                        goto next;
                if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
                        /*
                         * Make next syscall/trap call fss_trapret
                         */
                        t->t_trapret = 1;
                        aston(t);
                        if (t->t_state == TS_ONPROC)
                                DTRACE_PROBE1(fss__onproc, fssproc_t *,
                                    fssproc);
                        goto next;
                }
                fss_newpri(fssproc, false);
                updated = 1;

                fss_umdpri = fssproc->fss_umdpri;

                /*
                 * Only dequeue the thread if it needs to be moved; otherwise
                 * it should just round-robin here.
                 */
                if (t->t_pri != fss_umdpri)
                        fss_change_priority(t, fssproc);
next:
                thread_unlock(t);
        }
        mutex_exit(&fss_listlock[i]);
        return (updated);
}

/*ARGSUSED*/
static int
fss_admin(caddr_t uaddr, cred_t *reqpcredp)
{
        fssadmin_t fssadmin;

        if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
                return (EFAULT);

        switch (fssadmin.fss_cmd) {
        case FSS_SETADMIN:
                if (secpolicy_dispadm(reqpcredp) != 0)
                        return (EPERM);
                if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
                        return (EINVAL);
                fss_quantum = fssadmin.fss_quantum;
                break;
        case FSS_GETADMIN:
                fssadmin.fss_quantum = fss_quantum;
                if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
                        return (EFAULT);
                break;
        default:
                return (EINVAL);
        }
        return (0);
}

static int
fss_getclinfo(void *infop)
{
        fssinfo_t *fssinfo = (fssinfo_t *)infop;
        fssinfo->fss_maxupri = fss_maxupri;
        return (0);
}

static int
fss_parmsin(void *parmsp)
{
        fssparms_t *fssparmsp = (fssparms_t *)parmsp;

        /*
         * Check validity of parameters.
         */
        if ((fssparmsp->fss_uprilim > fss_maxupri ||
            fssparmsp->fss_uprilim < -fss_maxupri) &&
            fssparmsp->fss_uprilim != FSS_NOCHANGE)
                return (EINVAL);

        if ((fssparmsp->fss_upri > fss_maxupri ||
            fssparmsp->fss_upri < -fss_maxupri) &&
            fssparmsp->fss_upri != FSS_NOCHANGE)
                return (EINVAL);

        return (0);
}

/*ARGSUSED*/
static int
fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
{
        return (0);
}

static int
fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
{
        fssparms_t *fssparmsp = (fssparms_t *)parmsp;
        int priflag = 0;
        int limflag = 0;
        uint_t cnt;
        pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];

        /*
         * FSS_NOCHANGE (-32768) is outside of the range of values for
         * fss_uprilim and fss_upri.  If the structure fssparms_t is changed,
         * FSS_NOCHANGE should be replaced by a flag word.
         */
        fssparmsp->fss_uprilim = FSS_NOCHANGE;
        fssparmsp->fss_upri = FSS_NOCHANGE;

        /*
         * Get the varargs parameter and check validity of parameters.
         */
        if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
                return (EINVAL);

        for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
                switch (vpp->pc_key) {
                case FSS_KY_UPRILIM:
                        if (limflag++)
                                return (EINVAL);
                        fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
                        if (fssparmsp->fss_uprilim > fss_maxupri ||
                            fssparmsp->fss_uprilim < -fss_maxupri)
                                return (EINVAL);
                        break;
                case FSS_KY_UPRI:
                        if (priflag++)
                                return (EINVAL);
                        fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
                        if (fssparmsp->fss_upri > fss_maxupri ||
                            fssparmsp->fss_upri < -fss_maxupri)
                                return (EINVAL);
                        break;
                default:
                        return (EINVAL);
                }
        }

        if (vaparmsp->pc_vaparmscnt == 0) {
                /*
                 * Use default parameters.
                 */
                fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
        }

        return (0);
}

/*
 * Copy all selected fair-sharing class parameters to the user.  The parameters
 * are specified by a key.
 */
static int
fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
{
        fssparms_t *fssparmsp = (fssparms_t *)parmsp;
        int priflag = 0;
        int limflag = 0;
        uint_t cnt;
        pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];

        ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));

        if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
                return (EINVAL);

        for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
                switch (vpp->pc_key) {
                case FSS_KY_UPRILIM:
                        if (limflag++)
                                return (EINVAL);
                        if (copyout(&fssparmsp->fss_uprilim,
                            (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
                                return (EFAULT);
                        break;
                case FSS_KY_UPRI:
                        if (priflag++)
                                return (EINVAL);
                        if (copyout(&fssparmsp->fss_upri,
                            (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
                                return (EFAULT);
                        break;
                default:
                        return (EINVAL);
                }
        }

        return (0);
}

/*
 * Return the user mode scheduling priority range.
 */
static int
fss_getclpri(pcpri_t *pcprip)
{
        pcprip->pc_clpmax = fss_maxupri;
        pcprip->pc_clpmin = -fss_maxupri;
        return (0);
}

static int
fss_alloc(void **p, int flag)
{
        void *bufp;

        if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
                return (ENOMEM);
        } else {
                *p = bufp;
                return (0);
        }
}

static void
fss_free(void *bufp)
{
        if (bufp)
                kmem_free(bufp, sizeof (fssproc_t));
}

/*
 * Thread functions
 */
static int
fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
    void *bufp)
{
        fssparms_t      *fssparmsp = (fssparms_t *)parmsp;
        fssproc_t       *fssproc;
        pri_t           reqfssuprilim;
        pri_t           reqfssupri;
        static uint32_t fssexists = 0;
        fsspset_t       *fsspset;
        fssproj_t       *fssproj;
        fsszone_t       *fsszone;
        kproject_t      *kpj;
        zone_t          *zone;
        int             fsszone_allocated = 0;

        fssproc = (fssproc_t *)bufp;
        ASSERT(fssproc != NULL);

        ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

        /*
         * Only root can move threads to FSS class.
         */
        if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
                return (EPERM);
        /*
         * Initialize the fssproc structure.
         */
        fssproc->fss_umdpri = fss_maxumdpri / 2;

        if (fssparmsp == NULL) {
                /*
                 * Use default values.
                 */
                fssproc->fss_nice = NZERO;
                fssproc->fss_uprilim = fssproc->fss_upri = 0;
        } else {
                /*
                 * Use supplied values.
                 */
                if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
                        reqfssuprilim = 0;
                } else {
                        if (fssparmsp->fss_uprilim > 0 &&
                            secpolicy_setpriority(reqpcredp) != 0)
                                return (EPERM);
                        reqfssuprilim = fssparmsp->fss_uprilim;
                }
                if (fssparmsp->fss_upri == FSS_NOCHANGE) {
                        reqfssupri = reqfssuprilim;
                } else {
                        if (fssparmsp->fss_upri > 0 &&
                            secpolicy_setpriority(reqpcredp) != 0)
                                return (EPERM);
                        /*
                         * Set the user priority to the requested value or
                         * the upri limit, whichever is lower.
                         */
                        reqfssupri = fssparmsp->fss_upri;
                        if (reqfssupri > reqfssuprilim)
                                reqfssupri = reqfssuprilim;
                }
                fssproc->fss_uprilim = reqfssuprilim;
                fssproc->fss_upri = reqfssupri;
                fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
                if (fssproc->fss_nice > FSS_NICE_MAX)
                        fssproc->fss_nice = FSS_NICE_MAX;
        }

        fssproc->fss_timeleft = fss_quantum;
        fssproc->fss_tp = t;
        cpucaps_sc_init(&fssproc->fss_caps);

        /*
         * Put a lock on our fsspset structure.
         */
        mutex_enter(&fsspsets_lock);
        fsspset = fss_find_fsspset(t->t_cpupart);
        mutex_enter(&fsspset->fssps_lock);
        mutex_exit(&fsspsets_lock);

        zone = ttoproc(t)->p_zone;
        if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
                if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
                    == NULL) {
                        mutex_exit(&fsspset->fssps_lock);
                        return (ENOMEM);
                } else {
                        fsszone_allocated = 1;
                        fss_insert_fsszone(fsspset, zone, fsszone);
                }
        }
        kpj = ttoproj(t);
        if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
                if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
                    == NULL) {
                        if (fsszone_allocated) {
                                fss_remove_fsszone(fsspset, fsszone);
                                kmem_free(fsszone, sizeof (fsszone_t));
                        }
                        mutex_exit(&fsspset->fssps_lock);
                        return (ENOMEM);
                } else {
                        fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
                }
        }
        fssproj->fssp_threads++;
        fssproc->fss_proj = fssproj;

        /*
         * Reset priority. Process goes to a "user mode" priority here
         * regardless of whether or not it has slept since entering the kernel.
         */
        thread_lock(t);
        t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
        t->t_cid = cid;
        t->t_cldata = (void *)fssproc;
        t->t_schedflag |= TS_RUNQMATCH;
        fss_change_priority(t, fssproc);
        if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
            t->t_state == TS_WAIT)
                fss_active(t);
        thread_unlock(t);

        mutex_exit(&fsspset->fssps_lock);

        /*
         * Link new structure into fssproc list.
         */
        FSS_LIST_INSERT(fssproc);

        /*
         * If this is the first fair-sharing thread to occur since boot,
         * we set up the initial call to fss_update() here. Use an atomic
         * compare-and-swap since that's easier and faster than a mutex
         * (but check with an ordinary load first since most of the time
         * this will already be done).
         */
        if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
                (void) timeout(fss_update, NULL, hz);

        return (0);
}

/*
 * Remove fssproc_t from the list.
 */
static void
fss_exitclass(void *procp)
{
        fssproc_t *fssproc = (fssproc_t *)procp;
        fssproj_t *fssproj;
        fsspset_t *fsspset;
        fsszone_t *fsszone;
        kthread_t *t = fssproc->fss_tp;

        /*
         * We should be either getting this thread off the deathrow or
         * this thread has already moved to another scheduling class and
         * we're being called with its old cldata buffer pointer.  In both
         * cases, the content of this buffer can not be changed while we're
         * here.
         */
        mutex_enter(&fsspsets_lock);
        thread_lock(t);
        if (t->t_cid != fss_cid) {
                /*
                 * We're being called as a result of the priocntl() system
                 * call -- someone is trying to move our thread to another
                 * scheduling class. We can't call fss_inactive() here
                 * because our thread's t_cldata pointer already points
                 * to another scheduling class specific data.
                 */
                ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

                fssproj = FSSPROC2FSSPROJ(fssproc);
                fsspset = FSSPROJ2FSSPSET(fssproj);
                fsszone = fssproj->fssp_fsszone;

                if (fssproc->fss_runnable) {
                        disp_lock_enter_high(&fsspset->fssps_displock);
                        if (--fssproj->fssp_runnable == 0) {
                                fsszone->fssz_shares -= fssproj->fssp_shares;
                                if (--fsszone->fssz_runnable == 0)
                                        fsspset->fssps_shares -=
                                            fsszone->fssz_rshares;
                        }
                        disp_lock_exit_high(&fsspset->fssps_displock);
                }
                thread_unlock(t);

                mutex_enter(&fsspset->fssps_lock);
                if (--fssproj->fssp_threads == 0) {
                        fss_remove_fssproj(fsspset, fssproj);
                        if (fsszone->fssz_nproj == 0)
                                kmem_free(fsszone, sizeof (fsszone_t));
                        kmem_free(fssproj, sizeof (fssproj_t));
                }
                mutex_exit(&fsspset->fssps_lock);

        } else {
                ASSERT(t->t_state == TS_FREE);
                /*
                 * We're being called from thread_free() when our thread
                 * is removed from the deathrow. There is nothing we need
                 * do here since everything should've been done earlier
                 * in fss_exit().
                 */
                thread_unlock(t);
        }
        mutex_exit(&fsspsets_lock);

        FSS_LIST_DELETE(fssproc);
        fss_free(fssproc);
}

/*ARGSUSED*/
static int
fss_canexit(kthread_t *t, cred_t *credp)
{
        /*
         * A thread is allowed to exit FSS only if we have sufficient
         * privileges.
         */
        if (credp != NULL && secpolicy_setpriority(credp) != 0)
                return (EPERM);
        else
                return (0);
}

/*
 * Initialize fair-share class specific proc structure for a child.
 */
static int
fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
{
        fssproc_t *pfssproc;    /* ptr to parent's fssproc structure    */
        fssproc_t *cfssproc;    /* ptr to child's fssproc structure     */
        fssproj_t *fssproj;
        fsspset_t *fsspset;

        ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
        ASSERT(ct->t_state == TS_STOPPED);

        cfssproc = (fssproc_t *)bufp;
        ASSERT(cfssproc != NULL);
        bzero(cfssproc, sizeof (fssproc_t));

        thread_lock(pt);
        pfssproc = FSSPROC(pt);
        fssproj = FSSPROC2FSSPROJ(pfssproc);
        fsspset = FSSPROJ2FSSPSET(fssproj);
        thread_unlock(pt);

        mutex_enter(&fsspset->fssps_lock);
        /*
         * Initialize child's fssproc structure.
         */
        thread_lock(pt);
        ASSERT(FSSPROJ(pt) == fssproj);
        cfssproc->fss_proj = fssproj;
        cfssproc->fss_timeleft = fss_quantum;
        cfssproc->fss_umdpri = pfssproc->fss_umdpri;
        cfssproc->fss_fsspri = 0;
        cfssproc->fss_uprilim = pfssproc->fss_uprilim;
        cfssproc->fss_upri = pfssproc->fss_upri;
        cfssproc->fss_tp = ct;
        cfssproc->fss_nice = pfssproc->fss_nice;
        cpucaps_sc_init(&cfssproc->fss_caps);

        cfssproc->fss_flags =
            pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
        ct->t_cldata = (void *)cfssproc;
        ct->t_schedflag |= TS_RUNQMATCH;
        thread_unlock(pt);

        fssproj->fssp_threads++;
        mutex_exit(&fsspset->fssps_lock);

        /*
         * Link new structure into fssproc hash table.
         */
        FSS_LIST_INSERT(cfssproc);
        return (0);
}

/*
 * Child is placed at back of dispatcher queue and parent gives up processor
 * so that the child runs first after the fork. This allows the child
 * immediately execing to break the multiple use of copy on write pages with no
 * disk home. The parent will get to steal them back rather than uselessly
 * copying them.
 */
static void
fss_forkret(kthread_t *t, kthread_t *ct)
{
        proc_t *pp = ttoproc(t);
        proc_t *cp = ttoproc(ct);
        fssproc_t *fssproc;

        ASSERT(t == curthread);
        ASSERT(MUTEX_HELD(&pidlock));

        /*
         * Grab the child's p_lock before dropping pidlock to ensure the
         * process does not disappear before we set it running.
         */
        mutex_enter(&cp->p_lock);
        continuelwps(cp);
        mutex_exit(&cp->p_lock);

        mutex_enter(&pp->p_lock);
        mutex_exit(&pidlock);
        continuelwps(pp);

        thread_lock(t);

        fssproc = FSSPROC(t);
        fss_newpri(fssproc, false);
        fssproc->fss_timeleft = fss_quantum;
        t->t_pri = fssproc->fss_umdpri;
        ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
        fssproc->fss_flags &= ~FSSKPRI;
        THREAD_TRANSITION(t);

        /*
         * We don't want to call fss_setrun(t) here because it may call
         * fss_active, which we don't need.
         */
        fssproc->fss_flags &= ~FSSBACKQ;

        if (t->t_disp_time != ddi_get_lbolt())
                setbackdq(t);
        else
                setfrontdq(t);

        thread_unlock(t);
        /*
         * Safe to drop p_lock now since it is safe to change
         * the scheduling class after this point.
         */
        mutex_exit(&pp->p_lock);

        swtch();
}

/*
 * Get the fair-sharing parameters of the thread pointed to by fssprocp into
 * the buffer pointed by fssparmsp.
 */
static void
fss_parmsget(kthread_t *t, void *parmsp)
{
        fssproc_t *fssproc = FSSPROC(t);
        fssparms_t *fssparmsp = (fssparms_t *)parmsp;

        fssparmsp->fss_uprilim = fssproc->fss_uprilim;
        fssparmsp->fss_upri = fssproc->fss_upri;
}

/*ARGSUSED*/
static int
fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
{
        char            nice;
        pri_t           reqfssuprilim;
        pri_t           reqfssupri;
        fssproc_t       *fssproc = FSSPROC(t);
        fssparms_t      *fssparmsp = (fssparms_t *)parmsp;

        ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));

        if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
                reqfssuprilim = fssproc->fss_uprilim;
        else
                reqfssuprilim = fssparmsp->fss_uprilim;

        if (fssparmsp->fss_upri == FSS_NOCHANGE)
                reqfssupri = fssproc->fss_upri;
        else
                reqfssupri = fssparmsp->fss_upri;

        /*
         * Make sure the user priority doesn't exceed the upri limit.
         */
        if (reqfssupri > reqfssuprilim)
                reqfssupri = reqfssuprilim;

        /*
         * Basic permissions enforced by generic kernel code for all classes
         * require that a thread attempting to change the scheduling parameters
         * of a target thread be privileged or have a real or effective UID
         * matching that of the target thread. We are not called unless these
         * basic permission checks have already passed. The fair-sharing class
         * requires in addition that the calling thread be privileged if it
         * is attempting to raise the upri limit above its current value.
         * This may have been checked previously but if our caller passed us
         * a non-NULL credential pointer we assume it hasn't and we check it
         * here.
         */
        if ((reqpcredp != NULL) &&
            (reqfssuprilim > fssproc->fss_uprilim) &&
            secpolicy_raisepriority(reqpcredp) != 0)
                return (EPERM);

        /*
         * Set fss_nice to the nice value corresponding to the user priority we
         * are setting.  Note that setting the nice field of the parameter
         * struct won't affect upri or nice.
         */
        nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
        if (nice > FSS_NICE_MAX)
                nice = FSS_NICE_MAX;

        thread_lock(t);

        fssproc->fss_uprilim = reqfssuprilim;
        fssproc->fss_upri = reqfssupri;
        fssproc->fss_nice = nice;
        fss_newpri(fssproc, false);

        if ((fssproc->fss_flags & FSSKPRI) != 0) {
                thread_unlock(t);
                return (0);
        }

        fss_change_priority(t, fssproc);
        thread_unlock(t);
        return (0);

}

/*
 * The thread is being stopped.
 */
/*ARGSUSED*/
static void
fss_stop(kthread_t *t, int why, int what)
{
        ASSERT(THREAD_LOCK_HELD(t));
        ASSERT(t == curthread);

        fss_inactive(t);
}

/*
 * The current thread is exiting, do necessary adjustments to its project
 */
static void
fss_exit(kthread_t *t)
{
        fsspset_t *fsspset;
        fssproj_t *fssproj;
        fssproc_t *fssproc;
        fsszone_t *fsszone;
        int free = 0;

        /*
         * Thread t here is either a current thread (in which case we hold
         * its process' p_lock), or a thread being destroyed by forklwp_fail(),
         * in which case we hold pidlock and thread is no longer on the
         * thread list.
         */
        ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));

        fssproc = FSSPROC(t);
        fssproj = FSSPROC2FSSPROJ(fssproc);
        fsspset = FSSPROJ2FSSPSET(fssproj);
        fsszone = fssproj->fssp_fsszone;

        mutex_enter(&fsspsets_lock);
        mutex_enter(&fsspset->fssps_lock);

        thread_lock(t);
        disp_lock_enter_high(&fsspset->fssps_displock);
        if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
                if (--fssproj->fssp_runnable == 0) {
                        fsszone->fssz_shares -= fssproj->fssp_shares;
                        if (--fsszone->fssz_runnable == 0)
                                fsspset->fssps_shares -= fsszone->fssz_rshares;
                }
                ASSERT(fssproc->fss_runnable == 1);
                fssproc->fss_runnable = 0;
        }
        if (--fssproj->fssp_threads == 0) {
                fss_remove_fssproj(fsspset, fssproj);
                free = 1;
        }
        disp_lock_exit_high(&fsspset->fssps_displock);
        fssproc->fss_proj = NULL;       /* mark this thread as already exited */
        thread_unlock(t);

        if (free) {
                if (fsszone->fssz_nproj == 0)
                        kmem_free(fsszone, sizeof (fsszone_t));
                kmem_free(fssproj, sizeof (fssproj_t));
        }
        mutex_exit(&fsspset->fssps_lock);
        mutex_exit(&fsspsets_lock);

        /*
         * A thread could be exiting in between clock ticks, so we need to
         * calculate how much CPU time it used since it was charged last time.
         *
         * CPU caps are not enforced on exiting processes - it is usually
         * desirable to exit as soon as possible to free resources.
         */
        if (CPUCAPS_ON()) {
                thread_lock(t);
                fssproc = FSSPROC(t);
                (void) cpucaps_charge(t, &fssproc->fss_caps,
                    CPUCAPS_CHARGE_ONLY);
                thread_unlock(t);
        }
}

static void
fss_nullsys()
{
}

/*
 * If thread is currently at a kernel mode priority (has slept) and is
 * returning to the userland we assign it the appropriate user mode priority
 * and time quantum here.  If we're lowering the thread's priority below that
 * of other runnable threads then we will set runrun via cpu_surrender() to
 * cause preemption.
 */
static void
fss_trapret(kthread_t *t)
{
        fssproc_t *fssproc = FSSPROC(t);
        cpu_t *cp = CPU;

        ASSERT(THREAD_LOCK_HELD(t));
        ASSERT(t == curthread);
        ASSERT(cp->cpu_dispthread == t);
        ASSERT(t->t_state == TS_ONPROC);

        t->t_kpri_req = 0;
        if (fssproc->fss_flags & FSSKPRI) {
                /*
                 * If thread has blocked in the kernel
                 */
                THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
                cp->cpu_dispatch_pri = DISP_PRIO(t);
                ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
                fssproc->fss_flags &= ~FSSKPRI;

                if (DISP_MUST_SURRENDER(t))
                        cpu_surrender(t);
        }
}

/*
 * Arrange for thread to be placed in appropriate location on dispatcher queue.
 * This is called with the current thread in TS_ONPROC and locked.
 */
static void
fss_preempt(kthread_t *t)
{
        fssproc_t *fssproc = FSSPROC(t);
        klwp_t *lwp;
        uint_t flags;

        ASSERT(t == curthread);
        ASSERT(THREAD_LOCK_HELD(curthread));
        ASSERT(t->t_state == TS_ONPROC);

        /*
         * If preempted in the kernel, make sure the thread has a kernel
         * priority if needed.
         */
        lwp = curthread->t_lwp;
        if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
                fssproc->fss_flags |= FSSKPRI;
                THREAD_CHANGE_PRI(t, minclsyspri);
                ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
                t->t_trapret = 1;       /* so that fss_trapret will run */
                aston(t);
        }

        /*
         * This thread may be placed on wait queue by CPU Caps. In this case we
         * do not need to do anything until it is removed from the wait queue.
         * Do not enforce CPU caps on threads running at a kernel priority
         */
        if (CPUCAPS_ON()) {
                (void) cpucaps_charge(t, &fssproc->fss_caps,
                    CPUCAPS_CHARGE_ENFORCE);

                if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
                        return;
        }

        /*
         * Check to see if we're doing "preemption control" here.  If
         * we are, and if the user has requested that this thread not
         * be preempted, and if preemptions haven't been put off for
         * too long, let the preemption happen here but try to make
         * sure the thread is rescheduled as soon as possible.  We do
         * this by putting it on the front of the highest priority run
         * queue in the FSS class.  If the preemption has been put off
         * for too long, clear the "nopreempt" bit and let the thread
         * be preempted.
         */
        if (t->t_schedctl && schedctl_get_nopreempt(t)) {
                if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
                        DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
                        if (!(fssproc->fss_flags & FSSKPRI)) {
                                /*
                                 * If not already remembered, remember current
                                 * priority for restoration in fss_yield().
                                 */
                                if (!(fssproc->fss_flags & FSSRESTORE)) {
                                        fssproc->fss_scpri = t->t_pri;
                                        fssproc->fss_flags |= FSSRESTORE;
                                }
                                THREAD_CHANGE_PRI(t, fss_maxumdpri);
                        }
                        schedctl_set_yield(t, 1);
                        setfrontdq(t);
                        return;
                } else {
                        if (fssproc->fss_flags & FSSRESTORE) {
                                THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
                                fssproc->fss_flags &= ~FSSRESTORE;
                        }
                        schedctl_set_nopreempt(t, 0);
                        DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
                        /*
                         * Fall through and be preempted below.
                         */
                }
        }

        flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);

        if (flags == FSSBACKQ) {
                fssproc->fss_timeleft = fss_quantum;
                fssproc->fss_flags &= ~FSSBACKQ;
                setbackdq(t);
        } else if (flags == (FSSBACKQ | FSSKPRI)) {
                fssproc->fss_flags &= ~FSSBACKQ;
                setbackdq(t);
        } else {
                setfrontdq(t);
        }
}

/*
 * Called when a thread is waking up and is to be placed on the run queue.
 */
static void
fss_setrun(kthread_t *t)
{
        fssproc_t *fssproc = FSSPROC(t);

        ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */

        if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
                fss_active(t);

        fssproc->fss_timeleft = fss_quantum;

        fssproc->fss_flags &= ~FSSBACKQ;
        /*
         * If previously were running at the kernel priority then keep that
         * priority and the fss_timeleft doesn't matter.
         */
        if ((fssproc->fss_flags & FSSKPRI) == 0)
                THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);

        if (t->t_disp_time != ddi_get_lbolt())
                setbackdq(t);
        else
                setfrontdq(t);
}

/*
 * Prepare thread for sleep. We reset the thread priority so it will run at the
 * kernel priority level when it wakes up.
 */
static void
fss_sleep(kthread_t *t)
{
        fssproc_t *fssproc = FSSPROC(t);

        ASSERT(t == curthread);
        ASSERT(THREAD_LOCK_HELD(t));

        ASSERT(t->t_state == TS_ONPROC);

        /*
         * Account for time spent on CPU before going to sleep.
         */
        (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);

        fss_inactive(t);

        /*
         * Assign a system priority to the thread and arrange for it to be
         * retained when the thread is next placed on the run queue (i.e.,
         * when it wakes up) instead of being given a new pri.  Also arrange
         * for trapret processing as the thread leaves the system call so it
         * will drop back to normal priority range.
         */
        if (t->t_kpri_req) {
                THREAD_CHANGE_PRI(t, minclsyspri);
                fssproc->fss_flags |= FSSKPRI;
                t->t_trapret = 1;       /* so that fss_trapret will run */
                aston(t);
        } else if (fssproc->fss_flags & FSSKPRI) {
                /*
                 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
                 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
                 * then slept again all without finishing the current system
                 * call so trapret won't have cleared FSSKPRI
                 */
                fssproc->fss_flags &= ~FSSKPRI;
                THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
                if (DISP_MUST_SURRENDER(curthread))
                        cpu_surrender(t);
        }
}

/*
 * A tick interrupt has ocurrend on a running thread. Check to see if our
 * time slice has expired.
 */
static void
fss_tick(kthread_t *t)
{
        fssproc_t *fssproc;
        fssproj_t *fssproj;
        bool call_cpu_surrender = false;
        bool cpucaps_enforce = false;

        ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));

        /*
         * It's safe to access fsspset and fssproj structures because we're
         * holding our p_lock here.
         */
        thread_lock(t);
        fssproc = FSSPROC(t);
        fssproj = FSSPROC2FSSPROJ(fssproc);
        if (fssproj != NULL) {
                fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
                disp_lock_enter_high(&fsspset->fssps_displock);
                fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
                fssproj->fssp_tick_cnt++;
                fssproc->fss_ticks++;
                disp_lock_exit_high(&fsspset->fssps_displock);
        }

        /*
         * Keep track of thread's project CPU usage.  Note that projects
         * get charged even when threads are running in the kernel.
         * Do not surrender CPU if running in the SYS class.
         */
        if (CPUCAPS_ON()) {
                cpucaps_enforce = cpucaps_charge(t,
                    &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
                    !(fssproc->fss_flags & FSSKPRI);
        }

        /*
         * A thread's execution time for threads running in the SYS class
         * is not tracked.
         */
        if ((fssproc->fss_flags & FSSKPRI) == 0) {
                /*
                 * If thread is not in kernel mode, decrement its fss_timeleft
                 */
                if (--fssproc->fss_timeleft <= 0) {
                        pri_t new_pri;

                        /*
                         * If we're doing preemption control and trying to
                         * avoid preempting this thread, just note that the
                         * thread should yield soon and let it keep running
                         * (unless it's been a while).
                         */
                        if (t->t_schedctl && schedctl_get_nopreempt(t)) {
                                if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
                                        DTRACE_SCHED1(schedctl__nopreempt,
                                            kthread_t *, t);
                                        schedctl_set_yield(t, 1);
                                        thread_unlock_nopreempt(t);
                                        return;
                                }
                        }
                        fssproc->fss_flags &= ~FSSRESTORE;

                        fss_newpri(fssproc, true);
                        new_pri = fssproc->fss_umdpri;
                        ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);

                        /*
                         * When the priority of a thread is changed, it may
                         * be necessary to adjust its position on a sleep queue
                         * or dispatch queue. The function thread_change_pri
                         * accomplishes this.
                         */
                        if (thread_change_pri(t, new_pri, 0)) {
                                fssproc->fss_timeleft = fss_quantum;
                        } else {
                                call_cpu_surrender = true;
                        }
                } else if (t->t_state == TS_ONPROC &&
                    t->t_pri < t->t_disp_queue->disp_maxrunpri) {
                        /*
                         * If there is a higher-priority thread which is
                         * waiting for a processor, then thread surrenders
                         * the processor.
                         */
                        call_cpu_surrender = true;
                }
        }

        if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
                /*
                 * The thread used more than half of its quantum, so assume that
                 * it used the whole quantum.
                 *
                 * Update thread's priority just before putting it on the wait
                 * queue so that it gets charged for the CPU time from its
                 * quantum even before that quantum expires.
                 */
                fss_newpri(fssproc, false);
                if (t->t_pri != fssproc->fss_umdpri)
                        fss_change_priority(t, fssproc);

                /*
                 * We need to call cpu_surrender for this thread due to cpucaps
                 * enforcement, but fss_change_priority may have already done
                 * so. In this case FSSBACKQ is set and there is no need to call
                 * cpu-surrender again.
                 */
                if (!(fssproc->fss_flags & FSSBACKQ))
                        call_cpu_surrender = true;
        }

        if (call_cpu_surrender) {
                fssproc->fss_flags |= FSSBACKQ;
                cpu_surrender(t);
        }

        thread_unlock_nopreempt(t);     /* clock thread can't be preempted */
}

/*
 * Processes waking up go to the back of their queue.  We don't need to assign
 * a time quantum here because thread is still at a kernel mode priority and
 * the time slicing is not done for threads running in the kernel after
 * sleeping.  The proper time quantum will be assigned by fss_trapret before the
 * thread returns to user mode.
 */
static void
fss_wakeup(kthread_t *t)
{
        fssproc_t *fssproc;

        ASSERT(THREAD_LOCK_HELD(t));
        ASSERT(t->t_state == TS_SLEEP);

        fss_active(t);

        fssproc = FSSPROC(t);
        fssproc->fss_flags &= ~FSSBACKQ;

        if (fssproc->fss_flags & FSSKPRI) {
                /*
                 * If we already have a kernel priority assigned, then we
                 * just use it.
                 */
                setbackdq(t);
        } else if (t->t_kpri_req) {
                /*
                 * Give thread a priority boost if we were asked.
                 */
                fssproc->fss_flags |= FSSKPRI;
                THREAD_CHANGE_PRI(t, minclsyspri);
                setbackdq(t);
                t->t_trapret = 1;       /* so that fss_trapret will run */
                aston(t);
        } else {
                /*
                 * Otherwise, we recalculate the priority.
                 */
                if (t->t_disp_time == ddi_get_lbolt()) {
                        setfrontdq(t);
                } else {
                        fssproc->fss_timeleft = fss_quantum;
                        THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
                        setbackdq(t);
                }
        }
}

/*
 * fss_donice() is called when a nice(1) command is issued on the thread to
 * alter the priority. The nice(1) command exists in Solaris for compatibility.
 * Thread priority adjustments should be done via priocntl(1).
 */
static int
fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
{
        int newnice;
        fssproc_t *fssproc = FSSPROC(t);
        fssparms_t fssparms;

        /*
         * If there is no change to priority, just return current setting.
         */
        if (incr == 0) {
                if (retvalp)
                        *retvalp = fssproc->fss_nice - NZERO;
                return (0);
        }

        if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
                return (EPERM);

        /*
         * Specifying a nice increment greater than the upper limit of
         * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
         * value being set to the upper limit.  We check for this before
         * computing the new value because otherwise we could get overflow
         * if a privileged user specified some ridiculous increment.
         */
        if (incr > FSS_NICE_MAX)
                incr = FSS_NICE_MAX;

        newnice = fssproc->fss_nice + incr;
        if (newnice > FSS_NICE_MAX)
                newnice = FSS_NICE_MAX;
        else if (newnice < FSS_NICE_MIN)
                newnice = FSS_NICE_MIN;

        fssparms.fss_uprilim = fssparms.fss_upri =
            -((newnice - NZERO) * fss_maxupri) / NZERO;

        /*
         * Reset the uprilim and upri values of the thread.
         */
        (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);

        /*
         * Although fss_parmsset already reset fss_nice it may not have been
         * set to precisely the value calculated above because fss_parmsset
         * determines the nice value from the user priority and we may have
         * truncated during the integer conversion from nice value to user
         * priority and back. We reset fss_nice to the value we calculated
         * above.
         */
        fssproc->fss_nice = (char)newnice;

        if (retvalp)
                *retvalp = newnice - NZERO;
        return (0);
}

/*
 * Increment the priority of the specified thread by incr and
 * return the new value in *retvalp.
 */
static int
fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
{
        int newpri;
        fssproc_t *fssproc = FSSPROC(t);
        fssparms_t fssparms;

        /*
         * If there is no change to priority, just return current setting.
         */
        if (incr == 0) {
                *retvalp = fssproc->fss_upri;
                return (0);
        }

        newpri = fssproc->fss_upri + incr;
        if (newpri > fss_maxupri || newpri < -fss_maxupri)
                return (EINVAL);

        *retvalp = newpri;
        fssparms.fss_uprilim = fssparms.fss_upri = newpri;

        /*
         * Reset the uprilim and upri values of the thread.
         */
        return (fss_parmsset(t, &fssparms, (id_t)0, cr));
}

/*
 * Return the global scheduling priority that would be assigned to a thread
 * entering the fair-sharing class with the fss_upri.
 */
/*ARGSUSED*/
static pri_t
fss_globpri(kthread_t *t)
{
        ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

        return (fss_maxumdpri / 2);
}

/*
 * Called from the yield(2) system call when a thread is yielding (surrendering)
 * the processor. The kernel thread is placed at the back of a dispatch queue.
 */
static void
fss_yield(kthread_t *t)
{
        fssproc_t *fssproc = FSSPROC(t);

        ASSERT(t == curthread);
        ASSERT(THREAD_LOCK_HELD(t));

        /*
         * Collect CPU usage spent before yielding
         */
        (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);

        /*
         * Clear the preemption control "yield" bit since the user is
         * doing a yield.
         */
        if (t->t_schedctl)
                schedctl_set_yield(t, 0);
        /*
         * If fss_preempt() artifically increased the thread's priority
         * to avoid preemption, restore the original priority now.
         */
        if (fssproc->fss_flags & FSSRESTORE) {
                THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
                fssproc->fss_flags &= ~FSSRESTORE;
        }
        if (fssproc->fss_timeleft < 0) {
                /*
                 * Time slice was artificially extended to avoid preemption,
                 * so pretend we're preempting it now.
                 */
                DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
                fssproc->fss_timeleft = fss_quantum;
        }
        fssproc->fss_flags &= ~FSSBACKQ;
        setbackdq(t);
}

void
fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
    fssbuf_t *zonebuf)
{
        kproject_t *kpj_new = kp;
        zone_t *zone = zp;
        fssproj_t *fssproj_old, *fssproj_new;
        fsspset_t *fsspset;
        kproject_t *kpj_old;
        fssproc_t *fssproc;
        fsszone_t *fsszone_old, *fsszone_new;
        int free = 0;
        int id;

        ASSERT(MUTEX_HELD(&cpu_lock));
        ASSERT(MUTEX_HELD(&pidlock));
        ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

        if (t->t_cid != fss_cid)
                return;

        fssproc = FSSPROC(t);
        mutex_enter(&fsspsets_lock);
        fssproj_old = FSSPROC2FSSPROJ(fssproc);
        if (fssproj_old == NULL) {
                mutex_exit(&fsspsets_lock);
                return;
        }

        fsspset = FSSPROJ2FSSPSET(fssproj_old);
        mutex_enter(&fsspset->fssps_lock);
        kpj_old = FSSPROJ2KPROJ(fssproj_old);
        fsszone_old = fssproj_old->fssp_fsszone;

        ASSERT(t->t_cpupart == fsspset->fssps_cpupart);

        if (kpj_old == kpj_new) {
                mutex_exit(&fsspset->fssps_lock);
                mutex_exit(&fsspsets_lock);
                return;
        }

        if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
                /*
                 * If the zone for the new project is not currently active on
                 * the cpu partition we're on, get one of the pre-allocated
                 * buffers and link it in our per-pset zone list.  Such buffers
                 * should already exist.
                 */
                for (id = 0; id < zonebuf->fssb_size; id++) {
                        if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
                                fss_insert_fsszone(fsspset, zone, fsszone_new);
                                zonebuf->fssb_list[id] = NULL;
                                break;
                        }
                }
        }
        ASSERT(fsszone_new != NULL);
        if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
                /*
                 * If our new project is not currently running
                 * on the cpu partition we're on, get one of the
                 * pre-allocated buffers and link it in our new cpu
                 * partition doubly linked list. Such buffers should already
                 * exist.
                 */
                for (id = 0; id < projbuf->fssb_size; id++) {
                        if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
                                fss_insert_fssproj(fsspset, kpj_new,
                                    fsszone_new, fssproj_new);
                                projbuf->fssb_list[id] = NULL;
                                break;
                        }
                }
        }
        ASSERT(fssproj_new != NULL);

        thread_lock(t);
        if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
            t->t_state == TS_WAIT)
                fss_inactive(t);
        ASSERT(fssproj_old->fssp_threads > 0);
        if (--fssproj_old->fssp_threads == 0) {
                fss_remove_fssproj(fsspset, fssproj_old);
                free = 1;
        }
        fssproc->fss_proj = fssproj_new;
        fssproc->fss_fsspri = 0;
        fssproj_new->fssp_threads++;
        if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
            t->t_state == TS_WAIT)
                fss_active(t);
        thread_unlock(t);
        if (free) {
                if (fsszone_old->fssz_nproj == 0)
                        kmem_free(fsszone_old, sizeof (fsszone_t));
                kmem_free(fssproj_old, sizeof (fssproj_t));
        }

        mutex_exit(&fsspset->fssps_lock);
        mutex_exit(&fsspsets_lock);
}

void
fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
    fssbuf_t *zonebuf)
{
        fsspset_t *fsspset_old, *fsspset_new;
        fssproj_t *fssproj_old, *fssproj_new;
        fsszone_t *fsszone_old, *fsszone_new;
        fssproc_t *fssproc;
        kproject_t *kpj;
        zone_t *zone;
        int id;

        ASSERT(MUTEX_HELD(&cpu_lock));
        ASSERT(MUTEX_HELD(&pidlock));
        ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

        if (t->t_cid != fss_cid)
                return;

        fssproc = FSSPROC(t);
        zone = ttoproc(t)->p_zone;
        mutex_enter(&fsspsets_lock);
        fssproj_old = FSSPROC2FSSPROJ(fssproc);
        if (fssproj_old == NULL) {
                mutex_exit(&fsspsets_lock);
                return;
        }
        fsszone_old = fssproj_old->fssp_fsszone;
        fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
        kpj = FSSPROJ2KPROJ(fssproj_old);

        if (fsspset_old->fssps_cpupart == newcp) {
                mutex_exit(&fsspsets_lock);
                return;
        }

        ASSERT(ttoproj(t) == kpj);

        fsspset_new = fss_find_fsspset(newcp);

        mutex_enter(&fsspset_new->fssps_lock);
        if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
                for (id = 0; id < zonebuf->fssb_size; id++) {
                        if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
                                fss_insert_fsszone(fsspset_new, zone,
                                    fsszone_new);
                                zonebuf->fssb_list[id] = NULL;
                                break;
                        }
                }
        }
        ASSERT(fsszone_new != NULL);
        if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
                for (id = 0; id < projbuf->fssb_size; id++) {
                        if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
                                fss_insert_fssproj(fsspset_new, kpj,
                                    fsszone_new, fssproj_new);
                                projbuf->fssb_list[id] = NULL;
                                break;
                        }
                }
        }
        ASSERT(fssproj_new != NULL);

        fssproj_new->fssp_threads++;
        thread_lock(t);
        if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
            t->t_state == TS_WAIT)
                fss_inactive(t);
        fssproc->fss_proj = fssproj_new;
        fssproc->fss_fsspri = 0;
        if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
            t->t_state == TS_WAIT)
                fss_active(t);
        thread_unlock(t);
        mutex_exit(&fsspset_new->fssps_lock);

        mutex_enter(&fsspset_old->fssps_lock);
        if (--fssproj_old->fssp_threads == 0) {
                fss_remove_fssproj(fsspset_old, fssproj_old);
                if (fsszone_old->fssz_nproj == 0)
                        kmem_free(fsszone_old, sizeof (fsszone_t));
                kmem_free(fssproj_old, sizeof (fssproj_t));
        }
        mutex_exit(&fsspset_old->fssps_lock);

        mutex_exit(&fsspsets_lock);
}