uts: make emu10k non-verbose

[unleashed.git] / kernel / disp / cpucaps.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/disp.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/atomic.h>
#include <sys/cpucaps_impl.h>
#include <sys/dtrace.h>
#include <sys/sdt.h>
#include <sys/debug.h>
#include <sys/rctl.h>
#include <sys/errno.h>

/*
 * CPU Caps implementation
 * =======================
 *
 * A CPU cap can be set on any project or any zone. Zone CPU cap limits the CPU
 * usage for all projects running inside the zone. If the zone CPU cap is set
 * below the project CPU cap, the latter will have no effect.
 *
 * When CPU usage of projects and/or zones reaches specified caps, threads in
 * them do not get scheduled and instead are placed on wait queues associated
 * with a cap. Such threads will start running again only when CPU usage drops
 * below the cap level. Each zone and each project has its own wait queue.
 *
 * When CPU cap is set, the kernel continously keeps track of CPU time used by
 * capped zones and/or projects over a short time interval and calculates their
 * current CPU usage as a percentage. When the accumulated usage reaches the CPU
 * cap, LWPs running in the user-land (when they are not holding any critical
 * kernel locks) are placed on special wait queues until their project's or
 * zone's CPU usage drops below the cap.
 *
 * The system maintains a list of all capped projects and all capped zones. On
 * every clock tick every active thread belonging to a capped project adds its
 * CPU usage to its project. Usage from all projects belonging to a capped zone
 * is aggregated to get the zone usage.
 *
 * When the current CPU usage is above the cap, a project or zone is considered
 * over-capped. Every user thread caught running in an over-capped project or
 * zone is marked by setting TS_PROJWAITQ flag in thread's t_schedflag field and
 * is requested to surrender its CPU. This causes scheduling class specific
 * CL_PREEMPT() callback to be invoked. The callback function places threads
 * marked as TS_PROJWAIT on a wait queue and calls switch().
 *
 * Threads are only placed on wait queues after trapping from user-land
 * (they could be holding some user locks, but no kernel locks) and while
 * returning from the trap back to the user-land when no kernel locks are held.
 * Putting threads on wait queues in random places while running in the
 * kernel might lead to all kinds of locking problems.
 *
 * Accounting
 * ==========
 *
 * Accounting of CPU usage is based on per-thread micro-state accounting data.
 * On every clock tick clock() adds new on-CPU time for every thread found on
 * CPU. Scheduling classes also add new on-CPU time for any thread leaving CPU.
 * New times means time since it was last accounted for. On-CPU times greater
 * than 1 tick are truncated to 1 tick.
 *
 * Project CPU usage is aggregated from all threads within the project.
 * Zone CPU usage is the sum of usages for all projects within the zone. Zone
 * CPU usage is calculated on every clock tick by walking list of projects and
 * adding their usage together.
 *
 * Decay
 * =====
 *
 * CPU usage is decayed by the caps_update() routine which is called once per
 * every clock tick. It walks lists of project caps and decays their usages by
 * one per cent. If CPU usage drops below cap levels, threads on the wait queue
 * are made runnable again, one thread per clock tick.
 *
 * Interfaces
 * ==========
 *
 * The CPU Caps facility provides the following interfaces to the rest of the
 * system:
 *
 *   cpucaps_project_add(kproject_t *)
 *
 * Notifies the framework of a new project. It should be put on the
 * capped_projects list if its zone has a cap.
 *
 *   cpucaps_project_remove(kproject_t *)
 *
 * Remove the association between the specified project and its cap.
 * Called right before the project is destroyed.
 *
 * cpucaps_project_set(kproject_t *, rctl_qty_t)
 *
 * Set project cap of the specified project to the specified value. Setting the
 * value to NOCAP is equivalent to removing the cap.
 *
 *   cpucaps_zone_set(zone_t *, rctl_qty_t)
 *
 * Set zone cap of the specified zone to the specified value. Setting the value
 * to NOCAP is equivalent to removing the cap.
 *
 *   cpucaps_zone_remove(zone_t *)
 *
 * Remove the association between the zone and its cap.
 *
 *   cpucaps_charge(kthread_id_t, caps_sc_t *, cpucaps_charge_t)
 *
 * Charges specified thread's project the amount of on-CPU time that it used.
 * If the third argument is CPUCAPS_CHARGE_ONLY returns False.
 * Otherwise returns True if project or zone should be penalized because its
 * project or zone is exceeding its cap. Also sets TS_PROJWAITQ or TS_ZONEWAITQ
 * bits in t_schedflag in this case.
 *
 *   CPUCAPS_ENFORCE(kthread_id_t *)
 *
 * Enforces CPU caps for a specified thread. Places LWPs running in LWP_USER
 * state on project or zone wait queues, as requested by TS_PROJWAITQ or
 * TS_ZONEWAITQ bits in t_schedflag. Returns True if the thread was placed on a
 * wait queue or False otherwise.
 *
 *   cpucaps_sc_init(caps_sc_t *)
 *
 * Initializes the scheduling-class specific CPU Caps data for a thread.
 *
 * LOCKS
 * =====
 *
 * all the individual caps structures and their lists are protected by a global
 * caps_lock mutex. The lock is grabbed either by clock() or by events modifying
 * caps, so it is usually uncontended. We avoid all blocking memory allocations
 * while holding caps_lock to prevent clock() from blocking.
 *
 * Thread state is protected by the thread lock. It protects the association
 * between a thread and its project and, as a consequence, to its zone. The
 * association can not break while thread lock is held, so the project or zone
 * cap are not going to disappear while thread lock is held.
 *
 * Cap usage field is protected by high-pil spin-lock cap_usagelock. It is
 * grabbed by scheduling classes already holding thread lock at high PIL and by
 * clock thread performing usage decay. We should do as little work as possible
 * while holding the lock since it may be very hot. All threads in the project
 * contend for the same cache line doing cap usage updates.
 */

/*
 * caps_lock protects list of capped projects and zones, changes in the cap
 * state and changes of the global cpucaps_enabled flag.
 *
 * Changing zone caps also sets cpucaps_busy to avoid races when a zone cap is
 * modified in parallel. This can be per-zone cap flag, but we don't keep any
 * cap state for now.
 */
static kmutex_t caps_lock;              /* lock to protect: */
static list_t capped_zones;             /* - list of zones with caps */
static list_t capped_projects;          /* - list of projects with caps */
boolean_t cpucaps_enabled;              /* - are there any caps defined? */
boolean_t cpucaps_busy;                 /* - is framework busy? */

/*
 * The accounting is based on the number of nanoseconds threads spend running
 * during a tick which is kept in the cap_tick_cost variable.
 */
static hrtime_t cap_tick_cost;

/*
 * How much of the usage value is decayed every clock tick
 * Decay one per cent of value per tick
 */
#define CAP_DECAY_FACTOR 100

/*
 * Scale the value and round it to the closest integer value
 */
#define ROUND_SCALE(x, y) (((x) + (y) / 2) / (y))

static void caps_update();

/*
 * CAP kstats.
 */
struct cap_kstat {
        kstat_named_t   cap_value;
        kstat_named_t   cap_usage;
        kstat_named_t   cap_nwait;
        kstat_named_t   cap_below;
        kstat_named_t   cap_above;
        kstat_named_t   cap_maxusage;
        kstat_named_t   cap_zonename;
} cap_kstat = {
        { "value",      KSTAT_DATA_UINT64 },
        { "usage",      KSTAT_DATA_UINT64 },
        { "nwait",      KSTAT_DATA_UINT64 },
        { "below_sec",  KSTAT_DATA_UINT64 },
        { "above_sec",  KSTAT_DATA_UINT64 },
        { "maxusage",   KSTAT_DATA_UINT64 },
        { "zonename",   KSTAT_DATA_STRING },
};


static kmutex_t cap_kstat_lock;
static int cap_kstat_update(kstat_t *, int);

/*
 * Initialize CPU caps infrastructure.
 *   - Initialize lists of capped zones and capped projects
 *   - Set cpucaps_clock_callout to NULL
 */
void
cpucaps_init()
{
        /*
         * Initialize global variables
         */
        cap_tick_cost = TICK_TO_NSEC((hrtime_t)1);

        list_create(&capped_zones, sizeof (cpucap_t),
            offsetof(cpucap_t, cap_link));
        list_create(&capped_projects, sizeof (cpucap_t),
            offsetof(cpucap_t, cap_link));

        cpucaps_enabled = B_FALSE;
        cpucaps_busy = B_FALSE;
        cpucaps_clock_callout = NULL;
}

/*
 * Initialize scheduling-class specific CPU Caps data.
 */
void
cpucaps_sc_init(caps_sc_t *csc)
{
        csc->csc_cputime = 0;
}

/*
 * Allocate and initialize cpucap structure
 */
static cpucap_t *
cap_alloc(void)
{
        cpucap_t *cap = kmem_zalloc(sizeof (cpucap_t), KM_SLEEP);

        DISP_LOCK_INIT(&cap->cap_usagelock);
        waitq_init(&cap->cap_waitq);

        return (cap);
}

/*
 * Free cpucap structure
 */
static void
cap_free(cpucap_t *cap)
{
        if (cap == NULL)
                return;

        /*
         * This cap should not be active
         */
        ASSERT(!list_link_active(&cap->cap_link));
        ASSERT(cap->cap_value == 0);
        ASSERT(!DISP_LOCK_HELD(&cap->cap_usagelock));

        waitq_fini(&cap->cap_waitq);
        DISP_LOCK_DESTROY(&cap->cap_usagelock);

        kmem_free(cap, sizeof (cpucap_t));
}

/*
 * Activate cap - insert into active list and unblock its
 * wait queue. Should be called with caps_lock held.
 * The cap_value field is set to the value supplied.
 */
static void
cap_enable(list_t *l, cpucap_t *cap, hrtime_t value)
{
        ASSERT(MUTEX_HELD(&caps_lock));

        /*
         * Cap can not be already enabled
         */
        ASSERT(!CAP_ENABLED(cap));
        ASSERT(!list_link_active(&cap->cap_link));

        list_insert_tail(l, cap);
        cap->cap_below = cap->cap_above = 0;
        cap->cap_maxusage = 0;
        cap->cap_usage = 0;
        cap->cap_value = value;
        waitq_unblock(&cap->cap_waitq);
        if (CPUCAPS_OFF()) {
                cpucaps_enabled = B_TRUE;
                cpucaps_clock_callout = caps_update;
        }
}

/*
 * Deactivate cap
 *   - Block its wait queue. This prevents any new threads from being
 *      enqueued there and moves all enqueued threads to the run queue.
 *   - Remove cap from list l.
 *   - Disable CPU caps globally if there are no capped projects or zones
 *
 * Should be called with caps_lock held.
 */
static void
cap_disable(list_t *l, cpucap_t *cap)
{
        ASSERT(MUTEX_HELD(&caps_lock));
        /*
         * Cap should be currently active
         */
        ASSERT(CPUCAPS_ON());
        ASSERT(list_link_active(&cap->cap_link));
        ASSERT(CAP_ENABLED(cap));

        waitq_block(&cap->cap_waitq);
        list_remove(l, cap);
        if (list_is_empty(&capped_projects) && list_is_empty(&capped_zones)) {
                cpucaps_enabled = B_FALSE;
                cpucaps_clock_callout = NULL;
        }
        cap->cap_value = 0;
        cap->cap_project = NULL;
        cap->cap_zone = NULL;
        if (cap->cap_kstat != NULL) {
                kstat_delete(cap->cap_kstat);
                cap->cap_kstat = NULL;
        }

}

/*
 * Enable cap for a project kpj
 * It is safe to enable already enabled project cap.
 * Should be called with caps_lock held.
 */
static void
cap_project_enable(kproject_t *kpj, hrtime_t value)
{
        cpucap_t *cap = kpj->kpj_cpucap;

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(cap != NULL);

        if (CAP_DISABLED(cap)) {
                ASSERT(cap->cap_kstat == NULL);
                cap_enable(&capped_projects, cap, value);
                cap->cap_project = kpj;
                cap->cap_zone = kpj->kpj_zone;

                /*
                 * Create cap kstats
                 */
                if ((cap->cap_kstat = rctl_kstat_create_project(kpj, "cpucaps",
                    KSTAT_TYPE_NAMED,
                    sizeof (cap_kstat) / sizeof (kstat_named_t),
                    KSTAT_FLAG_VIRTUAL)) != NULL) {
                        cap->cap_kstat->ks_data_size +=
                            strlen(cap->cap_zone->zone_name) + 1;
                        cap->cap_kstat->ks_lock = &cap_kstat_lock;
                        cap->cap_kstat->ks_data = &cap_kstat;
                        cap->cap_kstat->ks_update = cap_kstat_update;
                        cap->cap_kstat->ks_private = cap;
                        kstat_install(cap->cap_kstat);
                }
        }
}

/*
 * Disable project cap.
 * It is safe to disable already disabled project cap.
 * Should be called with caps_lock held.
 */
static void
cap_project_disable(kproject_t *kpj)
{
        cpucap_t *cap = kpj->kpj_cpucap;

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(cap != NULL);
        ASSERT(cap->cap_project == kpj);

        if (CAP_ENABLED(cap))
                cap_disable(&capped_projects, cap);
}

/*
 * Enable cap for a zone
 * It is safe to enable already enabled zone cap.
 * Should be called with caps_lock held.
 */
static void
cap_zone_enable(zone_t *zone, hrtime_t value)
{
        cpucap_t *cap = zone->zone_cpucap;

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(cap != NULL);

        if (CAP_DISABLED(cap)) {
                ASSERT(cap->cap_kstat == NULL);
                cap_enable(&capped_zones, cap, value);
                cap->cap_zone = zone;

                /*
                 * Create cap kstats
                 */
                if ((cap->cap_kstat = rctl_kstat_create_zone(zone, "cpucaps",
                    KSTAT_TYPE_NAMED,
                    sizeof (cap_kstat) / sizeof (kstat_named_t),
                    KSTAT_FLAG_VIRTUAL)) != NULL) {
                        cap->cap_kstat->ks_data_size +=
                            strlen(cap->cap_zone->zone_name) + 1;
                        cap->cap_kstat->ks_lock = &cap_kstat_lock;
                        cap->cap_kstat->ks_data = &cap_kstat;
                        cap->cap_kstat->ks_update = cap_kstat_update;
                        cap->cap_kstat->ks_private = cap;
                        kstat_install(cap->cap_kstat);
                }
        }
}

/*
 * Disable zone cap.
 * It is safe to disable already disabled zone cap.
 * Should be called with caps_lock held.
 */
static void
cap_zone_disable(zone_t *zone)
{
        cpucap_t *cap = zone->zone_cpucap;

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(cap != NULL);
        ASSERT(cap->cap_zone == zone);

        if (CAP_ENABLED(cap))
                cap_disable(&capped_zones, cap);
}

/*
 * Apply specified callback to all caps contained in the list `l'.
 */
static void
cap_walk(list_t *l, void (*cb)(cpucap_t *, int64_t))
{
        static uint64_t cpucap_walk_gen;
        cpucap_t *cap;

        ASSERT(MUTEX_HELD(&caps_lock));

        for (cap = list_head(l); cap != NULL; cap = list_next(l, cap)) {
                (*cb)(cap, cpucap_walk_gen);
        }

        atomic_inc_64(&cpucap_walk_gen);
}

/*
 * If cap limit is not reached, make one thread from wait queue runnable.
 * The waitq_isempty check is performed without the waitq lock. If a new thread
 * is placed on the waitq right after the check, it will be picked up during the
 * next invocation of cap_poke_waitq().
 */
/* ARGSUSED */
static void
cap_poke_waitq(cpucap_t *cap, int64_t gen)
{
        ASSERT(MUTEX_HELD(&caps_lock));

        if (cap->cap_usage >= cap->cap_value) {
                cap->cap_above++;
        } else {
                waitq_t *wq = &cap->cap_waitq;

                cap->cap_below++;

                if (!waitq_isempty(wq))
                        waitq_runone(wq);
        }
}

/*
 * The callback function called for every cap on capped_projects list.
 * Decay cap usage by CAP_DECAY_FACTOR
 * Add this cap project usage to its zone usage.
 * Kick off a thread from the cap waitq if cap is not reached.
 */
static void
cap_project_usage_walker(cpucap_t *cap, int64_t gen)
{
        zone_t          *zone = cap->cap_zone;
        hrtime_t        cap_usage = cap->cap_usage;

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(cap->cap_project->kpj_cpucap == cap);
        ASSERT(zone == cap->cap_project->kpj_zone);
        ASSERT(CAP_ENABLED(cap));

        /*
         * Set or clear the CAP_REACHED flag based on the current usage.
         * Only projects having their own caps are ever marked as CAP_REACHED.
         */
        cap_poke_waitq(cap, 0);

        /*
         * Add project's CPU usage to our zone's CPU usage.
         */
        if (ZONE_IS_CAPPED(zone)) {
                cpucap_t *zcap = zone->zone_cpucap;

                ASSERT(zcap->cap_zone == zone);

                /*
                 * If we haven't reset this zone's usage during this clock tick
                 * yet, then do it now. The cap_gen field is used to check
                 * whether this is the first zone's project we see during this
                 * tick or a subsequent one.
                 */
                if (zcap->cap_gen != gen) {
                        if (zcap->cap_usage > zcap->cap_maxusage)
                                zcap->cap_maxusage = zcap->cap_usage;
                        zcap->cap_usage = 0;
                        zcap->cap_gen = gen;
                }
                DTRACE_PROBE2(cpucaps__zusage, cpucap_t *, zcap,
                    hrtime_t, cap_usage);
                zcap->cap_usage += cap_usage;
                /* Check for overflows */
                if (zcap->cap_usage < 0)
                        zcap->cap_usage = MAX_USAGE - 1;
        }

        /*
         * Decay project usage.
         */
        disp_lock_enter(&cap->cap_usagelock);
        cap->cap_usage -= ROUND_SCALE(cap_usage, CAP_DECAY_FACTOR);
        disp_lock_exit(&cap->cap_usagelock);
}

/*
 * On every clock tick walk the list of project caps and update the CPU usage.
 * Also walk the list of zone caps checking whether any threads should
 * transition from wait queue to run queue.
 *
 * This function gets called by the clock thread directly when there are any
 * defined caps. The only lock that it grabs is caps_lock. Nothing else grabs
 * caps_lock for long periods of time, so there should be almost no contention
 * for it.
 */
static void
caps_update()
{
        mutex_enter(&caps_lock);
        cap_walk(&capped_projects, cap_project_usage_walker);
        cap_walk(&capped_zones, cap_poke_waitq);
        mutex_exit(&caps_lock);
}

/*
 * The function is called for each project in a zone when the zone cap is
 * modified. It enables project caps if zone cap is enabled and disables if the
 * zone cap is disabled and project doesn't have its own cap.
 *
 * For each project that does not have cpucap structure allocated it allocates a
 * new structure and assigns to kpj->cpu_cap. The allocation is performed
 * without holding caps_lock to avoid using KM_SLEEP allocation with caps_lock
 * held.
 */
static int
cap_project_zone_modify_walker(kproject_t *kpj, void *arg)
{
        cpucap_t *project_cap = NULL;
        cpucap_t *zone_cap = (cpucap_t *)arg;

        ASSERT(zone_cap != NULL);

        if (kpj->kpj_cpucap == NULL) {
                /*
                 * This is the first time any cap was established for this
                 * project. Allocate a new cpucap structure for it.
                 */
                project_cap = cap_alloc();
        }

        mutex_enter(&caps_lock);

        /*
         * Double-check that kpj_cpucap is still NULL - now with caps_lock held
         * and assign the newly allocated cpucap structure to it.
         */
        if (kpj->kpj_cpucap == NULL) {
                kpj->kpj_cpucap = project_cap;
        } else if (project_cap != NULL) {
                cap_free(project_cap);
        }

        project_cap = kpj->kpj_cpucap;

        if (CAP_DISABLED(zone_cap)) {
                /*
                 * Remove all projects in this zone without caps
                 * from the capped_projects list.
                 */
                if (project_cap->cap_value == MAX_USAGE) {
                        cap_project_disable(kpj);
                }
        } else if (CAP_DISABLED(project_cap)) {
                /*
                 * Add the project to capped_projects list.
                 */
                ASSERT(project_cap->cap_value == 0);
                cap_project_enable(kpj, MAX_USAGE);
        }
        mutex_exit(&caps_lock);

        return (0);
}

/*
 * Set zone cap to cap_val
 * If cap_val is equal to NOCAP, disable zone cap.
 *
 * If this is the first time a cap is set on a zone, allocate cpucap structure
 * without holding caps_lock to avoid KM_SLEEP allocation with caps_lock held.
 */
int
cpucaps_zone_set(zone_t *zone, rctl_qty_t cap_val)
{
        cpucap_t *cap = NULL;
        hrtime_t value;

        if (cap_val == 0)
                return (EINVAL);

        ASSERT(cap_val <= MAXCAP);
        if (cap_val > MAXCAP)
                cap_val = MAXCAP;

        /*
         * Nothing to do if trying to disable a cap on a zone when caps are off
         * or a zone which does not have a cap yet.
         */
        if ((CPUCAPS_OFF() || !ZONE_IS_CAPPED(zone)) && (cap_val == NOCAP))
                return (0);

        if (zone->zone_cpucap == NULL)
                cap = cap_alloc();

        mutex_enter(&caps_lock);

        if (cpucaps_busy) {
                mutex_exit(&caps_lock);
                return (EBUSY);
        }

        /*
         * Double-check whether zone->zone_cpucap is NULL, now with caps_lock
         * held. If it is still NULL, assign a newly allocated cpucap to it.
         */
        if (zone->zone_cpucap == NULL) {
                zone->zone_cpucap = cap;
        } else if (cap != NULL) {
                cap_free(cap);
        }

        cap = zone->zone_cpucap;
        value = cap_val * cap_tick_cost;
        if (value < 0)
                value = MAX_USAGE;

        /* Nothing to do if the value is staying the same */
        if (value == cap->cap_value) {
                mutex_exit(&caps_lock);
                return (0);
        }

        /*
         * Clear cap statistics since the cap value itself changes.
         */
        cap->cap_above = cap->cap_below = 0;


        if (cap_val == NOCAP) {
                if (CAP_ENABLED(cap)) {
                        /*
                         * Remove cap for the zone
                         */
                        cap_zone_disable(zone);
                        cpucaps_busy = B_TRUE;
                        mutex_exit(&caps_lock);
                        /*
                         * Disable caps for all project belonging to this zone
                         * unless they have their own cap.
                         */
                        (void) project_walk_all(zone->zone_id,
                            cap_project_zone_modify_walker, cap);

                        mutex_enter(&caps_lock);
                        cpucaps_busy = B_FALSE;
                }
        } else if (CAP_DISABLED(cap)) {
                /*
                 * Set a cap on a zone which previously was not capped.
                 */
                cap_zone_enable(zone, value);
                cpucaps_busy = B_TRUE;
                mutex_exit(&caps_lock);

                /*
                 * Enable cap for all projects belonging to this zone.
                 */
                (void) project_walk_all(zone->zone_id,
                    cap_project_zone_modify_walker, cap);

                mutex_enter(&caps_lock);
                cpucaps_busy = B_FALSE;
        } else {
                /*
                 * No state transitions, just change the value
                 */
                cap->cap_value = value;
        }

        ASSERT(MUTEX_HELD(&caps_lock));
        ASSERT(!cpucaps_busy);
        mutex_exit(&caps_lock);

        return (0);
}

/*
 * The project is going away so disable its cap.
 */
void
cpucaps_project_remove(kproject_t *kpj)
{
        mutex_enter(&caps_lock);
        if (PROJECT_IS_CAPPED(kpj))
                cap_project_disable(kpj);
        if (kpj->kpj_cpucap != NULL) {
                cap_free(kpj->kpj_cpucap);
                kpj->kpj_cpucap = NULL;
        }
        mutex_exit(&caps_lock);
}

/*
 * The zone is going away, so disable its cap.
 */
void
cpucaps_zone_remove(zone_t *zone)
{
        mutex_enter(&caps_lock);
        while (ZONE_IS_CAPPED(zone)) {
                mutex_exit(&caps_lock);
                (void) cpucaps_zone_set(zone, NOCAP);
                mutex_enter(&caps_lock);
        }
        if (zone->zone_cpucap != NULL) {
                cap_free(zone->zone_cpucap);
                zone->zone_cpucap = NULL;
        }
        mutex_exit(&caps_lock);
}

/*
 * New project was created. It should be put on the capped_projects list if
 * its zone has a cap.
 */
void
cpucaps_project_add(kproject_t *kpj)
{
        cpucap_t *cap = NULL;

        if (CPUCAPS_OFF() || !ZONE_IS_CAPPED(kpj->kpj_zone))
                return;

        /*
         * This project was never capped before, so allocate its cap structure.
         */
        if (kpj->kpj_cpucap == NULL)
                cap = cap_alloc();

        mutex_enter(&caps_lock);
        /*
         * Double-check with caps_lock held
         */
        if (kpj->kpj_cpucap == NULL) {
                kpj->kpj_cpucap = cap;
        } else if (cap != NULL) {
                cap_free(cap);
        }

        if (ZONE_IS_CAPPED(kpj->kpj_zone))
                cap_project_enable(kpj, MAX_USAGE);

        mutex_exit(&caps_lock);
}

/*
 * Set project cap to cap_val
 * If cap_val is equal to NOCAP, disable project cap.
 *
 * If this is the first time a cap is set on a project, allocate cpucap
 * structure without holding caps_lock to avoid KM_SLEEP allocation with
 * caps_lock held.
 */
int
cpucaps_project_set(kproject_t *kpj, rctl_qty_t cap_val)
{
        cpucap_t *cap = NULL;
        hrtime_t value;

        if (cap_val == 0)
                return (EINVAL);

        ASSERT(cap_val <= MAXCAP);
        if (cap_val > MAXCAP)
                cap_val = MAXCAP;

        /*
         * Nothing to do if trying to disable project cap and caps are not
         * enabled or if trying to disable cap on a project that does not have
         * cap enabled.
         */
        if ((cap_val == NOCAP) && (CPUCAPS_OFF() || !PROJECT_IS_CAPPED(kpj)))
                return (0);

        if (kpj->kpj_cpucap == NULL) {
                /*
                 * This project was never capped before, so allocate its cap
                 * structure.
                 */
                cap = cap_alloc();
        }

        mutex_enter(&caps_lock);

        /*
         * Double-check with caps_lock held.
         */
        if (kpj->kpj_cpucap == NULL) {
                kpj->kpj_cpucap = cap;
        } else if (cap != NULL) {
                cap_free(cap);
        }

        /*
         * Get the actual pointer to the project cap.
         */
        cap = kpj->kpj_cpucap;
        value = cap_val * cap_tick_cost;
        if (value < 0)
                value = MAX_USAGE;

        /*
         * Nothing to do if the value is not changing
         */
        if (value == cap->cap_value) {
                mutex_exit(&caps_lock);
                return (0);
        }

        /*
         * Clear cap statistics since the cap value itself changes.
         */
        cap->cap_above = cap->cap_below = 0;
        cap->cap_maxusage = 0;

        if (cap_val != NOCAP) {
                /*
                 * Enable this cap if it is not already enabled.
                 */
                if (CAP_DISABLED(cap))
                        cap_project_enable(kpj, value);
                else
                        cap->cap_value = value;
        } else if (CAP_ENABLED(cap)) {
                /*
                 * User requested to drop a cap on the project. If it is part of
                 * capped zone, keep the cap and set the value to MAX_USAGE,
                 * otherwise disable the cap.
                 */
                if (ZONE_IS_CAPPED(kpj->kpj_zone)) {
                        cap->cap_value = MAX_USAGE;
                } else {
                        cap_project_disable(kpj);
                }
        }
        mutex_exit(&caps_lock);

        return (0);
}

/*
 * Get cap usage.
 */
static rctl_qty_t
cap_get(cpucap_t *cap)
{
        return (cap != NULL ? (rctl_qty_t)(cap->cap_usage / cap_tick_cost) : 0);
}

/*
 * Get current project usage.
 */
rctl_qty_t
cpucaps_project_get(kproject_t *kpj)
{
        return (cap_get(kpj->kpj_cpucap));
}

/*
 * Get current zone usage.
 */
rctl_qty_t
cpucaps_zone_get(zone_t *zone)
{
        return (cap_get(zone->zone_cpucap));
}

/*
 * Charge project of thread t the time thread t spent on CPU since previously
 * adjusted.
 *
 * Record the current on-CPU time in the csc structure.
 *
 * Do not adjust for more than one tick worth of time.
 *
 * It is possible that the project cap is being disabled while this routine is
 * executed. This should not cause any issues since the association between the
 * thread and its project is protected by thread lock.
 */
static void
caps_charge_adjust(kthread_id_t t, caps_sc_t *csc)
{
        kproject_t      *kpj = ttoproj(t);
        hrtime_t        new_usage;
        hrtime_t        usage_delta;

        ASSERT(THREAD_LOCK_HELD(t));
        ASSERT(kpj->kpj_cpucap != NULL);

        /* Get on-CPU time since birth of a thread */
        new_usage = mstate_thread_onproc_time(t);

        /* Time spent on CPU since last checked */
        usage_delta = new_usage - csc->csc_cputime;

        /* Save the accumulated on-CPU time */
        csc->csc_cputime = new_usage;

        /* Charge at most one tick worth of on-CPU time */
        if (usage_delta > cap_tick_cost)
                usage_delta = cap_tick_cost;

        /* Add usage_delta to the project usage value. */
        if (usage_delta > 0) {
                cpucap_t *cap = kpj->kpj_cpucap;

                DTRACE_PROBE2(cpucaps__project__charge,
                    kthread_id_t, t, hrtime_t, usage_delta);

                disp_lock_enter_high(&cap->cap_usagelock);
                cap->cap_usage += usage_delta;

                /* Check for overflows */
                if (cap->cap_usage < 0)
                        cap->cap_usage = MAX_USAGE - 1;

                disp_lock_exit_high(&cap->cap_usagelock);

                /*
                 * cap_maxusage is only kept for observability. Move it outside
                 * the lock to reduce the time spent while holding the lock.
                 */
                if (cap->cap_usage > cap->cap_maxusage)
                        cap->cap_maxusage = cap->cap_usage;
        }
}

/*
 * Charge thread's project and return True if project or zone should be
 * penalized because its project or zone is exceeding its cap. Also sets
 * TS_PROJWAITQ or TS_ZONEWAITQ in this case.
 *
 * It is possible that the project cap is being disabled while this routine is
 * executed. This should not cause any issues since the association between the
 * thread and its project is protected by thread lock. It will still set
 * TS_PROJECTWAITQ/TS_ZONEWAITQ in this case but cpucaps_enforce will not place
 * anything on the blocked wait queue.
 *
 */
boolean_t
cpucaps_charge(kthread_id_t t, caps_sc_t *csc, cpucaps_charge_t charge_type)
{
        kproject_t      *kpj = ttoproj(t);
        klwp_t          *lwp = t->t_lwp;
        zone_t          *zone;
        cpucap_t        *project_cap;
        boolean_t       rc = B_FALSE;

        ASSERT(THREAD_LOCK_HELD(t));

        /* Nothing to do for projects that are not capped. */
        if (lwp == NULL || !PROJECT_IS_CAPPED(kpj))
                return (B_FALSE);

        caps_charge_adjust(t, csc);

        /*
         * The caller only requested to charge the project usage, no enforcement
         * part.
         */
        if (charge_type == CPUCAPS_CHARGE_ONLY)
                return (B_FALSE);

        project_cap = kpj->kpj_cpucap;

        if (project_cap->cap_usage >= project_cap->cap_value) {
                t->t_schedflag |= TS_PROJWAITQ;
                rc = B_TRUE;
        } else if (t->t_schedflag & TS_PROJWAITQ) {
                t->t_schedflag &= ~TS_PROJWAITQ;
        }

        zone = ttozone(t);
        if (!ZONE_IS_CAPPED(zone)) {
                if (t->t_schedflag & TS_ZONEWAITQ)
                        t->t_schedflag &= ~TS_ZONEWAITQ;
        } else {
                cpucap_t *zone_cap = zone->zone_cpucap;

                if (zone_cap->cap_usage >= zone_cap->cap_value) {
                        t->t_schedflag |= TS_ZONEWAITQ;
                        rc = B_TRUE;
                } else if (t->t_schedflag & TS_ZONEWAITQ) {
                        t->t_schedflag &= ~TS_ZONEWAITQ;
                }
        }


        return (rc);
}

/*
 * Enforce CPU caps. If got preempted in the user-land, we know that thread does
 * not hold any kernel locks, so enqueue ourselves on the waitq, if needed.
 *
 * CPU Caps are only enforced for user threads.
 *
 * Threads flagged with TS_PROJWAITQ are placed on their project wait queues and
 * threads marked with TS_ZONEWAITQ are placed on their zone wait queue.
 *
 * It is possible that by the time we enter cpucaps_enforce() the cap is already
 * disabled. In this case waitq_enqueue() fails and doesn't enqueue anything. We
 * still clear TS_PROJWAITQ/TS_ZONEWAITQ flags in this case since they no longer
 * apply.
 */
boolean_t
cpucaps_enforce(kthread_t *t)
{
        klwp_t *lwp = t->t_lwp;

        ASSERT(THREAD_LOCK_HELD(t));

        if (lwp != NULL && lwp->lwp_state == LWP_USER) {
                if (t->t_schedflag & TS_PROJWAITQ) {
                        ASSERT(ttoproj(t)->kpj_cpucap != NULL);
                        t->t_schedflag &= ~TS_ANYWAITQ;
                        if (waitq_enqueue(&(ttoproj(t)->kpj_cpucap->cap_waitq),
                            t)) {
                                return (B_TRUE);
                        }
                }
                if (t->t_schedflag & TS_ZONEWAITQ) {
                        ASSERT(ttozone(t)->zone_cpucap != NULL);
                        t->t_schedflag &= ~TS_ZONEWAITQ;
                        if (waitq_enqueue(&(ttozone(t)->zone_cpucap->cap_waitq),
                            t)) {
                                return (B_TRUE);
                        }
                }
        }

        /*
         * The thread is not enqueued on the wait queue.
         */
        return (B_FALSE);
}

/*
 * Convert internal cap statistics into values exported by cap kstat.
 */
static int
cap_kstat_update(kstat_t *ksp, int rw)
{
        struct cap_kstat *capsp = &cap_kstat;
        cpucap_t *cap = ksp->ks_private;
        clock_t tick_sec = SEC_TO_TICK(1);
        char *zonename = cap->cap_zone->zone_name;

        if (rw == KSTAT_WRITE)
                return (EACCES);

        capsp->cap_value.value.ui64 =
            ROUND_SCALE(cap->cap_value, cap_tick_cost);
        capsp->cap_usage.value.ui64 =
            ROUND_SCALE(cap->cap_usage, cap_tick_cost);
        capsp->cap_maxusage.value.ui64 =
            ROUND_SCALE(cap->cap_maxusage, cap_tick_cost);
        capsp->cap_nwait.value.ui64 = cap->cap_waitq.wq_count;
        capsp->cap_below.value.ui64 = ROUND_SCALE(cap->cap_below, tick_sec);
        capsp->cap_above.value.ui64 = ROUND_SCALE(cap->cap_above, tick_sec);
        kstat_named_setstr(&capsp->cap_zonename, zonename);

        return (0);
}