common/disp/cpucaps.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/disp.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/atomic.h>
#include <sys/cpucaps_impl.h>
#include <sys/dtrace.h>
#include <sys/sdt.h>
#include <sys/debug.h>
#include <sys/rctl.h>
#include <sys/errno.h>

/*
 * CPU Caps implementation
 * =======================
 *
 * A CPU cap can be set on any project or any zone. Zone CPU cap limits the CPU
 * usage for all projects running inside the zone. If the zone CPU cap is set
 * below the project CPU cap, the latter will have no effect.
 *
 * When CPU usage of projects and/or zones reaches specified caps, threads in
 * them do not get scheduled and instead are placed on wait queues associated
 * with a cap. Such threads will start running again only when CPU usage drops
 * below the cap level. Each zone and each project has its own wait queue.
 *
 * When CPU cap is set, the kernel continously keeps track of CPU time used by
 * capped zones and/or projects over a short time interval and calculates their
 * current CPU usage as a percentage. When the accumulated usage reaches the CPU
 * cap, LWPs running in the user-land (when they are not holding any critical
 * kernel locks) are placed on special wait queues until their project's or
 * zone's CPU usage drops below the cap.
 *
 * The system maintains a list of all capped projects and all capped zones. On
 * every clock tick every active thread belonging to a capped project adds its
 * CPU usage to its project. Usage from all projects belonging to a capped zone
 * is aggregated to get the zone usage.
 *
 * When the current CPU usage is above the cap, a project or zone is considered
 * over-capped. Every user thread caught running in an over-capped project or
 * zone is marked by setting TS_PROJWAITQ flag in thread's t_schedflag field and
 * is requested to surrender its CPU. This causes scheduling class specific
 * CL_PREEMPT() callback to be invoked. The callback function places threads
 * marked as TS_PROJWAIT on a wait queue and calls switch().
 *
 * Threads are only placed on wait queues after trapping from user-land
 * (they could be holding some user locks, but no kernel locks) and while
 * returning from the trap back to the user-land when no kernel locks are held.
 * Putting threads on wait queues in random places while running in the
 * kernel might lead to all kinds of locking problems.
 *
 * Accounting
 * ==========
 *
 * Accounting of CPU usage is based on per-thread micro-state accounting data.
 * On every clock tick clock() adds new on-CPU time for every thread found on
 * CPU. Scheduling classes also add new on-CPU time for any thread leaving CPU.
 * New times means time since it was last accounted for. On-CPU times greater
 * than 1 tick are truncated to 1 tick.
 *
 * Project CPU usage is aggregated from all threads within the project.
 * Zone CPU usage is the sum of usages for all projects within the zone. Zone
 * CPU usage is calculated on every clock tick by walking list of projects and
 * adding their usage together.
 *
 * Decay
 * =====
 *
 * CPU usage is decayed by the caps_update() routine which is called once per
 * every clock tick. It walks lists of project caps and decays their usages by
 * one per cent. If CPU usage drops below cap levels, threads on the wait queue
 * are made runnable again, one thread per clock tick.
 *
 * Interfaces
 * ==========
 *
 * The CPU Caps facility provides the following interfaces to the rest of the
 * system:
 *
 *   cpucaps_project_add(kproject_t *)
 *
 * Notifies the framework of a new project. It should be put on the
 * capped_projects list if its zone has a cap.
 *
 *   cpucaps_project_remove(kproject_t *)
 *
 * Remove the association between the specified project and its cap.
 * Called right before the project is destroyed.
 *
 * cpucaps_project_set(kproject_t *, rctl_qty_t)
 *
 * Set project cap of the specified project to the specified value. Setting the
 * value to NOCAP is equivalent to removing the cap.
 *
 *   cpucaps_zone_set(zone_t *, rctl_qty_t)
 *
 * Set zone cap of the specified zone to the specified value. Setting the value
 * to NOCAP is equivalent to removing the cap.
 *
 *   cpucaps_zone_remove(zone_t *)
 *
 * Remove the association between the zone and its cap.
 *
 *   cpucaps_charge(kthread_id_t, caps_sc_t *, cpucaps_charge_t)
 *
 * Charges specified thread's project the amount of on-CPU time that it used.
 * If the third argument is CPUCAPS_CHARGE_ONLY returns False.
 * Otherwise returns True if project or zone should be penalized because its
 * project or zone is exceeding its cap. Also sets TS_PROJWAITQ or TS_ZONEWAITQ
 * bits in t_schedflag in this case.
 *
 *   CPUCAPS_ENFORCE(kthread_id_t *)
 *
 * Enforces CPU caps for a specified thread. Places LWPs running in LWP_USER
 * state on project or zone wait queues, as requested by TS_PROJWAITQ or
 * TS_ZONEWAITQ bits in t_schedflag. Returns True if the thread was placed on a
 * wait queue or False otherwise.
 *
 *   cpucaps_sc_init(caps_sc_t *)
 *
 * Initializes the scheduling-class specific CPU Caps data for a thread.
 *
 * LOCKS
 * =====
 *
 * all the individual caps structures and their lists are protected by a global
 * caps_lock mutex. The lock is grabbed either by clock() or by events modifying
 * caps, so it is usually uncontended. We avoid all blocking memory allocations
 * while holding caps_lock to prevent clock() from blocking.
 *
 * Thread state is protected by the thread lock. It protects the association
 * between a thread and its project and, as a consequence, to its zone. The
 * association can not break while thread lock is held, so the project or zone
 * cap are not going to disappear while thread lock is held.
 *
 * Cap usage field is protected by high-pil spin-lock cap_usagelock. It is
 * grabbed by scheduling classes already holding thread lock at high PIL and by
 * clock thread performing usage decay. We should do as little work as possible
 * while holding the lock since it may be very hot. All threads in the project
 * contend for the same cache line doing cap usage updates.
 */

/*
 * caps_lock protects list of capped projects and zones, changes in the cap
 * state and changes of the global cpucaps_enabled flag.
 *
 * Changing zone caps also sets cpucaps_busy to avoid races when a zone cap is
 * modified in parallel. This can be per-zone cap flag, but we don't keep any
 * cap state for now.
 */
static kmutex_t caps_lock;		/* lock to protect: */
static list_t capped_zones;		/* - list of zones with caps */
static list_t capped_projects;		/* - list of projects with caps */
boolean_t cpucaps_enabled;		/* - are there any caps defined? */
boolean_t cpucaps_busy;			/* - is framework busy? */

/*
 * The accounting is based on the number of nanoseconds threads spend running
 * during a tick which is kept in the cap_tick_cost variable.
 */
static hrtime_t cap_tick_cost;

/*
 * How much of the usage value is decayed every clock tick
 * Decay one per cent of value per tick
 */
#define	CAP_DECAY_FACTOR 100

/*
 * Scale the value and round it to the closest integer value
 */
#define	ROUND_SCALE(x, y) (((x) + (y) / 2) / (y))

static void caps_update();

/*
 * CAP kstats.
 */
struct cap_kstat {
	kstat_named_t	cap_value;
	kstat_named_t	cap_usage;
	kstat_named_t	cap_nwait;
	kstat_named_t	cap_below;
	kstat_named_t	cap_above;
	kstat_named_t	cap_maxusage;
	kstat_named_t	cap_zonename;
} cap_kstat = {
	{ "value",	KSTAT_DATA_UINT64 },
	{ "usage",	KSTAT_DATA_UINT64 },
	{ "nwait",	KSTAT_DATA_UINT64 },
	{ "below_sec",	KSTAT_DATA_UINT64 },
	{ "above_sec",	KSTAT_DATA_UINT64 },
	{ "maxusage",	KSTAT_DATA_UINT64 },
	{ "zonename",	KSTAT_DATA_STRING },
};


static kmutex_t cap_kstat_lock;
static int cap_kstat_update(kstat_t *, int);

/*
 * Initialize CPU caps infrastructure.
 *   - Initialize lists of capped zones and capped projects
 *   - Set cpucaps_clock_callout to NULL
 */
void
cpucaps_init()
{
	/*
	 * Initialize global variables
	 */
	cap_tick_cost = TICK_TO_NSEC((hrtime_t)1);

	list_create(&capped_zones, sizeof (cpucap_t),
	    offsetof(cpucap_t, cap_link));
	list_create(&capped_projects, sizeof (cpucap_t),
	    offsetof(cpucap_t, cap_link));

	cpucaps_enabled = B_FALSE;
	cpucaps_busy = B_FALSE;
	cpucaps_clock_callout = NULL;
}

/*
 * Initialize scheduling-class specific CPU Caps data.
 */
void
cpucaps_sc_init(caps_sc_t *csc)
{
	csc->csc_cputime = 0;
}

/*
 * Allocate and initialize cpucap structure
 */
static cpucap_t *
cap_alloc(void)
{
	cpucap_t *cap = kmem_zalloc(sizeof (cpucap_t), KM_SLEEP);

	DISP_LOCK_INIT(&cap->cap_usagelock);
	waitq_init(&cap->cap_waitq);

	return (cap);
}

/*
 * Free cpucap structure
 */
static void
cap_free(cpucap_t *cap)
{
	if (cap == NULL)
		return;

	/*
	 * This cap should not be active
	 */
	ASSERT(!list_link_active(&cap->cap_link));
	ASSERT(cap->cap_value == 0);
	ASSERT(!DISP_LOCK_HELD(&cap->cap_usagelock));

	waitq_fini(&cap->cap_waitq);
	DISP_LOCK_DESTROY(&cap->cap_usagelock);

	kmem_free(cap, sizeof (cpucap_t));
}

/*
 * Activate cap - insert into active list and unblock its
 * wait queue. Should be called with caps_lock held.
 * The cap_value field is set to the value supplied.
 */
static void
cap_enable(list_t *l, cpucap_t *cap, hrtime_t value)
{
	ASSERT(MUTEX_HELD(&caps_lock));

	/*
	 * Cap can not be already enabled
	 */
	ASSERT(!CAP_ENABLED(cap));
	ASSERT(!list_link_active(&cap->cap_link));

	list_insert_tail(l, cap);
	cap->cap_below = cap->cap_above = 0;
	cap->cap_maxusage = 0;
	cap->cap_usage = 0;
	cap->cap_value = value;
	waitq_unblock(&cap->cap_waitq);
	if (CPUCAPS_OFF()) {
		cpucaps_enabled = B_TRUE;
		cpucaps_clock_callout = caps_update;
	}
}

/*
 * Deactivate cap
 *   - Block its wait queue. This prevents any new threads from being
 *	enqueued there and moves all enqueued threads to the run queue.
 *   - Remove cap from list l.
 *   - Disable CPU caps globally if there are no capped projects or zones
 *
 * Should be called with caps_lock held.
 */
static void
cap_disable(list_t *l, cpucap_t *cap)
{
	ASSERT(MUTEX_HELD(&caps_lock));
	/*
	 * Cap should be currently active
	 */
	ASSERT(CPUCAPS_ON());
	ASSERT(list_link_active(&cap->cap_link));
	ASSERT(CAP_ENABLED(cap));

	waitq_block(&cap->cap_waitq);
	list_remove(l, cap);
	if (list_is_empty(&capped_projects) && list_is_empty(&capped_zones)) {
		cpucaps_enabled = B_FALSE;
		cpucaps_clock_callout = NULL;
	}
	cap->cap_value = 0;
	cap->cap_project = NULL;
	cap->cap_zone = NULL;
	if (cap->cap_kstat != NULL) {
		kstat_delete(cap->cap_kstat);
		cap->cap_kstat = NULL;
	}

}

/*
 * Enable cap for a project kpj
 * It is safe to enable already enabled project cap.
 * Should be called with caps_lock held.
 */
static void
cap_project_enable(kproject_t *kpj, hrtime_t value)
{
	cpucap_t *cap = kpj->kpj_cpucap;

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(cap != NULL);

	if (CAP_DISABLED(cap)) {
		ASSERT(cap->cap_kstat == NULL);
		cap_enable(&capped_projects, cap, value);
		cap->cap_project = kpj;
		cap->cap_zone = kpj->kpj_zone;

		/*
		 * Create cap kstats
		 */
		if ((cap->cap_kstat = rctl_kstat_create_project(kpj, "cpucaps",
		    KSTAT_TYPE_NAMED,
		    sizeof (cap_kstat) / sizeof (kstat_named_t),
		    KSTAT_FLAG_VIRTUAL)) != NULL) {
		    cap->cap_kstat->ks_data_size +=
			strlen(cap->cap_zone->zone_name) + 1;
		    cap->cap_kstat->ks_lock = &cap_kstat_lock;
		    cap->cap_kstat->ks_data = &cap_kstat;
		    cap->cap_kstat->ks_update = cap_kstat_update;
		    cap->cap_kstat->ks_private = cap;
		    kstat_install(cap->cap_kstat);
		}
	}
}

/*
 * Disable project cap.
 * It is safe to disable already disabled project cap.
 * Should be called with caps_lock held.
 */
static void
cap_project_disable(kproject_t *kpj)
{
	cpucap_t *cap = kpj->kpj_cpucap;

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(cap != NULL);
	ASSERT(cap->cap_project == kpj);

	if (CAP_ENABLED(cap))
		cap_disable(&capped_projects, cap);
}

/*
 * Enable cap for a zone
 * It is safe to enable already enabled zone cap.
 * Should be called with caps_lock held.
 */
static void
cap_zone_enable(zone_t *zone, hrtime_t value)
{
	cpucap_t *cap = zone->zone_cpucap;

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(cap != NULL);

	if (CAP_DISABLED(cap)) {
		ASSERT(cap->cap_kstat == NULL);
		cap_enable(&capped_zones, cap, value);
		cap->cap_zone = zone;

		/*
		 * Create cap kstats
		 */
		if ((cap->cap_kstat = rctl_kstat_create_zone(zone, "cpucaps",
		    KSTAT_TYPE_NAMED,
		    sizeof (cap_kstat) / sizeof (kstat_named_t),
		    KSTAT_FLAG_VIRTUAL)) != NULL) {
		    cap->cap_kstat->ks_data_size +=
			strlen(cap->cap_zone->zone_name) + 1;
		    cap->cap_kstat->ks_lock = &cap_kstat_lock;
		    cap->cap_kstat->ks_data = &cap_kstat;
		    cap->cap_kstat->ks_update = cap_kstat_update;
		    cap->cap_kstat->ks_private = cap;
		    kstat_install(cap->cap_kstat);
		}
	}
}

/*
 * Disable zone cap.
 * It is safe to disable already disabled zone cap.
 * Should be called with caps_lock held.
 */
static void
cap_zone_disable(zone_t *zone)
{
	cpucap_t *cap = zone->zone_cpucap;

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(cap != NULL);
	ASSERT(cap->cap_zone == zone);

	if (CAP_ENABLED(cap))
		cap_disable(&capped_zones, cap);
}

/*
 * Apply specified callback to all caps contained in the list `l'.
 */
static void
cap_walk(list_t *l, void (*cb)(cpucap_t *))
{
	cpucap_t *cap;

	ASSERT(MUTEX_HELD(&caps_lock));

	for (cap = list_head(l); cap != NULL; cap = list_next(l, cap)) {
		(*cb)(cap);
	}
}

/*
 * If cap limit is not reached, make one thread from wait queue runnable.
 * The waitq_isempty check is performed without the waitq lock. If a new thread
 * is placed on the waitq right after the check, it will be picked up during the
 * next invocation of cap_poke_waitq().
 */
static void
cap_poke_waitq(cpucap_t *cap)
{
	ASSERT(MUTEX_HELD(&caps_lock));

	if (cap->cap_usage >= cap->cap_value) {
		cap->cap_above++;
	} else {
		waitq_t *wq = &cap->cap_waitq;

		cap->cap_below++;

		if (!waitq_isempty(wq))
			waitq_runone(wq);
	}
}

/*
 * The callback function called for every cap on capped_projects list.
 * Decay cap usage by CAP_DECAY_FACTOR
 * Add this cap project usage to its zone usage.
 * Kick off a thread from the cap waitq if cap is not reached.
 */
static void
cap_project_usage_walker(cpucap_t *cap)
{
	zone_t		*zone = cap->cap_zone;
	hrtime_t	cap_usage = cap->cap_usage;

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(cap->cap_project->kpj_cpucap == cap);
	ASSERT(zone == cap->cap_project->kpj_zone);
	ASSERT(CAP_ENABLED(cap));

	/*
	 * Set or clear the CAP_REACHED flag based on the current usage.
	 * Only projects having their own caps are ever marked as CAP_REACHED.
	 */
	cap_poke_waitq(cap);

	/*
	 * Add project's CPU usage to our zone's CPU usage.
	 */
	if (ZONE_IS_CAPPED(zone)) {
		cpucap_t *zcap = zone->zone_cpucap;

		ASSERT(zcap->cap_zone == zone);

		/*
		 * If we haven't reset this zone's usage during this clock tick
		 * yet, then do it now. The cap_lbolt field is used to check
		 * whether this is the first zone's project we see during this
		 * tick or a subsequent one.
		 */
		if (zcap->cap_lbolt != lbolt64) {
			if (zcap->cap_usage > zcap->cap_maxusage)
				zcap->cap_maxusage = zcap->cap_usage;
			zcap->cap_usage = 0;
		}
		DTRACE_PROBE2(cpucaps__zusage, cpucap_t *, zcap,
		    hrtime_t, cap_usage);
		zcap->cap_usage += cap_usage;
		/* Check for overflows */
		if (zcap->cap_usage < 0)
			zcap->cap_usage = MAX_USAGE - 1;
	}

	/*
	 * Decay project usage.
	 */
	disp_lock_enter(&cap->cap_usagelock);
	cap->cap_usage -= ROUND_SCALE(cap_usage, CAP_DECAY_FACTOR);
	disp_lock_exit(&cap->cap_usagelock);
}

/*
 * On every clock tick walk the list of project caps and update the CPU usage.
 * Also walk the list of zone caps checking whether any threads should
 * transition from wait queue to run queue.
 *
 * This function gets called by the clock thread directly when there are any
 * defined caps. The only lock that it grabs is caps_lock. Nothing else grabs
 * caps_lock for long periods of time, so there should be almost no contention
 * for it.
 */
static void
caps_update()
{
	mutex_enter(&caps_lock);
	cap_walk(&capped_projects, cap_project_usage_walker);
	cap_walk(&capped_zones, cap_poke_waitq);
	mutex_exit(&caps_lock);
}

/*
 * The function is called for each project in a zone when the zone cap is
 * modified. It enables project caps if zone cap is enabled and disables if the
 * zone cap is disabled and project doesn't have its own cap.
 *
 * For each project that does not have cpucap structure allocated it allocates a
 * new structure and assigns to kpj->cpu_cap. The allocation is performed
 * without holding caps_lock to avoid using KM_SLEEP allocation with caps_lock
 * held.
 */
static int
cap_project_zone_modify_walker(kproject_t *kpj, void *arg)
{
	cpucap_t *project_cap = NULL;
	cpucap_t *zone_cap = (cpucap_t *)arg;

	ASSERT(zone_cap != NULL);

	if (kpj->kpj_cpucap == NULL) {
		/*
		 * This is the first time any cap was established for this
		 * project. Allocate a new cpucap structure for it.
		 */
		project_cap = cap_alloc();
	}

	mutex_enter(&caps_lock);

	/*
	 * Double-check that kpj_cpucap is still NULL - now with caps_lock held
	 * and assign the newly allocated cpucap structure to it.
	 */
	if (kpj->kpj_cpucap == NULL) {
		kpj->kpj_cpucap = project_cap;
	} else if (project_cap != NULL) {
		cap_free(project_cap);
	}

	project_cap = kpj->kpj_cpucap;

	if (CAP_DISABLED(zone_cap)) {
		/*
		 * Remove all projects in this zone without caps
		 * from the capped_projects list.
		 */
		if (project_cap->cap_value == MAX_USAGE) {
			cap_project_disable(kpj);
		}
	} else if (CAP_DISABLED(project_cap)) {
		/*
		 * Add the project to capped_projects list.
		 */
		ASSERT(project_cap->cap_value == 0);
		cap_project_enable(kpj, MAX_USAGE);
	}
	mutex_exit(&caps_lock);

	return (0);
}

/*
 * Set zone cap to cap_val
 * If cap_val is equal to NOCAP, disable zone cap.
 *
 * If this is the first time a cap is set on a zone, allocate cpucap structure
 * without holding caps_lock to avoid KM_SLEEP allocation with caps_lock held.
 */
int
cpucaps_zone_set(zone_t *zone, rctl_qty_t cap_val)
{
	cpucap_t *cap = NULL;
	hrtime_t value;

	if (cap_val == 0)
		return (EINVAL);

	ASSERT(cap_val <= MAXCAP);
	if (cap_val > MAXCAP)
		cap_val = MAXCAP;

	/*
	 * Nothing to do if trying to disable a cap on a zone when caps are off
	 * or a zone which does not have a cap yet.
	 */
	if ((CPUCAPS_OFF() || !ZONE_IS_CAPPED(zone)) && (cap_val == NOCAP))
		return (0);

	if (zone->zone_cpucap == NULL)
		cap = cap_alloc();

	mutex_enter(&caps_lock);

	if (cpucaps_busy) {
		mutex_exit(&caps_lock);
		return (EBUSY);
	}

	/*
	 * Double-check whether zone->zone_cpucap is NULL, now with caps_lock
	 * held. If it is still NULL, assign a newly allocated cpucap to it.
	 */
	if (zone->zone_cpucap == NULL) {
		zone->zone_cpucap = cap;
	} else if (cap != NULL) {
		cap_free(cap);
	}

	cap = zone->zone_cpucap;
	value = cap_val * cap_tick_cost;
	if (value < 0)
		value = MAX_USAGE;

	/* Nothing to do if the value is staying the same */
	if (value == cap->cap_value) {
		mutex_exit(&caps_lock);
		return (0);
	}

	/*
	 * Clear cap statistics since the cap value itself changes.
	 */
	cap->cap_above = cap->cap_below = 0;


	if (cap_val == NOCAP) {
		if (CAP_ENABLED(cap)) {
			/*
			 * Remove cap for the zone
			 */
			cap_zone_disable(zone);
			cpucaps_busy = B_TRUE;
			mutex_exit(&caps_lock);
			/*
			 * Disable caps for all project belonging to this zone
			 * unless they have their own cap.
			 */
			(void) project_walk_all(zone->zone_id,
			    cap_project_zone_modify_walker, cap);

			mutex_enter(&caps_lock);
			cpucaps_busy = B_FALSE;
		}
	} else if (CAP_DISABLED(cap)) {
		/*
		 * Set a cap on a zone which previously was not capped.
		 */
		cap_zone_enable(zone, value);
		cpucaps_busy = B_TRUE;
		mutex_exit(&caps_lock);

		/*
		 * Enable cap for all projects belonging to this zone.
		 */
		(void) project_walk_all(zone->zone_id,
		    cap_project_zone_modify_walker, cap);

		mutex_enter(&caps_lock);
		cpucaps_busy = B_FALSE;
	} else {
		/*
		 * No state transitions, just change the value
		 */
		cap->cap_value = value;
	}

	ASSERT(MUTEX_HELD(&caps_lock));
	ASSERT(!cpucaps_busy);
	mutex_exit(&caps_lock);

	return (0);
}

/*
 * The project is going away so disable its cap.
 */
void
cpucaps_project_remove(kproject_t *kpj)
{
	mutex_enter(&caps_lock);
	if (PROJECT_IS_CAPPED(kpj))
		cap_project_disable(kpj);
	if (kpj->kpj_cpucap != NULL) {
		cap_free(kpj->kpj_cpucap);
		kpj->kpj_cpucap = NULL;
	}
	mutex_exit(&caps_lock);
}

/*
 * The zone is going away, so disable its cap.
 */
void
cpucaps_zone_remove(zone_t *zone)
{
	mutex_enter(&caps_lock);
	while (ZONE_IS_CAPPED(zone)) {
		mutex_exit(&caps_lock);
		(void) cpucaps_zone_set(zone, NOCAP);
		mutex_enter(&caps_lock);
	}
	if (zone->zone_cpucap != NULL) {
		cap_free(zone->zone_cpucap);
		zone->zone_cpucap = NULL;
	}
	mutex_exit(&caps_lock);
}

/*
 * New project was created. It should be put on the capped_projects list if
 * its zone has a cap.
 */
void
cpucaps_project_add(kproject_t *kpj)
{
	cpucap_t *cap = NULL;

	if (CPUCAPS_OFF() || !ZONE_IS_CAPPED(kpj->kpj_zone))
		return;

	/*
	 * This project was never capped before, so allocate its cap structure.
	 */
	if (kpj->kpj_cpucap == NULL)
		cap = cap_alloc();

	mutex_enter(&caps_lock);
	/*
	 * Double-check with caps_lock held
	 */
	if (kpj->kpj_cpucap == NULL) {
		kpj->kpj_cpucap = cap;
	} else if (cap != NULL) {
		cap_free(cap);
	}

	if (ZONE_IS_CAPPED(kpj->kpj_zone))
		cap_project_enable(kpj, MAX_USAGE);

	mutex_exit(&caps_lock);
}

/*
 * Set project cap to cap_val
 * If cap_val is equal to NOCAP, disable project cap.
 *
 * If this is the first time a cap is set on a project, allocate cpucap
 * structure without holding caps_lock to avoid KM_SLEEP allocation with
 * caps_lock held.
 */
int
cpucaps_project_set(kproject_t *kpj, rctl_qty_t cap_val)
{
	cpucap_t *cap = NULL;
	hrtime_t value;

	if (cap_val == 0)
		return (EINVAL);

	ASSERT(cap_val <= MAXCAP);
	if (cap_val > MAXCAP)
		cap_val = MAXCAP;

	/*
	 * Nothing to do if trying to disable project cap and caps are not
	 * enabled or if trying to disable cap on a project that does not have
	 * cap enabled.
	 */
	if ((cap_val == NOCAP) && (CPUCAPS_OFF() || !PROJECT_IS_CAPPED(kpj)))
		return (0);

	if (kpj->kpj_cpucap == NULL) {
		/*
		 * This project was never capped before, so allocate its cap
		 * structure.
		 */
		cap = cap_alloc();
	}

	mutex_enter(&caps_lock);

	/*
	 * Double-check with caps_lock held.
	 */
	if (kpj->kpj_cpucap == NULL) {
		kpj->kpj_cpucap = cap;
	} else if (cap != NULL) {
		cap_free(cap);
	}

	/*
	 * Get the actual pointer to the project cap.
	 */
	cap = kpj->kpj_cpucap;
	value = cap_val * cap_tick_cost;
	if (value < 0)
		value = MAX_USAGE;

	/*
	 * Nothing to do if the value is not changing
	 */
	if (value == cap->cap_value) {
		mutex_exit(&caps_lock);
		return (0);
	}

	/*
	 * Clear cap statistics since the cap value itself changes.
	 */
	cap->cap_above = cap->cap_below = 0;
	cap->cap_maxusage = 0;

	if (cap_val != NOCAP) {
		/*
		 * Enable this cap if it is not already enabled.
		 */
		if (CAP_DISABLED(cap))
			cap_project_enable(kpj, value);
		else
			cap->cap_value = value;
	} else if (CAP_ENABLED(cap)) {
		/*
		 * User requested to drop a cap on the project. If it is part of
		 * capped zone, keep the cap and set the value to MAX_USAGE,
		 * otherwise disable the cap.
		 */
		if (ZONE_IS_CAPPED(kpj->kpj_zone)) {
			cap->cap_value = MAX_USAGE;
		} else {
			cap_project_disable(kpj);
		}
	}
	mutex_exit(&caps_lock);

	return (0);
}

/*
 * Get cap usage.
 */
static rctl_qty_t
cap_get(cpucap_t *cap)
{
	return (cap != NULL ? (rctl_qty_t)(cap->cap_usage / cap_tick_cost) : 0);
}

/*
 * Get current project usage.
 */
rctl_qty_t
cpucaps_project_get(kproject_t *kpj)
{
	return (cap_get(kpj->kpj_cpucap));
}

/*
 * Get current zone usage.
 */
rctl_qty_t
cpucaps_zone_get(zone_t *zone)
{
	return (cap_get(zone->zone_cpucap));
}

/*
 * Charge project of thread t the time thread t spent on CPU since previously
 * adjusted.
 *
 * Record the current on-CPU time in the csc structure.
 *
 * Do not adjust for more than one tick worth of time.
 *
 */
static void
caps_charge_adjust(kthread_id_t t, caps_sc_t *csc)
{
	kproject_t	*kpj = ttoproj(t);
	hrtime_t	new_usage;
	hrtime_t	usage_delta;

	ASSERT(THREAD_LOCK_HELD(t));
	ASSERT(PROJECT_IS_CAPPED(kpj));

	/* Get on-CPU time since birth of a thread */
	new_usage = mstate_thread_onproc_time(t);

	/* Time spent on CPU since last checked */
	usage_delta = new_usage - csc->csc_cputime;

	/* Save the accumulated on-CPU time */
	csc->csc_cputime = new_usage;

	/* Charge at most one tick worth of on-CPU time */
	if (usage_delta > cap_tick_cost)
		usage_delta = cap_tick_cost;

	/* Add usage_delta to the project usage value. */
	if (usage_delta > 0) {
		cpucap_t *cap = kpj->kpj_cpucap;

		DTRACE_PROBE2(cpucaps__project__charge,
		    kthread_id_t, t, hrtime_t, usage_delta);

		disp_lock_enter_high(&cap->cap_usagelock);
		cap->cap_usage += usage_delta;

		/* Check for overflows */
		if (cap->cap_usage < 0)
			cap->cap_usage = MAX_USAGE - 1;

		disp_lock_exit_high(&cap->cap_usagelock);

		/*
		 * cap_maxusage is only kept for observability. Move it outside
		 * the lock to reduce the time spent while holding the lock.
		 */
		if (cap->cap_usage > cap->cap_maxusage)
			cap->cap_maxusage = cap->cap_usage;
	}
}

/*
 * Charge thread's project and return True if project or zone should be
 * penalized because its project or zone is exceeding its cap. Also sets
 * TS_PROJWAITQ or TS_ZONEWAITQ in this case.
 */
boolean_t
cpucaps_charge(kthread_id_t t, caps_sc_t *csc, cpucaps_charge_t charge_type)
{
	kproject_t	*kpj = ttoproj(t);
	klwp_t		*lwp = t->t_lwp;
	zone_t		*zone;
	cpucap_t	*project_cap;
	boolean_t	rc = B_FALSE;

	ASSERT(THREAD_LOCK_HELD(t));

	/* Nothing to do for projects that are not capped. */
	if (lwp == NULL || !PROJECT_IS_CAPPED(kpj))
		return (B_FALSE);

	caps_charge_adjust(t, csc);

	/*
	 * The caller only requested to charge the project usage, no enforcement
	 * part.
	 */
	if (charge_type == CPUCAPS_CHARGE_ONLY)
		return (B_FALSE);

	project_cap = kpj->kpj_cpucap;

	if (project_cap->cap_usage >= project_cap->cap_value) {
		t->t_schedflag |= TS_PROJWAITQ;
		rc = B_TRUE;
	} else if (t->t_schedflag & TS_PROJWAITQ) {
		t->t_schedflag &= ~TS_PROJWAITQ;
	}

	zone = ttozone(t);
	if (!ZONE_IS_CAPPED(zone)) {
		if (t->t_schedflag & TS_ZONEWAITQ)
			t->t_schedflag &= ~TS_ZONEWAITQ;
	} else {
		cpucap_t *zone_cap = zone->zone_cpucap;

		if (zone_cap->cap_usage >= zone_cap->cap_value) {
			t->t_schedflag |= TS_ZONEWAITQ;
			rc = B_TRUE;
		} else if (t->t_schedflag & TS_ZONEWAITQ) {
			t->t_schedflag &= ~TS_ZONEWAITQ;
		}
	}


	return (rc);
}

/*
 * Enforce CPU caps. If got preempted in the user-land, we know that thread does
 * not hold any kernel locks, so enqueue ourselves on the waitq, if needed.
 *
 * CPU Caps are only enforced for user threads.
 *
 * Threads flagged with TS_PROJWAITQ are placed on their project wait queues and
 * threads marked with TS_ZONEWAITQ are placed on their zone wait queue.
 *
 * It is possible that by the time we enter cpucaps_enforce() the cap is already
 * disabled. In this case waitq_enqueue() fails and doesn't enqueue anything. We
 * still clear TS_PROJWAITQ/TS_ZONEWAITQ flags in this case since they no longer
 * apply.
 */
boolean_t
cpucaps_enforce(kthread_t *t)
{
	klwp_t *lwp = t->t_lwp;

	ASSERT(THREAD_LOCK_HELD(t));

	if (lwp != NULL && lwp->lwp_state == LWP_USER) {
		if (t->t_schedflag & TS_PROJWAITQ) {
			ASSERT(ttoproj(t)->kpj_cpucap != NULL);
			t->t_schedflag &= ~TS_ANYWAITQ;
			if (waitq_enqueue(&(ttoproj(t)->kpj_cpucap->cap_waitq),
				t)) {
				return (B_TRUE);
			}
		}
		if (t->t_schedflag & TS_ZONEWAITQ) {
			ASSERT(ttozone(t)->zone_cpucap != NULL);
			t->t_schedflag &= ~TS_ZONEWAITQ;
			if (waitq_enqueue(&(ttozone(t)->zone_cpucap->cap_waitq),
				t)) {
				return (B_TRUE);
			}
		}
	}

	/*
	 * The thread is not enqueued on the wait queue.
	 */
	return (B_FALSE);
}

/*
 * Convert internal cap statistics into values exported by cap kstat.
 */
static int
cap_kstat_update(kstat_t *ksp, int rw)
{
	struct cap_kstat *capsp = &cap_kstat;
	cpucap_t *cap = ksp->ks_private;
	clock_t	tick_sec = SEC_TO_TICK(1);
	char *zonename = cap->cap_zone->zone_name;

	if (rw == KSTAT_WRITE)
		return (EACCES);

	capsp->cap_value.value.ui64 =
	    ROUND_SCALE(cap->cap_value, cap_tick_cost);
	capsp->cap_usage.value.ui64 =
	    ROUND_SCALE(cap->cap_usage, cap_tick_cost);
	capsp->cap_maxusage.value.ui64 =
	    ROUND_SCALE(cap->cap_maxusage, cap_tick_cost);
	capsp->cap_nwait.value.ui64 = cap->cap_waitq.wq_count;
	capsp->cap_below.value.ui64 = ROUND_SCALE(cap->cap_below, tick_sec);
	capsp->cap_above.value.ui64 = ROUND_SCALE(cap->cap_above, tick_sec);
	kstat_named_setstr(&capsp->cap_zonename, zonename);

	return (0);
}