common/os/clock_tick.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/thread.h>
#include <sys/proc.h>
#include <sys/task.h>
#include <sys/cmn_err.h>
#include <sys/class.h>
#include <sys/sdt.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
#include <sys/clock_tick.h>
#include <sys/clock_impl.h>
#include <sys/sysmacros.h>
#include <vm/rm.h>

/*
 * This file contains the implementation of clock tick accounting for threads.
 * Every tick, user threads running on various CPUs are located and charged
 * with a tick to account for their use of CPU time.
 *
 * Every tick, the clock() handler calls clock_tick_schedule() to perform tick
 * accounting for all the threads in the system. Tick accounting is done in
 * two phases:
 *
 * Tick scheduling	Done in clock_tick_schedule(). In this phase, cross
 *			calls are scheduled to multiple CPUs to perform
 *			multi-threaded tick accounting. The CPUs are chosen
 *			on a rotational basis so as to distribute the tick
 *			accounting load evenly across all CPUs.
 *
 * Tick execution	Done in clock_tick_execute(). In this phase, tick
 *			accounting is actually performed by softint handlers
 *			on multiple CPUs.
 *
 * This implementation gives us a multi-threaded tick processing facility that
 * is suitable for configurations with a large number of CPUs. On smaller
 * configurations it may be desirable to let the processing be single-threaded
 * and just allow clock() to do it as it has been done traditionally. To
 * facilitate this, a variable, clock_tick_threshold, is defined. Platforms
 * that desire multi-threading should set this variable to something
 * appropriate. A recommended value may be found in clock_tick.h. At boot time,
 * if the number of CPUs is greater than clock_tick_threshold, multi-threading
 * kicks in. Note that this is a decision made at boot time. If more CPUs
 * are dynamically added later on to exceed the threshold, no attempt is made
 * to switch to multi-threaded. Similarly, if CPUs are removed dynamically
 * no attempt is made to switch to single-threaded. This is to keep the
 * implementation simple. Also note that the threshold can be changed for a
 * specific customer configuration via /etc/system.
 *
 * The boot time decision is reflected in clock_tick_single_threaded.
 */

/*
 * clock_tick_threshold
 *	If the number of CPUs at boot time exceeds this threshold,
 *	multi-threaded tick accounting kicks in.
 *
 * clock_tick_ncpus
 *	The number of CPUs in a set. Each set is scheduled for tick execution
 *	on a separate processor.
 *
 * clock_tick_single_threaded
 *	Indicates whether or not tick accounting is single threaded.
 *
 * clock_tick_total_cpus
 *	Total number of online CPUs.
 *
 * clock_tick_cpus
 *	Array of online CPU pointers.
 *
 * clock_tick_cpu
 *	Per-CPU, cache-aligned data structures to facilitate multi-threading.
 *
 * clock_tick_active
 *	Counter that indicates the number of active tick processing softints
 *	in the system.
 *
 * clock_tick_pending
 *	Number of pending ticks that need to be accounted by the softint
 *	handlers.
 *
 * clock_tick_lock
 *	Mutex to synchronize between clock_tick_schedule() and
 *	CPU online/offline.
 *
 * clock_cpu_id
 *	CPU id of the clock() CPU. Used to detect when the clock CPU
 *	is offlined.
 *
 * clock_tick_online_cpuset
 *	CPU set of all online processors that can be X-called.
 *
 * clock_tick_proc_max
 *	Each process is allowed to accumulate a few ticks before checking
 *	for the task CPU time resource limit. We lower the number of calls
 *	to rctl_test() to make tick accounting more scalable. The tradeoff
 *	is that the limit may not get enforced in a timely manner. This is
 *	typically not a problem.
 *
 * clock_tick_set
 *	Per-set structures. Each structure contains the range of CPUs
 *	to be processed for the set.
 *
 * clock_tick_nsets;
 *	Number of sets.
 *
 * clock_tick_scan
 *	Where to begin the scan for single-threaded mode. In multi-threaded,
 *	the clock_tick_set itself contains a field for this.
 */
int			clock_tick_threshold;
int			clock_tick_ncpus;
int			clock_tick_single_threaded;
int			clock_tick_total_cpus;
cpu_t			*clock_tick_cpus[NCPU];
clock_tick_cpu_t	*clock_tick_cpu[NCPU];
ulong_t			clock_tick_active;
int			clock_tick_pending;
kmutex_t		clock_tick_lock;
processorid_t		clock_cpu_id;
cpuset_t		clock_tick_online_cpuset;
clock_t			clock_tick_proc_max;
clock_tick_set_t	*clock_tick_set;
int			clock_tick_nsets;
int			clock_tick_scan;
ulong_t			clock_tick_intr;

static uint_t	clock_tick_execute(caddr_t, caddr_t);
static void	clock_tick_execute_common(int, int, int, clock_t, int);

#define	CLOCK_TICK_ALIGN	64	/* cache alignment */

/*
 * Clock tick initialization is done in two phases:
 *
 * 1. Before clock_init() is called, clock_tick_init_pre() is called to set
 *    up single-threading so the clock() can begin to do its job.
 *
 * 2. After the slave CPUs are initialized at boot time, we know the number
 *    of CPUs. clock_tick_init_post() is called to set up multi-threading if
 *    required.
 */
void
clock_tick_init_pre(void)
{
	clock_tick_cpu_t	*ctp;
	int			i, n;
	clock_tick_set_t	*csp;
	uintptr_t		buf;
	size_t			size;

	clock_tick_single_threaded = 1;

	size = P2ROUNDUP(sizeof (clock_tick_cpu_t), CLOCK_TICK_ALIGN);
	buf = (uintptr_t)kmem_zalloc(size * NCPU + CLOCK_TICK_ALIGN, KM_SLEEP);
	buf = P2ROUNDUP(buf, CLOCK_TICK_ALIGN);

	/*
	 * Perform initialization in case multi-threading is chosen later.
	 */
	if (&create_softint != NULL) {
		clock_tick_intr = create_softint(LOCK_LEVEL,
		    clock_tick_execute, (caddr_t)NULL);
	}
	for (i = 0; i < NCPU; i++, buf += size) {
		ctp = (clock_tick_cpu_t *)buf;
		clock_tick_cpu[i] = ctp;
		mutex_init(&ctp->ct_lock, NULL, MUTEX_DEFAULT, NULL);
		if (&create_softint != NULL) {
			ctp->ct_intr = clock_tick_intr;
		}
		ctp->ct_pending = 0;
	}

	mutex_init(&clock_tick_lock, NULL, MUTEX_DEFAULT, NULL);

	/*
	 * Compute clock_tick_ncpus here. We need it to compute the
	 * maximum number of tick sets we need to support.
	 */
	ASSERT(clock_tick_ncpus >= 0);
	if (clock_tick_ncpus == 0)
		clock_tick_ncpus = CLOCK_TICK_NCPUS;
	if (clock_tick_ncpus > max_ncpus)
		clock_tick_ncpus = max_ncpus;

	/*
	 * Allocate and initialize the tick sets.
	 */
	n = (max_ncpus + clock_tick_ncpus - 1)/clock_tick_ncpus;
	clock_tick_set = kmem_zalloc(sizeof (clock_tick_set_t) * n, KM_SLEEP);
	for (i = 0; i < n; i++) {
		csp = &clock_tick_set[i];
		csp->ct_start = i * clock_tick_ncpus;
		csp->ct_scan = csp->ct_start;
		csp->ct_end = csp->ct_start;
	}
}

void
clock_tick_init_post(void)
{
	/*
	 * If a platform does not provide create_softint() and invoke_softint(),
	 * then we assume single threaded.
	 */
	if (&invoke_softint == NULL)
		clock_tick_threshold = 0;

	ASSERT(clock_tick_threshold >= 0);

	if (clock_tick_threshold == 0)
		clock_tick_threshold = max_ncpus;

	/*
	 * If a platform does not specify a threshold or if the number of CPUs
	 * at boot time does not exceed the threshold, tick accounting remains
	 * single-threaded.
	 */
	if (ncpus <= clock_tick_threshold) {
		clock_tick_ncpus = max_ncpus;
		clock_tick_proc_max = 1;
		return;
	}

	/*
	 * OK. Multi-thread tick processing. If a platform has not specified
	 * the CPU set size for multi-threading, then use the default value.
	 * This value has been arrived through measurements on large
	 * configuration systems.
	 */
	clock_tick_single_threaded = 0;
	if (clock_tick_proc_max == 0) {
		clock_tick_proc_max = CLOCK_TICK_PROC_MAX;
		if (hires_tick)
			clock_tick_proc_max *= 10;
	}
}

static void
clock_tick_schedule_one(clock_tick_set_t *csp, int pending, processorid_t cid)
{
	clock_tick_cpu_t	*ctp;

	ASSERT(&invoke_softint != NULL);

	atomic_inc_ulong(&clock_tick_active);

	/*
	 * Schedule tick accounting for a set of CPUs.
	 */
	ctp = clock_tick_cpu[cid];
	mutex_enter(&ctp->ct_lock);
	ctp->ct_lbolt = LBOLT_NO_ACCOUNT;
	ctp->ct_pending += pending;
	ctp->ct_start = csp->ct_start;
	ctp->ct_end = csp->ct_end;
	ctp->ct_scan = csp->ct_scan;
	mutex_exit(&ctp->ct_lock);

	invoke_softint(cid, ctp->ct_intr);
	/*
	 * Return without waiting for the softint to finish.
	 */
}

static void
clock_tick_process(cpu_t *cp, clock_t mylbolt, int pending)
{
	kthread_t	*t;
	kmutex_t	*plockp;
	int		notick, intr;
	klwp_id_t	lwp;

	/*
	 * The locking here is rather tricky. thread_free_prevent()
	 * prevents the thread returned from being freed while we
	 * are looking at it. We can then check if the thread
	 * is exiting and get the appropriate p_lock if it
	 * is not.  We have to be careful, though, because
	 * the _process_ can still be freed while we've
	 * prevented thread free.  To avoid touching the
	 * proc structure we put a pointer to the p_lock in the
	 * thread structure.  The p_lock is persistent so we
	 * can acquire it even if the process is gone.  At that
	 * point we can check (again) if the thread is exiting
	 * and either drop the lock or do the tick processing.
	 */
	t = cp->cpu_thread;	/* Current running thread */
	if (CPU == cp) {
		/*
		 * 't' will be the tick processing thread on this
		 * CPU.  Use the pinned thread (if any) on this CPU
		 * as the target of the clock tick.
		 */
		if (t->t_intr != NULL)
			t = t->t_intr;
	}

	/*
	 * We use thread_free_prevent to keep the currently running
	 * thread from being freed or recycled while we're
	 * looking at it.
	 */
	thread_free_prevent(t);
	/*
	 * We cannot hold the cpu_lock to prevent the
	 * cpu_active from changing in the clock interrupt.
	 * As long as we don't block (or don't get pre-empted)
	 * the cpu_list will not change (all threads are paused
	 * before list modification).
	 */
	if (CLOCK_TICK_CPU_OFFLINE(cp)) {
		thread_free_allow(t);
		return;
	}

	/*
	 * Make sure the thread is still on the CPU.
	 */
	if ((t != cp->cpu_thread) &&
	    ((cp != CPU) || (t != cp->cpu_thread->t_intr))) {
		/*
		 * We could not locate the thread. Skip this CPU. Race
		 * conditions while performing these checks are benign.
		 * These checks are not perfect and they don't need
		 * to be.
		 */
		thread_free_allow(t);
		return;
	}

	intr = t->t_flag & T_INTR_THREAD;
	lwp = ttolwp(t);
	if (lwp == NULL || (t->t_proc_flag & TP_LWPEXIT) || intr) {
		/*
		 * Thread is exiting (or uninteresting) so don't
		 * do tick processing.
		 */
		thread_free_allow(t);
		return;
	}

	/*
	 * OK, try to grab the process lock.  See
	 * comments above for why we're not using
	 * ttoproc(t)->p_lockp here.
	 */
	plockp = t->t_plockp;
	mutex_enter(plockp);
	/* See above comment. */
	if (CLOCK_TICK_CPU_OFFLINE(cp)) {
		mutex_exit(plockp);
		thread_free_allow(t);
		return;
	}

	/*
	 * The thread may have exited between when we
	 * checked above, and when we got the p_lock.
	 */
	if (t->t_proc_flag & TP_LWPEXIT) {
		mutex_exit(plockp);
		thread_free_allow(t);
		return;
	}

	/*
	 * Either we have the p_lock for the thread's process,
	 * or we don't care about the thread structure any more.
	 * Either way we can allow thread free.
	 */
	thread_free_allow(t);

	/*
	 * If we haven't done tick processing for this
	 * lwp, then do it now. Since we don't hold the
	 * lwp down on a CPU it can migrate and show up
	 * more than once, hence the lbolt check. mylbolt
	 * is copied at the time of tick scheduling to prevent
	 * lbolt mismatches.
	 *
	 * Also, make sure that it's okay to perform the
	 * tick processing before calling clock_tick.
	 * Setting notick to a TRUE value (ie. not 0)
	 * results in tick processing not being performed for
	 * that thread.
	 */
	notick = ((cp->cpu_flags & CPU_QUIESCED) || CPU_ON_INTR(cp) ||
	    (cp->cpu_dispthread == cp->cpu_idle_thread));

	if ((!notick) && (t->t_lbolt < mylbolt)) {
		t->t_lbolt = mylbolt;
		clock_tick(t, pending);
	}

	mutex_exit(plockp);
}

void
clock_tick_schedule(int one_sec)
{
	ulong_t			active;
	int			i, end;
	clock_tick_set_t	*csp;
	cpu_t			*cp;

	if (clock_cpu_id != CPU->cpu_id)
		clock_cpu_id = CPU->cpu_id;

	if (clock_tick_single_threaded) {
		/*
		 * Each tick cycle, start the scan from a different
		 * CPU for the sake of fairness.
		 */
		end = clock_tick_total_cpus;
		clock_tick_scan++;
		if (clock_tick_scan >= end)
			clock_tick_scan = 0;

		clock_tick_execute_common(0, clock_tick_scan, end,
		    LBOLT_NO_ACCOUNT, 1);

		return;
	}

	/*
	 * If the previous invocation of handlers is not yet finished, then
	 * simply increment a pending count and return. Eventually when they
	 * finish, the pending count is passed down to the next set of
	 * handlers to process. This way, ticks that have already elapsed
	 * in the past are handled as quickly as possible to minimize the
	 * chances of threads getting away before their pending ticks are
	 * accounted. The other benefit is that if the pending count is
	 * more than one, it can be handled by a single invocation of
	 * clock_tick(). This is a good optimization for large configuration
	 * busy systems where tick accounting can get backed up for various
	 * reasons.
	 */
	clock_tick_pending++;

	active = clock_tick_active;
	active = atomic_cas_ulong(&clock_tick_active, active, active);
	if (active)
		return;

	/*
	 * We want to handle the clock CPU here. If we
	 * scheduled the accounting for the clock CPU to another
	 * processor, that processor will find only the clock() thread
	 * running and not account for any user thread below it. Also,
	 * we want to handle this before we block on anything and allow
	 * the pinned thread below the current thread to escape.
	 */
	clock_tick_process(CPU, LBOLT_NO_ACCOUNT, clock_tick_pending);

	mutex_enter(&clock_tick_lock);

	/*
	 * Schedule each set on a separate processor.
	 */
	cp = clock_cpu_list;
	for (i = 0; i < clock_tick_nsets; i++) {
		csp = &clock_tick_set[i];

		/*
		 * Pick the next online CPU in list for scheduling tick
		 * accounting. The clock_tick_lock is held by the caller.
		 * So, CPU online/offline cannot muck with this while
		 * we are picking our CPU to X-call.
		 */
		if (cp == CPU)
			cp = cp->cpu_next_onln;

		/*
		 * Each tick cycle, start the scan from a different
		 * CPU for the sake of fairness.
		 */
		csp->ct_scan++;
		if (csp->ct_scan >= csp->ct_end)
			csp->ct_scan = csp->ct_start;

		clock_tick_schedule_one(csp, clock_tick_pending, cp->cpu_id);

		cp = cp->cpu_next_onln;
	}

	if (one_sec) {
		/*
		 * Move the CPU pointer around every second. This is so
		 * all the CPUs can be X-called in a round-robin fashion
		 * to evenly distribute the X-calls. We don't do this
		 * at a faster rate than this because we don't want
		 * to affect cache performance negatively.
		 */
		clock_cpu_list = clock_cpu_list->cpu_next_onln;
	}

	mutex_exit(&clock_tick_lock);

	clock_tick_pending = 0;
}

static void
clock_tick_execute_common(int start, int scan, int end, clock_t mylbolt,
	int pending)
{
	cpu_t		*cp;
	int		i;

	ASSERT((start <= scan) && (scan <= end));

	/*
	 * Handle the thread on current CPU first. This is to prevent a
	 * pinned thread from escaping if we ever block on something.
	 * Note that in the single-threaded mode, this handles the clock
	 * CPU.
	 */
	clock_tick_process(CPU, mylbolt, pending);

	/*
	 * Perform tick accounting for the threads running on
	 * the scheduled CPUs.
	 */
	for (i = scan; i < end; i++) {
		cp = clock_tick_cpus[i];
		if ((cp == NULL) || (cp == CPU) || (cp->cpu_id == clock_cpu_id))
			continue;
		clock_tick_process(cp, mylbolt, pending);
	}

	for (i = start; i < scan; i++) {
		cp = clock_tick_cpus[i];
		if ((cp == NULL) || (cp == CPU) || (cp->cpu_id == clock_cpu_id))
			continue;
		clock_tick_process(cp, mylbolt, pending);
	}
}

/*ARGSUSED*/
static uint_t
clock_tick_execute(caddr_t arg1, caddr_t arg2)
{
	clock_tick_cpu_t	*ctp;
	int			start, scan, end, pending;
	clock_t			mylbolt;

	/*
	 * We could have raced with cpu offline. We don't want to
	 * process anything on an offlined CPU. If we got blocked
	 * on anything, we may not get scheduled when we wakeup
	 * later on.
	 */
	if (!CLOCK_TICK_XCALL_SAFE(CPU))
		goto out;

	ctp = clock_tick_cpu[CPU->cpu_id];

	mutex_enter(&ctp->ct_lock);
	pending = ctp->ct_pending;
	if (pending == 0) {
		/*
		 * If a CPU is busy at LOCK_LEVEL, then an invocation
		 * of this softint may be queued for some time. In that case,
		 * clock_tick_active will not be incremented.
		 * clock_tick_schedule() will then assume that the previous
		 * invocation is done and post a new softint. The first one
		 * that gets in will reset the pending count so the
		 * second one is a noop.
		 */
		mutex_exit(&ctp->ct_lock);
		goto out;
	}
	ctp->ct_pending = 0;
	start = ctp->ct_start;
	end = ctp->ct_end;
	scan = ctp->ct_scan;
	mylbolt = ctp->ct_lbolt;
	mutex_exit(&ctp->ct_lock);

	clock_tick_execute_common(start, scan, end, mylbolt, pending);

out:
	/*
	 * Signal completion to the clock handler.
	 */
	atomic_dec_ulong(&clock_tick_active);

	return (1);
}

/*ARGSUSED*/
static int
clock_tick_cpu_setup(cpu_setup_t what, int cid, void *arg)
{
	cpu_t			*cp, *ncp;
	int			i, set;
	clock_tick_set_t	*csp;

	/*
	 * This function performs some computations at CPU offline/online
	 * time. The computed values are used during tick scheduling and
	 * execution phases. This avoids having to compute things on
	 * an every tick basis. The other benefit is that we perform the
	 * computations only for onlined CPUs (not offlined ones). As a
	 * result, no tick processing is attempted for offlined CPUs.
	 *
	 * Also, cpu_offline() calls this function before checking for
	 * active interrupt threads. This allows us to avoid posting
	 * cross calls to CPUs that are being offlined.
	 */

	cp = cpu[cid];

	mutex_enter(&clock_tick_lock);

	switch (what) {
	case CPU_ON:
		clock_tick_cpus[clock_tick_total_cpus] = cp;
		set = clock_tick_total_cpus / clock_tick_ncpus;
		csp = &clock_tick_set[set];
		csp->ct_end++;
		clock_tick_total_cpus++;
		clock_tick_nsets =
		    (clock_tick_total_cpus + clock_tick_ncpus - 1) /
		    clock_tick_ncpus;
		CPUSET_ADD(clock_tick_online_cpuset, cp->cpu_id);
		membar_sync();
		break;

	case CPU_OFF:
		if (&sync_softint != NULL)
			sync_softint(clock_tick_online_cpuset);
		CPUSET_DEL(clock_tick_online_cpuset, cp->cpu_id);
		clock_tick_total_cpus--;
		clock_tick_cpus[clock_tick_total_cpus] = NULL;
		clock_tick_nsets =
		    (clock_tick_total_cpus + clock_tick_ncpus - 1) /
		    clock_tick_ncpus;
		set = clock_tick_total_cpus / clock_tick_ncpus;
		csp = &clock_tick_set[set];
		csp->ct_end--;

		i = 0;
		ncp = cpu_active;
		do {
			if (cp == ncp)
				continue;
			clock_tick_cpus[i] = ncp;
			i++;
		} while ((ncp = ncp->cpu_next_onln) != cpu_active);
		ASSERT(i == clock_tick_total_cpus);
		membar_sync();
		break;

	default:
		break;
	}

	mutex_exit(&clock_tick_lock);

	return (0);
}


void
clock_tick_mp_init(void)
{
	cpu_t	*cp;

	mutex_enter(&cpu_lock);

	cp = cpu_active;
	do {
		(void) clock_tick_cpu_setup(CPU_ON, cp->cpu_id, NULL);
	} while ((cp = cp->cpu_next_onln) != cpu_active);

	register_cpu_setup_func(clock_tick_cpu_setup, NULL);

	mutex_exit(&cpu_lock);
}