common/os/clock.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved	*/

/*
 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
 */

#include <sys/param.h>
#include <sys/t_lock.h>
#include <sys/types.h>
#include <sys/tuneable.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/cpuvar.h>
#include <sys/lgrp.h>
#include <sys/user.h>
#include <sys/proc.h>
#include <sys/callo.h>
#include <sys/kmem.h>
#include <sys/var.h>
#include <sys/cmn_err.h>
#include <sys/swap.h>
#include <sys/vmsystm.h>
#include <sys/class.h>
#include <sys/time.h>
#include <sys/debug.h>
#include <sys/vtrace.h>
#include <sys/spl.h>
#include <sys/atomic.h>
#include <sys/dumphdr.h>
#include <sys/archsystm.h>
#include <sys/fs/swapnode.h>
#include <sys/panic.h>
#include <sys/disp.h>
#include <sys/msacct.h>
#include <sys/mem_cage.h>

#include <vm/page.h>
#include <vm/anon.h>
#include <vm/rm.h>
#include <sys/cyclic.h>
#include <sys/cpupart.h>
#include <sys/rctl.h>
#include <sys/task.h>
#include <sys/sdt.h>
#include <sys/ddi_periodic.h>
#include <sys/random.h>
#include <sys/modctl.h>
#include <sys/zone.h>

/*
 * for NTP support
 */
#include <sys/timex.h>
#include <sys/inttypes.h>

#include <sys/sunddi.h>
#include <sys/clock_impl.h>

/*
 * clock() is called straight from the clock cyclic; see clock_init().
 *
 * Functions:
 *	reprime clock
 *	maintain date
 *	jab the scheduler
 */

extern kcondvar_t	fsflush_cv;
extern sysinfo_t	sysinfo;
extern vminfo_t	vminfo;
extern int	idleswtch;	/* flag set while idle in pswtch() */
extern hrtime_t volatile devinfo_freeze;

/*
 * high-precision avenrun values.  These are needed to make the
 * regular avenrun values accurate.
 */
static uint64_t hp_avenrun[3];
int	avenrun[3];		/* FSCALED average run queue lengths */
time_t	time;	/* time in seconds since 1970 - for compatibility only */

static struct loadavg_s loadavg;
/*
 * Phase/frequency-lock loop (PLL/FLL) definitions
 *
 * The following variables are read and set by the ntp_adjtime() system
 * call.
 *
 * time_state shows the state of the system clock, with values defined
 * in the timex.h header file.
 *
 * time_status shows the status of the system clock, with bits defined
 * in the timex.h header file.
 *
 * time_offset is used by the PLL/FLL to adjust the system time in small
 * increments.
 *
 * time_constant determines the bandwidth or "stiffness" of the PLL.
 *
 * time_tolerance determines maximum frequency error or tolerance of the
 * CPU clock oscillator and is a property of the architecture; however,
 * in principle it could change as result of the presence of external
 * discipline signals, for instance.
 *
 * time_precision is usually equal to the kernel tick variable; however,
 * in cases where a precision clock counter or external clock is
 * available, the resolution can be much less than this and depend on
 * whether the external clock is working or not.
 *
 * time_maxerror is initialized by a ntp_adjtime() call and increased by
 * the kernel once each second to reflect the maximum error bound
 * growth.
 *
 * time_esterror is set and read by the ntp_adjtime() call, but
 * otherwise not used by the kernel.
 */
int32_t time_state = TIME_OK;	/* clock state */
int32_t time_status = STA_UNSYNC;	/* clock status bits */
int32_t time_offset = 0;		/* time offset (us) */
int32_t time_constant = 0;		/* pll time constant */
int32_t time_tolerance = MAXFREQ;	/* frequency tolerance (scaled ppm) */
int32_t time_precision = 1;	/* clock precision (us) */
int32_t time_maxerror = MAXPHASE;	/* maximum error (us) */
int32_t time_esterror = MAXPHASE;	/* estimated error (us) */

/*
 * The following variables establish the state of the PLL/FLL and the
 * residual time and frequency offset of the local clock. The scale
 * factors are defined in the timex.h header file.
 *
 * time_phase and time_freq are the phase increment and the frequency
 * increment, respectively, of the kernel time variable.
 *
 * time_freq is set via ntp_adjtime() from a value stored in a file when
 * the synchronization daemon is first started. Its value is retrieved
 * via ntp_adjtime() and written to the file about once per hour by the
 * daemon.
 *
 * time_adj is the adjustment added to the value of tick at each timer
 * interrupt and is recomputed from time_phase and time_freq at each
 * seconds rollover.
 *
 * time_reftime is the second's portion of the system time at the last
 * call to ntp_adjtime(). It is used to adjust the time_freq variable
 * and to increase the time_maxerror as the time since last update
 * increases.
 */
int32_t time_phase = 0;		/* phase offset (scaled us) */
int32_t time_freq = 0;		/* frequency offset (scaled ppm) */
int32_t time_adj = 0;		/* tick adjust (scaled 1 / hz) */
int32_t time_reftime = 0;		/* time at last adjustment (s) */

/*
 * The scale factors of the following variables are defined in the
 * timex.h header file.
 *
 * pps_time contains the time at each calibration interval, as read by
 * microtime(). pps_count counts the seconds of the calibration
 * interval, the duration of which is nominally pps_shift in powers of
 * two.
 *
 * pps_offset is the time offset produced by the time median filter
 * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
 * this filter.
 *
 * pps_freq is the frequency offset produced by the frequency median
 * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
 * by this filter.
 *
 * pps_usec is latched from a high resolution counter or external clock
 * at pps_time. Here we want the hardware counter contents only, not the
 * contents plus the time_tv.usec as usual.
 *
 * pps_valid counts the number of seconds since the last PPS update. It
 * is used as a watchdog timer to disable the PPS discipline should the
 * PPS signal be lost.
 *
 * pps_glitch counts the number of seconds since the beginning of an
 * offset burst more than tick/2 from current nominal offset. It is used
 * mainly to suppress error bursts due to priority conflicts between the
 * PPS interrupt and timer interrupt.
 *
 * pps_intcnt counts the calibration intervals for use in the interval-
 * adaptation algorithm. It's just too complicated for words.
 */
struct timeval pps_time;	/* kernel time at last interval */
int32_t pps_tf[] = {0, 0, 0};	/* pps time offset median filter (us) */
int32_t pps_offset = 0;		/* pps time offset (us) */
int32_t pps_jitter = MAXTIME;	/* time dispersion (jitter) (us) */
int32_t pps_ff[] = {0, 0, 0};	/* pps frequency offset median filter */
int32_t pps_freq = 0;		/* frequency offset (scaled ppm) */
int32_t pps_stabil = MAXFREQ;	/* frequency dispersion (scaled ppm) */
int32_t pps_usec = 0;		/* microsec counter at last interval */
int32_t pps_valid = PPS_VALID;	/* pps signal watchdog counter */
int32_t pps_glitch = 0;		/* pps signal glitch counter */
int32_t pps_count = 0;		/* calibration interval counter (s) */
int32_t pps_shift = PPS_SHIFT;	/* interval duration (s) (shift) */
int32_t pps_intcnt = 0;		/* intervals at current duration */

/*
 * PPS signal quality monitors
 *
 * pps_jitcnt counts the seconds that have been discarded because the
 * jitter measured by the time median filter exceeds the limit MAXTIME
 * (100 us).
 *
 * pps_calcnt counts the frequency calibration intervals, which are
 * variable from 4 s to 256 s.
 *
 * pps_errcnt counts the calibration intervals which have been discarded
 * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
 * calibration interval jitter exceeds two ticks.
 *
 * pps_stbcnt counts the calibration intervals that have been discarded
 * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
 */
int32_t pps_jitcnt = 0;		/* jitter limit exceeded */
int32_t pps_calcnt = 0;		/* calibration intervals */
int32_t pps_errcnt = 0;		/* calibration errors */
int32_t pps_stbcnt = 0;		/* stability limit exceeded */

kcondvar_t lbolt_cv;

/*
 * Hybrid lbolt implementation:
 *
 * The service historically provided by the lbolt and lbolt64 variables has
 * been replaced by the ddi_get_lbolt() and ddi_get_lbolt64() routines, and the
 * original symbols removed from the system. The once clock driven variables are
 * now implemented in an event driven fashion, backed by gethrtime() coarsed to
 * the appropriate clock resolution. The default event driven implementation is
 * complemented by a cyclic driven one, active only during periods of intense
 * activity around the DDI lbolt routines, when a lbolt specific cyclic is
 * reprogramed to fire at a clock tick interval to serve consumers of lbolt who
 * rely on the original low cost of consulting a memory position.
 *
 * The implementation uses the number of calls to these routines and the
 * frequency of these to determine when to transition from event to cyclic
 * driven and vice-versa. These values are kept on a per CPU basis for
 * scalability reasons and to prevent CPUs from constantly invalidating a single
 * cache line when modifying a global variable. The transition from event to
 * cyclic mode happens once the thresholds are crossed, and activity on any CPU
 * can cause such transition.
 *
 * The lbolt_hybrid function pointer is called by ddi_get_lbolt() and
 * ddi_get_lbolt64(), and will point to lbolt_event_driven() or
 * lbolt_cyclic_driven() according to the current mode. When the thresholds
 * are exceeded, lbolt_event_driven() will reprogram the lbolt cyclic to
 * fire at a nsec_per_tick interval and increment an internal variable at
 * each firing. lbolt_hybrid will then point to lbolt_cyclic_driven(), which
 * will simply return the value of such variable. lbolt_cyclic() will attempt
 * to shut itself off at each threshold interval (sampling period for calls
 * to the DDI lbolt routines), and return to the event driven mode, but will
 * be prevented from doing so if lbolt_cyclic_driven() is being heavily used.
 *
 * lbolt_bootstrap is used during boot to serve lbolt consumers who don't wait
 * for the cyclic subsystem to be intialized.
 *
 */
int64_t lbolt_bootstrap(void);
int64_t lbolt_event_driven(void);
int64_t lbolt_cyclic_driven(void);
int64_t (*lbolt_hybrid)(void) = lbolt_bootstrap;
uint_t lbolt_ev_to_cyclic(caddr_t, caddr_t);

/*
 * lbolt's cyclic, installed by clock_init().
 */
static void lbolt_cyclic(void);

/*
 * Tunable to keep lbolt in cyclic driven mode. This will prevent the system
 * from switching back to event driven, once it reaches cyclic mode.
 */
static boolean_t lbolt_cyc_only = B_FALSE;

/*
 * Cache aligned, per CPU structure with lbolt usage statistics.
 */
static lbolt_cpu_t *lb_cpu;

/*
 * Single, cache aligned, structure with all the information required by
 * the lbolt implementation.
 */
lbolt_info_t *lb_info;


int one_sec = 1; /* turned on once every second */
static int fsflushcnt;	/* counter for t_fsflushr */
int	dosynctodr = 1;	/* patchable; enable/disable sync to TOD chip */
int	tod_needsync = 0;	/* need to sync tod chip with software time */
static int tod_broken = 0;	/* clock chip doesn't work */
time_t	boot_time = 0;		/* Boot time in seconds since 1970 */
cyclic_id_t clock_cyclic;	/* clock()'s cyclic_id */
cyclic_id_t deadman_cyclic;	/* deadman()'s cyclic_id */

extern void	clock_tick_schedule(int);

static int lgrp_ticks;		/* counter to schedule lgrp load calcs */

/*
 * for tod fault detection
 */
#define	TOD_REF_FREQ		((longlong_t)(NANOSEC))
#define	TOD_STALL_THRESHOLD	(TOD_REF_FREQ * 3 / 2)
#define	TOD_JUMP_THRESHOLD	(TOD_REF_FREQ / 2)
#define	TOD_FILTER_N		4
#define	TOD_FILTER_SETTLE	(4 * TOD_FILTER_N)
static enum tod_fault_type tod_faulted = TOD_NOFAULT;

static int tod_status_flag = 0;		/* used by tod_validate() */

static hrtime_t prev_set_tick = 0;	/* gethrtime() prior to tod_set() */
static time_t prev_set_tod = 0;		/* tv_sec value passed to tod_set() */

/* patchable via /etc/system */
int tod_validate_enable = 1;

/* Diagnose/Limit messages about delay(9F) called from interrupt context */
int			delay_from_interrupt_diagnose = 0;
volatile uint32_t	delay_from_interrupt_msg = 20;

/*
 * On non-SPARC systems, TOD validation must be deferred until gethrtime
 * returns non-zero values (after mach_clkinit's execution).
 * On SPARC systems, it must be deferred until after hrtime_base
 * and hres_last_tick are set (in the first invocation of hres_tick).
 * Since in both cases the prerequisites occur before the invocation of
 * tod_get() in clock(), the deferment is lifted there.
 */
static boolean_t tod_validate_deferred = B_TRUE;

/*
 * tod_fault_table[] must be aligned with
 * enum tod_fault_type in systm.h
 */
static char *tod_fault_table[] = {
	"Reversed",			/* TOD_REVERSED */
	"Stalled",			/* TOD_STALLED */
	"Jumped",			/* TOD_JUMPED */
	"Changed in Clock Rate",	/* TOD_RATECHANGED */
	"Is Read-Only"			/* TOD_RDONLY */
	/*
	 * no strings needed for TOD_NOFAULT
	 */
};

/*
 * test hook for tod broken detection in tod_validate
 */
int tod_unit_test = 0;
time_t tod_test_injector;

#define	CLOCK_ADJ_HIST_SIZE	4

static int	adj_hist_entry;

int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];

static void calcloadavg(int, uint64_t *);
static int genloadavg(struct loadavg_s *);
static void loadavg_update();

void (*cmm_clock_callout)() = NULL;
void (*cpucaps_clock_callout)() = NULL;

extern clock_t clock_tick_proc_max;

static int64_t deadman_counter = 0;

static void
clock(void)
{
	kthread_t	*t;
	uint_t	nrunnable;
	uint_t	w_io;
	cpu_t	*cp;
	cpupart_t *cpupart;
	extern	void	set_freemem();
	void	(*funcp)();
	int32_t ltemp;
	int64_t lltemp;
	int s;
	int do_lgrp_load;
	int i;
	clock_t now = LBOLT_NO_ACCOUNT;	/* current tick */

	if (panicstr)
		return;

	/*
	 * Make sure that 'freemem' do not drift too far from the truth
	 */
	set_freemem();


	/*
	 * Before the section which is repeated is executed, we do
	 * the time delta processing which occurs every clock tick
	 *
	 * There is additional processing which happens every time
	 * the nanosecond counter rolls over which is described
	 * below - see the section which begins with : if (one_sec)
	 *
	 * This section marks the beginning of the precision-kernel
	 * code fragment.
	 *
	 * First, compute the phase adjustment. If the low-order bits
	 * (time_phase) of the update overflow, bump the higher order
	 * bits (time_update).
	 */
	time_phase += time_adj;
	if (time_phase <= -FINEUSEC) {
		ltemp = -time_phase / SCALE_PHASE;
		time_phase += ltemp * SCALE_PHASE;
		s = hr_clock_lock();
		timedelta -= ltemp * (NANOSEC/MICROSEC);
		hr_clock_unlock(s);
	} else if (time_phase >= FINEUSEC) {
		ltemp = time_phase / SCALE_PHASE;
		time_phase -= ltemp * SCALE_PHASE;
		s = hr_clock_lock();
		timedelta += ltemp * (NANOSEC/MICROSEC);
		hr_clock_unlock(s);
	}

	/*
	 * End of precision-kernel code fragment which is processed
	 * every timer interrupt.
	 *
	 * Continue with the interrupt processing as scheduled.
	 */
	/*
	 * Count the number of runnable threads and the number waiting
	 * for some form of I/O to complete -- gets added to
	 * sysinfo.waiting.  To know the state of the system, must add
	 * wait counts from all CPUs.  Also add up the per-partition
	 * statistics.
	 */
	w_io = 0;
	nrunnable = 0;

	/*
	 * keep track of when to update lgrp/part loads
	 */

	do_lgrp_load = 0;
	if (lgrp_ticks++ >= hz / 10) {
		lgrp_ticks = 0;
		do_lgrp_load = 1;
	}

	if (one_sec) {
		loadavg_update();
		deadman_counter++;
	}

	/*
	 * First count the threads waiting on kpreempt queues in each
	 * CPU partition.
	 */

	cpupart = cp_list_head;
	do {
		uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;

		cpupart->cp_updates++;
		nrunnable += cpupart_nrunnable;
		cpupart->cp_nrunnable_cum += cpupart_nrunnable;
		if (one_sec) {
			cpupart->cp_nrunning = 0;
			cpupart->cp_nrunnable = cpupart_nrunnable;
		}
	} while ((cpupart = cpupart->cp_next) != cp_list_head);


	/* Now count the per-CPU statistics. */
	cp = cpu_list;
	do {
		uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;

		nrunnable += cpu_nrunnable;
		cpupart = cp->cpu_part;
		cpupart->cp_nrunnable_cum += cpu_nrunnable;
		if (one_sec) {
			cpupart->cp_nrunnable += cpu_nrunnable;
			/*
			 * Update user, system, and idle cpu times.
			 */
			cpupart->cp_nrunning++;
			/*
			 * w_io is used to update sysinfo.waiting during
			 * one_second processing below.  Only gather w_io
			 * information when we walk the list of cpus if we're
			 * going to perform one_second processing.
			 */
			w_io += CPU_STATS(cp, sys.iowait);
		}

		if (one_sec && (cp->cpu_flags & CPU_EXISTS)) {
			int i, load, change;
			hrtime_t intracct, intrused;
			const hrtime_t maxnsec = 1000000000;
			const int precision = 100;

			/*
			 * Estimate interrupt load on this cpu each second.
			 * Computes cpu_intrload as %utilization (0-99).
			 */

			/* add up interrupt time from all micro states */
			for (intracct = 0, i = 0; i < NCMSTATES; i++)
				intracct += cp->cpu_intracct[i];
			scalehrtime(&intracct);

			/* compute nsec used in the past second */
			intrused = intracct - cp->cpu_intrlast;
			cp->cpu_intrlast = intracct;

			/* limit the value for safety (and the first pass) */
			if (intrused >= maxnsec)
				intrused = maxnsec - 1;

			/* calculate %time in interrupt */
			load = (precision * intrused) / maxnsec;
			ASSERT(load >= 0 && load < precision);
			change = cp->cpu_intrload - load;

			/* jump to new max, or decay the old max */
			if (change < 0)
				cp->cpu_intrload = load;
			else if (change > 0)
				cp->cpu_intrload -= (change + 3) / 4;

			DTRACE_PROBE3(cpu_intrload,
			    cpu_t *, cp,
			    hrtime_t, intracct,
			    hrtime_t, intrused);
		}

		if (do_lgrp_load &&
		    (cp->cpu_flags & CPU_EXISTS)) {
			/*
			 * When updating the lgroup's load average,
			 * account for the thread running on the CPU.
			 * If the CPU is the current one, then we need
			 * to account for the underlying thread which
			 * got the clock interrupt not the thread that is
			 * handling the interrupt and caculating the load
			 * average
			 */
			t = cp->cpu_thread;
			if (CPU == cp)
				t = t->t_intr;

			/*
			 * Account for the load average for this thread if
			 * it isn't the idle thread or it is on the interrupt
			 * stack and not the current CPU handling the clock
			 * interrupt
			 */
			if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
			    CPU_ON_INTR(cp))) {
				if (t->t_lpl == cp->cpu_lpl) {
					/* local thread */
					cpu_nrunnable++;
				} else {
					/*
					 * This is a remote thread, charge it
					 * against its home lgroup.  Note that
					 * we notice that a thread is remote
					 * only if it's currently executing.
					 * This is a reasonable approximation,
					 * since queued remote threads are rare.
					 * Note also that if we didn't charge
					 * it to its home lgroup, remote
					 * execution would often make a system
					 * appear balanced even though it was
					 * not, and thread placement/migration
					 * would often not be done correctly.
					 */
					lgrp_loadavg(t->t_lpl,
					    LGRP_LOADAVG_IN_THREAD_MAX, 0);
				}
			}
			lgrp_loadavg(cp->cpu_lpl,
			    cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
		}
	} while ((cp = cp->cpu_next) != cpu_list);

	clock_tick_schedule(one_sec);

	/*
	 * Check for a callout that needs be called from the clock
	 * thread to support the membership protocol in a clustered
	 * system.  Copy the function pointer so that we can reset
	 * this to NULL if needed.
	 */
	if ((funcp = cmm_clock_callout) != NULL)
		(*funcp)();

	if ((funcp = cpucaps_clock_callout) != NULL)
		(*funcp)();

	/*
	 * Wakeup the cageout thread waiters once per second.
	 */
	if (one_sec)
		kcage_tick();

	if (one_sec) {

		int drift, absdrift;
		timestruc_t tod;
		int s;

		/*
		 * Beginning of precision-kernel code fragment executed
		 * every second.
		 *
		 * On rollover of the second the phase adjustment to be
		 * used for the next second is calculated.  Also, the
		 * maximum error is increased by the tolerance.  If the
		 * PPS frequency discipline code is present, the phase is
		 * increased to compensate for the CPU clock oscillator
		 * frequency error.
		 *
		 * On a 32-bit machine and given parameters in the timex.h
		 * header file, the maximum phase adjustment is +-512 ms
		 * and maximum frequency offset is (a tad less than)
		 * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
		 */
		time_maxerror += time_tolerance / SCALE_USEC;

		/*
		 * Leap second processing. If in leap-insert state at
		 * the end of the day, the system clock is set back one
		 * second; if in leap-delete state, the system clock is
		 * set ahead one second. The microtime() routine or
		 * external clock driver will insure that reported time
		 * is always monotonic. The ugly divides should be
		 * replaced.
		 */
		switch (time_state) {

		case TIME_OK:
			if (time_status & STA_INS)
				time_state = TIME_INS;
			else if (time_status & STA_DEL)
				time_state = TIME_DEL;
			break;

		case TIME_INS:
			if (hrestime.tv_sec % 86400 == 0) {
				s = hr_clock_lock();
				hrestime.tv_sec--;
				hr_clock_unlock(s);
				time_state = TIME_OOP;
			}
			break;

		case TIME_DEL:
			if ((hrestime.tv_sec + 1) % 86400 == 0) {
				s = hr_clock_lock();
				hrestime.tv_sec++;
				hr_clock_unlock(s);
				time_state = TIME_WAIT;
			}
			break;

		case TIME_OOP:
			time_state = TIME_WAIT;
			break;

		case TIME_WAIT:
			if (!(time_status & (STA_INS | STA_DEL)))
				time_state = TIME_OK;
		default:
			break;
		}

		/*
		 * Compute the phase adjustment for the next second. In
		 * PLL mode, the offset is reduced by a fixed factor
		 * times the time constant. In FLL mode the offset is
		 * used directly. In either mode, the maximum phase
		 * adjustment for each second is clamped so as to spread
		 * the adjustment over not more than the number of
		 * seconds between updates.
		 */
		if (time_offset == 0)
			time_adj = 0;
		else if (time_offset < 0) {
			lltemp = -time_offset;
			if (!(time_status & STA_FLL)) {
				if ((1 << time_constant) >= SCALE_KG)
					lltemp *= (1 << time_constant) /
					    SCALE_KG;
				else
					lltemp = (lltemp / SCALE_KG) >>
					    time_constant;
			}
			if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
				lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
			time_offset += lltemp;
			time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
		} else {
			lltemp = time_offset;
			if (!(time_status & STA_FLL)) {
				if ((1 << time_constant) >= SCALE_KG)
					lltemp *= (1 << time_constant) /
					    SCALE_KG;
				else
					lltemp = (lltemp / SCALE_KG) >>
					    time_constant;
			}
			if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
				lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
			time_offset -= lltemp;
			time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
		}

		/*
		 * Compute the frequency estimate and additional phase
		 * adjustment due to frequency error for the next
		 * second. When the PPS signal is engaged, gnaw on the
		 * watchdog counter and update the frequency computed by
		 * the pll and the PPS signal.
		 */
		pps_valid++;
		if (pps_valid == PPS_VALID) {
			pps_jitter = MAXTIME;
			pps_stabil = MAXFREQ;
			time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
			    STA_PPSWANDER | STA_PPSERROR);
		}
		lltemp = time_freq + pps_freq;

		if (lltemp)
			time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);

		/*
		 * End of precision kernel-code fragment
		 *
		 * The section below should be modified if we are planning
		 * to use NTP for synchronization.
		 *
		 * Note: the clock synchronization code now assumes
		 * the following:
		 *   - if dosynctodr is 1, then compute the drift between
		 *	the tod chip and software time and adjust one or
		 *	the other depending on the circumstances
		 *
		 *   - if dosynctodr is 0, then the tod chip is independent
		 *	of the software clock and should not be adjusted,
		 *	but allowed to free run.  this allows NTP to sync.
		 *	hrestime without any interference from the tod chip.
		 */

		tod_validate_deferred = B_FALSE;
		mutex_enter(&tod_lock);
		tod = tod_get();
		drift = tod.tv_sec - hrestime.tv_sec;
		absdrift = (drift >= 0) ? drift : -drift;
		if (tod_needsync || absdrift > 1) {
			int s;
			if (absdrift > 2) {
				if (!tod_broken && tod_faulted == TOD_NOFAULT) {
					s = hr_clock_lock();
					hrestime = tod;
					membar_enter();	/* hrestime visible */
					timedelta = 0;
					timechanged++;
					tod_needsync = 0;
					hr_clock_unlock(s);
					callout_hrestime();

				}
			} else {
				if (tod_needsync || !dosynctodr) {
					gethrestime(&tod);
					tod_set(tod);
					s = hr_clock_lock();
					if (timedelta == 0)
						tod_needsync = 0;
					hr_clock_unlock(s);
				} else {
					/*
					 * If the drift is 2 seconds on the
					 * money, then the TOD is adjusting
					 * the clock;  record that.
					 */
					clock_adj_hist[adj_hist_entry++ %
					    CLOCK_ADJ_HIST_SIZE] = now;
					s = hr_clock_lock();
					timedelta = (int64_t)drift*NANOSEC;
					hr_clock_unlock(s);
				}
			}
		}
		one_sec = 0;
		time = gethrestime_sec();  /* for crusty old kmem readers */
		mutex_exit(&tod_lock);

		/*
		 * Some drivers still depend on this... XXX
		 */
		cv_broadcast(&lbolt_cv);

		vminfo.freemem += freemem;
		{
			pgcnt_t maxswap, resv, free;
			pgcnt_t avail =
			    MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);

			maxswap = k_anoninfo.ani_mem_resv +
			    k_anoninfo.ani_max +avail;
			/* Update ani_free */
			set_anoninfo();
			free = k_anoninfo.ani_free + avail;
			resv = k_anoninfo.ani_phys_resv +
			    k_anoninfo.ani_mem_resv;

			vminfo.swap_resv += resv;
			/* number of reserved and allocated pages */
#ifdef	DEBUG
			if (maxswap < free)
				cmn_err(CE_WARN, "clock: maxswap < free");
			if (maxswap < resv)
				cmn_err(CE_WARN, "clock: maxswap < resv");
#endif
			vminfo.swap_alloc += maxswap - free;
			vminfo.swap_avail += maxswap - resv;
			vminfo.swap_free += free;
		}
		vminfo.updates++;
		if (nrunnable) {
			sysinfo.runque += nrunnable;
			sysinfo.runocc++;
		}
		if (nswapped) {
			sysinfo.swpque += nswapped;
			sysinfo.swpocc++;
		}
		sysinfo.waiting += w_io;
		sysinfo.updates++;

		/*
		 * Wake up fsflush to write out DELWRI
		 * buffers, dirty pages and other cached
		 * administrative data, e.g. inodes.
		 */
		if (--fsflushcnt <= 0) {
			fsflushcnt = tune.t_fsflushr;
			cv_signal(&fsflush_cv);
		}

		vmmeter();
		calcloadavg(genloadavg(&loadavg), hp_avenrun);
		for (i = 0; i < 3; i++)
			/*
			 * At the moment avenrun[] can only hold 31
			 * bits of load average as it is a signed
			 * int in the API. We need to ensure that
			 * hp_avenrun[i] >> (16 - FSHIFT) will not be
			 * too large. If it is, we put the largest value
			 * that we can use into avenrun[i]. This is
			 * kludgey, but about all we can do until we
			 * avenrun[] is declared as an array of uint64[]
			 */
			if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
				avenrun[i] = (int32_t)(hp_avenrun[i] >>
				    (16 - FSHIFT));
			else
				avenrun[i] = 0x7fffffff;

		cpupart = cp_list_head;
		do {
			calcloadavg(genloadavg(&cpupart->cp_loadavg),
			    cpupart->cp_hp_avenrun);
		} while ((cpupart = cpupart->cp_next) != cp_list_head);

		/*
		 * Wake up the swapper thread if necessary.
		 */
		if (runin ||
		    (runout && (avefree < desfree || wake_sched_sec))) {
			t = &t0;
			thread_lock(t);
			if (t->t_state == TS_STOPPED) {
				runin = runout = 0;
				wake_sched_sec = 0;
				t->t_whystop = 0;
				t->t_whatstop = 0;
				t->t_schedflag &= ~TS_ALLSTART;
				THREAD_TRANSITION(t);
				setfrontdq(t);
			}
			thread_unlock(t);
		}
	}

	/*
	 * Wake up the swapper if any high priority swapped-out threads
	 * became runable during the last tick.
	 */
	if (wake_sched) {
		t = &t0;
		thread_lock(t);
		if (t->t_state == TS_STOPPED) {
			runin = runout = 0;
			wake_sched = 0;
			t->t_whystop = 0;
			t->t_whatstop = 0;
			t->t_schedflag &= ~TS_ALLSTART;
			THREAD_TRANSITION(t);
			setfrontdq(t);
		}
		thread_unlock(t);
	}
}

void
clock_init(void)
{
	cyc_handler_t clk_hdlr, lbolt_hdlr;
	cyc_time_t clk_when, lbolt_when;
	int i, sz;
	intptr_t buf;

	/*
	 * Setup handler and timer for the clock cyclic.
	 */
	clk_hdlr.cyh_func = (cyc_func_t)clock;
	clk_hdlr.cyh_level = CY_LOCK_LEVEL;
	clk_hdlr.cyh_arg = NULL;

	clk_when.cyt_when = 0;
	clk_when.cyt_interval = nsec_per_tick;

	/*
	 * The lbolt cyclic will be reprogramed to fire at a nsec_per_tick
	 * interval to satisfy performance needs of the DDI lbolt consumers.
	 * It is off by default.
	 */
	lbolt_hdlr.cyh_func = (cyc_func_t)lbolt_cyclic;
	lbolt_hdlr.cyh_level = CY_LOCK_LEVEL;
	lbolt_hdlr.cyh_arg = NULL;

	lbolt_when.cyt_interval = nsec_per_tick;

	/*
	 * Allocate cache line aligned space for the per CPU lbolt data and
	 * lbolt info structures, and initialize them with their default
	 * values. Note that these structures are also cache line sized.
	 */
	sz = sizeof (lbolt_info_t) + CPU_CACHE_COHERENCE_SIZE;
	buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
	lb_info = (lbolt_info_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);

	if (hz != HZ_DEFAULT)
		lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL *
		    hz/HZ_DEFAULT;
	else
		lb_info->lbi_thresh_interval = LBOLT_THRESH_INTERVAL;

	lb_info->lbi_thresh_calls = LBOLT_THRESH_CALLS;

	sz = (sizeof (lbolt_cpu_t) * max_ncpus) + CPU_CACHE_COHERENCE_SIZE;
	buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
	lb_cpu = (lbolt_cpu_t *)P2ROUNDUP(buf, CPU_CACHE_COHERENCE_SIZE);

	for (i = 0; i < max_ncpus; i++)
		lb_cpu[i].lbc_counter = lb_info->lbi_thresh_calls;

	/*
	 * Install the softint used to switch between event and cyclic driven
	 * lbolt. We use a soft interrupt to make sure the context of the
	 * cyclic reprogram call is safe.
	 */
	lbolt_softint_add();

	/*
	 * Since the hybrid lbolt implementation is based on a hardware counter
	 * that is reset at every hardware reboot and that we'd like to have
	 * the lbolt value starting at zero after both a hardware and a fast
	 * reboot, we calculate the number of clock ticks the system's been up
	 * and store it in the lbi_debug_time field of the lbolt info structure.
	 * The value of this field will be subtracted from lbolt before
	 * returning it.
	 */
	lb_info->lbi_internal = lb_info->lbi_debug_time =
	    (gethrtime()/nsec_per_tick);

	/*
	 * lbolt_hybrid points at lbolt_bootstrap until now. The LBOLT_* macros
	 * and lbolt_debug_{enter,return} use this value as an indication that
	 * the initializaion above hasn't been completed. Setting lbolt_hybrid
	 * to either lbolt_{cyclic,event}_driven here signals those code paths
	 * that the lbolt related structures can be used.
	 */
	if (lbolt_cyc_only) {
		lbolt_when.cyt_when = 0;
		lbolt_hybrid = lbolt_cyclic_driven;
	} else {
		lbolt_when.cyt_when = CY_INFINITY;
		lbolt_hybrid = lbolt_event_driven;
	}

	/*
	 * Grab cpu_lock and install all three cyclics.
	 */
	mutex_enter(&cpu_lock);

	clock_cyclic = cyclic_add(&clk_hdlr, &clk_when);
	lb_info->id.lbi_cyclic_id = cyclic_add(&lbolt_hdlr, &lbolt_when);

	mutex_exit(&cpu_lock);
}

/*
 * Called before calcloadavg to get 10-sec moving loadavg together
 */

static int
genloadavg(struct loadavg_s *avgs)
{
	int avg;
	int spos; /* starting position */
	int cpos; /* moving current position */
	int i;
	int slen;
	hrtime_t hr_avg;

	/* 10-second snapshot, calculate first positon */
	if (avgs->lg_len == 0) {
		return (0);
	}
	slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;

	spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
	    S_LOADAVG_SZ + (avgs->lg_cur - 1);
	for (i = hr_avg = 0; i < slen; i++) {
		cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
		hr_avg += avgs->lg_loads[cpos];
	}

	hr_avg = hr_avg / slen;
	avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);

	return (avg);
}

/*
 * Run every second from clock () to update the loadavg count available to the
 * system and cpu-partitions.
 *
 * This works by sampling the previous usr, sys, wait time elapsed,
 * computing a delta, and adding that delta to the elapsed usr, sys,
 * wait increase.
 */

static void
loadavg_update()
{
	cpu_t *cp;
	cpupart_t *cpupart;
	hrtime_t cpu_total;
	int prev;

	cp = cpu_list;
	loadavg.lg_total = 0;

	/*
	 * first pass totals up per-cpu statistics for system and cpu
	 * partitions
	 */

	do {
		struct loadavg_s *lavg;

		lavg = &cp->cpu_loadavg;

		cpu_total = cp->cpu_acct[CMS_USER] +
		    cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
		/* compute delta against last total */
		scalehrtime(&cpu_total);
		prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
		    S_LOADAVG_SZ + (lavg->lg_cur - 1);
		if (lavg->lg_loads[prev] <= 0) {
			lavg->lg_loads[lavg->lg_cur] = cpu_total;
			cpu_total = 0;
		} else {
			lavg->lg_loads[lavg->lg_cur] = cpu_total;
			cpu_total = cpu_total - lavg->lg_loads[prev];
			if (cpu_total < 0)
				cpu_total = 0;
		}

		lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
		lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
		    lavg->lg_len + 1 : S_LOADAVG_SZ;

		loadavg.lg_total += cpu_total;
		cp->cpu_part->cp_loadavg.lg_total += cpu_total;

	} while ((cp = cp->cpu_next) != cpu_list);

	loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
	loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
	loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
	    loadavg.lg_len + 1 : S_LOADAVG_SZ;
	/*
	 * Second pass updates counts
	 */
	cpupart = cp_list_head;

	do {
		struct loadavg_s *lavg;

		lavg = &cpupart->cp_loadavg;
		lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
		lavg->lg_total = 0;
		lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
		lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
		    lavg->lg_len + 1 : S_LOADAVG_SZ;

	} while ((cpupart = cpupart->cp_next) != cp_list_head);

	/*
	 * Third pass totals up per-zone statistics.
	 */
	zone_loadavg_update();
}

/*
 * clock_update() - local clock update
 *
 * This routine is called by ntp_adjtime() to update the local clock
 * phase and frequency. The implementation is of an
 * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
 * routine computes new time and frequency offset estimates for each
 * call.  The PPS signal itself determines the new time offset,
 * instead of the calling argument.  Presumably, calls to
 * ntp_adjtime() occur only when the caller believes the local clock
 * is valid within some bound (+-128 ms with NTP). If the caller's
 * time is far different than the PPS time, an argument will ensue,
 * and it's not clear who will lose.
 *
 * For uncompensated quartz crystal oscillatores and nominal update
 * intervals less than 1024 s, operation should be in phase-lock mode
 * (STA_FLL = 0), where the loop is disciplined to phase. For update
 * intervals greater than this, operation should be in frequency-lock
 * mode (STA_FLL = 1), where the loop is disciplined to frequency.
 *
 * Note: mutex(&tod_lock) is in effect.
 */
void
clock_update(int offset)
{
	int ltemp, mtemp, s;

	ASSERT(MUTEX_HELD(&tod_lock));

	if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
		return;
	ltemp = offset;
	if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
		ltemp = pps_offset;

	/*
	 * Scale the phase adjustment and clamp to the operating range.
	 */
	if (ltemp > MAXPHASE)
		time_offset = MAXPHASE * SCALE_UPDATE;
	else if (ltemp < -MAXPHASE)
		time_offset = -(MAXPHASE * SCALE_UPDATE);
	else
		time_offset = ltemp * SCALE_UPDATE;

	/*
	 * Select whether the frequency is to be controlled and in which
	 * mode (PLL or FLL). Clamp to the operating range. Ugly
	 * multiply/divide should be replaced someday.
	 */
	if (time_status & STA_FREQHOLD || time_reftime == 0)
		time_reftime = hrestime.tv_sec;

	mtemp = hrestime.tv_sec - time_reftime;
	time_reftime = hrestime.tv_sec;

	if (time_status & STA_FLL) {
		if (mtemp >= MINSEC) {
			ltemp = ((time_offset / mtemp) * (SCALE_USEC /
			    SCALE_UPDATE));
			if (ltemp)
				time_freq += ltemp / SCALE_KH;
		}
	} else {
		if (mtemp < MAXSEC) {
			ltemp *= mtemp;
			if (ltemp)
				time_freq += (int)(((int64_t)ltemp *
				    SCALE_USEC) / SCALE_KF)
				    / (1 << (time_constant * 2));
		}
	}
	if (time_freq > time_tolerance)
		time_freq = time_tolerance;
	else if (time_freq < -time_tolerance)
		time_freq = -time_tolerance;

	s = hr_clock_lock();
	tod_needsync = 1;
	hr_clock_unlock(s);
}

/*
 * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
 *
 * This routine is called at each PPS interrupt in order to discipline
 * the CPU clock oscillator to the PPS signal. It measures the PPS phase
 * and leaves it in a handy spot for the clock() routine. It
 * integrates successive PPS phase differences and calculates the
 * frequency offset. This is used in clock() to discipline the CPU
 * clock oscillator so that intrinsic frequency error is cancelled out.
 * The code requires the caller to capture the time and hardware counter
 * value at the on-time PPS signal transition.
 *
 * Note that, on some Unix systems, this routine runs at an interrupt
 * priority level higher than the timer interrupt routine clock().
 * Therefore, the variables used are distinct from the clock()
 * variables, except for certain exceptions: The PPS frequency pps_freq
 * and phase pps_offset variables are determined by this routine and
 * updated atomically. The time_tolerance variable can be considered a
 * constant, since it is infrequently changed, and then only when the
 * PPS signal is disabled. The watchdog counter pps_valid is updated
 * once per second by clock() and is atomically cleared in this
 * routine.
 *
 * tvp is the time of the last tick; usec is a microsecond count since the
 * last tick.
 *
 * Note: In Solaris systems, the tick value is actually given by
 *       usec_per_tick.  This is called from the serial driver cdintr(),
 *	 or equivalent, at a high PIL.  Because the kernel keeps a
 *	 highresolution time, the following code can accept either
 *	 the traditional argument pair, or the current highres timestamp
 *       in tvp and zero in usec.
 */
void
ddi_hardpps(struct timeval *tvp, int usec)
{
	int u_usec, v_usec, bigtick;
	time_t cal_sec;
	int cal_usec;

	/*
	 * An occasional glitch can be produced when the PPS interrupt
	 * occurs in the clock() routine before the time variable is
	 * updated. Here the offset is discarded when the difference
	 * between it and the last one is greater than tick/2, but not
	 * if the interval since the first discard exceeds 30 s.
	 */
	time_status |= STA_PPSSIGNAL;
	time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
	pps_valid = 0;
	u_usec = -tvp->tv_usec;
	if (u_usec < -(MICROSEC/2))
		u_usec += MICROSEC;
	v_usec = pps_offset - u_usec;
	if (v_usec < 0)
		v_usec = -v_usec;
	if (v_usec > (usec_per_tick >> 1)) {
		if (pps_glitch > MAXGLITCH) {
			pps_glitch = 0;
			pps_tf[2] = u_usec;
			pps_tf[1] = u_usec;
		} else {
			pps_glitch++;
			u_usec = pps_offset;
		}
	} else
		pps_glitch = 0;

	/*
	 * A three-stage median filter is used to help deglitch the pps
	 * time. The median sample becomes the time offset estimate; the
	 * difference between the other two samples becomes the time
	 * dispersion (jitter) estimate.
	 */
	pps_tf[2] = pps_tf[1];
	pps_tf[1] = pps_tf[0];
	pps_tf[0] = u_usec;
	if (pps_tf[0] > pps_tf[1]) {
		if (pps_tf[1] > pps_tf[2]) {
			pps_offset = pps_tf[1];		/* 0 1 2 */
			v_usec = pps_tf[0] - pps_tf[2];
		} else if (pps_tf[2] > pps_tf[0]) {
			pps_offset = pps_tf[0];		/* 2 0 1 */
			v_usec = pps_tf[2] - pps_tf[1];
		} else {
			pps_offset = pps_tf[2];		/* 0 2 1 */
			v_usec = pps_tf[0] - pps_tf[1];
		}
	} else {
		if (pps_tf[1] < pps_tf[2]) {
			pps_offset = pps_tf[1];		/* 2 1 0 */
			v_usec = pps_tf[2] - pps_tf[0];
		} else  if (pps_tf[2] < pps_tf[0]) {
			pps_offset = pps_tf[0];		/* 1 0 2 */
			v_usec = pps_tf[1] - pps_tf[2];
		} else {
			pps_offset = pps_tf[2];		/* 1 2 0 */
			v_usec = pps_tf[1] - pps_tf[0];
		}
	}
	if (v_usec > MAXTIME)
		pps_jitcnt++;
	v_usec = (v_usec << PPS_AVG) - pps_jitter;
	pps_jitter += v_usec / (1 << PPS_AVG);
	if (pps_jitter > (MAXTIME >> 1))
		time_status |= STA_PPSJITTER;

	/*
	 * During the calibration interval adjust the starting time when
	 * the tick overflows. At the end of the interval compute the
	 * duration of the interval and the difference of the hardware
	 * counters at the beginning and end of the interval. This code
	 * is deliciously complicated by the fact valid differences may
	 * exceed the value of tick when using long calibration
	 * intervals and small ticks. Note that the counter can be
	 * greater than tick if caught at just the wrong instant, but
	 * the values returned and used here are correct.
	 */
	bigtick = (int)usec_per_tick * SCALE_USEC;
	pps_usec -= pps_freq;
	if (pps_usec >= bigtick)
		pps_usec -= bigtick;
	if (pps_usec < 0)
		pps_usec += bigtick;
	pps_time.tv_sec++;
	pps_count++;
	if (pps_count < (1 << pps_shift))
		return;
	pps_count = 0;
	pps_calcnt++;
	u_usec = usec * SCALE_USEC;
	v_usec = pps_usec - u_usec;
	if (v_usec >= bigtick >> 1)
		v_usec -= bigtick;
	if (v_usec < -(bigtick >> 1))
		v_usec += bigtick;
	if (v_usec < 0)
		v_usec = -(-v_usec >> pps_shift);
	else
		v_usec = v_usec >> pps_shift;
	pps_usec = u_usec;
	cal_sec = tvp->tv_sec;
	cal_usec = tvp->tv_usec;
	cal_sec -= pps_time.tv_sec;
	cal_usec -= pps_time.tv_usec;
	if (cal_usec < 0) {
		cal_usec += MICROSEC;
		cal_sec--;
	}
	pps_time = *tvp;

	/*
	 * Check for lost interrupts, noise, excessive jitter and
	 * excessive frequency error. The number of timer ticks during
	 * the interval may vary +-1 tick. Add to this a margin of one
	 * tick for the PPS signal jitter and maximum frequency
	 * deviation. If the limits are exceeded, the calibration
	 * interval is reset to the minimum and we start over.
	 */
	u_usec = (int)usec_per_tick << 1;
	if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
	    (cal_sec == 0 && cal_usec < u_usec)) ||
	    v_usec > time_tolerance || v_usec < -time_tolerance) {
		pps_errcnt++;
		pps_shift = PPS_SHIFT;
		pps_intcnt = 0;
		time_status |= STA_PPSERROR;
		return;
	}

	/*
	 * A three-stage median filter is used to help deglitch the pps
	 * frequency. The median sample becomes the frequency offset
	 * estimate; the difference between the other two samples
	 * becomes the frequency dispersion (stability) estimate.
	 */
	pps_ff[2] = pps_ff[1];
	pps_ff[1] = pps_ff[0];
	pps_ff[0] = v_usec;
	if (pps_ff[0] > pps_ff[1]) {
		if (pps_ff[1] > pps_ff[2]) {
			u_usec = pps_ff[1];		/* 0 1 2 */
			v_usec = pps_ff[0] - pps_ff[2];
		} else if (pps_ff[2] > pps_ff[0]) {
			u_usec = pps_ff[0];		/* 2 0 1 */
			v_usec = pps_ff[2] - pps_ff[1];
		} else {
			u_usec = pps_ff[2];		/* 0 2 1 */
			v_usec = pps_ff[0] - pps_ff[1];
		}
	} else {
		if (pps_ff[1] < pps_ff[2]) {
			u_usec = pps_ff[1];		/* 2 1 0 */
			v_usec = pps_ff[2] - pps_ff[0];
		} else  if (pps_ff[2] < pps_ff[0]) {
			u_usec = pps_ff[0];		/* 1 0 2 */
			v_usec = pps_ff[1] - pps_ff[2];
		} else {
			u_usec = pps_ff[2];		/* 1 2 0 */
			v_usec = pps_ff[1] - pps_ff[0];
		}
	}

	/*
	 * Here the frequency dispersion (stability) is updated. If it
	 * is less than one-fourth the maximum (MAXFREQ), the frequency
	 * offset is updated as well, but clamped to the tolerance. It
	 * will be processed later by the clock() routine.
	 */
	v_usec = (v_usec >> 1) - pps_stabil;
	if (v_usec < 0)
		pps_stabil -= -v_usec >> PPS_AVG;
	else
		pps_stabil += v_usec >> PPS_AVG;
	if (pps_stabil > MAXFREQ >> 2) {
		pps_stbcnt++;
		time_status |= STA_PPSWANDER;
		return;
	}
	if (time_status & STA_PPSFREQ) {
		if (u_usec < 0) {
			pps_freq -= -u_usec >> PPS_AVG;
			if (pps_freq < -time_tolerance)
				pps_freq = -time_tolerance;
			u_usec = -u_usec;
		} else {
			pps_freq += u_usec >> PPS_AVG;
			if (pps_freq > time_tolerance)
				pps_freq = time_tolerance;
		}
	}

	/*
	 * Here the calibration interval is adjusted. If the maximum
	 * time difference is greater than tick / 4, reduce the interval
	 * by half. If this is not the case for four consecutive
	 * intervals, double the interval.
	 */
	if (u_usec << pps_shift > bigtick >> 2) {
		pps_intcnt = 0;
		if (pps_shift > PPS_SHIFT)
			pps_shift--;
	} else if (pps_intcnt >= 4) {
		pps_intcnt = 0;
		if (pps_shift < PPS_SHIFTMAX)
			pps_shift++;
	} else
		pps_intcnt++;

	/*
	 * If recovering from kmdb, then make sure the tod chip gets resynced.
	 * If we took an early exit above, then we don't yet have a stable
	 * calibration signal to lock onto, so don't mark the tod for sync
	 * until we get all the way here.
	 */
	{
		int s = hr_clock_lock();

		tod_needsync = 1;
		hr_clock_unlock(s);
	}
}

/*
 * Handle clock tick processing for a thread.
 * Check for timer action, enforce CPU rlimit, do profiling etc.
 */
void
clock_tick(kthread_t *t, int pending)
{
	struct proc *pp;
	klwp_id_t    lwp;
	struct as *as;
	clock_t	ticks;
	int	poke = 0;		/* notify another CPU */
	int	user_mode;
	size_t	 rss;
	int i, total_usec, usec;
	rctl_qty_t secs;

	ASSERT(pending > 0);

	/* Must be operating on a lwp/thread */
	if ((lwp = ttolwp(t)) == NULL) {
		panic("clock_tick: no lwp");
		/*NOTREACHED*/
	}

	for (i = 0; i < pending; i++) {
		CL_TICK(t);	/* Class specific tick processing */
		DTRACE_SCHED1(tick, kthread_t *, t);
	}

	pp = ttoproc(t);

	/* pp->p_lock makes sure that the thread does not exit */
	ASSERT(MUTEX_HELD(&pp->p_lock));

	user_mode = (lwp->lwp_state == LWP_USER);

	ticks = (pp->p_utime + pp->p_stime) % hz;
	/*
	 * Update process times. Should use high res clock and state
	 * changes instead of statistical sampling method. XXX
	 */
	if (user_mode) {
		pp->p_utime += pending;
	} else {
		pp->p_stime += pending;
	}

	pp->p_ttime += pending;
	as = pp->p_as;

	/*
	 * Update user profiling statistics. Get the pc from the
	 * lwp when the AST happens.
	 */
	if (pp->p_prof.pr_scale) {
		atomic_add_32(&lwp->lwp_oweupc, (int32_t)pending);
		if (user_mode) {
			poke = 1;
			aston(t);
		}
	}

	/*
	 * If CPU was in user state, process lwp-virtual time
	 * interval timer. The value passed to itimerdecr() has to be
	 * in microseconds and has to be less than one second. Hence
	 * this loop.
	 */
	total_usec = usec_per_tick * pending;
	while (total_usec > 0) {
		usec = MIN(total_usec, (MICROSEC - 1));
		if (user_mode &&
		    timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
		    itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec) == 0) {
			poke = 1;
			sigtoproc(pp, t, SIGVTALRM);
		}
		total_usec -= usec;
	}

	/*
	 * If CPU was in user state, process lwp-profile
	 * interval timer.
	 */
	total_usec = usec_per_tick * pending;
	while (total_usec > 0) {
		usec = MIN(total_usec, (MICROSEC - 1));
		if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
		    itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec) == 0) {
			poke = 1;
			sigtoproc(pp, t, SIGPROF);
		}
		total_usec -= usec;
	}

	/*
	 * Enforce CPU resource controls:
	 *   (a) process.max-cpu-time resource control
	 *
	 * Perform the check only if we have accumulated more a second.
	 */
	if ((ticks + pending) >= hz) {
		(void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
		    (pp->p_utime + pp->p_stime)/hz, RCA_UNSAFE_SIGINFO);
	}

	/*
	 *   (b) task.max-cpu-time resource control
	 *
	 * If we have accumulated enough ticks, increment the task CPU
	 * time usage and test for the resource limit. This minimizes the
	 * number of calls to the rct_test(). The task CPU time mutex
	 * is highly contentious as many processes can be sharing a task.
	 */
	if (pp->p_ttime >= clock_tick_proc_max) {
		secs = task_cpu_time_incr(pp->p_task, pp->p_ttime);
		pp->p_ttime = 0;
		if (secs) {
			(void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls,
			    pp, secs, RCA_UNSAFE_SIGINFO);
		}
	}

	/*
	 * Update memory usage for the currently running process.
	 */
	rss = rm_asrss(as);
	PTOU(pp)->u_mem += rss;
	if (rss > PTOU(pp)->u_mem_max)
		PTOU(pp)->u_mem_max = rss;

	/*
	 * Notify the CPU the thread is running on.
	 */
	if (poke && t->t_cpu != CPU)
		poke_cpu(t->t_cpu->cpu_id);
}

void
profil_tick(uintptr_t upc)
{
	int ticks;
	proc_t *p = ttoproc(curthread);
	klwp_t *lwp = ttolwp(curthread);
	struct prof *pr = &p->p_prof;

	do {
		ticks = lwp->lwp_oweupc;
	} while (atomic_cas_32(&lwp->lwp_oweupc, ticks, 0) != ticks);

	mutex_enter(&p->p_pflock);
	if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
		/*
		 * Old-style profiling
		 */
		uint16_t *slot = pr->pr_base;
		uint16_t old, new;
		if (pr->pr_scale != 2) {
			uintptr_t delta = upc - pr->pr_off;
			uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
			    (((delta & 0xffff) * pr->pr_scale) >> 16);
			if (byteoff >= (uintptr_t)pr->pr_size) {
				mutex_exit(&p->p_pflock);
				return;
			}
			slot += byteoff / sizeof (uint16_t);
		}
		if (fuword16(slot, &old) < 0 ||
		    (new = old + ticks) > SHRT_MAX ||
		    suword16(slot, new) < 0) {
			pr->pr_scale = 0;
		}
	} else if (pr->pr_scale == 1) {
		/*
		 * PC Sampling
		 */
		model_t model = lwp_getdatamodel(lwp);
		int result;
#ifdef __lint
		model = model;
#endif
		while (ticks-- > 0) {
			if (pr->pr_samples == pr->pr_size) {
				/* buffer full, turn off sampling */
				pr->pr_scale = 0;
				break;
			}
			switch (SIZEOF_PTR(model)) {
			case sizeof (uint32_t):
				result = suword32(pr->pr_base, (uint32_t)upc);
				break;
#ifdef _LP64
			case sizeof (uint64_t):
				result = suword64(pr->pr_base, (uint64_t)upc);
				break;
#endif
			default:
				cmn_err(CE_WARN, "profil_tick: unexpected "
				    "data model");
				result = -1;
				break;
			}
			if (result != 0) {
				pr->pr_scale = 0;
				break;
			}
			pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
			pr->pr_samples++;
		}
	}
	mutex_exit(&p->p_pflock);
}

static void
delay_wakeup(void *arg)
{
	kthread_t	*t = arg;

	mutex_enter(&t->t_delay_lock);
	cv_signal(&t->t_delay_cv);
	mutex_exit(&t->t_delay_lock);
}

/*
 * The delay(9F) man page indicates that it can only be called from user or
 * kernel context - detect and diagnose bad calls. The following macro will
 * produce a limited number of messages identifying bad callers.  This is done
 * in a macro so that caller() is meaningful. When a bad caller is identified,
 * switching to 'drv_usecwait(TICK_TO_USEC(ticks));' may be appropriate.
 */
#define	DELAY_CONTEXT_CHECK()	{					\
	uint32_t	m;						\
	char		*f;						\
	ulong_t		off;						\
									\
	m = delay_from_interrupt_msg;					\
	if (delay_from_interrupt_diagnose && servicing_interrupt() &&	\
	    !panicstr && !devinfo_freeze &&				\
	    atomic_cas_32(&delay_from_interrupt_msg, m ? m : 1, m-1)) {	\
		f = modgetsymname((uintptr_t)caller(), &off);		\
		cmn_err(CE_WARN, "delay(9F) called from "		\
		    "interrupt context: %s`%s",				\
		    mod_containing_pc(caller()), f ? f : "...");	\
	}								\
}

/*
 * delay_common: common delay code.
 */
static void
delay_common(clock_t ticks)
{
	kthread_t	*t = curthread;
	clock_t		deadline;
	clock_t		timeleft;
	callout_id_t	id;

	/* If timeouts aren't running all we can do is spin. */
	if (panicstr || devinfo_freeze) {
		/* Convert delay(9F) call into drv_usecwait(9F) call. */
		if (ticks > 0)
			drv_usecwait(TICK_TO_USEC(ticks));
		return;
	}

	deadline = ddi_get_lbolt() + ticks;
	while ((timeleft = deadline - ddi_get_lbolt()) > 0) {
		mutex_enter(&t->t_delay_lock);
		id = timeout_default(delay_wakeup, t, timeleft);
		cv_wait(&t->t_delay_cv, &t->t_delay_lock);
		mutex_exit(&t->t_delay_lock);
		(void) untimeout_default(id, 0);
	}
}

/*
 * Delay specified number of clock ticks.
 */
void
delay(clock_t ticks)
{
	DELAY_CONTEXT_CHECK();

	delay_common(ticks);
}

/*
 * Delay a random number of clock ticks between 1 and ticks.
 */
void
delay_random(clock_t ticks)
{
	int	r;

	DELAY_CONTEXT_CHECK();

	(void) random_get_pseudo_bytes((void *)&r, sizeof (r));
	if (ticks == 0)
		ticks = 1;
	ticks = (r % ticks) + 1;
	delay_common(ticks);
}

/*
 * Like delay, but interruptible by a signal.
 */
int
delay_sig(clock_t ticks)
{
	kthread_t	*t = curthread;
	clock_t		deadline;
	clock_t		rc;

	/* If timeouts aren't running all we can do is spin. */
	if (panicstr || devinfo_freeze) {
		if (ticks > 0)
			drv_usecwait(TICK_TO_USEC(ticks));
		return (0);
	}

	deadline = ddi_get_lbolt() + ticks;
	mutex_enter(&t->t_delay_lock);
	do {
		rc = cv_timedwait_sig(&t->t_delay_cv,
		    &t->t_delay_lock, deadline);
		/* loop until past deadline or signaled */
	} while (rc > 0);
	mutex_exit(&t->t_delay_lock);
	if (rc == 0)
		return (EINTR);
	return (0);
}


#define	SECONDS_PER_DAY 86400

/*
 * Initialize the system time based on the TOD chip.  approx is used as
 * an approximation of time (e.g. from the filesystem) in the event that
 * the TOD chip has been cleared or is unresponsive.  An approx of -1
 * means the filesystem doesn't keep time.
 */
void
clkset(time_t approx)
{
	timestruc_t ts;
	int spl;
	int set_clock = 0;

	mutex_enter(&tod_lock);
	ts = tod_get();

	if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
		/*
		 * If the TOD chip is reporting some time after 1971,
		 * then it probably didn't lose power or become otherwise
		 * cleared in the recent past;  check to assure that
		 * the time coming from the filesystem isn't in the future
		 * according to the TOD chip.
		 */
		if (approx != -1 && approx > ts.tv_sec) {
			cmn_err(CE_WARN, "Last shutdown is later "
			    "than time on time-of-day chip; check date.");
		}
	} else {
		/*
		 * If the TOD chip isn't giving correct time, set it to the
		 * greater of i) approx and ii) 1987. That way if approx
		 * is negative or is earlier than 1987, we set the clock
		 * back to a time when Oliver North, ALF and Dire Straits
		 * were all on the collective brain:  1987.
		 */
		timestruc_t tmp;
		time_t diagnose_date = (1987 - 1970) * 365 * SECONDS_PER_DAY;
		ts.tv_sec = (approx > diagnose_date ? approx : diagnose_date);
		ts.tv_nsec = 0;

		/*
		 * Attempt to write the new time to the TOD chip.  Set spl high
		 * to avoid getting preempted between the tod_set and tod_get.
		 */
		spl = splhi();
		tod_set(ts);
		tmp = tod_get();
		splx(spl);

		if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
			tod_broken = 1;
			dosynctodr = 0;
			cmn_err(CE_WARN, "Time-of-day chip unresponsive.");
		} else {
			cmn_err(CE_WARN, "Time-of-day chip had "
			    "incorrect date; check and reset.");
		}
		set_clock = 1;
	}

	if (!boot_time) {
		boot_time = ts.tv_sec;
		set_clock = 1;
	}

	if (set_clock)
		set_hrestime(&ts);

	mutex_exit(&tod_lock);
}

int	timechanged;	/* for testing if the system time has been reset */

void
set_hrestime(timestruc_t *ts)
{
	int spl = hr_clock_lock();
	hrestime = *ts;
	membar_enter();	/* hrestime must be visible before timechanged++ */
	timedelta = 0;
	timechanged++;
	hr_clock_unlock(spl);
	callout_hrestime();
}

static uint_t deadman_seconds;
static uint32_t deadman_panics;
static int deadman_enabled = 0;
static int deadman_panic_timers = 1;

static void
deadman(void)
{
	if (panicstr) {
		/*
		 * During panic, other CPUs besides the panic
		 * master continue to handle cyclics and some other
		 * interrupts.  The code below is intended to be
		 * single threaded, so any CPU other than the master
		 * must keep out.
		 */
		if (CPU->cpu_id != panic_cpu.cpu_id)
			return;

		if (!deadman_panic_timers)
			return; /* allow all timers to be manually disabled */

		/*
		 * If we are generating a crash dump or syncing filesystems and
		 * the corresponding timer is set, decrement it and re-enter
		 * the panic code to abort it and advance to the next state.
		 * The panic states and triggers are explained in panic.c.
		 */
		if (panic_dump) {
			if (dump_timeleft && (--dump_timeleft == 0)) {
				panic("panic dump timeout");
				/*NOTREACHED*/
			}
		} else if (panic_sync) {
			if (sync_timeleft && (--sync_timeleft == 0)) {
				panic("panic sync timeout");
				/*NOTREACHED*/
			}
		}

		return;
	}

	if (deadman_counter != CPU->cpu_deadman_counter) {
		CPU->cpu_deadman_counter = deadman_counter;
		CPU->cpu_deadman_countdown = deadman_seconds;
		return;
	}

	if (--CPU->cpu_deadman_countdown > 0)
		return;

	/*
	 * Regardless of whether or not we actually bring the system down,
	 * bump the deadman_panics variable.
	 *
	 * N.B. deadman_panics is incremented once for each CPU that
	 * passes through here.  It's expected that all the CPUs will
	 * detect this condition within one second of each other, so
	 * when deadman_enabled is off, deadman_panics will
	 * typically be a multiple of the total number of CPUs in
	 * the system.
	 */
	atomic_inc_32(&deadman_panics);

	if (!deadman_enabled) {
		CPU->cpu_deadman_countdown = deadman_seconds;
		return;
	}

	/*
	 * If we're here, we want to bring the system down.
	 */
	panic("deadman: timed out after %d seconds of clock "
	    "inactivity", deadman_seconds);
	/*NOTREACHED*/
}

/*ARGSUSED*/
static void
deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
{
	cpu->cpu_deadman_counter = 0;
	cpu->cpu_deadman_countdown = deadman_seconds;

	hdlr->cyh_func = (cyc_func_t)deadman;
	hdlr->cyh_level = CY_HIGH_LEVEL;
	hdlr->cyh_arg = NULL;

	/*
	 * Stagger the CPUs so that they don't all run deadman() at
	 * the same time.  Simplest reason to do this is to make it
	 * more likely that only one CPU will panic in case of a
	 * timeout.  This is (strictly speaking) an aesthetic, not a
	 * technical consideration.
	 */
	when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
	when->cyt_interval = NANOSEC;
}


void
deadman_init(void)
{
	cyc_omni_handler_t hdlr;

	if (deadman_seconds == 0)
		deadman_seconds = snoop_interval / MICROSEC;

	if (snooping)
		deadman_enabled = 1;

	hdlr.cyo_online = deadman_online;
	hdlr.cyo_offline = NULL;
	hdlr.cyo_arg = NULL;

	mutex_enter(&cpu_lock);
	deadman_cyclic = cyclic_add_omni(&hdlr);
	mutex_exit(&cpu_lock);
}

/*
 * tod_fault() is for updating tod validate mechanism state:
 * (1) TOD_NOFAULT: for resetting the state to 'normal'.
 *     currently used for debugging only
 * (2) The following four cases detected by tod validate mechanism:
 *       TOD_REVERSED: current tod value is less than previous value.
 *       TOD_STALLED: current tod value hasn't advanced.
 *       TOD_JUMPED: current tod value advanced too far from previous value.
 *       TOD_RATECHANGED: the ratio between average tod delta and
 *       average tick delta has changed.
 * (3) TOD_RDONLY: when the TOD clock is not writeable e.g. because it is
 *     a virtual TOD provided by a hypervisor.
 */
enum tod_fault_type
tod_fault(enum tod_fault_type ftype, int off)
{
	ASSERT(MUTEX_HELD(&tod_lock));

	if (tod_faulted != ftype) {
		switch (ftype) {
		case TOD_NOFAULT:
			plat_tod_fault(TOD_NOFAULT);
			cmn_err(CE_NOTE, "Restarted tracking "
			    "Time of Day clock.");
			tod_faulted = ftype;
			break;
		case TOD_REVERSED:
		case TOD_JUMPED:
			if (tod_faulted == TOD_NOFAULT) {
				plat_tod_fault(ftype);
				cmn_err(CE_WARN, "Time of Day clock error: "
				    "reason [%s by 0x%x]. -- "
				    " Stopped tracking Time Of Day clock.",
				    tod_fault_table[ftype], off);
				tod_faulted = ftype;
			}
			break;
		case TOD_STALLED:
		case TOD_RATECHANGED:
			if (tod_faulted == TOD_NOFAULT) {
				plat_tod_fault(ftype);
				cmn_err(CE_WARN, "Time of Day clock error: "
				    "reason [%s]. -- "
				    " Stopped tracking Time Of Day clock.",
				    tod_fault_table[ftype]);
				tod_faulted = ftype;
			}
			break;
		case TOD_RDONLY:
			if (tod_faulted == TOD_NOFAULT) {
				plat_tod_fault(ftype);
				cmn_err(CE_NOTE, "!Time of Day clock is "
				    "Read-Only; set of Date/Time will not "
				    "persist across reboot.");
				tod_faulted = ftype;
			}
			break;
		default:
			break;
		}
	}
	return (tod_faulted);
}

/*
 * Two functions that allow tod_status_flag to be manipulated by functions
 * external to this file.
 */

void
tod_status_set(int tod_flag)
{
	tod_status_flag |= tod_flag;
}

void
tod_status_clear(int tod_flag)
{
	tod_status_flag &= ~tod_flag;
}

/*
 * Record a timestamp and the value passed to tod_set().  The next call to
 * tod_validate() can use these values, prev_set_tick and prev_set_tod,
 * when checking the timestruc_t returned by tod_get().  Ordinarily,
 * tod_validate() will use prev_tick and prev_tod for this task but these
 * become obsolete, and will be re-assigned with the prev_set_* values,
 * in the case when the TOD is re-written.
 */
void
tod_set_prev(timestruc_t ts)
{
	if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
	    tod_validate_deferred) {
		return;
	}
	prev_set_tick = gethrtime();
	/*
	 * A negative value will be set to zero in utc_to_tod() so we fake
	 * a zero here in such a case.  This would need to change if the
	 * behavior of utc_to_tod() changes.
	 */
	prev_set_tod = ts.tv_sec < 0 ? 0 : ts.tv_sec;
}

/*
 * tod_validate() is used for checking values returned by tod_get().
 * Four error cases can be detected by this routine:
 *   TOD_REVERSED: current tod value is less than previous.
 *   TOD_STALLED: current tod value hasn't advanced.
 *   TOD_JUMPED: current tod value advanced too far from previous value.
 *   TOD_RATECHANGED: the ratio between average tod delta and
 *   average tick delta has changed.
 */
time_t
tod_validate(time_t tod)
{
	time_t diff_tod;
	hrtime_t diff_tick;

	long dtick;
	int dtick_delta;

	int off = 0;
	enum tod_fault_type tod_bad = TOD_NOFAULT;

	static int firsttime = 1;

	static time_t prev_tod = 0;
	static hrtime_t prev_tick = 0;
	static long dtick_avg = TOD_REF_FREQ;

	int cpr_resume_done = 0;
	int dr_resume_done = 0;

	hrtime_t tick = gethrtime();

	ASSERT(MUTEX_HELD(&tod_lock));

	/*
	 * tod_validate_enable is patchable via /etc/system.
	 * If TOD is already faulted, or if TOD validation is deferred,
	 * there is nothing to do.
	 */
	if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
	    tod_validate_deferred) {
		return (tod);
	}

	/*
	 * If this is the first time through, we just need to save the tod
	 * we were called with and hrtime so we can use them next time to
	 * validate tod_get().
	 */
	if (firsttime) {
		firsttime = 0;
		prev_tod = tod;
		prev_tick = tick;
		return (tod);
	}

	/*
	 * Handle any flags that have been turned on by tod_status_set().
	 * In the case where a tod_set() is done and then a subsequent
	 * tod_get() fails (ie, both TOD_SET_DONE and TOD_GET_FAILED are
	 * true), we treat the TOD_GET_FAILED with precedence by switching
	 * off the flag, returning tod and leaving TOD_SET_DONE asserted
	 * until such time as tod_get() completes successfully.
	 */
	if (tod_status_flag & TOD_GET_FAILED) {
		/*
		 * tod_get() has encountered an issue, possibly transitory,
		 * when reading TOD.  We'll just return the incoming tod
		 * value (which is actually hrestime.tv_sec in this case)
		 * and when we get a genuine tod, following a successful
		 * tod_get(), we can validate using prev_tod and prev_tick.
		 */
		tod_status_flag &= ~TOD_GET_FAILED;
		return (tod);
	} else if (tod_status_flag & TOD_SET_DONE) {
		/*
		 * TOD has been modified.  Just before the TOD was written,
		 * tod_set_prev() saved tod and hrtime; we can now use
		 * those values, prev_set_tod and prev_set_tick, to validate
		 * the incoming tod that's just been read.
		 */
		prev_tod = prev_set_tod;
		prev_tick = prev_set_tick;
		dtick_avg = TOD_REF_FREQ;
		tod_status_flag &= ~TOD_SET_DONE;
		/*
		 * If a tod_set() preceded a cpr_suspend() without an
		 * intervening tod_validate(), we need to ensure that a
		 * TOD_JUMPED condition is ignored.
		 * Note this isn't a concern in the case of DR as we've
		 * just reassigned dtick_avg, above.
		 */
		if (tod_status_flag & TOD_CPR_RESUME_DONE) {
			cpr_resume_done = 1;
			tod_status_flag &= ~TOD_CPR_RESUME_DONE;
		}
	} else if (tod_status_flag & TOD_CPR_RESUME_DONE) {
		/*
		 * The system's coming back from a checkpoint resume.
		 */
		cpr_resume_done = 1;
		tod_status_flag &= ~TOD_CPR_RESUME_DONE;
		/*
		 * We need to handle the possibility of a CPR suspend
		 * operation having been initiated whilst a DR event was
		 * in-flight.
		 */
		if (tod_status_flag & TOD_DR_RESUME_DONE) {
			dr_resume_done = 1;
			tod_status_flag &= ~TOD_DR_RESUME_DONE;
		}
	} else if (tod_status_flag & TOD_DR_RESUME_DONE) {
		/*
		 * A Dynamic Reconfiguration event has taken place.
		 */
		dr_resume_done = 1;
		tod_status_flag &= ~TOD_DR_RESUME_DONE;
	}

	/* test hook */
	switch (tod_unit_test) {
	case 1: /* for testing jumping tod */
		tod += tod_test_injector;
		tod_unit_test = 0;
		break;
	case 2:	/* for testing stuck tod bit */
		tod |= 1 << tod_test_injector;
		tod_unit_test = 0;
		break;
	case 3:	/* for testing stalled tod */
		tod = prev_tod;
		tod_unit_test = 0;
		break;
	case 4:	/* reset tod fault status */
		(void) tod_fault(TOD_NOFAULT, 0);
		tod_unit_test = 0;
		break;
	default:
		break;
	}

	diff_tod = tod - prev_tod;
	diff_tick = tick - prev_tick;

	ASSERT(diff_tick >= 0);

	if (diff_tod < 0) {
		/* ERROR - tod reversed */
		tod_bad = TOD_REVERSED;
		off = (int)(prev_tod - tod);
	} else if (diff_tod == 0) {
		/* tod did not advance */
		if (diff_tick > TOD_STALL_THRESHOLD) {
			/* ERROR - tod stalled */
			tod_bad = TOD_STALLED;
		} else {
			/*
			 * Make sure we don't update prev_tick
			 * so that diff_tick is calculated since
			 * the first diff_tod == 0
			 */
			return (tod);
		}
	} else {
		/* calculate dtick */
		dtick = diff_tick / diff_tod;

		/* update dtick averages */
		dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);

		/*
		 * Calculate dtick_delta as
		 * variation from reference freq in quartiles
		 */
		dtick_delta = (dtick_avg - TOD_REF_FREQ) /
		    (TOD_REF_FREQ >> 2);

		/*
		 * Even with a perfectly functioning TOD device,
		 * when the number of elapsed seconds is low the
		 * algorithm can calculate a rate that is beyond
		 * tolerance, causing an error.  The algorithm is
		 * inaccurate when elapsed time is low (less than
		 * 5 seconds).
		 */
		if (diff_tod > 4) {
			if (dtick < TOD_JUMP_THRESHOLD) {
				/*
				 * If we've just done a CPR resume, we detect
				 * a jump in the TOD but, actually, what's
				 * happened is that the TOD has been increasing
				 * whilst the system was suspended and the tick
				 * count hasn't kept up.  We consider the first
				 * occurrence of this after a resume as normal
				 * and ignore it; otherwise, in a non-resume
				 * case, we regard it as a TOD problem.
				 */
				if (!cpr_resume_done) {
					/* ERROR - tod jumped */
					tod_bad = TOD_JUMPED;
					off = (int)diff_tod;
				}
			}
			if (dtick_delta) {
				/*
				 * If we've just done a DR resume, dtick_avg
				 * can go a bit askew so we reset it and carry
				 * on; otherwise, the TOD is in error.
				 */
				if (dr_resume_done) {
					dtick_avg = TOD_REF_FREQ;
				} else {
					/* ERROR - change in clock rate */
					tod_bad = TOD_RATECHANGED;
				}
			}
		}
	}

	if (tod_bad != TOD_NOFAULT) {
		(void) tod_fault(tod_bad, off);

		/*
		 * Disable dosynctodr since we are going to fault
		 * the TOD chip anyway here
		 */
		dosynctodr = 0;

		/*
		 * Set tod to the correct value from hrestime
		 */
		tod = hrestime.tv_sec;
	}

	prev_tod = tod;
	prev_tick = tick;
	return (tod);
}

static void
calcloadavg(int nrun, uint64_t *hp_ave)
{
	static int64_t f[3] = { 135, 27, 9 };
	uint_t i;
	int64_t q, r;

	/*
	 * Compute load average over the last 1, 5, and 15 minutes
	 * (60, 300, and 900 seconds).  The constants in f[3] are for
	 * exponential decay:
	 * (1 - exp(-1/60)) << 13 = 135,
	 * (1 - exp(-1/300)) << 13 = 27,
	 * (1 - exp(-1/900)) << 13 = 9.
	 */

	/*
	 * a little hoop-jumping to avoid integer overflow
	 */
	for (i = 0; i < 3; i++) {
		q = (hp_ave[i]  >> 16) << 7;
		r = (hp_ave[i]  & 0xffff) << 7;
		hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
	}
}

/*
 * lbolt_hybrid() is used by ddi_get_lbolt() and ddi_get_lbolt64() to
 * calculate the value of lbolt according to the current mode. In the event
 * driven mode (the default), lbolt is calculated by dividing the current hires
 * time by the number of nanoseconds per clock tick. In the cyclic driven mode
 * an internal variable is incremented at each firing of the lbolt cyclic
 * and returned by lbolt_cyclic_driven().
 *
 * The system will transition from event to cyclic driven mode when the number
 * of calls to lbolt_event_driven() exceeds the (per CPU) threshold within a
 * window of time. It does so by reprograming lbolt_cyclic from CY_INFINITY to
 * nsec_per_tick. The lbolt cyclic will remain ON while at least one CPU is
 * causing enough activity to cross the thresholds.
 */
int64_t
lbolt_bootstrap(void)
{
	return (0);
}

/* ARGSUSED */
uint_t
lbolt_ev_to_cyclic(caddr_t arg1, caddr_t arg2)
{
	hrtime_t ts, exp;
	int ret;

	ASSERT(lbolt_hybrid != lbolt_cyclic_driven);

	kpreempt_disable();

	ts = gethrtime();
	lb_info->lbi_internal = (ts/nsec_per_tick);

	/*
	 * Align the next expiration to a clock tick boundary.
	 */
	exp = ts + nsec_per_tick - 1;
	exp = (exp/nsec_per_tick) * nsec_per_tick;

	ret = cyclic_reprogram(lb_info->id.lbi_cyclic_id, exp);
	ASSERT(ret);

	lbolt_hybrid = lbolt_cyclic_driven;
	lb_info->lbi_cyc_deactivate = B_FALSE;
	lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;

	kpreempt_enable();

	ret = atomic_dec_32_nv(&lb_info->lbi_token);
	ASSERT(ret == 0);

	return (1);
}

int64_t
lbolt_event_driven(void)
{
	hrtime_t ts;
	int64_t lb;
	int ret, cpu = CPU->cpu_seqid;

	ts = gethrtime();
	ASSERT(ts > 0);

	ASSERT(nsec_per_tick > 0);
	lb = (ts/nsec_per_tick);

	/*
	 * Switch to cyclic mode if the number of calls to this routine
	 * has reached the threshold within the interval.
	 */
	if ((lb - lb_cpu[cpu].lbc_cnt_start) < lb_info->lbi_thresh_interval) {

		if (--lb_cpu[cpu].lbc_counter == 0) {
			/*
			 * Reached the threshold within the interval, reset
			 * the usage statistics.
			 */
			lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
			lb_cpu[cpu].lbc_cnt_start = lb;

			/*
			 * Make sure only one thread reprograms the
			 * lbolt cyclic and changes the mode.
			 */
			if (panicstr == NULL &&
			    atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {

				if (lbolt_hybrid == lbolt_cyclic_driven) {
					ret = atomic_dec_32_nv(
					    &lb_info->lbi_token);
					ASSERT(ret == 0);
				} else {
					lbolt_softint_post();
				}
			}
		}
	} else {
		/*
		 * Exceeded the interval, reset the usage statistics.
		 */
		lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
		lb_cpu[cpu].lbc_cnt_start = lb;
	}

	ASSERT(lb >= lb_info->lbi_debug_time);

	return (lb - lb_info->lbi_debug_time);
}

int64_t
lbolt_cyclic_driven(void)
{
	int64_t lb = lb_info->lbi_internal;
	int cpu;

	/*
	 * If a CPU has already prevented the lbolt cyclic from deactivating
	 * itself, don't bother tracking the usage. Otherwise check if we're
	 * within the interval and how the per CPU counter is doing.
	 */
	if (lb_info->lbi_cyc_deactivate) {
		cpu = CPU->cpu_seqid;
		if ((lb - lb_cpu[cpu].lbc_cnt_start) <
		    lb_info->lbi_thresh_interval) {

			if (lb_cpu[cpu].lbc_counter == 0)
				/*
				 * Reached the threshold within the interval,
				 * prevent the lbolt cyclic from turning itself
				 * off.
				 */
				lb_info->lbi_cyc_deactivate = B_FALSE;
			else
				lb_cpu[cpu].lbc_counter--;
		} else {
			/*
			 * Only reset the usage statistics when we have
			 * exceeded the interval.
			 */
			lb_cpu[cpu].lbc_counter = lb_info->lbi_thresh_calls;
			lb_cpu[cpu].lbc_cnt_start = lb;
		}
	}

	ASSERT(lb >= lb_info->lbi_debug_time);

	return (lb - lb_info->lbi_debug_time);
}

/*
 * The lbolt_cyclic() routine will fire at a nsec_per_tick interval to satisfy
 * performance needs of ddi_get_lbolt() and ddi_get_lbolt64() consumers.
 * It is inactive by default, and will be activated when switching from event
 * to cyclic driven lbolt. The cyclic will turn itself off unless signaled
 * by lbolt_cyclic_driven().
 */
static void
lbolt_cyclic(void)
{
	int ret;

	lb_info->lbi_internal++;

	if (!lbolt_cyc_only) {

		if (lb_info->lbi_cyc_deactivate) {
			/*
			 * Switching from cyclic to event driven mode.
			 */
			if (panicstr == NULL &&
			    atomic_cas_32(&lb_info->lbi_token, 0, 1) == 0) {

				if (lbolt_hybrid == lbolt_event_driven) {
					ret = atomic_dec_32_nv(
					    &lb_info->lbi_token);
					ASSERT(ret == 0);
					return;
				}

				kpreempt_disable();

				lbolt_hybrid = lbolt_event_driven;
				ret = cyclic_reprogram(
				    lb_info->id.lbi_cyclic_id,
				    CY_INFINITY);
				ASSERT(ret);

				kpreempt_enable();

				ret = atomic_dec_32_nv(&lb_info->lbi_token);
				ASSERT(ret == 0);
			}
		}

		/*
		 * The lbolt cyclic should not try to deactivate itself before
		 * the sampling period has elapsed.
		 */
		if (lb_info->lbi_internal - lb_info->lbi_cyc_deac_start >=
		    lb_info->lbi_thresh_interval) {
			lb_info->lbi_cyc_deactivate = B_TRUE;
			lb_info->lbi_cyc_deac_start = lb_info->lbi_internal;
		}
	}
}

/*
 * Since the lbolt service was historically cyclic driven, it must be 'stopped'
 * when the system drops into the kernel debugger. lbolt_debug_entry() is
 * called by the KDI system claim callbacks to record a hires timestamp at
 * debug enter time. lbolt_debug_return() is called by the sistem release
 * callbacks to account for the time spent in the debugger. The value is then
 * accumulated in the lb_info structure and used by lbolt_event_driven() and
 * lbolt_cyclic_driven(), as well as the mdb_get_lbolt() routine.
 */
void
lbolt_debug_entry(void)
{
	if (lbolt_hybrid != lbolt_bootstrap) {
		ASSERT(lb_info != NULL);
		lb_info->lbi_debug_ts = gethrtime();
	}
}

/*
 * Calculate the time spent in the debugger and add it to the lbolt info
 * structure. We also update the internal lbolt value in case we were in
 * cyclic driven mode going in.
 */
void
lbolt_debug_return(void)
{
	hrtime_t ts;

	if (lbolt_hybrid != lbolt_bootstrap) {
		ASSERT(lb_info != NULL);
		ASSERT(nsec_per_tick > 0);

		ts = gethrtime();
		lb_info->lbi_internal = (ts/nsec_per_tick);
		lb_info->lbi_debug_time +=
		    ((ts - lb_info->lbi_debug_ts)/nsec_per_tick);

		lb_info->lbi_debug_ts = 0;
	}
}