xref: /titanic_44/usr/src/uts/common/os/clock.c (revision e824d57f8160a27ac5e650005c7a4f037109c2be)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
22 /*	  All Rights Reserved	*/
23 
24 
25 /*
26  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
27  * Use is subject to license terms.
28  */
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 #include <sys/param.h>
33 #include <sys/t_lock.h>
34 #include <sys/types.h>
35 #include <sys/tuneable.h>
36 #include <sys/sysmacros.h>
37 #include <sys/systm.h>
38 #include <sys/cpuvar.h>
39 #include <sys/lgrp.h>
40 #include <sys/user.h>
41 #include <sys/proc.h>
42 #include <sys/callo.h>
43 #include <sys/kmem.h>
44 #include <sys/var.h>
45 #include <sys/cmn_err.h>
46 #include <sys/swap.h>
47 #include <sys/vmsystm.h>
48 #include <sys/class.h>
49 #include <sys/time.h>
50 #include <sys/debug.h>
51 #include <sys/vtrace.h>
52 #include <sys/spl.h>
53 #include <sys/atomic.h>
54 #include <sys/dumphdr.h>
55 #include <sys/archsystm.h>
56 #include <sys/fs/swapnode.h>
57 #include <sys/panic.h>
58 #include <sys/disp.h>
59 #include <sys/msacct.h>
60 #include <sys/mem_cage.h>
61 
62 #include <vm/page.h>
63 #include <vm/anon.h>
64 #include <vm/rm.h>
65 #include <sys/cyclic.h>
66 #include <sys/cpupart.h>
67 #include <sys/rctl.h>
68 #include <sys/task.h>
69 #include <sys/chip.h>
70 #include <sys/sdt.h>
71 
72 #ifdef __sparc
73 #include <sys/wdt.h>
74 #endif
75 
76 /*
77  * for NTP support
78  */
79 #include <sys/timex.h>
80 #include <sys/inttypes.h>
81 
82 /*
83  * clock is called straight from
84  * the real time clock interrupt.
85  *
86  * Functions:
87  *	reprime clock
88  *	schedule callouts
89  *	maintain date
90  *	jab the scheduler
91  */
92 
93 extern kcondvar_t	fsflush_cv;
94 extern sysinfo_t	sysinfo;
95 extern vminfo_t	vminfo;
96 extern int	idleswtch;	/* flag set while idle in pswtch() */
97 
98 /*
99  * high-precision avenrun values.  These are needed to make the
100  * regular avenrun values accurate.
101  */
102 static uint64_t hp_avenrun[3];
103 int	avenrun[3];		/* FSCALED average run queue lengths */
104 time_t	time;	/* time in seconds since 1970 - for compatibility only */
105 
106 static struct loadavg_s loadavg;
107 /*
108  * Phase/frequency-lock loop (PLL/FLL) definitions
109  *
110  * The following variables are read and set by the ntp_adjtime() system
111  * call.
112  *
113  * time_state shows the state of the system clock, with values defined
114  * in the timex.h header file.
115  *
116  * time_status shows the status of the system clock, with bits defined
117  * in the timex.h header file.
118  *
119  * time_offset is used by the PLL/FLL to adjust the system time in small
120  * increments.
121  *
122  * time_constant determines the bandwidth or "stiffness" of the PLL.
123  *
124  * time_tolerance determines maximum frequency error or tolerance of the
125  * CPU clock oscillator and is a property of the architecture; however,
126  * in principle it could change as result of the presence of external
127  * discipline signals, for instance.
128  *
129  * time_precision is usually equal to the kernel tick variable; however,
130  * in cases where a precision clock counter or external clock is
131  * available, the resolution can be much less than this and depend on
132  * whether the external clock is working or not.
133  *
134  * time_maxerror is initialized by a ntp_adjtime() call and increased by
135  * the kernel once each second to reflect the maximum error bound
136  * growth.
137  *
138  * time_esterror is set and read by the ntp_adjtime() call, but
139  * otherwise not used by the kernel.
140  */
141 int32_t time_state = TIME_OK;	/* clock state */
142 int32_t time_status = STA_UNSYNC;	/* clock status bits */
143 int32_t time_offset = 0;		/* time offset (us) */
144 int32_t time_constant = 0;		/* pll time constant */
145 int32_t time_tolerance = MAXFREQ;	/* frequency tolerance (scaled ppm) */
146 int32_t time_precision = 1;	/* clock precision (us) */
147 int32_t time_maxerror = MAXPHASE;	/* maximum error (us) */
148 int32_t time_esterror = MAXPHASE;	/* estimated error (us) */
149 
150 /*
151  * The following variables establish the state of the PLL/FLL and the
152  * residual time and frequency offset of the local clock. The scale
153  * factors are defined in the timex.h header file.
154  *
155  * time_phase and time_freq are the phase increment and the frequency
156  * increment, respectively, of the kernel time variable.
157  *
158  * time_freq is set via ntp_adjtime() from a value stored in a file when
159  * the synchronization daemon is first started. Its value is retrieved
160  * via ntp_adjtime() and written to the file about once per hour by the
161  * daemon.
162  *
163  * time_adj is the adjustment added to the value of tick at each timer
164  * interrupt and is recomputed from time_phase and time_freq at each
165  * seconds rollover.
166  *
167  * time_reftime is the second's portion of the system time at the last
168  * call to ntp_adjtime(). It is used to adjust the time_freq variable
169  * and to increase the time_maxerror as the time since last update
170  * increases.
171  */
172 int32_t time_phase = 0;		/* phase offset (scaled us) */
173 int32_t time_freq = 0;		/* frequency offset (scaled ppm) */
174 int32_t time_adj = 0;		/* tick adjust (scaled 1 / hz) */
175 int32_t time_reftime = 0;		/* time at last adjustment (s) */
176 
177 /*
178  * The scale factors of the following variables are defined in the
179  * timex.h header file.
180  *
181  * pps_time contains the time at each calibration interval, as read by
182  * microtime(). pps_count counts the seconds of the calibration
183  * interval, the duration of which is nominally pps_shift in powers of
184  * two.
185  *
186  * pps_offset is the time offset produced by the time median filter
187  * pps_tf[], while pps_jitter is the dispersion (jitter) measured by
188  * this filter.
189  *
190  * pps_freq is the frequency offset produced by the frequency median
191  * filter pps_ff[], while pps_stabil is the dispersion (wander) measured
192  * by this filter.
193  *
194  * pps_usec is latched from a high resolution counter or external clock
195  * at pps_time. Here we want the hardware counter contents only, not the
196  * contents plus the time_tv.usec as usual.
197  *
198  * pps_valid counts the number of seconds since the last PPS update. It
199  * is used as a watchdog timer to disable the PPS discipline should the
200  * PPS signal be lost.
201  *
202  * pps_glitch counts the number of seconds since the beginning of an
203  * offset burst more than tick/2 from current nominal offset. It is used
204  * mainly to suppress error bursts due to priority conflicts between the
205  * PPS interrupt and timer interrupt.
206  *
207  * pps_intcnt counts the calibration intervals for use in the interval-
208  * adaptation algorithm. It's just too complicated for words.
209  */
210 struct timeval pps_time;	/* kernel time at last interval */
211 int32_t pps_tf[] = {0, 0, 0};	/* pps time offset median filter (us) */
212 int32_t pps_offset = 0;		/* pps time offset (us) */
213 int32_t pps_jitter = MAXTIME;	/* time dispersion (jitter) (us) */
214 int32_t pps_ff[] = {0, 0, 0};	/* pps frequency offset median filter */
215 int32_t pps_freq = 0;		/* frequency offset (scaled ppm) */
216 int32_t pps_stabil = MAXFREQ;	/* frequency dispersion (scaled ppm) */
217 int32_t pps_usec = 0;		/* microsec counter at last interval */
218 int32_t pps_valid = PPS_VALID;	/* pps signal watchdog counter */
219 int32_t pps_glitch = 0;		/* pps signal glitch counter */
220 int32_t pps_count = 0;		/* calibration interval counter (s) */
221 int32_t pps_shift = PPS_SHIFT;	/* interval duration (s) (shift) */
222 int32_t pps_intcnt = 0;		/* intervals at current duration */
223 
224 /*
225  * PPS signal quality monitors
226  *
227  * pps_jitcnt counts the seconds that have been discarded because the
228  * jitter measured by the time median filter exceeds the limit MAXTIME
229  * (100 us).
230  *
231  * pps_calcnt counts the frequency calibration intervals, which are
232  * variable from 4 s to 256 s.
233  *
234  * pps_errcnt counts the calibration intervals which have been discarded
235  * because the wander exceeds the limit MAXFREQ (100 ppm) or where the
236  * calibration interval jitter exceeds two ticks.
237  *
238  * pps_stbcnt counts the calibration intervals that have been discarded
239  * because the frequency wander exceeds the limit MAXFREQ / 4 (25 us).
240  */
241 int32_t pps_jitcnt = 0;		/* jitter limit exceeded */
242 int32_t pps_calcnt = 0;		/* calibration intervals */
243 int32_t pps_errcnt = 0;		/* calibration errors */
244 int32_t pps_stbcnt = 0;		/* stability limit exceeded */
245 
246 /* The following variables require no explicit locking */
247 volatile clock_t lbolt;		/* time in Hz since last boot */
248 volatile int64_t lbolt64;	/* lbolt64 won't wrap for 2.9 billion yrs */
249 
250 kcondvar_t lbolt_cv;
251 int one_sec = 1; /* turned on once every second */
252 static int fsflushcnt;	/* counter for t_fsflushr */
253 int	dosynctodr = 1;	/* patchable; enable/disable sync to TOD chip */
254 int	tod_needsync = 0;	/* need to sync tod chip with software time */
255 static int tod_broken = 0;	/* clock chip doesn't work */
256 time_t	boot_time = 0;		/* Boot time in seconds since 1970 */
257 cyclic_id_t clock_cyclic;	/* clock()'s cyclic_id */
258 cyclic_id_t deadman_cyclic;	/* deadman()'s cyclic_id */
259 
260 static int lgrp_ticks;		/* counter to schedule lgrp load calcs */
261 
262 /*
263  * rechoose_interval_history is used to detect when rechoose_interval's
264  * value has changed (via hotpatching for example), so that the
265  * cached values in the cpu structures may be updated.
266  */
267 static int rechoose_interval_history = RECHOOSE_INTERVAL;
268 
269 /*
270  * for tod fault detection
271  */
272 #define	TOD_REF_FREQ		((longlong_t)(NANOSEC))
273 #define	TOD_STALL_THRESHOLD	(TOD_REF_FREQ * 3 / 2)
274 #define	TOD_JUMP_THRESHOLD	(TOD_REF_FREQ / 2)
275 #define	TOD_FILTER_N		4
276 #define	TOD_FILTER_SETTLE	(4 * TOD_FILTER_N)
277 static int tod_faulted = TOD_NOFAULT;
278 static int tod_fault_reset_flag = 0;
279 
280 /* patchable via /etc/system */
281 int tod_validate_enable = 1;
282 
283 /*
284  * On non-SPARC systems, TOD validation must be deferred until gethrtime
285  * returns non-zero values (after mach_clkinit's execution).
286  * On SPARC systems, it must be deferred until after hrtime_base
287  * and hres_last_tick are set (in the first invocation of hres_tick).
288  * Since in both cases the prerequisites occur before the invocation of
289  * tod_get() in clock(), the deferment is lifted there.
290  */
291 static boolean_t tod_validate_deferred = B_TRUE;
292 
293 /*
294  * tod_fault_table[] must be aligned with
295  * enum tod_fault_type in systm.h
296  */
297 static char *tod_fault_table[] = {
298 	"Reversed",			/* TOD_REVERSED */
299 	"Stalled",			/* TOD_STALLED */
300 	"Jumped",			/* TOD_JUMPED */
301 	"Changed in Clock Rate"		/* TOD_RATECHANGED */
302 	/*
303 	 * no strings needed for TOD_NOFAULT
304 	 */
305 };
306 
307 /*
308  * test hook for tod broken detection in tod_validate
309  */
310 int tod_unit_test = 0;
311 time_t tod_test_injector;
312 
313 #define	CLOCK_ADJ_HIST_SIZE	4
314 
315 static int	adj_hist_entry;
316 
317 int64_t clock_adj_hist[CLOCK_ADJ_HIST_SIZE];
318 
319 static void clock_tick(kthread_t *);
320 static void calcloadavg(int, uint64_t *);
321 static int genloadavg(struct loadavg_s *);
322 static void loadavg_update();
323 
324 void (*cmm_clock_callout)() = NULL;
325 
326 #ifdef	KSLICE
327 int kslice = KSLICE;
328 #endif
329 
330 static void
331 clock(void)
332 {
333 	kthread_t	*t;
334 	kmutex_t	*plockp;	/* pointer to thread's process lock */
335 	int	pinned_intr = 0;
336 	uint_t	nrunnable, nrunning;
337 	uint_t	w_io;
338 	cpu_t	*cp;
339 	cpupart_t *cpupart;
340 	int	exiting;
341 	extern void set_anoninfo();
342 	extern	void	set_freemem();
343 	void	(*funcp)();
344 	int32_t ltemp;
345 	int64_t lltemp;
346 	int s;
347 	int do_lgrp_load;
348 	int rechoose_update = 0;
349 	int rechoose;
350 	int i;
351 
352 	if (panicstr)
353 		return;
354 
355 	set_anoninfo();
356 	/*
357 	 * Make sure that 'freemem' do not drift too far from the truth
358 	 */
359 	set_freemem();
360 
361 
362 	/*
363 	 * Before the section which is repeated is executed, we do
364 	 * the time delta processing which occurs every clock tick
365 	 *
366 	 * There is additional processing which happens every time
367 	 * the nanosecond counter rolls over which is described
368 	 * below - see the section which begins with : if (one_sec)
369 	 *
370 	 * This section marks the beginning of the precision-kernel
371 	 * code fragment.
372 	 *
373 	 * First, compute the phase adjustment. If the low-order bits
374 	 * (time_phase) of the update overflow, bump the higher order
375 	 * bits (time_update).
376 	 */
377 	time_phase += time_adj;
378 	if (time_phase <= -FINEUSEC) {
379 		ltemp = -time_phase / SCALE_PHASE;
380 		time_phase += ltemp * SCALE_PHASE;
381 		s = hr_clock_lock();
382 		timedelta -= ltemp * (NANOSEC/MICROSEC);
383 		hr_clock_unlock(s);
384 	} else if (time_phase >= FINEUSEC) {
385 		ltemp = time_phase / SCALE_PHASE;
386 		time_phase -= ltemp * SCALE_PHASE;
387 		s = hr_clock_lock();
388 		timedelta += ltemp * (NANOSEC/MICROSEC);
389 		hr_clock_unlock(s);
390 	}
391 
392 	/*
393 	 * End of precision-kernel code fragment which is processed
394 	 * every timer interrupt.
395 	 *
396 	 * Continue with the interrupt processing as scheduled.
397 	 *
398 	 * Did we pin another interrupt thread?  Need to check this before
399 	 * grabbing any adaptive locks, since if we block on a lock the
400 	 * pinned thread could escape.  Note that this is just a heuristic;
401 	 * if we take multiple laps though clock() without returning from
402 	 * the interrupt because we have another clock tick pending, then
403 	 * the pinned interrupt could be released by one of the previous
404 	 * laps.  The only consequence is that the CPU will be counted as
405 	 * in idle (or wait) state once the pinned interrupt is released.
406 	 * Since this accounting is inaccurate by nature, this isn't a big
407 	 * deal --- but we should try to get it right in the common case
408 	 * where we only call clock() once per interrupt.
409 	 */
410 	if (curthread->t_intr != NULL)
411 		pinned_intr = (curthread->t_intr->t_flag & T_INTR_THREAD);
412 
413 	/*
414 	 * Count the number of runnable threads and the number waiting
415 	 * for some form of I/O to complete -- gets added to
416 	 * sysinfo.waiting.  To know the state of the system, must add
417 	 * wait counts from all CPUs.  Also add up the per-partition
418 	 * statistics.
419 	 */
420 	w_io = 0;
421 	nrunnable = 0;
422 
423 	/*
424 	 * keep track of when to update lgrp/part loads
425 	 */
426 
427 	do_lgrp_load = 0;
428 	if (lgrp_ticks++ >= hz / 10) {
429 		lgrp_ticks = 0;
430 		do_lgrp_load = 1;
431 	}
432 
433 	/*
434 	 * The dispatcher tunable rechoose_interval may be hot-patched.
435 	 * Note if it has a new value. If so, the effective rechoose_interval
436 	 * cached in the cpu structures needs to be updated.
437 	 * If needed we'll do this during the walk of the cpu_list below.
438 	 */
439 	if (rechoose_interval != rechoose_interval_history) {
440 		rechoose_interval_history = rechoose_interval;
441 		rechoose_update = 1;
442 	}
443 
444 	if (one_sec)
445 		loadavg_update();
446 
447 
448 	/*
449 	 * First count the threads waiting on kpreempt queues in each
450 	 * CPU partition.
451 	 */
452 
453 	cpupart = cp_list_head;
454 	do {
455 		uint_t cpupart_nrunnable = cpupart->cp_kp_queue.disp_nrunnable;
456 
457 		cpupart->cp_updates++;
458 		nrunnable += cpupart_nrunnable;
459 		cpupart->cp_nrunnable_cum += cpupart_nrunnable;
460 		if (one_sec) {
461 			cpupart->cp_nrunning = 0;
462 			cpupart->cp_nrunnable = cpupart_nrunnable;
463 		}
464 	} while ((cpupart = cpupart->cp_next) != cp_list_head);
465 
466 
467 	/* Now count the per-CPU statistics. */
468 	cp = cpu_list;
469 	do {
470 		uint_t cpu_nrunnable = cp->cpu_disp->disp_nrunnable;
471 
472 		nrunnable += cpu_nrunnable;
473 		cpupart = cp->cpu_part;
474 		cpupart->cp_nrunnable_cum += cpu_nrunnable;
475 		if (one_sec)
476 			cpupart->cp_nrunnable += cpu_nrunnable;
477 		if (do_lgrp_load &&
478 		    (cp->cpu_flags & CPU_EXISTS)) {
479 			/*
480 			 * When updating the lgroup's load average,
481 			 * account for the thread running on the CPU.
482 			 * If the CPU is the current one, then we need
483 			 * to account for the underlying thread which
484 			 * got the clock interrupt not the thread that is
485 			 * handling the interrupt and caculating the load
486 			 * average
487 			 */
488 			t = cp->cpu_thread;
489 			if (CPU == cp)
490 				t = t->t_intr;
491 
492 			/*
493 			 * Account for the load average for this thread if
494 			 * it isn't the idle thread or it is on the interrupt
495 			 * stack and not the current CPU handling the clock
496 			 * interrupt
497 			 */
498 			if ((t && t != cp->cpu_idle_thread) || (CPU != cp &&
499 			    CPU_ON_INTR(cp))) {
500 				if (t->t_lpl == cp->cpu_lpl) {
501 					/* local thread */
502 					cpu_nrunnable++;
503 				} else {
504 					/*
505 					 * This is a remote thread, charge it
506 					 * against its home lgroup.  Note that
507 					 * we notice that a thread is remote
508 					 * only if it's currently executing.
509 					 * This is a reasonable approximation,
510 					 * since queued remote threads are rare.
511 					 * Note also that if we didn't charge
512 					 * it to its home lgroup, remote
513 					 * execution would often make a system
514 					 * appear balanced even though it was
515 					 * not, and thread placement/migration
516 					 * would often not be done correctly.
517 					 */
518 					lgrp_loadavg(t->t_lpl,
519 					    LGRP_LOADAVG_IN_THREAD_MAX, 0);
520 				}
521 			}
522 			lgrp_loadavg(cp->cpu_lpl,
523 			    cpu_nrunnable * LGRP_LOADAVG_IN_THREAD_MAX, 1);
524 		}
525 		/*
526 		 * The platform may define a per physical processor
527 		 * adjustment of rechoose_interval. The effective
528 		 * (base + adjustment) rechoose_interval is cached
529 		 * in the cpu structures for efficiency. Above we detect
530 		 * if the cached values need updating, and here is where
531 		 * the update happens.
532 		 */
533 		if (rechoose_update) {
534 			rechoose = rechoose_interval +
535 				cp->cpu_chip->chip_rechoose_adj;
536 			cp->cpu_rechoose = (rechoose < 0) ? 0 : rechoose;
537 		}
538 	} while ((cp = cp->cpu_next) != cpu_list);
539 
540 	/*
541 	 * Do tick processing for all the active threads running in
542 	 * the system.
543 	 */
544 	cp = cpu_list;
545 	nrunning = 0;
546 	do {
547 		klwp_id_t lwp;
548 		int intr;
549 		int thread_away;
550 
551 		/*
552 		 * Don't do any tick processing on CPUs that
553 		 * aren't even in the system or aren't up yet.
554 		 */
555 		if ((cp->cpu_flags & CPU_EXISTS) == 0) {
556 			continue;
557 		}
558 
559 		/*
560 		 * The locking here is rather tricky.  We use
561 		 * thread_free_lock to keep the currently running
562 		 * thread from being freed or recycled while we're
563 		 * looking at it.  We can then check if the thread
564 		 * is exiting and get the appropriate p_lock if it
565 		 * is not.  We have to be careful, though, because
566 		 * the _process_ can still be freed while we're
567 		 * holding thread_free_lock.  To avoid touching the
568 		 * proc structure we put a pointer to the p_lock in the
569 		 * thread structure.  The p_lock is persistent so we
570 		 * can acquire it even if the process is gone.  At that
571 		 * point we can check (again) if the thread is exiting
572 		 * and either drop the lock or do the tick processing.
573 		 */
574 		mutex_enter(&thread_free_lock);
575 		/*
576 		 * We cannot hold the cpu_lock to prevent the
577 		 * cpu_list from changing in the clock interrupt.
578 		 * As long as we don't block (or don't get pre-empted)
579 		 * the cpu_list will not change (all threads are paused
580 		 * before list modification). If the list does change
581 		 * any deleted cpu structures will remain with cpu_next
582 		 * set to NULL, hence the following test.
583 		 */
584 		if (cp->cpu_next == NULL) {
585 			mutex_exit(&thread_free_lock);
586 			break;
587 		}
588 		t = cp->cpu_thread;	/* Current running thread */
589 		if (CPU == cp) {
590 			/*
591 			 * 't' will be the clock interrupt thread on this
592 			 * CPU.  Use the pinned thread (if any) on this CPU
593 			 * as the target of the clock tick.  If we pinned
594 			 * an interrupt, though, just keep using the clock
595 			 * interrupt thread since the formerly pinned one
596 			 * may have gone away.  One interrupt thread is as
597 			 * good as another, and this means we don't have
598 			 * to continue to check pinned_intr in subsequent
599 			 * code.
600 			 */
601 			ASSERT(t == curthread);
602 			if (t->t_intr != NULL && !pinned_intr)
603 				t = t->t_intr;
604 		}
605 
606 		intr = t->t_flag & T_INTR_THREAD;
607 		lwp = ttolwp(t);
608 		if (lwp == NULL || (t->t_proc_flag & TP_LWPEXIT) || intr) {
609 			/*
610 			 * Thread is exiting (or uninteresting) so don't
611 			 * do tick processing or grab p_lock.  Once we
612 			 * drop thread_free_lock we can't look inside the
613 			 * thread or lwp structure, since the thread may
614 			 * have gone away.
615 			 */
616 			exiting = 1;
617 		} else {
618 			/*
619 			 * OK, try to grab the process lock.  See
620 			 * comments above for why we're not using
621 			 * ttoproc(t)->p_lockp here.
622 			 */
623 			plockp = t->t_plockp;
624 			mutex_enter(plockp);
625 			/* See above comment. */
626 			if (cp->cpu_next == NULL) {
627 				mutex_exit(plockp);
628 				mutex_exit(&thread_free_lock);
629 				break;
630 			}
631 			/*
632 			 * The thread may have exited between when we
633 			 * checked above, and when we got the p_lock.
634 			 */
635 			if (t->t_proc_flag & TP_LWPEXIT) {
636 				mutex_exit(plockp);
637 				exiting = 1;
638 			} else {
639 				exiting = 0;
640 			}
641 		}
642 		/*
643 		 * Either we have the p_lock for the thread's process,
644 		 * or we don't care about the thread structure any more.
645 		 * Either way we can drop thread_free_lock.
646 		 */
647 		mutex_exit(&thread_free_lock);
648 
649 		/*
650 		 * Update user, system, and idle cpu times.
651 		 */
652 		if (one_sec) {
653 			nrunning++;
654 			cp->cpu_part->cp_nrunning++;
655 		}
656 		/*
657 		 * If we haven't done tick processing for this
658 		 * lwp, then do it now. Since we don't hold the
659 		 * lwp down on a CPU it can migrate and show up
660 		 * more than once, hence the lbolt check.
661 		 *
662 		 * Also, make sure that it's okay to perform the
663 		 * tick processing before calling clock_tick.
664 		 * Setting thread_away to a TRUE value (ie. not 0)
665 		 * results in tick processing not being performed for
666 		 * that thread.  Or, in other words, keeps the thread
667 		 * away from clock_tick processing.
668 		 */
669 		thread_away = ((cp->cpu_flags & CPU_QUIESCED) ||
670 		    CPU_ON_INTR(cp) || intr ||
671 		    (cp->cpu_dispthread == cp->cpu_idle_thread) || exiting);
672 
673 		if ((!thread_away) && (lbolt - t->t_lbolt != 0)) {
674 			t->t_lbolt = lbolt;
675 			clock_tick(t);
676 		}
677 
678 #ifdef KSLICE
679 		/*
680 		 * Ah what the heck, give this kid a taste of the real
681 		 * world and yank the rug out from under it.
682 		 * But, only if we are running UniProcessor.
683 		 */
684 		if ((kslice) && (ncpus == 1)) {
685 			aston(t);
686 			cp->cpu_runrun = 1;
687 			cp->cpu_kprunrun = 1;
688 		}
689 #endif
690 		if (!exiting)
691 			mutex_exit(plockp);
692 	} while ((cp = cp->cpu_next) != cpu_list);
693 
694 	/*
695 	 * bump time in ticks
696 	 *
697 	 * We rely on there being only one clock thread and hence
698 	 * don't need a lock to protect lbolt.
699 	 */
700 	lbolt++;
701 	atomic_add_64((uint64_t *)&lbolt64, (int64_t)1);
702 
703 	/*
704 	 * Check for a callout that needs be called from the clock
705 	 * thread to support the membership protocol in a clustered
706 	 * system.  Copy the function pointer so that we can reset
707 	 * this to NULL if needed.
708 	 */
709 	if ((funcp = cmm_clock_callout) != NULL)
710 		(*funcp)();
711 
712 	/*
713 	 * Wakeup the cageout thread waiters once per second.
714 	 */
715 	if (one_sec)
716 		kcage_tick();
717 
718 	/*
719 	 * Schedule timeout() requests if any are due at this time.
720 	 */
721 	callout_schedule();
722 
723 	if (one_sec) {
724 
725 		int drift, absdrift;
726 		timestruc_t tod;
727 		int s;
728 
729 		/*
730 		 * Beginning of precision-kernel code fragment executed
731 		 * every second.
732 		 *
733 		 * On rollover of the second the phase adjustment to be
734 		 * used for the next second is calculated.  Also, the
735 		 * maximum error is increased by the tolerance.  If the
736 		 * PPS frequency discipline code is present, the phase is
737 		 * increased to compensate for the CPU clock oscillator
738 		 * frequency error.
739 		 *
740 		 * On a 32-bit machine and given parameters in the timex.h
741 		 * header file, the maximum phase adjustment is +-512 ms
742 		 * and maximum frequency offset is (a tad less than)
743 		 * +-512 ppm. On a 64-bit machine, you shouldn't need to ask.
744 		 */
745 		time_maxerror += time_tolerance / SCALE_USEC;
746 
747 		/*
748 		 * Leap second processing. If in leap-insert state at
749 		 * the end of the day, the system clock is set back one
750 		 * second; if in leap-delete state, the system clock is
751 		 * set ahead one second. The microtime() routine or
752 		 * external clock driver will insure that reported time
753 		 * is always monotonic. The ugly divides should be
754 		 * replaced.
755 		 */
756 		switch (time_state) {
757 
758 		case TIME_OK:
759 			if (time_status & STA_INS)
760 				time_state = TIME_INS;
761 			else if (time_status & STA_DEL)
762 				time_state = TIME_DEL;
763 			break;
764 
765 		case TIME_INS:
766 			if (hrestime.tv_sec % 86400 == 0) {
767 				s = hr_clock_lock();
768 				hrestime.tv_sec--;
769 				hr_clock_unlock(s);
770 				time_state = TIME_OOP;
771 			}
772 			break;
773 
774 		case TIME_DEL:
775 			if ((hrestime.tv_sec + 1) % 86400 == 0) {
776 				s = hr_clock_lock();
777 				hrestime.tv_sec++;
778 				hr_clock_unlock(s);
779 				time_state = TIME_WAIT;
780 			}
781 			break;
782 
783 		case TIME_OOP:
784 			time_state = TIME_WAIT;
785 			break;
786 
787 		case TIME_WAIT:
788 			if (!(time_status & (STA_INS | STA_DEL)))
789 				time_state = TIME_OK;
790 		default:
791 			break;
792 		}
793 
794 		/*
795 		 * Compute the phase adjustment for the next second. In
796 		 * PLL mode, the offset is reduced by a fixed factor
797 		 * times the time constant. In FLL mode the offset is
798 		 * used directly. In either mode, the maximum phase
799 		 * adjustment for each second is clamped so as to spread
800 		 * the adjustment over not more than the number of
801 		 * seconds between updates.
802 		 */
803 		if (time_offset == 0)
804 			time_adj = 0;
805 		else if (time_offset < 0) {
806 			lltemp = -time_offset;
807 			if (!(time_status & STA_FLL)) {
808 				if ((1 << time_constant) >= SCALE_KG)
809 					lltemp *= (1 << time_constant) /
810 					    SCALE_KG;
811 				else
812 					lltemp = (lltemp / SCALE_KG) >>
813 					    time_constant;
814 			}
815 			if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
816 				lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
817 			time_offset += lltemp;
818 			time_adj = -(lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
819 		} else {
820 			lltemp = time_offset;
821 			if (!(time_status & STA_FLL)) {
822 				if ((1 << time_constant) >= SCALE_KG)
823 					lltemp *= (1 << time_constant) /
824 					    SCALE_KG;
825 				else
826 					lltemp = (lltemp / SCALE_KG) >>
827 					    time_constant;
828 			}
829 			if (lltemp > (MAXPHASE / MINSEC) * SCALE_UPDATE)
830 				lltemp = (MAXPHASE / MINSEC) * SCALE_UPDATE;
831 			time_offset -= lltemp;
832 			time_adj = (lltemp * SCALE_PHASE) / hz / SCALE_UPDATE;
833 		}
834 
835 		/*
836 		 * Compute the frequency estimate and additional phase
837 		 * adjustment due to frequency error for the next
838 		 * second. When the PPS signal is engaged, gnaw on the
839 		 * watchdog counter and update the frequency computed by
840 		 * the pll and the PPS signal.
841 		 */
842 		pps_valid++;
843 		if (pps_valid == PPS_VALID) {
844 			pps_jitter = MAXTIME;
845 			pps_stabil = MAXFREQ;
846 			time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
847 			    STA_PPSWANDER | STA_PPSERROR);
848 		}
849 		lltemp = time_freq + pps_freq;
850 
851 		if (lltemp)
852 			time_adj += (lltemp * SCALE_PHASE) / (SCALE_USEC * hz);
853 
854 		/*
855 		 * End of precision kernel-code fragment
856 		 *
857 		 * The section below should be modified if we are planning
858 		 * to use NTP for synchronization.
859 		 *
860 		 * Note: the clock synchronization code now assumes
861 		 * the following:
862 		 *   - if dosynctodr is 1, then compute the drift between
863 		 *	the tod chip and software time and adjust one or
864 		 *	the other depending on the circumstances
865 		 *
866 		 *   - if dosynctodr is 0, then the tod chip is independent
867 		 *	of the software clock and should not be adjusted,
868 		 *	but allowed to free run.  this allows NTP to sync.
869 		 *	hrestime without any interference from the tod chip.
870 		 */
871 
872 		tod_validate_deferred = B_FALSE;
873 		mutex_enter(&tod_lock);
874 		tod = tod_get();
875 		drift = tod.tv_sec - hrestime.tv_sec;
876 		absdrift = (drift >= 0) ? drift : -drift;
877 		if (tod_needsync || absdrift > 1) {
878 			int s;
879 			if (absdrift > 2) {
880 				if (!tod_broken && tod_faulted == TOD_NOFAULT) {
881 					s = hr_clock_lock();
882 					hrestime = tod;
883 					membar_enter();	/* hrestime visible */
884 					timedelta = 0;
885 					timechanged++;
886 					tod_needsync = 0;
887 					hr_clock_unlock(s);
888 				}
889 			} else {
890 				if (tod_needsync || !dosynctodr) {
891 					gethrestime(&tod);
892 					tod_set(tod);
893 					s = hr_clock_lock();
894 					if (timedelta == 0)
895 						tod_needsync = 0;
896 					hr_clock_unlock(s);
897 				} else {
898 					/*
899 					 * If the drift is 2 seconds on the
900 					 * money, then the TOD is adjusting
901 					 * the clock;  record that.
902 					 */
903 					clock_adj_hist[adj_hist_entry++ %
904 					    CLOCK_ADJ_HIST_SIZE] = lbolt64;
905 					s = hr_clock_lock();
906 					timedelta = (int64_t)drift*NANOSEC;
907 					hr_clock_unlock(s);
908 				}
909 			}
910 		}
911 		one_sec = 0;
912 		time = gethrestime_sec();  /* for crusty old kmem readers */
913 		mutex_exit(&tod_lock);
914 
915 		/*
916 		 * Some drivers still depend on this... XXX
917 		 */
918 		cv_broadcast(&lbolt_cv);
919 
920 		sysinfo.updates++;
921 		vminfo.freemem += freemem;
922 		{
923 			pgcnt_t maxswap, resv, free;
924 			pgcnt_t avail =
925 			    MAX((spgcnt_t)(availrmem - swapfs_minfree), 0);
926 
927 			maxswap = k_anoninfo.ani_mem_resv
928 					+ k_anoninfo.ani_max +avail;
929 			free = k_anoninfo.ani_free + avail;
930 			resv = k_anoninfo.ani_phys_resv +
931 			    k_anoninfo.ani_mem_resv;
932 
933 			vminfo.swap_resv += resv;
934 			/* number of reserved and allocated pages */
935 #ifdef	DEBUG
936 			if (maxswap < free)
937 				cmn_err(CE_WARN, "clock: maxswap < free");
938 			if (maxswap < resv)
939 				cmn_err(CE_WARN, "clock: maxswap < resv");
940 #endif
941 			vminfo.swap_alloc += maxswap - free;
942 			vminfo.swap_avail += maxswap - resv;
943 			vminfo.swap_free += free;
944 		}
945 		if (nrunnable) {
946 			sysinfo.runque += nrunnable;
947 			sysinfo.runocc++;
948 		}
949 		if (nswapped) {
950 			sysinfo.swpque += nswapped;
951 			sysinfo.swpocc++;
952 		}
953 		sysinfo.waiting += w_io;
954 
955 		/*
956 		 * Wake up fsflush to write out DELWRI
957 		 * buffers, dirty pages and other cached
958 		 * administrative data, e.g. inodes.
959 		 */
960 		if (--fsflushcnt <= 0) {
961 			fsflushcnt = tune.t_fsflushr;
962 			cv_signal(&fsflush_cv);
963 		}
964 
965 		vmmeter();
966 		calcloadavg(genloadavg(&loadavg), hp_avenrun);
967 		for (i = 0; i < 3; i++)
968 			/*
969 			 * At the moment avenrun[] can only hold 31
970 			 * bits of load average as it is a signed
971 			 * int in the API. We need to ensure that
972 			 * hp_avenrun[i] >> (16 - FSHIFT) will not be
973 			 * too large. If it is, we put the largest value
974 			 * that we can use into avenrun[i]. This is
975 			 * kludgey, but about all we can do until we
976 			 * avenrun[] is declared as an array of uint64[]
977 			 */
978 			if (hp_avenrun[i] < ((uint64_t)1<<(31+16-FSHIFT)))
979 				avenrun[i] = (int32_t)(hp_avenrun[i] >>
980 				    (16 - FSHIFT));
981 			else
982 				avenrun[i] = 0x7fffffff;
983 
984 		cpupart = cp_list_head;
985 		do {
986 			calcloadavg(genloadavg(&cpupart->cp_loadavg),
987 			    cpupart->cp_hp_avenrun);
988 		} while ((cpupart = cpupart->cp_next) != cp_list_head);
989 
990 		/*
991 		 * Wake up the swapper thread if necessary.
992 		 */
993 		if (runin ||
994 		    (runout && (avefree < desfree || wake_sched_sec))) {
995 			t = &t0;
996 			thread_lock(t);
997 			if (t->t_state == TS_STOPPED) {
998 				runin = runout = 0;
999 				wake_sched_sec = 0;
1000 				t->t_whystop = 0;
1001 				t->t_whatstop = 0;
1002 				t->t_schedflag &= ~TS_ALLSTART;
1003 				THREAD_TRANSITION(t);
1004 				setfrontdq(t);
1005 			}
1006 			thread_unlock(t);
1007 		}
1008 	}
1009 
1010 	/*
1011 	 * Wake up the swapper if any high priority swapped-out threads
1012 	 * became runable during the last tick.
1013 	 */
1014 	if (wake_sched) {
1015 		t = &t0;
1016 		thread_lock(t);
1017 		if (t->t_state == TS_STOPPED) {
1018 			runin = runout = 0;
1019 			wake_sched = 0;
1020 			t->t_whystop = 0;
1021 			t->t_whatstop = 0;
1022 			t->t_schedflag &= ~TS_ALLSTART;
1023 			THREAD_TRANSITION(t);
1024 			setfrontdq(t);
1025 		}
1026 		thread_unlock(t);
1027 	}
1028 }
1029 
1030 void
1031 clock_init(void)
1032 {
1033 	cyc_handler_t hdlr;
1034 	cyc_time_t when;
1035 
1036 	hdlr.cyh_func = (cyc_func_t)clock;
1037 	hdlr.cyh_level = CY_LOCK_LEVEL;
1038 	hdlr.cyh_arg = NULL;
1039 
1040 	when.cyt_when = 0;
1041 	when.cyt_interval = nsec_per_tick;
1042 
1043 	mutex_enter(&cpu_lock);
1044 	clock_cyclic = cyclic_add(&hdlr, &when);
1045 	mutex_exit(&cpu_lock);
1046 }
1047 
1048 /*
1049  * Called before calcloadavg to get 10-sec moving loadavg together
1050  */
1051 
1052 static int
1053 genloadavg(struct loadavg_s *avgs)
1054 {
1055 	int avg;
1056 	int spos; /* starting position */
1057 	int cpos; /* moving current position */
1058 	int i;
1059 	int slen;
1060 	hrtime_t hr_avg;
1061 
1062 	/* 10-second snapshot, calculate first positon */
1063 	if (avgs->lg_len == 0) {
1064 		return (0);
1065 	}
1066 	slen = avgs->lg_len < S_MOVAVG_SZ ? avgs->lg_len : S_MOVAVG_SZ;
1067 
1068 	spos = (avgs->lg_cur - 1) >= 0 ? avgs->lg_cur - 1 :
1069 	    S_LOADAVG_SZ + (avgs->lg_cur - 1);
1070 	for (i = hr_avg = 0; i < slen; i++) {
1071 		cpos = (spos - i) >= 0 ? spos - i : S_LOADAVG_SZ + (spos - i);
1072 		hr_avg += avgs->lg_loads[cpos];
1073 	}
1074 
1075 	hr_avg = hr_avg / slen;
1076 	avg = hr_avg / (NANOSEC / LGRP_LOADAVG_IN_THREAD_MAX);
1077 
1078 	return (avg);
1079 }
1080 
1081 /*
1082  * Run every second from clock () to update the loadavg count available to the
1083  * system and cpu-partitions.
1084  *
1085  * This works by sampling the previous usr, sys, wait time elapsed,
1086  * computing a delta, and adding that delta to the elapsed usr, sys,
1087  * wait increase.
1088  */
1089 
1090 static void
1091 loadavg_update()
1092 {
1093 	cpu_t *cp;
1094 	cpupart_t *cpupart;
1095 	hrtime_t cpu_total;
1096 	int prev;
1097 
1098 	cp = cpu_list;
1099 	loadavg.lg_total = 0;
1100 
1101 	/*
1102 	 * first pass totals up per-cpu statistics for system and cpu
1103 	 * partitions
1104 	 */
1105 
1106 	do {
1107 		struct loadavg_s *lavg;
1108 
1109 		lavg = &cp->cpu_loadavg;
1110 
1111 		cpu_total = cp->cpu_acct[CMS_USER] +
1112 		    cp->cpu_acct[CMS_SYSTEM] + cp->cpu_waitrq;
1113 		/* compute delta against last total */
1114 		scalehrtime(&cpu_total);
1115 		prev = (lavg->lg_cur - 1) >= 0 ? lavg->lg_cur - 1 :
1116 		    S_LOADAVG_SZ + (lavg->lg_cur - 1);
1117 		if (lavg->lg_loads[prev] <= 0) {
1118 			lavg->lg_loads[lavg->lg_cur] = cpu_total;
1119 			cpu_total = 0;
1120 		} else {
1121 			lavg->lg_loads[lavg->lg_cur] = cpu_total;
1122 			cpu_total = cpu_total - lavg->lg_loads[prev];
1123 			if (cpu_total < 0)
1124 				cpu_total = 0;
1125 		}
1126 
1127 		lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1128 		lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1129 		    lavg->lg_len + 1 : S_LOADAVG_SZ;
1130 
1131 		loadavg.lg_total += cpu_total;
1132 		cp->cpu_part->cp_loadavg.lg_total += cpu_total;
1133 
1134 	} while ((cp = cp->cpu_next) != cpu_list);
1135 
1136 	loadavg.lg_loads[loadavg.lg_cur] = loadavg.lg_total;
1137 	loadavg.lg_cur = (loadavg.lg_cur + 1) % S_LOADAVG_SZ;
1138 	loadavg.lg_len = (loadavg.lg_len + 1) < S_LOADAVG_SZ ?
1139 	    loadavg.lg_len + 1 : S_LOADAVG_SZ;
1140 	/*
1141 	 * Second pass updates counts
1142 	 */
1143 	cpupart = cp_list_head;
1144 
1145 	do {
1146 		struct loadavg_s *lavg;
1147 
1148 		lavg = &cpupart->cp_loadavg;
1149 		lavg->lg_loads[lavg->lg_cur] = lavg->lg_total;
1150 		lavg->lg_total = 0;
1151 		lavg->lg_cur = (lavg->lg_cur + 1) % S_LOADAVG_SZ;
1152 		lavg->lg_len = (lavg->lg_len + 1) < S_LOADAVG_SZ ?
1153 		    lavg->lg_len + 1 : S_LOADAVG_SZ;
1154 
1155 	} while ((cpupart = cpupart->cp_next) != cp_list_head);
1156 
1157 }
1158 
1159 /*
1160  * clock_update() - local clock update
1161  *
1162  * This routine is called by ntp_adjtime() to update the local clock
1163  * phase and frequency. The implementation is of an
1164  * adaptive-parameter, hybrid phase/frequency-lock loop (PLL/FLL). The
1165  * routine computes new time and frequency offset estimates for each
1166  * call.  The PPS signal itself determines the new time offset,
1167  * instead of the calling argument.  Presumably, calls to
1168  * ntp_adjtime() occur only when the caller believes the local clock
1169  * is valid within some bound (+-128 ms with NTP). If the caller's
1170  * time is far different than the PPS time, an argument will ensue,
1171  * and it's not clear who will lose.
1172  *
1173  * For uncompensated quartz crystal oscillatores and nominal update
1174  * intervals less than 1024 s, operation should be in phase-lock mode
1175  * (STA_FLL = 0), where the loop is disciplined to phase. For update
1176  * intervals greater than this, operation should be in frequency-lock
1177  * mode (STA_FLL = 1), where the loop is disciplined to frequency.
1178  *
1179  * Note: mutex(&tod_lock) is in effect.
1180  */
1181 void
1182 clock_update(int offset)
1183 {
1184 	int ltemp, mtemp, s;
1185 
1186 	ASSERT(MUTEX_HELD(&tod_lock));
1187 
1188 	if (!(time_status & STA_PLL) && !(time_status & STA_PPSTIME))
1189 		return;
1190 	ltemp = offset;
1191 	if ((time_status & STA_PPSTIME) && (time_status & STA_PPSSIGNAL))
1192 		ltemp = pps_offset;
1193 
1194 	/*
1195 	 * Scale the phase adjustment and clamp to the operating range.
1196 	 */
1197 	if (ltemp > MAXPHASE)
1198 		time_offset = MAXPHASE * SCALE_UPDATE;
1199 	else if (ltemp < -MAXPHASE)
1200 		time_offset = -(MAXPHASE * SCALE_UPDATE);
1201 	else
1202 		time_offset = ltemp * SCALE_UPDATE;
1203 
1204 	/*
1205 	 * Select whether the frequency is to be controlled and in which
1206 	 * mode (PLL or FLL). Clamp to the operating range. Ugly
1207 	 * multiply/divide should be replaced someday.
1208 	 */
1209 	if (time_status & STA_FREQHOLD || time_reftime == 0)
1210 		time_reftime = hrestime.tv_sec;
1211 
1212 	mtemp = hrestime.tv_sec - time_reftime;
1213 	time_reftime = hrestime.tv_sec;
1214 
1215 	if (time_status & STA_FLL) {
1216 		if (mtemp >= MINSEC) {
1217 			ltemp = ((time_offset / mtemp) * (SCALE_USEC /
1218 			    SCALE_UPDATE));
1219 			if (ltemp)
1220 				time_freq += ltemp / SCALE_KH;
1221 		}
1222 	} else {
1223 		if (mtemp < MAXSEC) {
1224 			ltemp *= mtemp;
1225 			if (ltemp)
1226 				time_freq += (int)(((int64_t)ltemp *
1227 				    SCALE_USEC) / SCALE_KF)
1228 				    / (1 << (time_constant * 2));
1229 		}
1230 	}
1231 	if (time_freq > time_tolerance)
1232 		time_freq = time_tolerance;
1233 	else if (time_freq < -time_tolerance)
1234 		time_freq = -time_tolerance;
1235 
1236 	s = hr_clock_lock();
1237 	tod_needsync = 1;
1238 	hr_clock_unlock(s);
1239 }
1240 
1241 /*
1242  * ddi_hardpps() - discipline CPU clock oscillator to external PPS signal
1243  *
1244  * This routine is called at each PPS interrupt in order to discipline
1245  * the CPU clock oscillator to the PPS signal. It measures the PPS phase
1246  * and leaves it in a handy spot for the clock() routine. It
1247  * integrates successive PPS phase differences and calculates the
1248  * frequency offset. This is used in clock() to discipline the CPU
1249  * clock oscillator so that intrinsic frequency error is cancelled out.
1250  * The code requires the caller to capture the time and hardware counter
1251  * value at the on-time PPS signal transition.
1252  *
1253  * Note that, on some Unix systems, this routine runs at an interrupt
1254  * priority level higher than the timer interrupt routine clock().
1255  * Therefore, the variables used are distinct from the clock()
1256  * variables, except for certain exceptions: The PPS frequency pps_freq
1257  * and phase pps_offset variables are determined by this routine and
1258  * updated atomically. The time_tolerance variable can be considered a
1259  * constant, since it is infrequently changed, and then only when the
1260  * PPS signal is disabled. The watchdog counter pps_valid is updated
1261  * once per second by clock() and is atomically cleared in this
1262  * routine.
1263  *
1264  * tvp is the time of the last tick; usec is a microsecond count since the
1265  * last tick.
1266  *
1267  * Note: In Solaris systems, the tick value is actually given by
1268  *       usec_per_tick.  This is called from the serial driver cdintr(),
1269  *	 or equivalent, at a high PIL.  Because the kernel keeps a
1270  *	 highresolution time, the following code can accept either
1271  *	 the traditional argument pair, or the current highres timestamp
1272  *       in tvp and zero in usec.
1273  */
1274 void
1275 ddi_hardpps(struct timeval *tvp, int usec)
1276 {
1277 	int u_usec, v_usec, bigtick;
1278 	time_t cal_sec;
1279 	int cal_usec;
1280 
1281 	/*
1282 	 * An occasional glitch can be produced when the PPS interrupt
1283 	 * occurs in the clock() routine before the time variable is
1284 	 * updated. Here the offset is discarded when the difference
1285 	 * between it and the last one is greater than tick/2, but not
1286 	 * if the interval since the first discard exceeds 30 s.
1287 	 */
1288 	time_status |= STA_PPSSIGNAL;
1289 	time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1290 	pps_valid = 0;
1291 	u_usec = -tvp->tv_usec;
1292 	if (u_usec < -(MICROSEC/2))
1293 		u_usec += MICROSEC;
1294 	v_usec = pps_offset - u_usec;
1295 	if (v_usec < 0)
1296 		v_usec = -v_usec;
1297 	if (v_usec > (usec_per_tick >> 1)) {
1298 		if (pps_glitch > MAXGLITCH) {
1299 			pps_glitch = 0;
1300 			pps_tf[2] = u_usec;
1301 			pps_tf[1] = u_usec;
1302 		} else {
1303 			pps_glitch++;
1304 			u_usec = pps_offset;
1305 		}
1306 	} else
1307 		pps_glitch = 0;
1308 
1309 	/*
1310 	 * A three-stage median filter is used to help deglitch the pps
1311 	 * time. The median sample becomes the time offset estimate; the
1312 	 * difference between the other two samples becomes the time
1313 	 * dispersion (jitter) estimate.
1314 	 */
1315 	pps_tf[2] = pps_tf[1];
1316 	pps_tf[1] = pps_tf[0];
1317 	pps_tf[0] = u_usec;
1318 	if (pps_tf[0] > pps_tf[1]) {
1319 		if (pps_tf[1] > pps_tf[2]) {
1320 			pps_offset = pps_tf[1];		/* 0 1 2 */
1321 			v_usec = pps_tf[0] - pps_tf[2];
1322 		} else if (pps_tf[2] > pps_tf[0]) {
1323 			pps_offset = pps_tf[0];		/* 2 0 1 */
1324 			v_usec = pps_tf[2] - pps_tf[1];
1325 		} else {
1326 			pps_offset = pps_tf[2];		/* 0 2 1 */
1327 			v_usec = pps_tf[0] - pps_tf[1];
1328 		}
1329 	} else {
1330 		if (pps_tf[1] < pps_tf[2]) {
1331 			pps_offset = pps_tf[1];		/* 2 1 0 */
1332 			v_usec = pps_tf[2] - pps_tf[0];
1333 		} else  if (pps_tf[2] < pps_tf[0]) {
1334 			pps_offset = pps_tf[0];		/* 1 0 2 */
1335 			v_usec = pps_tf[1] - pps_tf[2];
1336 		} else {
1337 			pps_offset = pps_tf[2];		/* 1 2 0 */
1338 			v_usec = pps_tf[1] - pps_tf[0];
1339 		}
1340 	}
1341 	if (v_usec > MAXTIME)
1342 		pps_jitcnt++;
1343 	v_usec = (v_usec << PPS_AVG) - pps_jitter;
1344 	pps_jitter += v_usec / (1 << PPS_AVG);
1345 	if (pps_jitter > (MAXTIME >> 1))
1346 		time_status |= STA_PPSJITTER;
1347 
1348 	/*
1349 	 * During the calibration interval adjust the starting time when
1350 	 * the tick overflows. At the end of the interval compute the
1351 	 * duration of the interval and the difference of the hardware
1352 	 * counters at the beginning and end of the interval. This code
1353 	 * is deliciously complicated by the fact valid differences may
1354 	 * exceed the value of tick when using long calibration
1355 	 * intervals and small ticks. Note that the counter can be
1356 	 * greater than tick if caught at just the wrong instant, but
1357 	 * the values returned and used here are correct.
1358 	 */
1359 	bigtick = (int)usec_per_tick * SCALE_USEC;
1360 	pps_usec -= pps_freq;
1361 	if (pps_usec >= bigtick)
1362 		pps_usec -= bigtick;
1363 	if (pps_usec < 0)
1364 		pps_usec += bigtick;
1365 	pps_time.tv_sec++;
1366 	pps_count++;
1367 	if (pps_count < (1 << pps_shift))
1368 		return;
1369 	pps_count = 0;
1370 	pps_calcnt++;
1371 	u_usec = usec * SCALE_USEC;
1372 	v_usec = pps_usec - u_usec;
1373 	if (v_usec >= bigtick >> 1)
1374 		v_usec -= bigtick;
1375 	if (v_usec < -(bigtick >> 1))
1376 		v_usec += bigtick;
1377 	if (v_usec < 0)
1378 		v_usec = -(-v_usec >> pps_shift);
1379 	else
1380 		v_usec = v_usec >> pps_shift;
1381 	pps_usec = u_usec;
1382 	cal_sec = tvp->tv_sec;
1383 	cal_usec = tvp->tv_usec;
1384 	cal_sec -= pps_time.tv_sec;
1385 	cal_usec -= pps_time.tv_usec;
1386 	if (cal_usec < 0) {
1387 		cal_usec += MICROSEC;
1388 		cal_sec--;
1389 	}
1390 	pps_time = *tvp;
1391 
1392 	/*
1393 	 * Check for lost interrupts, noise, excessive jitter and
1394 	 * excessive frequency error. The number of timer ticks during
1395 	 * the interval may vary +-1 tick. Add to this a margin of one
1396 	 * tick for the PPS signal jitter and maximum frequency
1397 	 * deviation. If the limits are exceeded, the calibration
1398 	 * interval is reset to the minimum and we start over.
1399 	 */
1400 	u_usec = (int)usec_per_tick << 1;
1401 	if (!((cal_sec == -1 && cal_usec > (MICROSEC - u_usec)) ||
1402 	    (cal_sec == 0 && cal_usec < u_usec)) ||
1403 	    v_usec > time_tolerance || v_usec < -time_tolerance) {
1404 		pps_errcnt++;
1405 		pps_shift = PPS_SHIFT;
1406 		pps_intcnt = 0;
1407 		time_status |= STA_PPSERROR;
1408 		return;
1409 	}
1410 
1411 	/*
1412 	 * A three-stage median filter is used to help deglitch the pps
1413 	 * frequency. The median sample becomes the frequency offset
1414 	 * estimate; the difference between the other two samples
1415 	 * becomes the frequency dispersion (stability) estimate.
1416 	 */
1417 	pps_ff[2] = pps_ff[1];
1418 	pps_ff[1] = pps_ff[0];
1419 	pps_ff[0] = v_usec;
1420 	if (pps_ff[0] > pps_ff[1]) {
1421 		if (pps_ff[1] > pps_ff[2]) {
1422 			u_usec = pps_ff[1];		/* 0 1 2 */
1423 			v_usec = pps_ff[0] - pps_ff[2];
1424 		} else if (pps_ff[2] > pps_ff[0]) {
1425 			u_usec = pps_ff[0];		/* 2 0 1 */
1426 			v_usec = pps_ff[2] - pps_ff[1];
1427 		} else {
1428 			u_usec = pps_ff[2];		/* 0 2 1 */
1429 			v_usec = pps_ff[0] - pps_ff[1];
1430 		}
1431 	} else {
1432 		if (pps_ff[1] < pps_ff[2]) {
1433 			u_usec = pps_ff[1];		/* 2 1 0 */
1434 			v_usec = pps_ff[2] - pps_ff[0];
1435 		} else  if (pps_ff[2] < pps_ff[0]) {
1436 			u_usec = pps_ff[0];		/* 1 0 2 */
1437 			v_usec = pps_ff[1] - pps_ff[2];
1438 		} else {
1439 			u_usec = pps_ff[2];		/* 1 2 0 */
1440 			v_usec = pps_ff[1] - pps_ff[0];
1441 		}
1442 	}
1443 
1444 	/*
1445 	 * Here the frequency dispersion (stability) is updated. If it
1446 	 * is less than one-fourth the maximum (MAXFREQ), the frequency
1447 	 * offset is updated as well, but clamped to the tolerance. It
1448 	 * will be processed later by the clock() routine.
1449 	 */
1450 	v_usec = (v_usec >> 1) - pps_stabil;
1451 	if (v_usec < 0)
1452 		pps_stabil -= -v_usec >> PPS_AVG;
1453 	else
1454 		pps_stabil += v_usec >> PPS_AVG;
1455 	if (pps_stabil > MAXFREQ >> 2) {
1456 		pps_stbcnt++;
1457 		time_status |= STA_PPSWANDER;
1458 		return;
1459 	}
1460 	if (time_status & STA_PPSFREQ) {
1461 		if (u_usec < 0) {
1462 			pps_freq -= -u_usec >> PPS_AVG;
1463 			if (pps_freq < -time_tolerance)
1464 				pps_freq = -time_tolerance;
1465 			u_usec = -u_usec;
1466 		} else {
1467 			pps_freq += u_usec >> PPS_AVG;
1468 			if (pps_freq > time_tolerance)
1469 				pps_freq = time_tolerance;
1470 		}
1471 	}
1472 
1473 	/*
1474 	 * Here the calibration interval is adjusted. If the maximum
1475 	 * time difference is greater than tick / 4, reduce the interval
1476 	 * by half. If this is not the case for four consecutive
1477 	 * intervals, double the interval.
1478 	 */
1479 	if (u_usec << pps_shift > bigtick >> 2) {
1480 		pps_intcnt = 0;
1481 		if (pps_shift > PPS_SHIFT)
1482 			pps_shift--;
1483 	} else if (pps_intcnt >= 4) {
1484 		pps_intcnt = 0;
1485 		if (pps_shift < PPS_SHIFTMAX)
1486 			pps_shift++;
1487 	} else
1488 		pps_intcnt++;
1489 
1490 	/*
1491 	 * If recovering from kmdb, then make sure the tod chip gets resynced.
1492 	 * If we took an early exit above, then we don't yet have a stable
1493 	 * calibration signal to lock onto, so don't mark the tod for sync
1494 	 * until we get all the way here.
1495 	 */
1496 	{
1497 		int s = hr_clock_lock();
1498 
1499 		tod_needsync = 1;
1500 		hr_clock_unlock(s);
1501 	}
1502 }
1503 
1504 /*
1505  * Handle clock tick processing for a thread.
1506  * Check for timer action, enforce CPU rlimit, do profiling etc.
1507  */
1508 void
1509 clock_tick(kthread_t *t)
1510 {
1511 	struct proc *pp;
1512 	klwp_id_t    lwp;
1513 	struct as *as;
1514 	clock_t	utime;
1515 	clock_t	stime;
1516 	int	poke = 0;		/* notify another CPU */
1517 	int	user_mode;
1518 	size_t	 rss;
1519 
1520 	/* Must be operating on a lwp/thread */
1521 	if ((lwp = ttolwp(t)) == NULL) {
1522 		panic("clock_tick: no lwp");
1523 		/*NOTREACHED*/
1524 	}
1525 
1526 	CL_TICK(t);	/* Class specific tick processing */
1527 	DTRACE_SCHED1(tick, kthread_t *, t);
1528 
1529 	pp = ttoproc(t);
1530 
1531 	/* pp->p_lock makes sure that the thread does not exit */
1532 	ASSERT(MUTEX_HELD(&pp->p_lock));
1533 
1534 	user_mode = (lwp->lwp_state == LWP_USER);
1535 
1536 	/*
1537 	 * Update process times. Should use high res clock and state
1538 	 * changes instead of statistical sampling method. XXX
1539 	 */
1540 	if (user_mode) {
1541 		pp->p_utime++;
1542 		pp->p_task->tk_cpu_time++;
1543 	} else {
1544 		pp->p_stime++;
1545 		pp->p_task->tk_cpu_time++;
1546 	}
1547 	as = pp->p_as;
1548 
1549 	/*
1550 	 * Update user profiling statistics. Get the pc from the
1551 	 * lwp when the AST happens.
1552 	 */
1553 	if (pp->p_prof.pr_scale) {
1554 		atomic_add_32(&lwp->lwp_oweupc, 1);
1555 		if (user_mode) {
1556 			poke = 1;
1557 			aston(t);
1558 		}
1559 	}
1560 
1561 	utime = pp->p_utime;
1562 	stime = pp->p_stime;
1563 
1564 	/*
1565 	 * If CPU was in user state, process lwp-virtual time
1566 	 * interval timer.
1567 	 */
1568 	if (user_mode &&
1569 	    timerisset(&lwp->lwp_timer[ITIMER_VIRTUAL].it_value) &&
1570 	    itimerdecr(&lwp->lwp_timer[ITIMER_VIRTUAL], usec_per_tick) == 0) {
1571 		poke = 1;
1572 		sigtoproc(pp, t, SIGVTALRM);
1573 	}
1574 
1575 	if (timerisset(&lwp->lwp_timer[ITIMER_PROF].it_value) &&
1576 	    itimerdecr(&lwp->lwp_timer[ITIMER_PROF], usec_per_tick) == 0) {
1577 		poke = 1;
1578 		sigtoproc(pp, t, SIGPROF);
1579 	}
1580 
1581 	/*
1582 	 * Enforce CPU resource controls:
1583 	 *   (a) process.max-cpu-time resource control
1584 	 */
1585 	(void) rctl_test(rctlproc_legacy[RLIMIT_CPU], pp->p_rctls, pp,
1586 	    (utime + stime)/hz, RCA_UNSAFE_SIGINFO);
1587 
1588 	/*
1589 	 *   (b) task.max-cpu-time resource control
1590 	 */
1591 	(void) rctl_test(rc_task_cpu_time, pp->p_task->tk_rctls, pp, 1,
1592 	    RCA_UNSAFE_SIGINFO);
1593 
1594 	/*
1595 	 * Update memory usage for the currently running process.
1596 	 */
1597 	rss = rm_asrss(as);
1598 	PTOU(pp)->u_mem += rss;
1599 	if (rss > PTOU(pp)->u_mem_max)
1600 		PTOU(pp)->u_mem_max = rss;
1601 
1602 	/*
1603 	 * Notify the CPU the thread is running on.
1604 	 */
1605 	if (poke && t->t_cpu != CPU)
1606 		poke_cpu(t->t_cpu->cpu_id);
1607 }
1608 
1609 void
1610 profil_tick(uintptr_t upc)
1611 {
1612 	int ticks;
1613 	proc_t *p = ttoproc(curthread);
1614 	klwp_t *lwp = ttolwp(curthread);
1615 	struct prof *pr = &p->p_prof;
1616 
1617 	do {
1618 		ticks = lwp->lwp_oweupc;
1619 	} while (cas32(&lwp->lwp_oweupc, ticks, 0) != ticks);
1620 
1621 	mutex_enter(&p->p_pflock);
1622 	if (pr->pr_scale >= 2 && upc >= pr->pr_off) {
1623 		/*
1624 		 * Old-style profiling
1625 		 */
1626 		uint16_t *slot = pr->pr_base;
1627 		uint16_t old, new;
1628 		if (pr->pr_scale != 2) {
1629 			uintptr_t delta = upc - pr->pr_off;
1630 			uintptr_t byteoff = ((delta >> 16) * pr->pr_scale) +
1631 			    (((delta & 0xffff) * pr->pr_scale) >> 16);
1632 			if (byteoff >= (uintptr_t)pr->pr_size) {
1633 				mutex_exit(&p->p_pflock);
1634 				return;
1635 			}
1636 			slot += byteoff / sizeof (uint16_t);
1637 		}
1638 		if (fuword16(slot, &old) < 0 ||
1639 		    (new = old + ticks) > SHRT_MAX ||
1640 		    suword16(slot, new) < 0) {
1641 			pr->pr_scale = 0;
1642 		}
1643 	} else if (pr->pr_scale == 1) {
1644 		/*
1645 		 * PC Sampling
1646 		 */
1647 		model_t model = lwp_getdatamodel(lwp);
1648 		int result;
1649 #ifdef __lint
1650 		model = model;
1651 #endif
1652 		while (ticks-- > 0) {
1653 			if (pr->pr_samples == pr->pr_size) {
1654 				/* buffer full, turn off sampling */
1655 				pr->pr_scale = 0;
1656 				break;
1657 			}
1658 			switch (SIZEOF_PTR(model)) {
1659 			case sizeof (uint32_t):
1660 				result = suword32(pr->pr_base, (uint32_t)upc);
1661 				break;
1662 #ifdef _LP64
1663 			case sizeof (uint64_t):
1664 				result = suword64(pr->pr_base, (uint64_t)upc);
1665 				break;
1666 #endif
1667 			default:
1668 				cmn_err(CE_WARN, "profil_tick: unexpected "
1669 				    "data model");
1670 				result = -1;
1671 				break;
1672 			}
1673 			if (result != 0) {
1674 				pr->pr_scale = 0;
1675 				break;
1676 			}
1677 			pr->pr_base = (caddr_t)pr->pr_base + SIZEOF_PTR(model);
1678 			pr->pr_samples++;
1679 		}
1680 	}
1681 	mutex_exit(&p->p_pflock);
1682 }
1683 
1684 static void
1685 delay_wakeup(void *arg)
1686 {
1687 	kthread_t *t = arg;
1688 
1689 	mutex_enter(&t->t_delay_lock);
1690 	cv_signal(&t->t_delay_cv);
1691 	mutex_exit(&t->t_delay_lock);
1692 }
1693 
1694 void
1695 delay(clock_t ticks)
1696 {
1697 	kthread_t *t = curthread;
1698 	clock_t deadline = lbolt + ticks;
1699 	clock_t timeleft;
1700 	timeout_id_t id;
1701 
1702 	if (panicstr && ticks > 0) {
1703 		/*
1704 		 * Timeouts aren't running, so all we can do is spin.
1705 		 */
1706 		drv_usecwait(TICK_TO_USEC(ticks));
1707 		return;
1708 	}
1709 
1710 	while ((timeleft = deadline - lbolt) > 0) {
1711 		mutex_enter(&t->t_delay_lock);
1712 		id = timeout(delay_wakeup, t, timeleft);
1713 		cv_wait(&t->t_delay_cv, &t->t_delay_lock);
1714 		mutex_exit(&t->t_delay_lock);
1715 		(void) untimeout(id);
1716 	}
1717 }
1718 
1719 /*
1720  * Like delay, but interruptible by a signal.
1721  */
1722 int
1723 delay_sig(clock_t ticks)
1724 {
1725 	clock_t deadline = lbolt + ticks;
1726 	clock_t rc;
1727 
1728 	mutex_enter(&curthread->t_delay_lock);
1729 	do {
1730 		rc = cv_timedwait_sig(&curthread->t_delay_cv,
1731 		    &curthread->t_delay_lock, deadline);
1732 	} while (rc > 0);
1733 	mutex_exit(&curthread->t_delay_lock);
1734 	if (rc == 0)
1735 		return (EINTR);
1736 	return (0);
1737 }
1738 
1739 #define	SECONDS_PER_DAY 86400
1740 
1741 /*
1742  * Initialize the system time based on the TOD chip.  approx is used as
1743  * an approximation of time (e.g. from the filesystem) in the event that
1744  * the TOD chip has been cleared or is unresponsive.  An approx of -1
1745  * means the filesystem doesn't keep time.
1746  */
1747 void
1748 clkset(time_t approx)
1749 {
1750 	timestruc_t ts;
1751 	int spl;
1752 	int set_clock = 0;
1753 
1754 	mutex_enter(&tod_lock);
1755 	ts = tod_get();
1756 
1757 	if (ts.tv_sec > 365 * SECONDS_PER_DAY) {
1758 		/*
1759 		 * If the TOD chip is reporting some time after 1971,
1760 		 * then it probably didn't lose power or become otherwise
1761 		 * cleared in the recent past;  check to assure that
1762 		 * the time coming from the filesystem isn't in the future
1763 		 * according to the TOD chip.
1764 		 */
1765 		if (approx != -1 && approx > ts.tv_sec) {
1766 			cmn_err(CE_WARN, "Last shutdown is later "
1767 			    "than time on time-of-day chip; check date.");
1768 		}
1769 	} else {
1770 		/*
1771 		 * If the TOD chip isn't giving correct time, then set it to
1772 		 * the time that was passed in as a rough estimate.  If we
1773 		 * don't have an estimate, then set the clock back to a time
1774 		 * when Oliver North, ALF and Dire Straits were all on the
1775 		 * collective brain:  1987.
1776 		 */
1777 		timestruc_t tmp;
1778 		if (approx == -1)
1779 			ts.tv_sec = (1987 - 1970) * 365 * SECONDS_PER_DAY;
1780 		else
1781 			ts.tv_sec = approx;
1782 		ts.tv_nsec = 0;
1783 
1784 		/*
1785 		 * Attempt to write the new time to the TOD chip.  Set spl high
1786 		 * to avoid getting preempted between the tod_set and tod_get.
1787 		 */
1788 		spl = splhi();
1789 		tod_set(ts);
1790 		tmp = tod_get();
1791 		splx(spl);
1792 
1793 		if (tmp.tv_sec != ts.tv_sec && tmp.tv_sec != ts.tv_sec + 1) {
1794 			tod_broken = 1;
1795 			dosynctodr = 0;
1796 			cmn_err(CE_WARN, "Time-of-day chip unresponsive;"
1797 			    " dead batteries?");
1798 		} else {
1799 			cmn_err(CE_WARN, "Time-of-day chip had "
1800 			    "incorrect date; check and reset.");
1801 		}
1802 		set_clock = 1;
1803 	}
1804 
1805 	if (!boot_time) {
1806 		boot_time = ts.tv_sec;
1807 		set_clock = 1;
1808 	}
1809 
1810 	if (set_clock)
1811 		set_hrestime(&ts);
1812 
1813 	mutex_exit(&tod_lock);
1814 }
1815 
1816 int	timechanged;	/* for testing if the system time has been reset */
1817 
1818 void
1819 set_hrestime(timestruc_t *ts)
1820 {
1821 	int spl = hr_clock_lock();
1822 	hrestime = *ts;
1823 	membar_enter();	/* hrestime must be visible before timechanged++ */
1824 	timedelta = 0;
1825 	timechanged++;
1826 	hr_clock_unlock(spl);
1827 }
1828 
1829 static uint_t deadman_seconds;
1830 static uint32_t deadman_panics;
1831 static int deadman_enabled = 0;
1832 static int deadman_panic_timers = 1;
1833 
1834 static void
1835 deadman(void)
1836 {
1837 	if (panicstr) {
1838 		/*
1839 		 * During panic, other CPUs besides the panic
1840 		 * master continue to handle cyclics and some other
1841 		 * interrupts.  The code below is intended to be
1842 		 * single threaded, so any CPU other than the master
1843 		 * must keep out.
1844 		 */
1845 		if (CPU->cpu_id != panic_cpu.cpu_id)
1846 			return;
1847 
1848 		/*
1849 		 * If we're panicking, the deadman cyclic continues to increase
1850 		 * lbolt in case the dump device driver relies on this for
1851 		 * timeouts.  Note that we rely on deadman() being invoked once
1852 		 * per second, and credit lbolt and lbolt64 with hz ticks each.
1853 		 */
1854 		lbolt += hz;
1855 		lbolt64 += hz;
1856 
1857 #ifdef __sparc
1858 		watchdog_pat();
1859 #endif
1860 
1861 		if (!deadman_panic_timers)
1862 			return; /* allow all timers to be manually disabled */
1863 
1864 		/*
1865 		 * If we are generating a crash dump or syncing filesystems and
1866 		 * the corresponding timer is set, decrement it and re-enter
1867 		 * the panic code to abort it and advance to the next state.
1868 		 * The panic states and triggers are explained in panic.c.
1869 		 */
1870 		if (panic_dump) {
1871 			if (dump_timeleft && (--dump_timeleft == 0)) {
1872 				panic("panic dump timeout");
1873 				/*NOTREACHED*/
1874 			}
1875 		} else if (panic_sync) {
1876 			if (sync_timeleft && (--sync_timeleft == 0)) {
1877 				panic("panic sync timeout");
1878 				/*NOTREACHED*/
1879 			}
1880 		}
1881 
1882 		return;
1883 	}
1884 
1885 	if (lbolt != CPU->cpu_deadman_lbolt) {
1886 		CPU->cpu_deadman_lbolt = lbolt;
1887 		CPU->cpu_deadman_countdown = deadman_seconds;
1888 		return;
1889 	}
1890 
1891 	if (CPU->cpu_deadman_countdown-- > 0)
1892 		return;
1893 
1894 	/*
1895 	 * Regardless of whether or not we actually bring the system down,
1896 	 * bump the deadman_panics variable.
1897 	 *
1898 	 * N.B. deadman_panics is incremented once for each CPU that
1899 	 * passes through here.  It's expected that all the CPUs will
1900 	 * detect this condition within one second of each other, so
1901 	 * when deadman_enabled is off, deadman_panics will
1902 	 * typically be a multiple of the total number of CPUs in
1903 	 * the system.
1904 	 */
1905 	atomic_add_32(&deadman_panics, 1);
1906 
1907 	if (!deadman_enabled) {
1908 		CPU->cpu_deadman_countdown = deadman_seconds;
1909 		return;
1910 	}
1911 
1912 	/*
1913 	 * If we're here, we want to bring the system down.
1914 	 */
1915 	panic("deadman: timed out after %d seconds of clock "
1916 	    "inactivity", deadman_seconds);
1917 	/*NOTREACHED*/
1918 }
1919 
1920 /*ARGSUSED*/
1921 static void
1922 deadman_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
1923 {
1924 	cpu->cpu_deadman_lbolt = 0;
1925 	cpu->cpu_deadman_countdown = deadman_seconds;
1926 
1927 	hdlr->cyh_func = (cyc_func_t)deadman;
1928 	hdlr->cyh_level = CY_HIGH_LEVEL;
1929 	hdlr->cyh_arg = NULL;
1930 
1931 	/*
1932 	 * Stagger the CPUs so that they don't all run deadman() at
1933 	 * the same time.  Simplest reason to do this is to make it
1934 	 * more likely that only one CPU will panic in case of a
1935 	 * timeout.  This is (strictly speaking) an aesthetic, not a
1936 	 * technical consideration.
1937 	 *
1938 	 * The interval must be one second in accordance with the
1939 	 * code in deadman() above to increase lbolt during panic.
1940 	 */
1941 	when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
1942 	when->cyt_interval = NANOSEC;
1943 }
1944 
1945 
1946 void
1947 deadman_init(void)
1948 {
1949 	cyc_omni_handler_t hdlr;
1950 
1951 	if (deadman_seconds == 0)
1952 		deadman_seconds = snoop_interval / MICROSEC;
1953 
1954 	if (snooping)
1955 		deadman_enabled = 1;
1956 
1957 	hdlr.cyo_online = deadman_online;
1958 	hdlr.cyo_offline = NULL;
1959 	hdlr.cyo_arg = NULL;
1960 
1961 	mutex_enter(&cpu_lock);
1962 	deadman_cyclic = cyclic_add_omni(&hdlr);
1963 	mutex_exit(&cpu_lock);
1964 }
1965 
1966 /*
1967  * tod_fault() is for updating tod validate mechanism state:
1968  * (1) TOD_NOFAULT: for resetting the state to 'normal'.
1969  *     currently used for debugging only
1970  * (2) The following four cases detected by tod validate mechanism:
1971  *       TOD_REVERSED: current tod value is less than previous value.
1972  *       TOD_STALLED: current tod value hasn't advanced.
1973  *       TOD_JUMPED: current tod value advanced too far from previous value.
1974  *       TOD_RATECHANGED: the ratio between average tod delta and
1975  *       average tick delta has changed.
1976  */
1977 enum tod_fault_type
1978 tod_fault(enum tod_fault_type ftype, int off)
1979 {
1980 	ASSERT(MUTEX_HELD(&tod_lock));
1981 
1982 	if (tod_faulted != ftype) {
1983 		switch (ftype) {
1984 		case TOD_NOFAULT:
1985 			plat_tod_fault(TOD_NOFAULT);
1986 			cmn_err(CE_NOTE, "Restarted tracking "
1987 					"Time of Day clock.");
1988 			tod_faulted = ftype;
1989 			break;
1990 		case TOD_REVERSED:
1991 		case TOD_JUMPED:
1992 			if (tod_faulted == TOD_NOFAULT) {
1993 				plat_tod_fault(ftype);
1994 				cmn_err(CE_WARN, "Time of Day clock error: "
1995 				    "reason [%s by 0x%x]. -- "
1996 				    " Stopped tracking Time Of Day clock.",
1997 				    tod_fault_table[ftype], off);
1998 				tod_faulted = ftype;
1999 			}
2000 			break;
2001 		case TOD_STALLED:
2002 		case TOD_RATECHANGED:
2003 			if (tod_faulted == TOD_NOFAULT) {
2004 				plat_tod_fault(ftype);
2005 				cmn_err(CE_WARN, "Time of Day clock error: "
2006 				    "reason [%s]. -- "
2007 				    " Stopped tracking Time Of Day clock.",
2008 				    tod_fault_table[ftype]);
2009 				tod_faulted = ftype;
2010 			}
2011 			break;
2012 		default:
2013 			break;
2014 		}
2015 	}
2016 	return (tod_faulted);
2017 }
2018 
2019 void
2020 tod_fault_reset()
2021 {
2022 	tod_fault_reset_flag = 1;
2023 }
2024 
2025 
2026 /*
2027  * tod_validate() is used for checking values returned by tod_get().
2028  * Four error cases can be detected by this routine:
2029  *   TOD_REVERSED: current tod value is less than previous.
2030  *   TOD_STALLED: current tod value hasn't advanced.
2031  *   TOD_JUMPED: current tod value advanced too far from previous value.
2032  *   TOD_RATECHANGED: the ratio between average tod delta and
2033  *   average tick delta has changed.
2034  */
2035 time_t
2036 tod_validate(time_t tod)
2037 {
2038 	time_t diff_tod;
2039 	hrtime_t diff_tick;
2040 
2041 	long dtick;
2042 	int dtick_delta;
2043 
2044 	int off = 0;
2045 	enum tod_fault_type tod_bad = TOD_NOFAULT;
2046 
2047 	static int firsttime = 1;
2048 
2049 	static time_t prev_tod = 0;
2050 	static hrtime_t prev_tick = 0;
2051 	static long dtick_avg = TOD_REF_FREQ;
2052 
2053 	hrtime_t tick = gethrtime();
2054 
2055 	ASSERT(MUTEX_HELD(&tod_lock));
2056 
2057 	/*
2058 	 * tod_validate_enable is patchable via /etc/system.
2059 	 * If TOD is already faulted, or if TOD validation is deferred,
2060 	 * there is nothing to do.
2061 	 */
2062 	if ((tod_validate_enable == 0) || (tod_faulted != TOD_NOFAULT) ||
2063 	    tod_validate_deferred) {
2064 		return (tod);
2065 	}
2066 
2067 	/*
2068 	 * Update prev_tod and prev_tick values for first run
2069 	 */
2070 	if (firsttime) {
2071 		firsttime = 0;
2072 		prev_tod = tod;
2073 		prev_tick = tick;
2074 		return (tod);
2075 	}
2076 
2077 	/*
2078 	 * For either of these conditions, we need to reset ourself
2079 	 * and start validation from zero since each condition
2080 	 * indicates that the TOD will be updated with new value
2081 	 * Also, note that tod_needsync will be reset in clock()
2082 	 */
2083 	if (tod_needsync || tod_fault_reset_flag) {
2084 		firsttime = 1;
2085 		prev_tod = 0;
2086 		prev_tick = 0;
2087 		dtick_avg = TOD_REF_FREQ;
2088 
2089 		if (tod_fault_reset_flag)
2090 			tod_fault_reset_flag = 0;
2091 
2092 		return (tod);
2093 	}
2094 
2095 	/* test hook */
2096 	switch (tod_unit_test) {
2097 	case 1: /* for testing jumping tod */
2098 		tod += tod_test_injector;
2099 		tod_unit_test = 0;
2100 		break;
2101 	case 2:	/* for testing stuck tod bit */
2102 		tod |= 1 << tod_test_injector;
2103 		tod_unit_test = 0;
2104 		break;
2105 	case 3:	/* for testing stalled tod */
2106 		tod = prev_tod;
2107 		tod_unit_test = 0;
2108 		break;
2109 	case 4:	/* reset tod fault status */
2110 		(void) tod_fault(TOD_NOFAULT, 0);
2111 		tod_unit_test = 0;
2112 		break;
2113 	default:
2114 		break;
2115 	}
2116 
2117 	diff_tod = tod - prev_tod;
2118 	diff_tick = tick - prev_tick;
2119 
2120 	ASSERT(diff_tick >= 0);
2121 
2122 	if (diff_tod < 0) {
2123 		/* ERROR - tod reversed */
2124 		tod_bad = TOD_REVERSED;
2125 		off = (int)(prev_tod - tod);
2126 	} else if (diff_tod == 0) {
2127 		/* tod did not advance */
2128 		if (diff_tick > TOD_STALL_THRESHOLD) {
2129 			/* ERROR - tod stalled */
2130 			tod_bad = TOD_STALLED;
2131 		} else {
2132 			/*
2133 			 * Make sure we don't update prev_tick
2134 			 * so that diff_tick is calculated since
2135 			 * the first diff_tod == 0
2136 			 */
2137 			return (tod);
2138 		}
2139 	} else {
2140 		/* calculate dtick */
2141 		dtick = diff_tick / diff_tod;
2142 
2143 		/* update dtick averages */
2144 		dtick_avg += ((dtick - dtick_avg) / TOD_FILTER_N);
2145 
2146 		/*
2147 		 * Calculate dtick_delta as
2148 		 * variation from reference freq in quartiles
2149 		 */
2150 		dtick_delta = (dtick_avg - TOD_REF_FREQ) /
2151 			(TOD_REF_FREQ >> 2);
2152 
2153 		/*
2154 		 * Even with a perfectly functioning TOD device,
2155 		 * when the number of elapsed seconds is low the
2156 		 * algorithm can calculate a rate that is beyond
2157 		 * tolerance, causing an error.  The algorithm is
2158 		 * inaccurate when elapsed time is low (less than
2159 		 * 5 seconds).
2160 		 */
2161 		if (diff_tod > 4) {
2162 			if (dtick < TOD_JUMP_THRESHOLD) {
2163 				/* ERROR - tod jumped */
2164 				tod_bad = TOD_JUMPED;
2165 				off = (int)diff_tod;
2166 			} else if (dtick_delta) {
2167 				/* ERROR - change in clock rate */
2168 				tod_bad = TOD_RATECHANGED;
2169 			}
2170 		}
2171 	}
2172 
2173 	if (tod_bad != TOD_NOFAULT) {
2174 		(void) tod_fault(tod_bad, off);
2175 
2176 		/*
2177 		 * Disable dosynctodr since we are going to fault
2178 		 * the TOD chip anyway here
2179 		 */
2180 		dosynctodr = 0;
2181 
2182 		/*
2183 		 * Set tod to the correct value from hrestime
2184 		 */
2185 		tod = hrestime.tv_sec;
2186 	}
2187 
2188 	prev_tod = tod;
2189 	prev_tick = tick;
2190 	return (tod);
2191 }
2192 
2193 static void
2194 calcloadavg(int nrun, uint64_t *hp_ave)
2195 {
2196 	static int64_t f[3] = { 135, 27, 9 };
2197 	uint_t i;
2198 	int64_t q, r;
2199 
2200 	/*
2201 	 * Compute load average over the last 1, 5, and 15 minutes
2202 	 * (60, 300, and 900 seconds).  The constants in f[3] are for
2203 	 * exponential decay:
2204 	 * (1 - exp(-1/60)) << 13 = 135,
2205 	 * (1 - exp(-1/300)) << 13 = 27,
2206 	 * (1 - exp(-1/900)) << 13 = 9.
2207 	 */
2208 
2209 	/*
2210 	 * a little hoop-jumping to avoid integer overflow
2211 	 */
2212 	for (i = 0; i < 3; i++) {
2213 		q = (hp_ave[i]  >> 16) << 7;
2214 		r = (hp_ave[i]  & 0xffff) << 7;
2215 		hp_ave[i] += ((nrun - q) * f[i] - ((r * f[i]) >> 16)) >> 4;
2216 	}
2217 }
2218