42 #include <sys/clock.h>
61  * ------------------------------------------------
63  * The Pentium family, like many chip architectures, has a high-resolution
85  *	only supports writing the low 32-bits of TSC, making it impractical
97  * In the absence of time-of-day clock adjustments, gethrtime() must stay in
99  * cannot drive its time-of-day source from TSC, and yet they must somehow be
101  * is called once per second from the interrupt which drives time-of-day.
115  * compiler to emit code that "optimized" away the while-loops below. The
139 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
145 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
159  * The cap of 1 second was chosen since it is the frequency at which the
173  * via /etc/system e.g. `set tsc_calibration="pit"`
183 /* The TSC frequency after calibration */
217 			tsc -= tsc_last;  in tsc_gethrtime()
218 		} else if (tsc >= tsc_last - 2*tsc_max_delta) {  in tsc_gethrtime()
271 		tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];  in tsc_gethrtime_delta()
277 			tsc -= tsc_last;  in tsc_gethrtime_delta()
278 		} else if (tsc >= tsc_last - 2 * tsc_max_delta) {  in tsc_gethrtime_delta()
299 	hrt = tsc_sync_tick_delta[CPU->cpu_id];  in tsc_gethrtime_tick_delta()
319 			tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];  in tsc_gethrtime_params()
333 		tsc -= last;  in tsc_gethrtime_params()
334 	} else if (tsc >= last - 2 * tsc_max_delta) {  in tsc_gethrtime_params()
384 			tsc += tsc_sync_tick_delta[CPU->cpu_id];  in dtrace_gethrtime()
392 			tsc -= tsc_last;  in dtrace_gethrtime()
393 		else if (tsc >= tsc_last - 2*tsc_max_delta)  in dtrace_gethrtime()
406 		 * If we're here, the clock lock is locked -- or it has been  in dtrace_gethrtime()
408 		 * tsc_tick() running on another CPU -- or it may be because  in dtrace_gethrtime()
433 			tsc += tsc_sync_tick_delta[CPU->cpu_id];  in dtrace_gethrtime()
441 			tsc -= shadow_tsc_last;  in dtrace_gethrtime()
442 		else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)  in dtrace_gethrtime()
517 	hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];  in tsc_gethrtimeunscaled_delta()
543  * such as bus contention.  Each sample set is independent per-CPU to control
567 	source = CPU->cpu_id;  in tsc_sync_master()
573 		tsc->master_tsc = tsc_read();  in tsc_sync_master()
578 		write_time =  mtsc_after - tsc->master_tsc;  in tsc_sync_master()
582 			tdelta = tsc->slave_tsc - mtsc_after;  in tsc_sync_master()
584 				tdelta = -tdelta;  in tsc_sync_master()
592 				tdelta = tsc->slave_tsc -  in tsc_sync_master()
593 				    (mtsc_after - (write_time/4));  in tsc_sync_master()
595 				tdelta = tsc->slave_tsc - mtsc_after;  in tsc_sync_master()
597 			last_delta = tsc_sync_tick_delta[source] - tdelta;  in tsc_sync_master()
602 		tsc->master_tsc = tsc->slave_tsc = write_time = 0;  in tsc_sync_master()
611 	last_delta = (last_delta < 0) ? -last_delta : last_delta;  in tsc_sync_master()
644 		/* Re-fill the cache line */  in tsc_sync_slave()
645 		s1 = tsc->master_tsc;  in tsc_sync_slave()
651 			 * slave are the same hyper-threaded CPU, we want the  in tsc_sync_slave()
655 		} while (tsc->master_tsc == 0);  in tsc_sync_slave()
656 		tsc->slave_tsc = s1;  in tsc_sync_slave()
669  * CY_HIGH_LEVEL interrupt.  (No longer just cpu0-only)
696 		now += tsc_sync_tick_delta[CPU->cpu_id];  in tsc_tick()
713 		 * Determine the number of TSC ticks since the last clock  in tsc_tick()
716 		delta = now - tsc_last;  in tsc_tick()
733 	 * cpu_freq_hz is the measured cpu frequency in hertz  in tsc_hrtimeinit()
741 	    (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);  in tsc_hrtimeinit()
743 	    (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);  in tsc_hrtimeinit()
748 	tsc_max_delta = tsc_read() - tsc;  in tsc_hrtimeinit()
803  * Functions to manage TSC and high-res time on suspend and resume.
818  * Take snapshots of the current time and do any other pre-suspend work.
883 			sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;  in tsc_resume()
888 			 * problem with the clock.  Assume at least  in tsc_resume()
911 			sleep_tsc -= now;  in tsc_resume()
935 	if (l->tscc_preference > r->tscc_preference)  in tsc_calibrate_cmp()
936 		return (-1);  in tsc_calibrate_cmp()
937 	if (l->tscc_preference < r->tscc_preference)  in tsc_calibrate_cmp()
941 	int c = strcmp(l->tscc_source, r->tscc_source);  in tsc_calibrate_cmp()
944 		return (-1);  in tsc_calibrate_cmp()
962 		if (strcasecmp(source, tsccp->tscc_source) == 0)  in tsc_calibrate_get_force()
982  * which will store the TSC frequency as measured by the PIT for the
1000 	 * we don't care about the results or saving the result -- tscc_pit.c  in tsc_pit_also()
1001 	 * saves the frequency in a global for the benefit of the APIC code.  in tsc_pit_also()
1003 	(void) pit->tscc_calibrate(&dummy);  in tsc_pit_also()
1014 	 * Every x86 system since the Pentium has TSC support. Since we  in tsc_calibrate()
1015 	 * only support 64-bit x86 systems, there should always be a TSC  in tsc_calibrate()
1019 		panic("System does not have TSC support");  in tsc_calibrate()
1033 	 * `set tsc_calibration="foo"` in /etc/system (preferring a bootloader  in tsc_calibrate()
1034 	 * supplied value over /etc/system).  in tsc_calibrate()
1042 		/* Overwrite any /etc/system supplied value */  in tsc_calibrate()
1059 			PRM_POINT("Forcing /etc/system specified TSC "  in tsc_calibrate()
1062 		PRM_DEBUGS(force->tscc_source);  in tsc_calibrate()
1064 		if (!force->tscc_calibrate(&tsc_freq))  in tsc_calibrate()
1103 		PRM_DEBUGS(tsccpp[i]->tscc_source);  in tsc_calibrate()
1104 		if (tsccpp[i]->tscc_calibrate(&tsc_freq)) {  in tsc_calibrate()
1109 			    tsccpp[i]->tscc_source, tsc_freq / 1000000);  in tsc_calibrate()
1129 	 * In case it's useful, we don't free tsccpp -- we're about to panic  in tsc_calibrate()