xref: /titanic_50/usr/src/uts/i86pc/os/timestamp.c (revision 86dd6e44e20b9551a25f95ce9ab4bd132bb87da3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  *
26  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27  * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
28  */
29 
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/disp.h>
34 #include <sys/var.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/x86_archext.h>
38 #include <sys/archsystm.h>
39 #include <sys/cpuvar.h>
40 #include <sys/psm_defs.h>
41 #include <sys/clock.h>
42 #include <sys/atomic.h>
43 #include <sys/lockstat.h>
44 #include <sys/smp_impldefs.h>
45 #include <sys/dtrace.h>
46 #include <sys/time.h>
47 #include <sys/panic.h>
48 #include <sys/cpu.h>
49 #include <sys/sdt.h>
50 
51 /*
52  * Using the Pentium's TSC register for gethrtime()
53  * ------------------------------------------------
54  *
55  * The Pentium family, like many chip architectures, has a high-resolution
56  * timestamp counter ("TSC") which increments once per CPU cycle.  The contents
57  * of the timestamp counter are read with the RDTSC instruction.
58  *
59  * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
60  * must be translated into nanoseconds in order to implement gethrtime().
61  * We avoid inducing floating point operations in this conversion by
62  * implementing the same nsec_scale algorithm as that found in the sun4u
63  * platform code.  The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
64  * a detailed description of the algorithm; the comment is not reproduced
65  * here.  This implementation differs only in its value for NSEC_SHIFT:
66  * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
67  * 60 MHz Pentiums.
68  *
69  * While TSC and %tick are both cycle counting registers, TSC's functionality
70  * falls short in several critical ways:
71  *
72  *  (a)	TSCs on different CPUs are not guaranteed to be in sync.  While in
73  *	practice they often _are_ in sync, this isn't guaranteed by the
74  *	architecture.
75  *
76  *  (b)	The TSC cannot be reliably set to an arbitrary value.  The architecture
77  *	only supports writing the low 32-bits of TSC, making it impractical
78  *	to rewrite.
79  *
80  *  (c)	The architecture doesn't have the capacity to interrupt based on
81  *	arbitrary values of TSC; there is no TICK_CMPR equivalent.
82  *
83  * Together, (a) and (b) imply that software must track the skew between
84  * TSCs and account for it (it is assumed that while there may exist skew,
85  * there does not exist drift).  To determine the skew between CPUs, we
86  * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
87  * the online operation calls tsc_sync_master().
88  *
89  * In the absence of time-of-day clock adjustments, gethrtime() must stay in
90  * sync with gettimeofday().  This is problematic; given (c), the software
91  * cannot drive its time-of-day source from TSC, and yet they must somehow be
92  * kept in sync.  We implement this by having a routine, tsc_tick(), which
93  * is called once per second from the interrupt which drives time-of-day.
94  *
95  * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
96  * atomically with nsec_scale under CLOCK_LOCK.  This assures that time
97  * monotonically increases.
98  */
99 
100 #define	NSEC_SHIFT 5
101 
102 static uint_t nsec_scale;
103 static uint_t nsec_unscale;
104 
105 /*
106  * These two variables used to be grouped together inside of a structure that
107  * lived on a single cache line. A regression (bug ID 4623398) caused the
108  * compiler to emit code that "optimized" away the while-loops below. The
109  * result was that no synchronization between the onlining and onlined CPUs
110  * took place.
111  */
112 static volatile int tsc_ready;
113 static volatile int tsc_sync_go;
114 
115 /*
116  * Used as indices into the tsc_sync_snaps[] array.
117  */
118 #define	TSC_MASTER		0
119 #define	TSC_SLAVE		1
120 
121 /*
122  * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
123  */
124 #define	TSC_SYNC_STOP		1
125 #define	TSC_SYNC_GO		2
126 #define	TSC_SYNC_DONE		3
127 #define	SYNC_ITERATIONS		10
128 
129 #define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) {	 	\
130 	unsigned int *_l = (unsigned int *)&(tsc); 	\
131 	(hrt) += mul32(_l[1], scale) << NSEC_SHIFT; 	\
132 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
133 }
134 
135 #define	TSC_CONVERT(tsc, hrt, scale) { 			\
136 	unsigned int *_l = (unsigned int *)&(tsc); 	\
137 	(hrt) = mul32(_l[1], scale) << NSEC_SHIFT; 	\
138 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
139 }
140 
141 int tsc_master_slave_sync_needed = 1;
142 
143 static int	tsc_max_delta;
144 static hrtime_t tsc_sync_tick_delta[NCPU];
145 typedef struct tsc_sync {
146 	volatile hrtime_t master_tsc, slave_tsc;
147 } tsc_sync_t;
148 static tsc_sync_t *tscp;
149 static hrtime_t largest_tsc_delta = 0;
150 static ulong_t shortest_write_time = ~0UL;
151 
152 static hrtime_t	tsc_last = 0;
153 static hrtime_t	tsc_last_jumped = 0;
154 static hrtime_t	tsc_hrtime_base = 0;
155 static int	tsc_jumped = 0;
156 static uint32_t	tsc_wayback = 0;
157 /*
158  * The cap of 1 second was chosen since it is the frequency at which the
159  * tsc_tick() function runs which means that when gethrtime() is called it
160  * should never be more than 1 second since tsc_last was updated.
161  */
162 static hrtime_t tsc_resume_cap;
163 static hrtime_t tsc_resume_cap_ns = NANOSEC;	 /* 1s */
164 
165 static hrtime_t	shadow_tsc_hrtime_base;
166 static hrtime_t	shadow_tsc_last;
167 static uint_t	shadow_nsec_scale;
168 static uint32_t	shadow_hres_lock;
169 int get_tsc_ready();
170 
171 static inline
172 hrtime_t tsc_protect(hrtime_t a) {
173 	if (a > tsc_resume_cap) {
174 		atomic_inc_32(&tsc_wayback);
175 		DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last,
176 		    uint32_t, tsc_wayback);
177 		return (tsc_resume_cap);
178 	}
179 	return (a);
180 }
181 
182 hrtime_t
183 tsc_gethrtime(void)
184 {
185 	uint32_t old_hres_lock;
186 	hrtime_t tsc, hrt;
187 
188 	do {
189 		old_hres_lock = hres_lock;
190 
191 		if ((tsc = tsc_read()) >= tsc_last) {
192 			/*
193 			 * It would seem to be obvious that this is true
194 			 * (that is, the past is less than the present),
195 			 * but it isn't true in the presence of suspend/resume
196 			 * cycles.  If we manage to call gethrtime()
197 			 * after a resume, but before the first call to
198 			 * tsc_tick(), we will see the jump.  In this case,
199 			 * we will simply use the value in TSC as the delta.
200 			 */
201 			tsc -= tsc_last;
202 		} else if (tsc >= tsc_last - 2*tsc_max_delta) {
203 			/*
204 			 * There is a chance that tsc_tick() has just run on
205 			 * another CPU, and we have drifted just enough so that
206 			 * we appear behind tsc_last.  In this case, force the
207 			 * delta to be zero.
208 			 */
209 			tsc = 0;
210 		} else {
211 			/*
212 			 * If we reach this else clause we assume that we have
213 			 * gone through a suspend/resume cycle and use the
214 			 * current tsc value as the delta.
215 			 *
216 			 * In rare cases we can reach this else clause due to
217 			 * a lack of monotonicity in the TSC value.  In such
218 			 * cases using the current TSC value as the delta would
219 			 * cause us to return a value ~2x of what it should
220 			 * be.  To protect against these cases we cap the
221 			 * suspend/resume delta at tsc_resume_cap.
222 			 */
223 			tsc = tsc_protect(tsc);
224 		}
225 
226 		hrt = tsc_hrtime_base;
227 
228 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
229 	} while ((old_hres_lock & ~1) != hres_lock);
230 
231 	return (hrt);
232 }
233 
234 hrtime_t
235 tsc_gethrtime_delta(void)
236 {
237 	uint32_t old_hres_lock;
238 	hrtime_t tsc, hrt;
239 	ulong_t flags;
240 
241 	do {
242 		old_hres_lock = hres_lock;
243 
244 		/*
245 		 * We need to disable interrupts here to assure that we
246 		 * don't migrate between the call to tsc_read() and
247 		 * adding the CPU's TSC tick delta. Note that disabling
248 		 * and reenabling preemption is forbidden here because
249 		 * we may be in the middle of a fast trap. In the amd64
250 		 * kernel we cannot tolerate preemption during a fast
251 		 * trap. See _update_sregs().
252 		 */
253 
254 		flags = clear_int_flag();
255 		tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
256 		restore_int_flag(flags);
257 
258 		/* See comments in tsc_gethrtime() above */
259 
260 		if (tsc >= tsc_last) {
261 			tsc -= tsc_last;
262 		} else if (tsc >= tsc_last - 2 * tsc_max_delta) {
263 			tsc = 0;
264 		} else {
265 			tsc = tsc_protect(tsc);
266 		}
267 
268 		hrt = tsc_hrtime_base;
269 
270 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
271 	} while ((old_hres_lock & ~1) != hres_lock);
272 
273 	return (hrt);
274 }
275 
276 hrtime_t
277 tsc_gethrtime_tick_delta(void)
278 {
279 	hrtime_t hrt;
280 	ulong_t flags;
281 
282 	flags = clear_int_flag();
283 	hrt = tsc_sync_tick_delta[CPU->cpu_id];
284 	restore_int_flag(flags);
285 
286 	return (hrt);
287 }
288 
289 /*
290  * This is similar to the above, but it cannot actually spin on hres_lock.
291  * As a result, it caches all of the variables it needs; if the variables
292  * don't change, it's done.
293  */
294 hrtime_t
295 dtrace_gethrtime(void)
296 {
297 	uint32_t old_hres_lock;
298 	hrtime_t tsc, hrt;
299 	ulong_t flags;
300 
301 	do {
302 		old_hres_lock = hres_lock;
303 
304 		/*
305 		 * Interrupts are disabled to ensure that the thread isn't
306 		 * migrated between the tsc_read() and adding the CPU's
307 		 * TSC tick delta.
308 		 */
309 		flags = clear_int_flag();
310 
311 		tsc = tsc_read();
312 
313 		if (gethrtimef == tsc_gethrtime_delta)
314 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
315 
316 		restore_int_flag(flags);
317 
318 		/*
319 		 * See the comments in tsc_gethrtime(), above.
320 		 */
321 		if (tsc >= tsc_last)
322 			tsc -= tsc_last;
323 		else if (tsc >= tsc_last - 2*tsc_max_delta)
324 			tsc = 0;
325 		else
326 			tsc = tsc_protect(tsc);
327 
328 		hrt = tsc_hrtime_base;
329 
330 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
331 
332 		if ((old_hres_lock & ~1) == hres_lock)
333 			break;
334 
335 		/*
336 		 * If we're here, the clock lock is locked -- or it has been
337 		 * unlocked and locked since we looked.  This may be due to
338 		 * tsc_tick() running on another CPU -- or it may be because
339 		 * some code path has ended up in dtrace_probe() with
340 		 * CLOCK_LOCK held.  We'll try to determine that we're in
341 		 * the former case by taking another lap if the lock has
342 		 * changed since when we first looked at it.
343 		 */
344 		if (old_hres_lock != hres_lock)
345 			continue;
346 
347 		/*
348 		 * So the lock was and is locked.  We'll use the old data
349 		 * instead.
350 		 */
351 		old_hres_lock = shadow_hres_lock;
352 
353 		/*
354 		 * Again, disable interrupts to ensure that the thread
355 		 * isn't migrated between the tsc_read() and adding
356 		 * the CPU's TSC tick delta.
357 		 */
358 		flags = clear_int_flag();
359 
360 		tsc = tsc_read();
361 
362 		if (gethrtimef == tsc_gethrtime_delta)
363 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
364 
365 		restore_int_flag(flags);
366 
367 		/*
368 		 * See the comments in tsc_gethrtime(), above.
369 		 */
370 		if (tsc >= shadow_tsc_last)
371 			tsc -= shadow_tsc_last;
372 		else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
373 			tsc = 0;
374 		else
375 			tsc = tsc_protect(tsc);
376 
377 		hrt = shadow_tsc_hrtime_base;
378 
379 		TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
380 	} while ((old_hres_lock & ~1) != shadow_hres_lock);
381 
382 	return (hrt);
383 }
384 
385 hrtime_t
386 tsc_gethrtimeunscaled(void)
387 {
388 	uint32_t old_hres_lock;
389 	hrtime_t tsc;
390 
391 	do {
392 		old_hres_lock = hres_lock;
393 
394 		/* See tsc_tick(). */
395 		tsc = tsc_read() + tsc_last_jumped;
396 	} while ((old_hres_lock & ~1) != hres_lock);
397 
398 	return (tsc);
399 }
400 
401 /*
402  * Convert a nanosecond based timestamp to tsc
403  */
404 uint64_t
405 tsc_unscalehrtime(hrtime_t nsec)
406 {
407 	hrtime_t tsc;
408 
409 	if (tsc_gethrtime_enable) {
410 		TSC_CONVERT(nsec, tsc, nsec_unscale);
411 		return (tsc);
412 	}
413 	return ((uint64_t)nsec);
414 }
415 
416 /* Convert a tsc timestamp to nanoseconds */
417 void
418 tsc_scalehrtime(hrtime_t *tsc)
419 {
420 	hrtime_t hrt;
421 	hrtime_t mytsc;
422 
423 	if (tsc == NULL)
424 		return;
425 	mytsc = *tsc;
426 
427 	TSC_CONVERT(mytsc, hrt, nsec_scale);
428 	*tsc  = hrt;
429 }
430 
431 hrtime_t
432 tsc_gethrtimeunscaled_delta(void)
433 {
434 	hrtime_t hrt;
435 	ulong_t flags;
436 
437 	/*
438 	 * Similarly to tsc_gethrtime_delta, we need to disable preemption
439 	 * to prevent migration between the call to tsc_gethrtimeunscaled
440 	 * and adding the CPU's hrtime delta. Note that disabling and
441 	 * reenabling preemption is forbidden here because we may be in the
442 	 * middle of a fast trap. In the amd64 kernel we cannot tolerate
443 	 * preemption during a fast trap. See _update_sregs().
444 	 */
445 
446 	flags = clear_int_flag();
447 	hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
448 	restore_int_flag(flags);
449 
450 	return (hrt);
451 }
452 
453 /*
454  * Called by the master in the TSC sync operation (usually the boot CPU).
455  * If the slave is discovered to have a skew, gethrtimef will be changed to
456  * point to tsc_gethrtime_delta(). Calculating skews is precise only when
457  * the master and slave TSCs are read simultaneously; however, there is no
458  * algorithm that can read both CPUs in perfect simultaneity. The proposed
459  * algorithm is an approximate method based on the behaviour of cache
460  * management. The slave CPU continuously reads TSC and then reads a global
461  * variable which the master CPU updates. The moment the master's update reaches
462  * the slave's visibility (being forced by an mfence operation) we use the TSC
463  * reading taken on the slave. A corresponding TSC read will be taken on the
464  * master as soon as possible after finishing the mfence operation. But the
465  * delay between causing the slave to notice the invalid cache line and the
466  * competion of mfence is not repeatable. This error is heuristically assumed
467  * to be 1/4th of the total write time as being measured by the two TSC reads
468  * on the master sandwiching the mfence. Furthermore, due to the nature of
469  * bus arbitration, contention on memory bus, etc., the time taken for the write
470  * to reflect globally can vary a lot. So instead of taking a single reading,
471  * a set of readings are taken and the one with least write time is chosen
472  * to calculate the final skew.
473  *
474  * TSC sync is disabled in the context of virtualization because the CPUs
475  * assigned to the guest are virtual CPUs which means the real CPUs on which
476  * guest runs keep changing during life time of guest OS. So we would end up
477  * calculating TSC skews for a set of CPUs during boot whereas the guest
478  * might migrate to a different set of physical CPUs at a later point of
479  * time.
480  */
481 void
482 tsc_sync_master(processorid_t slave)
483 {
484 	ulong_t flags, source, min_write_time = ~0UL;
485 	hrtime_t write_time, x, mtsc_after, tdelta;
486 	tsc_sync_t *tsc = tscp;
487 	int cnt;
488 	int hwtype;
489 
490 	hwtype = get_hwenv();
491 	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
492 		return;
493 
494 	flags = clear_int_flag();
495 	source = CPU->cpu_id;
496 
497 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
498 		while (tsc_sync_go != TSC_SYNC_GO)
499 			SMT_PAUSE();
500 
501 		tsc->master_tsc = tsc_read();
502 		membar_enter();
503 		mtsc_after = tsc_read();
504 		while (tsc_sync_go != TSC_SYNC_DONE)
505 			SMT_PAUSE();
506 		write_time =  mtsc_after - tsc->master_tsc;
507 		if (write_time <= min_write_time) {
508 			min_write_time = write_time;
509 			/*
510 			 * Apply heuristic adjustment only if the calculated
511 			 * delta is > 1/4th of the write time.
512 			 */
513 			x = tsc->slave_tsc - mtsc_after;
514 			if (x < 0)
515 				x = -x;
516 			if (x > (min_write_time/4))
517 				/*
518 				 * Subtract 1/4th of the measured write time
519 				 * from the master's TSC value, as an estimate
520 				 * of how late the mfence completion came
521 				 * after the slave noticed the cache line
522 				 * change.
523 				 */
524 				tdelta = tsc->slave_tsc -
525 				    (mtsc_after - (min_write_time/4));
526 			else
527 				tdelta = tsc->slave_tsc - mtsc_after;
528 			tsc_sync_tick_delta[slave] =
529 			    tsc_sync_tick_delta[source] - tdelta;
530 		}
531 
532 		tsc->master_tsc = tsc->slave_tsc = write_time = 0;
533 		membar_enter();
534 		tsc_sync_go = TSC_SYNC_STOP;
535 	}
536 	if (tdelta < 0)
537 		tdelta = -tdelta;
538 	if (tdelta > largest_tsc_delta)
539 		largest_tsc_delta = tdelta;
540 	if (min_write_time < shortest_write_time)
541 		shortest_write_time = min_write_time;
542 	/*
543 	 * Enable delta variants of tsc functions if the largest of all chosen
544 	 * deltas is > smallest of the write time.
545 	 */
546 	if (largest_tsc_delta > shortest_write_time) {
547 		gethrtimef = tsc_gethrtime_delta;
548 		gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
549 	}
550 	restore_int_flag(flags);
551 }
552 
553 /*
554  * Called by a CPU which has just been onlined.  It is expected that the CPU
555  * performing the online operation will call tsc_sync_master().
556  *
557  * TSC sync is disabled in the context of virtualization. See comments
558  * above tsc_sync_master.
559  */
560 void
561 tsc_sync_slave(void)
562 {
563 	ulong_t flags;
564 	hrtime_t s1;
565 	tsc_sync_t *tsc = tscp;
566 	int cnt;
567 	int hwtype;
568 
569 	hwtype = get_hwenv();
570 	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
571 		return;
572 
573 	flags = clear_int_flag();
574 
575 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
576 		/* Re-fill the cache line */
577 		s1 = tsc->master_tsc;
578 		membar_enter();
579 		tsc_sync_go = TSC_SYNC_GO;
580 		do {
581 			/*
582 			 * Do not put an SMT_PAUSE here. For instance,
583 			 * if the master and slave are really the same
584 			 * hyper-threaded CPU, then you want the master
585 			 * to yield to the slave as quickly as possible here,
586 			 * but not the other way.
587 			 */
588 			s1 = tsc_read();
589 		} while (tsc->master_tsc == 0);
590 		tsc->slave_tsc = s1;
591 		membar_enter();
592 		tsc_sync_go = TSC_SYNC_DONE;
593 
594 		while (tsc_sync_go != TSC_SYNC_STOP)
595 			SMT_PAUSE();
596 	}
597 
598 	restore_int_flag(flags);
599 }
600 
601 /*
602  * Called once per second on a CPU from the cyclic subsystem's
603  * CY_HIGH_LEVEL interrupt.  (No longer just cpu0-only)
604  */
605 void
606 tsc_tick(void)
607 {
608 	hrtime_t now, delta;
609 	ushort_t spl;
610 
611 	/*
612 	 * Before we set the new variables, we set the shadow values.  This
613 	 * allows for lock free operation in dtrace_gethrtime().
614 	 */
615 	lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
616 	    ipltospl(CBE_HIGH_PIL), &spl);
617 
618 	shadow_tsc_hrtime_base = tsc_hrtime_base;
619 	shadow_tsc_last = tsc_last;
620 	shadow_nsec_scale = nsec_scale;
621 
622 	shadow_hres_lock++;
623 	splx(spl);
624 
625 	CLOCK_LOCK(&spl);
626 
627 	now = tsc_read();
628 
629 	if (gethrtimef == tsc_gethrtime_delta)
630 		now += tsc_sync_tick_delta[CPU->cpu_id];
631 
632 	if (now < tsc_last) {
633 		/*
634 		 * The TSC has just jumped into the past.  We assume that
635 		 * this is due to a suspend/resume cycle, and we're going
636 		 * to use the _current_ value of TSC as the delta.  This
637 		 * will keep tsc_hrtime_base correct.  We're also going to
638 		 * assume that rate of tsc does not change after a suspend
639 		 * resume (i.e nsec_scale remains the same).
640 		 */
641 		delta = now;
642 		delta = tsc_protect(delta);
643 		tsc_last_jumped += tsc_last;
644 		tsc_jumped = 1;
645 	} else {
646 		/*
647 		 * Determine the number of TSC ticks since the last clock
648 		 * tick, and add that to the hrtime base.
649 		 */
650 		delta = now - tsc_last;
651 	}
652 
653 	TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
654 	tsc_last = now;
655 
656 	CLOCK_UNLOCK(spl);
657 }
658 
659 void
660 tsc_hrtimeinit(uint64_t cpu_freq_hz)
661 {
662 	extern int gethrtime_hires;
663 	longlong_t tsc;
664 	ulong_t flags;
665 
666 	/*
667 	 * cpu_freq_hz is the measured cpu frequency in hertz
668 	 */
669 
670 	/*
671 	 * We can't accommodate CPUs slower than 31.25 MHz.
672 	 */
673 	ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
674 	nsec_scale =
675 	    (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
676 	nsec_unscale =
677 	    (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
678 
679 	flags = clear_int_flag();
680 	tsc = tsc_read();
681 	(void) tsc_gethrtime();
682 	tsc_max_delta = tsc_read() - tsc;
683 	restore_int_flag(flags);
684 	gethrtimef = tsc_gethrtime;
685 	gethrtimeunscaledf = tsc_gethrtimeunscaled;
686 	scalehrtimef = tsc_scalehrtime;
687 	unscalehrtimef = tsc_unscalehrtime;
688 	hrtime_tick = tsc_tick;
689 	gethrtime_hires = 1;
690 	/*
691 	 * Allocate memory for the structure used in the tsc sync logic.
692 	 * This structure should be aligned on a multiple of cache line size.
693 	 */
694 	tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
695 
696 	/*
697 	 * Convert the TSC resume cap ns value into its unscaled TSC value.
698 	 * See tsc_gethrtime().
699 	 */
700 	if (tsc_resume_cap == 0)
701 		TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale);
702 }
703 
704 int
705 get_tsc_ready()
706 {
707 	return (tsc_ready);
708 }
709 
710 /*
711  * Adjust all the deltas by adding the passed value to the array.
712  * Then use the "delt" versions of the the gethrtime functions.
713  * Note that 'tdelta' _could_ be a negative number, which should
714  * reduce the values in the array (used, for example, if the Solaris
715  * instance was moved by a virtual manager to a machine with a higher
716  * value of tsc).
717  */
718 void
719 tsc_adjust_delta(hrtime_t tdelta)
720 {
721 	int		i;
722 
723 	for (i = 0; i < NCPU; i++) {
724 		tsc_sync_tick_delta[i] += tdelta;
725 	}
726 
727 	gethrtimef = tsc_gethrtime_delta;
728 	gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
729 }
730 
731 /*
732  * Functions to manage TSC and high-res time on suspend and resume.
733  */
734 
735 /*
736  * declarations needed for time adjustment
737  */
738 extern void	rtcsync(void);
739 extern tod_ops_t *tod_ops;
740 /* There must be a better way than exposing nsec_scale! */
741 extern uint_t	nsec_scale;
742 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
743 static timestruc_t tsc_saved_ts;
744 static int	tsc_needs_resume = 0;	/* We only want to do this once. */
745 int		tsc_delta_onsuspend = 0;
746 int		tsc_adjust_seconds = 1;
747 int		tsc_suspend_count = 0;
748 int		tsc_resume_in_cyclic = 0;
749 
750 /*
751  * Let timestamp.c know that we are suspending.  It needs to take
752  * snapshots of the current time, and do any pre-suspend work.
753  */
754 void
755 tsc_suspend(void)
756 {
757 /*
758  * What we need to do here, is to get the time we suspended, so that we
759  * know how much we should add to the resume.
760  * This routine is called by each CPU, so we need to handle reentry.
761  */
762 	if (tsc_gethrtime_enable) {
763 		/*
764 		 * We put the tsc_read() inside the lock as it
765 		 * as no locking constraints, and it puts the
766 		 * aquired value closer to the time stamp (in
767 		 * case we delay getting the lock).
768 		 */
769 		mutex_enter(&tod_lock);
770 		tsc_saved_tsc = tsc_read();
771 		tsc_saved_ts = TODOP_GET(tod_ops);
772 		mutex_exit(&tod_lock);
773 		/* We only want to do this once. */
774 		if (tsc_needs_resume == 0) {
775 			if (tsc_delta_onsuspend) {
776 				tsc_adjust_delta(tsc_saved_tsc);
777 			} else {
778 				tsc_adjust_delta(nsec_scale);
779 			}
780 			tsc_suspend_count++;
781 		}
782 	}
783 
784 	invalidate_cache();
785 	tsc_needs_resume = 1;
786 }
787 
788 /*
789  * Restore all timestamp state based on the snapshots taken at
790  * suspend time.
791  */
792 void
793 tsc_resume(void)
794 {
795 	/*
796 	 * We only need to (and want to) do this once.  So let the first
797 	 * caller handle this (we are locked by the cpu lock), as it
798 	 * is preferential that we get the earliest sync.
799 	 */
800 	if (tsc_needs_resume) {
801 		/*
802 		 * If using the TSC, adjust the delta based on how long
803 		 * we were sleeping (or away).  We also adjust for
804 		 * migration and a grown TSC.
805 		 */
806 		if (tsc_saved_tsc != 0) {
807 			timestruc_t	ts;
808 			hrtime_t	now, sleep_tsc = 0;
809 			int		sleep_sec;
810 			extern void	tsc_tick(void);
811 			extern uint64_t cpu_freq_hz;
812 
813 			/* tsc_read() MUST be before TODOP_GET() */
814 			mutex_enter(&tod_lock);
815 			now = tsc_read();
816 			ts = TODOP_GET(tod_ops);
817 			mutex_exit(&tod_lock);
818 
819 			/* Compute seconds of sleep time */
820 			sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
821 
822 			/*
823 			 * If the saved sec is less that or equal to
824 			 * the current ts, then there is likely a
825 			 * problem with the clock.  Assume at least
826 			 * one second has passed, so that time goes forward.
827 			 */
828 			if (sleep_sec <= 0) {
829 				sleep_sec = 1;
830 			}
831 
832 			/* How many TSC's should have occured while sleeping */
833 			if (tsc_adjust_seconds)
834 				sleep_tsc = sleep_sec * cpu_freq_hz;
835 
836 			/*
837 			 * We also want to subtract from the "sleep_tsc"
838 			 * the current value of tsc_read(), so that our
839 			 * adjustment accounts for the amount of time we
840 			 * have been resumed _or_ an adjustment based on
841 			 * the fact that we didn't actually power off the
842 			 * CPU (migration is another issue, but _should_
843 			 * also comply with this calculation).  If the CPU
844 			 * never powered off, then:
845 			 *    'now == sleep_tsc + saved_tsc'
846 			 * and the delta will effectively be "0".
847 			 */
848 			sleep_tsc -= now;
849 			if (tsc_delta_onsuspend) {
850 				tsc_adjust_delta(sleep_tsc);
851 			} else {
852 				tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
853 			}
854 			tsc_saved_tsc = 0;
855 
856 			tsc_tick();
857 		}
858 		tsc_needs_resume = 0;
859 	}
860 
861 }
862