1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27 * Copyright 2016 Joyent, Inc.
28 */
29
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/disp.h>
34 #include <sys/var.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/x86_archext.h>
38 #include <sys/archsystm.h>
39 #include <sys/cpuvar.h>
40 #include <sys/psm_defs.h>
41 #include <sys/clock.h>
42 #include <sys/atomic.h>
43 #include <sys/lockstat.h>
44 #include <sys/smp_impldefs.h>
45 #include <sys/dtrace.h>
46 #include <sys/time.h>
47 #include <sys/panic.h>
48 #include <sys/cpu.h>
49 #include <sys/comm_page.h>
50
51 /*
52 * Using the Pentium's TSC register for gethrtime()
53 * ------------------------------------------------
54 *
55 * The Pentium family, like many chip architectures, has a high-resolution
56 * timestamp counter ("TSC") which increments once per CPU cycle. The contents
57 * of the timestamp counter are read with the RDTSC instruction.
58 *
59 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
60 * must be translated into nanoseconds in order to implement gethrtime().
61 * We avoid inducing floating point operations in this conversion by
62 * implementing the same nsec_scale algorithm as that found in the sun4u
63 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
64 * a detailed description of the algorithm; the comment is not reproduced
65 * here. This implementation differs only in its value for NSEC_SHIFT:
66 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
67 * 60 MHz Pentiums.
68 *
69 * While TSC and %tick are both cycle counting registers, TSC's functionality
70 * falls short in several critical ways:
71 *
72 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in
73 * practice they often _are_ in sync, this isn't guaranteed by the
74 * architecture.
75 *
76 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture
77 * only supports writing the low 32-bits of TSC, making it impractical
78 * to rewrite.
79 *
80 * (c) The architecture doesn't have the capacity to interrupt based on
81 * arbitrary values of TSC; there is no TICK_CMPR equivalent.
82 *
83 * Together, (a) and (b) imply that software must track the skew between
84 * TSCs and account for it (it is assumed that while there may exist skew,
85 * there does not exist drift). To determine the skew between CPUs, we
86 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
87 * the online operation calls tsc_sync_master().
88 *
89 * In the absence of time-of-day clock adjustments, gethrtime() must stay in
90 * sync with gettimeofday(). This is problematic; given (c), the software
91 * cannot drive its time-of-day source from TSC, and yet they must somehow be
92 * kept in sync. We implement this by having a routine, tsc_tick(), which
93 * is called once per second from the interrupt which drives time-of-day.
94 *
95 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
96 * atomically with nsec_scale under CLOCK_LOCK. This assures that time
97 * monotonically increases.
98 */
99
100 #define NSEC_SHIFT 5
101
102 static uint_t nsec_unscale;
103
104 /*
105 * These two variables used to be grouped together inside of a structure that
106 * lived on a single cache line. A regression (bug ID 4623398) caused the
107 * compiler to emit code that "optimized" away the while-loops below. The
108 * result was that no synchronization between the onlining and onlined CPUs
109 * took place.
110 */
111 static volatile int tsc_ready;
112 static volatile int tsc_sync_go;
113
114 /*
115 * Used as indices into the tsc_sync_snaps[] array.
116 */
117 #define TSC_MASTER 0
118 #define TSC_SLAVE 1
119
120 /*
121 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
122 */
123 #define TSC_SYNC_STOP 1
124 #define TSC_SYNC_GO 2
125 #define TSC_SYNC_DONE 3
126 #define SYNC_ITERATIONS 10
127
128 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \
129 unsigned int *_l = (unsigned int *)&(tsc); \
130 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \
131 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
132 }
133
134 #define TSC_CONVERT(tsc, hrt, scale) { \
135 unsigned int *_l = (unsigned int *)&(tsc); \
136 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \
137 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
138 }
139
140 int tsc_master_slave_sync_needed = 1;
141
142 typedef struct tsc_sync {
143 volatile hrtime_t master_tsc, slave_tsc;
144 } tsc_sync_t;
145 static tsc_sync_t *tscp;
146 static hrtime_t largest_tsc_delta = 0;
147 static ulong_t shortest_write_time = ~0UL;
148
149 static hrtime_t tsc_last_jumped = 0;
150 static int tsc_jumped = 0;
151
152 static hrtime_t shadow_tsc_hrtime_base;
153 static hrtime_t shadow_tsc_last;
154 static uint_t shadow_nsec_scale;
155 static uint32_t shadow_hres_lock;
156 int get_tsc_ready();
157
158 hrtime_t
tsc_gethrtime(void)159 tsc_gethrtime(void)
160 {
161 uint32_t old_hres_lock;
162 hrtime_t tsc, hrt;
163
164 do {
165 old_hres_lock = hres_lock;
166
167 if ((tsc = tsc_read()) >= tsc_last) {
168 /*
169 * It would seem to be obvious that this is true
170 * (that is, the past is less than the present),
171 * but it isn't true in the presence of suspend/resume
172 * cycles. If we manage to call gethrtime()
173 * after a resume, but before the first call to
174 * tsc_tick(), we will see the jump. In this case,
175 * we will simply use the value in TSC as the delta.
176 */
177 tsc -= tsc_last;
178 } else if (tsc >= tsc_last - 2*tsc_max_delta) {
179 /*
180 * There is a chance that tsc_tick() has just run on
181 * another CPU, and we have drifted just enough so that
182 * we appear behind tsc_last. In this case, force the
183 * delta to be zero.
184 */
185 tsc = 0;
186 }
187
188 hrt = tsc_hrtime_base;
189
190 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
191 } while ((old_hres_lock & ~1) != hres_lock);
192
193 return (hrt);
194 }
195
196 hrtime_t
tsc_gethrtime_delta(void)197 tsc_gethrtime_delta(void)
198 {
199 uint32_t old_hres_lock;
200 hrtime_t tsc, hrt;
201 ulong_t flags;
202
203 do {
204 old_hres_lock = hres_lock;
205
206 /*
207 * We need to disable interrupts here to assure that we
208 * don't migrate between the call to tsc_read() and
209 * adding the CPU's TSC tick delta. Note that disabling
210 * and reenabling preemption is forbidden here because
211 * we may be in the middle of a fast trap. In the amd64
212 * kernel we cannot tolerate preemption during a fast
213 * trap. See _update_sregs().
214 */
215
216 flags = clear_int_flag();
217 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
218 restore_int_flag(flags);
219
220 /* See comments in tsc_gethrtime() above */
221
222 if (tsc >= tsc_last) {
223 tsc -= tsc_last;
224 } else if (tsc >= tsc_last - 2 * tsc_max_delta) {
225 tsc = 0;
226 }
227
228 hrt = tsc_hrtime_base;
229
230 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
231 } while ((old_hres_lock & ~1) != hres_lock);
232
233 return (hrt);
234 }
235
236 hrtime_t
tsc_gethrtime_tick_delta(void)237 tsc_gethrtime_tick_delta(void)
238 {
239 hrtime_t hrt;
240 ulong_t flags;
241
242 flags = clear_int_flag();
243 hrt = tsc_sync_tick_delta[CPU->cpu_id];
244 restore_int_flag(flags);
245
246 return (hrt);
247 }
248
249 /*
250 * This is similar to the above, but it cannot actually spin on hres_lock.
251 * As a result, it caches all of the variables it needs; if the variables
252 * don't change, it's done.
253 */
254 hrtime_t
dtrace_gethrtime(void)255 dtrace_gethrtime(void)
256 {
257 uint32_t old_hres_lock;
258 hrtime_t tsc, hrt;
259 ulong_t flags;
260
261 do {
262 old_hres_lock = hres_lock;
263
264 /*
265 * Interrupts are disabled to ensure that the thread isn't
266 * migrated between the tsc_read() and adding the CPU's
267 * TSC tick delta.
268 */
269 flags = clear_int_flag();
270
271 tsc = tsc_read();
272
273 if (gethrtimef == tsc_gethrtime_delta)
274 tsc += tsc_sync_tick_delta[CPU->cpu_id];
275
276 restore_int_flag(flags);
277
278 /*
279 * See the comments in tsc_gethrtime(), above.
280 */
281 if (tsc >= tsc_last)
282 tsc -= tsc_last;
283 else if (tsc >= tsc_last - 2*tsc_max_delta)
284 tsc = 0;
285
286 hrt = tsc_hrtime_base;
287
288 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
289
290 if ((old_hres_lock & ~1) == hres_lock)
291 break;
292
293 /*
294 * If we're here, the clock lock is locked -- or it has been
295 * unlocked and locked since we looked. This may be due to
296 * tsc_tick() running on another CPU -- or it may be because
297 * some code path has ended up in dtrace_probe() with
298 * CLOCK_LOCK held. We'll try to determine that we're in
299 * the former case by taking another lap if the lock has
300 * changed since when we first looked at it.
301 */
302 if (old_hres_lock != hres_lock)
303 continue;
304
305 /*
306 * So the lock was and is locked. We'll use the old data
307 * instead.
308 */
309 old_hres_lock = shadow_hres_lock;
310
311 /*
312 * Again, disable interrupts to ensure that the thread
313 * isn't migrated between the tsc_read() and adding
314 * the CPU's TSC tick delta.
315 */
316 flags = clear_int_flag();
317
318 tsc = tsc_read();
319
320 if (gethrtimef == tsc_gethrtime_delta)
321 tsc += tsc_sync_tick_delta[CPU->cpu_id];
322
323 restore_int_flag(flags);
324
325 /*
326 * See the comments in tsc_gethrtime(), above.
327 */
328 if (tsc >= shadow_tsc_last)
329 tsc -= shadow_tsc_last;
330 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
331 tsc = 0;
332
333 hrt = shadow_tsc_hrtime_base;
334
335 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
336 } while ((old_hres_lock & ~1) != shadow_hres_lock);
337
338 return (hrt);
339 }
340
341 hrtime_t
tsc_gethrtimeunscaled(void)342 tsc_gethrtimeunscaled(void)
343 {
344 uint32_t old_hres_lock;
345 hrtime_t tsc;
346
347 do {
348 old_hres_lock = hres_lock;
349
350 /* See tsc_tick(). */
351 tsc = tsc_read() + tsc_last_jumped;
352 } while ((old_hres_lock & ~1) != hres_lock);
353
354 return (tsc);
355 }
356
357 /*
358 * Convert a nanosecond based timestamp to tsc
359 */
360 uint64_t
tsc_unscalehrtime(hrtime_t nsec)361 tsc_unscalehrtime(hrtime_t nsec)
362 {
363 hrtime_t tsc;
364
365 if (tsc_gethrtime_enable) {
366 TSC_CONVERT(nsec, tsc, nsec_unscale);
367 return (tsc);
368 }
369 return ((uint64_t)nsec);
370 }
371
372 /* Convert a tsc timestamp to nanoseconds */
373 void
tsc_scalehrtime(hrtime_t * tsc)374 tsc_scalehrtime(hrtime_t *tsc)
375 {
376 hrtime_t hrt;
377 hrtime_t mytsc;
378
379 if (tsc == NULL)
380 return;
381 mytsc = *tsc;
382
383 TSC_CONVERT(mytsc, hrt, nsec_scale);
384 *tsc = hrt;
385 }
386
387 hrtime_t
tsc_gethrtimeunscaled_delta(void)388 tsc_gethrtimeunscaled_delta(void)
389 {
390 hrtime_t hrt;
391 ulong_t flags;
392
393 /*
394 * Similarly to tsc_gethrtime_delta, we need to disable preemption
395 * to prevent migration between the call to tsc_gethrtimeunscaled
396 * and adding the CPU's hrtime delta. Note that disabling and
397 * reenabling preemption is forbidden here because we may be in the
398 * middle of a fast trap. In the amd64 kernel we cannot tolerate
399 * preemption during a fast trap. See _update_sregs().
400 */
401
402 flags = clear_int_flag();
403 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
404 restore_int_flag(flags);
405
406 return (hrt);
407 }
408
409 /*
410 * Called by the master in the TSC sync operation (usually the boot CPU).
411 * If the slave is discovered to have a skew, gethrtimef will be changed to
412 * point to tsc_gethrtime_delta(). Calculating skews is precise only when
413 * the master and slave TSCs are read simultaneously; however, there is no
414 * algorithm that can read both CPUs in perfect simultaneity. The proposed
415 * algorithm is an approximate method based on the behaviour of cache
416 * management. The slave CPU continuously reads TSC and then reads a global
417 * variable which the master CPU updates. The moment the master's update reaches
418 * the slave's visibility (being forced by an mfence operation) we use the TSC
419 * reading taken on the slave. A corresponding TSC read will be taken on the
420 * master as soon as possible after finishing the mfence operation. But the
421 * delay between causing the slave to notice the invalid cache line and the
422 * competion of mfence is not repeatable. This error is heuristically assumed
423 * to be 1/4th of the total write time as being measured by the two TSC reads
424 * on the master sandwiching the mfence. Furthermore, due to the nature of
425 * bus arbitration, contention on memory bus, etc., the time taken for the write
426 * to reflect globally can vary a lot. So instead of taking a single reading,
427 * a set of readings are taken and the one with least write time is chosen
428 * to calculate the final skew.
429 *
430 * TSC sync is disabled in the context of virtualization because the CPUs
431 * assigned to the guest are virtual CPUs which means the real CPUs on which
432 * guest runs keep changing during life time of guest OS. So we would end up
433 * calculating TSC skews for a set of CPUs during boot whereas the guest
434 * might migrate to a different set of physical CPUs at a later point of
435 * time.
436 */
437 void
tsc_sync_master(processorid_t slave)438 tsc_sync_master(processorid_t slave)
439 {
440 ulong_t flags, source, min_write_time = ~0UL;
441 hrtime_t write_time, x, mtsc_after, tdelta;
442 tsc_sync_t *tsc = tscp;
443 int cnt;
444 int hwtype;
445
446 hwtype = get_hwenv();
447 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
448 return;
449
450 flags = clear_int_flag();
451 source = CPU->cpu_id;
452
453 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
454 while (tsc_sync_go != TSC_SYNC_GO)
455 SMT_PAUSE();
456
457 tsc->master_tsc = tsc_read();
458 membar_enter();
459 mtsc_after = tsc_read();
460 while (tsc_sync_go != TSC_SYNC_DONE)
461 SMT_PAUSE();
462 write_time = mtsc_after - tsc->master_tsc;
463 if (write_time <= min_write_time) {
464 min_write_time = write_time;
465 /*
466 * Apply heuristic adjustment only if the calculated
467 * delta is > 1/4th of the write time.
468 */
469 x = tsc->slave_tsc - mtsc_after;
470 if (x < 0)
471 x = -x;
472 if (x > (min_write_time/4))
473 /*
474 * Subtract 1/4th of the measured write time
475 * from the master's TSC value, as an estimate
476 * of how late the mfence completion came
477 * after the slave noticed the cache line
478 * change.
479 */
480 tdelta = tsc->slave_tsc -
481 (mtsc_after - (min_write_time/4));
482 else
483 tdelta = tsc->slave_tsc - mtsc_after;
484 tsc_sync_tick_delta[slave] =
485 tsc_sync_tick_delta[source] - tdelta;
486 }
487
488 tsc->master_tsc = tsc->slave_tsc = write_time = 0;
489 membar_enter();
490 tsc_sync_go = TSC_SYNC_STOP;
491 }
492 if (tdelta < 0)
493 tdelta = -tdelta;
494 if (tdelta > largest_tsc_delta)
495 largest_tsc_delta = tdelta;
496 if (min_write_time < shortest_write_time)
497 shortest_write_time = min_write_time;
498 /*
499 * Enable delta variants of tsc functions if the largest of all chosen
500 * deltas is > smallest of the write time.
501 */
502 if (largest_tsc_delta > shortest_write_time) {
503 gethrtimef = tsc_gethrtime_delta;
504 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
505 tsc_ncpu = NCPU;
506 }
507 restore_int_flag(flags);
508 }
509
510 /*
511 * Called by a CPU which has just been onlined. It is expected that the CPU
512 * performing the online operation will call tsc_sync_master().
513 *
514 * TSC sync is disabled in the context of virtualization. See comments
515 * above tsc_sync_master.
516 */
517 void
tsc_sync_slave(void)518 tsc_sync_slave(void)
519 {
520 ulong_t flags;
521 hrtime_t s1;
522 tsc_sync_t *tsc = tscp;
523 int cnt;
524 int hwtype;
525
526 hwtype = get_hwenv();
527 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
528 return;
529
530 flags = clear_int_flag();
531
532 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
533 /* Re-fill the cache line */
534 s1 = tsc->master_tsc;
535 membar_enter();
536 tsc_sync_go = TSC_SYNC_GO;
537 do {
538 /*
539 * Do not put an SMT_PAUSE here. For instance,
540 * if the master and slave are really the same
541 * hyper-threaded CPU, then you want the master
542 * to yield to the slave as quickly as possible here,
543 * but not the other way.
544 */
545 s1 = tsc_read();
546 } while (tsc->master_tsc == 0);
547 tsc->slave_tsc = s1;
548 membar_enter();
549 tsc_sync_go = TSC_SYNC_DONE;
550
551 while (tsc_sync_go != TSC_SYNC_STOP)
552 SMT_PAUSE();
553 }
554
555 restore_int_flag(flags);
556 }
557
558 /*
559 * Called once per second on a CPU from the cyclic subsystem's
560 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only)
561 */
562 void
tsc_tick(void)563 tsc_tick(void)
564 {
565 hrtime_t now, delta;
566 ushort_t spl;
567
568 /*
569 * Before we set the new variables, we set the shadow values. This
570 * allows for lock free operation in dtrace_gethrtime().
571 */
572 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
573 ipltospl(CBE_HIGH_PIL), &spl);
574
575 shadow_tsc_hrtime_base = tsc_hrtime_base;
576 shadow_tsc_last = tsc_last;
577 shadow_nsec_scale = nsec_scale;
578
579 shadow_hres_lock++;
580 splx(spl);
581
582 CLOCK_LOCK(&spl);
583
584 now = tsc_read();
585
586 if (gethrtimef == tsc_gethrtime_delta)
587 now += tsc_sync_tick_delta[CPU->cpu_id];
588
589 if (now < tsc_last) {
590 /*
591 * The TSC has just jumped into the past. We assume that
592 * this is due to a suspend/resume cycle, and we're going
593 * to use the _current_ value of TSC as the delta. This
594 * will keep tsc_hrtime_base correct. We're also going to
595 * assume that rate of tsc does not change after a suspend
596 * resume (i.e nsec_scale remains the same).
597 */
598 delta = now;
599 tsc_last_jumped += tsc_last;
600 tsc_jumped = 1;
601 } else {
602 /*
603 * Determine the number of TSC ticks since the last clock
604 * tick, and add that to the hrtime base.
605 */
606 delta = now - tsc_last;
607 }
608
609 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
610 tsc_last = now;
611
612 CLOCK_UNLOCK(spl);
613 }
614
615 void
tsc_hrtimeinit(uint64_t cpu_freq_hz)616 tsc_hrtimeinit(uint64_t cpu_freq_hz)
617 {
618 extern int gethrtime_hires;
619 longlong_t tsc;
620 ulong_t flags;
621
622 /*
623 * cpu_freq_hz is the measured cpu frequency in hertz
624 */
625
626 /*
627 * We can't accommodate CPUs slower than 31.25 MHz.
628 */
629 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
630 nsec_scale =
631 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
632 nsec_unscale =
633 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
634
635 flags = clear_int_flag();
636 tsc = tsc_read();
637 (void) tsc_gethrtime();
638 tsc_max_delta = tsc_read() - tsc;
639 restore_int_flag(flags);
640 gethrtimef = tsc_gethrtime;
641 gethrtimeunscaledf = tsc_gethrtimeunscaled;
642 scalehrtimef = tsc_scalehrtime;
643 unscalehrtimef = tsc_unscalehrtime;
644 hrtime_tick = tsc_tick;
645 gethrtime_hires = 1;
646 /*
647 * Being part of the comm page, tsc_ncpu communicates the published
648 * length of the tsc_sync_tick_delta array. This is kept zeroed to
649 * ignore the absent delta data while the TSCs are synced.
650 */
651 tsc_ncpu = 0;
652 /*
653 * Allocate memory for the structure used in the tsc sync logic.
654 * This structure should be aligned on a multiple of cache line size.
655 */
656 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
657 }
658
659 int
get_tsc_ready()660 get_tsc_ready()
661 {
662 return (tsc_ready);
663 }
664
665 /*
666 * Adjust all the deltas by adding the passed value to the array.
667 * Then use the "delt" versions of the the gethrtime functions.
668 * Note that 'tdelta' _could_ be a negative number, which should
669 * reduce the values in the array (used, for example, if the Solaris
670 * instance was moved by a virtual manager to a machine with a higher
671 * value of tsc).
672 */
673 void
tsc_adjust_delta(hrtime_t tdelta)674 tsc_adjust_delta(hrtime_t tdelta)
675 {
676 int i;
677
678 for (i = 0; i < NCPU; i++) {
679 tsc_sync_tick_delta[i] += tdelta;
680 }
681
682 gethrtimef = tsc_gethrtime_delta;
683 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
684 tsc_ncpu = NCPU;
685 }
686
687 /*
688 * Functions to manage TSC and high-res time on suspend and resume.
689 */
690
691 /*
692 * declarations needed for time adjustment
693 */
694 extern void rtcsync(void);
695 extern tod_ops_t *tod_ops;
696 /* There must be a better way than exposing nsec_scale! */
697 extern uint_t nsec_scale;
698 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
699 static timestruc_t tsc_saved_ts;
700 static int tsc_needs_resume = 0; /* We only want to do this once. */
701 int tsc_delta_onsuspend = 0;
702 int tsc_adjust_seconds = 1;
703 int tsc_suspend_count = 0;
704 int tsc_resume_in_cyclic = 0;
705
706 /*
707 * Let timestamp.c know that we are suspending. It needs to take
708 * snapshots of the current time, and do any pre-suspend work.
709 */
710 void
tsc_suspend(void)711 tsc_suspend(void)
712 {
713 /*
714 * What we need to do here, is to get the time we suspended, so that we
715 * know how much we should add to the resume.
716 * This routine is called by each CPU, so we need to handle reentry.
717 */
718 if (tsc_gethrtime_enable) {
719 /*
720 * We put the tsc_read() inside the lock as it
721 * as no locking constraints, and it puts the
722 * aquired value closer to the time stamp (in
723 * case we delay getting the lock).
724 */
725 mutex_enter(&tod_lock);
726 tsc_saved_tsc = tsc_read();
727 tsc_saved_ts = TODOP_GET(tod_ops);
728 mutex_exit(&tod_lock);
729 /* We only want to do this once. */
730 if (tsc_needs_resume == 0) {
731 if (tsc_delta_onsuspend) {
732 tsc_adjust_delta(tsc_saved_tsc);
733 } else {
734 tsc_adjust_delta(nsec_scale);
735 }
736 tsc_suspend_count++;
737 }
738 }
739
740 invalidate_cache();
741 tsc_needs_resume = 1;
742 }
743
744 /*
745 * Restore all timestamp state based on the snapshots taken at
746 * suspend time.
747 */
748 void
tsc_resume(void)749 tsc_resume(void)
750 {
751 /*
752 * We only need to (and want to) do this once. So let the first
753 * caller handle this (we are locked by the cpu lock), as it
754 * is preferential that we get the earliest sync.
755 */
756 if (tsc_needs_resume) {
757 /*
758 * If using the TSC, adjust the delta based on how long
759 * we were sleeping (or away). We also adjust for
760 * migration and a grown TSC.
761 */
762 if (tsc_saved_tsc != 0) {
763 timestruc_t ts;
764 hrtime_t now, sleep_tsc = 0;
765 int sleep_sec;
766 extern void tsc_tick(void);
767 extern uint64_t cpu_freq_hz;
768
769 /* tsc_read() MUST be before TODOP_GET() */
770 mutex_enter(&tod_lock);
771 now = tsc_read();
772 ts = TODOP_GET(tod_ops);
773 mutex_exit(&tod_lock);
774
775 /* Compute seconds of sleep time */
776 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
777
778 /*
779 * If the saved sec is less that or equal to
780 * the current ts, then there is likely a
781 * problem with the clock. Assume at least
782 * one second has passed, so that time goes forward.
783 */
784 if (sleep_sec <= 0) {
785 sleep_sec = 1;
786 }
787
788 /* How many TSC's should have occured while sleeping */
789 if (tsc_adjust_seconds)
790 sleep_tsc = sleep_sec * cpu_freq_hz;
791
792 /*
793 * We also want to subtract from the "sleep_tsc"
794 * the current value of tsc_read(), so that our
795 * adjustment accounts for the amount of time we
796 * have been resumed _or_ an adjustment based on
797 * the fact that we didn't actually power off the
798 * CPU (migration is another issue, but _should_
799 * also comply with this calculation). If the CPU
800 * never powered off, then:
801 * 'now == sleep_tsc + saved_tsc'
802 * and the delta will effectively be "0".
803 */
804 sleep_tsc -= now;
805 if (tsc_delta_onsuspend) {
806 tsc_adjust_delta(sleep_tsc);
807 } else {
808 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
809 }
810 tsc_saved_tsc = 0;
811
812 tsc_tick();
813 }
814 tsc_needs_resume = 0;
815 }
816
817 }
818