1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 #include <sys/types.h> 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/disp.h> 33 #include <sys/var.h> 34 #include <sys/cmn_err.h> 35 #include <sys/debug.h> 36 #include <sys/x86_archext.h> 37 #include <sys/archsystm.h> 38 #include <sys/cpuvar.h> 39 #include <sys/psm_defs.h> 40 #include <sys/clock.h> 41 #include <sys/atomic.h> 42 #include <sys/lockstat.h> 43 #include <sys/smp_impldefs.h> 44 #include <sys/dtrace.h> 45 #include <sys/time.h> 46 #include <sys/panic.h> 47 #include <sys/cpu.h> 48 49 /* 50 * Using the Pentium's TSC register for gethrtime() 51 * ------------------------------------------------ 52 * 53 * The Pentium family, like many chip architectures, has a high-resolution 54 * timestamp counter ("TSC") which increments once per CPU cycle. The contents 55 * of the timestamp counter are read with the RDTSC instruction. 56 * 57 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 58 * must be translated into nanoseconds in order to implement gethrtime(). 59 * We avoid inducing floating point operations in this conversion by 60 * implementing the same nsec_scale algorithm as that found in the sun4u 61 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 62 * a detailed description of the algorithm; the comment is not reproduced 63 * here. This implementation differs only in its value for NSEC_SHIFT: 64 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 65 * 60 MHz Pentiums. 66 * 67 * While TSC and %tick are both cycle counting registers, TSC's functionality 68 * falls short in several critical ways: 69 * 70 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 71 * practice they often _are_ in sync, this isn't guaranteed by the 72 * architecture. 73 * 74 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 75 * only supports writing the low 32-bits of TSC, making it impractical 76 * to rewrite. 77 * 78 * (c) The architecture doesn't have the capacity to interrupt based on 79 * arbitrary values of TSC; there is no TICK_CMPR equivalent. 80 * 81 * Together, (a) and (b) imply that software must track the skew between 82 * TSCs and account for it (it is assumed that while there may exist skew, 83 * there does not exist drift). To determine the skew between CPUs, we 84 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 85 * the online operation calls tsc_sync_master(). 86 * 87 * In the absence of time-of-day clock adjustments, gethrtime() must stay in 88 * sync with gettimeofday(). This is problematic; given (c), the software 89 * cannot drive its time-of-day source from TSC, and yet they must somehow be 90 * kept in sync. We implement this by having a routine, tsc_tick(), which 91 * is called once per second from the interrupt which drives time-of-day. 92 * 93 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 94 * atomically with nsec_scale under CLOCK_LOCK. This assures that time 95 * monotonically increases. 96 */ 97 98 #define NSEC_SHIFT 5 99 100 static uint_t nsec_scale; 101 static uint_t nsec_unscale; 102 103 /* 104 * These two variables used to be grouped together inside of a structure that 105 * lived on a single cache line. A regression (bug ID 4623398) caused the 106 * compiler to emit code that "optimized" away the while-loops below. The 107 * result was that no synchronization between the onlining and onlined CPUs 108 * took place. 109 */ 110 static volatile int tsc_ready; 111 static volatile int tsc_sync_go; 112 113 /* 114 * Used as indices into the tsc_sync_snaps[] array. 115 */ 116 #define TSC_MASTER 0 117 #define TSC_SLAVE 1 118 119 /* 120 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 121 */ 122 #define TSC_SYNC_STOP 1 123 #define TSC_SYNC_GO 2 124 #define TSC_SYNC_DONE 3 125 #define SYNC_ITERATIONS 10 126 127 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 128 unsigned int *_l = (unsigned int *)&(tsc); \ 129 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 130 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 131 } 132 133 #define TSC_CONVERT(tsc, hrt, scale) { \ 134 unsigned int *_l = (unsigned int *)&(tsc); \ 135 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 136 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 137 } 138 139 int tsc_master_slave_sync_needed = 1; 140 141 static int tsc_max_delta; 142 static hrtime_t tsc_sync_tick_delta[NCPU]; 143 typedef struct tsc_sync { 144 volatile hrtime_t master_tsc, slave_tsc; 145 } tsc_sync_t; 146 static tsc_sync_t *tscp; 147 static hrtime_t largest_tsc_delta = 0; 148 static ulong_t shortest_write_time = ~0UL; 149 150 static hrtime_t tsc_last = 0; 151 static hrtime_t tsc_last_jumped = 0; 152 static hrtime_t tsc_hrtime_base = 0; 153 static int tsc_jumped = 0; 154 155 static hrtime_t shadow_tsc_hrtime_base; 156 static hrtime_t shadow_tsc_last; 157 static uint_t shadow_nsec_scale; 158 static uint32_t shadow_hres_lock; 159 int get_tsc_ready(); 160 161 hrtime_t 162 tsc_gethrtime(void) 163 { 164 uint32_t old_hres_lock; 165 hrtime_t tsc, hrt; 166 167 do { 168 old_hres_lock = hres_lock; 169 170 if ((tsc = tsc_read()) >= tsc_last) { 171 /* 172 * It would seem to be obvious that this is true 173 * (that is, the past is less than the present), 174 * but it isn't true in the presence of suspend/resume 175 * cycles. If we manage to call gethrtime() 176 * after a resume, but before the first call to 177 * tsc_tick(), we will see the jump. In this case, 178 * we will simply use the value in TSC as the delta. 179 */ 180 tsc -= tsc_last; 181 } else if (tsc >= tsc_last - 2*tsc_max_delta) { 182 /* 183 * There is a chance that tsc_tick() has just run on 184 * another CPU, and we have drifted just enough so that 185 * we appear behind tsc_last. In this case, force the 186 * delta to be zero. 187 */ 188 tsc = 0; 189 } 190 191 hrt = tsc_hrtime_base; 192 193 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 194 } while ((old_hres_lock & ~1) != hres_lock); 195 196 return (hrt); 197 } 198 199 hrtime_t 200 tsc_gethrtime_delta(void) 201 { 202 uint32_t old_hres_lock; 203 hrtime_t tsc, hrt; 204 ulong_t flags; 205 206 do { 207 old_hres_lock = hres_lock; 208 209 /* 210 * We need to disable interrupts here to assure that we 211 * don't migrate between the call to tsc_read() and 212 * adding the CPU's TSC tick delta. Note that disabling 213 * and reenabling preemption is forbidden here because 214 * we may be in the middle of a fast trap. In the amd64 215 * kernel we cannot tolerate preemption during a fast 216 * trap. See _update_sregs(). 217 */ 218 219 flags = clear_int_flag(); 220 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 221 restore_int_flag(flags); 222 223 /* See comments in tsc_gethrtime() above */ 224 225 if (tsc >= tsc_last) { 226 tsc -= tsc_last; 227 } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 228 tsc = 0; 229 } 230 231 hrt = tsc_hrtime_base; 232 233 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 234 } while ((old_hres_lock & ~1) != hres_lock); 235 236 return (hrt); 237 } 238 239 /* 240 * This is similar to the above, but it cannot actually spin on hres_lock. 241 * As a result, it caches all of the variables it needs; if the variables 242 * don't change, it's done. 243 */ 244 hrtime_t 245 dtrace_gethrtime(void) 246 { 247 uint32_t old_hres_lock; 248 hrtime_t tsc, hrt; 249 ulong_t flags; 250 251 do { 252 old_hres_lock = hres_lock; 253 254 /* 255 * Interrupts are disabled to ensure that the thread isn't 256 * migrated between the tsc_read() and adding the CPU's 257 * TSC tick delta. 258 */ 259 flags = clear_int_flag(); 260 261 tsc = tsc_read(); 262 263 if (gethrtimef == tsc_gethrtime_delta) 264 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 265 266 restore_int_flag(flags); 267 268 /* 269 * See the comments in tsc_gethrtime(), above. 270 */ 271 if (tsc >= tsc_last) 272 tsc -= tsc_last; 273 else if (tsc >= tsc_last - 2*tsc_max_delta) 274 tsc = 0; 275 276 hrt = tsc_hrtime_base; 277 278 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 279 280 if ((old_hres_lock & ~1) == hres_lock) 281 break; 282 283 /* 284 * If we're here, the clock lock is locked -- or it has been 285 * unlocked and locked since we looked. This may be due to 286 * tsc_tick() running on another CPU -- or it may be because 287 * some code path has ended up in dtrace_probe() with 288 * CLOCK_LOCK held. We'll try to determine that we're in 289 * the former case by taking another lap if the lock has 290 * changed since when we first looked at it. 291 */ 292 if (old_hres_lock != hres_lock) 293 continue; 294 295 /* 296 * So the lock was and is locked. We'll use the old data 297 * instead. 298 */ 299 old_hres_lock = shadow_hres_lock; 300 301 /* 302 * Again, disable interrupts to ensure that the thread 303 * isn't migrated between the tsc_read() and adding 304 * the CPU's TSC tick delta. 305 */ 306 flags = clear_int_flag(); 307 308 tsc = tsc_read(); 309 310 if (gethrtimef == tsc_gethrtime_delta) 311 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 312 313 restore_int_flag(flags); 314 315 /* 316 * See the comments in tsc_gethrtime(), above. 317 */ 318 if (tsc >= shadow_tsc_last) 319 tsc -= shadow_tsc_last; 320 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 321 tsc = 0; 322 323 hrt = shadow_tsc_hrtime_base; 324 325 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 326 } while ((old_hres_lock & ~1) != shadow_hres_lock); 327 328 return (hrt); 329 } 330 331 hrtime_t 332 tsc_gethrtimeunscaled(void) 333 { 334 uint32_t old_hres_lock; 335 hrtime_t tsc; 336 337 do { 338 old_hres_lock = hres_lock; 339 340 /* See tsc_tick(). */ 341 tsc = tsc_read() + tsc_last_jumped; 342 } while ((old_hres_lock & ~1) != hres_lock); 343 344 return (tsc); 345 } 346 347 /* 348 * Convert a nanosecond based timestamp to tsc 349 */ 350 uint64_t 351 tsc_unscalehrtime(hrtime_t nsec) 352 { 353 hrtime_t tsc; 354 355 if (tsc_gethrtime_enable) { 356 TSC_CONVERT(nsec, tsc, nsec_unscale); 357 return (tsc); 358 } 359 return ((uint64_t)nsec); 360 } 361 362 /* Convert a tsc timestamp to nanoseconds */ 363 void 364 tsc_scalehrtime(hrtime_t *tsc) 365 { 366 hrtime_t hrt; 367 hrtime_t mytsc; 368 369 if (tsc == NULL) 370 return; 371 mytsc = *tsc; 372 373 TSC_CONVERT(mytsc, hrt, nsec_scale); 374 *tsc = hrt; 375 } 376 377 hrtime_t 378 tsc_gethrtimeunscaled_delta(void) 379 { 380 hrtime_t hrt; 381 ulong_t flags; 382 383 /* 384 * Similarly to tsc_gethrtime_delta, we need to disable preemption 385 * to prevent migration between the call to tsc_gethrtimeunscaled 386 * and adding the CPU's hrtime delta. Note that disabling and 387 * reenabling preemption is forbidden here because we may be in the 388 * middle of a fast trap. In the amd64 kernel we cannot tolerate 389 * preemption during a fast trap. See _update_sregs(). 390 */ 391 392 flags = clear_int_flag(); 393 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 394 restore_int_flag(flags); 395 396 return (hrt); 397 } 398 399 /* 400 * Called by the master in the TSC sync operation (usually the boot CPU). 401 * If the slave is discovered to have a skew, gethrtimef will be changed to 402 * point to tsc_gethrtime_delta(). Calculating skews is precise only when 403 * the master and slave TSCs are read simultaneously; however, there is no 404 * algorithm that can read both CPUs in perfect simultaneity. The proposed 405 * algorithm is an approximate method based on the behaviour of cache 406 * management. The slave CPU continuously reads TSC and then reads a global 407 * variable which the master CPU updates. The moment the master's update reaches 408 * the slave's visibility (being forced by an mfence operation) we use the TSC 409 * reading taken on the slave. A corresponding TSC read will be taken on the 410 * master as soon as possible after finishing the mfence operation. But the 411 * delay between causing the slave to notice the invalid cache line and the 412 * competion of mfence is not repeatable. This error is heuristically assumed 413 * to be 1/4th of the total write time as being measured by the two TSC reads 414 * on the master sandwiching the mfence. Furthermore, due to the nature of 415 * bus arbitration, contention on memory bus, etc., the time taken for the write 416 * to reflect globally can vary a lot. So instead of taking a single reading, 417 * a set of readings are taken and the one with least write time is chosen 418 * to calculate the final skew. 419 * 420 * TSC sync is disabled in the context of virtualization because the CPUs 421 * assigned to the guest are virtual CPUs which means the real CPUs on which 422 * guest runs keep changing during life time of guest OS. So we would end up 423 * calculating TSC skews for a set of CPUs during boot whereas the guest 424 * might migrate to a different set of physical CPUs at a later point of 425 * time. 426 */ 427 void 428 tsc_sync_master(processorid_t slave) 429 { 430 ulong_t flags, source, min_write_time = ~0UL; 431 hrtime_t write_time, x, mtsc_after, tdelta; 432 tsc_sync_t *tsc = tscp; 433 int cnt; 434 int hwtype; 435 436 hwtype = get_hwenv(); 437 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 438 return; 439 440 flags = clear_int_flag(); 441 source = CPU->cpu_id; 442 443 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 444 while (tsc_sync_go != TSC_SYNC_GO) 445 SMT_PAUSE(); 446 447 tsc->master_tsc = tsc_read(); 448 membar_enter(); 449 mtsc_after = tsc_read(); 450 while (tsc_sync_go != TSC_SYNC_DONE) 451 SMT_PAUSE(); 452 write_time = mtsc_after - tsc->master_tsc; 453 if (write_time <= min_write_time) { 454 min_write_time = write_time; 455 /* 456 * Apply heuristic adjustment only if the calculated 457 * delta is > 1/4th of the write time. 458 */ 459 x = tsc->slave_tsc - mtsc_after; 460 if (x < 0) 461 x = -x; 462 if (x > (min_write_time/4)) 463 /* 464 * Subtract 1/4th of the measured write time 465 * from the master's TSC value, as an estimate 466 * of how late the mfence completion came 467 * after the slave noticed the cache line 468 * change. 469 */ 470 tdelta = tsc->slave_tsc - 471 (mtsc_after - (min_write_time/4)); 472 else 473 tdelta = tsc->slave_tsc - mtsc_after; 474 tsc_sync_tick_delta[slave] = 475 tsc_sync_tick_delta[source] - tdelta; 476 } 477 478 tsc->master_tsc = tsc->slave_tsc = write_time = 0; 479 membar_enter(); 480 tsc_sync_go = TSC_SYNC_STOP; 481 } 482 if (tdelta < 0) 483 tdelta = -tdelta; 484 if (tdelta > largest_tsc_delta) 485 largest_tsc_delta = tdelta; 486 if (min_write_time < shortest_write_time) 487 shortest_write_time = min_write_time; 488 /* 489 * Enable delta variants of tsc functions if the largest of all chosen 490 * deltas is > smallest of the write time. 491 */ 492 if (largest_tsc_delta > shortest_write_time) { 493 gethrtimef = tsc_gethrtime_delta; 494 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 495 } 496 restore_int_flag(flags); 497 } 498 499 /* 500 * Called by a CPU which has just been onlined. It is expected that the CPU 501 * performing the online operation will call tsc_sync_master(). 502 * 503 * TSC sync is disabled in the context of virtualization. See comments 504 * above tsc_sync_master. 505 */ 506 void 507 tsc_sync_slave(void) 508 { 509 ulong_t flags; 510 hrtime_t s1; 511 tsc_sync_t *tsc = tscp; 512 int cnt; 513 int hwtype; 514 515 hwtype = get_hwenv(); 516 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 517 return; 518 519 flags = clear_int_flag(); 520 521 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 522 /* Re-fill the cache line */ 523 s1 = tsc->master_tsc; 524 membar_enter(); 525 tsc_sync_go = TSC_SYNC_GO; 526 do { 527 /* 528 * Do not put an SMT_PAUSE here. For instance, 529 * if the master and slave are really the same 530 * hyper-threaded CPU, then you want the master 531 * to yield to the slave as quickly as possible here, 532 * but not the other way. 533 */ 534 s1 = tsc_read(); 535 } while (tsc->master_tsc == 0); 536 tsc->slave_tsc = s1; 537 membar_enter(); 538 tsc_sync_go = TSC_SYNC_DONE; 539 540 while (tsc_sync_go != TSC_SYNC_STOP) 541 SMT_PAUSE(); 542 } 543 544 restore_int_flag(flags); 545 } 546 547 /* 548 * Called once per second on a CPU from the cyclic subsystem's 549 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 550 */ 551 void 552 tsc_tick(void) 553 { 554 hrtime_t now, delta; 555 ushort_t spl; 556 557 /* 558 * Before we set the new variables, we set the shadow values. This 559 * allows for lock free operation in dtrace_gethrtime(). 560 */ 561 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 562 ipltospl(CBE_HIGH_PIL), &spl); 563 564 shadow_tsc_hrtime_base = tsc_hrtime_base; 565 shadow_tsc_last = tsc_last; 566 shadow_nsec_scale = nsec_scale; 567 568 shadow_hres_lock++; 569 splx(spl); 570 571 CLOCK_LOCK(&spl); 572 573 now = tsc_read(); 574 575 if (gethrtimef == tsc_gethrtime_delta) 576 now += tsc_sync_tick_delta[CPU->cpu_id]; 577 578 if (now < tsc_last) { 579 /* 580 * The TSC has just jumped into the past. We assume that 581 * this is due to a suspend/resume cycle, and we're going 582 * to use the _current_ value of TSC as the delta. This 583 * will keep tsc_hrtime_base correct. We're also going to 584 * assume that rate of tsc does not change after a suspend 585 * resume (i.e nsec_scale remains the same). 586 */ 587 delta = now; 588 tsc_last_jumped += tsc_last; 589 tsc_jumped = 1; 590 } else { 591 /* 592 * Determine the number of TSC ticks since the last clock 593 * tick, and add that to the hrtime base. 594 */ 595 delta = now - tsc_last; 596 } 597 598 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 599 tsc_last = now; 600 601 CLOCK_UNLOCK(spl); 602 } 603 604 void 605 tsc_hrtimeinit(uint64_t cpu_freq_hz) 606 { 607 extern int gethrtime_hires; 608 longlong_t tsc; 609 ulong_t flags; 610 611 /* 612 * cpu_freq_hz is the measured cpu frequency in hertz 613 */ 614 615 /* 616 * We can't accommodate CPUs slower than 31.25 MHz. 617 */ 618 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 619 nsec_scale = 620 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 621 nsec_unscale = 622 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC); 623 624 flags = clear_int_flag(); 625 tsc = tsc_read(); 626 (void) tsc_gethrtime(); 627 tsc_max_delta = tsc_read() - tsc; 628 restore_int_flag(flags); 629 gethrtimef = tsc_gethrtime; 630 gethrtimeunscaledf = tsc_gethrtimeunscaled; 631 scalehrtimef = tsc_scalehrtime; 632 unscalehrtimef = tsc_unscalehrtime; 633 hrtime_tick = tsc_tick; 634 gethrtime_hires = 1; 635 /* 636 * Allocate memory for the structure used in the tsc sync logic. 637 * This structure should be aligned on a multiple of cache line size. 638 */ 639 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); 640 } 641 642 int 643 get_tsc_ready() 644 { 645 return (tsc_ready); 646 } 647 648 /* 649 * Adjust all the deltas by adding the passed value to the array. 650 * Then use the "delt" versions of the the gethrtime functions. 651 * Note that 'tdelta' _could_ be a negative number, which should 652 * reduce the values in the array (used, for example, if the Solaris 653 * instance was moved by a virtual manager to a machine with a higher 654 * value of tsc). 655 */ 656 void 657 tsc_adjust_delta(hrtime_t tdelta) 658 { 659 int i; 660 661 for (i = 0; i < NCPU; i++) { 662 tsc_sync_tick_delta[i] += tdelta; 663 } 664 665 gethrtimef = tsc_gethrtime_delta; 666 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 667 } 668 669 /* 670 * Functions to manage TSC and high-res time on suspend and resume. 671 */ 672 673 /* 674 * declarations needed for time adjustment 675 */ 676 extern void rtcsync(void); 677 extern tod_ops_t *tod_ops; 678 /* There must be a better way than exposing nsec_scale! */ 679 extern uint_t nsec_scale; 680 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 681 static timestruc_t tsc_saved_ts; 682 static int tsc_needs_resume = 0; /* We only want to do this once. */ 683 int tsc_delta_onsuspend = 0; 684 int tsc_adjust_seconds = 1; 685 int tsc_suspend_count = 0; 686 int tsc_resume_in_cyclic = 0; 687 688 /* 689 * Let timestamp.c know that we are suspending. It needs to take 690 * snapshots of the current time, and do any pre-suspend work. 691 */ 692 void 693 tsc_suspend(void) 694 { 695 /* 696 * What we need to do here, is to get the time we suspended, so that we 697 * know how much we should add to the resume. 698 * This routine is called by each CPU, so we need to handle reentry. 699 */ 700 if (tsc_gethrtime_enable) { 701 /* 702 * We put the tsc_read() inside the lock as it 703 * as no locking constraints, and it puts the 704 * aquired value closer to the time stamp (in 705 * case we delay getting the lock). 706 */ 707 mutex_enter(&tod_lock); 708 tsc_saved_tsc = tsc_read(); 709 tsc_saved_ts = TODOP_GET(tod_ops); 710 mutex_exit(&tod_lock); 711 /* We only want to do this once. */ 712 if (tsc_needs_resume == 0) { 713 if (tsc_delta_onsuspend) { 714 tsc_adjust_delta(tsc_saved_tsc); 715 } else { 716 tsc_adjust_delta(nsec_scale); 717 } 718 tsc_suspend_count++; 719 } 720 } 721 722 invalidate_cache(); 723 tsc_needs_resume = 1; 724 } 725 726 /* 727 * Restore all timestamp state based on the snapshots taken at 728 * suspend time. 729 */ 730 void 731 tsc_resume(void) 732 { 733 /* 734 * We only need to (and want to) do this once. So let the first 735 * caller handle this (we are locked by the cpu lock), as it 736 * is preferential that we get the earliest sync. 737 */ 738 if (tsc_needs_resume) { 739 /* 740 * If using the TSC, adjust the delta based on how long 741 * we were sleeping (or away). We also adjust for 742 * migration and a grown TSC. 743 */ 744 if (tsc_saved_tsc != 0) { 745 timestruc_t ts; 746 hrtime_t now, sleep_tsc = 0; 747 int sleep_sec; 748 extern void tsc_tick(void); 749 extern uint64_t cpu_freq_hz; 750 751 /* tsc_read() MUST be before TODOP_GET() */ 752 mutex_enter(&tod_lock); 753 now = tsc_read(); 754 ts = TODOP_GET(tod_ops); 755 mutex_exit(&tod_lock); 756 757 /* Compute seconds of sleep time */ 758 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 759 760 /* 761 * If the saved sec is less that or equal to 762 * the current ts, then there is likely a 763 * problem with the clock. Assume at least 764 * one second has passed, so that time goes forward. 765 */ 766 if (sleep_sec <= 0) { 767 sleep_sec = 1; 768 } 769 770 /* How many TSC's should have occured while sleeping */ 771 if (tsc_adjust_seconds) 772 sleep_tsc = sleep_sec * cpu_freq_hz; 773 774 /* 775 * We also want to subtract from the "sleep_tsc" 776 * the current value of tsc_read(), so that our 777 * adjustment accounts for the amount of time we 778 * have been resumed _or_ an adjustment based on 779 * the fact that we didn't actually power off the 780 * CPU (migration is another issue, but _should_ 781 * also comply with this calculation). If the CPU 782 * never powered off, then: 783 * 'now == sleep_tsc + saved_tsc' 784 * and the delta will effectively be "0". 785 */ 786 sleep_tsc -= now; 787 if (tsc_delta_onsuspend) { 788 tsc_adjust_delta(sleep_tsc); 789 } else { 790 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 791 } 792 tsc_saved_tsc = 0; 793 794 tsc_tick(); 795 } 796 tsc_needs_resume = 0; 797 } 798 799 } 800