1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/disp.h> 33 #include <sys/var.h> 34 #include <sys/cmn_err.h> 35 #include <sys/debug.h> 36 #include <sys/x86_archext.h> 37 #include <sys/archsystm.h> 38 #include <sys/cpuvar.h> 39 #include <sys/psm_defs.h> 40 #include <sys/clock.h> 41 #include <sys/atomic.h> 42 #include <sys/lockstat.h> 43 #include <sys/smp_impldefs.h> 44 #include <sys/dtrace.h> 45 #include <sys/time.h> 46 #include <sys/panic.h> 47 48 /* 49 * Using the Pentium's TSC register for gethrtime() 50 * ------------------------------------------------ 51 * 52 * The Pentium family, like many chip architectures, has a high-resolution 53 * timestamp counter ("TSC") which increments once per CPU cycle. The contents 54 * of the timestamp counter are read with the RDTSC instruction. 55 * 56 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 57 * must be translated into nanoseconds in order to implement gethrtime(). 58 * We avoid inducing floating point operations in this conversion by 59 * implementing the same nsec_scale algorithm as that found in the sun4u 60 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 61 * a detailed description of the algorithm; the comment is not reproduced 62 * here. This implementation differs only in its value for NSEC_SHIFT: 63 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 64 * 60 MHz Pentiums. 65 * 66 * While TSC and %tick are both cycle counting registers, TSC's functionality 67 * falls short in several critical ways: 68 * 69 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 70 * practice they often _are_ in sync, this isn't guaranteed by the 71 * architecture. 72 * 73 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 74 * only supports writing the low 32-bits of TSC, making it impractical 75 * to rewrite. 76 * 77 * (c) The architecture doesn't have the capacity to interrupt based on 78 * arbitrary values of TSC; there is no TICK_CMPR equivalent. 79 * 80 * Together, (a) and (b) imply that software must track the skew between 81 * TSCs and account for it (it is assumed that while there may exist skew, 82 * there does not exist drift). To determine the skew between CPUs, we 83 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 84 * the online operation calls tsc_sync_master(). Once both CPUs are ready, 85 * the master sets a shared flag, and each reads its TSC register. To reduce 86 * bias, we then wait until both CPUs are ready again, but this time the 87 * slave sets the shared flag, and each reads its TSC register again. The 88 * master compares the average of the two sample values, and, if observable 89 * skew is found, changes the gethrtimef function pointer to point to a 90 * gethrtime() implementation which will take the discovered skew into 91 * consideration. 92 * 93 * In the absence of time-of-day clock adjustments, gethrtime() must stay in 94 * sync with gettimeofday(). This is problematic; given (c), the software 95 * cannot drive its time-of-day source from TSC, and yet they must somehow be 96 * kept in sync. We implement this by having a routine, tsc_tick(), which 97 * is called once per second from the interrupt which drives time-of-day. 98 * tsc_tick() recalculates nsec_scale based on the number of the CPU cycles 99 * since boot versus the number of seconds since boot. This algorithm 100 * becomes more accurate over time and converges quickly; the error in 101 * nsec_scale is typically under 1 ppm less than 10 seconds after boot, and 102 * is less than 100 ppb 1 minute after boot. 103 * 104 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 105 * atomically with nsec_scale under CLOCK_LOCK. This assures that time 106 * monotonically increases. 107 */ 108 109 #define NSEC_SHIFT 5 110 111 static uint_t nsec_scale; 112 113 /* 114 * These two variables used to be grouped together inside of a structure that 115 * lived on a single cache line. A regression (bug ID 4623398) caused the 116 * compiler to emit code that "optimized" away the while-loops below. The 117 * result was that no synchronization between the onlining and onlined CPUs 118 * took place. 119 */ 120 static volatile int tsc_ready; 121 static volatile int tsc_sync_go; 122 123 /* 124 * Used as indices into the tsc_sync_snaps[] array. 125 */ 126 #define TSC_MASTER 0 127 #define TSC_SLAVE 1 128 129 /* 130 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 131 */ 132 #define TSC_SYNC_STOP 1 133 #define TSC_SYNC_GO 2 134 #define TSC_SYNC_AGAIN 3 135 136 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 137 unsigned int *_l = (unsigned int *)&(tsc); \ 138 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 139 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 140 } 141 142 #define TSC_CONVERT(tsc, hrt, scale) { \ 143 unsigned int *_l = (unsigned int *)&(tsc); \ 144 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 145 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 146 } 147 148 int tsc_master_slave_sync_needed = 1; 149 150 static int tsc_max_delta; 151 static hrtime_t tsc_sync_snaps[2]; 152 static hrtime_t tsc_sync_delta[NCPU]; 153 static hrtime_t tsc_sync_tick_delta[NCPU]; 154 static hrtime_t tsc_last = 0; 155 static hrtime_t tsc_last_jumped = 0; 156 static hrtime_t tsc_hrtime_base = 0; 157 static int tsc_jumped = 0; 158 159 static hrtime_t shadow_tsc_hrtime_base; 160 static hrtime_t shadow_tsc_last; 161 static uint_t shadow_nsec_scale; 162 static uint32_t shadow_hres_lock; 163 int get_tsc_ready(); 164 165 hrtime_t 166 tsc_gethrtime(void) 167 { 168 uint32_t old_hres_lock; 169 hrtime_t tsc, hrt; 170 171 do { 172 old_hres_lock = hres_lock; 173 174 if ((tsc = tsc_read()) >= tsc_last) { 175 /* 176 * It would seem to be obvious that this is true 177 * (that is, the past is less than the present), 178 * but it isn't true in the presence of suspend/resume 179 * cycles. If we manage to call gethrtime() 180 * after a resume, but before the first call to 181 * tsc_tick(), we will see the jump. In this case, 182 * we will simply use the value in TSC as the delta. 183 */ 184 tsc -= tsc_last; 185 } else if (tsc >= tsc_last - 2*tsc_max_delta) { 186 /* 187 * There is a chance that tsc_tick() has just run on 188 * another CPU, and we have drifted just enough so that 189 * we appear behind tsc_last. In this case, force the 190 * delta to be zero. 191 */ 192 tsc = 0; 193 } 194 195 hrt = tsc_hrtime_base; 196 197 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 198 } while ((old_hres_lock & ~1) != hres_lock); 199 200 return (hrt); 201 } 202 203 hrtime_t 204 tsc_gethrtime_delta(void) 205 { 206 uint32_t old_hres_lock; 207 hrtime_t tsc, hrt; 208 int flags; 209 210 do { 211 old_hres_lock = hres_lock; 212 213 /* 214 * We need to disable interrupts here to assure that we 215 * don't migrate between the call to tsc_read() and 216 * adding the CPU's TSC tick delta. Note that disabling 217 * and reenabling preemption is forbidden here because 218 * we may be in the middle of a fast trap. In the amd64 219 * kernel we cannot tolerate preemption during a fast 220 * trap. See _update_sregs(). 221 */ 222 223 flags = clear_int_flag(); 224 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 225 restore_int_flag(flags); 226 227 /* See comments in tsc_gethrtime() above */ 228 229 if (tsc >= tsc_last) { 230 tsc -= tsc_last; 231 } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 232 tsc = 0; 233 } 234 235 hrt = tsc_hrtime_base; 236 237 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 238 } while ((old_hres_lock & ~1) != hres_lock); 239 240 return (hrt); 241 } 242 243 /* 244 * This is similar to the above, but it cannot actually spin on hres_lock. 245 * As a result, it caches all of the variables it needs; if the variables 246 * don't change, it's done. 247 */ 248 hrtime_t 249 dtrace_gethrtime(void) 250 { 251 uint32_t old_hres_lock; 252 hrtime_t tsc, hrt; 253 int flags; 254 255 do { 256 old_hres_lock = hres_lock; 257 258 /* 259 * Interrupts are disabled to ensure that the thread isn't 260 * migrated between the tsc_read() and adding the CPU's 261 * TSC tick delta. 262 */ 263 flags = clear_int_flag(); 264 265 tsc = tsc_read(); 266 267 if (gethrtimef == tsc_gethrtime_delta) 268 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 269 270 restore_int_flag(flags); 271 272 /* 273 * See the comments in tsc_gethrtime(), above. 274 */ 275 if (tsc >= tsc_last) 276 tsc -= tsc_last; 277 else if (tsc >= tsc_last - 2*tsc_max_delta) 278 tsc = 0; 279 280 hrt = tsc_hrtime_base; 281 282 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 283 284 if ((old_hres_lock & ~1) == hres_lock) 285 break; 286 287 /* 288 * If we're here, the clock lock is locked -- or it has been 289 * unlocked and locked since we looked. This may be due to 290 * tsc_tick() running on another CPU -- or it may be because 291 * some code path has ended up in dtrace_probe() with 292 * CLOCK_LOCK held. We'll try to determine that we're in 293 * the former case by taking another lap if the lock has 294 * changed since when we first looked at it. 295 */ 296 if (old_hres_lock != hres_lock) 297 continue; 298 299 /* 300 * So the lock was and is locked. We'll use the old data 301 * instead. 302 */ 303 old_hres_lock = shadow_hres_lock; 304 305 /* 306 * Again, disable interrupts to ensure that the thread 307 * isn't migrated between the tsc_read() and adding 308 * the CPU's TSC tick delta. 309 */ 310 flags = clear_int_flag(); 311 312 tsc = tsc_read(); 313 314 if (gethrtimef == tsc_gethrtime_delta) 315 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 316 317 restore_int_flag(flags); 318 319 /* 320 * See the comments in tsc_gethrtime(), above. 321 */ 322 if (tsc >= shadow_tsc_last) 323 tsc -= shadow_tsc_last; 324 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 325 tsc = 0; 326 327 hrt = shadow_tsc_hrtime_base; 328 329 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 330 } while ((old_hres_lock & ~1) != shadow_hres_lock); 331 332 return (hrt); 333 } 334 335 hrtime_t 336 tsc_gethrtimeunscaled(void) 337 { 338 uint32_t old_hres_lock; 339 hrtime_t tsc; 340 341 do { 342 old_hres_lock = hres_lock; 343 344 /* See tsc_tick(). */ 345 tsc = tsc_read() + tsc_last_jumped; 346 } while ((old_hres_lock & ~1) != hres_lock); 347 348 return (tsc); 349 } 350 351 352 /* Convert a tsc timestamp to nanoseconds */ 353 void 354 tsc_scalehrtime(hrtime_t *tsc) 355 { 356 hrtime_t hrt; 357 hrtime_t mytsc; 358 359 if (tsc == NULL) 360 return; 361 mytsc = *tsc; 362 363 TSC_CONVERT(mytsc, hrt, nsec_scale); 364 *tsc = hrt; 365 } 366 367 hrtime_t 368 tsc_gethrtimeunscaled_delta(void) 369 { 370 hrtime_t hrt; 371 int flags; 372 373 /* 374 * Similarly to tsc_gethrtime_delta, we need to disable preemption 375 * to prevent migration between the call to tsc_gethrtimeunscaled 376 * and adding the CPU's hrtime delta. Note that disabling and 377 * reenabling preemption is forbidden here because we may be in the 378 * middle of a fast trap. In the amd64 kernel we cannot tolerate 379 * preemption during a fast trap. See _update_sregs(). 380 */ 381 382 flags = clear_int_flag(); 383 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 384 restore_int_flag(flags); 385 386 return (hrt); 387 } 388 389 /* 390 * Called by the master after the sync operation is complete. If the 391 * slave is discovered to lag, gethrtimef will be changed to point to 392 * tsc_gethrtime_delta(). 393 */ 394 static void 395 tsc_digest(processorid_t target) 396 { 397 hrtime_t tdelta, hdelta = 0; 398 int max = tsc_max_delta; 399 processorid_t source = CPU->cpu_id; 400 int update; 401 402 update = tsc_sync_delta[source] != 0 || 403 gethrtimef == tsc_gethrtime_delta; 404 405 /* 406 * We divide by 2 since each of the data points is the sum of two TSC 407 * reads; this takes the average of the two. 408 */ 409 tdelta = (tsc_sync_snaps[TSC_SLAVE] - tsc_sync_snaps[TSC_MASTER]) / 2; 410 if ((tdelta > max) || ((tdelta >= 0) && update)) { 411 TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 412 tsc_sync_delta[target] = tsc_sync_delta[source] - hdelta; 413 tsc_sync_tick_delta[target] = tsc_sync_tick_delta[source] 414 -tdelta; 415 gethrtimef = tsc_gethrtime_delta; 416 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 417 return; 418 } 419 420 tdelta = -tdelta; 421 if ((tdelta > max) || update) { 422 TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 423 tsc_sync_delta[target] = tsc_sync_delta[source] + hdelta; 424 tsc_sync_tick_delta[target] = tsc_sync_tick_delta[source] 425 + tdelta; 426 gethrtimef = tsc_gethrtime_delta; 427 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 428 } 429 430 } 431 432 /* 433 * Called by a CPU which has just performed an online operation on another 434 * CPU. It is expected that the newly onlined CPU will call tsc_sync_slave(). 435 */ 436 void 437 tsc_sync_master(processorid_t slave) 438 { 439 ulong_t flags; 440 hrtime_t hrt; 441 442 if (!tsc_master_slave_sync_needed) 443 return; 444 445 ASSERT(tsc_sync_go != TSC_SYNC_GO); 446 447 flags = clear_int_flag(); 448 449 /* 450 * Wait for the slave CPU to arrive. 451 */ 452 while (tsc_ready != TSC_SYNC_GO) 453 continue; 454 455 /* 456 * Tell the slave CPU to begin reading its TSC; read our own. 457 */ 458 tsc_sync_go = TSC_SYNC_GO; 459 hrt = tsc_read(); 460 461 /* 462 * Tell the slave that we're ready, and wait for the slave to tell us 463 * to read our TSC again. 464 */ 465 tsc_ready = TSC_SYNC_AGAIN; 466 while (tsc_sync_go != TSC_SYNC_AGAIN) 467 continue; 468 469 hrt += tsc_read(); 470 tsc_sync_snaps[TSC_MASTER] = hrt; 471 472 /* 473 * Wait for the slave to finish reading its TSC. 474 */ 475 while (tsc_ready != TSC_SYNC_STOP) 476 continue; 477 478 /* 479 * At this point, both CPUs have performed their tsc_read() calls. 480 * We'll digest it now before letting the slave CPU return. 481 */ 482 tsc_digest(slave); 483 tsc_sync_go = TSC_SYNC_STOP; 484 485 restore_int_flag(flags); 486 } 487 488 /* 489 * Called by a CPU which has just been onlined. It is expected that the CPU 490 * performing the online operation will call tsc_sync_master(). 491 */ 492 void 493 tsc_sync_slave(void) 494 { 495 ulong_t flags; 496 hrtime_t hrt; 497 498 if (!tsc_master_slave_sync_needed) 499 return; 500 501 ASSERT(tsc_sync_go != TSC_SYNC_GO); 502 503 flags = clear_int_flag(); 504 505 /* to test tsc_gethrtime_delta, add wrmsr(REG_TSC, 0) here */ 506 507 /* 508 * Tell the master CPU that we're ready, and wait for the master to 509 * tell us to begin reading our TSC. 510 */ 511 tsc_ready = TSC_SYNC_GO; 512 while (tsc_sync_go != TSC_SYNC_GO) 513 continue; 514 515 hrt = tsc_read(); 516 517 /* 518 * Wait for the master CPU to be ready to read its TSC again. 519 */ 520 while (tsc_ready != TSC_SYNC_AGAIN) 521 continue; 522 523 /* 524 * Tell the master CPU to read its TSC again; read ours again. 525 */ 526 tsc_sync_go = TSC_SYNC_AGAIN; 527 528 hrt += tsc_read(); 529 tsc_sync_snaps[TSC_SLAVE] = hrt; 530 531 /* 532 * Tell the master that we're done, and wait to be dismissed. 533 */ 534 tsc_ready = TSC_SYNC_STOP; 535 while (tsc_sync_go != TSC_SYNC_STOP) 536 continue; 537 538 restore_int_flag(flags); 539 } 540 541 /* 542 * Called once per second on a CPU from the cyclic subsystem's 543 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 544 */ 545 void 546 tsc_tick(void) 547 { 548 hrtime_t now, delta; 549 ushort_t spl; 550 551 /* 552 * Before we set the new variables, we set the shadow values. This 553 * allows for lock free operation in dtrace_gethrtime(). 554 */ 555 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 556 ipltospl(CBE_HIGH_PIL), &spl); 557 558 shadow_tsc_hrtime_base = tsc_hrtime_base; 559 shadow_tsc_last = tsc_last; 560 shadow_nsec_scale = nsec_scale; 561 562 shadow_hres_lock++; 563 splx(spl); 564 565 CLOCK_LOCK(&spl); 566 567 now = tsc_read(); 568 569 if (gethrtimef == tsc_gethrtime_delta) 570 now += tsc_sync_tick_delta[CPU->cpu_id]; 571 572 if (now < tsc_last) { 573 /* 574 * The TSC has just jumped into the past. We assume that 575 * this is due to a suspend/resume cycle, and we're going 576 * to use the _current_ value of TSC as the delta. This 577 * will keep tsc_hrtime_base correct. We're also going to 578 * assume that rate of tsc does not change after a suspend 579 * resume (i.e nsec_scale remains the same). 580 */ 581 delta = now; 582 tsc_last_jumped += tsc_last; 583 tsc_jumped = 1; 584 } else { 585 /* 586 * Determine the number of TSC ticks since the last clock 587 * tick, and add that to the hrtime base. 588 */ 589 delta = now - tsc_last; 590 } 591 592 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 593 tsc_last = now; 594 595 CLOCK_UNLOCK(spl); 596 } 597 598 void 599 tsc_hrtimeinit(uint64_t cpu_freq_hz) 600 { 601 extern int gethrtime_hires; 602 longlong_t tsc; 603 ulong_t flags; 604 605 /* 606 * cpu_freq_hz is the measured cpu frequency in hertz 607 */ 608 609 /* 610 * We can't accommodate CPUs slower than 31.25 MHz. 611 */ 612 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 613 nsec_scale = 614 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 615 616 flags = clear_int_flag(); 617 tsc = tsc_read(); 618 (void) tsc_gethrtime(); 619 tsc_max_delta = tsc_read() - tsc; 620 restore_int_flag(flags); 621 gethrtimef = tsc_gethrtime; 622 gethrtimeunscaledf = tsc_gethrtimeunscaled; 623 scalehrtimef = tsc_scalehrtime; 624 hrtime_tick = tsc_tick; 625 gethrtime_hires = 1; 626 } 627 628 int 629 get_tsc_ready() 630 { 631 return (tsc_ready); 632 } 633 634 /* 635 * Adjust all the deltas by adding the passed value to the array. 636 * Then use the "delt" versions of the the gethrtime functions. 637 * Note that 'tdelta' _could_ be a negative number, which should 638 * reduce the values in the array (used, for example, if the Solaris 639 * instance was moved by a virtual manager to a machine with a higher 640 * value of tsc). 641 */ 642 void 643 tsc_adjust_delta(hrtime_t tdelta) 644 { 645 int i; 646 hrtime_t hdelta = 0; 647 648 TSC_CONVERT(tdelta, hdelta, nsec_scale); 649 650 for (i = 0; i < NCPU; i++) { 651 tsc_sync_delta[i] += hdelta; 652 tsc_sync_tick_delta[i] += tdelta; 653 } 654 655 gethrtimef = tsc_gethrtime_delta; 656 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 657 } 658 659 /* 660 * Functions to manage TSC and high-res time on suspend and resume. 661 */ 662 663 /* 664 * declarations needed for time adjustment 665 */ 666 extern void rtcsync(void); 667 extern tod_ops_t *tod_ops; 668 /* There must be a better way than exposing nsec_scale! */ 669 extern uint_t nsec_scale; 670 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 671 static timestruc_t tsc_saved_ts; 672 static int tsc_needs_resume = 0; /* We only want to do this once. */ 673 int tsc_delta_onsuspend = 0; 674 int tsc_adjust_seconds = 1; 675 int tsc_suspend_count = 0; 676 int tsc_resume_in_cyclic = 0; 677 678 /* 679 * Let timestamp.c know that we are suspending. It needs to take 680 * snapshots of the current time, and do any pre-suspend work. 681 */ 682 void 683 tsc_suspend(void) 684 { 685 /* 686 * What we need to do here, is to get the time we suspended, so that we 687 * know how much we should add to the resume. 688 * This routine is called by each CPU, so we need to handle reentry. 689 */ 690 if (tsc_gethrtime_enable) { 691 /* 692 * We put the tsc_read() inside the lock as it 693 * as no locking constraints, and it puts the 694 * aquired value closer to the time stamp (in 695 * case we delay getting the lock). 696 */ 697 mutex_enter(&tod_lock); 698 tsc_saved_tsc = tsc_read(); 699 tsc_saved_ts = TODOP_GET(tod_ops); 700 mutex_exit(&tod_lock); 701 /* We only want to do this once. */ 702 if (tsc_needs_resume == 0) { 703 if (tsc_delta_onsuspend) { 704 tsc_adjust_delta(tsc_saved_tsc); 705 } else { 706 tsc_adjust_delta(nsec_scale); 707 } 708 tsc_suspend_count++; 709 } 710 } 711 712 invalidate_cache(); 713 tsc_needs_resume = 1; 714 } 715 716 /* 717 * Restore all timestamp state based on the snapshots taken at 718 * suspend time. 719 */ 720 void 721 tsc_resume(void) 722 { 723 /* 724 * We only need to (and want to) do this once. So let the first 725 * caller handle this (we are locked by the cpu lock), as it 726 * is preferential that we get the earliest sync. 727 */ 728 if (tsc_needs_resume) { 729 /* 730 * If using the TSC, adjust the delta based on how long 731 * we were sleeping (or away). We also adjust for 732 * migration and a grown TSC. 733 */ 734 if (tsc_saved_tsc != 0) { 735 timestruc_t ts; 736 hrtime_t now, sleep_tsc = 0; 737 int sleep_sec; 738 extern void tsc_tick(void); 739 extern uint64_t cpu_freq_hz; 740 741 /* tsc_read() MUST be before TODOP_GET() */ 742 mutex_enter(&tod_lock); 743 now = tsc_read(); 744 ts = TODOP_GET(tod_ops); 745 mutex_exit(&tod_lock); 746 747 /* Compute seconds of sleep time */ 748 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 749 750 /* 751 * If the saved sec is less that or equal to 752 * the current ts, then there is likely a 753 * problem with the clock. Assume at least 754 * one second has passed, so that time goes forward. 755 */ 756 if (sleep_sec <= 0) { 757 sleep_sec = 1; 758 } 759 760 /* How many TSC's should have occured while sleeping */ 761 if (tsc_adjust_seconds) 762 sleep_tsc = sleep_sec * cpu_freq_hz; 763 764 /* 765 * We also want to subtract from the "sleep_tsc" 766 * the current value of tsc_read(), so that our 767 * adjustment accounts for the amount of time we 768 * have been resumed _or_ an adjustment based on 769 * the fact that we didn't actually power off the 770 * CPU (migration is another issue, but _should_ 771 * also comply with this calculation). If the CPU 772 * never powered off, then: 773 * 'now == sleep_tsc + saved_tsc' 774 * and the delta will effectively be "0". 775 */ 776 sleep_tsc -= now; 777 if (tsc_delta_onsuspend) { 778 tsc_adjust_delta(sleep_tsc); 779 } else { 780 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 781 } 782 tsc_saved_tsc = 0; 783 784 tsc_tick(); 785 } 786 tsc_needs_resume = 0; 787 } 788 789 } 790