1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/disp.h> 31 #include <sys/var.h> 32 #include <sys/cmn_err.h> 33 #include <sys/debug.h> 34 #include <sys/x86_archext.h> 35 #include <sys/archsystm.h> 36 #include <sys/cpuvar.h> 37 #include <sys/psm_defs.h> 38 #include <sys/clock.h> 39 #include <sys/atomic.h> 40 #include <sys/lockstat.h> 41 #include <sys/smp_impldefs.h> 42 #include <sys/dtrace.h> 43 #include <sys/time.h> 44 #include <sys/panic.h> 45 #include <sys/cpu.h> 46 47 /* 48 * Using the Pentium's TSC register for gethrtime() 49 * ------------------------------------------------ 50 * 51 * The Pentium family, like many chip architectures, has a high-resolution 52 * timestamp counter ("TSC") which increments once per CPU cycle. The contents 53 * of the timestamp counter are read with the RDTSC instruction. 54 * 55 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 56 * must be translated into nanoseconds in order to implement gethrtime(). 57 * We avoid inducing floating point operations in this conversion by 58 * implementing the same nsec_scale algorithm as that found in the sun4u 59 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 60 * a detailed description of the algorithm; the comment is not reproduced 61 * here. This implementation differs only in its value for NSEC_SHIFT: 62 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 63 * 60 MHz Pentiums. 64 * 65 * While TSC and %tick are both cycle counting registers, TSC's functionality 66 * falls short in several critical ways: 67 * 68 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 69 * practice they often _are_ in sync, this isn't guaranteed by the 70 * architecture. 71 * 72 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 73 * only supports writing the low 32-bits of TSC, making it impractical 74 * to rewrite. 75 * 76 * (c) The architecture doesn't have the capacity to interrupt based on 77 * arbitrary values of TSC; there is no TICK_CMPR equivalent. 78 * 79 * Together, (a) and (b) imply that software must track the skew between 80 * TSCs and account for it (it is assumed that while there may exist skew, 81 * there does not exist drift). To determine the skew between CPUs, we 82 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 83 * the online operation calls tsc_sync_master(). 84 * 85 * In the absence of time-of-day clock adjustments, gethrtime() must stay in 86 * sync with gettimeofday(). This is problematic; given (c), the software 87 * cannot drive its time-of-day source from TSC, and yet they must somehow be 88 * kept in sync. We implement this by having a routine, tsc_tick(), which 89 * is called once per second from the interrupt which drives time-of-day. 90 * 91 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 92 * atomically with nsec_scale under CLOCK_LOCK. This assures that time 93 * monotonically increases. 94 */ 95 96 #define NSEC_SHIFT 5 97 98 static uint_t nsec_scale; 99 100 /* 101 * These two variables used to be grouped together inside of a structure that 102 * lived on a single cache line. A regression (bug ID 4623398) caused the 103 * compiler to emit code that "optimized" away the while-loops below. The 104 * result was that no synchronization between the onlining and onlined CPUs 105 * took place. 106 */ 107 static volatile int tsc_ready; 108 static volatile int tsc_sync_go; 109 110 /* 111 * Used as indices into the tsc_sync_snaps[] array. 112 */ 113 #define TSC_MASTER 0 114 #define TSC_SLAVE 1 115 116 /* 117 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 118 */ 119 #define TSC_SYNC_STOP 1 120 #define TSC_SYNC_GO 2 121 #define TSC_SYNC_DONE 3 122 #define SYNC_ITERATIONS 10 123 124 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 125 unsigned int *_l = (unsigned int *)&(tsc); \ 126 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 127 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 128 } 129 130 #define TSC_CONVERT(tsc, hrt, scale) { \ 131 unsigned int *_l = (unsigned int *)&(tsc); \ 132 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 133 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 134 } 135 136 int tsc_master_slave_sync_needed = 1; 137 138 static int tsc_max_delta; 139 static hrtime_t tsc_sync_tick_delta[NCPU]; 140 typedef struct tsc_sync { 141 volatile hrtime_t master_tsc, slave_tsc; 142 } tsc_sync_t; 143 static tsc_sync_t *tscp; 144 static hrtime_t largest_tsc_delta = 0; 145 static ulong_t shortest_write_time = ~0UL; 146 147 static hrtime_t tsc_last = 0; 148 static hrtime_t tsc_last_jumped = 0; 149 static hrtime_t tsc_hrtime_base = 0; 150 static int tsc_jumped = 0; 151 152 static hrtime_t shadow_tsc_hrtime_base; 153 static hrtime_t shadow_tsc_last; 154 static uint_t shadow_nsec_scale; 155 static uint32_t shadow_hres_lock; 156 int get_tsc_ready(); 157 158 hrtime_t 159 tsc_gethrtime(void) 160 { 161 uint32_t old_hres_lock; 162 hrtime_t tsc, hrt; 163 164 do { 165 old_hres_lock = hres_lock; 166 167 if ((tsc = tsc_read()) >= tsc_last) { 168 /* 169 * It would seem to be obvious that this is true 170 * (that is, the past is less than the present), 171 * but it isn't true in the presence of suspend/resume 172 * cycles. If we manage to call gethrtime() 173 * after a resume, but before the first call to 174 * tsc_tick(), we will see the jump. In this case, 175 * we will simply use the value in TSC as the delta. 176 */ 177 tsc -= tsc_last; 178 } else if (tsc >= tsc_last - 2*tsc_max_delta) { 179 /* 180 * There is a chance that tsc_tick() has just run on 181 * another CPU, and we have drifted just enough so that 182 * we appear behind tsc_last. In this case, force the 183 * delta to be zero. 184 */ 185 tsc = 0; 186 } 187 188 hrt = tsc_hrtime_base; 189 190 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 191 } while ((old_hres_lock & ~1) != hres_lock); 192 193 return (hrt); 194 } 195 196 hrtime_t 197 tsc_gethrtime_delta(void) 198 { 199 uint32_t old_hres_lock; 200 hrtime_t tsc, hrt; 201 ulong_t flags; 202 203 do { 204 old_hres_lock = hres_lock; 205 206 /* 207 * We need to disable interrupts here to assure that we 208 * don't migrate between the call to tsc_read() and 209 * adding the CPU's TSC tick delta. Note that disabling 210 * and reenabling preemption is forbidden here because 211 * we may be in the middle of a fast trap. In the amd64 212 * kernel we cannot tolerate preemption during a fast 213 * trap. See _update_sregs(). 214 */ 215 216 flags = clear_int_flag(); 217 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 218 restore_int_flag(flags); 219 220 /* See comments in tsc_gethrtime() above */ 221 222 if (tsc >= tsc_last) { 223 tsc -= tsc_last; 224 } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 225 tsc = 0; 226 } 227 228 hrt = tsc_hrtime_base; 229 230 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 231 } while ((old_hres_lock & ~1) != hres_lock); 232 233 return (hrt); 234 } 235 236 /* 237 * This is similar to the above, but it cannot actually spin on hres_lock. 238 * As a result, it caches all of the variables it needs; if the variables 239 * don't change, it's done. 240 */ 241 hrtime_t 242 dtrace_gethrtime(void) 243 { 244 uint32_t old_hres_lock; 245 hrtime_t tsc, hrt; 246 ulong_t flags; 247 248 do { 249 old_hres_lock = hres_lock; 250 251 /* 252 * Interrupts are disabled to ensure that the thread isn't 253 * migrated between the tsc_read() and adding the CPU's 254 * TSC tick delta. 255 */ 256 flags = clear_int_flag(); 257 258 tsc = tsc_read(); 259 260 if (gethrtimef == tsc_gethrtime_delta) 261 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 262 263 restore_int_flag(flags); 264 265 /* 266 * See the comments in tsc_gethrtime(), above. 267 */ 268 if (tsc >= tsc_last) 269 tsc -= tsc_last; 270 else if (tsc >= tsc_last - 2*tsc_max_delta) 271 tsc = 0; 272 273 hrt = tsc_hrtime_base; 274 275 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 276 277 if ((old_hres_lock & ~1) == hres_lock) 278 break; 279 280 /* 281 * If we're here, the clock lock is locked -- or it has been 282 * unlocked and locked since we looked. This may be due to 283 * tsc_tick() running on another CPU -- or it may be because 284 * some code path has ended up in dtrace_probe() with 285 * CLOCK_LOCK held. We'll try to determine that we're in 286 * the former case by taking another lap if the lock has 287 * changed since when we first looked at it. 288 */ 289 if (old_hres_lock != hres_lock) 290 continue; 291 292 /* 293 * So the lock was and is locked. We'll use the old data 294 * instead. 295 */ 296 old_hres_lock = shadow_hres_lock; 297 298 /* 299 * Again, disable interrupts to ensure that the thread 300 * isn't migrated between the tsc_read() and adding 301 * the CPU's TSC tick delta. 302 */ 303 flags = clear_int_flag(); 304 305 tsc = tsc_read(); 306 307 if (gethrtimef == tsc_gethrtime_delta) 308 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 309 310 restore_int_flag(flags); 311 312 /* 313 * See the comments in tsc_gethrtime(), above. 314 */ 315 if (tsc >= shadow_tsc_last) 316 tsc -= shadow_tsc_last; 317 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 318 tsc = 0; 319 320 hrt = shadow_tsc_hrtime_base; 321 322 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 323 } while ((old_hres_lock & ~1) != shadow_hres_lock); 324 325 return (hrt); 326 } 327 328 hrtime_t 329 tsc_gethrtimeunscaled(void) 330 { 331 uint32_t old_hres_lock; 332 hrtime_t tsc; 333 334 do { 335 old_hres_lock = hres_lock; 336 337 /* See tsc_tick(). */ 338 tsc = tsc_read() + tsc_last_jumped; 339 } while ((old_hres_lock & ~1) != hres_lock); 340 341 return (tsc); 342 } 343 344 345 /* Convert a tsc timestamp to nanoseconds */ 346 void 347 tsc_scalehrtime(hrtime_t *tsc) 348 { 349 hrtime_t hrt; 350 hrtime_t mytsc; 351 352 if (tsc == NULL) 353 return; 354 mytsc = *tsc; 355 356 TSC_CONVERT(mytsc, hrt, nsec_scale); 357 *tsc = hrt; 358 } 359 360 hrtime_t 361 tsc_gethrtimeunscaled_delta(void) 362 { 363 hrtime_t hrt; 364 ulong_t flags; 365 366 /* 367 * Similarly to tsc_gethrtime_delta, we need to disable preemption 368 * to prevent migration between the call to tsc_gethrtimeunscaled 369 * and adding the CPU's hrtime delta. Note that disabling and 370 * reenabling preemption is forbidden here because we may be in the 371 * middle of a fast trap. In the amd64 kernel we cannot tolerate 372 * preemption during a fast trap. See _update_sregs(). 373 */ 374 375 flags = clear_int_flag(); 376 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 377 restore_int_flag(flags); 378 379 return (hrt); 380 } 381 382 /* 383 * Called by the master in the TSC sync operation (usually the boot CPU). 384 * If the slave is discovered to have a skew, gethrtimef will be changed to 385 * point to tsc_gethrtime_delta(). Calculating skews is precise only when 386 * the master and slave TSCs are read simultaneously; however, there is no 387 * algorithm that can read both CPUs in perfect simultaneity. The proposed 388 * algorithm is an approximate method based on the behaviour of cache 389 * management. The slave CPU continuously reads TSC and then reads a global 390 * variable which the master CPU updates. The moment the master's update reaches 391 * the slave's visibility (being forced by an mfence operation) we use the TSC 392 * reading taken on the slave. A corresponding TSC read will be taken on the 393 * master as soon as possible after finishing the mfence operation. But the 394 * delay between causing the slave to notice the invalid cache line and the 395 * competion of mfence is not repeatable. This error is heuristically assumed 396 * to be 1/4th of the total write time as being measured by the two TSC reads 397 * on the master sandwiching the mfence. Furthermore, due to the nature of 398 * bus arbitration, contention on memory bus, etc., the time taken for the write 399 * to reflect globally can vary a lot. So instead of taking a single reading, 400 * a set of readings are taken and the one with least write time is chosen 401 * to calculate the final skew. 402 * 403 * TSC sync is disabled in the context of virtualization because the CPUs 404 * assigned to the guest are virtual CPUs which means the real CPUs on which 405 * guest runs keep changing during life time of guest OS. So we would end up 406 * calculating TSC skews for a set of CPUs during boot whereas the guest 407 * might migrate to a different set of physical CPUs at a later point of 408 * time. 409 */ 410 void 411 tsc_sync_master(processorid_t slave) 412 { 413 ulong_t flags, source, min_write_time = ~0UL; 414 hrtime_t write_time, x, mtsc_after, tdelta; 415 tsc_sync_t *tsc = tscp; 416 int cnt; 417 int hwtype; 418 419 hwtype = get_hwenv(); 420 if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM || 421 hwtype == HW_VMWARE) 422 return; 423 424 flags = clear_int_flag(); 425 source = CPU->cpu_id; 426 427 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 428 while (tsc_sync_go != TSC_SYNC_GO) 429 SMT_PAUSE(); 430 431 tsc->master_tsc = tsc_read(); 432 membar_enter(); 433 mtsc_after = tsc_read(); 434 while (tsc_sync_go != TSC_SYNC_DONE) 435 SMT_PAUSE(); 436 write_time = mtsc_after - tsc->master_tsc; 437 if (write_time <= min_write_time) { 438 min_write_time = write_time; 439 /* 440 * Apply heuristic adjustment only if the calculated 441 * delta is > 1/4th of the write time. 442 */ 443 x = tsc->slave_tsc - mtsc_after; 444 if (x < 0) 445 x = -x; 446 if (x > (min_write_time/4)) 447 /* 448 * Subtract 1/4th of the measured write time 449 * from the master's TSC value, as an estimate 450 * of how late the mfence completion came 451 * after the slave noticed the cache line 452 * change. 453 */ 454 tdelta = tsc->slave_tsc - 455 (mtsc_after - (min_write_time/4)); 456 else 457 tdelta = tsc->slave_tsc - mtsc_after; 458 tsc_sync_tick_delta[slave] = 459 tsc_sync_tick_delta[source] - tdelta; 460 } 461 462 tsc->master_tsc = tsc->slave_tsc = write_time = 0; 463 membar_enter(); 464 tsc_sync_go = TSC_SYNC_STOP; 465 } 466 if (tdelta < 0) 467 tdelta = -tdelta; 468 if (tdelta > largest_tsc_delta) 469 largest_tsc_delta = tdelta; 470 if (min_write_time < shortest_write_time) 471 shortest_write_time = min_write_time; 472 /* 473 * Enable delta variants of tsc functions if the largest of all chosen 474 * deltas is > smallest of the write time. 475 */ 476 if (largest_tsc_delta > shortest_write_time) { 477 gethrtimef = tsc_gethrtime_delta; 478 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 479 } 480 restore_int_flag(flags); 481 } 482 483 /* 484 * Called by a CPU which has just been onlined. It is expected that the CPU 485 * performing the online operation will call tsc_sync_master(). 486 * 487 * TSC sync is disabled in the context of virtualization. See comments 488 * above tsc_sync_master. 489 */ 490 void 491 tsc_sync_slave(void) 492 { 493 ulong_t flags; 494 hrtime_t s1; 495 tsc_sync_t *tsc = tscp; 496 int cnt; 497 int hwtype; 498 499 hwtype = get_hwenv(); 500 if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM || 501 hwtype == HW_VMWARE) 502 return; 503 504 flags = clear_int_flag(); 505 506 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 507 /* Re-fill the cache line */ 508 s1 = tsc->master_tsc; 509 membar_enter(); 510 tsc_sync_go = TSC_SYNC_GO; 511 do { 512 /* 513 * Do not put an SMT_PAUSE here. For instance, 514 * if the master and slave are really the same 515 * hyper-threaded CPU, then you want the master 516 * to yield to the slave as quickly as possible here, 517 * but not the other way. 518 */ 519 s1 = tsc_read(); 520 } while (tsc->master_tsc == 0); 521 tsc->slave_tsc = s1; 522 membar_enter(); 523 tsc_sync_go = TSC_SYNC_DONE; 524 525 while (tsc_sync_go != TSC_SYNC_STOP) 526 SMT_PAUSE(); 527 } 528 529 restore_int_flag(flags); 530 } 531 532 /* 533 * Called once per second on a CPU from the cyclic subsystem's 534 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 535 */ 536 void 537 tsc_tick(void) 538 { 539 hrtime_t now, delta; 540 ushort_t spl; 541 542 /* 543 * Before we set the new variables, we set the shadow values. This 544 * allows for lock free operation in dtrace_gethrtime(). 545 */ 546 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 547 ipltospl(CBE_HIGH_PIL), &spl); 548 549 shadow_tsc_hrtime_base = tsc_hrtime_base; 550 shadow_tsc_last = tsc_last; 551 shadow_nsec_scale = nsec_scale; 552 553 shadow_hres_lock++; 554 splx(spl); 555 556 CLOCK_LOCK(&spl); 557 558 now = tsc_read(); 559 560 if (gethrtimef == tsc_gethrtime_delta) 561 now += tsc_sync_tick_delta[CPU->cpu_id]; 562 563 if (now < tsc_last) { 564 /* 565 * The TSC has just jumped into the past. We assume that 566 * this is due to a suspend/resume cycle, and we're going 567 * to use the _current_ value of TSC as the delta. This 568 * will keep tsc_hrtime_base correct. We're also going to 569 * assume that rate of tsc does not change after a suspend 570 * resume (i.e nsec_scale remains the same). 571 */ 572 delta = now; 573 tsc_last_jumped += tsc_last; 574 tsc_jumped = 1; 575 } else { 576 /* 577 * Determine the number of TSC ticks since the last clock 578 * tick, and add that to the hrtime base. 579 */ 580 delta = now - tsc_last; 581 } 582 583 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 584 tsc_last = now; 585 586 CLOCK_UNLOCK(spl); 587 } 588 589 void 590 tsc_hrtimeinit(uint64_t cpu_freq_hz) 591 { 592 extern int gethrtime_hires; 593 longlong_t tsc; 594 ulong_t flags; 595 596 /* 597 * cpu_freq_hz is the measured cpu frequency in hertz 598 */ 599 600 /* 601 * We can't accommodate CPUs slower than 31.25 MHz. 602 */ 603 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 604 nsec_scale = 605 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 606 607 flags = clear_int_flag(); 608 tsc = tsc_read(); 609 (void) tsc_gethrtime(); 610 tsc_max_delta = tsc_read() - tsc; 611 restore_int_flag(flags); 612 gethrtimef = tsc_gethrtime; 613 gethrtimeunscaledf = tsc_gethrtimeunscaled; 614 scalehrtimef = tsc_scalehrtime; 615 hrtime_tick = tsc_tick; 616 gethrtime_hires = 1; 617 /* 618 * Allocate memory for the structure used in the tsc sync logic. 619 * This structure should be aligned on a multiple of cache line size. 620 */ 621 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); 622 } 623 624 int 625 get_tsc_ready() 626 { 627 return (tsc_ready); 628 } 629 630 /* 631 * Adjust all the deltas by adding the passed value to the array. 632 * Then use the "delt" versions of the the gethrtime functions. 633 * Note that 'tdelta' _could_ be a negative number, which should 634 * reduce the values in the array (used, for example, if the Solaris 635 * instance was moved by a virtual manager to a machine with a higher 636 * value of tsc). 637 */ 638 void 639 tsc_adjust_delta(hrtime_t tdelta) 640 { 641 int i; 642 643 for (i = 0; i < NCPU; i++) { 644 tsc_sync_tick_delta[i] += tdelta; 645 } 646 647 gethrtimef = tsc_gethrtime_delta; 648 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 649 } 650 651 /* 652 * Functions to manage TSC and high-res time on suspend and resume. 653 */ 654 655 /* 656 * declarations needed for time adjustment 657 */ 658 extern void rtcsync(void); 659 extern tod_ops_t *tod_ops; 660 /* There must be a better way than exposing nsec_scale! */ 661 extern uint_t nsec_scale; 662 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 663 static timestruc_t tsc_saved_ts; 664 static int tsc_needs_resume = 0; /* We only want to do this once. */ 665 int tsc_delta_onsuspend = 0; 666 int tsc_adjust_seconds = 1; 667 int tsc_suspend_count = 0; 668 int tsc_resume_in_cyclic = 0; 669 670 /* 671 * Let timestamp.c know that we are suspending. It needs to take 672 * snapshots of the current time, and do any pre-suspend work. 673 */ 674 void 675 tsc_suspend(void) 676 { 677 /* 678 * What we need to do here, is to get the time we suspended, so that we 679 * know how much we should add to the resume. 680 * This routine is called by each CPU, so we need to handle reentry. 681 */ 682 if (tsc_gethrtime_enable) { 683 /* 684 * We put the tsc_read() inside the lock as it 685 * as no locking constraints, and it puts the 686 * aquired value closer to the time stamp (in 687 * case we delay getting the lock). 688 */ 689 mutex_enter(&tod_lock); 690 tsc_saved_tsc = tsc_read(); 691 tsc_saved_ts = TODOP_GET(tod_ops); 692 mutex_exit(&tod_lock); 693 /* We only want to do this once. */ 694 if (tsc_needs_resume == 0) { 695 if (tsc_delta_onsuspend) { 696 tsc_adjust_delta(tsc_saved_tsc); 697 } else { 698 tsc_adjust_delta(nsec_scale); 699 } 700 tsc_suspend_count++; 701 } 702 } 703 704 invalidate_cache(); 705 tsc_needs_resume = 1; 706 } 707 708 /* 709 * Restore all timestamp state based on the snapshots taken at 710 * suspend time. 711 */ 712 void 713 tsc_resume(void) 714 { 715 /* 716 * We only need to (and want to) do this once. So let the first 717 * caller handle this (we are locked by the cpu lock), as it 718 * is preferential that we get the earliest sync. 719 */ 720 if (tsc_needs_resume) { 721 /* 722 * If using the TSC, adjust the delta based on how long 723 * we were sleeping (or away). We also adjust for 724 * migration and a grown TSC. 725 */ 726 if (tsc_saved_tsc != 0) { 727 timestruc_t ts; 728 hrtime_t now, sleep_tsc = 0; 729 int sleep_sec; 730 extern void tsc_tick(void); 731 extern uint64_t cpu_freq_hz; 732 733 /* tsc_read() MUST be before TODOP_GET() */ 734 mutex_enter(&tod_lock); 735 now = tsc_read(); 736 ts = TODOP_GET(tod_ops); 737 mutex_exit(&tod_lock); 738 739 /* Compute seconds of sleep time */ 740 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 741 742 /* 743 * If the saved sec is less that or equal to 744 * the current ts, then there is likely a 745 * problem with the clock. Assume at least 746 * one second has passed, so that time goes forward. 747 */ 748 if (sleep_sec <= 0) { 749 sleep_sec = 1; 750 } 751 752 /* How many TSC's should have occured while sleeping */ 753 if (tsc_adjust_seconds) 754 sleep_tsc = sleep_sec * cpu_freq_hz; 755 756 /* 757 * We also want to subtract from the "sleep_tsc" 758 * the current value of tsc_read(), so that our 759 * adjustment accounts for the amount of time we 760 * have been resumed _or_ an adjustment based on 761 * the fact that we didn't actually power off the 762 * CPU (migration is another issue, but _should_ 763 * also comply with this calculation). If the CPU 764 * never powered off, then: 765 * 'now == sleep_tsc + saved_tsc' 766 * and the delta will effectively be "0". 767 */ 768 sleep_tsc -= now; 769 if (tsc_delta_onsuspend) { 770 tsc_adjust_delta(sleep_tsc); 771 } else { 772 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 773 } 774 tsc_saved_tsc = 0; 775 776 tsc_tick(); 777 } 778 tsc_needs_resume = 0; 779 } 780 781 } 782