1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 27 * Copyright (c) 2014, 2016 by Delphix. All rights reserved. 28 * Copyright 2018 Joyent, Inc. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/disp.h> 35 #include <sys/var.h> 36 #include <sys/cmn_err.h> 37 #include <sys/debug.h> 38 #include <sys/x86_archext.h> 39 #include <sys/archsystm.h> 40 #include <sys/cpuvar.h> 41 #include <sys/psm_defs.h> 42 #include <sys/clock.h> 43 #include <sys/atomic.h> 44 #include <sys/lockstat.h> 45 #include <sys/smp_impldefs.h> 46 #include <sys/dtrace.h> 47 #include <sys/time.h> 48 #include <sys/panic.h> 49 #include <sys/cpu.h> 50 #include <sys/sdt.h> 51 #include <sys/comm_page.h> 52 53 /* 54 * Using the Pentium's TSC register for gethrtime() 55 * ------------------------------------------------ 56 * 57 * The Pentium family, like many chip architectures, has a high-resolution 58 * timestamp counter ("TSC") which increments once per CPU cycle. The contents 59 * of the timestamp counter are read with the RDTSC instruction. 60 * 61 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 62 * must be translated into nanoseconds in order to implement gethrtime(). 63 * We avoid inducing floating point operations in this conversion by 64 * implementing the same nsec_scale algorithm as that found in the sun4u 65 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 66 * a detailed description of the algorithm; the comment is not reproduced 67 * here. This implementation differs only in its value for NSEC_SHIFT: 68 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 69 * 60 MHz Pentiums. 70 * 71 * While TSC and %tick are both cycle counting registers, TSC's functionality 72 * falls short in several critical ways: 73 * 74 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 75 * practice they often _are_ in sync, this isn't guaranteed by the 76 * architecture. 77 * 78 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 79 * only supports writing the low 32-bits of TSC, making it impractical 80 * to rewrite. 81 * 82 * (c) The architecture doesn't have the capacity to interrupt based on 83 * arbitrary values of TSC; there is no TICK_CMPR equivalent. 84 * 85 * Together, (a) and (b) imply that software must track the skew between 86 * TSCs and account for it (it is assumed that while there may exist skew, 87 * there does not exist drift). To determine the skew between CPUs, we 88 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 89 * the online operation calls tsc_sync_master(). 90 * 91 * In the absence of time-of-day clock adjustments, gethrtime() must stay in 92 * sync with gettimeofday(). This is problematic; given (c), the software 93 * cannot drive its time-of-day source from TSC, and yet they must somehow be 94 * kept in sync. We implement this by having a routine, tsc_tick(), which 95 * is called once per second from the interrupt which drives time-of-day. 96 * 97 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 98 * atomically with nsec_scale under CLOCK_LOCK. This assures that time 99 * monotonically increases. 100 */ 101 102 #define NSEC_SHIFT 5 103 104 static uint_t nsec_unscale; 105 106 /* 107 * These two variables used to be grouped together inside of a structure that 108 * lived on a single cache line. A regression (bug ID 4623398) caused the 109 * compiler to emit code that "optimized" away the while-loops below. The 110 * result was that no synchronization between the onlining and onlined CPUs 111 * took place. 112 */ 113 static volatile int tsc_ready; 114 static volatile int tsc_sync_go; 115 116 /* 117 * Used as indices into the tsc_sync_snaps[] array. 118 */ 119 #define TSC_MASTER 0 120 #define TSC_SLAVE 1 121 122 /* 123 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 124 */ 125 #define TSC_SYNC_STOP 1 126 #define TSC_SYNC_GO 2 127 #define TSC_SYNC_DONE 3 128 #define SYNC_ITERATIONS 10 129 130 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 131 unsigned int *_l = (unsigned int *)&(tsc); \ 132 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 133 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 134 } 135 136 #define TSC_CONVERT(tsc, hrt, scale) { \ 137 unsigned int *_l = (unsigned int *)&(tsc); \ 138 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 139 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 140 } 141 142 int tsc_master_slave_sync_needed = 1; 143 144 typedef struct tsc_sync { 145 volatile hrtime_t master_tsc, slave_tsc; 146 } tsc_sync_t; 147 static tsc_sync_t *tscp; 148 149 static hrtime_t tsc_last_jumped = 0; 150 static int tsc_jumped = 0; 151 static uint32_t tsc_wayback = 0; 152 /* 153 * The cap of 1 second was chosen since it is the frequency at which the 154 * tsc_tick() function runs which means that when gethrtime() is called it 155 * should never be more than 1 second since tsc_last was updated. 156 */ 157 static hrtime_t tsc_resume_cap_ns = NANOSEC; /* 1s */ 158 159 static hrtime_t shadow_tsc_hrtime_base; 160 static hrtime_t shadow_tsc_last; 161 static uint_t shadow_nsec_scale; 162 static uint32_t shadow_hres_lock; 163 int get_tsc_ready(); 164 165 static inline 166 hrtime_t tsc_protect(hrtime_t a) { 167 if (a > tsc_resume_cap) { 168 atomic_inc_32(&tsc_wayback); 169 DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last, 170 uint32_t, tsc_wayback); 171 return (tsc_resume_cap); 172 } 173 return (a); 174 } 175 176 hrtime_t 177 tsc_gethrtime(void) 178 { 179 uint32_t old_hres_lock; 180 hrtime_t tsc, hrt; 181 182 do { 183 old_hres_lock = hres_lock; 184 185 if ((tsc = tsc_read()) >= tsc_last) { 186 /* 187 * It would seem to be obvious that this is true 188 * (that is, the past is less than the present), 189 * but it isn't true in the presence of suspend/resume 190 * cycles. If we manage to call gethrtime() 191 * after a resume, but before the first call to 192 * tsc_tick(), we will see the jump. In this case, 193 * we will simply use the value in TSC as the delta. 194 */ 195 tsc -= tsc_last; 196 } else if (tsc >= tsc_last - 2*tsc_max_delta) { 197 /* 198 * There is a chance that tsc_tick() has just run on 199 * another CPU, and we have drifted just enough so that 200 * we appear behind tsc_last. In this case, force the 201 * delta to be zero. 202 */ 203 tsc = 0; 204 } else { 205 /* 206 * If we reach this else clause we assume that we have 207 * gone through a suspend/resume cycle and use the 208 * current tsc value as the delta. 209 * 210 * In rare cases we can reach this else clause due to 211 * a lack of monotonicity in the TSC value. In such 212 * cases using the current TSC value as the delta would 213 * cause us to return a value ~2x of what it should 214 * be. To protect against these cases we cap the 215 * suspend/resume delta at tsc_resume_cap. 216 */ 217 tsc = tsc_protect(tsc); 218 } 219 220 hrt = tsc_hrtime_base; 221 222 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 223 } while ((old_hres_lock & ~1) != hres_lock); 224 225 return (hrt); 226 } 227 228 hrtime_t 229 tsc_gethrtime_delta(void) 230 { 231 uint32_t old_hres_lock; 232 hrtime_t tsc, hrt; 233 ulong_t flags; 234 235 do { 236 old_hres_lock = hres_lock; 237 238 /* 239 * We need to disable interrupts here to assure that we 240 * don't migrate between the call to tsc_read() and 241 * adding the CPU's TSC tick delta. Note that disabling 242 * and reenabling preemption is forbidden here because 243 * we may be in the middle of a fast trap. In the amd64 244 * kernel we cannot tolerate preemption during a fast 245 * trap. See _update_sregs(). 246 */ 247 248 flags = clear_int_flag(); 249 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 250 restore_int_flag(flags); 251 252 /* See comments in tsc_gethrtime() above */ 253 254 if (tsc >= tsc_last) { 255 tsc -= tsc_last; 256 } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 257 tsc = 0; 258 } else { 259 tsc = tsc_protect(tsc); 260 } 261 262 hrt = tsc_hrtime_base; 263 264 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 265 } while ((old_hres_lock & ~1) != hres_lock); 266 267 return (hrt); 268 } 269 270 hrtime_t 271 tsc_gethrtime_tick_delta(void) 272 { 273 hrtime_t hrt; 274 ulong_t flags; 275 276 flags = clear_int_flag(); 277 hrt = tsc_sync_tick_delta[CPU->cpu_id]; 278 restore_int_flag(flags); 279 280 return (hrt); 281 } 282 283 /* Calculate the hrtime while exposing the parameters of that calculation. */ 284 hrtime_t 285 tsc_gethrtime_params(uint64_t *tscp, uint32_t *scalep, uint8_t *shiftp) 286 { 287 uint32_t old_hres_lock, scale; 288 hrtime_t tsc, last, base; 289 290 do { 291 old_hres_lock = hres_lock; 292 293 if (gethrtimef == tsc_gethrtime_delta) { 294 ulong_t flags; 295 296 flags = clear_int_flag(); 297 tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 298 restore_int_flag(flags); 299 } else { 300 tsc = tsc_read(); 301 } 302 303 last = tsc_last; 304 base = tsc_hrtime_base; 305 scale = nsec_scale; 306 307 } while ((old_hres_lock & ~1) != hres_lock); 308 309 /* See comments in tsc_gethrtime() above */ 310 if (tsc >= last) { 311 tsc -= last; 312 } else if (tsc >= last - 2 * tsc_max_delta) { 313 tsc = 0; 314 } else { 315 tsc = tsc_protect(tsc); 316 } 317 318 TSC_CONVERT_AND_ADD(tsc, base, nsec_scale); 319 320 if (tscp != NULL) { 321 /* 322 * Do not simply communicate the delta applied to the hrtime 323 * base, but rather the effective TSC measurement. 324 */ 325 *tscp = tsc + last; 326 } 327 if (scalep != NULL) { 328 *scalep = scale; 329 } 330 if (shiftp != NULL) { 331 *shiftp = NSEC_SHIFT; 332 } 333 334 return (base); 335 } 336 337 /* 338 * This is similar to tsc_gethrtime_delta, but it cannot actually spin on 339 * hres_lock. As a result, it caches all of the variables it needs; if the 340 * variables don't change, it's done. 341 */ 342 hrtime_t 343 dtrace_gethrtime(void) 344 { 345 uint32_t old_hres_lock; 346 hrtime_t tsc, hrt; 347 ulong_t flags; 348 349 do { 350 old_hres_lock = hres_lock; 351 352 /* 353 * Interrupts are disabled to ensure that the thread isn't 354 * migrated between the tsc_read() and adding the CPU's 355 * TSC tick delta. 356 */ 357 flags = clear_int_flag(); 358 359 tsc = tsc_read(); 360 361 if (gethrtimef == tsc_gethrtime_delta) 362 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 363 364 restore_int_flag(flags); 365 366 /* 367 * See the comments in tsc_gethrtime(), above. 368 */ 369 if (tsc >= tsc_last) 370 tsc -= tsc_last; 371 else if (tsc >= tsc_last - 2*tsc_max_delta) 372 tsc = 0; 373 else 374 tsc = tsc_protect(tsc); 375 376 hrt = tsc_hrtime_base; 377 378 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 379 380 if ((old_hres_lock & ~1) == hres_lock) 381 break; 382 383 /* 384 * If we're here, the clock lock is locked -- or it has been 385 * unlocked and locked since we looked. This may be due to 386 * tsc_tick() running on another CPU -- or it may be because 387 * some code path has ended up in dtrace_probe() with 388 * CLOCK_LOCK held. We'll try to determine that we're in 389 * the former case by taking another lap if the lock has 390 * changed since when we first looked at it. 391 */ 392 if (old_hres_lock != hres_lock) 393 continue; 394 395 /* 396 * So the lock was and is locked. We'll use the old data 397 * instead. 398 */ 399 old_hres_lock = shadow_hres_lock; 400 401 /* 402 * Again, disable interrupts to ensure that the thread 403 * isn't migrated between the tsc_read() and adding 404 * the CPU's TSC tick delta. 405 */ 406 flags = clear_int_flag(); 407 408 tsc = tsc_read(); 409 410 if (gethrtimef == tsc_gethrtime_delta) 411 tsc += tsc_sync_tick_delta[CPU->cpu_id]; 412 413 restore_int_flag(flags); 414 415 /* 416 * See the comments in tsc_gethrtime(), above. 417 */ 418 if (tsc >= shadow_tsc_last) 419 tsc -= shadow_tsc_last; 420 else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 421 tsc = 0; 422 else 423 tsc = tsc_protect(tsc); 424 425 hrt = shadow_tsc_hrtime_base; 426 427 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 428 } while ((old_hres_lock & ~1) != shadow_hres_lock); 429 430 return (hrt); 431 } 432 433 hrtime_t 434 tsc_gethrtimeunscaled(void) 435 { 436 uint32_t old_hres_lock; 437 hrtime_t tsc; 438 439 do { 440 old_hres_lock = hres_lock; 441 442 /* See tsc_tick(). */ 443 tsc = tsc_read() + tsc_last_jumped; 444 } while ((old_hres_lock & ~1) != hres_lock); 445 446 return (tsc); 447 } 448 449 /* 450 * Convert a nanosecond based timestamp to tsc 451 */ 452 uint64_t 453 tsc_unscalehrtime(hrtime_t nsec) 454 { 455 hrtime_t tsc; 456 457 if (tsc_gethrtime_enable) { 458 TSC_CONVERT(nsec, tsc, nsec_unscale); 459 return (tsc); 460 } 461 return ((uint64_t)nsec); 462 } 463 464 /* Convert a tsc timestamp to nanoseconds */ 465 void 466 tsc_scalehrtime(hrtime_t *tsc) 467 { 468 hrtime_t hrt; 469 hrtime_t mytsc; 470 471 if (tsc == NULL) 472 return; 473 mytsc = *tsc; 474 475 TSC_CONVERT(mytsc, hrt, nsec_scale); 476 *tsc = hrt; 477 } 478 479 hrtime_t 480 tsc_gethrtimeunscaled_delta(void) 481 { 482 hrtime_t hrt; 483 ulong_t flags; 484 485 /* 486 * Similarly to tsc_gethrtime_delta, we need to disable preemption 487 * to prevent migration between the call to tsc_gethrtimeunscaled 488 * and adding the CPU's hrtime delta. Note that disabling and 489 * reenabling preemption is forbidden here because we may be in the 490 * middle of a fast trap. In the amd64 kernel we cannot tolerate 491 * preemption during a fast trap. See _update_sregs(). 492 */ 493 494 flags = clear_int_flag(); 495 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 496 restore_int_flag(flags); 497 498 return (hrt); 499 } 500 501 /* 502 * TSC Sync Master 503 * 504 * Typically called on the boot CPU, this attempts to quantify TSC skew between 505 * different CPUs. If an appreciable difference is found, gethrtimef will be 506 * changed to point to tsc_gethrtime_delta(). 507 * 508 * Calculating skews is precise only when the master and slave TSCs are read 509 * simultaneously; however, there is no algorithm that can read both CPUs in 510 * perfect simultaneity. The proposed algorithm is an approximate method based 511 * on the behaviour of cache management. The slave CPU continuously polls the 512 * TSC while reading a global variable updated by the master CPU. The latest 513 * TSC reading is saved when the master's update (forced via mfence) reaches 514 * visibility on the slave. The master will also take a TSC reading 515 * immediately following the mfence. 516 * 517 * While the delay between cache line invalidation on the slave and mfence 518 * completion on the master is not repeatable, the error is heuristically 519 * assumed to be 1/4th of the write time recorded by the master. Multiple 520 * samples are taken to control for the variance caused by external factors 521 * such as bus contention. Each sample set is independent per-CPU to control 522 * for differing memory latency on NUMA systems. 523 * 524 * TSC sync is disabled in the context of virtualization because the CPUs 525 * assigned to the guest are virtual CPUs which means the real CPUs on which 526 * guest runs keep changing during life time of guest OS. So we would end up 527 * calculating TSC skews for a set of CPUs during boot whereas the guest 528 * might migrate to a different set of physical CPUs at a later point of 529 * time. 530 */ 531 void 532 tsc_sync_master(processorid_t slave) 533 { 534 ulong_t flags, source, min_write_time = ~0UL; 535 hrtime_t write_time, mtsc_after, last_delta = 0; 536 tsc_sync_t *tsc = tscp; 537 int cnt; 538 int hwtype; 539 540 hwtype = get_hwenv(); 541 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 542 return; 543 544 flags = clear_int_flag(); 545 source = CPU->cpu_id; 546 547 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 548 while (tsc_sync_go != TSC_SYNC_GO) 549 SMT_PAUSE(); 550 551 tsc->master_tsc = tsc_read(); 552 membar_enter(); 553 mtsc_after = tsc_read(); 554 while (tsc_sync_go != TSC_SYNC_DONE) 555 SMT_PAUSE(); 556 write_time = mtsc_after - tsc->master_tsc; 557 if (write_time <= min_write_time) { 558 hrtime_t tdelta; 559 560 tdelta = tsc->slave_tsc - mtsc_after; 561 if (tdelta < 0) 562 tdelta = -tdelta; 563 /* 564 * If the margin exists, subtract 1/4th of the measured 565 * write time from the master's TSC value. This is an 566 * estimate of how late the mfence completion came 567 * after the slave noticed the cache line change. 568 */ 569 if (tdelta > (write_time/4)) { 570 tdelta = tsc->slave_tsc - 571 (mtsc_after - (write_time/4)); 572 } else { 573 tdelta = tsc->slave_tsc - mtsc_after; 574 } 575 last_delta = tsc_sync_tick_delta[source] - tdelta; 576 tsc_sync_tick_delta[slave] = last_delta; 577 min_write_time = write_time; 578 } 579 580 tsc->master_tsc = tsc->slave_tsc = write_time = 0; 581 membar_enter(); 582 tsc_sync_go = TSC_SYNC_STOP; 583 } 584 585 /* 586 * Only enable the delta variants of the TSC functions if the measured 587 * skew is greater than the fastest write time. 588 */ 589 last_delta = (last_delta < 0) ? -last_delta : last_delta; 590 if (last_delta > min_write_time) { 591 gethrtimef = tsc_gethrtime_delta; 592 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 593 tsc_ncpu = NCPU; 594 } 595 restore_int_flag(flags); 596 } 597 598 /* 599 * TSC Sync Slave 600 * 601 * Called by a CPU which has just been onlined. It is expected that the CPU 602 * performing the online operation will call tsc_sync_master(). 603 * 604 * Like tsc_sync_master, this logic is skipped on virtualized platforms. 605 */ 606 void 607 tsc_sync_slave(void) 608 { 609 ulong_t flags; 610 hrtime_t s1; 611 tsc_sync_t *tsc = tscp; 612 int cnt; 613 int hwtype; 614 615 hwtype = get_hwenv(); 616 if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 617 return; 618 619 flags = clear_int_flag(); 620 621 for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 622 /* Re-fill the cache line */ 623 s1 = tsc->master_tsc; 624 membar_enter(); 625 tsc_sync_go = TSC_SYNC_GO; 626 do { 627 /* 628 * Do not put an SMT_PAUSE here. If the master and 629 * slave are the same hyper-threaded CPU, we want the 630 * master to yield as quickly as possible to the slave. 631 */ 632 s1 = tsc_read(); 633 } while (tsc->master_tsc == 0); 634 tsc->slave_tsc = s1; 635 membar_enter(); 636 tsc_sync_go = TSC_SYNC_DONE; 637 638 while (tsc_sync_go != TSC_SYNC_STOP) 639 SMT_PAUSE(); 640 } 641 642 restore_int_flag(flags); 643 } 644 645 /* 646 * Called once per second on a CPU from the cyclic subsystem's 647 * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 648 */ 649 void 650 tsc_tick(void) 651 { 652 hrtime_t now, delta; 653 ushort_t spl; 654 655 /* 656 * Before we set the new variables, we set the shadow values. This 657 * allows for lock free operation in dtrace_gethrtime(). 658 */ 659 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 660 ipltospl(CBE_HIGH_PIL), &spl); 661 662 shadow_tsc_hrtime_base = tsc_hrtime_base; 663 shadow_tsc_last = tsc_last; 664 shadow_nsec_scale = nsec_scale; 665 666 shadow_hres_lock++; 667 splx(spl); 668 669 CLOCK_LOCK(&spl); 670 671 now = tsc_read(); 672 673 if (gethrtimef == tsc_gethrtime_delta) 674 now += tsc_sync_tick_delta[CPU->cpu_id]; 675 676 if (now < tsc_last) { 677 /* 678 * The TSC has just jumped into the past. We assume that 679 * this is due to a suspend/resume cycle, and we're going 680 * to use the _current_ value of TSC as the delta. This 681 * will keep tsc_hrtime_base correct. We're also going to 682 * assume that rate of tsc does not change after a suspend 683 * resume (i.e nsec_scale remains the same). 684 */ 685 delta = now; 686 delta = tsc_protect(delta); 687 tsc_last_jumped += tsc_last; 688 tsc_jumped = 1; 689 } else { 690 /* 691 * Determine the number of TSC ticks since the last clock 692 * tick, and add that to the hrtime base. 693 */ 694 delta = now - tsc_last; 695 } 696 697 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 698 tsc_last = now; 699 700 CLOCK_UNLOCK(spl); 701 } 702 703 void 704 tsc_hrtimeinit(uint64_t cpu_freq_hz) 705 { 706 extern int gethrtime_hires; 707 longlong_t tsc; 708 ulong_t flags; 709 710 /* 711 * cpu_freq_hz is the measured cpu frequency in hertz 712 */ 713 714 /* 715 * We can't accommodate CPUs slower than 31.25 MHz. 716 */ 717 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 718 nsec_scale = 719 (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 720 nsec_unscale = 721 (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC); 722 723 flags = clear_int_flag(); 724 tsc = tsc_read(); 725 (void) tsc_gethrtime(); 726 tsc_max_delta = tsc_read() - tsc; 727 restore_int_flag(flags); 728 gethrtimef = tsc_gethrtime; 729 gethrtimeunscaledf = tsc_gethrtimeunscaled; 730 scalehrtimef = tsc_scalehrtime; 731 unscalehrtimef = tsc_unscalehrtime; 732 hrtime_tick = tsc_tick; 733 gethrtime_hires = 1; 734 /* 735 * Being part of the comm page, tsc_ncpu communicates the published 736 * length of the tsc_sync_tick_delta array. This is kept zeroed to 737 * ignore the absent delta data while the TSCs are synced. 738 */ 739 tsc_ncpu = 0; 740 /* 741 * Allocate memory for the structure used in the tsc sync logic. 742 * This structure should be aligned on a multiple of cache line size. 743 */ 744 tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); 745 746 /* 747 * Convert the TSC resume cap ns value into its unscaled TSC value. 748 * See tsc_gethrtime(). 749 */ 750 if (tsc_resume_cap == 0) 751 TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale); 752 } 753 754 int 755 get_tsc_ready() 756 { 757 return (tsc_ready); 758 } 759 760 /* 761 * Adjust all the deltas by adding the passed value to the array and activate 762 * the "delta" versions of the gethrtime functions. It is possible that the 763 * adjustment could be negative. Such may occur if the SunOS instance was 764 * moved by a virtual manager to a machine with a higher value of TSC. 765 */ 766 void 767 tsc_adjust_delta(hrtime_t tdelta) 768 { 769 int i; 770 771 for (i = 0; i < NCPU; i++) { 772 tsc_sync_tick_delta[i] += tdelta; 773 } 774 775 gethrtimef = tsc_gethrtime_delta; 776 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 777 tsc_ncpu = NCPU; 778 } 779 780 /* 781 * Functions to manage TSC and high-res time on suspend and resume. 782 */ 783 784 /* tod_ops from "uts/i86pc/io/todpc_subr.c" */ 785 extern tod_ops_t *tod_ops; 786 787 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 788 static timestruc_t tsc_saved_ts; 789 static int tsc_needs_resume = 0; /* We only want to do this once. */ 790 int tsc_delta_onsuspend = 0; 791 int tsc_adjust_seconds = 1; 792 int tsc_suspend_count = 0; 793 int tsc_resume_in_cyclic = 0; 794 795 /* 796 * Take snapshots of the current time and do any other pre-suspend work. 797 */ 798 void 799 tsc_suspend(void) 800 { 801 /* 802 * We need to collect the time at which we suspended here so we know 803 * now much should be added during the resume. This is called by each 804 * CPU, so reentry must be properly handled. 805 */ 806 if (tsc_gethrtime_enable) { 807 /* 808 * Perform the tsc_read after acquiring the lock to make it as 809 * accurate as possible in the face of contention. 810 */ 811 mutex_enter(&tod_lock); 812 tsc_saved_tsc = tsc_read(); 813 tsc_saved_ts = TODOP_GET(tod_ops); 814 mutex_exit(&tod_lock); 815 /* We only want to do this once. */ 816 if (tsc_needs_resume == 0) { 817 if (tsc_delta_onsuspend) { 818 tsc_adjust_delta(tsc_saved_tsc); 819 } else { 820 tsc_adjust_delta(nsec_scale); 821 } 822 tsc_suspend_count++; 823 } 824 } 825 826 invalidate_cache(); 827 tsc_needs_resume = 1; 828 } 829 830 /* 831 * Restore all timestamp state based on the snapshots taken at suspend time. 832 */ 833 void 834 tsc_resume(void) 835 { 836 /* 837 * We only need to (and want to) do this once. So let the first 838 * caller handle this (we are locked by the cpu lock), as it 839 * is preferential that we get the earliest sync. 840 */ 841 if (tsc_needs_resume) { 842 /* 843 * If using the TSC, adjust the delta based on how long 844 * we were sleeping (or away). We also adjust for 845 * migration and a grown TSC. 846 */ 847 if (tsc_saved_tsc != 0) { 848 timestruc_t ts; 849 hrtime_t now, sleep_tsc = 0; 850 int sleep_sec; 851 extern void tsc_tick(void); 852 extern uint64_t cpu_freq_hz; 853 854 /* tsc_read() MUST be before TODOP_GET() */ 855 mutex_enter(&tod_lock); 856 now = tsc_read(); 857 ts = TODOP_GET(tod_ops); 858 mutex_exit(&tod_lock); 859 860 /* Compute seconds of sleep time */ 861 sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 862 863 /* 864 * If the saved sec is less that or equal to 865 * the current ts, then there is likely a 866 * problem with the clock. Assume at least 867 * one second has passed, so that time goes forward. 868 */ 869 if (sleep_sec <= 0) { 870 sleep_sec = 1; 871 } 872 873 /* How many TSC's should have occured while sleeping */ 874 if (tsc_adjust_seconds) 875 sleep_tsc = sleep_sec * cpu_freq_hz; 876 877 /* 878 * We also want to subtract from the "sleep_tsc" 879 * the current value of tsc_read(), so that our 880 * adjustment accounts for the amount of time we 881 * have been resumed _or_ an adjustment based on 882 * the fact that we didn't actually power off the 883 * CPU (migration is another issue, but _should_ 884 * also comply with this calculation). If the CPU 885 * never powered off, then: 886 * 'now == sleep_tsc + saved_tsc' 887 * and the delta will effectively be "0". 888 */ 889 sleep_tsc -= now; 890 if (tsc_delta_onsuspend) { 891 tsc_adjust_delta(sleep_tsc); 892 } else { 893 tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 894 } 895 tsc_saved_tsc = 0; 896 897 tsc_tick(); 898 } 899 tsc_needs_resume = 0; 900 } 901 902 } 903