1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/disp.h> 33 #include <sys/var.h> 34 #include <sys/cmn_err.h> 35 #include <sys/debug.h> 36 #include <sys/x86_archext.h> 37 #include <sys/archsystm.h> 38 #include <sys/cpuvar.h> 39 #include <sys/psm_defs.h> 40 #include <sys/clock.h> 41 #include <sys/atomic.h> 42 #include <sys/lockstat.h> 43 #include <sys/smp_impldefs.h> 44 #include <sys/dtrace.h> 45 #include <sys/time.h> 46 47 /* 48 * Using the Pentium's TSC register for gethrtime() 49 * ------------------------------------------------ 50 * 51 * The Pentium family, like many chip architectures, has a high-resolution 52 * timestamp counter ("TSC") which increments once per CPU cycle. The contents 53 * of the timestamp counter are read with the RDTSC instruction. 54 * 55 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 56 * must be translated into nanoseconds in order to implement gethrtime(). 57 * We avoid inducing floating point operations in this conversion by 58 * implementing the same nsec_scale algorithm as that found in the sun4u 59 * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 60 * a detailed description of the algorithm; the comment is not reproduced 61 * here. This implementation differs only in its value for NSEC_SHIFT: 62 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 63 * 60 MHz Pentiums. 64 * 65 * While TSC and %tick are both cycle counting registers, TSC's functionality 66 * falls short in several critical ways: 67 * 68 * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 69 * practice they often _are_ in sync, this isn't guaranteed by the 70 * architecture. 71 * 72 * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 73 * only supports writing the low 32-bits of TSC, making it impractical 74 * to rewrite. 75 * 76 * (c) The architecture doesn't have the capacity to interrupt based on 77 * arbitrary values of TSC; there is no TICK_CMPR equivalent. 78 * 79 * Together, (a) and (b) imply that software must track the skew between 80 * TSCs and account for it (it is assumed that while there may exist skew, 81 * there does not exist drift). To determine the skew between CPUs, we 82 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 83 * the online operation calls tsc_sync_master(). Once both CPUs are ready, 84 * the master sets a shared flag, and each reads its TSC register. To reduce 85 * bias, we then wait until both CPUs are ready again, but this time the 86 * slave sets the shared flag, and each reads its TSC register again. The 87 * master compares the average of the two sample values, and, if observable 88 * skew is found, changes the gethrtimef function pointer to point to a 89 * gethrtime() implementation which will take the discovered skew into 90 * consideration. 91 * 92 * In the absence of time-of-day clock adjustments, gethrtime() must stay in 93 * sync with gettimeofday(). This is problematic; given (c), the software 94 * cannot drive its time-of-day source from TSC, and yet they must somehow be 95 * kept in sync. We implement this by having a routine, tsc_tick(), which 96 * is called once per second from the interrupt which drives time-of-day. 97 * tsc_tick() recalculates nsec_scale based on the number of the CPU cycles 98 * since boot versus the number of seconds since boot. This algorithm 99 * becomes more accurate over time and converges quickly; the error in 100 * nsec_scale is typically under 1 ppm less than 10 seconds after boot, and 101 * is less than 100 ppb 1 minute after boot. 102 * 103 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 104 * atomically with nsec_scale under CLOCK_LOCK. This assures that time 105 * monotonically increases. 106 */ 107 108 #define NSEC_SHIFT 5 109 110 static uint_t nsec_scale; 111 112 /* 113 * These two variables used to be grouped together inside of a structure that 114 * lived on a single cache line. A regression (bug ID 4623398) caused the 115 * compiler to emit code that "optimized" away the while-loops below. The 116 * result was that no synchronization between the onlining and onlined CPUs 117 * took place. 118 */ 119 static volatile int tsc_ready; 120 static volatile int tsc_sync_go; 121 122 /* 123 * Used as indices into the tsc_sync_snaps[] array. 124 */ 125 #define TSC_MASTER 0 126 #define TSC_SLAVE 1 127 128 /* 129 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 130 */ 131 #define TSC_SYNC_STOP 1 132 #define TSC_SYNC_GO 2 133 #define TSC_SYNC_AGAIN 3 134 135 /* 136 * XX64 Is the faster way to do this with a 64-bit ABI? 137 */ 138 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 139 unsigned int *_l = (unsigned int *)&(tsc); \ 140 (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 141 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 142 } 143 144 #define TSC_CONVERT(tsc, hrt, scale) { \ 145 unsigned int *_l = (unsigned int *)&(tsc); \ 146 (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 147 (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 148 } 149 150 151 152 static int tsc_max_delta; 153 static hrtime_t tsc_sync_snaps[2]; 154 static hrtime_t tsc_sync_delta[NCPU]; 155 static hrtime_t tsc_sync_tick_delta[NCPU]; 156 static hrtime_t tsc_last = 0; 157 static hrtime_t tsc_last_jumped = 0; 158 static hrtime_t tsc_hrtime_base = 0; 159 static int tsc_jumped = 0; 160 161 static hrtime_t shadow_tsc_hrtime_base; 162 static hrtime_t shadow_tsc_last; 163 static uint_t shadow_nsec_scale; 164 static uint32_t shadow_hres_lock; 165 166 /* 167 * Called by the master after the sync operation is complete. If the 168 * slave is discovered to lag, gethrtimef will be changed to point to 169 * tsc_gethrtime_delta(). 170 */ 171 static void 172 tsc_digest(processorid_t target) 173 { 174 hrtime_t tdelta, hdelta = 0; 175 int max = tsc_max_delta; 176 processorid_t source = CPU->cpu_id; 177 int update; 178 179 update = tsc_sync_delta[source] != 0 || 180 gethrtimef == tsc_gethrtime_delta; 181 182 /* 183 * We divide by 2 since each of the data points is the sum of two TSC 184 * reads; this takes the average of the two. 185 */ 186 tdelta = (tsc_sync_snaps[TSC_SLAVE] - tsc_sync_snaps[TSC_MASTER]) / 2; 187 if ((tdelta > max) || ((tdelta >= 0) && update)) { 188 TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 189 tsc_sync_delta[target] = tsc_sync_delta[source] - hdelta; 190 tsc_sync_tick_delta[target] = -tdelta; 191 gethrtimef = tsc_gethrtime_delta; 192 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 193 return; 194 } 195 196 tdelta = -tdelta; 197 if ((tdelta > max) || update) { 198 TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 199 tsc_sync_delta[target] = tsc_sync_delta[source] + hdelta; 200 tsc_sync_tick_delta[target] = tdelta; 201 gethrtimef = tsc_gethrtime_delta; 202 gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 203 } 204 205 } 206 207 /* 208 * Called by a CPU which has just performed an online operation on another 209 * CPU. It is expected that the newly onlined CPU will call tsc_sync_slave(). 210 */ 211 void 212 tsc_sync_master(processorid_t slave) 213 { 214 int flags; 215 hrtime_t hrt; 216 217 ASSERT(tsc_sync_go != TSC_SYNC_GO); 218 219 flags = clear_int_flag(); 220 221 /* 222 * Wait for the slave CPU to arrive. 223 */ 224 while (tsc_ready != TSC_SYNC_GO) 225 continue; 226 227 /* 228 * Tell the slave CPU to begin reading its TSC; read our own. 229 */ 230 tsc_sync_go = TSC_SYNC_GO; 231 hrt = tsc_read(); 232 233 /* 234 * Tell the slave that we're ready, and wait for the slave to tell us 235 * to read our TSC again. 236 */ 237 tsc_ready = TSC_SYNC_AGAIN; 238 while (tsc_sync_go != TSC_SYNC_AGAIN) 239 continue; 240 241 hrt += tsc_read(); 242 tsc_sync_snaps[TSC_MASTER] = hrt; 243 244 /* 245 * Wait for the slave to finish reading its TSC. 246 */ 247 while (tsc_ready != TSC_SYNC_STOP) 248 continue; 249 250 /* 251 * At this point, both CPUs have performed their tsc_read() calls. 252 * We'll digest it now before letting the slave CPU return. 253 */ 254 tsc_digest(slave); 255 tsc_sync_go = TSC_SYNC_STOP; 256 257 restore_int_flag(flags); 258 } 259 260 /* 261 * Called by a CPU which has just been onlined. It is expected that the CPU 262 * performing the online operation will call tsc_sync_master(). 263 */ 264 void 265 tsc_sync_slave(void) 266 { 267 int flags; 268 hrtime_t hrt; 269 270 ASSERT(tsc_sync_go != TSC_SYNC_GO); 271 272 flags = clear_int_flag(); 273 274 /* 275 * Tell the master CPU that we're ready, and wait for the master to 276 * tell us to begin reading our TSC. 277 */ 278 tsc_ready = TSC_SYNC_GO; 279 while (tsc_sync_go != TSC_SYNC_GO) 280 continue; 281 282 hrt = tsc_read(); 283 284 /* 285 * Wait for the master CPU to be ready to read its TSC again. 286 */ 287 while (tsc_ready != TSC_SYNC_AGAIN) 288 continue; 289 290 /* 291 * Tell the master CPU to read its TSC again; read ours again. 292 */ 293 tsc_sync_go = TSC_SYNC_AGAIN; 294 295 hrt += tsc_read(); 296 tsc_sync_snaps[TSC_SLAVE] = hrt; 297 298 /* 299 * Tell the master that we're done, and wait to be dismissed. 300 */ 301 tsc_ready = TSC_SYNC_STOP; 302 while (tsc_sync_go != TSC_SYNC_STOP) 303 continue; 304 305 restore_int_flag(flags); 306 } 307 308 void 309 tsc_hrtimeinit(uint64_t cpu_freq_hz) 310 { 311 longlong_t tsc; 312 int flags; 313 314 /* 315 * cpu_freq_hz is the measured cpu frequency in hertz 316 */ 317 318 /* 319 * We can't accommodate CPUs slower than 31.25 MHz. 320 */ 321 ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 322 nsec_scale = 323 (uint_t) 324 (((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 325 326 flags = clear_int_flag(); 327 tsc = tsc_read(); 328 (void) tsc_gethrtime(); 329 tsc_max_delta = tsc_read() - tsc; 330 restore_int_flag(flags); 331 } 332 333 /* 334 * Called once per second on CPU 0 from the cyclic subsystem's CY_HIGH_LEVEL 335 * interrupt. 336 */ 337 void 338 tsc_tick(void) 339 { 340 hrtime_t now, delta; 341 ushort_t spl; 342 343 /* 344 * Before we set the new variables, we set the shadow values. This 345 * allows for lock free operation in dtrace_gethrtime(). 346 */ 347 lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 348 ipltospl(CBE_HIGH_PIL), &spl); 349 350 shadow_tsc_hrtime_base = tsc_hrtime_base; 351 shadow_tsc_last = tsc_last; 352 shadow_nsec_scale = nsec_scale; 353 354 shadow_hres_lock++; 355 splx(spl); 356 357 CLOCK_LOCK(&spl); 358 359 now = tsc_read(); 360 361 if (now < tsc_last) { 362 /* 363 * The TSC has just jumped into the past. We assume that 364 * this is due to a suspend/resume cycle, and we're going 365 * to use the _current_ value of TSC as the delta. This 366 * will keep tsc_hrtime_base correct. We're also going to 367 * assume that rate of tsc does not change after a suspend 368 * resume (i.e nsec_scale remains the same). 369 */ 370 delta = now; 371 tsc_last_jumped += tsc_last; 372 tsc_jumped = 1; 373 } else { 374 /* 375 * Determine the number of TSC ticks since the last clock 376 * tick, and add that to the hrtime base. 377 */ 378 delta = now - tsc_last; 379 } 380 381 TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 382 tsc_last = now; 383 384 CLOCK_UNLOCK(spl); 385 } 386 387 hrtime_t 388 tsc_gethrtime(void) 389 { 390 uint32_t old_hres_lock; 391 hrtime_t tsc, hrt; 392 393 do { 394 old_hres_lock = hres_lock; 395 396 if ((tsc = tsc_read()) >= tsc_last) { 397 /* 398 * It would seem to be obvious that this is true 399 * (that is, the past is less than the present), 400 * but it isn't true in the presence of suspend/resume 401 * cycles. If we manage to call gethrtime() 402 * after a resume, but before the first call to 403 * tsc_tick(), we will see the jump. In this case, 404 * we will simply use the value in TSC as the delta. 405 */ 406 tsc -= tsc_last; 407 } else if (tsc >= tsc_last - 2*tsc_max_delta) { 408 /* 409 * There is a chance that tsc_tick() has just run on 410 * another CPU, and we have drifted just enough so that 411 * we appear behind tsc_last. In this case, force the 412 * delta to be zero. 413 */ 414 tsc = 0; 415 } 416 417 hrt = tsc_hrtime_base; 418 419 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 420 } while ((old_hres_lock & ~1) != hres_lock); 421 422 return (hrt); 423 } 424 425 /* 426 * This is similar to the above, but it cannot actually spin on hres_lock. 427 * As a result, it caches all of the variables it needs; if the variables 428 * don't change, it's done. 429 */ 430 hrtime_t 431 dtrace_gethrtime(void) 432 { 433 uint32_t old_hres_lock; 434 hrtime_t tsc, hrt; 435 436 do { 437 old_hres_lock = hres_lock; 438 439 /* 440 * See the comments in tsc_gethrtime(), above. 441 */ 442 if ((tsc = tsc_read()) >= tsc_last) 443 tsc -= tsc_last; 444 else if (tsc >= tsc_last - 2*tsc_max_delta) 445 tsc = 0; 446 447 hrt = tsc_hrtime_base; 448 449 TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 450 451 if ((old_hres_lock & ~1) == hres_lock) 452 break; 453 454 /* 455 * If we're here, the clock lock is locked -- or it has been 456 * unlocked and locked since we looked. This may be due to 457 * tsc_tick() running on another CPU -- or it may be because 458 * some code path has ended up in dtrace_probe() with 459 * CLOCK_LOCK held. We'll try to determine that we're in 460 * the former case by taking another lap if the lock has 461 * changed since when we first looked at it. 462 */ 463 if (old_hres_lock != hres_lock) 464 continue; 465 466 /* 467 * So the lock was and is locked. We'll use the old data 468 * instead. 469 */ 470 old_hres_lock = shadow_hres_lock; 471 472 /* 473 * See the comments in tsc_gethrtime(), above. 474 */ 475 if ((tsc = tsc_read()) >= shadow_tsc_last) 476 tsc -= shadow_tsc_last; 477 else if (tsc >= shadow_tsc_last - 2*tsc_max_delta) 478 tsc = 0; 479 480 hrt = shadow_tsc_hrtime_base; 481 482 TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 483 } while ((old_hres_lock & ~1) != shadow_hres_lock); 484 485 return (hrt); 486 } 487 488 hrtime_t 489 tsc_gethrtime_delta(void) 490 { 491 hrtime_t hrt; 492 int flags; 493 494 /* 495 * We need to disable interrupts here to assure that we don't migrate 496 * between the call to tsc_gethrtime() and adding the CPU's hrtime 497 * delta. Note that disabling and reenabling preemption is forbidden 498 * here because we may be in the middle of a fast trap. In the amd64 499 * kernel we cannot tolerate preemption during a fast trap. See 500 * _update_sregs(). 501 */ 502 503 flags = clear_int_flag(); 504 hrt = tsc_gethrtime() + tsc_sync_delta[CPU->cpu_id]; 505 restore_int_flag(flags); 506 507 return (hrt); 508 } 509 510 extern uint64_t cpu_freq_hz; 511 extern int tsc_gethrtime_enable; 512 513 /* 514 * The following converts nanoseconds of highres-time to ticks 515 */ 516 517 static uint64_t 518 hrtime2tick(hrtime_t ts) 519 { 520 hrtime_t q = ts / NANOSEC; 521 hrtime_t r = ts - (q * NANOSEC); 522 523 return (q * cpu_freq_hz + ((r * cpu_freq_hz) / NANOSEC)); 524 } 525 526 /* 527 * This is used to convert scaled high-res time from nanoseconds to 528 * unscaled hardware ticks. (Read from hardware timestamp counter) 529 */ 530 531 uint64_t 532 unscalehrtime(hrtime_t ts) 533 { 534 if (tsc_gethrtime_enable) { 535 uint64_t unscale = 0; 536 hrtime_t rescale; 537 hrtime_t diff = ts; 538 539 while (diff > (nsec_per_tick)) { 540 unscale += hrtime2tick(diff); 541 rescale = unscale; 542 scalehrtime(&rescale); 543 diff = ts - rescale; 544 } 545 546 return (unscale); 547 } 548 return (0); 549 } 550 551 552 hrtime_t 553 tsc_gethrtimeunscaled(void) 554 { 555 uint32_t old_hres_lock; 556 hrtime_t tsc; 557 558 do { 559 old_hres_lock = hres_lock; 560 561 if ((tsc = tsc_read()) < tsc_last) { 562 /* 563 * see comments in tsc_gethrtime 564 */ 565 tsc += tsc_last_jumped; 566 } 567 568 } while ((old_hres_lock & ~1) != hres_lock); 569 570 return (tsc); 571 } 572 573 574 /* Convert a tsc timestamp to nanoseconds */ 575 void 576 tsc_scalehrtime(hrtime_t *tsc) 577 { 578 hrtime_t hrt; 579 hrtime_t mytsc; 580 581 if (tsc == NULL) 582 return; 583 mytsc = *tsc; 584 585 TSC_CONVERT(mytsc, hrt, nsec_scale); 586 *tsc = hrt; 587 } 588 589 hrtime_t 590 tsc_gethrtimeunscaled_delta(void) 591 { 592 hrtime_t hrt; 593 int flags; 594 595 /* 596 * Similarly to tsc_gethrtime_delta, we need to disable preemption 597 * to prevent migration between the call to tsc_gethrtimeunscaled 598 * and adding the CPU's hrtime delta. Note that disabling and 599 * reenabling preemption is forbidden here because we may be in the 600 * middle of a fast trap. In the amd64 kernel we cannot tolerate 601 * preemption during a fast trap. See _update_sregs(). 602 */ 603 604 flags = clear_int_flag(); 605 hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 606 restore_int_flag(flags); 607 608 return (hrt); 609 } 610