1 #include <linux/kernel.h> 2 #include <linux/sched.h> 3 #include <linux/init.h> 4 #include <linux/module.h> 5 #include <linux/timer.h> 6 #include <linux/acpi_pmtmr.h> 7 #include <linux/cpufreq.h> 8 #include <linux/dmi.h> 9 #include <linux/delay.h> 10 #include <linux/clocksource.h> 11 #include <linux/percpu.h> 12 13 #include <asm/hpet.h> 14 #include <asm/timer.h> 15 #include <asm/vgtod.h> 16 #include <asm/time.h> 17 #include <asm/delay.h> 18 19 unsigned int cpu_khz; /* TSC clocks / usec, not used here */ 20 EXPORT_SYMBOL(cpu_khz); 21 unsigned int tsc_khz; 22 EXPORT_SYMBOL(tsc_khz); 23 24 /* 25 * TSC can be unstable due to cpufreq or due to unsynced TSCs 26 */ 27 static int tsc_unstable; 28 29 /* native_sched_clock() is called before tsc_init(), so 30 we must start with the TSC soft disabled to prevent 31 erroneous rdtsc usage on !cpu_has_tsc processors */ 32 static int tsc_disabled = -1; 33 34 /* 35 * Scheduler clock - returns current time in nanosec units. 36 */ 37 u64 native_sched_clock(void) 38 { 39 u64 this_offset; 40 41 /* 42 * Fall back to jiffies if there's no TSC available: 43 * ( But note that we still use it if the TSC is marked 44 * unstable. We do this because unlike Time Of Day, 45 * the scheduler clock tolerates small errors and it's 46 * very important for it to be as fast as the platform 47 * can achive it. ) 48 */ 49 if (unlikely(tsc_disabled)) { 50 /* No locking but a rare wrong value is not a big deal: */ 51 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); 52 } 53 54 /* read the Time Stamp Counter: */ 55 rdtscll(this_offset); 56 57 /* return the value in ns */ 58 return cycles_2_ns(this_offset); 59 } 60 61 /* We need to define a real function for sched_clock, to override the 62 weak default version */ 63 #ifdef CONFIG_PARAVIRT 64 unsigned long long sched_clock(void) 65 { 66 return paravirt_sched_clock(); 67 } 68 #else 69 unsigned long long 70 sched_clock(void) __attribute__((alias("native_sched_clock"))); 71 #endif 72 73 int check_tsc_unstable(void) 74 { 75 return tsc_unstable; 76 } 77 EXPORT_SYMBOL_GPL(check_tsc_unstable); 78 79 #ifdef CONFIG_X86_TSC 80 int __init notsc_setup(char *str) 81 { 82 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, " 83 "cannot disable TSC completely.\n"); 84 tsc_disabled = 1; 85 return 1; 86 } 87 #else 88 /* 89 * disable flag for tsc. Takes effect by clearing the TSC cpu flag 90 * in cpu/common.c 91 */ 92 int __init notsc_setup(char *str) 93 { 94 setup_clear_cpu_cap(X86_FEATURE_TSC); 95 return 1; 96 } 97 #endif 98 99 __setup("notsc", notsc_setup); 100 101 #define MAX_RETRIES 5 102 #define SMI_TRESHOLD 50000 103 104 /* 105 * Read TSC and the reference counters. Take care of SMI disturbance 106 */ 107 static u64 tsc_read_refs(u64 *pm, u64 *hpet) 108 { 109 u64 t1, t2; 110 int i; 111 112 for (i = 0; i < MAX_RETRIES; i++) { 113 t1 = get_cycles(); 114 if (hpet) 115 *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; 116 else 117 *pm = acpi_pm_read_early(); 118 t2 = get_cycles(); 119 if ((t2 - t1) < SMI_TRESHOLD) 120 return t2; 121 } 122 return ULLONG_MAX; 123 } 124 125 /* 126 * Try to calibrate the TSC against the Programmable 127 * Interrupt Timer and return the frequency of the TSC 128 * in kHz. 129 * 130 * Return ULONG_MAX on failure to calibrate. 131 */ 132 static unsigned long pit_calibrate_tsc(void) 133 { 134 u64 tsc, t1, t2, delta; 135 unsigned long tscmin, tscmax; 136 int pitcnt; 137 138 /* Set the Gate high, disable speaker */ 139 outb((inb(0x61) & ~0x02) | 0x01, 0x61); 140 141 /* 142 * Setup CTC channel 2* for mode 0, (interrupt on terminal 143 * count mode), binary count. Set the latch register to 50ms 144 * (LSB then MSB) to begin countdown. 145 */ 146 outb(0xb0, 0x43); 147 outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42); 148 outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42); 149 150 tsc = t1 = t2 = get_cycles(); 151 152 pitcnt = 0; 153 tscmax = 0; 154 tscmin = ULONG_MAX; 155 while ((inb(0x61) & 0x20) == 0) { 156 t2 = get_cycles(); 157 delta = t2 - tsc; 158 tsc = t2; 159 if ((unsigned long) delta < tscmin) 160 tscmin = (unsigned int) delta; 161 if ((unsigned long) delta > tscmax) 162 tscmax = (unsigned int) delta; 163 pitcnt++; 164 } 165 166 /* 167 * Sanity checks: 168 * 169 * If we were not able to read the PIT more than 5000 170 * times, then we have been hit by a massive SMI 171 * 172 * If the maximum is 10 times larger than the minimum, 173 * then we got hit by an SMI as well. 174 */ 175 if (pitcnt < 5000 || tscmax > 10 * tscmin) 176 return ULONG_MAX; 177 178 /* Calculate the PIT value */ 179 delta = t2 - t1; 180 do_div(delta, 50); 181 return delta; 182 } 183 184 185 /** 186 * native_calibrate_tsc - calibrate the tsc on boot 187 */ 188 unsigned long native_calibrate_tsc(void) 189 { 190 u64 tsc1, tsc2, delta, pm1, pm2, hpet1, hpet2; 191 unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; 192 unsigned long flags; 193 int hpet = is_hpet_enabled(), i; 194 195 /* 196 * Run 5 calibration loops to get the lowest frequency value 197 * (the best estimate). We use two different calibration modes 198 * here: 199 * 200 * 1) PIT loop. We set the PIT Channel 2 to oneshot mode and 201 * load a timeout of 50ms. We read the time right after we 202 * started the timer and wait until the PIT count down reaches 203 * zero. In each wait loop iteration we read the TSC and check 204 * the delta to the previous read. We keep track of the min 205 * and max values of that delta. The delta is mostly defined 206 * by the IO time of the PIT access, so we can detect when a 207 * SMI/SMM disturbance happend between the two reads. If the 208 * maximum time is significantly larger than the minimum time, 209 * then we discard the result and have another try. 210 * 211 * 2) Reference counter. If available we use the HPET or the 212 * PMTIMER as a reference to check the sanity of that value. 213 * We use separate TSC readouts and check inside of the 214 * reference read for a SMI/SMM disturbance. We dicard 215 * disturbed values here as well. We do that around the PIT 216 * calibration delay loop as we have to wait for a certain 217 * amount of time anyway. 218 */ 219 for (i = 0; i < 5; i++) { 220 unsigned long tsc_pit_khz; 221 222 /* 223 * Read the start value and the reference count of 224 * hpet/pmtimer when available. Then do the PIT 225 * calibration, which will take at least 50ms, and 226 * read the end value. 227 */ 228 local_irq_save(flags); 229 tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL); 230 tsc_pit_khz = pit_calibrate_tsc(); 231 tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL); 232 local_irq_restore(flags); 233 234 /* Pick the lowest PIT TSC calibration so far */ 235 tsc_pit_min = min(tsc_pit_min, tsc_pit_khz); 236 237 /* hpet or pmtimer available ? */ 238 if (!hpet && !pm1 && !pm2) 239 continue; 240 241 /* Check, whether the sampling was disturbed by an SMI */ 242 if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX) 243 continue; 244 245 tsc2 = (tsc2 - tsc1) * 1000000LL; 246 247 if (hpet) { 248 if (hpet2 < hpet1) 249 hpet2 += 0x100000000ULL; 250 hpet2 -= hpet1; 251 tsc1 = ((u64)hpet2 * hpet_readl(HPET_PERIOD)); 252 do_div(tsc1, 1000000); 253 } else { 254 if (pm2 < pm1) 255 pm2 += (u64)ACPI_PM_OVRRUN; 256 pm2 -= pm1; 257 tsc1 = pm2 * 1000000000LL; 258 do_div(tsc1, PMTMR_TICKS_PER_SEC); 259 } 260 261 do_div(tsc2, tsc1); 262 tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2); 263 } 264 265 /* 266 * Now check the results. 267 */ 268 if (tsc_pit_min == ULONG_MAX) { 269 /* PIT gave no useful value */ 270 printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n"); 271 272 /* We don't have an alternative source, disable TSC */ 273 if (!hpet && !pm1 && !pm2) { 274 printk("TSC: No reference (HPET/PMTIMER) available\n"); 275 return 0; 276 } 277 278 /* The alternative source failed as well, disable TSC */ 279 if (tsc_ref_min == ULONG_MAX) { 280 printk(KERN_WARNING "TSC: HPET/PMTIMER calibration " 281 "failed due to SMI disturbance.\n"); 282 return 0; 283 } 284 285 /* Use the alternative source */ 286 printk(KERN_INFO "TSC: using %s reference calibration\n", 287 hpet ? "HPET" : "PMTIMER"); 288 289 return tsc_ref_min; 290 } 291 292 /* We don't have an alternative source, use the PIT calibration value */ 293 if (!hpet && !pm1 && !pm2) { 294 printk(KERN_INFO "TSC: Using PIT calibration value\n"); 295 return tsc_pit_min; 296 } 297 298 /* The alternative source failed, use the PIT calibration value */ 299 if (tsc_ref_min == ULONG_MAX) { 300 printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed due " 301 "to SMI disturbance. Using PIT calibration\n"); 302 return tsc_pit_min; 303 } 304 305 /* Check the reference deviation */ 306 delta = ((u64) tsc_pit_min) * 100; 307 do_div(delta, tsc_ref_min); 308 309 /* 310 * If both calibration results are inside a 5% window, the we 311 * use the lower frequency of those as it is probably the 312 * closest estimate. 313 */ 314 if (delta >= 95 && delta <= 105) { 315 printk(KERN_INFO "TSC: PIT calibration confirmed by %s.\n", 316 hpet ? "HPET" : "PMTIMER"); 317 printk(KERN_INFO "TSC: using %s calibration value\n", 318 tsc_pit_min <= tsc_ref_min ? "PIT" : 319 hpet ? "HPET" : "PMTIMER"); 320 return tsc_pit_min <= tsc_ref_min ? tsc_pit_min : tsc_ref_min; 321 } 322 323 printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n", 324 hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min); 325 326 /* 327 * The calibration values differ too much. In doubt, we use 328 * the PIT value as we know that there are PMTIMERs around 329 * running at double speed. 330 */ 331 printk(KERN_INFO "TSC: Using PIT calibration value\n"); 332 return tsc_pit_min; 333 } 334 335 #ifdef CONFIG_X86_32 336 /* Only called from the Powernow K7 cpu freq driver */ 337 int recalibrate_cpu_khz(void) 338 { 339 #ifndef CONFIG_SMP 340 unsigned long cpu_khz_old = cpu_khz; 341 342 if (cpu_has_tsc) { 343 tsc_khz = calibrate_tsc(); 344 cpu_khz = tsc_khz; 345 cpu_data(0).loops_per_jiffy = 346 cpufreq_scale(cpu_data(0).loops_per_jiffy, 347 cpu_khz_old, cpu_khz); 348 return 0; 349 } else 350 return -ENODEV; 351 #else 352 return -ENODEV; 353 #endif 354 } 355 356 EXPORT_SYMBOL(recalibrate_cpu_khz); 357 358 #endif /* CONFIG_X86_32 */ 359 360 /* Accelerators for sched_clock() 361 * convert from cycles(64bits) => nanoseconds (64bits) 362 * basic equation: 363 * ns = cycles / (freq / ns_per_sec) 364 * ns = cycles * (ns_per_sec / freq) 365 * ns = cycles * (10^9 / (cpu_khz * 10^3)) 366 * ns = cycles * (10^6 / cpu_khz) 367 * 368 * Then we use scaling math (suggested by george@mvista.com) to get: 369 * ns = cycles * (10^6 * SC / cpu_khz) / SC 370 * ns = cycles * cyc2ns_scale / SC 371 * 372 * And since SC is a constant power of two, we can convert the div 373 * into a shift. 374 * 375 * We can use khz divisor instead of mhz to keep a better precision, since 376 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. 377 * (mathieu.desnoyers@polymtl.ca) 378 * 379 * -johnstul@us.ibm.com "math is hard, lets go shopping!" 380 */ 381 382 DEFINE_PER_CPU(unsigned long, cyc2ns); 383 384 static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) 385 { 386 unsigned long long tsc_now, ns_now; 387 unsigned long flags, *scale; 388 389 local_irq_save(flags); 390 sched_clock_idle_sleep_event(); 391 392 scale = &per_cpu(cyc2ns, cpu); 393 394 rdtscll(tsc_now); 395 ns_now = __cycles_2_ns(tsc_now); 396 397 if (cpu_khz) 398 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz; 399 400 sched_clock_idle_wakeup_event(0); 401 local_irq_restore(flags); 402 } 403 404 #ifdef CONFIG_CPU_FREQ 405 406 /* Frequency scaling support. Adjust the TSC based timer when the cpu frequency 407 * changes. 408 * 409 * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's 410 * not that important because current Opteron setups do not support 411 * scaling on SMP anyroads. 412 * 413 * Should fix up last_tsc too. Currently gettimeofday in the 414 * first tick after the change will be slightly wrong. 415 */ 416 417 static unsigned int ref_freq; 418 static unsigned long loops_per_jiffy_ref; 419 static unsigned long tsc_khz_ref; 420 421 static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 422 void *data) 423 { 424 struct cpufreq_freqs *freq = data; 425 unsigned long *lpj, dummy; 426 427 if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC)) 428 return 0; 429 430 lpj = &dummy; 431 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 432 #ifdef CONFIG_SMP 433 lpj = &cpu_data(freq->cpu).loops_per_jiffy; 434 #else 435 lpj = &boot_cpu_data.loops_per_jiffy; 436 #endif 437 438 if (!ref_freq) { 439 ref_freq = freq->old; 440 loops_per_jiffy_ref = *lpj; 441 tsc_khz_ref = tsc_khz; 442 } 443 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || 444 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || 445 (val == CPUFREQ_RESUMECHANGE)) { 446 *lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); 447 448 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); 449 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 450 mark_tsc_unstable("cpufreq changes"); 451 } 452 453 set_cyc2ns_scale(tsc_khz, freq->cpu); 454 455 return 0; 456 } 457 458 static struct notifier_block time_cpufreq_notifier_block = { 459 .notifier_call = time_cpufreq_notifier 460 }; 461 462 static int __init cpufreq_tsc(void) 463 { 464 if (!cpu_has_tsc) 465 return 0; 466 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 467 return 0; 468 cpufreq_register_notifier(&time_cpufreq_notifier_block, 469 CPUFREQ_TRANSITION_NOTIFIER); 470 return 0; 471 } 472 473 core_initcall(cpufreq_tsc); 474 475 #endif /* CONFIG_CPU_FREQ */ 476 477 /* clocksource code */ 478 479 static struct clocksource clocksource_tsc; 480 481 /* 482 * We compare the TSC to the cycle_last value in the clocksource 483 * structure to avoid a nasty time-warp. This can be observed in a 484 * very small window right after one CPU updated cycle_last under 485 * xtime/vsyscall_gtod lock and the other CPU reads a TSC value which 486 * is smaller than the cycle_last reference value due to a TSC which 487 * is slighty behind. This delta is nowhere else observable, but in 488 * that case it results in a forward time jump in the range of hours 489 * due to the unsigned delta calculation of the time keeping core 490 * code, which is necessary to support wrapping clocksources like pm 491 * timer. 492 */ 493 static cycle_t read_tsc(void) 494 { 495 cycle_t ret = (cycle_t)get_cycles(); 496 497 return ret >= clocksource_tsc.cycle_last ? 498 ret : clocksource_tsc.cycle_last; 499 } 500 501 #ifdef CONFIG_X86_64 502 static cycle_t __vsyscall_fn vread_tsc(void) 503 { 504 cycle_t ret = (cycle_t)vget_cycles(); 505 506 return ret >= __vsyscall_gtod_data.clock.cycle_last ? 507 ret : __vsyscall_gtod_data.clock.cycle_last; 508 } 509 #endif 510 511 static struct clocksource clocksource_tsc = { 512 .name = "tsc", 513 .rating = 300, 514 .read = read_tsc, 515 .mask = CLOCKSOURCE_MASK(64), 516 .shift = 22, 517 .flags = CLOCK_SOURCE_IS_CONTINUOUS | 518 CLOCK_SOURCE_MUST_VERIFY, 519 #ifdef CONFIG_X86_64 520 .vread = vread_tsc, 521 #endif 522 }; 523 524 void mark_tsc_unstable(char *reason) 525 { 526 if (!tsc_unstable) { 527 tsc_unstable = 1; 528 printk("Marking TSC unstable due to %s\n", reason); 529 /* Change only the rating, when not registered */ 530 if (clocksource_tsc.mult) 531 clocksource_change_rating(&clocksource_tsc, 0); 532 else 533 clocksource_tsc.rating = 0; 534 } 535 } 536 537 EXPORT_SYMBOL_GPL(mark_tsc_unstable); 538 539 static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d) 540 { 541 printk(KERN_NOTICE "%s detected: marking TSC unstable.\n", 542 d->ident); 543 tsc_unstable = 1; 544 return 0; 545 } 546 547 /* List of systems that have known TSC problems */ 548 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = { 549 { 550 .callback = dmi_mark_tsc_unstable, 551 .ident = "IBM Thinkpad 380XD", 552 .matches = { 553 DMI_MATCH(DMI_BOARD_VENDOR, "IBM"), 554 DMI_MATCH(DMI_BOARD_NAME, "2635FA0"), 555 }, 556 }, 557 {} 558 }; 559 560 /* 561 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC 562 */ 563 #ifdef CONFIG_MGEODE_LX 564 /* RTSC counts during suspend */ 565 #define RTSC_SUSP 0x100 566 567 static void __init check_geode_tsc_reliable(void) 568 { 569 unsigned long res_low, res_high; 570 571 rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high); 572 if (res_low & RTSC_SUSP) 573 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY; 574 } 575 #else 576 static inline void check_geode_tsc_reliable(void) { } 577 #endif 578 579 /* 580 * Make an educated guess if the TSC is trustworthy and synchronized 581 * over all CPUs. 582 */ 583 __cpuinit int unsynchronized_tsc(void) 584 { 585 if (!cpu_has_tsc || tsc_unstable) 586 return 1; 587 588 #ifdef CONFIG_SMP 589 if (apic_is_clustered_box()) 590 return 1; 591 #endif 592 593 if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 594 return 0; 595 /* 596 * Intel systems are normally all synchronized. 597 * Exceptions must mark TSC as unstable: 598 */ 599 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 600 /* assume multi socket systems are not synchronized: */ 601 if (num_possible_cpus() > 1) 602 tsc_unstable = 1; 603 } 604 605 return tsc_unstable; 606 } 607 608 static void __init init_tsc_clocksource(void) 609 { 610 clocksource_tsc.mult = clocksource_khz2mult(tsc_khz, 611 clocksource_tsc.shift); 612 /* lower the rating if we already know its unstable: */ 613 if (check_tsc_unstable()) { 614 clocksource_tsc.rating = 0; 615 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS; 616 } 617 clocksource_register(&clocksource_tsc); 618 } 619 620 void __init tsc_init(void) 621 { 622 u64 lpj; 623 int cpu; 624 625 if (!cpu_has_tsc) 626 return; 627 628 tsc_khz = calibrate_tsc(); 629 cpu_khz = tsc_khz; 630 631 if (!tsc_khz) { 632 mark_tsc_unstable("could not calculate TSC khz"); 633 return; 634 } 635 636 #ifdef CONFIG_X86_64 637 if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) && 638 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)) 639 cpu_khz = calibrate_cpu(); 640 #endif 641 642 lpj = ((u64)tsc_khz * 1000); 643 do_div(lpj, HZ); 644 lpj_fine = lpj; 645 646 printk("Detected %lu.%03lu MHz processor.\n", 647 (unsigned long)cpu_khz / 1000, 648 (unsigned long)cpu_khz % 1000); 649 650 /* 651 * Secondary CPUs do not run through tsc_init(), so set up 652 * all the scale factors for all CPUs, assuming the same 653 * speed as the bootup CPU. (cpufreq notifiers will fix this 654 * up if their speed diverges) 655 */ 656 for_each_possible_cpu(cpu) 657 set_cyc2ns_scale(cpu_khz, cpu); 658 659 if (tsc_disabled > 0) 660 return; 661 662 /* now allow native_sched_clock() to use rdtsc */ 663 tsc_disabled = 0; 664 665 use_tsc_delay(); 666 /* Check and install the TSC clocksource */ 667 dmi_check_system(bad_tsc_dmi_table); 668 669 if (unsynchronized_tsc()) 670 mark_tsc_unstable("TSCs unsynchronized"); 671 672 check_geode_tsc_reliable(); 673 init_tsc_clocksource(); 674 } 675 676