1 /* 2 * check TSC synchronization. 3 * 4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar 5 * 6 * We check whether all boot CPUs have their TSC's synchronized, 7 * print a warning if not and turn off the TSC clock-source. 8 * 9 * The warp-check is point-to-point between two CPUs, the CPU 10 * initiating the bootup is the 'source CPU', the freshly booting 11 * CPU is the 'target CPU'. 12 * 13 * Only two CPUs may participate - they can enter in any order. 14 * ( The serial nature of the boot logic and the CPU hotplug lock 15 * protects against more than 2 CPUs entering this code. ) 16 */ 17 #include <linux/spinlock.h> 18 #include <linux/kernel.h> 19 #include <linux/init.h> 20 #include <linux/smp.h> 21 #include <linux/nmi.h> 22 #include <asm/tsc.h> 23 24 /* 25 * Entry/exit counters that make sure that both CPUs 26 * run the measurement code at once: 27 */ 28 static __cpuinitdata atomic_t start_count; 29 static __cpuinitdata atomic_t stop_count; 30 31 /* 32 * We use a raw spinlock in this exceptional case, because 33 * we want to have the fastest, inlined, non-debug version 34 * of a critical section, to be able to prove TSC time-warps: 35 */ 36 static __cpuinitdata raw_spinlock_t sync_lock = __RAW_SPIN_LOCK_UNLOCKED; 37 static __cpuinitdata cycles_t last_tsc; 38 static __cpuinitdata cycles_t max_warp; 39 static __cpuinitdata int nr_warps; 40 41 /* 42 * TSC-warp measurement loop running on both CPUs: 43 */ 44 static __cpuinit void check_tsc_warp(void) 45 { 46 cycles_t start, now, prev, end; 47 int i; 48 49 rdtsc_barrier(); 50 start = get_cycles(); 51 rdtsc_barrier(); 52 /* 53 * The measurement runs for 20 msecs: 54 */ 55 end = start + tsc_khz * 20ULL; 56 now = start; 57 58 for (i = 0; ; i++) { 59 /* 60 * We take the global lock, measure TSC, save the 61 * previous TSC that was measured (possibly on 62 * another CPU) and update the previous TSC timestamp. 63 */ 64 __raw_spin_lock(&sync_lock); 65 prev = last_tsc; 66 rdtsc_barrier(); 67 now = get_cycles(); 68 rdtsc_barrier(); 69 last_tsc = now; 70 __raw_spin_unlock(&sync_lock); 71 72 /* 73 * Be nice every now and then (and also check whether 74 * measurement is done [we also insert a 10 million 75 * loops safety exit, so we dont lock up in case the 76 * TSC readout is totally broken]): 77 */ 78 if (unlikely(!(i & 7))) { 79 if (now > end || i > 10000000) 80 break; 81 cpu_relax(); 82 touch_nmi_watchdog(); 83 } 84 /* 85 * Outside the critical section we can now see whether 86 * we saw a time-warp of the TSC going backwards: 87 */ 88 if (unlikely(prev > now)) { 89 __raw_spin_lock(&sync_lock); 90 max_warp = max(max_warp, prev - now); 91 nr_warps++; 92 __raw_spin_unlock(&sync_lock); 93 } 94 } 95 WARN(!(now-start), 96 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", 97 now-start, end-start); 98 } 99 100 /* 101 * Source CPU calls into this - it waits for the freshly booted 102 * target CPU to arrive and then starts the measurement: 103 */ 104 void __cpuinit check_tsc_sync_source(int cpu) 105 { 106 int cpus = 2; 107 108 /* 109 * No need to check if we already know that the TSC is not 110 * synchronized: 111 */ 112 if (unsynchronized_tsc()) 113 return; 114 115 printk(KERN_INFO "checking TSC synchronization [CPU#%d -> CPU#%d]:", 116 smp_processor_id(), cpu); 117 118 /* 119 * Reset it - in case this is a second bootup: 120 */ 121 atomic_set(&stop_count, 0); 122 123 /* 124 * Wait for the target to arrive: 125 */ 126 while (atomic_read(&start_count) != cpus-1) 127 cpu_relax(); 128 /* 129 * Trigger the target to continue into the measurement too: 130 */ 131 atomic_inc(&start_count); 132 133 check_tsc_warp(); 134 135 while (atomic_read(&stop_count) != cpus-1) 136 cpu_relax(); 137 138 if (nr_warps) { 139 printk("\n"); 140 printk(KERN_WARNING "Measured %Ld cycles TSC warp between CPUs," 141 " turning off TSC clock.\n", max_warp); 142 mark_tsc_unstable("check_tsc_sync_source failed"); 143 } else { 144 printk(" passed.\n"); 145 } 146 147 /* 148 * Reset it - just in case we boot another CPU later: 149 */ 150 atomic_set(&start_count, 0); 151 nr_warps = 0; 152 max_warp = 0; 153 last_tsc = 0; 154 155 /* 156 * Let the target continue with the bootup: 157 */ 158 atomic_inc(&stop_count); 159 } 160 161 /* 162 * Freshly booted CPUs call into this: 163 */ 164 void __cpuinit check_tsc_sync_target(void) 165 { 166 int cpus = 2; 167 168 if (unsynchronized_tsc()) 169 return; 170 171 /* 172 * Register this CPU's participation and wait for the 173 * source CPU to start the measurement: 174 */ 175 atomic_inc(&start_count); 176 while (atomic_read(&start_count) != cpus) 177 cpu_relax(); 178 179 check_tsc_warp(); 180 181 /* 182 * Ok, we are done: 183 */ 184 atomic_inc(&stop_count); 185 186 /* 187 * Wait for the source CPU to print stuff: 188 */ 189 while (atomic_read(&stop_count) != cpus) 190 cpu_relax(); 191 } 192 #undef NR_LOOPS 193 194