1 // SPDX-License-Identifier: GPL-2.0 2 /* calibrate.c: default delay calibration 3 * 4 * Excised from init/main.c 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 #include <linux/delay.h> 9 #include <linux/init.h> 10 #include <linux/jiffies.h> 11 #include <linux/kstrtox.h> 12 #include <linux/percpu.h> 13 #include <linux/printk.h> 14 #include <linux/smp.h> 15 #include <linux/stddef.h> 16 #include <linux/timex.h> 17 18 unsigned long lpj_fine; 19 unsigned long preset_lpj; 20 21 static int __init lpj_setup(char *str) 22 { 23 return kstrtoul(str, 0, &preset_lpj) == 0; 24 } 25 26 __setup("lpj=", lpj_setup); 27 28 #ifdef ARCH_HAS_READ_CURRENT_TIMER 29 30 /* This routine uses the read_current_timer() routine and gets the 31 * loops per jiffy directly, instead of guessing it using delay(). 32 * Also, this code tries to handle non-maskable asynchronous events 33 * (like SMIs) 34 */ 35 #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100)) 36 #define MAX_DIRECT_CALIBRATION_RETRIES 5 37 38 static unsigned long calibrate_delay_direct(void) 39 { 40 unsigned long pre_start, start, post_start; 41 unsigned long pre_end, end, post_end; 42 unsigned long start_jiffies; 43 unsigned long timer_rate_min, timer_rate_max; 44 unsigned long good_timer_sum = 0; 45 unsigned long good_timer_count = 0; 46 unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES]; 47 int max = -1; /* index of measured_times with max/min values or not set */ 48 int min = -1; 49 int i; 50 51 if (read_current_timer(&pre_start) < 0 ) 52 return 0; 53 54 /* 55 * A simple loop like 56 * while ( jiffies < start_jiffies+1) 57 * start = read_current_timer(); 58 * will not do. As we don't really know whether jiffy switch 59 * happened first or timer_value was read first. And some asynchronous 60 * event can happen between these two events introducing errors in lpj. 61 * 62 * So, we do 63 * 1. pre_start <- When we are sure that jiffy switch hasn't happened 64 * 2. check jiffy switch 65 * 3. start <- timer value before or after jiffy switch 66 * 4. post_start <- When we are sure that jiffy switch has happened 67 * 68 * Note, we don't know anything about order of 2 and 3. 69 * Now, by looking at post_start and pre_start difference, we can 70 * check whether any asynchronous event happened or not 71 */ 72 73 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) { 74 pre_start = 0; 75 read_current_timer(&start); 76 start_jiffies = jiffies; 77 while (time_before_eq(jiffies, start_jiffies + 1)) { 78 pre_start = start; 79 read_current_timer(&start); 80 } 81 read_current_timer(&post_start); 82 83 pre_end = 0; 84 end = post_start; 85 while (time_before_eq(jiffies, start_jiffies + 1 + 86 DELAY_CALIBRATION_TICKS)) { 87 pre_end = end; 88 read_current_timer(&end); 89 } 90 read_current_timer(&post_end); 91 92 timer_rate_max = (post_end - pre_start) / 93 DELAY_CALIBRATION_TICKS; 94 timer_rate_min = (pre_end - post_start) / 95 DELAY_CALIBRATION_TICKS; 96 97 /* 98 * If the upper limit and lower limit of the timer_rate is 99 * >= 12.5% apart, redo calibration. 100 */ 101 if (start >= post_end) 102 printk(KERN_NOTICE "calibrate_delay_direct() ignoring " 103 "timer_rate as we had a TSC wrap around" 104 " start=%lu >=post_end=%lu\n", 105 start, post_end); 106 if (start < post_end && pre_start != 0 && pre_end != 0 && 107 (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) { 108 good_timer_count++; 109 good_timer_sum += timer_rate_max; 110 measured_times[i] = timer_rate_max; 111 if (max < 0 || timer_rate_max > measured_times[max]) 112 max = i; 113 if (min < 0 || timer_rate_max < measured_times[min]) 114 min = i; 115 } else 116 measured_times[i] = 0; 117 118 } 119 120 /* 121 * Find the maximum & minimum - if they differ too much throw out the 122 * one with the largest difference from the mean and try again... 123 */ 124 while (good_timer_count > 1) { 125 unsigned long estimate; 126 unsigned long maxdiff; 127 128 /* compute the estimate */ 129 estimate = (good_timer_sum/good_timer_count); 130 maxdiff = estimate >> 3; 131 132 /* if range is within 12% let's take it */ 133 if ((measured_times[max] - measured_times[min]) < maxdiff) 134 return estimate; 135 136 /* ok - drop the worse value and try again... */ 137 good_timer_sum = 0; 138 good_timer_count = 0; 139 if ((measured_times[max] - estimate) < 140 (estimate - measured_times[min])) { 141 printk(KERN_NOTICE "calibrate_delay_direct() dropping " 142 "min bogoMips estimate %d = %lu\n", 143 min, measured_times[min]); 144 measured_times[min] = 0; 145 min = max; 146 } else { 147 printk(KERN_NOTICE "calibrate_delay_direct() dropping " 148 "max bogoMips estimate %d = %lu\n", 149 max, measured_times[max]); 150 measured_times[max] = 0; 151 max = min; 152 } 153 154 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) { 155 if (measured_times[i] == 0) 156 continue; 157 good_timer_count++; 158 good_timer_sum += measured_times[i]; 159 if (measured_times[i] < measured_times[min]) 160 min = i; 161 if (measured_times[i] > measured_times[max]) 162 max = i; 163 } 164 165 } 166 167 printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good " 168 "estimate for loops_per_jiffy.\nProbably due to long platform " 169 "interrupts. Consider using \"lpj=\" boot option.\n"); 170 return 0; 171 } 172 #else 173 static unsigned long calibrate_delay_direct(void) 174 { 175 return 0; 176 } 177 #endif 178 179 /* 180 * This is the number of bits of precision for the loops_per_jiffy. Each 181 * time we refine our estimate after the first takes 1.5/HZ seconds, so try 182 * to start with a good estimate. 183 * For the boot cpu we can skip the delay calibration and assign it a value 184 * calculated based on the timer frequency. 185 * For the rest of the CPUs we cannot assume that the timer frequency is same as 186 * the cpu frequency, hence do the calibration for those. 187 */ 188 #define LPS_PREC 8 189 190 static unsigned long calibrate_delay_converge(void) 191 { 192 /* First stage - slowly accelerate to find initial bounds */ 193 unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit; 194 int trials = 0, band = 0, trial_in_band = 0; 195 196 lpj = (1<<12); 197 198 /* wait for "start of" clock tick */ 199 ticks = jiffies; 200 while (ticks == jiffies) 201 ; /* nothing */ 202 /* Go .. */ 203 ticks = jiffies; 204 do { 205 if (++trial_in_band == (1<<band)) { 206 ++band; 207 trial_in_band = 0; 208 } 209 __delay(lpj * band); 210 trials += band; 211 } while (ticks == jiffies); 212 /* 213 * We overshot, so retreat to a clear underestimate. Then estimate 214 * the largest likely undershoot. This defines our chop bounds. 215 */ 216 trials -= band; 217 loopadd_base = lpj * band; 218 lpj_base = lpj * trials; 219 220 recalibrate: 221 lpj = lpj_base; 222 loopadd = loopadd_base; 223 224 /* 225 * Do a binary approximation to get lpj set to 226 * equal one clock (up to LPS_PREC bits) 227 */ 228 chop_limit = lpj >> LPS_PREC; 229 while (loopadd > chop_limit) { 230 lpj += loopadd; 231 ticks = jiffies; 232 while (ticks == jiffies) 233 ; /* nothing */ 234 ticks = jiffies; 235 __delay(lpj); 236 if (jiffies != ticks) /* longer than 1 tick */ 237 lpj -= loopadd; 238 loopadd >>= 1; 239 } 240 /* 241 * If we incremented every single time possible, presume we've 242 * massively underestimated initially, and retry with a higher 243 * start, and larger range. (Only seen on x86_64, due to SMIs) 244 */ 245 if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) { 246 lpj_base = lpj; 247 loopadd_base <<= 2; 248 goto recalibrate; 249 } 250 251 return lpj; 252 } 253 254 static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 }; 255 256 /* 257 * Check if cpu calibration delay is already known. For example, 258 * some processors with multi-core sockets may have all cores 259 * with the same calibration delay. 260 * 261 * Architectures should override this function if a faster calibration 262 * method is available. 263 */ 264 unsigned long __attribute__((weak)) calibrate_delay_is_known(void) 265 { 266 return 0; 267 } 268 269 /* 270 * Indicate the cpu delay calibration is done. This can be used by 271 * architectures to stop accepting delay timer registrations after this point. 272 */ 273 274 void __attribute__((weak)) calibration_delay_done(void) 275 { 276 } 277 278 void calibrate_delay(void) 279 { 280 unsigned long lpj; 281 static bool printed; 282 int this_cpu = smp_processor_id(); 283 284 if (per_cpu(cpu_loops_per_jiffy, this_cpu)) { 285 lpj = per_cpu(cpu_loops_per_jiffy, this_cpu); 286 if (!printed) 287 pr_info("Calibrating delay loop (skipped) " 288 "already calibrated this CPU"); 289 } else if (preset_lpj) { 290 lpj = preset_lpj; 291 if (!printed) 292 pr_info("Calibrating delay loop (skipped) " 293 "preset value.. "); 294 } else if ((!printed) && lpj_fine) { 295 lpj = lpj_fine; 296 pr_info("Calibrating delay loop (skipped), " 297 "value calculated using timer frequency.. "); 298 } else if ((lpj = calibrate_delay_is_known())) { 299 ; 300 } else if ((lpj = calibrate_delay_direct()) != 0) { 301 if (!printed) 302 pr_info("Calibrating delay using timer " 303 "specific routine.. "); 304 } else { 305 if (!printed) 306 pr_info("Calibrating delay loop... "); 307 lpj = calibrate_delay_converge(); 308 } 309 per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj; 310 if (!printed) 311 pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n", 312 lpj/(500000/HZ), 313 (lpj/(5000/HZ)) % 100, lpj); 314 315 loops_per_jiffy = lpj; 316 printed = true; 317 318 calibration_delay_done(); 319 } 320