1 // SPDX-License-Identifier: GPL-2.0 2 /* calibrate.c: default delay calibration 3 * 4 * Excised from init/main.c 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 */ 7 8 #include <linux/delay.h> 9 #include <linux/init.h> 10 #include <linux/jiffies.h> 11 #include <linux/kstrtox.h> 12 #include <linux/percpu.h> 13 #include <linux/printk.h> 14 #include <linux/smp.h> 15 #include <linux/stddef.h> 16 17 unsigned long lpj_fine; 18 unsigned long preset_lpj; 19 20 static int __init lpj_setup(char *str) 21 { 22 return kstrtoul(str, 0, &preset_lpj) == 0; 23 } 24 25 __setup("lpj=", lpj_setup); 26 27 #ifdef CONFIG_ARCH_HAS_DELAY_TIMER 28 29 /* This routine uses the delay_read_timer() routine and gets the 30 * loops per jiffy directly, instead of guessing it using delay(). 31 * Also, this code tries to handle non-maskable asynchronous events 32 * (like SMIs) 33 */ 34 #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100)) 35 #define MAX_DIRECT_CALIBRATION_RETRIES 5 36 37 static unsigned long calibrate_delay_direct(void) 38 { 39 unsigned long pre_start, start, post_start; 40 unsigned long pre_end, end, post_end; 41 unsigned long start_jiffies; 42 unsigned long timer_rate_min, timer_rate_max; 43 unsigned long good_timer_sum = 0; 44 unsigned long good_timer_count = 0; 45 unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES]; 46 int max = -1; /* index of measured_times with max/min values or not set */ 47 int min = -1; 48 int i; 49 50 if (!delay_read_timer(&pre_start)) 51 return 0; 52 53 /* 54 * A simple loop like 55 * while ( jiffies < start_jiffies+1) 56 * start = delay_read_timer(); 57 * will not do. As we don't really know whether jiffy switch 58 * happened first or timer_value was read first. And some asynchronous 59 * event can happen between these two events introducing errors in lpj. 60 * 61 * So, we do 62 * 1. pre_start <- When we are sure that jiffy switch hasn't happened 63 * 2. check jiffy switch 64 * 3. start <- timer value before or after jiffy switch 65 * 4. post_start <- When we are sure that jiffy switch has happened 66 * 67 * Note, we don't know anything about order of 2 and 3. 68 * Now, by looking at post_start and pre_start difference, we can 69 * check whether any asynchronous event happened or not 70 */ 71 72 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) { 73 pre_start = 0; 74 delay_read_timer(&start); 75 start_jiffies = jiffies; 76 while (time_before_eq(jiffies, start_jiffies + 1)) { 77 pre_start = start; 78 delay_read_timer(&start); 79 } 80 delay_read_timer(&post_start); 81 82 pre_end = 0; 83 end = post_start; 84 while (time_before_eq(jiffies, start_jiffies + 1 + 85 DELAY_CALIBRATION_TICKS)) { 86 pre_end = end; 87 delay_read_timer(&end); 88 } 89 delay_read_timer(&post_end); 90 91 timer_rate_max = (post_end - pre_start) / 92 DELAY_CALIBRATION_TICKS; 93 timer_rate_min = (pre_end - post_start) / 94 DELAY_CALIBRATION_TICKS; 95 96 /* 97 * If the upper limit and lower limit of the timer_rate is 98 * >= 12.5% apart, redo calibration. 99 */ 100 if (start >= post_end) 101 printk(KERN_NOTICE "calibrate_delay_direct() ignoring " 102 "timer_rate as we had a TSC wrap around" 103 " start=%lu >=post_end=%lu\n", 104 start, post_end); 105 if (start < post_end && pre_start != 0 && pre_end != 0 && 106 (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) { 107 good_timer_count++; 108 good_timer_sum += timer_rate_max; 109 measured_times[i] = timer_rate_max; 110 if (max < 0 || timer_rate_max > measured_times[max]) 111 max = i; 112 if (min < 0 || timer_rate_max < measured_times[min]) 113 min = i; 114 } else 115 measured_times[i] = 0; 116 117 } 118 119 /* 120 * Find the maximum & minimum - if they differ too much throw out the 121 * one with the largest difference from the mean and try again... 122 */ 123 while (good_timer_count > 1) { 124 unsigned long estimate; 125 unsigned long maxdiff; 126 127 /* compute the estimate */ 128 estimate = (good_timer_sum/good_timer_count); 129 maxdiff = estimate >> 3; 130 131 /* if range is within 12% let's take it */ 132 if ((measured_times[max] - measured_times[min]) < maxdiff) 133 return estimate; 134 135 /* ok - drop the worse value and try again... */ 136 good_timer_sum = 0; 137 good_timer_count = 0; 138 if ((measured_times[max] - estimate) < 139 (estimate - measured_times[min])) { 140 printk(KERN_NOTICE "calibrate_delay_direct() dropping " 141 "min bogoMips estimate %d = %lu\n", 142 min, measured_times[min]); 143 measured_times[min] = 0; 144 min = max; 145 } else { 146 printk(KERN_NOTICE "calibrate_delay_direct() dropping " 147 "max bogoMips estimate %d = %lu\n", 148 max, measured_times[max]); 149 measured_times[max] = 0; 150 max = min; 151 } 152 153 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) { 154 if (measured_times[i] == 0) 155 continue; 156 good_timer_count++; 157 good_timer_sum += measured_times[i]; 158 if (measured_times[i] < measured_times[min]) 159 min = i; 160 if (measured_times[i] > measured_times[max]) 161 max = i; 162 } 163 164 } 165 166 printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good " 167 "estimate for loops_per_jiffy.\nProbably due to long platform " 168 "interrupts. Consider using \"lpj=\" boot option.\n"); 169 return 0; 170 } 171 #else 172 static unsigned long calibrate_delay_direct(void) 173 { 174 return 0; 175 } 176 #endif 177 178 /* 179 * This is the number of bits of precision for the loops_per_jiffy. Each 180 * time we refine our estimate after the first takes 1.5/HZ seconds, so try 181 * to start with a good estimate. 182 * For the boot cpu we can skip the delay calibration and assign it a value 183 * calculated based on the timer frequency. 184 * For the rest of the CPUs we cannot assume that the timer frequency is same as 185 * the cpu frequency, hence do the calibration for those. 186 */ 187 #define LPS_PREC 8 188 189 static unsigned long calibrate_delay_converge(void) 190 { 191 /* First stage - slowly accelerate to find initial bounds */ 192 unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit; 193 int trials = 0, band = 0, trial_in_band = 0; 194 195 lpj = (1<<12); 196 197 /* wait for "start of" clock tick */ 198 ticks = jiffies; 199 while (ticks == jiffies) 200 ; /* nothing */ 201 /* Go .. */ 202 ticks = jiffies; 203 do { 204 if (++trial_in_band == (1<<band)) { 205 ++band; 206 trial_in_band = 0; 207 } 208 __delay(lpj * band); 209 trials += band; 210 } while (ticks == jiffies); 211 /* 212 * We overshot, so retreat to a clear underestimate. Then estimate 213 * the largest likely undershoot. This defines our chop bounds. 214 */ 215 trials -= band; 216 loopadd_base = lpj * band; 217 lpj_base = lpj * trials; 218 219 recalibrate: 220 lpj = lpj_base; 221 loopadd = loopadd_base; 222 223 /* 224 * Do a binary approximation to get lpj set to 225 * equal one clock (up to LPS_PREC bits) 226 */ 227 chop_limit = lpj >> LPS_PREC; 228 while (loopadd > chop_limit) { 229 lpj += loopadd; 230 ticks = jiffies; 231 while (ticks == jiffies) 232 ; /* nothing */ 233 ticks = jiffies; 234 __delay(lpj); 235 if (jiffies != ticks) /* longer than 1 tick */ 236 lpj -= loopadd; 237 loopadd >>= 1; 238 } 239 /* 240 * If we incremented every single time possible, presume we've 241 * massively underestimated initially, and retry with a higher 242 * start, and larger range. (Only seen on x86_64, due to SMIs) 243 */ 244 if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) { 245 lpj_base = lpj; 246 loopadd_base <<= 2; 247 goto recalibrate; 248 } 249 250 return lpj; 251 } 252 253 static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 }; 254 255 /* 256 * Check if cpu calibration delay is already known. For example, 257 * some processors with multi-core sockets may have all cores 258 * with the same calibration delay. 259 * 260 * Architectures should override this function if a faster calibration 261 * method is available. 262 */ 263 unsigned long __attribute__((weak)) calibrate_delay_is_known(void) 264 { 265 return 0; 266 } 267 268 /* 269 * Indicate the cpu delay calibration is done. This can be used by 270 * architectures to stop accepting delay timer registrations after this point. 271 */ 272 273 void __attribute__((weak)) calibration_delay_done(void) 274 { 275 } 276 277 void calibrate_delay(void) 278 { 279 unsigned long lpj; 280 static bool printed; 281 int this_cpu = smp_processor_id(); 282 283 if (per_cpu(cpu_loops_per_jiffy, this_cpu)) { 284 lpj = per_cpu(cpu_loops_per_jiffy, this_cpu); 285 if (!printed) 286 pr_info("Calibrating delay loop (skipped) " 287 "already calibrated this CPU"); 288 } else if (preset_lpj) { 289 lpj = preset_lpj; 290 if (!printed) 291 pr_info("Calibrating delay loop (skipped) " 292 "preset value.. "); 293 } else if ((!printed) && lpj_fine) { 294 lpj = lpj_fine; 295 pr_info("Calibrating delay loop (skipped), " 296 "value calculated using timer frequency.. "); 297 } else if ((lpj = calibrate_delay_is_known())) { 298 ; 299 } else if ((lpj = calibrate_delay_direct()) != 0) { 300 if (!printed) 301 pr_info("Calibrating delay using timer " 302 "specific routine.. "); 303 } else { 304 if (!printed) 305 pr_info("Calibrating delay loop... "); 306 lpj = calibrate_delay_converge(); 307 } 308 per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj; 309 if (!printed) 310 pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n", 311 lpj/(500000/HZ), 312 (lpj/(5000/HZ)) % 100, lpj); 313 314 loops_per_jiffy = lpj; 315 printed = true; 316 317 calibration_delay_done(); 318 } 319