1 // SPDX-License-Identifier: GPL-2.0
2 /* calibrate.c: default delay calibration
3 *
4 * Excised from init/main.c
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8 #include <linux/delay.h>
9 #include <linux/init.h>
10 #include <linux/jiffies.h>
11 #include <linux/kstrtox.h>
12 #include <linux/percpu.h>
13 #include <linux/printk.h>
14 #include <linux/smp.h>
15 #include <linux/stddef.h>
16 #include <linux/timex.h>
17
18 unsigned long lpj_fine;
19 unsigned long preset_lpj;
20
lpj_setup(char * str)21 static int __init lpj_setup(char *str)
22 {
23 return kstrtoul(str, 0, &preset_lpj) == 0;
24 }
25
26 __setup("lpj=", lpj_setup);
27
28 #ifdef ARCH_HAS_READ_CURRENT_TIMER
29
30 /* This routine uses the read_current_timer() routine and gets the
31 * loops per jiffy directly, instead of guessing it using delay().
32 * Also, this code tries to handle non-maskable asynchronous events
33 * (like SMIs)
34 */
35 #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100))
36 #define MAX_DIRECT_CALIBRATION_RETRIES 5
37
calibrate_delay_direct(void)38 static unsigned long calibrate_delay_direct(void)
39 {
40 unsigned long pre_start, start, post_start;
41 unsigned long pre_end, end, post_end;
42 unsigned long start_jiffies;
43 unsigned long timer_rate_min, timer_rate_max;
44 unsigned long good_timer_sum = 0;
45 unsigned long good_timer_count = 0;
46 unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES];
47 int max = -1; /* index of measured_times with max/min values or not set */
48 int min = -1;
49 int i;
50
51 if (read_current_timer(&pre_start) < 0 )
52 return 0;
53
54 /*
55 * A simple loop like
56 * while ( jiffies < start_jiffies+1)
57 * start = read_current_timer();
58 * will not do. As we don't really know whether jiffy switch
59 * happened first or timer_value was read first. And some asynchronous
60 * event can happen between these two events introducing errors in lpj.
61 *
62 * So, we do
63 * 1. pre_start <- When we are sure that jiffy switch hasn't happened
64 * 2. check jiffy switch
65 * 3. start <- timer value before or after jiffy switch
66 * 4. post_start <- When we are sure that jiffy switch has happened
67 *
68 * Note, we don't know anything about order of 2 and 3.
69 * Now, by looking at post_start and pre_start difference, we can
70 * check whether any asynchronous event happened or not
71 */
72
73 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
74 pre_start = 0;
75 read_current_timer(&start);
76 start_jiffies = jiffies;
77 while (time_before_eq(jiffies, start_jiffies + 1)) {
78 pre_start = start;
79 read_current_timer(&start);
80 }
81 read_current_timer(&post_start);
82
83 pre_end = 0;
84 end = post_start;
85 while (time_before_eq(jiffies, start_jiffies + 1 +
86 DELAY_CALIBRATION_TICKS)) {
87 pre_end = end;
88 read_current_timer(&end);
89 }
90 read_current_timer(&post_end);
91
92 timer_rate_max = (post_end - pre_start) /
93 DELAY_CALIBRATION_TICKS;
94 timer_rate_min = (pre_end - post_start) /
95 DELAY_CALIBRATION_TICKS;
96
97 /*
98 * If the upper limit and lower limit of the timer_rate is
99 * >= 12.5% apart, redo calibration.
100 */
101 if (start >= post_end)
102 printk(KERN_NOTICE "calibrate_delay_direct() ignoring "
103 "timer_rate as we had a TSC wrap around"
104 " start=%lu >=post_end=%lu\n",
105 start, post_end);
106 if (start < post_end && pre_start != 0 && pre_end != 0 &&
107 (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) {
108 good_timer_count++;
109 good_timer_sum += timer_rate_max;
110 measured_times[i] = timer_rate_max;
111 if (max < 0 || timer_rate_max > measured_times[max])
112 max = i;
113 if (min < 0 || timer_rate_max < measured_times[min])
114 min = i;
115 } else
116 measured_times[i] = 0;
117
118 }
119
120 /*
121 * Find the maximum & minimum - if they differ too much throw out the
122 * one with the largest difference from the mean and try again...
123 */
124 while (good_timer_count > 1) {
125 unsigned long estimate;
126 unsigned long maxdiff;
127
128 /* compute the estimate */
129 estimate = (good_timer_sum/good_timer_count);
130 maxdiff = estimate >> 3;
131
132 /* if range is within 12% let's take it */
133 if ((measured_times[max] - measured_times[min]) < maxdiff)
134 return estimate;
135
136 /* ok - drop the worse value and try again... */
137 good_timer_sum = 0;
138 good_timer_count = 0;
139 if ((measured_times[max] - estimate) <
140 (estimate - measured_times[min])) {
141 printk(KERN_NOTICE "calibrate_delay_direct() dropping "
142 "min bogoMips estimate %d = %lu\n",
143 min, measured_times[min]);
144 measured_times[min] = 0;
145 min = max;
146 } else {
147 printk(KERN_NOTICE "calibrate_delay_direct() dropping "
148 "max bogoMips estimate %d = %lu\n",
149 max, measured_times[max]);
150 measured_times[max] = 0;
151 max = min;
152 }
153
154 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
155 if (measured_times[i] == 0)
156 continue;
157 good_timer_count++;
158 good_timer_sum += measured_times[i];
159 if (measured_times[i] < measured_times[min])
160 min = i;
161 if (measured_times[i] > measured_times[max])
162 max = i;
163 }
164
165 }
166
167 printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good "
168 "estimate for loops_per_jiffy.\nProbably due to long platform "
169 "interrupts. Consider using \"lpj=\" boot option.\n");
170 return 0;
171 }
172 #else
calibrate_delay_direct(void)173 static unsigned long calibrate_delay_direct(void)
174 {
175 return 0;
176 }
177 #endif
178
179 /*
180 * This is the number of bits of precision for the loops_per_jiffy. Each
181 * time we refine our estimate after the first takes 1.5/HZ seconds, so try
182 * to start with a good estimate.
183 * For the boot cpu we can skip the delay calibration and assign it a value
184 * calculated based on the timer frequency.
185 * For the rest of the CPUs we cannot assume that the timer frequency is same as
186 * the cpu frequency, hence do the calibration for those.
187 */
188 #define LPS_PREC 8
189
calibrate_delay_converge(void)190 static unsigned long calibrate_delay_converge(void)
191 {
192 /* First stage - slowly accelerate to find initial bounds */
193 unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit;
194 int trials = 0, band = 0, trial_in_band = 0;
195
196 lpj = (1<<12);
197
198 /* wait for "start of" clock tick */
199 ticks = jiffies;
200 while (ticks == jiffies)
201 ; /* nothing */
202 /* Go .. */
203 ticks = jiffies;
204 do {
205 if (++trial_in_band == (1<<band)) {
206 ++band;
207 trial_in_band = 0;
208 }
209 __delay(lpj * band);
210 trials += band;
211 } while (ticks == jiffies);
212 /*
213 * We overshot, so retreat to a clear underestimate. Then estimate
214 * the largest likely undershoot. This defines our chop bounds.
215 */
216 trials -= band;
217 loopadd_base = lpj * band;
218 lpj_base = lpj * trials;
219
220 recalibrate:
221 lpj = lpj_base;
222 loopadd = loopadd_base;
223
224 /*
225 * Do a binary approximation to get lpj set to
226 * equal one clock (up to LPS_PREC bits)
227 */
228 chop_limit = lpj >> LPS_PREC;
229 while (loopadd > chop_limit) {
230 lpj += loopadd;
231 ticks = jiffies;
232 while (ticks == jiffies)
233 ; /* nothing */
234 ticks = jiffies;
235 __delay(lpj);
236 if (jiffies != ticks) /* longer than 1 tick */
237 lpj -= loopadd;
238 loopadd >>= 1;
239 }
240 /*
241 * If we incremented every single time possible, presume we've
242 * massively underestimated initially, and retry with a higher
243 * start, and larger range. (Only seen on x86_64, due to SMIs)
244 */
245 if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) {
246 lpj_base = lpj;
247 loopadd_base <<= 2;
248 goto recalibrate;
249 }
250
251 return lpj;
252 }
253
254 static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 };
255
256 /*
257 * Check if cpu calibration delay is already known. For example,
258 * some processors with multi-core sockets may have all cores
259 * with the same calibration delay.
260 *
261 * Architectures should override this function if a faster calibration
262 * method is available.
263 */
calibrate_delay_is_known(void)264 unsigned long __attribute__((weak)) calibrate_delay_is_known(void)
265 {
266 return 0;
267 }
268
269 /*
270 * Indicate the cpu delay calibration is done. This can be used by
271 * architectures to stop accepting delay timer registrations after this point.
272 */
273
calibration_delay_done(void)274 void __attribute__((weak)) calibration_delay_done(void)
275 {
276 }
277
calibrate_delay(void)278 void calibrate_delay(void)
279 {
280 unsigned long lpj;
281 static bool printed;
282 int this_cpu = smp_processor_id();
283
284 if (per_cpu(cpu_loops_per_jiffy, this_cpu)) {
285 lpj = per_cpu(cpu_loops_per_jiffy, this_cpu);
286 if (!printed)
287 pr_info("Calibrating delay loop (skipped) "
288 "already calibrated this CPU");
289 } else if (preset_lpj) {
290 lpj = preset_lpj;
291 if (!printed)
292 pr_info("Calibrating delay loop (skipped) "
293 "preset value.. ");
294 } else if ((!printed) && lpj_fine) {
295 lpj = lpj_fine;
296 pr_info("Calibrating delay loop (skipped), "
297 "value calculated using timer frequency.. ");
298 } else if ((lpj = calibrate_delay_is_known())) {
299 ;
300 } else if ((lpj = calibrate_delay_direct()) != 0) {
301 if (!printed)
302 pr_info("Calibrating delay using timer "
303 "specific routine.. ");
304 } else {
305 if (!printed)
306 pr_info("Calibrating delay loop... ");
307 lpj = calibrate_delay_converge();
308 }
309 per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj;
310 if (!printed)
311 pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n",
312 lpj/(500000/HZ),
313 (lpj/(5000/HZ)) % 100, lpj);
314
315 loops_per_jiffy = lpj;
316 printed = true;
317
318 calibration_delay_done();
319 }
320