xref: /freebsd/sys/x86/x86/tsc.c (revision a0e793cbf1951d07fc47a0d9ea389d7dacba5213)
1 /*-
2  * Copyright (c) 1998-2003 Poul-Henning Kamp
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include "opt_compat.h"
31 #include "opt_clock.h"
32 
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/cpu.h>
36 #include <sys/limits.h>
37 #include <sys/malloc.h>
38 #include <sys/systm.h>
39 #include <sys/sysctl.h>
40 #include <sys/time.h>
41 #include <sys/timetc.h>
42 #include <sys/kernel.h>
43 #include <sys/power.h>
44 #include <sys/smp.h>
45 #include <sys/vdso.h>
46 #include <machine/clock.h>
47 #include <machine/cputypes.h>
48 #include <machine/md_var.h>
49 #include <machine/specialreg.h>
50 
51 #include "cpufreq_if.h"
52 
53 uint64_t	tsc_freq;
54 int		tsc_is_invariant;
55 int		tsc_perf_stat;
56 
57 static eventhandler_tag tsc_levels_tag, tsc_pre_tag, tsc_post_tag;
58 
59 SYSCTL_INT(_kern_timecounter, OID_AUTO, invariant_tsc, CTLFLAG_RDTUN,
60     &tsc_is_invariant, 0, "Indicates whether the TSC is P-state invariant");
61 
62 #ifdef SMP
63 int	smp_tsc;
64 SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0,
65     "Indicates whether the TSC is safe to use in SMP mode");
66 
67 int	smp_tsc_adjust = 0;
68 SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc_adjust, CTLFLAG_RDTUN,
69     &smp_tsc_adjust, 0, "Try to adjust TSC on APs to match BSP");
70 #endif
71 
72 static int	tsc_shift = 1;
73 SYSCTL_INT(_kern_timecounter, OID_AUTO, tsc_shift, CTLFLAG_RDTUN,
74     &tsc_shift, 0, "Shift to pre-apply for the maximum TSC frequency");
75 
76 static int	tsc_disabled;
77 SYSCTL_INT(_machdep, OID_AUTO, disable_tsc, CTLFLAG_RDTUN, &tsc_disabled, 0,
78     "Disable x86 Time Stamp Counter");
79 
80 static int	tsc_skip_calibration;
81 SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN,
82     &tsc_skip_calibration, 0, "Disable TSC frequency calibration");
83 
84 static void tsc_freq_changed(void *arg, const struct cf_level *level,
85     int status);
86 static void tsc_freq_changing(void *arg, const struct cf_level *level,
87     int *status);
88 static unsigned tsc_get_timecount(struct timecounter *tc);
89 static inline unsigned tsc_get_timecount_low(struct timecounter *tc);
90 static unsigned tsc_get_timecount_lfence(struct timecounter *tc);
91 static unsigned tsc_get_timecount_low_lfence(struct timecounter *tc);
92 static unsigned tsc_get_timecount_mfence(struct timecounter *tc);
93 static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc);
94 static void tsc_levels_changed(void *arg, int unit);
95 
96 static struct timecounter tsc_timecounter = {
97 	tsc_get_timecount,	/* get_timecount */
98 	0,			/* no poll_pps */
99 	~0u,			/* counter_mask */
100 	0,			/* frequency */
101 	"TSC",			/* name */
102 	800,			/* quality (adjusted in code) */
103 };
104 
105 #define	VMW_HVMAGIC		0x564d5868
106 #define	VMW_HVPORT		0x5658
107 #define	VMW_HVCMD_GETVERSION	10
108 #define	VMW_HVCMD_GETHZ		45
109 
110 static __inline void
111 vmware_hvcall(u_int cmd, u_int *p)
112 {
113 
114 	__asm __volatile("inl %w3, %0"
115 	: "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
116 	: "0" (VMW_HVMAGIC), "1" (UINT_MAX), "2" (cmd), "3" (VMW_HVPORT)
117 	: "memory");
118 }
119 
120 static int
121 tsc_freq_vmware(void)
122 {
123 	char hv_sig[13];
124 	u_int regs[4];
125 	char *p;
126 	u_int hv_high;
127 	int i;
128 
129 	/*
130 	 * [RFC] CPUID usage for interaction between Hypervisors and Linux.
131 	 * http://lkml.org/lkml/2008/10/1/246
132 	 *
133 	 * KB1009458: Mechanisms to determine if software is running in
134 	 * a VMware virtual machine
135 	 * http://kb.vmware.com/kb/1009458
136 	 */
137 	hv_high = 0;
138 	if ((cpu_feature2 & CPUID2_HV) != 0) {
139 		do_cpuid(0x40000000, regs);
140 		hv_high = regs[0];
141 		for (i = 1, p = hv_sig; i < 4; i++, p += sizeof(regs) / 4)
142 			memcpy(p, &regs[i], sizeof(regs[i]));
143 		*p = '\0';
144 		if (bootverbose) {
145 			/*
146 			 * HV vendor	ID string
147 			 * ------------+--------------
148 			 * KVM		"KVMKVMKVM"
149 			 * Microsoft	"Microsoft Hv"
150 			 * VMware	"VMwareVMware"
151 			 * Xen		"XenVMMXenVMM"
152 			 */
153 			printf("Hypervisor: Origin = \"%s\"\n", hv_sig);
154 		}
155 		if (strncmp(hv_sig, "VMwareVMware", 12) != 0)
156 			return (0);
157 	} else {
158 		p = getenv("smbios.system.serial");
159 		if (p == NULL)
160 			return (0);
161 		if (strncmp(p, "VMware-", 7) != 0 &&
162 		    strncmp(p, "VMW", 3) != 0) {
163 			freeenv(p);
164 			return (0);
165 		}
166 		freeenv(p);
167 		vmware_hvcall(VMW_HVCMD_GETVERSION, regs);
168 		if (regs[1] != VMW_HVMAGIC)
169 			return (0);
170 	}
171 	if (hv_high >= 0x40000010) {
172 		do_cpuid(0x40000010, regs);
173 		tsc_freq = regs[0] * 1000;
174 	} else {
175 		vmware_hvcall(VMW_HVCMD_GETHZ, regs);
176 		if (regs[1] != UINT_MAX)
177 			tsc_freq = regs[0] | ((uint64_t)regs[1] << 32);
178 	}
179 	tsc_is_invariant = 1;
180 	return (1);
181 }
182 
183 static void
184 tsc_freq_intel(void)
185 {
186 	char brand[48];
187 	u_int regs[4];
188 	uint64_t freq;
189 	char *p;
190 	u_int i;
191 
192 	/*
193 	 * Intel Processor Identification and the CPUID Instruction
194 	 * Application Note 485.
195 	 * http://www.intel.com/assets/pdf/appnote/241618.pdf
196 	 */
197 	if (cpu_exthigh >= 0x80000004) {
198 		p = brand;
199 		for (i = 0x80000002; i < 0x80000005; i++) {
200 			do_cpuid(i, regs);
201 			memcpy(p, regs, sizeof(regs));
202 			p += sizeof(regs);
203 		}
204 		p = NULL;
205 		for (i = 0; i < sizeof(brand) - 1; i++)
206 			if (brand[i] == 'H' && brand[i + 1] == 'z')
207 				p = brand + i;
208 		if (p != NULL) {
209 			p -= 5;
210 			switch (p[4]) {
211 			case 'M':
212 				i = 1;
213 				break;
214 			case 'G':
215 				i = 1000;
216 				break;
217 			case 'T':
218 				i = 1000000;
219 				break;
220 			default:
221 				return;
222 			}
223 #define	C2D(c)	((c) - '0')
224 			if (p[1] == '.') {
225 				freq = C2D(p[0]) * 1000;
226 				freq += C2D(p[2]) * 100;
227 				freq += C2D(p[3]) * 10;
228 				freq *= i * 1000;
229 			} else {
230 				freq = C2D(p[0]) * 1000;
231 				freq += C2D(p[1]) * 100;
232 				freq += C2D(p[2]) * 10;
233 				freq += C2D(p[3]);
234 				freq *= i * 1000000;
235 			}
236 #undef C2D
237 			tsc_freq = freq;
238 		}
239 	}
240 }
241 
242 static void
243 probe_tsc_freq(void)
244 {
245 	u_int regs[4];
246 	uint64_t tsc1, tsc2;
247 
248 	if (cpu_high >= 6) {
249 		do_cpuid(6, regs);
250 		if ((regs[2] & CPUID_PERF_STAT) != 0) {
251 			/*
252 			 * XXX Some emulators expose host CPUID without actual
253 			 * support for these MSRs.  We must test whether they
254 			 * really work.
255 			 */
256 			wrmsr(MSR_MPERF, 0);
257 			wrmsr(MSR_APERF, 0);
258 			DELAY(10);
259 			if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0)
260 				tsc_perf_stat = 1;
261 		}
262 	}
263 
264 	if (tsc_freq_vmware())
265 		return;
266 
267 	switch (cpu_vendor_id) {
268 	case CPU_VENDOR_AMD:
269 		if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 ||
270 		    (vm_guest == VM_GUEST_NO &&
271 		    CPUID_TO_FAMILY(cpu_id) >= 0x10))
272 			tsc_is_invariant = 1;
273 		if (cpu_feature & CPUID_SSE2) {
274 			tsc_timecounter.tc_get_timecount =
275 			    tsc_get_timecount_mfence;
276 		}
277 		break;
278 	case CPU_VENDOR_INTEL:
279 		if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 ||
280 		    (vm_guest == VM_GUEST_NO &&
281 		    ((CPUID_TO_FAMILY(cpu_id) == 0x6 &&
282 		    CPUID_TO_MODEL(cpu_id) >= 0xe) ||
283 		    (CPUID_TO_FAMILY(cpu_id) == 0xf &&
284 		    CPUID_TO_MODEL(cpu_id) >= 0x3))))
285 			tsc_is_invariant = 1;
286 		if (cpu_feature & CPUID_SSE2) {
287 			tsc_timecounter.tc_get_timecount =
288 			    tsc_get_timecount_lfence;
289 		}
290 		break;
291 	case CPU_VENDOR_CENTAUR:
292 		if (vm_guest == VM_GUEST_NO &&
293 		    CPUID_TO_FAMILY(cpu_id) == 0x6 &&
294 		    CPUID_TO_MODEL(cpu_id) >= 0xf &&
295 		    (rdmsr(0x1203) & 0x100000000ULL) == 0)
296 			tsc_is_invariant = 1;
297 		if (cpu_feature & CPUID_SSE2) {
298 			tsc_timecounter.tc_get_timecount =
299 			    tsc_get_timecount_lfence;
300 		}
301 		break;
302 	}
303 
304 	if (tsc_skip_calibration) {
305 		if (cpu_vendor_id == CPU_VENDOR_INTEL)
306 			tsc_freq_intel();
307 		return;
308 	}
309 
310 	if (bootverbose)
311 	        printf("Calibrating TSC clock ... ");
312 	tsc1 = rdtsc();
313 	DELAY(1000000);
314 	tsc2 = rdtsc();
315 	tsc_freq = tsc2 - tsc1;
316 	if (bootverbose)
317 		printf("TSC clock: %ju Hz\n", (intmax_t)tsc_freq);
318 }
319 
320 void
321 init_TSC(void)
322 {
323 
324 	if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
325 		return;
326 
327 	probe_tsc_freq();
328 
329 	/*
330 	 * Inform CPU accounting about our boot-time clock rate.  This will
331 	 * be updated if someone loads a cpufreq driver after boot that
332 	 * discovers a new max frequency.
333 	 */
334 	if (tsc_freq != 0)
335 		set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant);
336 
337 	if (tsc_is_invariant)
338 		return;
339 
340 	/* Register to find out about changes in CPU frequency. */
341 	tsc_pre_tag = EVENTHANDLER_REGISTER(cpufreq_pre_change,
342 	    tsc_freq_changing, NULL, EVENTHANDLER_PRI_FIRST);
343 	tsc_post_tag = EVENTHANDLER_REGISTER(cpufreq_post_change,
344 	    tsc_freq_changed, NULL, EVENTHANDLER_PRI_FIRST);
345 	tsc_levels_tag = EVENTHANDLER_REGISTER(cpufreq_levels_changed,
346 	    tsc_levels_changed, NULL, EVENTHANDLER_PRI_ANY);
347 }
348 
349 #ifdef SMP
350 
351 /*
352  * RDTSC is not a serializing instruction, and does not drain
353  * instruction stream, so we need to drain the stream before executing
354  * it.  It could be fixed by use of RDTSCP, except the instruction is
355  * not available everywhere.
356  *
357  * Use CPUID for draining in the boot-time SMP constistency test.  The
358  * timecounters use MFENCE for AMD CPUs, and LFENCE for others (Intel
359  * and VIA) when SSE2 is present, and nothing on older machines which
360  * also do not issue RDTSC prematurely.  There, testing for SSE2 and
361  * vendor is too cumbersome, and we learn about TSC presence from CPUID.
362  *
363  * Do not use do_cpuid(), since we do not need CPUID results, which
364  * have to be written into memory with do_cpuid().
365  */
366 #define	TSC_READ(x)							\
367 static void								\
368 tsc_read_##x(void *arg)							\
369 {									\
370 	uint64_t *tsc = arg;						\
371 	u_int cpu = PCPU_GET(cpuid);					\
372 									\
373 	__asm __volatile("cpuid" : : : "eax", "ebx", "ecx", "edx");	\
374 	tsc[cpu * 3 + x] = rdtsc();					\
375 }
376 TSC_READ(0)
377 TSC_READ(1)
378 TSC_READ(2)
379 #undef TSC_READ
380 
381 #define	N	1000
382 
383 static void
384 comp_smp_tsc(void *arg)
385 {
386 	uint64_t *tsc;
387 	int64_t d1, d2;
388 	u_int cpu = PCPU_GET(cpuid);
389 	u_int i, j, size;
390 
391 	size = (mp_maxid + 1) * 3;
392 	for (i = 0, tsc = arg; i < N; i++, tsc += size)
393 		CPU_FOREACH(j) {
394 			if (j == cpu)
395 				continue;
396 			d1 = tsc[cpu * 3 + 1] - tsc[j * 3];
397 			d2 = tsc[cpu * 3 + 2] - tsc[j * 3 + 1];
398 			if (d1 <= 0 || d2 <= 0) {
399 				smp_tsc = 0;
400 				return;
401 			}
402 		}
403 }
404 
405 static void
406 adj_smp_tsc(void *arg)
407 {
408 	uint64_t *tsc;
409 	int64_t d, min, max;
410 	u_int cpu = PCPU_GET(cpuid);
411 	u_int first, i, size;
412 
413 	first = CPU_FIRST();
414 	if (cpu == first)
415 		return;
416 	min = INT64_MIN;
417 	max = INT64_MAX;
418 	size = (mp_maxid + 1) * 3;
419 	for (i = 0, tsc = arg; i < N; i++, tsc += size) {
420 		d = tsc[first * 3] - tsc[cpu * 3 + 1];
421 		if (d > min)
422 			min = d;
423 		d = tsc[first * 3 + 1] - tsc[cpu * 3 + 2];
424 		if (d > min)
425 			min = d;
426 		d = tsc[first * 3 + 1] - tsc[cpu * 3];
427 		if (d < max)
428 			max = d;
429 		d = tsc[first * 3 + 2] - tsc[cpu * 3 + 1];
430 		if (d < max)
431 			max = d;
432 	}
433 	if (min > max)
434 		return;
435 	d = min / 2 + max / 2;
436 	__asm __volatile (
437 		"movl $0x10, %%ecx\n\t"
438 		"rdmsr\n\t"
439 		"addl %%edi, %%eax\n\t"
440 		"adcl %%esi, %%edx\n\t"
441 		"wrmsr\n"
442 		: /* No output */
443 		: "D" ((uint32_t)d), "S" ((uint32_t)(d >> 32))
444 		: "ax", "cx", "dx", "cc"
445 	);
446 }
447 
448 static int
449 test_tsc(void)
450 {
451 	uint64_t *data, *tsc;
452 	u_int i, size, adj;
453 
454 	if ((!smp_tsc && !tsc_is_invariant) || vm_guest)
455 		return (-100);
456 	size = (mp_maxid + 1) * 3;
457 	data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK);
458 	adj = 0;
459 retry:
460 	for (i = 0, tsc = data; i < N; i++, tsc += size)
461 		smp_rendezvous(tsc_read_0, tsc_read_1, tsc_read_2, tsc);
462 	smp_tsc = 1;	/* XXX */
463 	smp_rendezvous(smp_no_rendevous_barrier, comp_smp_tsc,
464 	    smp_no_rendevous_barrier, data);
465 	if (!smp_tsc && adj < smp_tsc_adjust) {
466 		adj++;
467 		smp_rendezvous(smp_no_rendevous_barrier, adj_smp_tsc,
468 		    smp_no_rendevous_barrier, data);
469 		goto retry;
470 	}
471 	free(data, M_TEMP);
472 	if (bootverbose)
473 		printf("SMP: %sed TSC synchronization test%s\n",
474 		    smp_tsc ? "pass" : "fail",
475 		    adj > 0 ? " after adjustment" : "");
476 	if (smp_tsc && tsc_is_invariant) {
477 		switch (cpu_vendor_id) {
478 		case CPU_VENDOR_AMD:
479 			/*
480 			 * Starting with Family 15h processors, TSC clock
481 			 * source is in the north bridge.  Check whether
482 			 * we have a single-socket/multi-core platform.
483 			 * XXX Need more work for complex cases.
484 			 */
485 			if (CPUID_TO_FAMILY(cpu_id) < 0x15 ||
486 			    (amd_feature2 & AMDID2_CMP) == 0 ||
487 			    smp_cpus > (cpu_procinfo2 & AMDID_CMP_CORES) + 1)
488 				break;
489 			return (1000);
490 		case CPU_VENDOR_INTEL:
491 			/*
492 			 * XXX Assume Intel platforms have synchronized TSCs.
493 			 */
494 			return (1000);
495 		}
496 		return (800);
497 	}
498 	return (-100);
499 }
500 
501 #undef N
502 
503 #else
504 
505 /*
506  * The function is not called, it is provided to avoid linking failure
507  * on uniprocessor kernel.
508  */
509 static int
510 test_tsc(void)
511 {
512 
513 	return (0);
514 }
515 
516 #endif /* SMP */
517 
518 static void
519 init_TSC_tc(void)
520 {
521 	uint64_t max_freq;
522 	int shift;
523 
524 	if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled)
525 		return;
526 
527 	/*
528 	 * Limit timecounter frequency to fit in an int and prevent it from
529 	 * overflowing too fast.
530 	 */
531 	max_freq = UINT_MAX;
532 
533 	/*
534 	 * We can not use the TSC if we support APM.  Precise timekeeping
535 	 * on an APM'ed machine is at best a fools pursuit, since
536 	 * any and all of the time spent in various SMM code can't
537 	 * be reliably accounted for.  Reading the RTC is your only
538 	 * source of reliable time info.  The i8254 loses too, of course,
539 	 * but we need to have some kind of time...
540 	 * We don't know at this point whether APM is going to be used
541 	 * or not, nor when it might be activated.  Play it safe.
542 	 */
543 	if (power_pm_get_type() == POWER_PM_TYPE_APM) {
544 		tsc_timecounter.tc_quality = -1000;
545 		if (bootverbose)
546 			printf("TSC timecounter disabled: APM enabled.\n");
547 		goto init;
548 	}
549 
550 	/*
551 	 * We cannot use the TSC if it stops incrementing in deep sleep.
552 	 * Currently only Intel CPUs are known for this problem unless
553 	 * the invariant TSC bit is set.
554 	 */
555 	if (cpu_can_deep_sleep && cpu_vendor_id == CPU_VENDOR_INTEL &&
556 	    (amd_pminfo & AMDPM_TSC_INVARIANT) == 0) {
557 		tsc_timecounter.tc_quality = -1000;
558 		tsc_timecounter.tc_flags |= TC_FLAGS_C3STOP;
559 		if (bootverbose)
560 			printf("TSC timecounter disabled: C3 enabled.\n");
561 		goto init;
562 	}
563 
564 	/*
565 	 * We can not use the TSC in SMP mode unless the TSCs on all CPUs
566 	 * are synchronized.  If the user is sure that the system has
567 	 * synchronized TSCs, set kern.timecounter.smp_tsc tunable to a
568 	 * non-zero value.  The TSC seems unreliable in virtualized SMP
569 	 * environments, so it is set to a negative quality in those cases.
570 	 */
571 	if (mp_ncpus > 1)
572 		tsc_timecounter.tc_quality = test_tsc();
573 	else if (tsc_is_invariant)
574 		tsc_timecounter.tc_quality = 1000;
575 	max_freq >>= tsc_shift;
576 
577 init:
578 	for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++)
579 		;
580 	if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
581 		if (cpu_vendor_id == CPU_VENDOR_AMD) {
582 			tsc_timecounter.tc_get_timecount = shift > 0 ?
583 			    tsc_get_timecount_low_mfence :
584 			    tsc_get_timecount_mfence;
585 		} else {
586 			tsc_timecounter.tc_get_timecount = shift > 0 ?
587 			    tsc_get_timecount_low_lfence :
588 			    tsc_get_timecount_lfence;
589 		}
590 	} else {
591 		tsc_timecounter.tc_get_timecount = shift > 0 ?
592 		    tsc_get_timecount_low : tsc_get_timecount;
593 	}
594 	if (shift > 0) {
595 		tsc_timecounter.tc_name = "TSC-low";
596 		if (bootverbose)
597 			printf("TSC timecounter discards lower %d bit(s)\n",
598 			    shift);
599 	}
600 	if (tsc_freq != 0) {
601 		tsc_timecounter.tc_frequency = tsc_freq >> shift;
602 		tsc_timecounter.tc_priv = (void *)(intptr_t)shift;
603 		tc_init(&tsc_timecounter);
604 	}
605 }
606 SYSINIT(tsc_tc, SI_SUB_SMP, SI_ORDER_ANY, init_TSC_tc, NULL);
607 
608 /*
609  * When cpufreq levels change, find out about the (new) max frequency.  We
610  * use this to update CPU accounting in case it got a lower estimate at boot.
611  */
612 static void
613 tsc_levels_changed(void *arg, int unit)
614 {
615 	device_t cf_dev;
616 	struct cf_level *levels;
617 	int count, error;
618 	uint64_t max_freq;
619 
620 	/* Only use values from the first CPU, assuming all are equal. */
621 	if (unit != 0)
622 		return;
623 
624 	/* Find the appropriate cpufreq device instance. */
625 	cf_dev = devclass_get_device(devclass_find("cpufreq"), unit);
626 	if (cf_dev == NULL) {
627 		printf("tsc_levels_changed() called but no cpufreq device?\n");
628 		return;
629 	}
630 
631 	/* Get settings from the device and find the max frequency. */
632 	count = 64;
633 	levels = malloc(count * sizeof(*levels), M_TEMP, M_NOWAIT);
634 	if (levels == NULL)
635 		return;
636 	error = CPUFREQ_LEVELS(cf_dev, levels, &count);
637 	if (error == 0 && count != 0) {
638 		max_freq = (uint64_t)levels[0].total_set.freq * 1000000;
639 		set_cputicker(rdtsc, max_freq, 1);
640 	} else
641 		printf("tsc_levels_changed: no max freq found\n");
642 	free(levels, M_TEMP);
643 }
644 
645 /*
646  * If the TSC timecounter is in use, veto the pending change.  It may be
647  * possible in the future to handle a dynamically-changing timecounter rate.
648  */
649 static void
650 tsc_freq_changing(void *arg, const struct cf_level *level, int *status)
651 {
652 
653 	if (*status != 0 || timecounter != &tsc_timecounter)
654 		return;
655 
656 	printf("timecounter TSC must not be in use when "
657 	    "changing frequencies; change denied\n");
658 	*status = EBUSY;
659 }
660 
661 /* Update TSC freq with the value indicated by the caller. */
662 static void
663 tsc_freq_changed(void *arg, const struct cf_level *level, int status)
664 {
665 	uint64_t freq;
666 
667 	/* If there was an error during the transition, don't do anything. */
668 	if (tsc_disabled || status != 0)
669 		return;
670 
671 	/* Total setting for this level gives the new frequency in MHz. */
672 	freq = (uint64_t)level->total_set.freq * 1000000;
673 	atomic_store_rel_64(&tsc_freq, freq);
674 	tsc_timecounter.tc_frequency =
675 	    freq >> (int)(intptr_t)tsc_timecounter.tc_priv;
676 }
677 
678 static int
679 sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS)
680 {
681 	int error;
682 	uint64_t freq;
683 
684 	freq = atomic_load_acq_64(&tsc_freq);
685 	if (freq == 0)
686 		return (EOPNOTSUPP);
687 	error = sysctl_handle_64(oidp, &freq, 0, req);
688 	if (error == 0 && req->newptr != NULL) {
689 		atomic_store_rel_64(&tsc_freq, freq);
690 		atomic_store_rel_64(&tsc_timecounter.tc_frequency,
691 		    freq >> (int)(intptr_t)tsc_timecounter.tc_priv);
692 	}
693 	return (error);
694 }
695 
696 SYSCTL_PROC(_machdep, OID_AUTO, tsc_freq, CTLTYPE_U64 | CTLFLAG_RW,
697     0, 0, sysctl_machdep_tsc_freq, "QU", "Time Stamp Counter frequency");
698 
699 static u_int
700 tsc_get_timecount(struct timecounter *tc __unused)
701 {
702 
703 	return (rdtsc32());
704 }
705 
706 static inline u_int
707 tsc_get_timecount_low(struct timecounter *tc)
708 {
709 	uint32_t rv;
710 
711 	__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
712 	    : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx");
713 	return (rv);
714 }
715 
716 static u_int
717 tsc_get_timecount_lfence(struct timecounter *tc __unused)
718 {
719 
720 	lfence();
721 	return (rdtsc32());
722 }
723 
724 static u_int
725 tsc_get_timecount_low_lfence(struct timecounter *tc)
726 {
727 
728 	lfence();
729 	return (tsc_get_timecount_low(tc));
730 }
731 
732 static u_int
733 tsc_get_timecount_mfence(struct timecounter *tc __unused)
734 {
735 
736 	mfence();
737 	return (rdtsc32());
738 }
739 
740 static u_int
741 tsc_get_timecount_low_mfence(struct timecounter *tc)
742 {
743 
744 	mfence();
745 	return (tsc_get_timecount_low(tc));
746 }
747 
748 uint32_t
749 cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
750 {
751 
752 	vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
753 	bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
754 	return (timecounter == &tsc_timecounter);
755 }
756 
757 #ifdef COMPAT_FREEBSD32
758 uint32_t
759 cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
760 {
761 
762 	vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
763 	bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res));
764 	return (timecounter == &tsc_timecounter);
765 }
766 #endif
767