1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_CPUID_API_H
3 #define _ASM_X86_CPUID_API_H
4
5 #include <asm/cpuid/types.h>
6
7 #include <linux/build_bug.h>
8 #include <linux/types.h>
9
10 #include <asm/processor.h>
11 #include <asm/string.h>
12
13 /*
14 * Raw CPUID accessors:
15 */
16
17 #ifdef CONFIG_X86_32
18 bool cpuid_feature(void);
19 #else
cpuid_feature(void)20 static inline bool cpuid_feature(void)
21 {
22 return true;
23 }
24 #endif
25
native_cpuid(u32 * eax,u32 * ebx,u32 * ecx,u32 * edx)26 static inline void native_cpuid(u32 *eax, u32 *ebx,
27 u32 *ecx, u32 *edx)
28 {
29 /* ecx is often an input as well as an output. */
30 asm volatile("cpuid"
31 : "=a" (*eax),
32 "=b" (*ebx),
33 "=c" (*ecx),
34 "=d" (*edx)
35 : "0" (*eax), "2" (*ecx)
36 : "memory");
37 }
38
39 #define NATIVE_CPUID_REG(reg) \
40 static inline u32 native_cpuid_##reg(u32 op) \
41 { \
42 u32 eax = op, ebx, ecx = 0, edx; \
43 \
44 native_cpuid(&eax, &ebx, &ecx, &edx); \
45 \
46 return reg; \
47 }
48
49 /*
50 * Native CPUID functions returning a single datum:
51 */
52 NATIVE_CPUID_REG(eax)
NATIVE_CPUID_REG(ebx)53 NATIVE_CPUID_REG(ebx)
54 NATIVE_CPUID_REG(ecx)
55 NATIVE_CPUID_REG(edx)
56
57 #ifdef CONFIG_PARAVIRT_XXL
58 # include <asm/paravirt.h>
59 #else
60 # define __cpuid native_cpuid
61 #endif
62
63 /*
64 * Generic CPUID function
65 *
66 * Clear ECX since some CPUs (Cyrix MII) do not set or clear ECX
67 * resulting in stale register contents being returned.
68 */
69 static inline void cpuid(u32 op,
70 u32 *eax, u32 *ebx,
71 u32 *ecx, u32 *edx)
72 {
73 *eax = op;
74 *ecx = 0;
75 __cpuid(eax, ebx, ecx, edx);
76 }
77
78 /* Some CPUID calls want 'count' to be placed in ECX */
cpuid_count(u32 op,int count,u32 * eax,u32 * ebx,u32 * ecx,u32 * edx)79 static inline void cpuid_count(u32 op, int count,
80 u32 *eax, u32 *ebx,
81 u32 *ecx, u32 *edx)
82 {
83 *eax = op;
84 *ecx = count;
85 __cpuid(eax, ebx, ecx, edx);
86 }
87
88 /*
89 * CPUID functions returning a single datum:
90 */
91
cpuid_eax(u32 op)92 static inline u32 cpuid_eax(u32 op)
93 {
94 u32 eax, ebx, ecx, edx;
95
96 cpuid(op, &eax, &ebx, &ecx, &edx);
97
98 return eax;
99 }
100
cpuid_ebx(u32 op)101 static inline u32 cpuid_ebx(u32 op)
102 {
103 u32 eax, ebx, ecx, edx;
104
105 cpuid(op, &eax, &ebx, &ecx, &edx);
106
107 return ebx;
108 }
109
cpuid_ecx(u32 op)110 static inline u32 cpuid_ecx(u32 op)
111 {
112 u32 eax, ebx, ecx, edx;
113
114 cpuid(op, &eax, &ebx, &ecx, &edx);
115
116 return ecx;
117 }
118
cpuid_edx(u32 op)119 static inline u32 cpuid_edx(u32 op)
120 {
121 u32 eax, ebx, ecx, edx;
122
123 cpuid(op, &eax, &ebx, &ecx, &edx);
124
125 return edx;
126 }
127
__cpuid_read(u32 leaf,u32 subleaf,u32 * regs)128 static inline void __cpuid_read(u32 leaf, u32 subleaf, u32 *regs)
129 {
130 regs[CPUID_EAX] = leaf;
131 regs[CPUID_ECX] = subleaf;
132 __cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX);
133 }
134
135 #define cpuid_read_subleaf(leaf, subleaf, regs) { \
136 static_assert(sizeof(*(regs)) == 16); \
137 __cpuid_read(leaf, subleaf, (u32 *)(regs)); \
138 }
139
140 #define cpuid_read(leaf, regs) { \
141 static_assert(sizeof(*(regs)) == 16); \
142 __cpuid_read(leaf, 0, (u32 *)(regs)); \
143 }
144
__cpuid_read_reg(u32 leaf,u32 subleaf,enum cpuid_regs_idx regidx,u32 * reg)145 static inline void __cpuid_read_reg(u32 leaf, u32 subleaf,
146 enum cpuid_regs_idx regidx, u32 *reg)
147 {
148 u32 regs[4];
149
150 __cpuid_read(leaf, subleaf, regs);
151 *reg = regs[regidx];
152 }
153
154 #define cpuid_subleaf_reg(leaf, subleaf, regidx, reg) { \
155 static_assert(sizeof(*(reg)) == 4); \
156 __cpuid_read_reg(leaf, subleaf, regidx, (u32 *)(reg)); \
157 }
158
159 #define cpuid_leaf_reg(leaf, regidx, reg) { \
160 static_assert(sizeof(*(reg)) == 4); \
161 __cpuid_read_reg(leaf, 0, regidx, (u32 *)(reg)); \
162 }
163
164 /*
165 * Hypervisor-related APIs:
166 */
167
cpuid_function_is_indexed(u32 function)168 static __always_inline bool cpuid_function_is_indexed(u32 function)
169 {
170 switch (function) {
171 case 4:
172 case 7:
173 case 0xb:
174 case 0xd:
175 case 0xf:
176 case 0x10:
177 case 0x12:
178 case 0x14:
179 case 0x17:
180 case 0x18:
181 case 0x1d:
182 case 0x1e:
183 case 0x1f:
184 case 0x24:
185 case 0x8000001d:
186 return true;
187 }
188
189 return false;
190 }
191
192 #define for_each_possible_cpuid_base_hypervisor(function) \
193 for (function = 0x40000000; function < 0x40010000; function += 0x100)
194
cpuid_base_hypervisor(const char * sig,u32 leaves)195 static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves)
196 {
197 u32 base, eax, signature[3];
198
199 for_each_possible_cpuid_base_hypervisor(base) {
200 cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);
201
202 /*
203 * This must not compile to "call memcmp" because it's called
204 * from PVH early boot code before instrumentation is set up
205 * and memcmp() itself may be instrumented.
206 */
207 if (!__inline_memcmp(sig, signature, 12) &&
208 (leaves == 0 || ((eax - base) >= leaves)))
209 return base;
210 }
211
212 return 0;
213 }
214
215 /*
216 * CPUID(0x2) parsing:
217 */
218
219 /**
220 * cpuid_leaf_0x2() - Return sanitized CPUID(0x2) register output
221 * @regs: Output parameter
222 *
223 * Query CPUID(0x2) and store its output in @regs. Force set any
224 * invalid 1-byte descriptor returned by the hardware to zero (the NULL
225 * cache/TLB descriptor) before returning it to the caller.
226 *
227 * Use for_each_cpuid_0x2_desc() to iterate over the register output in
228 * parsed form.
229 */
cpuid_leaf_0x2(union leaf_0x2_regs * regs)230 static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs)
231 {
232 cpuid_read(0x2, regs);
233
234 /*
235 * All Intel CPUs must report an iteration count of 1. In case
236 * of bogus hardware, treat all returned descriptors as NULL.
237 */
238 if (regs->desc[0] != 0x01) {
239 for (int i = 0; i < 4; i++)
240 regs->regv[i] = 0;
241 return;
242 }
243
244 /*
245 * The most significant bit (MSB) of each register must be clear.
246 * If a register is invalid, replace its descriptors with NULL.
247 */
248 for (int i = 0; i < 4; i++) {
249 if (regs->reg[i].invalid)
250 regs->regv[i] = 0;
251 }
252 }
253
254 /**
255 * for_each_cpuid_0x2_desc() - Iterator for parsed CPUID(0x2) descriptors
256 * @_regs: CPUID(0x2) register output, as returned by cpuid_leaf_0x2()
257 * @_ptr: u8 pointer, for macro internal use only
258 * @_desc: Pointer to the parsed CPUID(0x2) descriptor at each iteration
259 *
260 * Loop over the 1-byte descriptors in the passed CPUID(0x2) output registers
261 * @_regs. Provide the parsed information for each descriptor through @_desc.
262 *
263 * To handle cache-specific descriptors, switch on @_desc->c_type. For TLB
264 * descriptors, switch on @_desc->t_type.
265 *
266 * Example usage for cache descriptors::
267 *
268 * const struct leaf_0x2_table *desc;
269 * union leaf_0x2_regs regs;
270 * u8 *ptr;
271 *
272 * cpuid_leaf_0x2(®s);
273 * for_each_cpuid_0x2_desc(regs, ptr, desc) {
274 * switch (desc->c_type) {
275 * ...
276 * }
277 * }
278 */
279 #define for_each_cpuid_0x2_desc(_regs, _ptr, _desc) \
280 for (_ptr = &(_regs).desc[1]; \
281 _ptr < &(_regs).desc[16] && (_desc = &cpuid_0x2_table[*_ptr]); \
282 _ptr++)
283
284 /*
285 * CPUID(0x80000006) parsing:
286 */
287
cpuid_amd_hygon_has_l3_cache(void)288 static inline bool cpuid_amd_hygon_has_l3_cache(void)
289 {
290 return cpuid_edx(0x80000006);
291 }
292
293 /*
294 * 'struct cpuid_leaves' accessors (without sanity checks):
295 *
296 * For internal use by the CPUID parser.
297 */
298
299 /* Return constified pointers for all call-site APIs */
300 #define __const_ptr(_ptr) \
301 ((const __typeof__(*(_ptr)) *)(_ptr))
302
303 #define __cpuid_leaves_subleaf(_leaves, _leaf, _subleaf) \
304 __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf)[0])
305
306 #define __cpuid_leaves_subleaf_n(_leaves, _leaf, _index) \
307 __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## n)[_index])
308
309 #define __cpuid_leaves_subleaf_info(_leaves, _leaf, _subleaf) \
310 __const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf ## _ ## info))
311
312 /*
313 * 'struct cpuid_table' accessors (with sanity checks):
314 *
315 * For internal use by the CPUID parser.
316 */
317
318 #define __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf) \
319 __cpuid_leaves_subleaf_info(&((_table)->leaves), _leaf, _subleaf)->nr_entries
320
321 #define __cpuid_table_subleaf_range_size(_table, _leaf) \
322 ARRAY_SIZE((_table)->leaves.leaf_ ## _leaf ## _n)
323
324 #define __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf) \
325 (((_subleaf) < (__cpuid_leaf_first_subleaf(_leaf))) || \
326 ((_subleaf) > (__cpuid_leaf_first_subleaf(_leaf) + \
327 __cpuid_table_subleaf_range_size(_table, _leaf) - 1)))
328
329 /* Return NULL if the parser did not fill that leaf. Check cpuid_subleaf(). */
330 #define __cpuid_table_subleaf(_table, _leaf, _subleaf) \
331 ({ \
332 unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf); \
333 \
334 (____f != 1) ? NULL : __cpuid_leaves_subleaf(&((_table)->leaves), _leaf, _subleaf); \
335 })
336
337 /*
338 * Return NULL if the CPUID parser did not fill this leaf, or if the given
339 * dynamic subleaf value is out of range. Check cpuid_subleaf_n().
340 */
341 #define __cpuid_table_subleaf_n(_table, _leaf, _subleaf) \
342 ({ \
343 unsigned int ____i = (_subleaf) - __cpuid_leaf_first_subleaf(_leaf); \
344 unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, n); \
345 \
346 /* CPUID parser might not have filled the entire subleaf range */ \
347 ((____i >= ____f) || __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf)) ? \
348 NULL : __cpuid_leaves_subleaf_n(&((_table)->leaves), _leaf, ____i); \
349 })
350
351 /*
352 * Compile-time checks for leaves with a subleaf range:
353 */
354
355 #define __cpuid_assert_subleaf_range(_cpuinfo, _leaf) \
356 static_assert(__cpuid_table_subleaf_range_size(&(_cpuinfo)->cpuid, _leaf) > 1)
357
358 #define __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf) \
359 BUILD_BUG_ON(__builtin_constant_p(_subleaf) && \
360 __cpuid_table_invalid_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf))
361
362 /*
363 * CPUID Parser Call-site APIs
364 *
365 * Call sites should use below APIs instead of invoking direct CPUID queries.
366 *
367 * Benefits include:
368 *
369 * - Return CPUID output as typed C structures that are auto-generated from a
370 * centralized database (see <asm/cpuid/leaf_types.h). Such data types have a
371 * full C99 bitfield layout per CPUID leaf/subleaf combination. Call sites
372 * can thus avoid doing ugly and cryptic bitwise operations on raw CPUID data.
373 *
374 * - Return cached, per-CPU, CPUID output. Below APIs do not invoke any CPUID
375 * queries, thus avoiding their side effects like serialization and VM exits.
376 * Call-site-specific hard coded constants and macros for caching CPUID query
377 * outputs can also be avoided.
378 *
379 * - Return sanitized CPUID data. Below APIs return NULL if the given CPUID
380 * leaf/subleaf input is not supported by hardware, or if the hardware CPUID
381 * output was deemed invalid by the CPUID parser. This centralizes all CPUID
382 * data sanitization in one place (the kernel's CPUID parser.)
383 *
384 * - A centralized global view of system CPUID data. Below APIs will reflect
385 * any kernel-enforced feature masking or overrides, unlike ad hoc parsing of
386 * raw CPUID output by drivers and individual call sites.
387 */
388
389 /*
390 * Call-site APIs for CPUID leaves with a single subleaf:
391 */
392
393 /**
394 * cpuid_subleaf() - Access parsed CPUID
395 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
396 * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x7, 0xf
397 * @_subleaf: CPUID subleaf, in compile-time decimal format; e.g. 0, 1, 3
398 *
399 * Returns a pointer to parsed CPUID output, from the CPUID table inside
400 * @_cpuinfo, as a <cpuid/leaf_types.h> data type: 'struct leaf_0xM_N', where
401 * 0xM is the token provided at @_leaf, and N is the token provided at
402 * @_subleaf; e.g. struct leaf_0x7_0.
403 *
404 * Returns NULL if the requested CPUID @_leaf/@_subleaf query output is not
405 * present at the parsed CPUID table inside @_cpuinfo. This can happen if:
406 *
407 * - The CPUID table inside @_cpuinfo has not yet been populated.
408 * - The CPUID table inside @_cpuinfo was populated, but the CPU does not
409 * implement the requested CPUID @_leaf/@_subleaf combination.
410 * - The CPUID table inside @_cpuinfo was populated, but the kernel's CPUID
411 * parser has predetermined that the requested CPUID @_leaf/@_subleaf
412 * hardware output is invalid or unsupported.
413 *
414 * Example usage::
415 *
416 * const struct leaf_0x7_0 *l7_0 = cpuid_subleaf(c, 0x7, 0);
417 * if (!l7_0) {
418 * // Handle error
419 * }
420 *
421 * const struct leaf_0x7_1 *l7_1 = cpuid_subleaf(c, 0x7, 1);
422 * if (!l7_1) {
423 * // Handle error
424 * }
425 */
426 #define cpuid_subleaf(_cpuinfo, _leaf, _subleaf) \
427 __cpuid_table_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf) \
428
429 /**
430 * cpuid_leaf() - Access parsed CPUID data
431 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
432 * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x0, 0x2, 0x80000000
433 *
434 * Similar to cpuid_subleaf(), but with a CPUID subleaf = 0.
435 *
436 * Example usage::
437 *
438 * const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0);
439 * if (!l0) {
440 * // Handle error
441 * }
442 *
443 * const struct leaf_0x80000000_0 *el0 = cpuid_leaf(c, 0x80000000);
444 * if (!el0) {
445 * // Handle error
446 * }
447 */
448 #define cpuid_leaf(_cpuinfo, _leaf) \
449 cpuid_subleaf(_cpuinfo, _leaf, 0)
450
451 /**
452 * cpuid_leaf_raw() - Access parsed CPUID data in raw format
453 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
454 * @_leaf: CPUID leaf, in compile-time 0xN format
455 *
456 * Similar to cpuid_leaf(), but returns a raw 'struct cpuid_regs' pointer to
457 * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
458 */
459 #define cpuid_leaf_raw(_cpuinfo, _leaf) \
460 ((const struct cpuid_regs *)(cpuid_leaf(_cpuinfo, _leaf)))
461
462 /*
463 * Call-site APIs for CPUID leaves with a subleaf range:
464 */
465
466 /**
467 * cpuid_subleaf_n() - Access parsed CPUID data for leaf with a subleaf range
468 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
469 * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
470 * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller
471 * than cpuid_subleaf_count(@_cpuinfo, @_leaf).
472 *
473 * Build-time errors will be emitted in the following cases:
474 *
475 * - @_leaf has no subleaf range. Leaves with a subleaf range have an '_n' type
476 * suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
477 *
478 * - @_subleaf is known at compile-time but is out of range.
479 *
480 * Example usage::
481 *
482 * const struct leaf_0x4_n *l4;
483 *
484 * for (int i = 0; i < cpuid_subleaf_count(c, 0x4); i++) {
485 * l4 = cpuid_subleaf_n(c, 0x4, i);
486 * if (!l4) {
487 * // Handle error
488 * }
489 * ...
490 * }
491 *
492 * Beside the standard error situations detailed at cpuid_subleaf(), this
493 * macro will also return NULL if @_subleaf is out of the leaf's subleaf range.
494 */
495 #define cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf) \
496 ({ \
497 __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \
498 __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf); \
499 __cpuid_table_subleaf_n(&(_cpuinfo)->cpuid, _leaf, _subleaf); \
500 })
501
502 /**
503 * cpuid_subleaf_n_raw() - Access parsed CPUID data for leaf with subleaf range
504 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
505 * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
506 * @_subleaf: Subleaf number, which can be passed dynamically. It must be smaller
507 * than cpuid_subleaf_count(@_cpuinfo, @_leaf).
508 *
509 * Similar to cpuid_subleaf_n(), but returns a raw 'struct cpuid_regs' pointer to
510 * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
511 */
512 #define cpuid_subleaf_n_raw(_cpuinfo, _leaf, _subleaf) \
513 ((const struct cpuid_regs *)cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf))
514
515 /**
516 * cpuid_subleaf_count() - Number of filled subleaves for @_leaf
517 * @_cpuinfo: CPU capability structure reference ('struct cpuinfo_x86')
518 * @_leaf: CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
519 *
520 * Return the number of subleaves filled by the CPUID parser for @_leaf.
521 *
522 * @_leaf must have subleaf range. Leaves with a subleaf range have an '_n' type
523 * suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
524 */
525 #define cpuid_subleaf_count(_cpuinfo, _leaf) \
526 ({ \
527 __cpuid_assert_subleaf_range(_cpuinfo, _leaf); \
528 __cpuid_table_nr_filled_subleaves(&(_cpuinfo)->cpuid, _leaf, n); \
529 })
530
531 /*
532 * CPUID parser exported APIs:
533 */
534
535 void cpuid_scan_cpu(struct cpuinfo_x86 *c);
536 void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf);
537 void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end);
538
539 #endif /* _ASM_X86_CPUID_API_H */
540