xref: /linux/drivers/idle/intel_idle.c (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * intel_idle.c - native hardware idle loop for modern Intel processors
4  *
5  * Copyright (c) 2013 - 2020, Intel Corporation.
6  * Len Brown <len.brown@intel.com>
7  * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8  */
9 
10 /*
11  * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
13  * make Linux more efficient on these processors, as intel_idle knows
14  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15  */
16 
17 /*
18  * Design Assumptions
19  *
20  * All CPUs have same idle states as boot CPU
21  *
22  * Chipset BM_STS (bus master status) bit is a NOP
23  *	for preventing entry into deep C-states
24  *
25  * CPU will flush caches as needed when entering a C-state via MWAIT
26  *	(in contrast to entering ACPI C3, in which case the WBINVD
27  *	instruction needs to be executed to flush the caches)
28  */
29 
30 /*
31  * Known limitations
32  *
33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
34  * to avoid complications with the lapic timer workaround.
35  * Have not seen issues with suspend, but may need same workaround here.
36  *
37  */
38 
39 /* un-comment DEBUG to enable pr_debug() statements */
40 /* #define DEBUG */
41 
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43 
44 #include <linux/acpi.h>
45 #include <linux/kernel.h>
46 #include <linux/cpuidle.h>
47 #include <linux/tick.h>
48 #include <linux/time64.h>
49 #include <trace/events/power.h>
50 #include <linux/sched.h>
51 #include <linux/sched/smt.h>
52 #include <linux/mutex.h>
53 #include <linux/notifier.h>
54 #include <linux/cpu.h>
55 #include <linux/moduleparam.h>
56 #include <linux/pm_qos.h>
57 #include <linux/sysfs.h>
58 #include <asm/cpuid/api.h>
59 #include <asm/cpu_device_id.h>
60 #include <asm/intel-family.h>
61 #include <asm/mwait.h>
62 #include <asm/spec-ctrl.h>
63 #include <asm/msr.h>
64 #include <asm/tsc.h>
65 #include <asm/fpu/api.h>
66 #include <asm/smp.h>
67 
68 static struct cpuidle_driver intel_idle_driver = {
69 	.name = "intel_idle",
70 	.owner = THIS_MODULE,
71 };
72 /* intel_idle.max_cstate=0 disables driver */
73 static int max_cstate = CPUIDLE_STATE_MAX - 1;
74 static unsigned int disabled_states_mask __read_mostly;
75 static bool force_irq_on __read_mostly;
76 static bool ibrs_off __read_mostly;
77 
78 /* The maximum allowed length for the 'table' module parameter  */
79 #define MAX_CMDLINE_TABLE_LEN 256
80 /* Maximum allowed C-state latency */
81 #define MAX_CMDLINE_LATENCY_US (5 * USEC_PER_MSEC)
82 /* Maximum allowed C-state target residency */
83 #define MAX_CMDLINE_RESIDENCY_US (100 * USEC_PER_MSEC)
84 
85 /* The Package C-State Limit bits in MSR_PKG_CST_CONFIG_CONTROL */
86 #define SKX_PKG_CST_LIMIT_MASK GENMASK(2, 0)
87 /* PC6 is enabled when Package C-State Limit >= this value */
88 #define SKX_PKG_CST_LIMIT_PC6 2
89 
90 static char cmdline_table_str[MAX_CMDLINE_TABLE_LEN] __read_mostly;
91 
92 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
93 
94 static unsigned long auto_demotion_disable_flags;
95 
96 static enum {
97 	C1E_PROMOTION_PRESERVE,
98 	C1E_PROMOTION_ENABLE,
99 	C1E_PROMOTION_DISABLE
100 } c1e_promotion = C1E_PROMOTION_PRESERVE;
101 
102 struct idle_cpu {
103 	struct cpuidle_state *state_table;
104 
105 	/*
106 	 * Hardware C-state auto-demotion may not always be optimal.
107 	 * Indicate which enable bits to clear here.
108 	 */
109 	unsigned long auto_demotion_disable_flags;
110 	bool disable_promotion_to_c1e;
111 	bool c1_demotion_supported;
112 	bool use_acpi;
113 };
114 
115 static bool c1_demotion_supported;
116 static DEFINE_MUTEX(c1_demotion_mutex);
117 
118 static struct device *sysfs_root __initdata;
119 
120 static const struct idle_cpu *icpu __initdata;
121 static struct cpuidle_state *cpuidle_state_table __initdata;
122 
123 /* C-states data from the 'intel_idle.table' cmdline parameter */
124 static struct cpuidle_state cmdline_states[CPUIDLE_STATE_MAX] __initdata;
125 
126 static unsigned int mwait_substates __initdata;
127 
128 /*
129  * Enable interrupts before entering the C-state. On some platforms and for
130  * some C-states, this may measurably decrease interrupt latency.
131  */
132 #define CPUIDLE_FLAG_IRQ_ENABLE		BIT(14)
133 
134 /*
135  * Enable this state by default even if the ACPI _CST does not list it.
136  */
137 #define CPUIDLE_FLAG_ALWAYS_ENABLE	BIT(15)
138 
139 /*
140  * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
141  * above.
142  */
143 #define CPUIDLE_FLAG_IBRS		BIT(16)
144 
145 /*
146  * Initialize large xstate for the C6-state entrance.
147  */
148 #define CPUIDLE_FLAG_INIT_XSTATE	BIT(17)
149 
150 /*
151  * Ignore the sub-state when matching mwait hints between the ACPI _CST and
152  * custom tables.
153  */
154 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH	BIT(18)
155 
156 /*
157  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
158  * the C-state (top nibble) and sub-state (bottom nibble)
159  * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
160  *
161  * We store the hint at the top of our "flags" for each state.
162  */
163 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
164 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
165 
166 static __always_inline int __intel_idle(struct cpuidle_device *dev,
167 					struct cpuidle_driver *drv,
168 					int index, bool irqoff)
169 {
170 	struct cpuidle_state *state = &drv->states[index];
171 	unsigned int eax = flg2MWAIT(state->flags);
172 	unsigned int ecx = 1*irqoff; /* break on interrupt flag */
173 
174 	mwait_idle_with_hints(eax, ecx);
175 
176 	return index;
177 }
178 
179 /**
180  * intel_idle - Ask the processor to enter the given idle state.
181  * @dev: cpuidle device of the target CPU.
182  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
183  * @index: Target idle state index.
184  *
185  * Use the MWAIT instruction to notify the processor that the CPU represented by
186  * @dev is idle and it can try to enter the idle state corresponding to @index.
187  *
188  * If the local APIC timer is not known to be reliable in the target idle state,
189  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
190  *
191  * Must be called under local_irq_disable().
192  */
193 static __cpuidle int intel_idle(struct cpuidle_device *dev,
194 				struct cpuidle_driver *drv, int index)
195 {
196 	return __intel_idle(dev, drv, index, true);
197 }
198 
199 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
200 				    struct cpuidle_driver *drv, int index)
201 {
202 	return __intel_idle(dev, drv, index, false);
203 }
204 
205 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
206 				     struct cpuidle_driver *drv, int index)
207 {
208 	bool smt_active = sched_smt_active();
209 	u64 spec_ctrl = spec_ctrl_current();
210 	int ret;
211 
212 	if (smt_active)
213 		__update_spec_ctrl(0);
214 
215 	ret = __intel_idle(dev, drv, index, true);
216 
217 	if (smt_active)
218 		__update_spec_ctrl(spec_ctrl);
219 
220 	return ret;
221 }
222 
223 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
224 				       struct cpuidle_driver *drv, int index)
225 {
226 	fpu_idle_fpregs();
227 	return __intel_idle(dev, drv, index, true);
228 }
229 
230 /**
231  * intel_idle_s2idle - Ask the processor to enter the given idle state.
232  * @dev: cpuidle device of the target CPU.
233  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
234  * @index: Target idle state index.
235  *
236  * Use the MWAIT instruction to notify the processor that the CPU represented by
237  * @dev is idle and it can try to enter the idle state corresponding to @index.
238  *
239  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
240  * scheduler tick and suspended scheduler clock on the target CPU.
241  */
242 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
243 				       struct cpuidle_driver *drv, int index)
244 {
245 	struct cpuidle_state *state = &drv->states[index];
246 	unsigned int eax = flg2MWAIT(state->flags);
247 	unsigned int ecx = 1; /* break on interrupt flag */
248 
249 	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
250 		fpu_idle_fpregs();
251 
252 	mwait_idle_with_hints(eax, ecx);
253 
254 	return 0;
255 }
256 
257 static void intel_idle_enter_dead(struct cpuidle_device *dev, int index)
258 {
259 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
260 	struct cpuidle_state *state = &drv->states[index];
261 	unsigned long eax = flg2MWAIT(state->flags);
262 
263 	mwait_play_dead(eax);
264 }
265 
266 /*
267  * States are indexed by the cstate number,
268  * which is also the index into the MWAIT hint array.
269  * Thus C0 is a dummy.
270  */
271 static struct cpuidle_state nehalem_cstates[] __initdata = {
272 	{
273 		.name = "C1",
274 		.desc = "MWAIT 0x00",
275 		.flags = MWAIT2flg(0x00),
276 		.exit_latency = 3,
277 		.target_residency = 6,
278 		.enter = intel_idle,
279 		.enter_s2idle = intel_idle_s2idle, },
280 	{
281 		.name = "C1E",
282 		.desc = "MWAIT 0x01",
283 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
284 		.exit_latency = 10,
285 		.target_residency = 20,
286 		.enter = intel_idle,
287 		.enter_s2idle = intel_idle_s2idle, },
288 	{
289 		.name = "C3",
290 		.desc = "MWAIT 0x10",
291 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
292 		.exit_latency = 20,
293 		.target_residency = 80,
294 		.enter = intel_idle,
295 		.enter_s2idle = intel_idle_s2idle, },
296 	{
297 		.name = "C6",
298 		.desc = "MWAIT 0x20",
299 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
300 		.exit_latency = 200,
301 		.target_residency = 800,
302 		.enter = intel_idle,
303 		.enter_s2idle = intel_idle_s2idle, },
304 	{
305 		.enter = NULL }
306 };
307 
308 static struct cpuidle_state snb_cstates[] __initdata = {
309 	{
310 		.name = "C1",
311 		.desc = "MWAIT 0x00",
312 		.flags = MWAIT2flg(0x00),
313 		.exit_latency = 2,
314 		.target_residency = 2,
315 		.enter = intel_idle,
316 		.enter_s2idle = intel_idle_s2idle, },
317 	{
318 		.name = "C1E",
319 		.desc = "MWAIT 0x01",
320 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
321 		.exit_latency = 10,
322 		.target_residency = 20,
323 		.enter = intel_idle,
324 		.enter_s2idle = intel_idle_s2idle, },
325 	{
326 		.name = "C3",
327 		.desc = "MWAIT 0x10",
328 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
329 		.exit_latency = 80,
330 		.target_residency = 211,
331 		.enter = intel_idle,
332 		.enter_s2idle = intel_idle_s2idle, },
333 	{
334 		.name = "C6",
335 		.desc = "MWAIT 0x20",
336 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
337 		.exit_latency = 104,
338 		.target_residency = 345,
339 		.enter = intel_idle,
340 		.enter_s2idle = intel_idle_s2idle, },
341 	{
342 		.name = "C7",
343 		.desc = "MWAIT 0x30",
344 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
345 		.exit_latency = 109,
346 		.target_residency = 345,
347 		.enter = intel_idle,
348 		.enter_s2idle = intel_idle_s2idle, },
349 	{
350 		.enter = NULL }
351 };
352 
353 static struct cpuidle_state byt_cstates[] __initdata = {
354 	{
355 		.name = "C1",
356 		.desc = "MWAIT 0x00",
357 		.flags = MWAIT2flg(0x00),
358 		.exit_latency = 1,
359 		.target_residency = 1,
360 		.enter = intel_idle,
361 		.enter_s2idle = intel_idle_s2idle, },
362 	{
363 		.name = "C6N",
364 		.desc = "MWAIT 0x58",
365 		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
366 		.exit_latency = 300,
367 		.target_residency = 275,
368 		.enter = intel_idle,
369 		.enter_s2idle = intel_idle_s2idle, },
370 	{
371 		.name = "C6S",
372 		.desc = "MWAIT 0x52",
373 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
374 		.exit_latency = 500,
375 		.target_residency = 560,
376 		.enter = intel_idle,
377 		.enter_s2idle = intel_idle_s2idle, },
378 	{
379 		.name = "C7",
380 		.desc = "MWAIT 0x60",
381 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
382 		.exit_latency = 1200,
383 		.target_residency = 4000,
384 		.enter = intel_idle,
385 		.enter_s2idle = intel_idle_s2idle, },
386 	{
387 		.name = "C7S",
388 		.desc = "MWAIT 0x64",
389 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
390 		.exit_latency = 10000,
391 		.target_residency = 20000,
392 		.enter = intel_idle,
393 		.enter_s2idle = intel_idle_s2idle, },
394 	{
395 		.enter = NULL }
396 };
397 
398 static struct cpuidle_state cht_cstates[] __initdata = {
399 	{
400 		.name = "C1",
401 		.desc = "MWAIT 0x00",
402 		.flags = MWAIT2flg(0x00),
403 		.exit_latency = 1,
404 		.target_residency = 1,
405 		.enter = intel_idle,
406 		.enter_s2idle = intel_idle_s2idle, },
407 	{
408 		.name = "C6N",
409 		.desc = "MWAIT 0x58",
410 		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
411 		.exit_latency = 80,
412 		.target_residency = 275,
413 		.enter = intel_idle,
414 		.enter_s2idle = intel_idle_s2idle, },
415 	{
416 		.name = "C6S",
417 		.desc = "MWAIT 0x52",
418 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
419 		.exit_latency = 200,
420 		.target_residency = 560,
421 		.enter = intel_idle,
422 		.enter_s2idle = intel_idle_s2idle, },
423 	{
424 		.name = "C7",
425 		.desc = "MWAIT 0x60",
426 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
427 		.exit_latency = 1200,
428 		.target_residency = 4000,
429 		.enter = intel_idle,
430 		.enter_s2idle = intel_idle_s2idle, },
431 	{
432 		.name = "C7S",
433 		.desc = "MWAIT 0x64",
434 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
435 		.exit_latency = 10000,
436 		.target_residency = 20000,
437 		.enter = intel_idle,
438 		.enter_s2idle = intel_idle_s2idle, },
439 	{
440 		.enter = NULL }
441 };
442 
443 static struct cpuidle_state ivb_cstates[] __initdata = {
444 	{
445 		.name = "C1",
446 		.desc = "MWAIT 0x00",
447 		.flags = MWAIT2flg(0x00),
448 		.exit_latency = 1,
449 		.target_residency = 1,
450 		.enter = intel_idle,
451 		.enter_s2idle = intel_idle_s2idle, },
452 	{
453 		.name = "C1E",
454 		.desc = "MWAIT 0x01",
455 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
456 		.exit_latency = 10,
457 		.target_residency = 20,
458 		.enter = intel_idle,
459 		.enter_s2idle = intel_idle_s2idle, },
460 	{
461 		.name = "C3",
462 		.desc = "MWAIT 0x10",
463 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
464 		.exit_latency = 59,
465 		.target_residency = 156,
466 		.enter = intel_idle,
467 		.enter_s2idle = intel_idle_s2idle, },
468 	{
469 		.name = "C6",
470 		.desc = "MWAIT 0x20",
471 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
472 		.exit_latency = 80,
473 		.target_residency = 300,
474 		.enter = intel_idle,
475 		.enter_s2idle = intel_idle_s2idle, },
476 	{
477 		.name = "C7",
478 		.desc = "MWAIT 0x30",
479 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
480 		.exit_latency = 87,
481 		.target_residency = 300,
482 		.enter = intel_idle,
483 		.enter_s2idle = intel_idle_s2idle, },
484 	{
485 		.enter = NULL }
486 };
487 
488 static struct cpuidle_state ivt_cstates[] __initdata = {
489 	{
490 		.name = "C1",
491 		.desc = "MWAIT 0x00",
492 		.flags = MWAIT2flg(0x00),
493 		.exit_latency = 1,
494 		.target_residency = 1,
495 		.enter = intel_idle,
496 		.enter_s2idle = intel_idle_s2idle, },
497 	{
498 		.name = "C1E",
499 		.desc = "MWAIT 0x01",
500 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
501 		.exit_latency = 10,
502 		.target_residency = 80,
503 		.enter = intel_idle,
504 		.enter_s2idle = intel_idle_s2idle, },
505 	{
506 		.name = "C3",
507 		.desc = "MWAIT 0x10",
508 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
509 		.exit_latency = 59,
510 		.target_residency = 156,
511 		.enter = intel_idle,
512 		.enter_s2idle = intel_idle_s2idle, },
513 	{
514 		.name = "C6",
515 		.desc = "MWAIT 0x20",
516 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
517 		.exit_latency = 82,
518 		.target_residency = 300,
519 		.enter = intel_idle,
520 		.enter_s2idle = intel_idle_s2idle, },
521 	{
522 		.enter = NULL }
523 };
524 
525 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
526 	{
527 		.name = "C1",
528 		.desc = "MWAIT 0x00",
529 		.flags = MWAIT2flg(0x00),
530 		.exit_latency = 1,
531 		.target_residency = 1,
532 		.enter = intel_idle,
533 		.enter_s2idle = intel_idle_s2idle, },
534 	{
535 		.name = "C1E",
536 		.desc = "MWAIT 0x01",
537 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
538 		.exit_latency = 10,
539 		.target_residency = 250,
540 		.enter = intel_idle,
541 		.enter_s2idle = intel_idle_s2idle, },
542 	{
543 		.name = "C3",
544 		.desc = "MWAIT 0x10",
545 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
546 		.exit_latency = 59,
547 		.target_residency = 300,
548 		.enter = intel_idle,
549 		.enter_s2idle = intel_idle_s2idle, },
550 	{
551 		.name = "C6",
552 		.desc = "MWAIT 0x20",
553 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
554 		.exit_latency = 84,
555 		.target_residency = 400,
556 		.enter = intel_idle,
557 		.enter_s2idle = intel_idle_s2idle, },
558 	{
559 		.enter = NULL }
560 };
561 
562 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
563 	{
564 		.name = "C1",
565 		.desc = "MWAIT 0x00",
566 		.flags = MWAIT2flg(0x00),
567 		.exit_latency = 1,
568 		.target_residency = 1,
569 		.enter = intel_idle,
570 		.enter_s2idle = intel_idle_s2idle, },
571 	{
572 		.name = "C1E",
573 		.desc = "MWAIT 0x01",
574 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
575 		.exit_latency = 10,
576 		.target_residency = 500,
577 		.enter = intel_idle,
578 		.enter_s2idle = intel_idle_s2idle, },
579 	{
580 		.name = "C3",
581 		.desc = "MWAIT 0x10",
582 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
583 		.exit_latency = 59,
584 		.target_residency = 600,
585 		.enter = intel_idle,
586 		.enter_s2idle = intel_idle_s2idle, },
587 	{
588 		.name = "C6",
589 		.desc = "MWAIT 0x20",
590 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
591 		.exit_latency = 88,
592 		.target_residency = 700,
593 		.enter = intel_idle,
594 		.enter_s2idle = intel_idle_s2idle, },
595 	{
596 		.enter = NULL }
597 };
598 
599 static struct cpuidle_state hsw_cstates[] __initdata = {
600 	{
601 		.name = "C1",
602 		.desc = "MWAIT 0x00",
603 		.flags = MWAIT2flg(0x00),
604 		.exit_latency = 2,
605 		.target_residency = 2,
606 		.enter = intel_idle,
607 		.enter_s2idle = intel_idle_s2idle, },
608 	{
609 		.name = "C1E",
610 		.desc = "MWAIT 0x01",
611 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
612 		.exit_latency = 10,
613 		.target_residency = 20,
614 		.enter = intel_idle,
615 		.enter_s2idle = intel_idle_s2idle, },
616 	{
617 		.name = "C3",
618 		.desc = "MWAIT 0x10",
619 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
620 		.exit_latency = 33,
621 		.target_residency = 100,
622 		.enter = intel_idle,
623 		.enter_s2idle = intel_idle_s2idle, },
624 	{
625 		.name = "C6",
626 		.desc = "MWAIT 0x20",
627 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
628 		.exit_latency = 133,
629 		.target_residency = 400,
630 		.enter = intel_idle,
631 		.enter_s2idle = intel_idle_s2idle, },
632 	{
633 		.name = "C7s",
634 		.desc = "MWAIT 0x32",
635 		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
636 		.exit_latency = 166,
637 		.target_residency = 500,
638 		.enter = intel_idle,
639 		.enter_s2idle = intel_idle_s2idle, },
640 	{
641 		.name = "C8",
642 		.desc = "MWAIT 0x40",
643 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
644 		.exit_latency = 300,
645 		.target_residency = 900,
646 		.enter = intel_idle,
647 		.enter_s2idle = intel_idle_s2idle, },
648 	{
649 		.name = "C9",
650 		.desc = "MWAIT 0x50",
651 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
652 		.exit_latency = 600,
653 		.target_residency = 1800,
654 		.enter = intel_idle,
655 		.enter_s2idle = intel_idle_s2idle, },
656 	{
657 		.name = "C10",
658 		.desc = "MWAIT 0x60",
659 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
660 		.exit_latency = 2600,
661 		.target_residency = 7700,
662 		.enter = intel_idle,
663 		.enter_s2idle = intel_idle_s2idle, },
664 	{
665 		.enter = NULL }
666 };
667 static struct cpuidle_state bdw_cstates[] __initdata = {
668 	{
669 		.name = "C1",
670 		.desc = "MWAIT 0x00",
671 		.flags = MWAIT2flg(0x00),
672 		.exit_latency = 2,
673 		.target_residency = 2,
674 		.enter = intel_idle,
675 		.enter_s2idle = intel_idle_s2idle, },
676 	{
677 		.name = "C1E",
678 		.desc = "MWAIT 0x01",
679 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
680 		.exit_latency = 10,
681 		.target_residency = 20,
682 		.enter = intel_idle,
683 		.enter_s2idle = intel_idle_s2idle, },
684 	{
685 		.name = "C3",
686 		.desc = "MWAIT 0x10",
687 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
688 		.exit_latency = 40,
689 		.target_residency = 100,
690 		.enter = intel_idle,
691 		.enter_s2idle = intel_idle_s2idle, },
692 	{
693 		.name = "C6",
694 		.desc = "MWAIT 0x20",
695 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
696 		.exit_latency = 133,
697 		.target_residency = 400,
698 		.enter = intel_idle,
699 		.enter_s2idle = intel_idle_s2idle, },
700 	{
701 		.name = "C7s",
702 		.desc = "MWAIT 0x32",
703 		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
704 		.exit_latency = 166,
705 		.target_residency = 500,
706 		.enter = intel_idle,
707 		.enter_s2idle = intel_idle_s2idle, },
708 	{
709 		.name = "C8",
710 		.desc = "MWAIT 0x40",
711 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
712 		.exit_latency = 300,
713 		.target_residency = 900,
714 		.enter = intel_idle,
715 		.enter_s2idle = intel_idle_s2idle, },
716 	{
717 		.name = "C9",
718 		.desc = "MWAIT 0x50",
719 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
720 		.exit_latency = 600,
721 		.target_residency = 1800,
722 		.enter = intel_idle,
723 		.enter_s2idle = intel_idle_s2idle, },
724 	{
725 		.name = "C10",
726 		.desc = "MWAIT 0x60",
727 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
728 		.exit_latency = 2600,
729 		.target_residency = 7700,
730 		.enter = intel_idle,
731 		.enter_s2idle = intel_idle_s2idle, },
732 	{
733 		.enter = NULL }
734 };
735 
736 static struct cpuidle_state skl_cstates[] __initdata = {
737 	{
738 		.name = "C1",
739 		.desc = "MWAIT 0x00",
740 		.flags = MWAIT2flg(0x00),
741 		.exit_latency = 2,
742 		.target_residency = 2,
743 		.enter = intel_idle,
744 		.enter_s2idle = intel_idle_s2idle, },
745 	{
746 		.name = "C1E",
747 		.desc = "MWAIT 0x01",
748 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
749 		.exit_latency = 10,
750 		.target_residency = 20,
751 		.enter = intel_idle,
752 		.enter_s2idle = intel_idle_s2idle, },
753 	{
754 		.name = "C3",
755 		.desc = "MWAIT 0x10",
756 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
757 		.exit_latency = 70,
758 		.target_residency = 100,
759 		.enter = intel_idle,
760 		.enter_s2idle = intel_idle_s2idle, },
761 	{
762 		.name = "C6",
763 		.desc = "MWAIT 0x20",
764 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
765 		.exit_latency = 85,
766 		.target_residency = 200,
767 		.enter = intel_idle,
768 		.enter_s2idle = intel_idle_s2idle, },
769 	{
770 		.name = "C7s",
771 		.desc = "MWAIT 0x33",
772 		.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
773 		.exit_latency = 124,
774 		.target_residency = 800,
775 		.enter = intel_idle,
776 		.enter_s2idle = intel_idle_s2idle, },
777 	{
778 		.name = "C8",
779 		.desc = "MWAIT 0x40",
780 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
781 		.exit_latency = 200,
782 		.target_residency = 800,
783 		.enter = intel_idle,
784 		.enter_s2idle = intel_idle_s2idle, },
785 	{
786 		.name = "C9",
787 		.desc = "MWAIT 0x50",
788 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
789 		.exit_latency = 480,
790 		.target_residency = 5000,
791 		.enter = intel_idle,
792 		.enter_s2idle = intel_idle_s2idle, },
793 	{
794 		.name = "C10",
795 		.desc = "MWAIT 0x60",
796 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
797 		.exit_latency = 890,
798 		.target_residency = 5000,
799 		.enter = intel_idle,
800 		.enter_s2idle = intel_idle_s2idle, },
801 	{
802 		.enter = NULL }
803 };
804 
805 static struct cpuidle_state skx_cstates[] __initdata = {
806 	{
807 		.name = "C1",
808 		.desc = "MWAIT 0x00",
809 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
810 		.exit_latency = 2,
811 		.target_residency = 2,
812 		.enter = intel_idle,
813 		.enter_s2idle = intel_idle_s2idle, },
814 	{
815 		.name = "C1E",
816 		.desc = "MWAIT 0x01",
817 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
818 		.exit_latency = 10,
819 		.target_residency = 20,
820 		.enter = intel_idle,
821 		.enter_s2idle = intel_idle_s2idle, },
822 	{
823 		.name = "C6",
824 		.desc = "MWAIT 0x20",
825 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
826 		.exit_latency = 133,
827 		.target_residency = 600,
828 		.enter = intel_idle,
829 		.enter_s2idle = intel_idle_s2idle, },
830 	{
831 		.enter = NULL }
832 };
833 
834 static struct cpuidle_state icx_cstates[] __initdata = {
835 	{
836 		.name = "C1",
837 		.desc = "MWAIT 0x00",
838 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
839 		.exit_latency = 1,
840 		.target_residency = 1,
841 		.enter = intel_idle,
842 		.enter_s2idle = intel_idle_s2idle, },
843 	{
844 		.name = "C1E",
845 		.desc = "MWAIT 0x01",
846 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
847 		.exit_latency = 4,
848 		.target_residency = 4,
849 		.enter = intel_idle,
850 		.enter_s2idle = intel_idle_s2idle, },
851 	{
852 		.name = "C6",
853 		.desc = "MWAIT 0x20",
854 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
855 		.exit_latency = 170,
856 		.target_residency = 600,
857 		.enter = intel_idle,
858 		.enter_s2idle = intel_idle_s2idle, },
859 	{
860 		.enter = NULL }
861 };
862 
863 /*
864  * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
865  * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
866  * But in this case there is effectively no C1, because C1 requests are
867  * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
868  * and C1E requests end up with C1, so there is effectively no C1E.
869  *
870  * By default we enable C1E and disable C1 by marking it with
871  * 'CPUIDLE_FLAG_UNUSABLE'.
872  */
873 static struct cpuidle_state adl_cstates[] __initdata = {
874 	{
875 		.name = "C1",
876 		.desc = "MWAIT 0x00",
877 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
878 		.exit_latency = 1,
879 		.target_residency = 1,
880 		.enter = intel_idle,
881 		.enter_s2idle = intel_idle_s2idle, },
882 	{
883 		.name = "C1E",
884 		.desc = "MWAIT 0x01",
885 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
886 		.exit_latency = 2,
887 		.target_residency = 4,
888 		.enter = intel_idle,
889 		.enter_s2idle = intel_idle_s2idle, },
890 	{
891 		.name = "C6",
892 		.desc = "MWAIT 0x20",
893 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
894 		.exit_latency = 220,
895 		.target_residency = 600,
896 		.enter = intel_idle,
897 		.enter_s2idle = intel_idle_s2idle, },
898 	{
899 		.name = "C8",
900 		.desc = "MWAIT 0x40",
901 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
902 		.exit_latency = 280,
903 		.target_residency = 800,
904 		.enter = intel_idle,
905 		.enter_s2idle = intel_idle_s2idle, },
906 	{
907 		.name = "C10",
908 		.desc = "MWAIT 0x60",
909 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
910 		.exit_latency = 680,
911 		.target_residency = 2000,
912 		.enter = intel_idle,
913 		.enter_s2idle = intel_idle_s2idle, },
914 	{
915 		.enter = NULL }
916 };
917 
918 static struct cpuidle_state adl_l_cstates[] __initdata = {
919 	{
920 		.name = "C1",
921 		.desc = "MWAIT 0x00",
922 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
923 		.exit_latency = 1,
924 		.target_residency = 1,
925 		.enter = intel_idle,
926 		.enter_s2idle = intel_idle_s2idle, },
927 	{
928 		.name = "C1E",
929 		.desc = "MWAIT 0x01",
930 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
931 		.exit_latency = 2,
932 		.target_residency = 4,
933 		.enter = intel_idle,
934 		.enter_s2idle = intel_idle_s2idle, },
935 	{
936 		.name = "C6",
937 		.desc = "MWAIT 0x20",
938 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
939 		.exit_latency = 170,
940 		.target_residency = 500,
941 		.enter = intel_idle,
942 		.enter_s2idle = intel_idle_s2idle, },
943 	{
944 		.name = "C8",
945 		.desc = "MWAIT 0x40",
946 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
947 		.exit_latency = 200,
948 		.target_residency = 600,
949 		.enter = intel_idle,
950 		.enter_s2idle = intel_idle_s2idle, },
951 	{
952 		.name = "C10",
953 		.desc = "MWAIT 0x60",
954 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
955 		.exit_latency = 230,
956 		.target_residency = 700,
957 		.enter = intel_idle,
958 		.enter_s2idle = intel_idle_s2idle, },
959 	{
960 		.enter = NULL }
961 };
962 
963 static struct cpuidle_state mtl_l_cstates[] __initdata = {
964 	{
965 		.name = "C1E",
966 		.desc = "MWAIT 0x01",
967 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
968 		.exit_latency = 1,
969 		.target_residency = 1,
970 		.enter = intel_idle,
971 		.enter_s2idle = intel_idle_s2idle, },
972 	{
973 		.name = "C6",
974 		.desc = "MWAIT 0x20",
975 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
976 		.exit_latency = 140,
977 		.target_residency = 420,
978 		.enter = intel_idle,
979 		.enter_s2idle = intel_idle_s2idle, },
980 	{
981 		.name = "C10",
982 		.desc = "MWAIT 0x60",
983 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
984 		.exit_latency = 310,
985 		.target_residency = 930,
986 		.enter = intel_idle,
987 		.enter_s2idle = intel_idle_s2idle, },
988 	{
989 		.enter = NULL }
990 };
991 
992 static struct cpuidle_state ptl_cstates[] __initdata = {
993 	{
994 		.name = "C1",
995 		.desc = "MWAIT 0x00",
996 		.flags = MWAIT2flg(0x00),
997 		.exit_latency = 1,
998 		.target_residency = 1,
999 		.enter = &intel_idle,
1000 		.enter_s2idle = intel_idle_s2idle, },
1001 	{
1002 		.name = "C1E",
1003 		.desc = "MWAIT 0x01",
1004 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1005 		.exit_latency = 10,
1006 		.target_residency = 10,
1007 		.enter = &intel_idle,
1008 		.enter_s2idle = intel_idle_s2idle, },
1009 	{
1010 		.name = "C6S",
1011 		.desc = "MWAIT 0x21",
1012 		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED,
1013 		.exit_latency = 300,
1014 		.target_residency = 300,
1015 		.enter = &intel_idle,
1016 		.enter_s2idle = intel_idle_s2idle, },
1017 	{
1018 		.name = "C10",
1019 		.desc = "MWAIT 0x60",
1020 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1021 		.exit_latency = 370,
1022 		.target_residency = 2500,
1023 		.enter = &intel_idle,
1024 		.enter_s2idle = intel_idle_s2idle, },
1025 	{
1026 		.enter = NULL }
1027 };
1028 
1029 static struct cpuidle_state gmt_cstates[] __initdata = {
1030 	{
1031 		.name = "C1",
1032 		.desc = "MWAIT 0x00",
1033 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
1034 		.exit_latency = 1,
1035 		.target_residency = 1,
1036 		.enter = intel_idle,
1037 		.enter_s2idle = intel_idle_s2idle, },
1038 	{
1039 		.name = "C1E",
1040 		.desc = "MWAIT 0x01",
1041 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1042 		.exit_latency = 2,
1043 		.target_residency = 4,
1044 		.enter = intel_idle,
1045 		.enter_s2idle = intel_idle_s2idle, },
1046 	{
1047 		.name = "C6",
1048 		.desc = "MWAIT 0x20",
1049 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1050 		.exit_latency = 195,
1051 		.target_residency = 585,
1052 		.enter = intel_idle,
1053 		.enter_s2idle = intel_idle_s2idle, },
1054 	{
1055 		.name = "C8",
1056 		.desc = "MWAIT 0x40",
1057 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1058 		.exit_latency = 260,
1059 		.target_residency = 1040,
1060 		.enter = intel_idle,
1061 		.enter_s2idle = intel_idle_s2idle, },
1062 	{
1063 		.name = "C10",
1064 		.desc = "MWAIT 0x60",
1065 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1066 		.exit_latency = 660,
1067 		.target_residency = 1980,
1068 		.enter = intel_idle,
1069 		.enter_s2idle = intel_idle_s2idle, },
1070 	{
1071 		.enter = NULL }
1072 };
1073 
1074 static struct cpuidle_state spr_cstates[] __initdata = {
1075 	{
1076 		.name = "C1",
1077 		.desc = "MWAIT 0x00",
1078 		.flags = MWAIT2flg(0x00),
1079 		.exit_latency = 1,
1080 		.target_residency = 1,
1081 		.enter = intel_idle,
1082 		.enter_s2idle = intel_idle_s2idle, },
1083 	{
1084 		.name = "C1E",
1085 		.desc = "MWAIT 0x01",
1086 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1087 		.exit_latency = 2,
1088 		.target_residency = 4,
1089 		.enter = intel_idle,
1090 		.enter_s2idle = intel_idle_s2idle, },
1091 	{
1092 		.name = "C6",
1093 		.desc = "MWAIT 0x20",
1094 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1095 					   CPUIDLE_FLAG_INIT_XSTATE,
1096 		.exit_latency = 290,
1097 		.target_residency = 800,
1098 		.enter = intel_idle,
1099 		.enter_s2idle = intel_idle_s2idle, },
1100 	{
1101 		.enter = NULL }
1102 };
1103 
1104 static struct cpuidle_state gnr_cstates[] __initdata = {
1105 	{
1106 		.name = "C1",
1107 		.desc = "MWAIT 0x00",
1108 		.flags = MWAIT2flg(0x00),
1109 		.exit_latency = 1,
1110 		.target_residency = 1,
1111 		.enter = intel_idle,
1112 		.enter_s2idle = intel_idle_s2idle, },
1113 	{
1114 		.name = "C1E",
1115 		.desc = "MWAIT 0x01",
1116 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1117 		.exit_latency = 4,
1118 		.target_residency = 4,
1119 		.enter = intel_idle,
1120 		.enter_s2idle = intel_idle_s2idle, },
1121 	{
1122 		.name = "C6",
1123 		.desc = "MWAIT 0x20",
1124 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1125 					   CPUIDLE_FLAG_INIT_XSTATE |
1126 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1127 		.exit_latency = 170,
1128 		.target_residency = 650,
1129 		.enter = intel_idle,
1130 		.enter_s2idle = intel_idle_s2idle, },
1131 	{
1132 		.name = "C6P",
1133 		.desc = "MWAIT 0x21",
1134 		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1135 					   CPUIDLE_FLAG_INIT_XSTATE |
1136 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1137 		.exit_latency = 210,
1138 		.target_residency = 1000,
1139 		.enter = intel_idle,
1140 		.enter_s2idle = intel_idle_s2idle, },
1141 	{
1142 		.enter = NULL }
1143 };
1144 
1145 static struct cpuidle_state gnrd_cstates[] __initdata = {
1146 	{
1147 		.name = "C1",
1148 		.desc = "MWAIT 0x00",
1149 		.flags = MWAIT2flg(0x00),
1150 		.exit_latency = 1,
1151 		.target_residency = 1,
1152 		.enter = intel_idle,
1153 		.enter_s2idle = intel_idle_s2idle, },
1154 	{
1155 		.name = "C1E",
1156 		.desc = "MWAIT 0x01",
1157 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1158 		.exit_latency = 4,
1159 		.target_residency = 4,
1160 		.enter = intel_idle,
1161 		.enter_s2idle = intel_idle_s2idle, },
1162 	{
1163 		.name = "C6",
1164 		.desc = "MWAIT 0x20",
1165 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1166 					   CPUIDLE_FLAG_INIT_XSTATE |
1167 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1168 		.exit_latency = 220,
1169 		.target_residency = 650,
1170 		.enter = intel_idle,
1171 		.enter_s2idle = intel_idle_s2idle, },
1172 	{
1173 		.name = "C6P",
1174 		.desc = "MWAIT 0x21",
1175 		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1176 					   CPUIDLE_FLAG_INIT_XSTATE |
1177 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1178 		.exit_latency = 240,
1179 		.target_residency = 750,
1180 		.enter = intel_idle,
1181 		.enter_s2idle = intel_idle_s2idle, },
1182 	{
1183 		.enter = NULL }
1184 };
1185 
1186 static struct cpuidle_state atom_cstates[] __initdata = {
1187 	{
1188 		.name = "C1E",
1189 		.desc = "MWAIT 0x00",
1190 		.flags = MWAIT2flg(0x00),
1191 		.exit_latency = 10,
1192 		.target_residency = 20,
1193 		.enter = intel_idle,
1194 		.enter_s2idle = intel_idle_s2idle, },
1195 	{
1196 		.name = "C2",
1197 		.desc = "MWAIT 0x10",
1198 		.flags = MWAIT2flg(0x10),
1199 		.exit_latency = 20,
1200 		.target_residency = 80,
1201 		.enter = intel_idle,
1202 		.enter_s2idle = intel_idle_s2idle, },
1203 	{
1204 		.name = "C4",
1205 		.desc = "MWAIT 0x30",
1206 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1207 		.exit_latency = 100,
1208 		.target_residency = 400,
1209 		.enter = intel_idle,
1210 		.enter_s2idle = intel_idle_s2idle, },
1211 	{
1212 		.name = "C6",
1213 		.desc = "MWAIT 0x52",
1214 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1215 		.exit_latency = 140,
1216 		.target_residency = 560,
1217 		.enter = intel_idle,
1218 		.enter_s2idle = intel_idle_s2idle, },
1219 	{
1220 		.enter = NULL }
1221 };
1222 static struct cpuidle_state tangier_cstates[] __initdata = {
1223 	{
1224 		.name = "C1",
1225 		.desc = "MWAIT 0x00",
1226 		.flags = MWAIT2flg(0x00),
1227 		.exit_latency = 1,
1228 		.target_residency = 4,
1229 		.enter = intel_idle,
1230 		.enter_s2idle = intel_idle_s2idle, },
1231 	{
1232 		.name = "C4",
1233 		.desc = "MWAIT 0x30",
1234 		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1235 		.exit_latency = 100,
1236 		.target_residency = 400,
1237 		.enter = intel_idle,
1238 		.enter_s2idle = intel_idle_s2idle, },
1239 	{
1240 		.name = "C6",
1241 		.desc = "MWAIT 0x52",
1242 		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1243 		.exit_latency = 140,
1244 		.target_residency = 560,
1245 		.enter = intel_idle,
1246 		.enter_s2idle = intel_idle_s2idle, },
1247 	{
1248 		.name = "C7",
1249 		.desc = "MWAIT 0x60",
1250 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1251 		.exit_latency = 1200,
1252 		.target_residency = 4000,
1253 		.enter = intel_idle,
1254 		.enter_s2idle = intel_idle_s2idle, },
1255 	{
1256 		.name = "C9",
1257 		.desc = "MWAIT 0x64",
1258 		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1259 		.exit_latency = 10000,
1260 		.target_residency = 20000,
1261 		.enter = intel_idle,
1262 		.enter_s2idle = intel_idle_s2idle, },
1263 	{
1264 		.enter = NULL }
1265 };
1266 static struct cpuidle_state avn_cstates[] __initdata = {
1267 	{
1268 		.name = "C1",
1269 		.desc = "MWAIT 0x00",
1270 		.flags = MWAIT2flg(0x00),
1271 		.exit_latency = 2,
1272 		.target_residency = 2,
1273 		.enter = intel_idle,
1274 		.enter_s2idle = intel_idle_s2idle, },
1275 	{
1276 		.name = "C6",
1277 		.desc = "MWAIT 0x51",
1278 		.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1279 		.exit_latency = 15,
1280 		.target_residency = 45,
1281 		.enter = intel_idle,
1282 		.enter_s2idle = intel_idle_s2idle, },
1283 	{
1284 		.enter = NULL }
1285 };
1286 static struct cpuidle_state knl_cstates[] __initdata = {
1287 	{
1288 		.name = "C1",
1289 		.desc = "MWAIT 0x00",
1290 		.flags = MWAIT2flg(0x00),
1291 		.exit_latency = 1,
1292 		.target_residency = 2,
1293 		.enter = intel_idle,
1294 		.enter_s2idle = intel_idle_s2idle },
1295 	{
1296 		.name = "C6",
1297 		.desc = "MWAIT 0x10",
1298 		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1299 		.exit_latency = 120,
1300 		.target_residency = 500,
1301 		.enter = intel_idle,
1302 		.enter_s2idle = intel_idle_s2idle },
1303 	{
1304 		.enter = NULL }
1305 };
1306 
1307 static struct cpuidle_state bxt_cstates[] __initdata = {
1308 	{
1309 		.name = "C1",
1310 		.desc = "MWAIT 0x00",
1311 		.flags = MWAIT2flg(0x00),
1312 		.exit_latency = 2,
1313 		.target_residency = 2,
1314 		.enter = intel_idle,
1315 		.enter_s2idle = intel_idle_s2idle, },
1316 	{
1317 		.name = "C1E",
1318 		.desc = "MWAIT 0x01",
1319 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1320 		.exit_latency = 10,
1321 		.target_residency = 20,
1322 		.enter = intel_idle,
1323 		.enter_s2idle = intel_idle_s2idle, },
1324 	{
1325 		.name = "C6",
1326 		.desc = "MWAIT 0x20",
1327 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1328 		.exit_latency = 133,
1329 		.target_residency = 133,
1330 		.enter = intel_idle,
1331 		.enter_s2idle = intel_idle_s2idle, },
1332 	{
1333 		.name = "C7s",
1334 		.desc = "MWAIT 0x31",
1335 		.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1336 		.exit_latency = 155,
1337 		.target_residency = 155,
1338 		.enter = intel_idle,
1339 		.enter_s2idle = intel_idle_s2idle, },
1340 	{
1341 		.name = "C8",
1342 		.desc = "MWAIT 0x40",
1343 		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1344 		.exit_latency = 1000,
1345 		.target_residency = 1000,
1346 		.enter = intel_idle,
1347 		.enter_s2idle = intel_idle_s2idle, },
1348 	{
1349 		.name = "C9",
1350 		.desc = "MWAIT 0x50",
1351 		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1352 		.exit_latency = 2000,
1353 		.target_residency = 2000,
1354 		.enter = intel_idle,
1355 		.enter_s2idle = intel_idle_s2idle, },
1356 	{
1357 		.name = "C10",
1358 		.desc = "MWAIT 0x60",
1359 		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1360 		.exit_latency = 10000,
1361 		.target_residency = 10000,
1362 		.enter = intel_idle,
1363 		.enter_s2idle = intel_idle_s2idle, },
1364 	{
1365 		.enter = NULL }
1366 };
1367 
1368 static struct cpuidle_state dnv_cstates[] __initdata = {
1369 	{
1370 		.name = "C1",
1371 		.desc = "MWAIT 0x00",
1372 		.flags = MWAIT2flg(0x00),
1373 		.exit_latency = 2,
1374 		.target_residency = 2,
1375 		.enter = intel_idle,
1376 		.enter_s2idle = intel_idle_s2idle, },
1377 	{
1378 		.name = "C1E",
1379 		.desc = "MWAIT 0x01",
1380 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1381 		.exit_latency = 10,
1382 		.target_residency = 20,
1383 		.enter = intel_idle,
1384 		.enter_s2idle = intel_idle_s2idle, },
1385 	{
1386 		.name = "C6",
1387 		.desc = "MWAIT 0x20",
1388 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1389 		.exit_latency = 50,
1390 		.target_residency = 500,
1391 		.enter = intel_idle,
1392 		.enter_s2idle = intel_idle_s2idle, },
1393 	{
1394 		.enter = NULL }
1395 };
1396 
1397 /*
1398  * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1399  * C6, and this is indicated in the CPUID mwait leaf.
1400  */
1401 static struct cpuidle_state snr_cstates[] __initdata = {
1402 	{
1403 		.name = "C1",
1404 		.desc = "MWAIT 0x00",
1405 		.flags = MWAIT2flg(0x00),
1406 		.exit_latency = 2,
1407 		.target_residency = 2,
1408 		.enter = intel_idle,
1409 		.enter_s2idle = intel_idle_s2idle, },
1410 	{
1411 		.name = "C1E",
1412 		.desc = "MWAIT 0x01",
1413 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1414 		.exit_latency = 15,
1415 		.target_residency = 25,
1416 		.enter = intel_idle,
1417 		.enter_s2idle = intel_idle_s2idle, },
1418 	{
1419 		.name = "C6",
1420 		.desc = "MWAIT 0x20",
1421 		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1422 		.exit_latency = 130,
1423 		.target_residency = 500,
1424 		.enter = intel_idle,
1425 		.enter_s2idle = intel_idle_s2idle, },
1426 	{
1427 		.enter = NULL }
1428 };
1429 
1430 static struct cpuidle_state grr_cstates[] __initdata = {
1431 	{
1432 		.name = "C1",
1433 		.desc = "MWAIT 0x00",
1434 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1435 		.exit_latency = 1,
1436 		.target_residency = 1,
1437 		.enter = intel_idle,
1438 		.enter_s2idle = intel_idle_s2idle, },
1439 	{
1440 		.name = "C1E",
1441 		.desc = "MWAIT 0x01",
1442 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1443 		.exit_latency = 2,
1444 		.target_residency = 10,
1445 		.enter = intel_idle,
1446 		.enter_s2idle = intel_idle_s2idle, },
1447 	{
1448 		.name = "C6S",
1449 		.desc = "MWAIT 0x22",
1450 		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1451 		.exit_latency = 140,
1452 		.target_residency = 500,
1453 		.enter = intel_idle,
1454 		.enter_s2idle = intel_idle_s2idle, },
1455 	{
1456 		.enter = NULL }
1457 };
1458 
1459 static struct cpuidle_state srf_cstates[] __initdata = {
1460 	{
1461 		.name = "C1",
1462 		.desc = "MWAIT 0x00",
1463 		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1464 		.exit_latency = 1,
1465 		.target_residency = 1,
1466 		.enter = intel_idle,
1467 		.enter_s2idle = intel_idle_s2idle, },
1468 	{
1469 		.name = "C1E",
1470 		.desc = "MWAIT 0x01",
1471 		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1472 		.exit_latency = 2,
1473 		.target_residency = 10,
1474 		.enter = intel_idle,
1475 		.enter_s2idle = intel_idle_s2idle, },
1476 	{
1477 		.name = "C6S",
1478 		.desc = "MWAIT 0x22",
1479 		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1480 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1481 		.exit_latency = 270,
1482 		.target_residency = 700,
1483 		.enter = intel_idle,
1484 		.enter_s2idle = intel_idle_s2idle, },
1485 	{
1486 		.name = "C6SP",
1487 		.desc = "MWAIT 0x23",
1488 		.flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1489 					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1490 		.exit_latency = 310,
1491 		.target_residency = 900,
1492 		.enter = intel_idle,
1493 		.enter_s2idle = intel_idle_s2idle, },
1494 	{
1495 		.enter = NULL }
1496 };
1497 
1498 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1499 	.state_table = nehalem_cstates,
1500 	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1501 	.disable_promotion_to_c1e = true,
1502 };
1503 
1504 static const struct idle_cpu idle_cpu_nhx __initconst = {
1505 	.state_table = nehalem_cstates,
1506 	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1507 	.disable_promotion_to_c1e = true,
1508 	.use_acpi = true,
1509 };
1510 
1511 static const struct idle_cpu idle_cpu_atom __initconst = {
1512 	.state_table = atom_cstates,
1513 };
1514 
1515 static const struct idle_cpu idle_cpu_tangier __initconst = {
1516 	.state_table = tangier_cstates,
1517 };
1518 
1519 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1520 	.state_table = atom_cstates,
1521 	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1522 };
1523 
1524 static const struct idle_cpu idle_cpu_snb __initconst = {
1525 	.state_table = snb_cstates,
1526 	.disable_promotion_to_c1e = true,
1527 };
1528 
1529 static const struct idle_cpu idle_cpu_snx __initconst = {
1530 	.state_table = snb_cstates,
1531 	.disable_promotion_to_c1e = true,
1532 	.use_acpi = true,
1533 };
1534 
1535 static const struct idle_cpu idle_cpu_byt __initconst = {
1536 	.state_table = byt_cstates,
1537 	.disable_promotion_to_c1e = true,
1538 };
1539 
1540 static const struct idle_cpu idle_cpu_cht __initconst = {
1541 	.state_table = cht_cstates,
1542 	.disable_promotion_to_c1e = true,
1543 };
1544 
1545 static const struct idle_cpu idle_cpu_ivb __initconst = {
1546 	.state_table = ivb_cstates,
1547 	.disable_promotion_to_c1e = true,
1548 };
1549 
1550 static const struct idle_cpu idle_cpu_ivt __initconst = {
1551 	.state_table = ivt_cstates,
1552 	.disable_promotion_to_c1e = true,
1553 	.use_acpi = true,
1554 };
1555 
1556 static const struct idle_cpu idle_cpu_hsw __initconst = {
1557 	.state_table = hsw_cstates,
1558 	.disable_promotion_to_c1e = true,
1559 };
1560 
1561 static const struct idle_cpu idle_cpu_hsx __initconst = {
1562 	.state_table = hsw_cstates,
1563 	.disable_promotion_to_c1e = true,
1564 	.use_acpi = true,
1565 };
1566 
1567 static const struct idle_cpu idle_cpu_bdw __initconst = {
1568 	.state_table = bdw_cstates,
1569 	.disable_promotion_to_c1e = true,
1570 };
1571 
1572 static const struct idle_cpu idle_cpu_bdx __initconst = {
1573 	.state_table = bdw_cstates,
1574 	.disable_promotion_to_c1e = true,
1575 	.use_acpi = true,
1576 };
1577 
1578 static const struct idle_cpu idle_cpu_skl __initconst = {
1579 	.state_table = skl_cstates,
1580 	.disable_promotion_to_c1e = true,
1581 };
1582 
1583 static const struct idle_cpu idle_cpu_skx __initconst = {
1584 	.state_table = skx_cstates,
1585 	.disable_promotion_to_c1e = true,
1586 	.use_acpi = true,
1587 };
1588 
1589 static const struct idle_cpu idle_cpu_icx __initconst = {
1590 	.state_table = icx_cstates,
1591 	.disable_promotion_to_c1e = true,
1592 	.use_acpi = true,
1593 };
1594 
1595 static const struct idle_cpu idle_cpu_adl __initconst = {
1596 	.state_table = adl_cstates,
1597 };
1598 
1599 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1600 	.state_table = adl_l_cstates,
1601 };
1602 
1603 static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1604 	.state_table = mtl_l_cstates,
1605 };
1606 
1607 static const struct idle_cpu idle_cpu_ptl __initconst = {
1608 	.state_table = ptl_cstates,
1609 };
1610 
1611 static const struct idle_cpu idle_cpu_gmt __initconst = {
1612 	.state_table = gmt_cstates,
1613 };
1614 
1615 static const struct idle_cpu idle_cpu_spr __initconst = {
1616 	.state_table = spr_cstates,
1617 	.disable_promotion_to_c1e = true,
1618 	.c1_demotion_supported = true,
1619 	.use_acpi = true,
1620 };
1621 
1622 static const struct idle_cpu idle_cpu_gnr __initconst = {
1623 	.state_table = gnr_cstates,
1624 	.disable_promotion_to_c1e = true,
1625 	.c1_demotion_supported = true,
1626 	.use_acpi = true,
1627 };
1628 
1629 static const struct idle_cpu idle_cpu_gnrd __initconst = {
1630 	.state_table = gnrd_cstates,
1631 	.disable_promotion_to_c1e = true,
1632 	.c1_demotion_supported = true,
1633 	.use_acpi = true,
1634 };
1635 
1636 static const struct idle_cpu idle_cpu_avn __initconst = {
1637 	.state_table = avn_cstates,
1638 	.disable_promotion_to_c1e = true,
1639 	.use_acpi = true,
1640 };
1641 
1642 static const struct idle_cpu idle_cpu_knl __initconst = {
1643 	.state_table = knl_cstates,
1644 	.use_acpi = true,
1645 };
1646 
1647 static const struct idle_cpu idle_cpu_bxt __initconst = {
1648 	.state_table = bxt_cstates,
1649 	.disable_promotion_to_c1e = true,
1650 };
1651 
1652 static const struct idle_cpu idle_cpu_dnv __initconst = {
1653 	.state_table = dnv_cstates,
1654 	.disable_promotion_to_c1e = true,
1655 	.use_acpi = true,
1656 };
1657 
1658 static const struct idle_cpu idle_cpu_tmt __initconst = {
1659 	.disable_promotion_to_c1e = true,
1660 };
1661 
1662 static const struct idle_cpu idle_cpu_snr __initconst = {
1663 	.state_table = snr_cstates,
1664 	.disable_promotion_to_c1e = true,
1665 	.use_acpi = true,
1666 };
1667 
1668 static const struct idle_cpu idle_cpu_grr __initconst = {
1669 	.state_table = grr_cstates,
1670 	.disable_promotion_to_c1e = true,
1671 	.c1_demotion_supported = true,
1672 	.use_acpi = true,
1673 };
1674 
1675 static const struct idle_cpu idle_cpu_srf __initconst = {
1676 	.state_table = srf_cstates,
1677 	.disable_promotion_to_c1e = true,
1678 	.c1_demotion_supported = true,
1679 	.use_acpi = true,
1680 };
1681 
1682 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1683 	X86_MATCH_VFM(INTEL_NEHALEM_EP,		&idle_cpu_nhx),
1684 	X86_MATCH_VFM(INTEL_NEHALEM,		&idle_cpu_nehalem),
1685 	X86_MATCH_VFM(INTEL_NEHALEM_G,		&idle_cpu_nehalem),
1686 	X86_MATCH_VFM(INTEL_WESTMERE,		&idle_cpu_nehalem),
1687 	X86_MATCH_VFM(INTEL_WESTMERE_EP,	&idle_cpu_nhx),
1688 	X86_MATCH_VFM(INTEL_NEHALEM_EX,		&idle_cpu_nhx),
1689 	X86_MATCH_VFM(INTEL_ATOM_BONNELL,	&idle_cpu_atom),
1690 	X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID,	&idle_cpu_lincroft),
1691 	X86_MATCH_VFM(INTEL_WESTMERE_EX,	&idle_cpu_nhx),
1692 	X86_MATCH_VFM(INTEL_SANDYBRIDGE,	&idle_cpu_snb),
1693 	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,	&idle_cpu_snx),
1694 	X86_MATCH_VFM(INTEL_ATOM_SALTWELL,	&idle_cpu_atom),
1695 	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT,	&idle_cpu_byt),
1696 	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1697 	X86_MATCH_VFM(INTEL_ATOM_AIRMONT,	&idle_cpu_cht),
1698 	X86_MATCH_VFM(INTEL_IVYBRIDGE,		&idle_cpu_ivb),
1699 	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,	&idle_cpu_ivt),
1700 	X86_MATCH_VFM(INTEL_HASWELL,		&idle_cpu_hsw),
1701 	X86_MATCH_VFM(INTEL_HASWELL_X,		&idle_cpu_hsx),
1702 	X86_MATCH_VFM(INTEL_HASWELL_L,		&idle_cpu_hsw),
1703 	X86_MATCH_VFM(INTEL_HASWELL_G,		&idle_cpu_hsw),
1704 	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D,	&idle_cpu_avn),
1705 	X86_MATCH_VFM(INTEL_BROADWELL,		&idle_cpu_bdw),
1706 	X86_MATCH_VFM(INTEL_BROADWELL_G,	&idle_cpu_bdw),
1707 	X86_MATCH_VFM(INTEL_BROADWELL_X,	&idle_cpu_bdx),
1708 	X86_MATCH_VFM(INTEL_BROADWELL_D,	&idle_cpu_bdx),
1709 	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&idle_cpu_skl),
1710 	X86_MATCH_VFM(INTEL_SKYLAKE,		&idle_cpu_skl),
1711 	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&idle_cpu_skl),
1712 	X86_MATCH_VFM(INTEL_KABYLAKE,		&idle_cpu_skl),
1713 	X86_MATCH_VFM(INTEL_SKYLAKE_X,		&idle_cpu_skx),
1714 	X86_MATCH_VFM(INTEL_ICELAKE_X,		&idle_cpu_icx),
1715 	X86_MATCH_VFM(INTEL_ICELAKE_D,		&idle_cpu_icx),
1716 	X86_MATCH_VFM(INTEL_ALDERLAKE,		&idle_cpu_adl),
1717 	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&idle_cpu_adl_l),
1718 	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&idle_cpu_mtl_l),
1719 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&idle_cpu_ptl),
1720 	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&idle_cpu_gmt),
1721 	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&idle_cpu_spr),
1722 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&idle_cpu_spr),
1723 	X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,	&idle_cpu_gnr),
1724 	X86_MATCH_VFM(INTEL_GRANITERAPIDS_D,	&idle_cpu_gnrd),
1725 	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,	&idle_cpu_knl),
1726 	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,	&idle_cpu_knl),
1727 	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,	&idle_cpu_bxt),
1728 	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS,	&idle_cpu_bxt),
1729 	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,	&idle_cpu_dnv),
1730 	X86_MATCH_VFM(INTEL_ATOM_TREMONT,       &idle_cpu_tmt),
1731 	X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,     &idle_cpu_tmt),
1732 	X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,	&idle_cpu_snr),
1733 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,	&idle_cpu_grr),
1734 	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,	&idle_cpu_srf),
1735 	X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X,	&idle_cpu_srf),
1736 	{}
1737 };
1738 
1739 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1740 	X86_MATCH_VENDOR_FAM_FEATURE(INTEL, X86_FAMILY_ANY, X86_FEATURE_MWAIT, NULL),
1741 	{}
1742 };
1743 
1744 static bool __init intel_idle_max_cstate_reached(int cstate)
1745 {
1746 	if (cstate + 1 > max_cstate) {
1747 		pr_info("max_cstate %d reached\n", max_cstate);
1748 		return true;
1749 	}
1750 	return false;
1751 }
1752 
1753 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1754 {
1755 	unsigned long eax = flg2MWAIT(state->flags);
1756 
1757 	if (boot_cpu_has(X86_FEATURE_ARAT))
1758 		return false;
1759 
1760 	/*
1761 	 * Switch over to one-shot tick broadcast if the target C-state
1762 	 * is deeper than C1.
1763 	 */
1764 	return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1765 }
1766 
1767 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1768 #include <acpi/processor.h>
1769 
1770 static bool no_acpi __read_mostly;
1771 module_param(no_acpi, bool, 0444);
1772 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1773 
1774 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1775 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1776 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1777 
1778 static bool no_native __read_mostly; /* No effect if no_acpi is set. */
1779 module_param_named(no_native, no_native, bool, 0444);
1780 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
1781 
1782 static struct acpi_processor_power acpi_state_table __initdata;
1783 static bool acpi_lpi_available __initdata;
1784 
1785 /**
1786  * intel_idle_cst_usable - Check if the _CST information can be used.
1787  *
1788  * Check if all of the C-states listed by _CST in the max_cstate range are
1789  * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1790  */
1791 static bool __init intel_idle_cst_usable(void)
1792 {
1793 	int cstate, limit;
1794 
1795 	limit = min3(CPUIDLE_STATE_MAX, max_cstate + 1, acpi_state_table.count);
1796 	for (cstate = 1; cstate < limit; cstate++) {
1797 		struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1798 
1799 		if (cx->entry_method != ACPI_CSTATE_FFH)
1800 			return false;
1801 	}
1802 
1803 	return true;
1804 }
1805 
1806 static bool __init intel_idle_acpi_extract_lpi_cstates(void)
1807 {
1808 	unsigned int cpu;
1809 
1810 	for_each_possible_cpu(cpu) {
1811 		struct acpi_processor *pr;
1812 
1813 		pr = per_cpu(processors, cpu);
1814 		if (!pr)
1815 			continue;
1816 
1817 		if (acpi_processor_extract_lpi_info(pr->handle,
1818 						    &acpi_state_table, true))
1819 			continue;
1820 
1821 		acpi_lpi_available = true;
1822 		return true;
1823 	}
1824 
1825 	pr_debug("No ACPI _LPI idle states\n");
1826 	return false;
1827 }
1828 
1829 static bool __init intel_idle_acpi_extract_cst_cstates(void)
1830 {
1831 	unsigned int cpu;
1832 
1833 	for_each_possible_cpu(cpu) {
1834 		struct acpi_processor *pr;
1835 
1836 		pr = per_cpu(processors, cpu);
1837 		if (!pr)
1838 			continue;
1839 
1840 		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1841 			continue;
1842 
1843 		acpi_state_table.count++;
1844 
1845 		if (!intel_idle_cst_usable())
1846 			continue;
1847 
1848 		return true;
1849 	}
1850 
1851 	pr_debug("ACPI _CST not found or not usable\n");
1852 	return false;
1853 }
1854 
1855 static bool __init intel_idle_acpi_extract_cstates(void)
1856 {
1857 	if (intel_idle_acpi_extract_lpi_cstates())
1858 		return true;
1859 
1860 	if (intel_idle_acpi_extract_cst_cstates())
1861 		return true;
1862 
1863 	return false;
1864 }
1865 
1866 static bool __init intel_idle_acpi_probe(void)
1867 {
1868 	if (no_acpi) {
1869 		pr_debug("Not allowed to use ACPI for C-states extraction\n");
1870 		return false;
1871 	}
1872 
1873 	if (intel_idle_acpi_extract_cstates() &&
1874 	    acpi_processor_claim_cst_control())
1875 		return true;
1876 
1877 	acpi_state_table.count = 0;
1878 	return false;
1879 }
1880 
1881 static void __init intel_idle_complete_state_init(struct cpuidle_state *state)
1882 {
1883 	if (intel_idle_state_needs_timer_stop(state))
1884 		state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1885 
1886 	state->enter = intel_idle;
1887 	state->enter_dead = intel_idle_enter_dead;
1888 	state->enter_s2idle = intel_idle_s2idle;
1889 }
1890 
1891 static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv)
1892 {
1893 	int index;
1894 
1895 	for (index = 0; index < acpi_state_table.count; index++) {
1896 		struct acpi_lpi_state *lpi_state;
1897 		struct cpuidle_state *state;
1898 
1899 		if (intel_idle_max_cstate_reached(index))
1900 			break;
1901 
1902 		lpi_state = &acpi_state_table.lpi_states[index];
1903 
1904 		state = &drv->states[drv->state_count++];
1905 
1906 		scnprintf(state->name, CPUIDLE_NAME_LEN, "C%d_LPI", index + 1);
1907 		strscpy(state->desc, lpi_state->desc);
1908 		state->exit_latency = lpi_state->wake_latency;
1909 		state->target_residency = lpi_state->min_residency;
1910 		state->flags = MWAIT2flg(lpi_state->address);
1911 		/*
1912 		 * Assume that entering any of the idle states extracted from
1913 		 * _LPI except for the first two will cause the TLB to be
1914 		 * flushed and let the core call leave_mm() for them upfront
1915 		 * to avoid unnecessary wakeups due to TLB shootdowns.
1916 		 */
1917 		if (index > 1)
1918 			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1919 
1920 		if (disabled_states_mask & BIT(index + 1))
1921 			state->flags |= CPUIDLE_FLAG_OFF;
1922 
1923 		intel_idle_complete_state_init(state);
1924 
1925 		pr_info("%s: MWAIT hint 0x%x\n", state->name, flg2MWAIT(state->flags));
1926 	}
1927 
1928 	/*
1929 	 * Assume the first idle state in the table to be C1 and if any deeper
1930 	 * idle states are exposed while X86_FEATURE_NONSTOP_TSC is unset, mark
1931 	 * the TSC as unstable.
1932 	 */
1933 	if (index > 1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1934 		mark_tsc_unstable("TSC halts in idle");
1935 }
1936 
1937 static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv)
1938 {
1939 	int cstate, limit = min(CPUIDLE_STATE_MAX, acpi_state_table.count);
1940 
1941 	/*
1942 	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1943 	 * the interesting states are ACPI_CSTATE_FFH.
1944 	 */
1945 	for (cstate = 1; cstate < limit; cstate++) {
1946 		struct acpi_processor_cx *cx;
1947 		struct cpuidle_state *state;
1948 
1949 		if (intel_idle_max_cstate_reached(cstate - 1))
1950 			break;
1951 
1952 		cx = &acpi_state_table.states[cstate];
1953 
1954 		state = &drv->states[drv->state_count++];
1955 
1956 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1957 		strscpy(state->desc, cx->desc);
1958 		state->exit_latency = cx->latency;
1959 		/*
1960 		 * For C1-type C-states use the same number for both the exit
1961 		 * latency and target residency, because that is the case for
1962 		 * C1 in the majority of the static C-states tables above.
1963 		 * For the other types of C-states, however, set the target
1964 		 * residency to 3 times the exit latency which should lead to
1965 		 * a reasonable balance between energy-efficiency and
1966 		 * performance in the majority of interesting cases.
1967 		 */
1968 		state->target_residency = cx->latency;
1969 		if (cx->type > ACPI_STATE_C1)
1970 			state->target_residency *= 3;
1971 
1972 		state->flags = MWAIT2flg(cx->address);
1973 		if (cx->type > ACPI_STATE_C2)
1974 			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1975 
1976 		if (disabled_states_mask & BIT(cstate))
1977 			state->flags |= CPUIDLE_FLAG_OFF;
1978 
1979 		if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1980 			mark_tsc_unstable("TSC halts in idle");
1981 
1982 		intel_idle_complete_state_init(state);
1983 	}
1984 }
1985 
1986 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1987 {
1988 	if (acpi_lpi_available)
1989 		intel_idle_init_cstates_acpi_lpi(drv);
1990 	else
1991 		intel_idle_init_cstates_acpi_cst(drv);
1992 }
1993 
1994 static bool __init intel_idle_acpi_hint_match(unsigned int flags, u32 acpi_hint,
1995 					      u32 table_hint)
1996 {
1997 	if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
1998 		acpi_hint &= ~MWAIT_SUBSTATE_MASK;
1999 		table_hint &= ~MWAIT_SUBSTATE_MASK;
2000 	}
2001 	return acpi_hint == table_hint;
2002 }
2003 
2004 static bool __init intel_idle_off_by_default_lpi(unsigned int flags, u32 mwait_hint)
2005 {
2006 	int index;
2007 
2008 	for (index = 0; index < acpi_state_table.count; index++) {
2009 		u32 acpi_hint = acpi_state_table.lpi_states[index].address;
2010 
2011 		if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
2012 			return false;
2013 	}
2014 	return true;
2015 }
2016 
2017 static bool __init intel_idle_off_by_default_cst(unsigned int flags, u32 mwait_hint)
2018 {
2019 	int cstate, limit = min(CPUIDLE_STATE_MAX, acpi_state_table.count);
2020 
2021 	/*
2022 	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
2023 	 * the interesting states are ACPI_CSTATE_FFH.
2024 	 */
2025 	for (cstate = 1; cstate < limit; cstate++) {
2026 		u32 acpi_hint = acpi_state_table.states[cstate].address;
2027 
2028 		if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
2029 			return false;
2030 	}
2031 	return true;
2032 }
2033 
2034 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
2035 {
2036 	/*
2037 	 * If there is no C-states information in the ACPI tables, do not
2038 	 * disable any C-states by default.
2039 	 */
2040 	if (!acpi_state_table.count)
2041 		return false;
2042 
2043 	if (acpi_lpi_available)
2044 		return intel_idle_off_by_default_lpi(flags, mwait_hint);
2045 
2046 	return intel_idle_off_by_default_cst(flags, mwait_hint);
2047 }
2048 
2049 static inline bool ignore_native(void)
2050 {
2051 	return no_native && !no_acpi;
2052 }
2053 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
2054 #define force_use_acpi	(false)
2055 
2056 static inline bool intel_idle_acpi_probe(void) { return false; }
2057 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
2058 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
2059 {
2060 	return false;
2061 }
2062 static inline bool ignore_native(void) { return false; }
2063 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
2064 
2065 /**
2066  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
2067  *
2068  * Tune IVT multi-socket targets.
2069  * Assumption: num_sockets == (max_package_num + 1).
2070  */
2071 static void __init ivt_idle_state_table_update(void)
2072 {
2073 	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
2074 	int cpu, package_num, num_sockets = 1;
2075 
2076 	for_each_online_cpu(cpu) {
2077 		package_num = topology_physical_package_id(cpu);
2078 		if (package_num + 1 > num_sockets) {
2079 			num_sockets = package_num + 1;
2080 
2081 			if (num_sockets > 4) {
2082 				cpuidle_state_table = ivt_cstates_8s;
2083 				return;
2084 			}
2085 		}
2086 	}
2087 
2088 	if (num_sockets > 2)
2089 		cpuidle_state_table = ivt_cstates_4s;
2090 
2091 	/* else, 1 and 2 socket systems use default ivt_cstates */
2092 }
2093 
2094 /**
2095  * irtl_2_usec - IRTL to microseconds conversion.
2096  * @irtl: IRTL MSR value.
2097  *
2098  * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
2099  */
2100 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
2101 {
2102 	static const unsigned int irtl_ns_units[] __initconst = {
2103 		1, 32, 1024, 32768, 1048576, 33554432, 0, 0
2104 	};
2105 	unsigned long long ns;
2106 
2107 	if (!irtl)
2108 		return 0;
2109 
2110 	ns = irtl_ns_units[(irtl >> 10) & 0x7];
2111 
2112 	return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
2113 }
2114 
2115 /**
2116  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
2117  *
2118  * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
2119  * definitive maximum latency and use the same value for target_residency.
2120  */
2121 static void __init bxt_idle_state_table_update(void)
2122 {
2123 	unsigned long long msr;
2124 	unsigned int usec;
2125 
2126 	rdmsrq(MSR_PKGC6_IRTL, msr);
2127 	usec = irtl_2_usec(msr);
2128 	if (usec) {
2129 		bxt_cstates[2].exit_latency = usec;
2130 		bxt_cstates[2].target_residency = usec;
2131 	}
2132 
2133 	rdmsrq(MSR_PKGC7_IRTL, msr);
2134 	usec = irtl_2_usec(msr);
2135 	if (usec) {
2136 		bxt_cstates[3].exit_latency = usec;
2137 		bxt_cstates[3].target_residency = usec;
2138 	}
2139 
2140 	rdmsrq(MSR_PKGC8_IRTL, msr);
2141 	usec = irtl_2_usec(msr);
2142 	if (usec) {
2143 		bxt_cstates[4].exit_latency = usec;
2144 		bxt_cstates[4].target_residency = usec;
2145 	}
2146 
2147 	rdmsrq(MSR_PKGC9_IRTL, msr);
2148 	usec = irtl_2_usec(msr);
2149 	if (usec) {
2150 		bxt_cstates[5].exit_latency = usec;
2151 		bxt_cstates[5].target_residency = usec;
2152 	}
2153 
2154 	rdmsrq(MSR_PKGC10_IRTL, msr);
2155 	usec = irtl_2_usec(msr);
2156 	if (usec) {
2157 		bxt_cstates[6].exit_latency = usec;
2158 		bxt_cstates[6].target_residency = usec;
2159 	}
2160 
2161 }
2162 
2163 /**
2164  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
2165  *
2166  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
2167  */
2168 static void __init sklh_idle_state_table_update(void)
2169 {
2170 	unsigned long long msr;
2171 	unsigned int eax, ebx, ecx, edx;
2172 
2173 
2174 	/* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
2175 	if (max_cstate <= 7)
2176 		return;
2177 
2178 	/* if PC10 not present in CPUID.MWAIT.EDX */
2179 	if ((mwait_substates & (0xF << 28)) == 0)
2180 		return;
2181 
2182 	rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
2183 
2184 	/* PC10 is not enabled in PKG C-state limit */
2185 	if ((msr & 0xF) != 8)
2186 		return;
2187 
2188 	ecx = 0;
2189 	cpuid(7, &eax, &ebx, &ecx, &edx);
2190 
2191 	/* if SGX is present */
2192 	if (ebx & (1 << 2)) {
2193 
2194 		rdmsrq(MSR_IA32_FEAT_CTL, msr);
2195 
2196 		/* if SGX is enabled */
2197 		if (msr & (1 << 18))
2198 			return;
2199 	}
2200 
2201 	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */
2202 	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */
2203 }
2204 
2205 /**
2206  * skx_is_pc6_disabled() - Check if PC6 is disabled in BIOS.
2207  *
2208  * Return: %true if PC6 is disabled, %false otherwise.
2209  */
2210 static bool __init skx_is_pc6_disabled(void)
2211 {
2212 	u64 msr;
2213 
2214 	rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
2215 
2216 	/*
2217 	 * 000b: C0/C1 (no package C-state support)
2218 	 * 001b: C2
2219 	 * 010b: C6 (non-retention)
2220 	 * 011b: C6 (retention)
2221 	 * 111b: No Package C state limits.
2222 	 */
2223 	return (msr & SKX_PKG_CST_LIMIT_MASK) < SKX_PKG_CST_LIMIT_PC6;
2224 }
2225 
2226 /**
2227  * skx_idle_state_table_update - Adjust the SKX/CLX idle states table.
2228  *
2229  * Adjust Sky Lake or Cascade Lake Xeon idle states if PC6 is disabled in BIOS.
2230  * Use the CC6 + PC0 latency and 3 times of that latency for target_residency.
2231  * This is consistent with how the intel_idle driver uses _CST to set the
2232  * target_residency.
2233  */
2234 static void __init skx_idle_state_table_update(void)
2235 {
2236 	if (skx_is_pc6_disabled()) {
2237 		skx_cstates[2].exit_latency = 92;
2238 		skx_cstates[2].target_residency = 276;
2239 	}
2240 }
2241 
2242 /**
2243  * spr_idle_state_table_update - Adjust Sapphire Rapids Xeon idle states table.
2244  *
2245  * By default, the C6 state assumes the worst-case scenario of package C6.
2246  * However, if PC6 is disabled in BIOS, update the numbers to match core C6.
2247  */
2248 static void __init spr_idle_state_table_update(void)
2249 {
2250 	if (skx_is_pc6_disabled()) {
2251 		spr_cstates[2].exit_latency = 190;
2252 		spr_cstates[2].target_residency = 600;
2253 	}
2254 }
2255 
2256 /**
2257  * drop_pc6_redundant_cstates() - Drop C-states redundant when PC6 is disabled.
2258  * @states: Idle states table to modify.
2259  *
2260  * When PC6 is disabled in BIOS, C-states that exist solely to enable PC6
2261  * entry (such as C6P or C6SP) become identical to shallower C-states like
2262  * C6, and are therefore redundant. Should be called only on systems with
2263  * multiple C6 flavors.
2264  */
2265 static void __init drop_pc6_redundant_cstates(struct cpuidle_state *states)
2266 {
2267 	int count;
2268 
2269 	if (!skx_is_pc6_disabled())
2270 		/* PC6 is not disabled, nothing to do */
2271 		return;
2272 
2273 	for (count = 0; states[count].enter; count++)
2274 		continue;
2275 
2276 	if (count < 2) {
2277 		pr_debug("Too few idle states to drop PC6-redundant states\n");
2278 		return;
2279 	}
2280 
2281 	/*
2282 	 * Sanity check: At this point all platforms with multiple C6 flavors
2283 	 * use the CPUIDLE_FLAG_PARTIAL_HINT_MATCH flag. And the last state in
2284 	 * the table is the one that becomes redundant when PC6 is disabled.
2285 	 */
2286 	if (!(states[count - 1].flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH)) {
2287 		pr_debug("Can't drop PC6-redundant states: unexpected flags\n");
2288 		return;
2289 	}
2290 
2291 	/*
2292 	 * On all current platforms with multiple C6 flavors, there is only one
2293 	 * C-state that becomes redundant when PC6 is disabled. This state is
2294 	 * the last one in the table. Drop it by marking it with
2295 	 * CPUIDLE_FLAG_UNUSABLE so that cpuidle excludes it when registering
2296 	 * idle states.
2297 	 */
2298 	pr_info("Dropping idle state %s because PC6 is disabled\n",
2299 		states[count - 1].name);
2300 	states[count - 1].flags |= CPUIDLE_FLAG_UNUSABLE;
2301 }
2302 
2303 /**
2304  * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2305  */
2306 static void __init byt_cht_auto_demotion_disable(void)
2307 {
2308 	wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2309 	wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2310 }
2311 
2312 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2313 {
2314 	unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2315 					MWAIT_CSTATE_MASK;
2316 	unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2317 					MWAIT_SUBSTATE_MASK;
2318 
2319 	/* Ignore the C-state if there are NO sub-states in CPUID for it. */
2320 	if (num_substates == 0)
2321 		return false;
2322 
2323 	if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2324 		mark_tsc_unstable("TSC halts in idle states deeper than C2");
2325 
2326 	return true;
2327 }
2328 
2329 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2330 {
2331 	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2332 		/*
2333 		 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2334 		 * is not currently supported but this driver.
2335 		 */
2336 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2337 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2338 		state->enter = intel_idle_xstate;
2339 		return;
2340 	}
2341 
2342 	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2343 			((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2344 		/*
2345 		 * IBRS mitigation requires that C-states are entered
2346 		 * with interrupts disabled.
2347 		 */
2348 		if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2349 			state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2350 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2351 		state->enter = intel_idle_ibrs;
2352 		return;
2353 	}
2354 
2355 	if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2356 		state->enter = intel_idle_irq;
2357 		return;
2358 	}
2359 
2360 	if (force_irq_on) {
2361 		pr_info("forced intel_idle_irq for state %d\n", cstate);
2362 		state->enter = intel_idle_irq;
2363 	}
2364 }
2365 
2366 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2367 {
2368 	int cstate;
2369 
2370 	switch (boot_cpu_data.x86_vfm) {
2371 	case INTEL_IVYBRIDGE_X:
2372 		ivt_idle_state_table_update();
2373 		break;
2374 	case INTEL_ATOM_GOLDMONT:
2375 	case INTEL_ATOM_GOLDMONT_PLUS:
2376 		bxt_idle_state_table_update();
2377 		break;
2378 	case INTEL_SKYLAKE:
2379 		sklh_idle_state_table_update();
2380 		break;
2381 	case INTEL_SKYLAKE_X:
2382 		skx_idle_state_table_update();
2383 		break;
2384 	case INTEL_SAPPHIRERAPIDS_X:
2385 	case INTEL_EMERALDRAPIDS_X:
2386 		spr_idle_state_table_update();
2387 		break;
2388 	case INTEL_ATOM_SILVERMONT:
2389 	case INTEL_ATOM_AIRMONT:
2390 		byt_cht_auto_demotion_disable();
2391 		break;
2392 	case INTEL_GRANITERAPIDS_D:
2393 	case INTEL_GRANITERAPIDS_X:
2394 	case INTEL_ATOM_CRESTMONT_X:
2395 	case INTEL_ATOM_DARKMONT_X:
2396 		drop_pc6_redundant_cstates(cpuidle_state_table);
2397 		break;
2398 	}
2399 
2400 	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2401 		struct cpuidle_state *state;
2402 		unsigned int mwait_hint;
2403 
2404 		if (intel_idle_max_cstate_reached(cstate))
2405 			break;
2406 
2407 		if (!cpuidle_state_table[cstate].enter &&
2408 		    !cpuidle_state_table[cstate].enter_s2idle)
2409 			break;
2410 
2411 		if (!cpuidle_state_table[cstate].enter_dead)
2412 			cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead;
2413 
2414 		/* If marked as unusable, skip this state. */
2415 		if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2416 			pr_debug("state %s is disabled\n",
2417 				 cpuidle_state_table[cstate].name);
2418 			continue;
2419 		}
2420 
2421 		mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2422 		if (!intel_idle_verify_cstate(mwait_hint))
2423 			continue;
2424 
2425 		/* Structure copy. */
2426 		drv->states[drv->state_count] = cpuidle_state_table[cstate];
2427 		state = &drv->states[drv->state_count];
2428 
2429 		state_update_enter_method(state, cstate);
2430 
2431 
2432 		if ((disabled_states_mask & BIT(drv->state_count)) ||
2433 		    ((icpu->use_acpi || force_use_acpi) &&
2434 		     intel_idle_off_by_default(state->flags, mwait_hint) &&
2435 		     !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2436 			state->flags |= CPUIDLE_FLAG_OFF;
2437 
2438 		if (intel_idle_state_needs_timer_stop(state))
2439 			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2440 
2441 		drv->state_count++;
2442 	}
2443 }
2444 
2445 /**
2446  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2447  * @drv: cpuidle driver structure to initialize.
2448  */
2449 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2450 {
2451 	cpuidle_poll_state_init(drv);
2452 
2453 	if (disabled_states_mask & BIT(0))
2454 		drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2455 
2456 	drv->state_count = 1;
2457 
2458 	if (icpu && icpu->state_table)
2459 		intel_idle_init_cstates_icpu(drv);
2460 	else
2461 		intel_idle_init_cstates_acpi(drv);
2462 }
2463 
2464 static void auto_demotion_disable(void)
2465 {
2466 	unsigned long long msr_bits;
2467 
2468 	rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2469 	msr_bits &= ~auto_demotion_disable_flags;
2470 	wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2471 }
2472 
2473 static void c1e_promotion_enable(void)
2474 {
2475 	unsigned long long msr_bits;
2476 
2477 	rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
2478 	msr_bits |= 0x2;
2479 	wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
2480 }
2481 
2482 static void c1e_promotion_disable(void)
2483 {
2484 	unsigned long long msr_bits;
2485 
2486 	rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
2487 	msr_bits &= ~0x2;
2488 	wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
2489 }
2490 
2491 /**
2492  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2493  * @cpu: CPU to initialize.
2494  *
2495  * Register a cpuidle device object for @cpu and update its MSRs in accordance
2496  * with the processor model flags.
2497  */
2498 static int intel_idle_cpu_init(unsigned int cpu)
2499 {
2500 	struct cpuidle_device *dev;
2501 
2502 	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2503 	dev->cpu = cpu;
2504 
2505 	if (cpuidle_register_device(dev)) {
2506 		pr_debug("cpuidle_register_device %d failed!\n", cpu);
2507 		return -EIO;
2508 	}
2509 
2510 	if (auto_demotion_disable_flags)
2511 		auto_demotion_disable();
2512 
2513 	if (c1e_promotion == C1E_PROMOTION_ENABLE)
2514 		c1e_promotion_enable();
2515 	else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2516 		c1e_promotion_disable();
2517 
2518 	return 0;
2519 }
2520 
2521 static int intel_idle_cpu_online(unsigned int cpu)
2522 {
2523 	struct cpuidle_device *dev;
2524 
2525 	if (!boot_cpu_has(X86_FEATURE_ARAT))
2526 		tick_broadcast_enable();
2527 
2528 	/*
2529 	 * Some systems can hotplug a cpu at runtime after
2530 	 * the kernel has booted, we have to initialize the
2531 	 * driver in this case
2532 	 */
2533 	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2534 	if (!dev->registered)
2535 		return intel_idle_cpu_init(cpu);
2536 
2537 	return 0;
2538 }
2539 
2540 /**
2541  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2542  */
2543 static void __init intel_idle_cpuidle_devices_uninit(void)
2544 {
2545 	int i;
2546 
2547 	for_each_online_cpu(i)
2548 		cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2549 }
2550 
2551 static void intel_c1_demotion_toggle(void *enable)
2552 {
2553 	unsigned long long msr_val;
2554 
2555 	rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2556 	/*
2557 	 * Enable/disable C1 undemotion along with C1 demotion, as this is the
2558 	 * most sensible configuration in general.
2559 	 */
2560 	if (enable)
2561 		msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE;
2562 	else
2563 		msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE);
2564 	wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2565 }
2566 
2567 static ssize_t intel_c1_demotion_store(struct device *dev,
2568 				       struct device_attribute *attr,
2569 				       const char *buf, size_t count)
2570 {
2571 	bool enable;
2572 	int err;
2573 
2574 	err = kstrtobool(buf, &enable);
2575 	if (err)
2576 		return err;
2577 
2578 	mutex_lock(&c1_demotion_mutex);
2579 	/* Enable/disable C1 demotion on all CPUs */
2580 	on_each_cpu(intel_c1_demotion_toggle, (void *)enable, 1);
2581 	mutex_unlock(&c1_demotion_mutex);
2582 
2583 	return count;
2584 }
2585 
2586 static ssize_t intel_c1_demotion_show(struct device *dev,
2587 				      struct device_attribute *attr, char *buf)
2588 {
2589 	unsigned long long msr_val;
2590 
2591 	/*
2592 	 * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other
2593 	 * configuration would be a BIOS bug.
2594 	 */
2595 	rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2596 	return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE));
2597 }
2598 static DEVICE_ATTR_RW(intel_c1_demotion);
2599 
2600 static int __init intel_idle_sysfs_init(void)
2601 {
2602 	int err;
2603 
2604 	if (!c1_demotion_supported)
2605 		return 0;
2606 
2607 	sysfs_root = bus_get_dev_root(&cpu_subsys);
2608 	if (!sysfs_root)
2609 		return 0;
2610 
2611 	err = sysfs_add_file_to_group(&sysfs_root->kobj,
2612 				      &dev_attr_intel_c1_demotion.attr,
2613 				      "cpuidle");
2614 	if (err) {
2615 		put_device(sysfs_root);
2616 		return err;
2617 	}
2618 
2619 	return 0;
2620 }
2621 
2622 static void __init intel_idle_sysfs_uninit(void)
2623 {
2624 	if (!sysfs_root)
2625 		return;
2626 
2627 	sysfs_remove_file_from_group(&sysfs_root->kobj,
2628 				     &dev_attr_intel_c1_demotion.attr,
2629 				     "cpuidle");
2630 	put_device(sysfs_root);
2631 }
2632 
2633  /**
2634   * get_cmdline_field - Get the current field from a cmdline string.
2635   * @args: The cmdline string to get the current field from.
2636   * @field: Pointer to the current field upon return.
2637   * @sep: The fields separator character.
2638   *
2639   * Examples:
2640   *   Input: args="C1:1:1,C1E:2:10", sep=':'
2641   *   Output: field="C1", return "1:1,C1E:2:10"
2642   *   Input: args="C1:1:1,C1E:2:10", sep=','
2643   *   Output: field="C1:1:1", return "C1E:2:10"
2644   *   Ipnut: args="::", sep=':'
2645   *   Output: field="", return ":"
2646   *
2647   * Return: The continuation of the cmdline string after the field or NULL.
2648   */
2649 static char *get_cmdline_field(char *args, char **field, char sep)
2650 {
2651 	unsigned int i;
2652 
2653 	for (i = 0; args[i] && !isspace(args[i]); i++) {
2654 		if (args[i] == sep)
2655 			break;
2656 	}
2657 
2658 	*field = args;
2659 
2660 	if (args[i] != sep)
2661 		return NULL;
2662 
2663 	args[i] = '\0';
2664 	return args + i + 1;
2665 }
2666 
2667 /**
2668  * validate_cmdline_cstate - Validate a C-state from cmdline.
2669  * @state: The C-state to validate.
2670  * @prev_state: The previous C-state in the table or NULL.
2671  *
2672  * Return: 0 if the C-state is valid or -EINVAL otherwise.
2673  */
2674 static int validate_cmdline_cstate(struct cpuidle_state *state,
2675 				   struct cpuidle_state *prev_state)
2676 {
2677 	if (state->exit_latency == 0)
2678 		/* Exit latency 0 can only be used for the POLL state */
2679 		return -EINVAL;
2680 
2681 	if (state->exit_latency > MAX_CMDLINE_LATENCY_US)
2682 		return -EINVAL;
2683 
2684 	if (state->target_residency > MAX_CMDLINE_RESIDENCY_US)
2685 		return -EINVAL;
2686 
2687 	if (state->target_residency < state->exit_latency)
2688 		return -EINVAL;
2689 
2690 	if (!prev_state)
2691 		return 0;
2692 
2693 	if (state->exit_latency <= prev_state->exit_latency)
2694 		return -EINVAL;
2695 
2696 	if (state->target_residency <= prev_state->target_residency)
2697 		return -EINVAL;
2698 
2699 	return 0;
2700 }
2701 
2702 /**
2703  * cmdline_table_adjust - Adjust the C-states table with data from cmdline.
2704  * @drv: cpuidle driver (assumed to point to intel_idle_driver).
2705  *
2706  * Adjust the C-states table with data from the 'intel_idle.table' module
2707  * parameter (if specified).
2708  */
2709 static void __init cmdline_table_adjust(struct cpuidle_driver *drv)
2710 {
2711 	char *args = cmdline_table_str;
2712 	struct cpuidle_state *state;
2713 	int i;
2714 
2715 	if (args[0] == '\0')
2716 		/* The 'intel_idle.table' module parameter was not specified */
2717 		return;
2718 
2719 	/* Create a copy of the C-states table */
2720 	for (i = 0; i < drv->state_count; i++)
2721 		cmdline_states[i] = drv->states[i];
2722 
2723 	/*
2724 	 * Adjust the C-states table copy with data from the 'intel_idle.table'
2725 	 * module parameter.
2726 	 */
2727 	while (args) {
2728 		char *fields, *name, *val;
2729 
2730 		/*
2731 		 * Get the next C-state definition, which is expected to be
2732 		 * '<name>:<latency_us>:<target_residency_us>'. Treat "empty"
2733 		 * fields as unchanged. For example,
2734 		 * '<name>::<target_residency_us>' leaves the latency unchanged.
2735 		 */
2736 		args = get_cmdline_field(args, &fields, ',');
2737 
2738 		/* name */
2739 		fields = get_cmdline_field(fields, &name, ':');
2740 		if (!fields)
2741 			goto error;
2742 
2743 		if (!strcmp(name, "POLL")) {
2744 			pr_err("Cannot adjust POLL\n");
2745 			continue;
2746 		}
2747 
2748 		/* Find the C-state by its name */
2749 		state = NULL;
2750 		for (i = 0; i < drv->state_count; i++) {
2751 			if (!strcmp(name, drv->states[i].name)) {
2752 				state = &cmdline_states[i];
2753 				break;
2754 			}
2755 		}
2756 
2757 		if (!state) {
2758 			pr_err("C-state '%s' was not found\n", name);
2759 			continue;
2760 		}
2761 
2762 		/* Latency */
2763 		fields = get_cmdline_field(fields, &val, ':');
2764 		if (!fields)
2765 			goto error;
2766 
2767 		if (*val) {
2768 			if (kstrtouint(val, 0, &state->exit_latency))
2769 				goto error;
2770 		}
2771 
2772 		/* Target residency */
2773 		fields = get_cmdline_field(fields, &val, ':');
2774 
2775 		if (*val) {
2776 			if (kstrtouint(val, 0, &state->target_residency))
2777 				goto error;
2778 		}
2779 
2780 		/*
2781 		 * Allow for 3 more fields, but ignore them. Helps to make
2782 		 * possible future extensions of the cmdline format backward
2783 		 * compatible.
2784 		 */
2785 		for (i = 0; fields && i < 3; i++) {
2786 			fields = get_cmdline_field(fields, &val, ':');
2787 			if (!fields)
2788 				break;
2789 		}
2790 
2791 		if (fields) {
2792 			pr_err("Too many fields for C-state '%s'\n", state->name);
2793 			goto error;
2794 		}
2795 
2796 		pr_info("C-state from cmdline: name=%s, latency=%u, residency=%u\n",
2797 			state->name, state->exit_latency, state->target_residency);
2798 	}
2799 
2800 	/* Validate the adjusted C-states, start with index 1 to skip POLL */
2801 	for (i = 1; i < drv->state_count; i++) {
2802 		struct cpuidle_state *prev_state;
2803 
2804 		state = &cmdline_states[i];
2805 		prev_state = &cmdline_states[i - 1];
2806 
2807 		if (validate_cmdline_cstate(state, prev_state)) {
2808 			pr_err("C-state '%s' validation failed\n", state->name);
2809 			goto error;
2810 		}
2811 	}
2812 
2813 	/* Copy the adjusted C-states table back */
2814 	for (i = 1; i < drv->state_count; i++)
2815 		drv->states[i] = cmdline_states[i];
2816 
2817 	pr_info("Adjusted C-states with data from 'intel_idle.table'\n");
2818 	return;
2819 
2820 error:
2821 	pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n");
2822 }
2823 
2824 #define INTEL_IDLE_INIT_QOS	20
2825 static struct pm_qos_request qos_req __initdata;
2826 
2827 static int __init intel_idle_init(void)
2828 {
2829 	const struct x86_cpu_id *id;
2830 	unsigned int eax, ebx, ecx;
2831 	int retval;
2832 
2833 	/* Do not load intel_idle at all for now if idle= is passed */
2834 	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2835 		return -ENODEV;
2836 
2837 	if (max_cstate == 0) {
2838 		pr_debug("disabled\n");
2839 		return -EPERM;
2840 	}
2841 
2842 	id = x86_match_cpu(intel_idle_ids);
2843 	if (id) {
2844 		if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2845 			pr_debug("Please enable MWAIT in BIOS SETUP\n");
2846 			return -ENODEV;
2847 		}
2848 	} else {
2849 		id = x86_match_cpu(intel_mwait_ids);
2850 		if (!id)
2851 			return -ENODEV;
2852 	}
2853 
2854 	cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);
2855 
2856 	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2857 	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2858 	    !mwait_substates)
2859 			return -ENODEV;
2860 
2861 	pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2862 
2863 	icpu = (const struct idle_cpu *)id->driver_data;
2864 	if (icpu && ignore_native()) {
2865 		pr_debug("ignoring native CPU idle states\n");
2866 		icpu = NULL;
2867 	}
2868 	if (icpu) {
2869 		if (icpu->state_table)
2870 			cpuidle_state_table = icpu->state_table;
2871 		else if (!intel_idle_acpi_probe())
2872 			return -ENODEV;
2873 
2874 		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2875 		if (icpu->disable_promotion_to_c1e)
2876 			c1e_promotion = C1E_PROMOTION_DISABLE;
2877 		if (icpu->c1_demotion_supported)
2878 			c1_demotion_supported = true;
2879 		if (icpu->use_acpi || force_use_acpi)
2880 			intel_idle_acpi_probe();
2881 	} else if (!intel_idle_acpi_probe()) {
2882 		return -ENODEV;
2883 	}
2884 
2885 	intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2886 	if (!intel_idle_cpuidle_devices)
2887 		return -ENOMEM;
2888 
2889 	intel_idle_cpuidle_driver_init(&intel_idle_driver);
2890 	cmdline_table_adjust(&intel_idle_driver);
2891 
2892 	retval = intel_idle_sysfs_init();
2893 	if (retval)
2894 		pr_warn("failed to initialized sysfs");
2895 
2896 	/*
2897 	 * Some platforms, in particular the Intel S1200BTL motherboard, have a
2898 	 * problem with using package idle states too early, so prevent that
2899 	 * from taking place until the device_initcall() phase is over.
2900 	 */
2901 	cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS);
2902 
2903 	retval = cpuidle_register_driver(&intel_idle_driver);
2904 	if (retval) {
2905 		struct cpuidle_driver *drv = cpuidle_get_driver();
2906 		printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2907 		       drv ? drv->name : "none");
2908 		goto init_driver_fail;
2909 	}
2910 
2911 	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2912 				   intel_idle_cpu_online, NULL);
2913 	if (retval < 0)
2914 		goto hp_setup_fail;
2915 
2916 	pr_debug("Local APIC timer is reliable in %s\n",
2917 		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2918 
2919 	arch_cpu_rescan_dead_smt_siblings();
2920 
2921 	return 0;
2922 
2923 hp_setup_fail:
2924 	intel_idle_cpuidle_devices_uninit();
2925 	cpuidle_unregister_driver(&intel_idle_driver);
2926 init_driver_fail:
2927 	if (cpu_latency_qos_request_active((&qos_req)))
2928 		cpu_latency_qos_remove_request(&qos_req);
2929 
2930 	intel_idle_sysfs_uninit();
2931 	free_percpu(intel_idle_cpuidle_devices);
2932 	return retval;
2933 
2934 }
2935 subsys_initcall_sync(intel_idle_init);
2936 
2937 static int __init intel_idle_init_complete(void)
2938 {
2939 	if (cpu_latency_qos_request_active((&qos_req)))
2940 		cpu_latency_qos_remove_request(&qos_req);
2941 
2942 	return 0;
2943 }
2944 device_initcall_sync(intel_idle_init_complete);
2945 
2946 /*
2947  * We are not really modular, but we used to support that.  Meaning we also
2948  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2949  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2950  * is the easiest way (currently) to continue doing that.
2951  */
2952 module_param(max_cstate, int, 0444);
2953 /*
2954  * The positions of the bits that are set in this number are the indices of the
2955  * idle states to be disabled by default (as reflected by the names of the
2956  * corresponding idle state directories in sysfs, "state0", "state1" ...
2957  * "state<i>" ..., where <i> is the index of the given state).
2958  */
2959 module_param_named(states_off, disabled_states_mask, uint, 0444);
2960 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2961 /*
2962  * Debugging option that forces the driver to enter all C-states with
2963  * interrupts enabled. Does not apply to C-states with
2964  * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2965  */
2966 module_param(force_irq_on, bool, 0444);
2967 /*
2968  * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2969  * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2970  */
2971 module_param(ibrs_off, bool, 0444);
2972 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
2973 
2974 /*
2975  * Define the C-states table from a user input string. Expected format is
2976  * 'name:latency:residency', where:
2977  * - name: The C-state name.
2978  * - latency: The C-state exit latency in us.
2979  * - residency: The C-state target residency in us.
2980  *
2981  * Multiple C-states can be defined by separating them with commas:
2982  * 'name1:latency1:residency1,name2:latency2:residency2'
2983  *
2984  * Example: intel_idle.table=C1:1:1,C1E:5:10,C6:100:600
2985  *
2986  * To leave latency or residency unchanged, use an empty field, for example:
2987  * 'C1:1:1,C1E::10' - leaves C1E latency unchanged.
2988  */
2989 module_param_string(table, cmdline_table_str, MAX_CMDLINE_TABLE_LEN, 0444);
2990 MODULE_PARM_DESC(table, "Build the C-states table from a user input string");
2991