1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013 - 2020, Intel Corporation.
6 * Len Brown <len.brown@intel.com>
7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8 */
9
10 /*
11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
13 * make Linux more efficient on these processors, as intel_idle knows
14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15 */
16
17 /*
18 * Design Assumptions
19 *
20 * All CPUs have same idle states as boot CPU
21 *
22 * Chipset BM_STS (bus master status) bit is a NOP
23 * for preventing entry into deep C-states
24 *
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
27 * instruction needs to be executed to flush the caches)
28 */
29
30 /*
31 * Known limitations
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39 /* un-comment DEBUG to enable pr_debug() statements */
40 /* #define DEBUG */
41
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44 #include <linux/acpi.h>
45 #include <linux/kernel.h>
46 #include <linux/cpuidle.h>
47 #include <linux/tick.h>
48 #include <trace/events/power.h>
49 #include <linux/sched.h>
50 #include <linux/sched/smt.h>
51 #include <linux/notifier.h>
52 #include <linux/cpu.h>
53 #include <linux/moduleparam.h>
54 #include <asm/cpuid.h>
55 #include <asm/cpu_device_id.h>
56 #include <asm/intel-family.h>
57 #include <asm/mwait.h>
58 #include <asm/spec-ctrl.h>
59 #include <asm/tsc.h>
60 #include <asm/fpu/api.h>
61 #include <asm/smp.h>
62
63 #define INTEL_IDLE_VERSION "0.5.1"
64
65 static struct cpuidle_driver intel_idle_driver = {
66 .name = "intel_idle",
67 .owner = THIS_MODULE,
68 };
69 /* intel_idle.max_cstate=0 disables driver */
70 static int max_cstate = CPUIDLE_STATE_MAX - 1;
71 static unsigned int disabled_states_mask __read_mostly;
72 static unsigned int preferred_states_mask __read_mostly;
73 static bool force_irq_on __read_mostly;
74 static bool ibrs_off __read_mostly;
75
76 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
77
78 static unsigned long auto_demotion_disable_flags;
79
80 static enum {
81 C1E_PROMOTION_PRESERVE,
82 C1E_PROMOTION_ENABLE,
83 C1E_PROMOTION_DISABLE
84 } c1e_promotion = C1E_PROMOTION_PRESERVE;
85
86 struct idle_cpu {
87 struct cpuidle_state *state_table;
88
89 /*
90 * Hardware C-state auto-demotion may not always be optimal.
91 * Indicate which enable bits to clear here.
92 */
93 unsigned long auto_demotion_disable_flags;
94 bool disable_promotion_to_c1e;
95 bool use_acpi;
96 };
97
98 static const struct idle_cpu *icpu __initdata;
99 static struct cpuidle_state *cpuidle_state_table __initdata;
100
101 static unsigned int mwait_substates __initdata;
102
103 /*
104 * Enable interrupts before entering the C-state. On some platforms and for
105 * some C-states, this may measurably decrease interrupt latency.
106 */
107 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
108
109 /*
110 * Enable this state by default even if the ACPI _CST does not list it.
111 */
112 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
113
114 /*
115 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
116 * above.
117 */
118 #define CPUIDLE_FLAG_IBRS BIT(16)
119
120 /*
121 * Initialize large xstate for the C6-state entrance.
122 */
123 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
124
125 /*
126 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
127 * custom tables.
128 */
129 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18)
130
131 /*
132 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
133 * the C-state (top nibble) and sub-state (bottom nibble)
134 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
135 *
136 * We store the hint at the top of our "flags" for each state.
137 */
138 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
139 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
140
__intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index,bool irqoff)141 static __always_inline int __intel_idle(struct cpuidle_device *dev,
142 struct cpuidle_driver *drv,
143 int index, bool irqoff)
144 {
145 struct cpuidle_state *state = &drv->states[index];
146 unsigned long eax = flg2MWAIT(state->flags);
147 unsigned long ecx = 1*irqoff; /* break on interrupt flag */
148
149 mwait_idle_with_hints(eax, ecx);
150
151 return index;
152 }
153
154 /**
155 * intel_idle - Ask the processor to enter the given idle state.
156 * @dev: cpuidle device of the target CPU.
157 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
158 * @index: Target idle state index.
159 *
160 * Use the MWAIT instruction to notify the processor that the CPU represented by
161 * @dev is idle and it can try to enter the idle state corresponding to @index.
162 *
163 * If the local APIC timer is not known to be reliable in the target idle state,
164 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
165 *
166 * Must be called under local_irq_disable().
167 */
intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)168 static __cpuidle int intel_idle(struct cpuidle_device *dev,
169 struct cpuidle_driver *drv, int index)
170 {
171 return __intel_idle(dev, drv, index, true);
172 }
173
intel_idle_irq(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)174 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
175 struct cpuidle_driver *drv, int index)
176 {
177 return __intel_idle(dev, drv, index, false);
178 }
179
intel_idle_ibrs(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)180 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
181 struct cpuidle_driver *drv, int index)
182 {
183 bool smt_active = sched_smt_active();
184 u64 spec_ctrl = spec_ctrl_current();
185 int ret;
186
187 if (smt_active)
188 __update_spec_ctrl(0);
189
190 ret = __intel_idle(dev, drv, index, true);
191
192 if (smt_active)
193 __update_spec_ctrl(spec_ctrl);
194
195 return ret;
196 }
197
intel_idle_xstate(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)198 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
199 struct cpuidle_driver *drv, int index)
200 {
201 fpu_idle_fpregs();
202 return __intel_idle(dev, drv, index, true);
203 }
204
205 /**
206 * intel_idle_s2idle - Ask the processor to enter the given idle state.
207 * @dev: cpuidle device of the target CPU.
208 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
209 * @index: Target idle state index.
210 *
211 * Use the MWAIT instruction to notify the processor that the CPU represented by
212 * @dev is idle and it can try to enter the idle state corresponding to @index.
213 *
214 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
215 * scheduler tick and suspended scheduler clock on the target CPU.
216 */
intel_idle_s2idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)217 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
218 struct cpuidle_driver *drv, int index)
219 {
220 unsigned long ecx = 1; /* break on interrupt flag */
221 struct cpuidle_state *state = &drv->states[index];
222 unsigned long eax = flg2MWAIT(state->flags);
223
224 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
225 fpu_idle_fpregs();
226
227 mwait_idle_with_hints(eax, ecx);
228
229 return 0;
230 }
231
intel_idle_enter_dead(struct cpuidle_device * dev,int index)232 static void intel_idle_enter_dead(struct cpuidle_device *dev, int index)
233 {
234 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
235 struct cpuidle_state *state = &drv->states[index];
236 unsigned long eax = flg2MWAIT(state->flags);
237
238 mwait_play_dead(eax);
239 }
240
241 /*
242 * States are indexed by the cstate number,
243 * which is also the index into the MWAIT hint array.
244 * Thus C0 is a dummy.
245 */
246 static struct cpuidle_state nehalem_cstates[] __initdata = {
247 {
248 .name = "C1",
249 .desc = "MWAIT 0x00",
250 .flags = MWAIT2flg(0x00),
251 .exit_latency = 3,
252 .target_residency = 6,
253 .enter = &intel_idle,
254 .enter_s2idle = intel_idle_s2idle, },
255 {
256 .name = "C1E",
257 .desc = "MWAIT 0x01",
258 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
259 .exit_latency = 10,
260 .target_residency = 20,
261 .enter = &intel_idle,
262 .enter_s2idle = intel_idle_s2idle, },
263 {
264 .name = "C3",
265 .desc = "MWAIT 0x10",
266 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
267 .exit_latency = 20,
268 .target_residency = 80,
269 .enter = &intel_idle,
270 .enter_s2idle = intel_idle_s2idle, },
271 {
272 .name = "C6",
273 .desc = "MWAIT 0x20",
274 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
275 .exit_latency = 200,
276 .target_residency = 800,
277 .enter = &intel_idle,
278 .enter_s2idle = intel_idle_s2idle, },
279 {
280 .enter = NULL }
281 };
282
283 static struct cpuidle_state snb_cstates[] __initdata = {
284 {
285 .name = "C1",
286 .desc = "MWAIT 0x00",
287 .flags = MWAIT2flg(0x00),
288 .exit_latency = 2,
289 .target_residency = 2,
290 .enter = &intel_idle,
291 .enter_s2idle = intel_idle_s2idle, },
292 {
293 .name = "C1E",
294 .desc = "MWAIT 0x01",
295 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
296 .exit_latency = 10,
297 .target_residency = 20,
298 .enter = &intel_idle,
299 .enter_s2idle = intel_idle_s2idle, },
300 {
301 .name = "C3",
302 .desc = "MWAIT 0x10",
303 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
304 .exit_latency = 80,
305 .target_residency = 211,
306 .enter = &intel_idle,
307 .enter_s2idle = intel_idle_s2idle, },
308 {
309 .name = "C6",
310 .desc = "MWAIT 0x20",
311 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
312 .exit_latency = 104,
313 .target_residency = 345,
314 .enter = &intel_idle,
315 .enter_s2idle = intel_idle_s2idle, },
316 {
317 .name = "C7",
318 .desc = "MWAIT 0x30",
319 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
320 .exit_latency = 109,
321 .target_residency = 345,
322 .enter = &intel_idle,
323 .enter_s2idle = intel_idle_s2idle, },
324 {
325 .enter = NULL }
326 };
327
328 static struct cpuidle_state byt_cstates[] __initdata = {
329 {
330 .name = "C1",
331 .desc = "MWAIT 0x00",
332 .flags = MWAIT2flg(0x00),
333 .exit_latency = 1,
334 .target_residency = 1,
335 .enter = &intel_idle,
336 .enter_s2idle = intel_idle_s2idle, },
337 {
338 .name = "C6N",
339 .desc = "MWAIT 0x58",
340 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
341 .exit_latency = 300,
342 .target_residency = 275,
343 .enter = &intel_idle,
344 .enter_s2idle = intel_idle_s2idle, },
345 {
346 .name = "C6S",
347 .desc = "MWAIT 0x52",
348 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
349 .exit_latency = 500,
350 .target_residency = 560,
351 .enter = &intel_idle,
352 .enter_s2idle = intel_idle_s2idle, },
353 {
354 .name = "C7",
355 .desc = "MWAIT 0x60",
356 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
357 .exit_latency = 1200,
358 .target_residency = 4000,
359 .enter = &intel_idle,
360 .enter_s2idle = intel_idle_s2idle, },
361 {
362 .name = "C7S",
363 .desc = "MWAIT 0x64",
364 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
365 .exit_latency = 10000,
366 .target_residency = 20000,
367 .enter = &intel_idle,
368 .enter_s2idle = intel_idle_s2idle, },
369 {
370 .enter = NULL }
371 };
372
373 static struct cpuidle_state cht_cstates[] __initdata = {
374 {
375 .name = "C1",
376 .desc = "MWAIT 0x00",
377 .flags = MWAIT2flg(0x00),
378 .exit_latency = 1,
379 .target_residency = 1,
380 .enter = &intel_idle,
381 .enter_s2idle = intel_idle_s2idle, },
382 {
383 .name = "C6N",
384 .desc = "MWAIT 0x58",
385 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
386 .exit_latency = 80,
387 .target_residency = 275,
388 .enter = &intel_idle,
389 .enter_s2idle = intel_idle_s2idle, },
390 {
391 .name = "C6S",
392 .desc = "MWAIT 0x52",
393 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
394 .exit_latency = 200,
395 .target_residency = 560,
396 .enter = &intel_idle,
397 .enter_s2idle = intel_idle_s2idle, },
398 {
399 .name = "C7",
400 .desc = "MWAIT 0x60",
401 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
402 .exit_latency = 1200,
403 .target_residency = 4000,
404 .enter = &intel_idle,
405 .enter_s2idle = intel_idle_s2idle, },
406 {
407 .name = "C7S",
408 .desc = "MWAIT 0x64",
409 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
410 .exit_latency = 10000,
411 .target_residency = 20000,
412 .enter = &intel_idle,
413 .enter_s2idle = intel_idle_s2idle, },
414 {
415 .enter = NULL }
416 };
417
418 static struct cpuidle_state ivb_cstates[] __initdata = {
419 {
420 .name = "C1",
421 .desc = "MWAIT 0x00",
422 .flags = MWAIT2flg(0x00),
423 .exit_latency = 1,
424 .target_residency = 1,
425 .enter = &intel_idle,
426 .enter_s2idle = intel_idle_s2idle, },
427 {
428 .name = "C1E",
429 .desc = "MWAIT 0x01",
430 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
431 .exit_latency = 10,
432 .target_residency = 20,
433 .enter = &intel_idle,
434 .enter_s2idle = intel_idle_s2idle, },
435 {
436 .name = "C3",
437 .desc = "MWAIT 0x10",
438 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
439 .exit_latency = 59,
440 .target_residency = 156,
441 .enter = &intel_idle,
442 .enter_s2idle = intel_idle_s2idle, },
443 {
444 .name = "C6",
445 .desc = "MWAIT 0x20",
446 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
447 .exit_latency = 80,
448 .target_residency = 300,
449 .enter = &intel_idle,
450 .enter_s2idle = intel_idle_s2idle, },
451 {
452 .name = "C7",
453 .desc = "MWAIT 0x30",
454 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
455 .exit_latency = 87,
456 .target_residency = 300,
457 .enter = &intel_idle,
458 .enter_s2idle = intel_idle_s2idle, },
459 {
460 .enter = NULL }
461 };
462
463 static struct cpuidle_state ivt_cstates[] __initdata = {
464 {
465 .name = "C1",
466 .desc = "MWAIT 0x00",
467 .flags = MWAIT2flg(0x00),
468 .exit_latency = 1,
469 .target_residency = 1,
470 .enter = &intel_idle,
471 .enter_s2idle = intel_idle_s2idle, },
472 {
473 .name = "C1E",
474 .desc = "MWAIT 0x01",
475 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
476 .exit_latency = 10,
477 .target_residency = 80,
478 .enter = &intel_idle,
479 .enter_s2idle = intel_idle_s2idle, },
480 {
481 .name = "C3",
482 .desc = "MWAIT 0x10",
483 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
484 .exit_latency = 59,
485 .target_residency = 156,
486 .enter = &intel_idle,
487 .enter_s2idle = intel_idle_s2idle, },
488 {
489 .name = "C6",
490 .desc = "MWAIT 0x20",
491 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
492 .exit_latency = 82,
493 .target_residency = 300,
494 .enter = &intel_idle,
495 .enter_s2idle = intel_idle_s2idle, },
496 {
497 .enter = NULL }
498 };
499
500 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
501 {
502 .name = "C1",
503 .desc = "MWAIT 0x00",
504 .flags = MWAIT2flg(0x00),
505 .exit_latency = 1,
506 .target_residency = 1,
507 .enter = &intel_idle,
508 .enter_s2idle = intel_idle_s2idle, },
509 {
510 .name = "C1E",
511 .desc = "MWAIT 0x01",
512 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
513 .exit_latency = 10,
514 .target_residency = 250,
515 .enter = &intel_idle,
516 .enter_s2idle = intel_idle_s2idle, },
517 {
518 .name = "C3",
519 .desc = "MWAIT 0x10",
520 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
521 .exit_latency = 59,
522 .target_residency = 300,
523 .enter = &intel_idle,
524 .enter_s2idle = intel_idle_s2idle, },
525 {
526 .name = "C6",
527 .desc = "MWAIT 0x20",
528 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
529 .exit_latency = 84,
530 .target_residency = 400,
531 .enter = &intel_idle,
532 .enter_s2idle = intel_idle_s2idle, },
533 {
534 .enter = NULL }
535 };
536
537 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
538 {
539 .name = "C1",
540 .desc = "MWAIT 0x00",
541 .flags = MWAIT2flg(0x00),
542 .exit_latency = 1,
543 .target_residency = 1,
544 .enter = &intel_idle,
545 .enter_s2idle = intel_idle_s2idle, },
546 {
547 .name = "C1E",
548 .desc = "MWAIT 0x01",
549 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
550 .exit_latency = 10,
551 .target_residency = 500,
552 .enter = &intel_idle,
553 .enter_s2idle = intel_idle_s2idle, },
554 {
555 .name = "C3",
556 .desc = "MWAIT 0x10",
557 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
558 .exit_latency = 59,
559 .target_residency = 600,
560 .enter = &intel_idle,
561 .enter_s2idle = intel_idle_s2idle, },
562 {
563 .name = "C6",
564 .desc = "MWAIT 0x20",
565 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
566 .exit_latency = 88,
567 .target_residency = 700,
568 .enter = &intel_idle,
569 .enter_s2idle = intel_idle_s2idle, },
570 {
571 .enter = NULL }
572 };
573
574 static struct cpuidle_state hsw_cstates[] __initdata = {
575 {
576 .name = "C1",
577 .desc = "MWAIT 0x00",
578 .flags = MWAIT2flg(0x00),
579 .exit_latency = 2,
580 .target_residency = 2,
581 .enter = &intel_idle,
582 .enter_s2idle = intel_idle_s2idle, },
583 {
584 .name = "C1E",
585 .desc = "MWAIT 0x01",
586 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
587 .exit_latency = 10,
588 .target_residency = 20,
589 .enter = &intel_idle,
590 .enter_s2idle = intel_idle_s2idle, },
591 {
592 .name = "C3",
593 .desc = "MWAIT 0x10",
594 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
595 .exit_latency = 33,
596 .target_residency = 100,
597 .enter = &intel_idle,
598 .enter_s2idle = intel_idle_s2idle, },
599 {
600 .name = "C6",
601 .desc = "MWAIT 0x20",
602 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
603 .exit_latency = 133,
604 .target_residency = 400,
605 .enter = &intel_idle,
606 .enter_s2idle = intel_idle_s2idle, },
607 {
608 .name = "C7s",
609 .desc = "MWAIT 0x32",
610 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
611 .exit_latency = 166,
612 .target_residency = 500,
613 .enter = &intel_idle,
614 .enter_s2idle = intel_idle_s2idle, },
615 {
616 .name = "C8",
617 .desc = "MWAIT 0x40",
618 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
619 .exit_latency = 300,
620 .target_residency = 900,
621 .enter = &intel_idle,
622 .enter_s2idle = intel_idle_s2idle, },
623 {
624 .name = "C9",
625 .desc = "MWAIT 0x50",
626 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
627 .exit_latency = 600,
628 .target_residency = 1800,
629 .enter = &intel_idle,
630 .enter_s2idle = intel_idle_s2idle, },
631 {
632 .name = "C10",
633 .desc = "MWAIT 0x60",
634 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
635 .exit_latency = 2600,
636 .target_residency = 7700,
637 .enter = &intel_idle,
638 .enter_s2idle = intel_idle_s2idle, },
639 {
640 .enter = NULL }
641 };
642 static struct cpuidle_state bdw_cstates[] __initdata = {
643 {
644 .name = "C1",
645 .desc = "MWAIT 0x00",
646 .flags = MWAIT2flg(0x00),
647 .exit_latency = 2,
648 .target_residency = 2,
649 .enter = &intel_idle,
650 .enter_s2idle = intel_idle_s2idle, },
651 {
652 .name = "C1E",
653 .desc = "MWAIT 0x01",
654 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
655 .exit_latency = 10,
656 .target_residency = 20,
657 .enter = &intel_idle,
658 .enter_s2idle = intel_idle_s2idle, },
659 {
660 .name = "C3",
661 .desc = "MWAIT 0x10",
662 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
663 .exit_latency = 40,
664 .target_residency = 100,
665 .enter = &intel_idle,
666 .enter_s2idle = intel_idle_s2idle, },
667 {
668 .name = "C6",
669 .desc = "MWAIT 0x20",
670 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
671 .exit_latency = 133,
672 .target_residency = 400,
673 .enter = &intel_idle,
674 .enter_s2idle = intel_idle_s2idle, },
675 {
676 .name = "C7s",
677 .desc = "MWAIT 0x32",
678 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
679 .exit_latency = 166,
680 .target_residency = 500,
681 .enter = &intel_idle,
682 .enter_s2idle = intel_idle_s2idle, },
683 {
684 .name = "C8",
685 .desc = "MWAIT 0x40",
686 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
687 .exit_latency = 300,
688 .target_residency = 900,
689 .enter = &intel_idle,
690 .enter_s2idle = intel_idle_s2idle, },
691 {
692 .name = "C9",
693 .desc = "MWAIT 0x50",
694 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
695 .exit_latency = 600,
696 .target_residency = 1800,
697 .enter = &intel_idle,
698 .enter_s2idle = intel_idle_s2idle, },
699 {
700 .name = "C10",
701 .desc = "MWAIT 0x60",
702 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
703 .exit_latency = 2600,
704 .target_residency = 7700,
705 .enter = &intel_idle,
706 .enter_s2idle = intel_idle_s2idle, },
707 {
708 .enter = NULL }
709 };
710
711 static struct cpuidle_state skl_cstates[] __initdata = {
712 {
713 .name = "C1",
714 .desc = "MWAIT 0x00",
715 .flags = MWAIT2flg(0x00),
716 .exit_latency = 2,
717 .target_residency = 2,
718 .enter = &intel_idle,
719 .enter_s2idle = intel_idle_s2idle, },
720 {
721 .name = "C1E",
722 .desc = "MWAIT 0x01",
723 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
724 .exit_latency = 10,
725 .target_residency = 20,
726 .enter = &intel_idle,
727 .enter_s2idle = intel_idle_s2idle, },
728 {
729 .name = "C3",
730 .desc = "MWAIT 0x10",
731 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
732 .exit_latency = 70,
733 .target_residency = 100,
734 .enter = &intel_idle,
735 .enter_s2idle = intel_idle_s2idle, },
736 {
737 .name = "C6",
738 .desc = "MWAIT 0x20",
739 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
740 .exit_latency = 85,
741 .target_residency = 200,
742 .enter = &intel_idle,
743 .enter_s2idle = intel_idle_s2idle, },
744 {
745 .name = "C7s",
746 .desc = "MWAIT 0x33",
747 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
748 .exit_latency = 124,
749 .target_residency = 800,
750 .enter = &intel_idle,
751 .enter_s2idle = intel_idle_s2idle, },
752 {
753 .name = "C8",
754 .desc = "MWAIT 0x40",
755 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
756 .exit_latency = 200,
757 .target_residency = 800,
758 .enter = &intel_idle,
759 .enter_s2idle = intel_idle_s2idle, },
760 {
761 .name = "C9",
762 .desc = "MWAIT 0x50",
763 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
764 .exit_latency = 480,
765 .target_residency = 5000,
766 .enter = &intel_idle,
767 .enter_s2idle = intel_idle_s2idle, },
768 {
769 .name = "C10",
770 .desc = "MWAIT 0x60",
771 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
772 .exit_latency = 890,
773 .target_residency = 5000,
774 .enter = &intel_idle,
775 .enter_s2idle = intel_idle_s2idle, },
776 {
777 .enter = NULL }
778 };
779
780 static struct cpuidle_state skx_cstates[] __initdata = {
781 {
782 .name = "C1",
783 .desc = "MWAIT 0x00",
784 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
785 .exit_latency = 2,
786 .target_residency = 2,
787 .enter = &intel_idle,
788 .enter_s2idle = intel_idle_s2idle, },
789 {
790 .name = "C1E",
791 .desc = "MWAIT 0x01",
792 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
793 .exit_latency = 10,
794 .target_residency = 20,
795 .enter = &intel_idle,
796 .enter_s2idle = intel_idle_s2idle, },
797 {
798 .name = "C6",
799 .desc = "MWAIT 0x20",
800 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
801 .exit_latency = 133,
802 .target_residency = 600,
803 .enter = &intel_idle,
804 .enter_s2idle = intel_idle_s2idle, },
805 {
806 .enter = NULL }
807 };
808
809 static struct cpuidle_state icx_cstates[] __initdata = {
810 {
811 .name = "C1",
812 .desc = "MWAIT 0x00",
813 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
814 .exit_latency = 1,
815 .target_residency = 1,
816 .enter = &intel_idle,
817 .enter_s2idle = intel_idle_s2idle, },
818 {
819 .name = "C1E",
820 .desc = "MWAIT 0x01",
821 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
822 .exit_latency = 4,
823 .target_residency = 4,
824 .enter = &intel_idle,
825 .enter_s2idle = intel_idle_s2idle, },
826 {
827 .name = "C6",
828 .desc = "MWAIT 0x20",
829 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
830 .exit_latency = 170,
831 .target_residency = 600,
832 .enter = &intel_idle,
833 .enter_s2idle = intel_idle_s2idle, },
834 {
835 .enter = NULL }
836 };
837
838 /*
839 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
840 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
841 * But in this case there is effectively no C1, because C1 requests are
842 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
843 * and C1E requests end up with C1, so there is effectively no C1E.
844 *
845 * By default we enable C1E and disable C1 by marking it with
846 * 'CPUIDLE_FLAG_UNUSABLE'.
847 */
848 static struct cpuidle_state adl_cstates[] __initdata = {
849 {
850 .name = "C1",
851 .desc = "MWAIT 0x00",
852 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
853 .exit_latency = 1,
854 .target_residency = 1,
855 .enter = &intel_idle,
856 .enter_s2idle = intel_idle_s2idle, },
857 {
858 .name = "C1E",
859 .desc = "MWAIT 0x01",
860 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
861 .exit_latency = 2,
862 .target_residency = 4,
863 .enter = &intel_idle,
864 .enter_s2idle = intel_idle_s2idle, },
865 {
866 .name = "C6",
867 .desc = "MWAIT 0x20",
868 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
869 .exit_latency = 220,
870 .target_residency = 600,
871 .enter = &intel_idle,
872 .enter_s2idle = intel_idle_s2idle, },
873 {
874 .name = "C8",
875 .desc = "MWAIT 0x40",
876 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
877 .exit_latency = 280,
878 .target_residency = 800,
879 .enter = &intel_idle,
880 .enter_s2idle = intel_idle_s2idle, },
881 {
882 .name = "C10",
883 .desc = "MWAIT 0x60",
884 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
885 .exit_latency = 680,
886 .target_residency = 2000,
887 .enter = &intel_idle,
888 .enter_s2idle = intel_idle_s2idle, },
889 {
890 .enter = NULL }
891 };
892
893 static struct cpuidle_state adl_l_cstates[] __initdata = {
894 {
895 .name = "C1",
896 .desc = "MWAIT 0x00",
897 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
898 .exit_latency = 1,
899 .target_residency = 1,
900 .enter = &intel_idle,
901 .enter_s2idle = intel_idle_s2idle, },
902 {
903 .name = "C1E",
904 .desc = "MWAIT 0x01",
905 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
906 .exit_latency = 2,
907 .target_residency = 4,
908 .enter = &intel_idle,
909 .enter_s2idle = intel_idle_s2idle, },
910 {
911 .name = "C6",
912 .desc = "MWAIT 0x20",
913 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
914 .exit_latency = 170,
915 .target_residency = 500,
916 .enter = &intel_idle,
917 .enter_s2idle = intel_idle_s2idle, },
918 {
919 .name = "C8",
920 .desc = "MWAIT 0x40",
921 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
922 .exit_latency = 200,
923 .target_residency = 600,
924 .enter = &intel_idle,
925 .enter_s2idle = intel_idle_s2idle, },
926 {
927 .name = "C10",
928 .desc = "MWAIT 0x60",
929 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
930 .exit_latency = 230,
931 .target_residency = 700,
932 .enter = &intel_idle,
933 .enter_s2idle = intel_idle_s2idle, },
934 {
935 .enter = NULL }
936 };
937
938 static struct cpuidle_state mtl_l_cstates[] __initdata = {
939 {
940 .name = "C1E",
941 .desc = "MWAIT 0x01",
942 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
943 .exit_latency = 1,
944 .target_residency = 1,
945 .enter = &intel_idle,
946 .enter_s2idle = intel_idle_s2idle, },
947 {
948 .name = "C6",
949 .desc = "MWAIT 0x20",
950 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
951 .exit_latency = 140,
952 .target_residency = 420,
953 .enter = &intel_idle,
954 .enter_s2idle = intel_idle_s2idle, },
955 {
956 .name = "C10",
957 .desc = "MWAIT 0x60",
958 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
959 .exit_latency = 310,
960 .target_residency = 930,
961 .enter = &intel_idle,
962 .enter_s2idle = intel_idle_s2idle, },
963 {
964 .enter = NULL }
965 };
966
967 static struct cpuidle_state gmt_cstates[] __initdata = {
968 {
969 .name = "C1",
970 .desc = "MWAIT 0x00",
971 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
972 .exit_latency = 1,
973 .target_residency = 1,
974 .enter = &intel_idle,
975 .enter_s2idle = intel_idle_s2idle, },
976 {
977 .name = "C1E",
978 .desc = "MWAIT 0x01",
979 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
980 .exit_latency = 2,
981 .target_residency = 4,
982 .enter = &intel_idle,
983 .enter_s2idle = intel_idle_s2idle, },
984 {
985 .name = "C6",
986 .desc = "MWAIT 0x20",
987 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
988 .exit_latency = 195,
989 .target_residency = 585,
990 .enter = &intel_idle,
991 .enter_s2idle = intel_idle_s2idle, },
992 {
993 .name = "C8",
994 .desc = "MWAIT 0x40",
995 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
996 .exit_latency = 260,
997 .target_residency = 1040,
998 .enter = &intel_idle,
999 .enter_s2idle = intel_idle_s2idle, },
1000 {
1001 .name = "C10",
1002 .desc = "MWAIT 0x60",
1003 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1004 .exit_latency = 660,
1005 .target_residency = 1980,
1006 .enter = &intel_idle,
1007 .enter_s2idle = intel_idle_s2idle, },
1008 {
1009 .enter = NULL }
1010 };
1011
1012 static struct cpuidle_state spr_cstates[] __initdata = {
1013 {
1014 .name = "C1",
1015 .desc = "MWAIT 0x00",
1016 .flags = MWAIT2flg(0x00),
1017 .exit_latency = 1,
1018 .target_residency = 1,
1019 .enter = &intel_idle,
1020 .enter_s2idle = intel_idle_s2idle, },
1021 {
1022 .name = "C1E",
1023 .desc = "MWAIT 0x01",
1024 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1025 .exit_latency = 2,
1026 .target_residency = 4,
1027 .enter = &intel_idle,
1028 .enter_s2idle = intel_idle_s2idle, },
1029 {
1030 .name = "C6",
1031 .desc = "MWAIT 0x20",
1032 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1033 CPUIDLE_FLAG_INIT_XSTATE,
1034 .exit_latency = 290,
1035 .target_residency = 800,
1036 .enter = &intel_idle,
1037 .enter_s2idle = intel_idle_s2idle, },
1038 {
1039 .enter = NULL }
1040 };
1041
1042 static struct cpuidle_state gnr_cstates[] __initdata = {
1043 {
1044 .name = "C1",
1045 .desc = "MWAIT 0x00",
1046 .flags = MWAIT2flg(0x00),
1047 .exit_latency = 1,
1048 .target_residency = 1,
1049 .enter = &intel_idle,
1050 .enter_s2idle = intel_idle_s2idle, },
1051 {
1052 .name = "C1E",
1053 .desc = "MWAIT 0x01",
1054 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1055 .exit_latency = 4,
1056 .target_residency = 4,
1057 .enter = &intel_idle,
1058 .enter_s2idle = intel_idle_s2idle, },
1059 {
1060 .name = "C6",
1061 .desc = "MWAIT 0x20",
1062 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1063 CPUIDLE_FLAG_INIT_XSTATE |
1064 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1065 .exit_latency = 170,
1066 .target_residency = 650,
1067 .enter = &intel_idle,
1068 .enter_s2idle = intel_idle_s2idle, },
1069 {
1070 .name = "C6P",
1071 .desc = "MWAIT 0x21",
1072 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1073 CPUIDLE_FLAG_INIT_XSTATE |
1074 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1075 .exit_latency = 210,
1076 .target_residency = 1000,
1077 .enter = &intel_idle,
1078 .enter_s2idle = intel_idle_s2idle, },
1079 {
1080 .enter = NULL }
1081 };
1082
1083 static struct cpuidle_state gnrd_cstates[] __initdata = {
1084 {
1085 .name = "C1",
1086 .desc = "MWAIT 0x00",
1087 .flags = MWAIT2flg(0x00),
1088 .exit_latency = 1,
1089 .target_residency = 1,
1090 .enter = &intel_idle,
1091 .enter_s2idle = intel_idle_s2idle, },
1092 {
1093 .name = "C1E",
1094 .desc = "MWAIT 0x01",
1095 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1096 .exit_latency = 4,
1097 .target_residency = 4,
1098 .enter = &intel_idle,
1099 .enter_s2idle = intel_idle_s2idle, },
1100 {
1101 .name = "C6",
1102 .desc = "MWAIT 0x20",
1103 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1104 CPUIDLE_FLAG_INIT_XSTATE |
1105 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1106 .exit_latency = 220,
1107 .target_residency = 650,
1108 .enter = &intel_idle,
1109 .enter_s2idle = intel_idle_s2idle, },
1110 {
1111 .name = "C6P",
1112 .desc = "MWAIT 0x21",
1113 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1114 CPUIDLE_FLAG_INIT_XSTATE |
1115 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1116 .exit_latency = 240,
1117 .target_residency = 750,
1118 .enter = &intel_idle,
1119 .enter_s2idle = intel_idle_s2idle, },
1120 {
1121 .enter = NULL }
1122 };
1123
1124 static struct cpuidle_state atom_cstates[] __initdata = {
1125 {
1126 .name = "C1E",
1127 .desc = "MWAIT 0x00",
1128 .flags = MWAIT2flg(0x00),
1129 .exit_latency = 10,
1130 .target_residency = 20,
1131 .enter = &intel_idle,
1132 .enter_s2idle = intel_idle_s2idle, },
1133 {
1134 .name = "C2",
1135 .desc = "MWAIT 0x10",
1136 .flags = MWAIT2flg(0x10),
1137 .exit_latency = 20,
1138 .target_residency = 80,
1139 .enter = &intel_idle,
1140 .enter_s2idle = intel_idle_s2idle, },
1141 {
1142 .name = "C4",
1143 .desc = "MWAIT 0x30",
1144 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1145 .exit_latency = 100,
1146 .target_residency = 400,
1147 .enter = &intel_idle,
1148 .enter_s2idle = intel_idle_s2idle, },
1149 {
1150 .name = "C6",
1151 .desc = "MWAIT 0x52",
1152 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1153 .exit_latency = 140,
1154 .target_residency = 560,
1155 .enter = &intel_idle,
1156 .enter_s2idle = intel_idle_s2idle, },
1157 {
1158 .enter = NULL }
1159 };
1160 static struct cpuidle_state tangier_cstates[] __initdata = {
1161 {
1162 .name = "C1",
1163 .desc = "MWAIT 0x00",
1164 .flags = MWAIT2flg(0x00),
1165 .exit_latency = 1,
1166 .target_residency = 4,
1167 .enter = &intel_idle,
1168 .enter_s2idle = intel_idle_s2idle, },
1169 {
1170 .name = "C4",
1171 .desc = "MWAIT 0x30",
1172 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1173 .exit_latency = 100,
1174 .target_residency = 400,
1175 .enter = &intel_idle,
1176 .enter_s2idle = intel_idle_s2idle, },
1177 {
1178 .name = "C6",
1179 .desc = "MWAIT 0x52",
1180 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1181 .exit_latency = 140,
1182 .target_residency = 560,
1183 .enter = &intel_idle,
1184 .enter_s2idle = intel_idle_s2idle, },
1185 {
1186 .name = "C7",
1187 .desc = "MWAIT 0x60",
1188 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1189 .exit_latency = 1200,
1190 .target_residency = 4000,
1191 .enter = &intel_idle,
1192 .enter_s2idle = intel_idle_s2idle, },
1193 {
1194 .name = "C9",
1195 .desc = "MWAIT 0x64",
1196 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1197 .exit_latency = 10000,
1198 .target_residency = 20000,
1199 .enter = &intel_idle,
1200 .enter_s2idle = intel_idle_s2idle, },
1201 {
1202 .enter = NULL }
1203 };
1204 static struct cpuidle_state avn_cstates[] __initdata = {
1205 {
1206 .name = "C1",
1207 .desc = "MWAIT 0x00",
1208 .flags = MWAIT2flg(0x00),
1209 .exit_latency = 2,
1210 .target_residency = 2,
1211 .enter = &intel_idle,
1212 .enter_s2idle = intel_idle_s2idle, },
1213 {
1214 .name = "C6",
1215 .desc = "MWAIT 0x51",
1216 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1217 .exit_latency = 15,
1218 .target_residency = 45,
1219 .enter = &intel_idle,
1220 .enter_s2idle = intel_idle_s2idle, },
1221 {
1222 .enter = NULL }
1223 };
1224 static struct cpuidle_state knl_cstates[] __initdata = {
1225 {
1226 .name = "C1",
1227 .desc = "MWAIT 0x00",
1228 .flags = MWAIT2flg(0x00),
1229 .exit_latency = 1,
1230 .target_residency = 2,
1231 .enter = &intel_idle,
1232 .enter_s2idle = intel_idle_s2idle },
1233 {
1234 .name = "C6",
1235 .desc = "MWAIT 0x10",
1236 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1237 .exit_latency = 120,
1238 .target_residency = 500,
1239 .enter = &intel_idle,
1240 .enter_s2idle = intel_idle_s2idle },
1241 {
1242 .enter = NULL }
1243 };
1244
1245 static struct cpuidle_state bxt_cstates[] __initdata = {
1246 {
1247 .name = "C1",
1248 .desc = "MWAIT 0x00",
1249 .flags = MWAIT2flg(0x00),
1250 .exit_latency = 2,
1251 .target_residency = 2,
1252 .enter = &intel_idle,
1253 .enter_s2idle = intel_idle_s2idle, },
1254 {
1255 .name = "C1E",
1256 .desc = "MWAIT 0x01",
1257 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1258 .exit_latency = 10,
1259 .target_residency = 20,
1260 .enter = &intel_idle,
1261 .enter_s2idle = intel_idle_s2idle, },
1262 {
1263 .name = "C6",
1264 .desc = "MWAIT 0x20",
1265 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1266 .exit_latency = 133,
1267 .target_residency = 133,
1268 .enter = &intel_idle,
1269 .enter_s2idle = intel_idle_s2idle, },
1270 {
1271 .name = "C7s",
1272 .desc = "MWAIT 0x31",
1273 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1274 .exit_latency = 155,
1275 .target_residency = 155,
1276 .enter = &intel_idle,
1277 .enter_s2idle = intel_idle_s2idle, },
1278 {
1279 .name = "C8",
1280 .desc = "MWAIT 0x40",
1281 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1282 .exit_latency = 1000,
1283 .target_residency = 1000,
1284 .enter = &intel_idle,
1285 .enter_s2idle = intel_idle_s2idle, },
1286 {
1287 .name = "C9",
1288 .desc = "MWAIT 0x50",
1289 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1290 .exit_latency = 2000,
1291 .target_residency = 2000,
1292 .enter = &intel_idle,
1293 .enter_s2idle = intel_idle_s2idle, },
1294 {
1295 .name = "C10",
1296 .desc = "MWAIT 0x60",
1297 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1298 .exit_latency = 10000,
1299 .target_residency = 10000,
1300 .enter = &intel_idle,
1301 .enter_s2idle = intel_idle_s2idle, },
1302 {
1303 .enter = NULL }
1304 };
1305
1306 static struct cpuidle_state dnv_cstates[] __initdata = {
1307 {
1308 .name = "C1",
1309 .desc = "MWAIT 0x00",
1310 .flags = MWAIT2flg(0x00),
1311 .exit_latency = 2,
1312 .target_residency = 2,
1313 .enter = &intel_idle,
1314 .enter_s2idle = intel_idle_s2idle, },
1315 {
1316 .name = "C1E",
1317 .desc = "MWAIT 0x01",
1318 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1319 .exit_latency = 10,
1320 .target_residency = 20,
1321 .enter = &intel_idle,
1322 .enter_s2idle = intel_idle_s2idle, },
1323 {
1324 .name = "C6",
1325 .desc = "MWAIT 0x20",
1326 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1327 .exit_latency = 50,
1328 .target_residency = 500,
1329 .enter = &intel_idle,
1330 .enter_s2idle = intel_idle_s2idle, },
1331 {
1332 .enter = NULL }
1333 };
1334
1335 /*
1336 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1337 * C6, and this is indicated in the CPUID mwait leaf.
1338 */
1339 static struct cpuidle_state snr_cstates[] __initdata = {
1340 {
1341 .name = "C1",
1342 .desc = "MWAIT 0x00",
1343 .flags = MWAIT2flg(0x00),
1344 .exit_latency = 2,
1345 .target_residency = 2,
1346 .enter = &intel_idle,
1347 .enter_s2idle = intel_idle_s2idle, },
1348 {
1349 .name = "C1E",
1350 .desc = "MWAIT 0x01",
1351 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1352 .exit_latency = 15,
1353 .target_residency = 25,
1354 .enter = &intel_idle,
1355 .enter_s2idle = intel_idle_s2idle, },
1356 {
1357 .name = "C6",
1358 .desc = "MWAIT 0x20",
1359 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1360 .exit_latency = 130,
1361 .target_residency = 500,
1362 .enter = &intel_idle,
1363 .enter_s2idle = intel_idle_s2idle, },
1364 {
1365 .enter = NULL }
1366 };
1367
1368 static struct cpuidle_state grr_cstates[] __initdata = {
1369 {
1370 .name = "C1",
1371 .desc = "MWAIT 0x00",
1372 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1373 .exit_latency = 1,
1374 .target_residency = 1,
1375 .enter = &intel_idle,
1376 .enter_s2idle = intel_idle_s2idle, },
1377 {
1378 .name = "C1E",
1379 .desc = "MWAIT 0x01",
1380 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1381 .exit_latency = 2,
1382 .target_residency = 10,
1383 .enter = &intel_idle,
1384 .enter_s2idle = intel_idle_s2idle, },
1385 {
1386 .name = "C6S",
1387 .desc = "MWAIT 0x22",
1388 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1389 .exit_latency = 140,
1390 .target_residency = 500,
1391 .enter = &intel_idle,
1392 .enter_s2idle = intel_idle_s2idle, },
1393 {
1394 .enter = NULL }
1395 };
1396
1397 static struct cpuidle_state srf_cstates[] __initdata = {
1398 {
1399 .name = "C1",
1400 .desc = "MWAIT 0x00",
1401 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1402 .exit_latency = 1,
1403 .target_residency = 1,
1404 .enter = &intel_idle,
1405 .enter_s2idle = intel_idle_s2idle, },
1406 {
1407 .name = "C1E",
1408 .desc = "MWAIT 0x01",
1409 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1410 .exit_latency = 2,
1411 .target_residency = 10,
1412 .enter = &intel_idle,
1413 .enter_s2idle = intel_idle_s2idle, },
1414 {
1415 .name = "C6S",
1416 .desc = "MWAIT 0x22",
1417 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1418 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1419 .exit_latency = 270,
1420 .target_residency = 700,
1421 .enter = &intel_idle,
1422 .enter_s2idle = intel_idle_s2idle, },
1423 {
1424 .name = "C6SP",
1425 .desc = "MWAIT 0x23",
1426 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1427 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1428 .exit_latency = 310,
1429 .target_residency = 900,
1430 .enter = &intel_idle,
1431 .enter_s2idle = intel_idle_s2idle, },
1432 {
1433 .enter = NULL }
1434 };
1435
1436 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1437 .state_table = nehalem_cstates,
1438 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1439 .disable_promotion_to_c1e = true,
1440 };
1441
1442 static const struct idle_cpu idle_cpu_nhx __initconst = {
1443 .state_table = nehalem_cstates,
1444 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1445 .disable_promotion_to_c1e = true,
1446 .use_acpi = true,
1447 };
1448
1449 static const struct idle_cpu idle_cpu_atom __initconst = {
1450 .state_table = atom_cstates,
1451 };
1452
1453 static const struct idle_cpu idle_cpu_tangier __initconst = {
1454 .state_table = tangier_cstates,
1455 };
1456
1457 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1458 .state_table = atom_cstates,
1459 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1460 };
1461
1462 static const struct idle_cpu idle_cpu_snb __initconst = {
1463 .state_table = snb_cstates,
1464 .disable_promotion_to_c1e = true,
1465 };
1466
1467 static const struct idle_cpu idle_cpu_snx __initconst = {
1468 .state_table = snb_cstates,
1469 .disable_promotion_to_c1e = true,
1470 .use_acpi = true,
1471 };
1472
1473 static const struct idle_cpu idle_cpu_byt __initconst = {
1474 .state_table = byt_cstates,
1475 .disable_promotion_to_c1e = true,
1476 };
1477
1478 static const struct idle_cpu idle_cpu_cht __initconst = {
1479 .state_table = cht_cstates,
1480 .disable_promotion_to_c1e = true,
1481 };
1482
1483 static const struct idle_cpu idle_cpu_ivb __initconst = {
1484 .state_table = ivb_cstates,
1485 .disable_promotion_to_c1e = true,
1486 };
1487
1488 static const struct idle_cpu idle_cpu_ivt __initconst = {
1489 .state_table = ivt_cstates,
1490 .disable_promotion_to_c1e = true,
1491 .use_acpi = true,
1492 };
1493
1494 static const struct idle_cpu idle_cpu_hsw __initconst = {
1495 .state_table = hsw_cstates,
1496 .disable_promotion_to_c1e = true,
1497 };
1498
1499 static const struct idle_cpu idle_cpu_hsx __initconst = {
1500 .state_table = hsw_cstates,
1501 .disable_promotion_to_c1e = true,
1502 .use_acpi = true,
1503 };
1504
1505 static const struct idle_cpu idle_cpu_bdw __initconst = {
1506 .state_table = bdw_cstates,
1507 .disable_promotion_to_c1e = true,
1508 };
1509
1510 static const struct idle_cpu idle_cpu_bdx __initconst = {
1511 .state_table = bdw_cstates,
1512 .disable_promotion_to_c1e = true,
1513 .use_acpi = true,
1514 };
1515
1516 static const struct idle_cpu idle_cpu_skl __initconst = {
1517 .state_table = skl_cstates,
1518 .disable_promotion_to_c1e = true,
1519 };
1520
1521 static const struct idle_cpu idle_cpu_skx __initconst = {
1522 .state_table = skx_cstates,
1523 .disable_promotion_to_c1e = true,
1524 .use_acpi = true,
1525 };
1526
1527 static const struct idle_cpu idle_cpu_icx __initconst = {
1528 .state_table = icx_cstates,
1529 .disable_promotion_to_c1e = true,
1530 .use_acpi = true,
1531 };
1532
1533 static const struct idle_cpu idle_cpu_adl __initconst = {
1534 .state_table = adl_cstates,
1535 };
1536
1537 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1538 .state_table = adl_l_cstates,
1539 };
1540
1541 static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1542 .state_table = mtl_l_cstates,
1543 };
1544
1545 static const struct idle_cpu idle_cpu_gmt __initconst = {
1546 .state_table = gmt_cstates,
1547 };
1548
1549 static const struct idle_cpu idle_cpu_spr __initconst = {
1550 .state_table = spr_cstates,
1551 .disable_promotion_to_c1e = true,
1552 .use_acpi = true,
1553 };
1554
1555 static const struct idle_cpu idle_cpu_gnr __initconst = {
1556 .state_table = gnr_cstates,
1557 .disable_promotion_to_c1e = true,
1558 .use_acpi = true,
1559 };
1560
1561 static const struct idle_cpu idle_cpu_gnrd __initconst = {
1562 .state_table = gnrd_cstates,
1563 .disable_promotion_to_c1e = true,
1564 .use_acpi = true,
1565 };
1566
1567 static const struct idle_cpu idle_cpu_avn __initconst = {
1568 .state_table = avn_cstates,
1569 .disable_promotion_to_c1e = true,
1570 .use_acpi = true,
1571 };
1572
1573 static const struct idle_cpu idle_cpu_knl __initconst = {
1574 .state_table = knl_cstates,
1575 .use_acpi = true,
1576 };
1577
1578 static const struct idle_cpu idle_cpu_bxt __initconst = {
1579 .state_table = bxt_cstates,
1580 .disable_promotion_to_c1e = true,
1581 };
1582
1583 static const struct idle_cpu idle_cpu_dnv __initconst = {
1584 .state_table = dnv_cstates,
1585 .disable_promotion_to_c1e = true,
1586 .use_acpi = true,
1587 };
1588
1589 static const struct idle_cpu idle_cpu_tmt __initconst = {
1590 .disable_promotion_to_c1e = true,
1591 };
1592
1593 static const struct idle_cpu idle_cpu_snr __initconst = {
1594 .state_table = snr_cstates,
1595 .disable_promotion_to_c1e = true,
1596 .use_acpi = true,
1597 };
1598
1599 static const struct idle_cpu idle_cpu_grr __initconst = {
1600 .state_table = grr_cstates,
1601 .disable_promotion_to_c1e = true,
1602 .use_acpi = true,
1603 };
1604
1605 static const struct idle_cpu idle_cpu_srf __initconst = {
1606 .state_table = srf_cstates,
1607 .disable_promotion_to_c1e = true,
1608 .use_acpi = true,
1609 };
1610
1611 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1612 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx),
1613 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem),
1614 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem),
1615 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem),
1616 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx),
1617 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx),
1618 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom),
1619 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft),
1620 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx),
1621 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb),
1622 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx),
1623 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom),
1624 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt),
1625 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1626 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht),
1627 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb),
1628 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt),
1629 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw),
1630 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx),
1631 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw),
1632 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw),
1633 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn),
1634 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw),
1635 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw),
1636 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx),
1637 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx),
1638 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl),
1639 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl),
1640 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl),
1641 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl),
1642 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx),
1643 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx),
1644 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx),
1645 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl),
1646 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l),
1647 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l),
1648 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt),
1649 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr),
1650 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr),
1651 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr),
1652 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd),
1653 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl),
1654 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl),
1655 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt),
1656 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1657 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv),
1658 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt),
1659 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt),
1660 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr),
1661 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr),
1662 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf),
1663 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf),
1664 {}
1665 };
1666
1667 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1668 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1669 {}
1670 };
1671
intel_idle_max_cstate_reached(int cstate)1672 static bool __init intel_idle_max_cstate_reached(int cstate)
1673 {
1674 if (cstate + 1 > max_cstate) {
1675 pr_info("max_cstate %d reached\n", max_cstate);
1676 return true;
1677 }
1678 return false;
1679 }
1680
intel_idle_state_needs_timer_stop(struct cpuidle_state * state)1681 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1682 {
1683 unsigned long eax = flg2MWAIT(state->flags);
1684
1685 if (boot_cpu_has(X86_FEATURE_ARAT))
1686 return false;
1687
1688 /*
1689 * Switch over to one-shot tick broadcast if the target C-state
1690 * is deeper than C1.
1691 */
1692 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1693 }
1694
1695 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1696 #include <acpi/processor.h>
1697
1698 static bool no_acpi __read_mostly;
1699 module_param(no_acpi, bool, 0444);
1700 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1701
1702 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1703 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1704 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1705
1706 static bool no_native __read_mostly; /* No effect if no_acpi is set. */
1707 module_param_named(no_native, no_native, bool, 0444);
1708 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
1709
1710 static struct acpi_processor_power acpi_state_table __initdata;
1711
1712 /**
1713 * intel_idle_cst_usable - Check if the _CST information can be used.
1714 *
1715 * Check if all of the C-states listed by _CST in the max_cstate range are
1716 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1717 */
intel_idle_cst_usable(void)1718 static bool __init intel_idle_cst_usable(void)
1719 {
1720 int cstate, limit;
1721
1722 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1723 acpi_state_table.count);
1724
1725 for (cstate = 1; cstate < limit; cstate++) {
1726 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1727
1728 if (cx->entry_method != ACPI_CSTATE_FFH)
1729 return false;
1730 }
1731
1732 return true;
1733 }
1734
intel_idle_acpi_cst_extract(void)1735 static bool __init intel_idle_acpi_cst_extract(void)
1736 {
1737 unsigned int cpu;
1738
1739 if (no_acpi) {
1740 pr_debug("Not allowed to use ACPI _CST\n");
1741 return false;
1742 }
1743
1744 for_each_possible_cpu(cpu) {
1745 struct acpi_processor *pr = per_cpu(processors, cpu);
1746
1747 if (!pr)
1748 continue;
1749
1750 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1751 continue;
1752
1753 acpi_state_table.count++;
1754
1755 if (!intel_idle_cst_usable())
1756 continue;
1757
1758 if (!acpi_processor_claim_cst_control())
1759 break;
1760
1761 return true;
1762 }
1763
1764 acpi_state_table.count = 0;
1765 pr_debug("ACPI _CST not found or not usable\n");
1766 return false;
1767 }
1768
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1769 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1770 {
1771 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1772
1773 /*
1774 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1775 * the interesting states are ACPI_CSTATE_FFH.
1776 */
1777 for (cstate = 1; cstate < limit; cstate++) {
1778 struct acpi_processor_cx *cx;
1779 struct cpuidle_state *state;
1780
1781 if (intel_idle_max_cstate_reached(cstate - 1))
1782 break;
1783
1784 cx = &acpi_state_table.states[cstate];
1785
1786 state = &drv->states[drv->state_count++];
1787
1788 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1789 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1790 state->exit_latency = cx->latency;
1791 /*
1792 * For C1-type C-states use the same number for both the exit
1793 * latency and target residency, because that is the case for
1794 * C1 in the majority of the static C-states tables above.
1795 * For the other types of C-states, however, set the target
1796 * residency to 3 times the exit latency which should lead to
1797 * a reasonable balance between energy-efficiency and
1798 * performance in the majority of interesting cases.
1799 */
1800 state->target_residency = cx->latency;
1801 if (cx->type > ACPI_STATE_C1)
1802 state->target_residency *= 3;
1803
1804 state->flags = MWAIT2flg(cx->address);
1805 if (cx->type > ACPI_STATE_C2)
1806 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1807
1808 if (disabled_states_mask & BIT(cstate))
1809 state->flags |= CPUIDLE_FLAG_OFF;
1810
1811 if (intel_idle_state_needs_timer_stop(state))
1812 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1813
1814 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1815 mark_tsc_unstable("TSC halts in idle");
1816
1817 state->enter = intel_idle;
1818 state->enter_dead = intel_idle_enter_dead;
1819 state->enter_s2idle = intel_idle_s2idle;
1820 }
1821 }
1822
intel_idle_off_by_default(unsigned int flags,u32 mwait_hint)1823 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1824 {
1825 int cstate, limit;
1826
1827 /*
1828 * If there are no _CST C-states, do not disable any C-states by
1829 * default.
1830 */
1831 if (!acpi_state_table.count)
1832 return false;
1833
1834 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1835 /*
1836 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1837 * the interesting states are ACPI_CSTATE_FFH.
1838 */
1839 for (cstate = 1; cstate < limit; cstate++) {
1840 u32 acpi_hint = acpi_state_table.states[cstate].address;
1841 u32 table_hint = mwait_hint;
1842
1843 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
1844 acpi_hint &= ~MWAIT_SUBSTATE_MASK;
1845 table_hint &= ~MWAIT_SUBSTATE_MASK;
1846 }
1847
1848 if (acpi_hint == table_hint)
1849 return false;
1850 }
1851 return true;
1852 }
1853
ignore_native(void)1854 static inline bool ignore_native(void)
1855 {
1856 return no_native && !no_acpi;
1857 }
1858 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1859 #define force_use_acpi (false)
1860
intel_idle_acpi_cst_extract(void)1861 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1862 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
intel_idle_off_by_default(unsigned int flags,u32 mwait_hint)1863 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1864 {
1865 return false;
1866 }
ignore_native(void)1867 static inline bool ignore_native(void) { return false; }
1868 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1869
1870 /**
1871 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1872 *
1873 * Tune IVT multi-socket targets.
1874 * Assumption: num_sockets == (max_package_num + 1).
1875 */
ivt_idle_state_table_update(void)1876 static void __init ivt_idle_state_table_update(void)
1877 {
1878 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1879 int cpu, package_num, num_sockets = 1;
1880
1881 for_each_online_cpu(cpu) {
1882 package_num = topology_physical_package_id(cpu);
1883 if (package_num + 1 > num_sockets) {
1884 num_sockets = package_num + 1;
1885
1886 if (num_sockets > 4) {
1887 cpuidle_state_table = ivt_cstates_8s;
1888 return;
1889 }
1890 }
1891 }
1892
1893 if (num_sockets > 2)
1894 cpuidle_state_table = ivt_cstates_4s;
1895
1896 /* else, 1 and 2 socket systems use default ivt_cstates */
1897 }
1898
1899 /**
1900 * irtl_2_usec - IRTL to microseconds conversion.
1901 * @irtl: IRTL MSR value.
1902 *
1903 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1904 */
irtl_2_usec(unsigned long long irtl)1905 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1906 {
1907 static const unsigned int irtl_ns_units[] __initconst = {
1908 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1909 };
1910 unsigned long long ns;
1911
1912 if (!irtl)
1913 return 0;
1914
1915 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1916
1917 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1918 }
1919
1920 /**
1921 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1922 *
1923 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1924 * definitive maximum latency and use the same value for target_residency.
1925 */
bxt_idle_state_table_update(void)1926 static void __init bxt_idle_state_table_update(void)
1927 {
1928 unsigned long long msr;
1929 unsigned int usec;
1930
1931 rdmsrl(MSR_PKGC6_IRTL, msr);
1932 usec = irtl_2_usec(msr);
1933 if (usec) {
1934 bxt_cstates[2].exit_latency = usec;
1935 bxt_cstates[2].target_residency = usec;
1936 }
1937
1938 rdmsrl(MSR_PKGC7_IRTL, msr);
1939 usec = irtl_2_usec(msr);
1940 if (usec) {
1941 bxt_cstates[3].exit_latency = usec;
1942 bxt_cstates[3].target_residency = usec;
1943 }
1944
1945 rdmsrl(MSR_PKGC8_IRTL, msr);
1946 usec = irtl_2_usec(msr);
1947 if (usec) {
1948 bxt_cstates[4].exit_latency = usec;
1949 bxt_cstates[4].target_residency = usec;
1950 }
1951
1952 rdmsrl(MSR_PKGC9_IRTL, msr);
1953 usec = irtl_2_usec(msr);
1954 if (usec) {
1955 bxt_cstates[5].exit_latency = usec;
1956 bxt_cstates[5].target_residency = usec;
1957 }
1958
1959 rdmsrl(MSR_PKGC10_IRTL, msr);
1960 usec = irtl_2_usec(msr);
1961 if (usec) {
1962 bxt_cstates[6].exit_latency = usec;
1963 bxt_cstates[6].target_residency = usec;
1964 }
1965
1966 }
1967
1968 /**
1969 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1970 *
1971 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1972 */
sklh_idle_state_table_update(void)1973 static void __init sklh_idle_state_table_update(void)
1974 {
1975 unsigned long long msr;
1976 unsigned int eax, ebx, ecx, edx;
1977
1978
1979 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1980 if (max_cstate <= 7)
1981 return;
1982
1983 /* if PC10 not present in CPUID.MWAIT.EDX */
1984 if ((mwait_substates & (0xF << 28)) == 0)
1985 return;
1986
1987 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1988
1989 /* PC10 is not enabled in PKG C-state limit */
1990 if ((msr & 0xF) != 8)
1991 return;
1992
1993 ecx = 0;
1994 cpuid(7, &eax, &ebx, &ecx, &edx);
1995
1996 /* if SGX is present */
1997 if (ebx & (1 << 2)) {
1998
1999 rdmsrl(MSR_IA32_FEAT_CTL, msr);
2000
2001 /* if SGX is enabled */
2002 if (msr & (1 << 18))
2003 return;
2004 }
2005
2006 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
2007 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
2008 }
2009
2010 /**
2011 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
2012 * idle states table.
2013 */
skx_idle_state_table_update(void)2014 static void __init skx_idle_state_table_update(void)
2015 {
2016 unsigned long long msr;
2017
2018 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
2019
2020 /*
2021 * 000b: C0/C1 (no package C-state support)
2022 * 001b: C2
2023 * 010b: C6 (non-retention)
2024 * 011b: C6 (retention)
2025 * 111b: No Package C state limits.
2026 */
2027 if ((msr & 0x7) < 2) {
2028 /*
2029 * Uses the CC6 + PC0 latency and 3 times of
2030 * latency for target_residency if the PC6
2031 * is disabled in BIOS. This is consistent
2032 * with how intel_idle driver uses _CST
2033 * to set the target_residency.
2034 */
2035 skx_cstates[2].exit_latency = 92;
2036 skx_cstates[2].target_residency = 276;
2037 }
2038 }
2039
2040 /**
2041 * adl_idle_state_table_update - Adjust AlderLake idle states table.
2042 */
adl_idle_state_table_update(void)2043 static void __init adl_idle_state_table_update(void)
2044 {
2045 /* Check if user prefers C1 over C1E. */
2046 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
2047 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
2048 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
2049
2050 /* Disable C1E by clearing the "C1E promotion" bit. */
2051 c1e_promotion = C1E_PROMOTION_DISABLE;
2052 return;
2053 }
2054
2055 /* Make sure C1E is enabled by default */
2056 c1e_promotion = C1E_PROMOTION_ENABLE;
2057 }
2058
2059 /**
2060 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
2061 */
spr_idle_state_table_update(void)2062 static void __init spr_idle_state_table_update(void)
2063 {
2064 unsigned long long msr;
2065
2066 /*
2067 * By default, the C6 state assumes the worst-case scenario of package
2068 * C6. However, if PC6 is disabled, we update the numbers to match
2069 * core C6.
2070 */
2071 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
2072
2073 /* Limit value 2 and above allow for PC6. */
2074 if ((msr & 0x7) < 2) {
2075 spr_cstates[2].exit_latency = 190;
2076 spr_cstates[2].target_residency = 600;
2077 }
2078 }
2079
2080 /**
2081 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2082 */
byt_cht_auto_demotion_disable(void)2083 static void __init byt_cht_auto_demotion_disable(void)
2084 {
2085 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2086 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2087 }
2088
intel_idle_verify_cstate(unsigned int mwait_hint)2089 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2090 {
2091 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2092 MWAIT_CSTATE_MASK;
2093 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2094 MWAIT_SUBSTATE_MASK;
2095
2096 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
2097 if (num_substates == 0)
2098 return false;
2099
2100 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2101 mark_tsc_unstable("TSC halts in idle states deeper than C2");
2102
2103 return true;
2104 }
2105
state_update_enter_method(struct cpuidle_state * state,int cstate)2106 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2107 {
2108 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2109 /*
2110 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2111 * is not currently supported but this driver.
2112 */
2113 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2114 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2115 state->enter = intel_idle_xstate;
2116 return;
2117 }
2118
2119 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2120 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2121 /*
2122 * IBRS mitigation requires that C-states are entered
2123 * with interrupts disabled.
2124 */
2125 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2126 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2127 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2128 state->enter = intel_idle_ibrs;
2129 return;
2130 }
2131
2132 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2133 state->enter = intel_idle_irq;
2134 return;
2135 }
2136
2137 if (force_irq_on) {
2138 pr_info("forced intel_idle_irq for state %d\n", cstate);
2139 state->enter = intel_idle_irq;
2140 }
2141 }
2142
intel_idle_init_cstates_icpu(struct cpuidle_driver * drv)2143 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2144 {
2145 int cstate;
2146
2147 switch (boot_cpu_data.x86_vfm) {
2148 case INTEL_IVYBRIDGE_X:
2149 ivt_idle_state_table_update();
2150 break;
2151 case INTEL_ATOM_GOLDMONT:
2152 case INTEL_ATOM_GOLDMONT_PLUS:
2153 bxt_idle_state_table_update();
2154 break;
2155 case INTEL_SKYLAKE:
2156 sklh_idle_state_table_update();
2157 break;
2158 case INTEL_SKYLAKE_X:
2159 skx_idle_state_table_update();
2160 break;
2161 case INTEL_SAPPHIRERAPIDS_X:
2162 case INTEL_EMERALDRAPIDS_X:
2163 spr_idle_state_table_update();
2164 break;
2165 case INTEL_ALDERLAKE:
2166 case INTEL_ALDERLAKE_L:
2167 case INTEL_ATOM_GRACEMONT:
2168 adl_idle_state_table_update();
2169 break;
2170 case INTEL_ATOM_SILVERMONT:
2171 case INTEL_ATOM_AIRMONT:
2172 byt_cht_auto_demotion_disable();
2173 break;
2174 }
2175
2176 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2177 struct cpuidle_state *state;
2178 unsigned int mwait_hint;
2179
2180 if (intel_idle_max_cstate_reached(cstate))
2181 break;
2182
2183 if (!cpuidle_state_table[cstate].enter &&
2184 !cpuidle_state_table[cstate].enter_s2idle)
2185 break;
2186
2187 if (!cpuidle_state_table[cstate].enter_dead)
2188 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead;
2189
2190 /* If marked as unusable, skip this state. */
2191 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2192 pr_debug("state %s is disabled\n",
2193 cpuidle_state_table[cstate].name);
2194 continue;
2195 }
2196
2197 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2198 if (!intel_idle_verify_cstate(mwait_hint))
2199 continue;
2200
2201 /* Structure copy. */
2202 drv->states[drv->state_count] = cpuidle_state_table[cstate];
2203 state = &drv->states[drv->state_count];
2204
2205 state_update_enter_method(state, cstate);
2206
2207
2208 if ((disabled_states_mask & BIT(drv->state_count)) ||
2209 ((icpu->use_acpi || force_use_acpi) &&
2210 intel_idle_off_by_default(state->flags, mwait_hint) &&
2211 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2212 state->flags |= CPUIDLE_FLAG_OFF;
2213
2214 if (intel_idle_state_needs_timer_stop(state))
2215 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2216
2217 drv->state_count++;
2218 }
2219 }
2220
2221 /**
2222 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2223 * @drv: cpuidle driver structure to initialize.
2224 */
intel_idle_cpuidle_driver_init(struct cpuidle_driver * drv)2225 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2226 {
2227 cpuidle_poll_state_init(drv);
2228
2229 if (disabled_states_mask & BIT(0))
2230 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2231
2232 drv->state_count = 1;
2233
2234 if (icpu && icpu->state_table)
2235 intel_idle_init_cstates_icpu(drv);
2236 else
2237 intel_idle_init_cstates_acpi(drv);
2238 }
2239
auto_demotion_disable(void)2240 static void auto_demotion_disable(void)
2241 {
2242 unsigned long long msr_bits;
2243
2244 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2245 msr_bits &= ~auto_demotion_disable_flags;
2246 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2247 }
2248
c1e_promotion_enable(void)2249 static void c1e_promotion_enable(void)
2250 {
2251 unsigned long long msr_bits;
2252
2253 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2254 msr_bits |= 0x2;
2255 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2256 }
2257
c1e_promotion_disable(void)2258 static void c1e_promotion_disable(void)
2259 {
2260 unsigned long long msr_bits;
2261
2262 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2263 msr_bits &= ~0x2;
2264 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2265 }
2266
2267 /**
2268 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2269 * @cpu: CPU to initialize.
2270 *
2271 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2272 * with the processor model flags.
2273 */
intel_idle_cpu_init(unsigned int cpu)2274 static int intel_idle_cpu_init(unsigned int cpu)
2275 {
2276 struct cpuidle_device *dev;
2277
2278 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2279 dev->cpu = cpu;
2280
2281 if (cpuidle_register_device(dev)) {
2282 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2283 return -EIO;
2284 }
2285
2286 if (auto_demotion_disable_flags)
2287 auto_demotion_disable();
2288
2289 if (c1e_promotion == C1E_PROMOTION_ENABLE)
2290 c1e_promotion_enable();
2291 else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2292 c1e_promotion_disable();
2293
2294 return 0;
2295 }
2296
intel_idle_cpu_online(unsigned int cpu)2297 static int intel_idle_cpu_online(unsigned int cpu)
2298 {
2299 struct cpuidle_device *dev;
2300
2301 if (!boot_cpu_has(X86_FEATURE_ARAT))
2302 tick_broadcast_enable();
2303
2304 /*
2305 * Some systems can hotplug a cpu at runtime after
2306 * the kernel has booted, we have to initialize the
2307 * driver in this case
2308 */
2309 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2310 if (!dev->registered)
2311 return intel_idle_cpu_init(cpu);
2312
2313 return 0;
2314 }
2315
2316 /**
2317 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2318 */
intel_idle_cpuidle_devices_uninit(void)2319 static void __init intel_idle_cpuidle_devices_uninit(void)
2320 {
2321 int i;
2322
2323 for_each_online_cpu(i)
2324 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2325 }
2326
intel_idle_init(void)2327 static int __init intel_idle_init(void)
2328 {
2329 const struct x86_cpu_id *id;
2330 unsigned int eax, ebx, ecx;
2331 int retval;
2332
2333 /* Do not load intel_idle at all for now if idle= is passed */
2334 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2335 return -ENODEV;
2336
2337 if (max_cstate == 0) {
2338 pr_debug("disabled\n");
2339 return -EPERM;
2340 }
2341
2342 id = x86_match_cpu(intel_idle_ids);
2343 if (id) {
2344 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2345 pr_debug("Please enable MWAIT in BIOS SETUP\n");
2346 return -ENODEV;
2347 }
2348 } else {
2349 id = x86_match_cpu(intel_mwait_ids);
2350 if (!id)
2351 return -ENODEV;
2352 }
2353
2354 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);
2355
2356 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2357 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2358 !mwait_substates)
2359 return -ENODEV;
2360
2361 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2362
2363 icpu = (const struct idle_cpu *)id->driver_data;
2364 if (icpu && ignore_native()) {
2365 pr_debug("ignoring native CPU idle states\n");
2366 icpu = NULL;
2367 }
2368 if (icpu) {
2369 if (icpu->state_table)
2370 cpuidle_state_table = icpu->state_table;
2371 else if (!intel_idle_acpi_cst_extract())
2372 return -ENODEV;
2373
2374 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2375 if (icpu->disable_promotion_to_c1e)
2376 c1e_promotion = C1E_PROMOTION_DISABLE;
2377 if (icpu->use_acpi || force_use_acpi)
2378 intel_idle_acpi_cst_extract();
2379 } else if (!intel_idle_acpi_cst_extract()) {
2380 return -ENODEV;
2381 }
2382
2383 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2384 boot_cpu_data.x86_model);
2385
2386 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2387 if (!intel_idle_cpuidle_devices)
2388 return -ENOMEM;
2389
2390 intel_idle_cpuidle_driver_init(&intel_idle_driver);
2391
2392 retval = cpuidle_register_driver(&intel_idle_driver);
2393 if (retval) {
2394 struct cpuidle_driver *drv = cpuidle_get_driver();
2395 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2396 drv ? drv->name : "none");
2397 goto init_driver_fail;
2398 }
2399
2400 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2401 intel_idle_cpu_online, NULL);
2402 if (retval < 0)
2403 goto hp_setup_fail;
2404
2405 pr_debug("Local APIC timer is reliable in %s\n",
2406 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2407
2408 return 0;
2409
2410 hp_setup_fail:
2411 intel_idle_cpuidle_devices_uninit();
2412 cpuidle_unregister_driver(&intel_idle_driver);
2413 init_driver_fail:
2414 free_percpu(intel_idle_cpuidle_devices);
2415 return retval;
2416
2417 }
2418 device_initcall(intel_idle_init);
2419
2420 /*
2421 * We are not really modular, but we used to support that. Meaning we also
2422 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2423 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2424 * is the easiest way (currently) to continue doing that.
2425 */
2426 module_param(max_cstate, int, 0444);
2427 /*
2428 * The positions of the bits that are set in this number are the indices of the
2429 * idle states to be disabled by default (as reflected by the names of the
2430 * corresponding idle state directories in sysfs, "state0", "state1" ...
2431 * "state<i>" ..., where <i> is the index of the given state).
2432 */
2433 module_param_named(states_off, disabled_states_mask, uint, 0444);
2434 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2435 /*
2436 * Some platforms come with mutually exclusive C-states, so that if one is
2437 * enabled, the other C-states must not be used. Example: C1 and C1E on
2438 * Sapphire Rapids platform. This parameter allows for selecting the
2439 * preferred C-states among the groups of mutually exclusive C-states - the
2440 * selected C-states will be registered, the other C-states from the mutually
2441 * exclusive group won't be registered. If the platform has no mutually
2442 * exclusive C-states, this parameter has no effect.
2443 */
2444 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2445 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2446 /*
2447 * Debugging option that forces the driver to enter all C-states with
2448 * interrupts enabled. Does not apply to C-states with
2449 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2450 */
2451 module_param(force_irq_on, bool, 0444);
2452 /*
2453 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2454 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2455 */
2456 module_param(ibrs_off, bool, 0444);
2457 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
2458