1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013 - 2020, Intel Corporation.
6 * Len Brown <len.brown@intel.com>
7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8 */
9
10 /*
11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
13 * make Linux more efficient on these processors, as intel_idle knows
14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15 */
16
17 /*
18 * Design Assumptions
19 *
20 * All CPUs have same idle states as boot CPU
21 *
22 * Chipset BM_STS (bus master status) bit is a NOP
23 * for preventing entry into deep C-states
24 *
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
27 * instruction needs to be executed to flush the caches)
28 */
29
30 /*
31 * Known limitations
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39 /* un-comment DEBUG to enable pr_debug() statements */
40 /* #define DEBUG */
41
42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44 #include <linux/acpi.h>
45 #include <linux/kernel.h>
46 #include <linux/cpuidle.h>
47 #include <linux/tick.h>
48 #include <trace/events/power.h>
49 #include <linux/sched.h>
50 #include <linux/sched/smt.h>
51 #include <linux/notifier.h>
52 #include <linux/cpu.h>
53 #include <linux/moduleparam.h>
54 #include <asm/cpu_device_id.h>
55 #include <asm/intel-family.h>
56 #include <asm/mwait.h>
57 #include <asm/spec-ctrl.h>
58 #include <asm/fpu/api.h>
59
60 #define INTEL_IDLE_VERSION "0.5.1"
61
62 static struct cpuidle_driver intel_idle_driver = {
63 .name = "intel_idle",
64 .owner = THIS_MODULE,
65 };
66 /* intel_idle.max_cstate=0 disables driver */
67 static int max_cstate = CPUIDLE_STATE_MAX - 1;
68 static unsigned int disabled_states_mask __read_mostly;
69 static unsigned int preferred_states_mask __read_mostly;
70 static bool force_irq_on __read_mostly;
71 static bool ibrs_off __read_mostly;
72
73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
74
75 static unsigned long auto_demotion_disable_flags;
76
77 static enum {
78 C1E_PROMOTION_PRESERVE,
79 C1E_PROMOTION_ENABLE,
80 C1E_PROMOTION_DISABLE
81 } c1e_promotion = C1E_PROMOTION_PRESERVE;
82
83 struct idle_cpu {
84 struct cpuidle_state *state_table;
85
86 /*
87 * Hardware C-state auto-demotion may not always be optimal.
88 * Indicate which enable bits to clear here.
89 */
90 unsigned long auto_demotion_disable_flags;
91 bool byt_auto_demotion_disable_flag;
92 bool disable_promotion_to_c1e;
93 bool use_acpi;
94 };
95
96 static const struct idle_cpu *icpu __initdata;
97 static struct cpuidle_state *cpuidle_state_table __initdata;
98
99 static unsigned int mwait_substates __initdata;
100
101 /*
102 * Enable interrupts before entering the C-state. On some platforms and for
103 * some C-states, this may measurably decrease interrupt latency.
104 */
105 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
106
107 /*
108 * Enable this state by default even if the ACPI _CST does not list it.
109 */
110 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
111
112 /*
113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
114 * above.
115 */
116 #define CPUIDLE_FLAG_IBRS BIT(16)
117
118 /*
119 * Initialize large xstate for the C6-state entrance.
120 */
121 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
122
123 /*
124 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
125 * the C-state (top nibble) and sub-state (bottom nibble)
126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
127 *
128 * We store the hint at the top of our "flags" for each state.
129 */
130 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
131 #define MWAIT2flg(eax) ((eax & 0xFF) << 24)
132
__intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index,bool irqoff)133 static __always_inline int __intel_idle(struct cpuidle_device *dev,
134 struct cpuidle_driver *drv,
135 int index, bool irqoff)
136 {
137 struct cpuidle_state *state = &drv->states[index];
138 unsigned long eax = flg2MWAIT(state->flags);
139 unsigned long ecx = 1*irqoff; /* break on interrupt flag */
140
141 mwait_idle_with_hints(eax, ecx);
142
143 return index;
144 }
145
146 /**
147 * intel_idle - Ask the processor to enter the given idle state.
148 * @dev: cpuidle device of the target CPU.
149 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
150 * @index: Target idle state index.
151 *
152 * Use the MWAIT instruction to notify the processor that the CPU represented by
153 * @dev is idle and it can try to enter the idle state corresponding to @index.
154 *
155 * If the local APIC timer is not known to be reliable in the target idle state,
156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
157 *
158 * Must be called under local_irq_disable().
159 */
intel_idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)160 static __cpuidle int intel_idle(struct cpuidle_device *dev,
161 struct cpuidle_driver *drv, int index)
162 {
163 return __intel_idle(dev, drv, index, true);
164 }
165
intel_idle_irq(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)166 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
167 struct cpuidle_driver *drv, int index)
168 {
169 return __intel_idle(dev, drv, index, false);
170 }
171
intel_idle_ibrs(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)172 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
173 struct cpuidle_driver *drv, int index)
174 {
175 bool smt_active = sched_smt_active();
176 u64 spec_ctrl = spec_ctrl_current();
177 int ret;
178
179 if (smt_active)
180 __update_spec_ctrl(0);
181
182 ret = __intel_idle(dev, drv, index, true);
183
184 if (smt_active)
185 __update_spec_ctrl(spec_ctrl);
186
187 return ret;
188 }
189
intel_idle_xstate(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)190 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
191 struct cpuidle_driver *drv, int index)
192 {
193 fpu_idle_fpregs();
194 return __intel_idle(dev, drv, index, true);
195 }
196
197 /**
198 * intel_idle_s2idle - Ask the processor to enter the given idle state.
199 * @dev: cpuidle device of the target CPU.
200 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
201 * @index: Target idle state index.
202 *
203 * Use the MWAIT instruction to notify the processor that the CPU represented by
204 * @dev is idle and it can try to enter the idle state corresponding to @index.
205 *
206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
207 * scheduler tick and suspended scheduler clock on the target CPU.
208 */
intel_idle_s2idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)209 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
210 struct cpuidle_driver *drv, int index)
211 {
212 unsigned long ecx = 1; /* break on interrupt flag */
213 struct cpuidle_state *state = &drv->states[index];
214 unsigned long eax = flg2MWAIT(state->flags);
215
216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
217 fpu_idle_fpregs();
218
219 mwait_idle_with_hints(eax, ecx);
220
221 return 0;
222 }
223
224 /*
225 * States are indexed by the cstate number,
226 * which is also the index into the MWAIT hint array.
227 * Thus C0 is a dummy.
228 */
229 static struct cpuidle_state nehalem_cstates[] __initdata = {
230 {
231 .name = "C1",
232 .desc = "MWAIT 0x00",
233 .flags = MWAIT2flg(0x00),
234 .exit_latency = 3,
235 .target_residency = 6,
236 .enter = &intel_idle,
237 .enter_s2idle = intel_idle_s2idle, },
238 {
239 .name = "C1E",
240 .desc = "MWAIT 0x01",
241 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
242 .exit_latency = 10,
243 .target_residency = 20,
244 .enter = &intel_idle,
245 .enter_s2idle = intel_idle_s2idle, },
246 {
247 .name = "C3",
248 .desc = "MWAIT 0x10",
249 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
250 .exit_latency = 20,
251 .target_residency = 80,
252 .enter = &intel_idle,
253 .enter_s2idle = intel_idle_s2idle, },
254 {
255 .name = "C6",
256 .desc = "MWAIT 0x20",
257 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
258 .exit_latency = 200,
259 .target_residency = 800,
260 .enter = &intel_idle,
261 .enter_s2idle = intel_idle_s2idle, },
262 {
263 .enter = NULL }
264 };
265
266 static struct cpuidle_state snb_cstates[] __initdata = {
267 {
268 .name = "C1",
269 .desc = "MWAIT 0x00",
270 .flags = MWAIT2flg(0x00),
271 .exit_latency = 2,
272 .target_residency = 2,
273 .enter = &intel_idle,
274 .enter_s2idle = intel_idle_s2idle, },
275 {
276 .name = "C1E",
277 .desc = "MWAIT 0x01",
278 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
279 .exit_latency = 10,
280 .target_residency = 20,
281 .enter = &intel_idle,
282 .enter_s2idle = intel_idle_s2idle, },
283 {
284 .name = "C3",
285 .desc = "MWAIT 0x10",
286 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
287 .exit_latency = 80,
288 .target_residency = 211,
289 .enter = &intel_idle,
290 .enter_s2idle = intel_idle_s2idle, },
291 {
292 .name = "C6",
293 .desc = "MWAIT 0x20",
294 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
295 .exit_latency = 104,
296 .target_residency = 345,
297 .enter = &intel_idle,
298 .enter_s2idle = intel_idle_s2idle, },
299 {
300 .name = "C7",
301 .desc = "MWAIT 0x30",
302 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
303 .exit_latency = 109,
304 .target_residency = 345,
305 .enter = &intel_idle,
306 .enter_s2idle = intel_idle_s2idle, },
307 {
308 .enter = NULL }
309 };
310
311 static struct cpuidle_state byt_cstates[] __initdata = {
312 {
313 .name = "C1",
314 .desc = "MWAIT 0x00",
315 .flags = MWAIT2flg(0x00),
316 .exit_latency = 1,
317 .target_residency = 1,
318 .enter = &intel_idle,
319 .enter_s2idle = intel_idle_s2idle, },
320 {
321 .name = "C6N",
322 .desc = "MWAIT 0x58",
323 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
324 .exit_latency = 300,
325 .target_residency = 275,
326 .enter = &intel_idle,
327 .enter_s2idle = intel_idle_s2idle, },
328 {
329 .name = "C6S",
330 .desc = "MWAIT 0x52",
331 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
332 .exit_latency = 500,
333 .target_residency = 560,
334 .enter = &intel_idle,
335 .enter_s2idle = intel_idle_s2idle, },
336 {
337 .name = "C7",
338 .desc = "MWAIT 0x60",
339 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
340 .exit_latency = 1200,
341 .target_residency = 4000,
342 .enter = &intel_idle,
343 .enter_s2idle = intel_idle_s2idle, },
344 {
345 .name = "C7S",
346 .desc = "MWAIT 0x64",
347 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
348 .exit_latency = 10000,
349 .target_residency = 20000,
350 .enter = &intel_idle,
351 .enter_s2idle = intel_idle_s2idle, },
352 {
353 .enter = NULL }
354 };
355
356 static struct cpuidle_state cht_cstates[] __initdata = {
357 {
358 .name = "C1",
359 .desc = "MWAIT 0x00",
360 .flags = MWAIT2flg(0x00),
361 .exit_latency = 1,
362 .target_residency = 1,
363 .enter = &intel_idle,
364 .enter_s2idle = intel_idle_s2idle, },
365 {
366 .name = "C6N",
367 .desc = "MWAIT 0x58",
368 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
369 .exit_latency = 80,
370 .target_residency = 275,
371 .enter = &intel_idle,
372 .enter_s2idle = intel_idle_s2idle, },
373 {
374 .name = "C6S",
375 .desc = "MWAIT 0x52",
376 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
377 .exit_latency = 200,
378 .target_residency = 560,
379 .enter = &intel_idle,
380 .enter_s2idle = intel_idle_s2idle, },
381 {
382 .name = "C7",
383 .desc = "MWAIT 0x60",
384 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
385 .exit_latency = 1200,
386 .target_residency = 4000,
387 .enter = &intel_idle,
388 .enter_s2idle = intel_idle_s2idle, },
389 {
390 .name = "C7S",
391 .desc = "MWAIT 0x64",
392 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
393 .exit_latency = 10000,
394 .target_residency = 20000,
395 .enter = &intel_idle,
396 .enter_s2idle = intel_idle_s2idle, },
397 {
398 .enter = NULL }
399 };
400
401 static struct cpuidle_state ivb_cstates[] __initdata = {
402 {
403 .name = "C1",
404 .desc = "MWAIT 0x00",
405 .flags = MWAIT2flg(0x00),
406 .exit_latency = 1,
407 .target_residency = 1,
408 .enter = &intel_idle,
409 .enter_s2idle = intel_idle_s2idle, },
410 {
411 .name = "C1E",
412 .desc = "MWAIT 0x01",
413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
414 .exit_latency = 10,
415 .target_residency = 20,
416 .enter = &intel_idle,
417 .enter_s2idle = intel_idle_s2idle, },
418 {
419 .name = "C3",
420 .desc = "MWAIT 0x10",
421 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
422 .exit_latency = 59,
423 .target_residency = 156,
424 .enter = &intel_idle,
425 .enter_s2idle = intel_idle_s2idle, },
426 {
427 .name = "C6",
428 .desc = "MWAIT 0x20",
429 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
430 .exit_latency = 80,
431 .target_residency = 300,
432 .enter = &intel_idle,
433 .enter_s2idle = intel_idle_s2idle, },
434 {
435 .name = "C7",
436 .desc = "MWAIT 0x30",
437 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
438 .exit_latency = 87,
439 .target_residency = 300,
440 .enter = &intel_idle,
441 .enter_s2idle = intel_idle_s2idle, },
442 {
443 .enter = NULL }
444 };
445
446 static struct cpuidle_state ivt_cstates[] __initdata = {
447 {
448 .name = "C1",
449 .desc = "MWAIT 0x00",
450 .flags = MWAIT2flg(0x00),
451 .exit_latency = 1,
452 .target_residency = 1,
453 .enter = &intel_idle,
454 .enter_s2idle = intel_idle_s2idle, },
455 {
456 .name = "C1E",
457 .desc = "MWAIT 0x01",
458 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
459 .exit_latency = 10,
460 .target_residency = 80,
461 .enter = &intel_idle,
462 .enter_s2idle = intel_idle_s2idle, },
463 {
464 .name = "C3",
465 .desc = "MWAIT 0x10",
466 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
467 .exit_latency = 59,
468 .target_residency = 156,
469 .enter = &intel_idle,
470 .enter_s2idle = intel_idle_s2idle, },
471 {
472 .name = "C6",
473 .desc = "MWAIT 0x20",
474 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
475 .exit_latency = 82,
476 .target_residency = 300,
477 .enter = &intel_idle,
478 .enter_s2idle = intel_idle_s2idle, },
479 {
480 .enter = NULL }
481 };
482
483 static struct cpuidle_state ivt_cstates_4s[] __initdata = {
484 {
485 .name = "C1",
486 .desc = "MWAIT 0x00",
487 .flags = MWAIT2flg(0x00),
488 .exit_latency = 1,
489 .target_residency = 1,
490 .enter = &intel_idle,
491 .enter_s2idle = intel_idle_s2idle, },
492 {
493 .name = "C1E",
494 .desc = "MWAIT 0x01",
495 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
496 .exit_latency = 10,
497 .target_residency = 250,
498 .enter = &intel_idle,
499 .enter_s2idle = intel_idle_s2idle, },
500 {
501 .name = "C3",
502 .desc = "MWAIT 0x10",
503 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
504 .exit_latency = 59,
505 .target_residency = 300,
506 .enter = &intel_idle,
507 .enter_s2idle = intel_idle_s2idle, },
508 {
509 .name = "C6",
510 .desc = "MWAIT 0x20",
511 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
512 .exit_latency = 84,
513 .target_residency = 400,
514 .enter = &intel_idle,
515 .enter_s2idle = intel_idle_s2idle, },
516 {
517 .enter = NULL }
518 };
519
520 static struct cpuidle_state ivt_cstates_8s[] __initdata = {
521 {
522 .name = "C1",
523 .desc = "MWAIT 0x00",
524 .flags = MWAIT2flg(0x00),
525 .exit_latency = 1,
526 .target_residency = 1,
527 .enter = &intel_idle,
528 .enter_s2idle = intel_idle_s2idle, },
529 {
530 .name = "C1E",
531 .desc = "MWAIT 0x01",
532 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
533 .exit_latency = 10,
534 .target_residency = 500,
535 .enter = &intel_idle,
536 .enter_s2idle = intel_idle_s2idle, },
537 {
538 .name = "C3",
539 .desc = "MWAIT 0x10",
540 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
541 .exit_latency = 59,
542 .target_residency = 600,
543 .enter = &intel_idle,
544 .enter_s2idle = intel_idle_s2idle, },
545 {
546 .name = "C6",
547 .desc = "MWAIT 0x20",
548 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
549 .exit_latency = 88,
550 .target_residency = 700,
551 .enter = &intel_idle,
552 .enter_s2idle = intel_idle_s2idle, },
553 {
554 .enter = NULL }
555 };
556
557 static struct cpuidle_state hsw_cstates[] __initdata = {
558 {
559 .name = "C1",
560 .desc = "MWAIT 0x00",
561 .flags = MWAIT2flg(0x00),
562 .exit_latency = 2,
563 .target_residency = 2,
564 .enter = &intel_idle,
565 .enter_s2idle = intel_idle_s2idle, },
566 {
567 .name = "C1E",
568 .desc = "MWAIT 0x01",
569 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
570 .exit_latency = 10,
571 .target_residency = 20,
572 .enter = &intel_idle,
573 .enter_s2idle = intel_idle_s2idle, },
574 {
575 .name = "C3",
576 .desc = "MWAIT 0x10",
577 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
578 .exit_latency = 33,
579 .target_residency = 100,
580 .enter = &intel_idle,
581 .enter_s2idle = intel_idle_s2idle, },
582 {
583 .name = "C6",
584 .desc = "MWAIT 0x20",
585 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
586 .exit_latency = 133,
587 .target_residency = 400,
588 .enter = &intel_idle,
589 .enter_s2idle = intel_idle_s2idle, },
590 {
591 .name = "C7s",
592 .desc = "MWAIT 0x32",
593 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
594 .exit_latency = 166,
595 .target_residency = 500,
596 .enter = &intel_idle,
597 .enter_s2idle = intel_idle_s2idle, },
598 {
599 .name = "C8",
600 .desc = "MWAIT 0x40",
601 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
602 .exit_latency = 300,
603 .target_residency = 900,
604 .enter = &intel_idle,
605 .enter_s2idle = intel_idle_s2idle, },
606 {
607 .name = "C9",
608 .desc = "MWAIT 0x50",
609 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
610 .exit_latency = 600,
611 .target_residency = 1800,
612 .enter = &intel_idle,
613 .enter_s2idle = intel_idle_s2idle, },
614 {
615 .name = "C10",
616 .desc = "MWAIT 0x60",
617 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
618 .exit_latency = 2600,
619 .target_residency = 7700,
620 .enter = &intel_idle,
621 .enter_s2idle = intel_idle_s2idle, },
622 {
623 .enter = NULL }
624 };
625 static struct cpuidle_state bdw_cstates[] __initdata = {
626 {
627 .name = "C1",
628 .desc = "MWAIT 0x00",
629 .flags = MWAIT2flg(0x00),
630 .exit_latency = 2,
631 .target_residency = 2,
632 .enter = &intel_idle,
633 .enter_s2idle = intel_idle_s2idle, },
634 {
635 .name = "C1E",
636 .desc = "MWAIT 0x01",
637 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
638 .exit_latency = 10,
639 .target_residency = 20,
640 .enter = &intel_idle,
641 .enter_s2idle = intel_idle_s2idle, },
642 {
643 .name = "C3",
644 .desc = "MWAIT 0x10",
645 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
646 .exit_latency = 40,
647 .target_residency = 100,
648 .enter = &intel_idle,
649 .enter_s2idle = intel_idle_s2idle, },
650 {
651 .name = "C6",
652 .desc = "MWAIT 0x20",
653 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
654 .exit_latency = 133,
655 .target_residency = 400,
656 .enter = &intel_idle,
657 .enter_s2idle = intel_idle_s2idle, },
658 {
659 .name = "C7s",
660 .desc = "MWAIT 0x32",
661 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
662 .exit_latency = 166,
663 .target_residency = 500,
664 .enter = &intel_idle,
665 .enter_s2idle = intel_idle_s2idle, },
666 {
667 .name = "C8",
668 .desc = "MWAIT 0x40",
669 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
670 .exit_latency = 300,
671 .target_residency = 900,
672 .enter = &intel_idle,
673 .enter_s2idle = intel_idle_s2idle, },
674 {
675 .name = "C9",
676 .desc = "MWAIT 0x50",
677 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
678 .exit_latency = 600,
679 .target_residency = 1800,
680 .enter = &intel_idle,
681 .enter_s2idle = intel_idle_s2idle, },
682 {
683 .name = "C10",
684 .desc = "MWAIT 0x60",
685 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
686 .exit_latency = 2600,
687 .target_residency = 7700,
688 .enter = &intel_idle,
689 .enter_s2idle = intel_idle_s2idle, },
690 {
691 .enter = NULL }
692 };
693
694 static struct cpuidle_state skl_cstates[] __initdata = {
695 {
696 .name = "C1",
697 .desc = "MWAIT 0x00",
698 .flags = MWAIT2flg(0x00),
699 .exit_latency = 2,
700 .target_residency = 2,
701 .enter = &intel_idle,
702 .enter_s2idle = intel_idle_s2idle, },
703 {
704 .name = "C1E",
705 .desc = "MWAIT 0x01",
706 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
707 .exit_latency = 10,
708 .target_residency = 20,
709 .enter = &intel_idle,
710 .enter_s2idle = intel_idle_s2idle, },
711 {
712 .name = "C3",
713 .desc = "MWAIT 0x10",
714 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
715 .exit_latency = 70,
716 .target_residency = 100,
717 .enter = &intel_idle,
718 .enter_s2idle = intel_idle_s2idle, },
719 {
720 .name = "C6",
721 .desc = "MWAIT 0x20",
722 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
723 .exit_latency = 85,
724 .target_residency = 200,
725 .enter = &intel_idle,
726 .enter_s2idle = intel_idle_s2idle, },
727 {
728 .name = "C7s",
729 .desc = "MWAIT 0x33",
730 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
731 .exit_latency = 124,
732 .target_residency = 800,
733 .enter = &intel_idle,
734 .enter_s2idle = intel_idle_s2idle, },
735 {
736 .name = "C8",
737 .desc = "MWAIT 0x40",
738 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
739 .exit_latency = 200,
740 .target_residency = 800,
741 .enter = &intel_idle,
742 .enter_s2idle = intel_idle_s2idle, },
743 {
744 .name = "C9",
745 .desc = "MWAIT 0x50",
746 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
747 .exit_latency = 480,
748 .target_residency = 5000,
749 .enter = &intel_idle,
750 .enter_s2idle = intel_idle_s2idle, },
751 {
752 .name = "C10",
753 .desc = "MWAIT 0x60",
754 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
755 .exit_latency = 890,
756 .target_residency = 5000,
757 .enter = &intel_idle,
758 .enter_s2idle = intel_idle_s2idle, },
759 {
760 .enter = NULL }
761 };
762
763 static struct cpuidle_state skx_cstates[] __initdata = {
764 {
765 .name = "C1",
766 .desc = "MWAIT 0x00",
767 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
768 .exit_latency = 2,
769 .target_residency = 2,
770 .enter = &intel_idle,
771 .enter_s2idle = intel_idle_s2idle, },
772 {
773 .name = "C1E",
774 .desc = "MWAIT 0x01",
775 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
776 .exit_latency = 10,
777 .target_residency = 20,
778 .enter = &intel_idle,
779 .enter_s2idle = intel_idle_s2idle, },
780 {
781 .name = "C6",
782 .desc = "MWAIT 0x20",
783 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
784 .exit_latency = 133,
785 .target_residency = 600,
786 .enter = &intel_idle,
787 .enter_s2idle = intel_idle_s2idle, },
788 {
789 .enter = NULL }
790 };
791
792 static struct cpuidle_state icx_cstates[] __initdata = {
793 {
794 .name = "C1",
795 .desc = "MWAIT 0x00",
796 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
797 .exit_latency = 1,
798 .target_residency = 1,
799 .enter = &intel_idle,
800 .enter_s2idle = intel_idle_s2idle, },
801 {
802 .name = "C1E",
803 .desc = "MWAIT 0x01",
804 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
805 .exit_latency = 4,
806 .target_residency = 4,
807 .enter = &intel_idle,
808 .enter_s2idle = intel_idle_s2idle, },
809 {
810 .name = "C6",
811 .desc = "MWAIT 0x20",
812 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
813 .exit_latency = 170,
814 .target_residency = 600,
815 .enter = &intel_idle,
816 .enter_s2idle = intel_idle_s2idle, },
817 {
818 .enter = NULL }
819 };
820
821 /*
822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
824 * But in this case there is effectively no C1, because C1 requests are
825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
826 * and C1E requests end up with C1, so there is effectively no C1E.
827 *
828 * By default we enable C1E and disable C1 by marking it with
829 * 'CPUIDLE_FLAG_UNUSABLE'.
830 */
831 static struct cpuidle_state adl_cstates[] __initdata = {
832 {
833 .name = "C1",
834 .desc = "MWAIT 0x00",
835 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
836 .exit_latency = 1,
837 .target_residency = 1,
838 .enter = &intel_idle,
839 .enter_s2idle = intel_idle_s2idle, },
840 {
841 .name = "C1E",
842 .desc = "MWAIT 0x01",
843 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
844 .exit_latency = 2,
845 .target_residency = 4,
846 .enter = &intel_idle,
847 .enter_s2idle = intel_idle_s2idle, },
848 {
849 .name = "C6",
850 .desc = "MWAIT 0x20",
851 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
852 .exit_latency = 220,
853 .target_residency = 600,
854 .enter = &intel_idle,
855 .enter_s2idle = intel_idle_s2idle, },
856 {
857 .name = "C8",
858 .desc = "MWAIT 0x40",
859 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
860 .exit_latency = 280,
861 .target_residency = 800,
862 .enter = &intel_idle,
863 .enter_s2idle = intel_idle_s2idle, },
864 {
865 .name = "C10",
866 .desc = "MWAIT 0x60",
867 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
868 .exit_latency = 680,
869 .target_residency = 2000,
870 .enter = &intel_idle,
871 .enter_s2idle = intel_idle_s2idle, },
872 {
873 .enter = NULL }
874 };
875
876 static struct cpuidle_state adl_l_cstates[] __initdata = {
877 {
878 .name = "C1",
879 .desc = "MWAIT 0x00",
880 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
881 .exit_latency = 1,
882 .target_residency = 1,
883 .enter = &intel_idle,
884 .enter_s2idle = intel_idle_s2idle, },
885 {
886 .name = "C1E",
887 .desc = "MWAIT 0x01",
888 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
889 .exit_latency = 2,
890 .target_residency = 4,
891 .enter = &intel_idle,
892 .enter_s2idle = intel_idle_s2idle, },
893 {
894 .name = "C6",
895 .desc = "MWAIT 0x20",
896 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
897 .exit_latency = 170,
898 .target_residency = 500,
899 .enter = &intel_idle,
900 .enter_s2idle = intel_idle_s2idle, },
901 {
902 .name = "C8",
903 .desc = "MWAIT 0x40",
904 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
905 .exit_latency = 200,
906 .target_residency = 600,
907 .enter = &intel_idle,
908 .enter_s2idle = intel_idle_s2idle, },
909 {
910 .name = "C10",
911 .desc = "MWAIT 0x60",
912 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
913 .exit_latency = 230,
914 .target_residency = 700,
915 .enter = &intel_idle,
916 .enter_s2idle = intel_idle_s2idle, },
917 {
918 .enter = NULL }
919 };
920
921 static struct cpuidle_state mtl_l_cstates[] __initdata = {
922 {
923 .name = "C1E",
924 .desc = "MWAIT 0x01",
925 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
926 .exit_latency = 1,
927 .target_residency = 1,
928 .enter = &intel_idle,
929 .enter_s2idle = intel_idle_s2idle, },
930 {
931 .name = "C6",
932 .desc = "MWAIT 0x20",
933 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
934 .exit_latency = 140,
935 .target_residency = 420,
936 .enter = &intel_idle,
937 .enter_s2idle = intel_idle_s2idle, },
938 {
939 .name = "C10",
940 .desc = "MWAIT 0x60",
941 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
942 .exit_latency = 310,
943 .target_residency = 930,
944 .enter = &intel_idle,
945 .enter_s2idle = intel_idle_s2idle, },
946 {
947 .enter = NULL }
948 };
949
950 static struct cpuidle_state gmt_cstates[] __initdata = {
951 {
952 .name = "C1",
953 .desc = "MWAIT 0x00",
954 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
955 .exit_latency = 1,
956 .target_residency = 1,
957 .enter = &intel_idle,
958 .enter_s2idle = intel_idle_s2idle, },
959 {
960 .name = "C1E",
961 .desc = "MWAIT 0x01",
962 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
963 .exit_latency = 2,
964 .target_residency = 4,
965 .enter = &intel_idle,
966 .enter_s2idle = intel_idle_s2idle, },
967 {
968 .name = "C6",
969 .desc = "MWAIT 0x20",
970 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
971 .exit_latency = 195,
972 .target_residency = 585,
973 .enter = &intel_idle,
974 .enter_s2idle = intel_idle_s2idle, },
975 {
976 .name = "C8",
977 .desc = "MWAIT 0x40",
978 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
979 .exit_latency = 260,
980 .target_residency = 1040,
981 .enter = &intel_idle,
982 .enter_s2idle = intel_idle_s2idle, },
983 {
984 .name = "C10",
985 .desc = "MWAIT 0x60",
986 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
987 .exit_latency = 660,
988 .target_residency = 1980,
989 .enter = &intel_idle,
990 .enter_s2idle = intel_idle_s2idle, },
991 {
992 .enter = NULL }
993 };
994
995 static struct cpuidle_state spr_cstates[] __initdata = {
996 {
997 .name = "C1",
998 .desc = "MWAIT 0x00",
999 .flags = MWAIT2flg(0x00),
1000 .exit_latency = 1,
1001 .target_residency = 1,
1002 .enter = &intel_idle,
1003 .enter_s2idle = intel_idle_s2idle, },
1004 {
1005 .name = "C1E",
1006 .desc = "MWAIT 0x01",
1007 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1008 .exit_latency = 2,
1009 .target_residency = 4,
1010 .enter = &intel_idle,
1011 .enter_s2idle = intel_idle_s2idle, },
1012 {
1013 .name = "C6",
1014 .desc = "MWAIT 0x20",
1015 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1016 CPUIDLE_FLAG_INIT_XSTATE,
1017 .exit_latency = 290,
1018 .target_residency = 800,
1019 .enter = &intel_idle,
1020 .enter_s2idle = intel_idle_s2idle, },
1021 {
1022 .enter = NULL }
1023 };
1024
1025 static struct cpuidle_state gnr_cstates[] __initdata = {
1026 {
1027 .name = "C1",
1028 .desc = "MWAIT 0x00",
1029 .flags = MWAIT2flg(0x00),
1030 .exit_latency = 1,
1031 .target_residency = 1,
1032 .enter = &intel_idle,
1033 .enter_s2idle = intel_idle_s2idle, },
1034 {
1035 .name = "C1E",
1036 .desc = "MWAIT 0x01",
1037 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1038 .exit_latency = 4,
1039 .target_residency = 4,
1040 .enter = &intel_idle,
1041 .enter_s2idle = intel_idle_s2idle, },
1042 {
1043 .name = "C6",
1044 .desc = "MWAIT 0x20",
1045 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1046 CPUIDLE_FLAG_INIT_XSTATE,
1047 .exit_latency = 170,
1048 .target_residency = 650,
1049 .enter = &intel_idle,
1050 .enter_s2idle = intel_idle_s2idle, },
1051 {
1052 .name = "C6P",
1053 .desc = "MWAIT 0x21",
1054 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1055 CPUIDLE_FLAG_INIT_XSTATE,
1056 .exit_latency = 210,
1057 .target_residency = 1000,
1058 .enter = &intel_idle,
1059 .enter_s2idle = intel_idle_s2idle, },
1060 {
1061 .enter = NULL }
1062 };
1063
1064 static struct cpuidle_state atom_cstates[] __initdata = {
1065 {
1066 .name = "C1E",
1067 .desc = "MWAIT 0x00",
1068 .flags = MWAIT2flg(0x00),
1069 .exit_latency = 10,
1070 .target_residency = 20,
1071 .enter = &intel_idle,
1072 .enter_s2idle = intel_idle_s2idle, },
1073 {
1074 .name = "C2",
1075 .desc = "MWAIT 0x10",
1076 .flags = MWAIT2flg(0x10),
1077 .exit_latency = 20,
1078 .target_residency = 80,
1079 .enter = &intel_idle,
1080 .enter_s2idle = intel_idle_s2idle, },
1081 {
1082 .name = "C4",
1083 .desc = "MWAIT 0x30",
1084 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1085 .exit_latency = 100,
1086 .target_residency = 400,
1087 .enter = &intel_idle,
1088 .enter_s2idle = intel_idle_s2idle, },
1089 {
1090 .name = "C6",
1091 .desc = "MWAIT 0x52",
1092 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1093 .exit_latency = 140,
1094 .target_residency = 560,
1095 .enter = &intel_idle,
1096 .enter_s2idle = intel_idle_s2idle, },
1097 {
1098 .enter = NULL }
1099 };
1100 static struct cpuidle_state tangier_cstates[] __initdata = {
1101 {
1102 .name = "C1",
1103 .desc = "MWAIT 0x00",
1104 .flags = MWAIT2flg(0x00),
1105 .exit_latency = 1,
1106 .target_residency = 4,
1107 .enter = &intel_idle,
1108 .enter_s2idle = intel_idle_s2idle, },
1109 {
1110 .name = "C4",
1111 .desc = "MWAIT 0x30",
1112 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1113 .exit_latency = 100,
1114 .target_residency = 400,
1115 .enter = &intel_idle,
1116 .enter_s2idle = intel_idle_s2idle, },
1117 {
1118 .name = "C6",
1119 .desc = "MWAIT 0x52",
1120 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1121 .exit_latency = 140,
1122 .target_residency = 560,
1123 .enter = &intel_idle,
1124 .enter_s2idle = intel_idle_s2idle, },
1125 {
1126 .name = "C7",
1127 .desc = "MWAIT 0x60",
1128 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1129 .exit_latency = 1200,
1130 .target_residency = 4000,
1131 .enter = &intel_idle,
1132 .enter_s2idle = intel_idle_s2idle, },
1133 {
1134 .name = "C9",
1135 .desc = "MWAIT 0x64",
1136 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1137 .exit_latency = 10000,
1138 .target_residency = 20000,
1139 .enter = &intel_idle,
1140 .enter_s2idle = intel_idle_s2idle, },
1141 {
1142 .enter = NULL }
1143 };
1144 static struct cpuidle_state avn_cstates[] __initdata = {
1145 {
1146 .name = "C1",
1147 .desc = "MWAIT 0x00",
1148 .flags = MWAIT2flg(0x00),
1149 .exit_latency = 2,
1150 .target_residency = 2,
1151 .enter = &intel_idle,
1152 .enter_s2idle = intel_idle_s2idle, },
1153 {
1154 .name = "C6",
1155 .desc = "MWAIT 0x51",
1156 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1157 .exit_latency = 15,
1158 .target_residency = 45,
1159 .enter = &intel_idle,
1160 .enter_s2idle = intel_idle_s2idle, },
1161 {
1162 .enter = NULL }
1163 };
1164 static struct cpuidle_state knl_cstates[] __initdata = {
1165 {
1166 .name = "C1",
1167 .desc = "MWAIT 0x00",
1168 .flags = MWAIT2flg(0x00),
1169 .exit_latency = 1,
1170 .target_residency = 2,
1171 .enter = &intel_idle,
1172 .enter_s2idle = intel_idle_s2idle },
1173 {
1174 .name = "C6",
1175 .desc = "MWAIT 0x10",
1176 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1177 .exit_latency = 120,
1178 .target_residency = 500,
1179 .enter = &intel_idle,
1180 .enter_s2idle = intel_idle_s2idle },
1181 {
1182 .enter = NULL }
1183 };
1184
1185 static struct cpuidle_state bxt_cstates[] __initdata = {
1186 {
1187 .name = "C1",
1188 .desc = "MWAIT 0x00",
1189 .flags = MWAIT2flg(0x00),
1190 .exit_latency = 2,
1191 .target_residency = 2,
1192 .enter = &intel_idle,
1193 .enter_s2idle = intel_idle_s2idle, },
1194 {
1195 .name = "C1E",
1196 .desc = "MWAIT 0x01",
1197 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1198 .exit_latency = 10,
1199 .target_residency = 20,
1200 .enter = &intel_idle,
1201 .enter_s2idle = intel_idle_s2idle, },
1202 {
1203 .name = "C6",
1204 .desc = "MWAIT 0x20",
1205 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1206 .exit_latency = 133,
1207 .target_residency = 133,
1208 .enter = &intel_idle,
1209 .enter_s2idle = intel_idle_s2idle, },
1210 {
1211 .name = "C7s",
1212 .desc = "MWAIT 0x31",
1213 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1214 .exit_latency = 155,
1215 .target_residency = 155,
1216 .enter = &intel_idle,
1217 .enter_s2idle = intel_idle_s2idle, },
1218 {
1219 .name = "C8",
1220 .desc = "MWAIT 0x40",
1221 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1222 .exit_latency = 1000,
1223 .target_residency = 1000,
1224 .enter = &intel_idle,
1225 .enter_s2idle = intel_idle_s2idle, },
1226 {
1227 .name = "C9",
1228 .desc = "MWAIT 0x50",
1229 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1230 .exit_latency = 2000,
1231 .target_residency = 2000,
1232 .enter = &intel_idle,
1233 .enter_s2idle = intel_idle_s2idle, },
1234 {
1235 .name = "C10",
1236 .desc = "MWAIT 0x60",
1237 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1238 .exit_latency = 10000,
1239 .target_residency = 10000,
1240 .enter = &intel_idle,
1241 .enter_s2idle = intel_idle_s2idle, },
1242 {
1243 .enter = NULL }
1244 };
1245
1246 static struct cpuidle_state dnv_cstates[] __initdata = {
1247 {
1248 .name = "C1",
1249 .desc = "MWAIT 0x00",
1250 .flags = MWAIT2flg(0x00),
1251 .exit_latency = 2,
1252 .target_residency = 2,
1253 .enter = &intel_idle,
1254 .enter_s2idle = intel_idle_s2idle, },
1255 {
1256 .name = "C1E",
1257 .desc = "MWAIT 0x01",
1258 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1259 .exit_latency = 10,
1260 .target_residency = 20,
1261 .enter = &intel_idle,
1262 .enter_s2idle = intel_idle_s2idle, },
1263 {
1264 .name = "C6",
1265 .desc = "MWAIT 0x20",
1266 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1267 .exit_latency = 50,
1268 .target_residency = 500,
1269 .enter = &intel_idle,
1270 .enter_s2idle = intel_idle_s2idle, },
1271 {
1272 .enter = NULL }
1273 };
1274
1275 /*
1276 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1277 * C6, and this is indicated in the CPUID mwait leaf.
1278 */
1279 static struct cpuidle_state snr_cstates[] __initdata = {
1280 {
1281 .name = "C1",
1282 .desc = "MWAIT 0x00",
1283 .flags = MWAIT2flg(0x00),
1284 .exit_latency = 2,
1285 .target_residency = 2,
1286 .enter = &intel_idle,
1287 .enter_s2idle = intel_idle_s2idle, },
1288 {
1289 .name = "C1E",
1290 .desc = "MWAIT 0x01",
1291 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1292 .exit_latency = 15,
1293 .target_residency = 25,
1294 .enter = &intel_idle,
1295 .enter_s2idle = intel_idle_s2idle, },
1296 {
1297 .name = "C6",
1298 .desc = "MWAIT 0x20",
1299 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1300 .exit_latency = 130,
1301 .target_residency = 500,
1302 .enter = &intel_idle,
1303 .enter_s2idle = intel_idle_s2idle, },
1304 {
1305 .enter = NULL }
1306 };
1307
1308 static struct cpuidle_state grr_cstates[] __initdata = {
1309 {
1310 .name = "C1",
1311 .desc = "MWAIT 0x00",
1312 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1313 .exit_latency = 1,
1314 .target_residency = 1,
1315 .enter = &intel_idle,
1316 .enter_s2idle = intel_idle_s2idle, },
1317 {
1318 .name = "C1E",
1319 .desc = "MWAIT 0x01",
1320 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1321 .exit_latency = 2,
1322 .target_residency = 10,
1323 .enter = &intel_idle,
1324 .enter_s2idle = intel_idle_s2idle, },
1325 {
1326 .name = "C6S",
1327 .desc = "MWAIT 0x22",
1328 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1329 .exit_latency = 140,
1330 .target_residency = 500,
1331 .enter = &intel_idle,
1332 .enter_s2idle = intel_idle_s2idle, },
1333 {
1334 .enter = NULL }
1335 };
1336
1337 static struct cpuidle_state srf_cstates[] __initdata = {
1338 {
1339 .name = "C1",
1340 .desc = "MWAIT 0x00",
1341 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1342 .exit_latency = 1,
1343 .target_residency = 1,
1344 .enter = &intel_idle,
1345 .enter_s2idle = intel_idle_s2idle, },
1346 {
1347 .name = "C1E",
1348 .desc = "MWAIT 0x01",
1349 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1350 .exit_latency = 2,
1351 .target_residency = 10,
1352 .enter = &intel_idle,
1353 .enter_s2idle = intel_idle_s2idle, },
1354 {
1355 .name = "C6S",
1356 .desc = "MWAIT 0x22",
1357 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1358 .exit_latency = 270,
1359 .target_residency = 700,
1360 .enter = &intel_idle,
1361 .enter_s2idle = intel_idle_s2idle, },
1362 {
1363 .name = "C6SP",
1364 .desc = "MWAIT 0x23",
1365 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED,
1366 .exit_latency = 310,
1367 .target_residency = 900,
1368 .enter = &intel_idle,
1369 .enter_s2idle = intel_idle_s2idle, },
1370 {
1371 .enter = NULL }
1372 };
1373
1374 static const struct idle_cpu idle_cpu_nehalem __initconst = {
1375 .state_table = nehalem_cstates,
1376 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1377 .disable_promotion_to_c1e = true,
1378 };
1379
1380 static const struct idle_cpu idle_cpu_nhx __initconst = {
1381 .state_table = nehalem_cstates,
1382 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1383 .disable_promotion_to_c1e = true,
1384 .use_acpi = true,
1385 };
1386
1387 static const struct idle_cpu idle_cpu_atom __initconst = {
1388 .state_table = atom_cstates,
1389 };
1390
1391 static const struct idle_cpu idle_cpu_tangier __initconst = {
1392 .state_table = tangier_cstates,
1393 };
1394
1395 static const struct idle_cpu idle_cpu_lincroft __initconst = {
1396 .state_table = atom_cstates,
1397 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1398 };
1399
1400 static const struct idle_cpu idle_cpu_snb __initconst = {
1401 .state_table = snb_cstates,
1402 .disable_promotion_to_c1e = true,
1403 };
1404
1405 static const struct idle_cpu idle_cpu_snx __initconst = {
1406 .state_table = snb_cstates,
1407 .disable_promotion_to_c1e = true,
1408 .use_acpi = true,
1409 };
1410
1411 static const struct idle_cpu idle_cpu_byt __initconst = {
1412 .state_table = byt_cstates,
1413 .disable_promotion_to_c1e = true,
1414 .byt_auto_demotion_disable_flag = true,
1415 };
1416
1417 static const struct idle_cpu idle_cpu_cht __initconst = {
1418 .state_table = cht_cstates,
1419 .disable_promotion_to_c1e = true,
1420 .byt_auto_demotion_disable_flag = true,
1421 };
1422
1423 static const struct idle_cpu idle_cpu_ivb __initconst = {
1424 .state_table = ivb_cstates,
1425 .disable_promotion_to_c1e = true,
1426 };
1427
1428 static const struct idle_cpu idle_cpu_ivt __initconst = {
1429 .state_table = ivt_cstates,
1430 .disable_promotion_to_c1e = true,
1431 .use_acpi = true,
1432 };
1433
1434 static const struct idle_cpu idle_cpu_hsw __initconst = {
1435 .state_table = hsw_cstates,
1436 .disable_promotion_to_c1e = true,
1437 };
1438
1439 static const struct idle_cpu idle_cpu_hsx __initconst = {
1440 .state_table = hsw_cstates,
1441 .disable_promotion_to_c1e = true,
1442 .use_acpi = true,
1443 };
1444
1445 static const struct idle_cpu idle_cpu_bdw __initconst = {
1446 .state_table = bdw_cstates,
1447 .disable_promotion_to_c1e = true,
1448 };
1449
1450 static const struct idle_cpu idle_cpu_bdx __initconst = {
1451 .state_table = bdw_cstates,
1452 .disable_promotion_to_c1e = true,
1453 .use_acpi = true,
1454 };
1455
1456 static const struct idle_cpu idle_cpu_skl __initconst = {
1457 .state_table = skl_cstates,
1458 .disable_promotion_to_c1e = true,
1459 };
1460
1461 static const struct idle_cpu idle_cpu_skx __initconst = {
1462 .state_table = skx_cstates,
1463 .disable_promotion_to_c1e = true,
1464 .use_acpi = true,
1465 };
1466
1467 static const struct idle_cpu idle_cpu_icx __initconst = {
1468 .state_table = icx_cstates,
1469 .disable_promotion_to_c1e = true,
1470 .use_acpi = true,
1471 };
1472
1473 static const struct idle_cpu idle_cpu_adl __initconst = {
1474 .state_table = adl_cstates,
1475 };
1476
1477 static const struct idle_cpu idle_cpu_adl_l __initconst = {
1478 .state_table = adl_l_cstates,
1479 };
1480
1481 static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1482 .state_table = mtl_l_cstates,
1483 };
1484
1485 static const struct idle_cpu idle_cpu_gmt __initconst = {
1486 .state_table = gmt_cstates,
1487 };
1488
1489 static const struct idle_cpu idle_cpu_spr __initconst = {
1490 .state_table = spr_cstates,
1491 .disable_promotion_to_c1e = true,
1492 .use_acpi = true,
1493 };
1494
1495 static const struct idle_cpu idle_cpu_gnr __initconst = {
1496 .state_table = gnr_cstates,
1497 .disable_promotion_to_c1e = true,
1498 .use_acpi = true,
1499 };
1500
1501 static const struct idle_cpu idle_cpu_avn __initconst = {
1502 .state_table = avn_cstates,
1503 .disable_promotion_to_c1e = true,
1504 .use_acpi = true,
1505 };
1506
1507 static const struct idle_cpu idle_cpu_knl __initconst = {
1508 .state_table = knl_cstates,
1509 .use_acpi = true,
1510 };
1511
1512 static const struct idle_cpu idle_cpu_bxt __initconst = {
1513 .state_table = bxt_cstates,
1514 .disable_promotion_to_c1e = true,
1515 };
1516
1517 static const struct idle_cpu idle_cpu_dnv __initconst = {
1518 .state_table = dnv_cstates,
1519 .disable_promotion_to_c1e = true,
1520 .use_acpi = true,
1521 };
1522
1523 static const struct idle_cpu idle_cpu_tmt __initconst = {
1524 .disable_promotion_to_c1e = true,
1525 };
1526
1527 static const struct idle_cpu idle_cpu_snr __initconst = {
1528 .state_table = snr_cstates,
1529 .disable_promotion_to_c1e = true,
1530 .use_acpi = true,
1531 };
1532
1533 static const struct idle_cpu idle_cpu_grr __initconst = {
1534 .state_table = grr_cstates,
1535 .disable_promotion_to_c1e = true,
1536 .use_acpi = true,
1537 };
1538
1539 static const struct idle_cpu idle_cpu_srf __initconst = {
1540 .state_table = srf_cstates,
1541 .disable_promotion_to_c1e = true,
1542 .use_acpi = true,
1543 };
1544
1545 static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1546 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx),
1547 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem),
1548 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem),
1549 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem),
1550 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx),
1551 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx),
1552 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom),
1553 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft),
1554 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx),
1555 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb),
1556 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx),
1557 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom),
1558 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt),
1559 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1560 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht),
1561 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb),
1562 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt),
1563 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw),
1564 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx),
1565 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw),
1566 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw),
1567 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn),
1568 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw),
1569 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw),
1570 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx),
1571 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx),
1572 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl),
1573 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl),
1574 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl),
1575 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl),
1576 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx),
1577 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx),
1578 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx),
1579 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl),
1580 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l),
1581 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l),
1582 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt),
1583 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr),
1584 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr),
1585 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr),
1586 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl),
1587 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl),
1588 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt),
1589 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1590 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv),
1591 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt),
1592 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt),
1593 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr),
1594 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr),
1595 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf),
1596 {}
1597 };
1598
1599 static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1600 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1601 {}
1602 };
1603
intel_idle_max_cstate_reached(int cstate)1604 static bool __init intel_idle_max_cstate_reached(int cstate)
1605 {
1606 if (cstate + 1 > max_cstate) {
1607 pr_info("max_cstate %d reached\n", max_cstate);
1608 return true;
1609 }
1610 return false;
1611 }
1612
intel_idle_state_needs_timer_stop(struct cpuidle_state * state)1613 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1614 {
1615 unsigned long eax = flg2MWAIT(state->flags);
1616
1617 if (boot_cpu_has(X86_FEATURE_ARAT))
1618 return false;
1619
1620 /*
1621 * Switch over to one-shot tick broadcast if the target C-state
1622 * is deeper than C1.
1623 */
1624 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1625 }
1626
1627 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1628 #include <acpi/processor.h>
1629
1630 static bool no_acpi __read_mostly;
1631 module_param(no_acpi, bool, 0444);
1632 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1633
1634 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1635 module_param_named(use_acpi, force_use_acpi, bool, 0444);
1636 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1637
1638 static struct acpi_processor_power acpi_state_table __initdata;
1639
1640 /**
1641 * intel_idle_cst_usable - Check if the _CST information can be used.
1642 *
1643 * Check if all of the C-states listed by _CST in the max_cstate range are
1644 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1645 */
intel_idle_cst_usable(void)1646 static bool __init intel_idle_cst_usable(void)
1647 {
1648 int cstate, limit;
1649
1650 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1651 acpi_state_table.count);
1652
1653 for (cstate = 1; cstate < limit; cstate++) {
1654 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1655
1656 if (cx->entry_method != ACPI_CSTATE_FFH)
1657 return false;
1658 }
1659
1660 return true;
1661 }
1662
intel_idle_acpi_cst_extract(void)1663 static bool __init intel_idle_acpi_cst_extract(void)
1664 {
1665 unsigned int cpu;
1666
1667 if (no_acpi) {
1668 pr_debug("Not allowed to use ACPI _CST\n");
1669 return false;
1670 }
1671
1672 for_each_possible_cpu(cpu) {
1673 struct acpi_processor *pr = per_cpu(processors, cpu);
1674
1675 if (!pr)
1676 continue;
1677
1678 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1679 continue;
1680
1681 acpi_state_table.count++;
1682
1683 if (!intel_idle_cst_usable())
1684 continue;
1685
1686 if (!acpi_processor_claim_cst_control())
1687 break;
1688
1689 return true;
1690 }
1691
1692 acpi_state_table.count = 0;
1693 pr_debug("ACPI _CST not found or not usable\n");
1694 return false;
1695 }
1696
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1697 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1698 {
1699 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1700
1701 /*
1702 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1703 * the interesting states are ACPI_CSTATE_FFH.
1704 */
1705 for (cstate = 1; cstate < limit; cstate++) {
1706 struct acpi_processor_cx *cx;
1707 struct cpuidle_state *state;
1708
1709 if (intel_idle_max_cstate_reached(cstate - 1))
1710 break;
1711
1712 cx = &acpi_state_table.states[cstate];
1713
1714 state = &drv->states[drv->state_count++];
1715
1716 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1717 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1718 state->exit_latency = cx->latency;
1719 /*
1720 * For C1-type C-states use the same number for both the exit
1721 * latency and target residency, because that is the case for
1722 * C1 in the majority of the static C-states tables above.
1723 * For the other types of C-states, however, set the target
1724 * residency to 3 times the exit latency which should lead to
1725 * a reasonable balance between energy-efficiency and
1726 * performance in the majority of interesting cases.
1727 */
1728 state->target_residency = cx->latency;
1729 if (cx->type > ACPI_STATE_C1)
1730 state->target_residency *= 3;
1731
1732 state->flags = MWAIT2flg(cx->address);
1733 if (cx->type > ACPI_STATE_C2)
1734 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1735
1736 if (disabled_states_mask & BIT(cstate))
1737 state->flags |= CPUIDLE_FLAG_OFF;
1738
1739 if (intel_idle_state_needs_timer_stop(state))
1740 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1741
1742 state->enter = intel_idle;
1743 state->enter_s2idle = intel_idle_s2idle;
1744 }
1745 }
1746
intel_idle_off_by_default(u32 mwait_hint)1747 static bool __init intel_idle_off_by_default(u32 mwait_hint)
1748 {
1749 int cstate, limit;
1750
1751 /*
1752 * If there are no _CST C-states, do not disable any C-states by
1753 * default.
1754 */
1755 if (!acpi_state_table.count)
1756 return false;
1757
1758 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1759 /*
1760 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1761 * the interesting states are ACPI_CSTATE_FFH.
1762 */
1763 for (cstate = 1; cstate < limit; cstate++) {
1764 if (acpi_state_table.states[cstate].address == mwait_hint)
1765 return false;
1766 }
1767 return true;
1768 }
1769 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1770 #define force_use_acpi (false)
1771
intel_idle_acpi_cst_extract(void)1772 static inline bool intel_idle_acpi_cst_extract(void) { return false; }
intel_idle_init_cstates_acpi(struct cpuidle_driver * drv)1773 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
intel_idle_off_by_default(u32 mwait_hint)1774 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1775 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1776
1777 /**
1778 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1779 *
1780 * Tune IVT multi-socket targets.
1781 * Assumption: num_sockets == (max_package_num + 1).
1782 */
ivt_idle_state_table_update(void)1783 static void __init ivt_idle_state_table_update(void)
1784 {
1785 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1786 int cpu, package_num, num_sockets = 1;
1787
1788 for_each_online_cpu(cpu) {
1789 package_num = topology_physical_package_id(cpu);
1790 if (package_num + 1 > num_sockets) {
1791 num_sockets = package_num + 1;
1792
1793 if (num_sockets > 4) {
1794 cpuidle_state_table = ivt_cstates_8s;
1795 return;
1796 }
1797 }
1798 }
1799
1800 if (num_sockets > 2)
1801 cpuidle_state_table = ivt_cstates_4s;
1802
1803 /* else, 1 and 2 socket systems use default ivt_cstates */
1804 }
1805
1806 /**
1807 * irtl_2_usec - IRTL to microseconds conversion.
1808 * @irtl: IRTL MSR value.
1809 *
1810 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1811 */
irtl_2_usec(unsigned long long irtl)1812 static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1813 {
1814 static const unsigned int irtl_ns_units[] __initconst = {
1815 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1816 };
1817 unsigned long long ns;
1818
1819 if (!irtl)
1820 return 0;
1821
1822 ns = irtl_ns_units[(irtl >> 10) & 0x7];
1823
1824 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1825 }
1826
1827 /**
1828 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1829 *
1830 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1831 * definitive maximum latency and use the same value for target_residency.
1832 */
bxt_idle_state_table_update(void)1833 static void __init bxt_idle_state_table_update(void)
1834 {
1835 unsigned long long msr;
1836 unsigned int usec;
1837
1838 rdmsrl(MSR_PKGC6_IRTL, msr);
1839 usec = irtl_2_usec(msr);
1840 if (usec) {
1841 bxt_cstates[2].exit_latency = usec;
1842 bxt_cstates[2].target_residency = usec;
1843 }
1844
1845 rdmsrl(MSR_PKGC7_IRTL, msr);
1846 usec = irtl_2_usec(msr);
1847 if (usec) {
1848 bxt_cstates[3].exit_latency = usec;
1849 bxt_cstates[3].target_residency = usec;
1850 }
1851
1852 rdmsrl(MSR_PKGC8_IRTL, msr);
1853 usec = irtl_2_usec(msr);
1854 if (usec) {
1855 bxt_cstates[4].exit_latency = usec;
1856 bxt_cstates[4].target_residency = usec;
1857 }
1858
1859 rdmsrl(MSR_PKGC9_IRTL, msr);
1860 usec = irtl_2_usec(msr);
1861 if (usec) {
1862 bxt_cstates[5].exit_latency = usec;
1863 bxt_cstates[5].target_residency = usec;
1864 }
1865
1866 rdmsrl(MSR_PKGC10_IRTL, msr);
1867 usec = irtl_2_usec(msr);
1868 if (usec) {
1869 bxt_cstates[6].exit_latency = usec;
1870 bxt_cstates[6].target_residency = usec;
1871 }
1872
1873 }
1874
1875 /**
1876 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1877 *
1878 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1879 */
sklh_idle_state_table_update(void)1880 static void __init sklh_idle_state_table_update(void)
1881 {
1882 unsigned long long msr;
1883 unsigned int eax, ebx, ecx, edx;
1884
1885
1886 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1887 if (max_cstate <= 7)
1888 return;
1889
1890 /* if PC10 not present in CPUID.MWAIT.EDX */
1891 if ((mwait_substates & (0xF << 28)) == 0)
1892 return;
1893
1894 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1895
1896 /* PC10 is not enabled in PKG C-state limit */
1897 if ((msr & 0xF) != 8)
1898 return;
1899
1900 ecx = 0;
1901 cpuid(7, &eax, &ebx, &ecx, &edx);
1902
1903 /* if SGX is present */
1904 if (ebx & (1 << 2)) {
1905
1906 rdmsrl(MSR_IA32_FEAT_CTL, msr);
1907
1908 /* if SGX is enabled */
1909 if (msr & (1 << 18))
1910 return;
1911 }
1912
1913 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
1914 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
1915 }
1916
1917 /**
1918 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1919 * idle states table.
1920 */
skx_idle_state_table_update(void)1921 static void __init skx_idle_state_table_update(void)
1922 {
1923 unsigned long long msr;
1924
1925 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1926
1927 /*
1928 * 000b: C0/C1 (no package C-state support)
1929 * 001b: C2
1930 * 010b: C6 (non-retention)
1931 * 011b: C6 (retention)
1932 * 111b: No Package C state limits.
1933 */
1934 if ((msr & 0x7) < 2) {
1935 /*
1936 * Uses the CC6 + PC0 latency and 3 times of
1937 * latency for target_residency if the PC6
1938 * is disabled in BIOS. This is consistent
1939 * with how intel_idle driver uses _CST
1940 * to set the target_residency.
1941 */
1942 skx_cstates[2].exit_latency = 92;
1943 skx_cstates[2].target_residency = 276;
1944 }
1945 }
1946
1947 /**
1948 * adl_idle_state_table_update - Adjust AlderLake idle states table.
1949 */
adl_idle_state_table_update(void)1950 static void __init adl_idle_state_table_update(void)
1951 {
1952 /* Check if user prefers C1 over C1E. */
1953 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
1954 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
1955 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
1956
1957 /* Disable C1E by clearing the "C1E promotion" bit. */
1958 c1e_promotion = C1E_PROMOTION_DISABLE;
1959 return;
1960 }
1961
1962 /* Make sure C1E is enabled by default */
1963 c1e_promotion = C1E_PROMOTION_ENABLE;
1964 }
1965
1966 /**
1967 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1968 */
spr_idle_state_table_update(void)1969 static void __init spr_idle_state_table_update(void)
1970 {
1971 unsigned long long msr;
1972
1973 /*
1974 * By default, the C6 state assumes the worst-case scenario of package
1975 * C6. However, if PC6 is disabled, we update the numbers to match
1976 * core C6.
1977 */
1978 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1979
1980 /* Limit value 2 and above allow for PC6. */
1981 if ((msr & 0x7) < 2) {
1982 spr_cstates[2].exit_latency = 190;
1983 spr_cstates[2].target_residency = 600;
1984 }
1985 }
1986
intel_idle_verify_cstate(unsigned int mwait_hint)1987 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1988 {
1989 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
1990 MWAIT_CSTATE_MASK;
1991 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1992 MWAIT_SUBSTATE_MASK;
1993
1994 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
1995 if (num_substates == 0)
1996 return false;
1997
1998 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1999 mark_tsc_unstable("TSC halts in idle states deeper than C2");
2000
2001 return true;
2002 }
2003
state_update_enter_method(struct cpuidle_state * state,int cstate)2004 static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2005 {
2006 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2007 /*
2008 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2009 * is not currently supported but this driver.
2010 */
2011 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2012 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2013 state->enter = intel_idle_xstate;
2014 return;
2015 }
2016
2017 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2018 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2019 /*
2020 * IBRS mitigation requires that C-states are entered
2021 * with interrupts disabled.
2022 */
2023 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2024 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2025 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2026 state->enter = intel_idle_ibrs;
2027 return;
2028 }
2029
2030 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2031 state->enter = intel_idle_irq;
2032 return;
2033 }
2034
2035 if (force_irq_on) {
2036 pr_info("forced intel_idle_irq for state %d\n", cstate);
2037 state->enter = intel_idle_irq;
2038 }
2039 }
2040
intel_idle_init_cstates_icpu(struct cpuidle_driver * drv)2041 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2042 {
2043 int cstate;
2044
2045 switch (boot_cpu_data.x86_vfm) {
2046 case INTEL_IVYBRIDGE_X:
2047 ivt_idle_state_table_update();
2048 break;
2049 case INTEL_ATOM_GOLDMONT:
2050 case INTEL_ATOM_GOLDMONT_PLUS:
2051 bxt_idle_state_table_update();
2052 break;
2053 case INTEL_SKYLAKE:
2054 sklh_idle_state_table_update();
2055 break;
2056 case INTEL_SKYLAKE_X:
2057 skx_idle_state_table_update();
2058 break;
2059 case INTEL_SAPPHIRERAPIDS_X:
2060 case INTEL_EMERALDRAPIDS_X:
2061 spr_idle_state_table_update();
2062 break;
2063 case INTEL_ALDERLAKE:
2064 case INTEL_ALDERLAKE_L:
2065 case INTEL_ATOM_GRACEMONT:
2066 adl_idle_state_table_update();
2067 break;
2068 }
2069
2070 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2071 struct cpuidle_state *state;
2072 unsigned int mwait_hint;
2073
2074 if (intel_idle_max_cstate_reached(cstate))
2075 break;
2076
2077 if (!cpuidle_state_table[cstate].enter &&
2078 !cpuidle_state_table[cstate].enter_s2idle)
2079 break;
2080
2081 /* If marked as unusable, skip this state. */
2082 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2083 pr_debug("state %s is disabled\n",
2084 cpuidle_state_table[cstate].name);
2085 continue;
2086 }
2087
2088 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2089 if (!intel_idle_verify_cstate(mwait_hint))
2090 continue;
2091
2092 /* Structure copy. */
2093 drv->states[drv->state_count] = cpuidle_state_table[cstate];
2094 state = &drv->states[drv->state_count];
2095
2096 state_update_enter_method(state, cstate);
2097
2098
2099 if ((disabled_states_mask & BIT(drv->state_count)) ||
2100 ((icpu->use_acpi || force_use_acpi) &&
2101 intel_idle_off_by_default(mwait_hint) &&
2102 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2103 state->flags |= CPUIDLE_FLAG_OFF;
2104
2105 if (intel_idle_state_needs_timer_stop(state))
2106 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2107
2108 drv->state_count++;
2109 }
2110
2111 if (icpu->byt_auto_demotion_disable_flag) {
2112 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2113 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2114 }
2115 }
2116
2117 /**
2118 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2119 * @drv: cpuidle driver structure to initialize.
2120 */
intel_idle_cpuidle_driver_init(struct cpuidle_driver * drv)2121 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2122 {
2123 cpuidle_poll_state_init(drv);
2124
2125 if (disabled_states_mask & BIT(0))
2126 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2127
2128 drv->state_count = 1;
2129
2130 if (icpu && icpu->state_table)
2131 intel_idle_init_cstates_icpu(drv);
2132 else
2133 intel_idle_init_cstates_acpi(drv);
2134 }
2135
auto_demotion_disable(void)2136 static void auto_demotion_disable(void)
2137 {
2138 unsigned long long msr_bits;
2139
2140 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2141 msr_bits &= ~auto_demotion_disable_flags;
2142 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2143 }
2144
c1e_promotion_enable(void)2145 static void c1e_promotion_enable(void)
2146 {
2147 unsigned long long msr_bits;
2148
2149 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2150 msr_bits |= 0x2;
2151 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2152 }
2153
c1e_promotion_disable(void)2154 static void c1e_promotion_disable(void)
2155 {
2156 unsigned long long msr_bits;
2157
2158 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2159 msr_bits &= ~0x2;
2160 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2161 }
2162
2163 /**
2164 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2165 * @cpu: CPU to initialize.
2166 *
2167 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2168 * with the processor model flags.
2169 */
intel_idle_cpu_init(unsigned int cpu)2170 static int intel_idle_cpu_init(unsigned int cpu)
2171 {
2172 struct cpuidle_device *dev;
2173
2174 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2175 dev->cpu = cpu;
2176
2177 if (cpuidle_register_device(dev)) {
2178 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2179 return -EIO;
2180 }
2181
2182 if (auto_demotion_disable_flags)
2183 auto_demotion_disable();
2184
2185 if (c1e_promotion == C1E_PROMOTION_ENABLE)
2186 c1e_promotion_enable();
2187 else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2188 c1e_promotion_disable();
2189
2190 return 0;
2191 }
2192
intel_idle_cpu_online(unsigned int cpu)2193 static int intel_idle_cpu_online(unsigned int cpu)
2194 {
2195 struct cpuidle_device *dev;
2196
2197 if (!boot_cpu_has(X86_FEATURE_ARAT))
2198 tick_broadcast_enable();
2199
2200 /*
2201 * Some systems can hotplug a cpu at runtime after
2202 * the kernel has booted, we have to initialize the
2203 * driver in this case
2204 */
2205 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2206 if (!dev->registered)
2207 return intel_idle_cpu_init(cpu);
2208
2209 return 0;
2210 }
2211
2212 /**
2213 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2214 */
intel_idle_cpuidle_devices_uninit(void)2215 static void __init intel_idle_cpuidle_devices_uninit(void)
2216 {
2217 int i;
2218
2219 for_each_online_cpu(i)
2220 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2221 }
2222
intel_idle_init(void)2223 static int __init intel_idle_init(void)
2224 {
2225 const struct x86_cpu_id *id;
2226 unsigned int eax, ebx, ecx;
2227 int retval;
2228
2229 /* Do not load intel_idle at all for now if idle= is passed */
2230 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2231 return -ENODEV;
2232
2233 if (max_cstate == 0) {
2234 pr_debug("disabled\n");
2235 return -EPERM;
2236 }
2237
2238 id = x86_match_cpu(intel_idle_ids);
2239 if (id) {
2240 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2241 pr_debug("Please enable MWAIT in BIOS SETUP\n");
2242 return -ENODEV;
2243 }
2244 } else {
2245 id = x86_match_cpu(intel_mwait_ids);
2246 if (!id)
2247 return -ENODEV;
2248 }
2249
2250 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2251 return -ENODEV;
2252
2253 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
2254
2255 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2256 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2257 !mwait_substates)
2258 return -ENODEV;
2259
2260 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2261
2262 icpu = (const struct idle_cpu *)id->driver_data;
2263 if (icpu) {
2264 if (icpu->state_table)
2265 cpuidle_state_table = icpu->state_table;
2266 else if (!intel_idle_acpi_cst_extract())
2267 return -ENODEV;
2268
2269 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2270 if (icpu->disable_promotion_to_c1e)
2271 c1e_promotion = C1E_PROMOTION_DISABLE;
2272 if (icpu->use_acpi || force_use_acpi)
2273 intel_idle_acpi_cst_extract();
2274 } else if (!intel_idle_acpi_cst_extract()) {
2275 return -ENODEV;
2276 }
2277
2278 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2279 boot_cpu_data.x86_model);
2280
2281 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2282 if (!intel_idle_cpuidle_devices)
2283 return -ENOMEM;
2284
2285 intel_idle_cpuidle_driver_init(&intel_idle_driver);
2286
2287 retval = cpuidle_register_driver(&intel_idle_driver);
2288 if (retval) {
2289 struct cpuidle_driver *drv = cpuidle_get_driver();
2290 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2291 drv ? drv->name : "none");
2292 goto init_driver_fail;
2293 }
2294
2295 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2296 intel_idle_cpu_online, NULL);
2297 if (retval < 0)
2298 goto hp_setup_fail;
2299
2300 pr_debug("Local APIC timer is reliable in %s\n",
2301 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2302
2303 return 0;
2304
2305 hp_setup_fail:
2306 intel_idle_cpuidle_devices_uninit();
2307 cpuidle_unregister_driver(&intel_idle_driver);
2308 init_driver_fail:
2309 free_percpu(intel_idle_cpuidle_devices);
2310 return retval;
2311
2312 }
2313 device_initcall(intel_idle_init);
2314
2315 /*
2316 * We are not really modular, but we used to support that. Meaning we also
2317 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2318 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2319 * is the easiest way (currently) to continue doing that.
2320 */
2321 module_param(max_cstate, int, 0444);
2322 /*
2323 * The positions of the bits that are set in this number are the indices of the
2324 * idle states to be disabled by default (as reflected by the names of the
2325 * corresponding idle state directories in sysfs, "state0", "state1" ...
2326 * "state<i>" ..., where <i> is the index of the given state).
2327 */
2328 module_param_named(states_off, disabled_states_mask, uint, 0444);
2329 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2330 /*
2331 * Some platforms come with mutually exclusive C-states, so that if one is
2332 * enabled, the other C-states must not be used. Example: C1 and C1E on
2333 * Sapphire Rapids platform. This parameter allows for selecting the
2334 * preferred C-states among the groups of mutually exclusive C-states - the
2335 * selected C-states will be registered, the other C-states from the mutually
2336 * exclusive group won't be registered. If the platform has no mutually
2337 * exclusive C-states, this parameter has no effect.
2338 */
2339 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2340 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2341 /*
2342 * Debugging option that forces the driver to enter all C-states with
2343 * interrupts enabled. Does not apply to C-states with
2344 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2345 */
2346 module_param(force_irq_on, bool, 0444);
2347 /*
2348 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2349 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2350 */
2351 module_param(ibrs_off, bool, 0444);
2352 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
2353