1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/sched/smt.h> 51 #include <linux/notifier.h> 52 #include <linux/cpu.h> 53 #include <linux/moduleparam.h> 54 #include <asm/cpuid.h> 55 #include <asm/cpu_device_id.h> 56 #include <asm/intel-family.h> 57 #include <asm/mwait.h> 58 #include <asm/spec-ctrl.h> 59 #include <asm/tsc.h> 60 #include <asm/fpu/api.h> 61 #include <asm/smp.h> 62 63 #define INTEL_IDLE_VERSION "0.5.1" 64 65 static struct cpuidle_driver intel_idle_driver = { 66 .name = "intel_idle", 67 .owner = THIS_MODULE, 68 }; 69 /* intel_idle.max_cstate=0 disables driver */ 70 static int max_cstate = CPUIDLE_STATE_MAX - 1; 71 static unsigned int disabled_states_mask __read_mostly; 72 static unsigned int preferred_states_mask __read_mostly; 73 static bool force_irq_on __read_mostly; 74 static bool ibrs_off __read_mostly; 75 76 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 77 78 static unsigned long auto_demotion_disable_flags; 79 80 static enum { 81 C1E_PROMOTION_PRESERVE, 82 C1E_PROMOTION_ENABLE, 83 C1E_PROMOTION_DISABLE 84 } c1e_promotion = C1E_PROMOTION_PRESERVE; 85 86 struct idle_cpu { 87 struct cpuidle_state *state_table; 88 89 /* 90 * Hardware C-state auto-demotion may not always be optimal. 91 * Indicate which enable bits to clear here. 92 */ 93 unsigned long auto_demotion_disable_flags; 94 bool byt_auto_demotion_disable_flag; 95 bool disable_promotion_to_c1e; 96 bool use_acpi; 97 }; 98 99 static const struct idle_cpu *icpu __initdata; 100 static struct cpuidle_state *cpuidle_state_table __initdata; 101 102 static unsigned int mwait_substates __initdata; 103 104 /* 105 * Enable interrupts before entering the C-state. On some platforms and for 106 * some C-states, this may measurably decrease interrupt latency. 107 */ 108 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 109 110 /* 111 * Enable this state by default even if the ACPI _CST does not list it. 112 */ 113 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 114 115 /* 116 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 117 * above. 118 */ 119 #define CPUIDLE_FLAG_IBRS BIT(16) 120 121 /* 122 * Initialize large xstate for the C6-state entrance. 123 */ 124 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 125 126 /* 127 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 128 * custom tables. 129 */ 130 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 131 132 /* 133 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 134 * the C-state (top nibble) and sub-state (bottom nibble) 135 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 136 * 137 * We store the hint at the top of our "flags" for each state. 138 */ 139 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 140 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 141 142 static __always_inline int __intel_idle(struct cpuidle_device *dev, 143 struct cpuidle_driver *drv, 144 int index, bool irqoff) 145 { 146 struct cpuidle_state *state = &drv->states[index]; 147 unsigned long eax = flg2MWAIT(state->flags); 148 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 149 150 mwait_idle_with_hints(eax, ecx); 151 152 return index; 153 } 154 155 /** 156 * intel_idle - Ask the processor to enter the given idle state. 157 * @dev: cpuidle device of the target CPU. 158 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 159 * @index: Target idle state index. 160 * 161 * Use the MWAIT instruction to notify the processor that the CPU represented by 162 * @dev is idle and it can try to enter the idle state corresponding to @index. 163 * 164 * If the local APIC timer is not known to be reliable in the target idle state, 165 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 166 * 167 * Must be called under local_irq_disable(). 168 */ 169 static __cpuidle int intel_idle(struct cpuidle_device *dev, 170 struct cpuidle_driver *drv, int index) 171 { 172 return __intel_idle(dev, drv, index, true); 173 } 174 175 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 176 struct cpuidle_driver *drv, int index) 177 { 178 return __intel_idle(dev, drv, index, false); 179 } 180 181 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 182 struct cpuidle_driver *drv, int index) 183 { 184 bool smt_active = sched_smt_active(); 185 u64 spec_ctrl = spec_ctrl_current(); 186 int ret; 187 188 if (smt_active) 189 __update_spec_ctrl(0); 190 191 ret = __intel_idle(dev, drv, index, true); 192 193 if (smt_active) 194 __update_spec_ctrl(spec_ctrl); 195 196 return ret; 197 } 198 199 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 200 struct cpuidle_driver *drv, int index) 201 { 202 fpu_idle_fpregs(); 203 return __intel_idle(dev, drv, index, true); 204 } 205 206 /** 207 * intel_idle_s2idle - Ask the processor to enter the given idle state. 208 * @dev: cpuidle device of the target CPU. 209 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 210 * @index: Target idle state index. 211 * 212 * Use the MWAIT instruction to notify the processor that the CPU represented by 213 * @dev is idle and it can try to enter the idle state corresponding to @index. 214 * 215 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 216 * scheduler tick and suspended scheduler clock on the target CPU. 217 */ 218 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 219 struct cpuidle_driver *drv, int index) 220 { 221 unsigned long ecx = 1; /* break on interrupt flag */ 222 struct cpuidle_state *state = &drv->states[index]; 223 unsigned long eax = flg2MWAIT(state->flags); 224 225 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 226 fpu_idle_fpregs(); 227 228 mwait_idle_with_hints(eax, ecx); 229 230 return 0; 231 } 232 233 static void intel_idle_enter_dead(struct cpuidle_device *dev, int index) 234 { 235 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 236 struct cpuidle_state *state = &drv->states[index]; 237 unsigned long eax = flg2MWAIT(state->flags); 238 239 mwait_play_dead(eax); 240 } 241 242 /* 243 * States are indexed by the cstate number, 244 * which is also the index into the MWAIT hint array. 245 * Thus C0 is a dummy. 246 */ 247 static struct cpuidle_state nehalem_cstates[] __initdata = { 248 { 249 .name = "C1", 250 .desc = "MWAIT 0x00", 251 .flags = MWAIT2flg(0x00), 252 .exit_latency = 3, 253 .target_residency = 6, 254 .enter = &intel_idle, 255 .enter_s2idle = intel_idle_s2idle, }, 256 { 257 .name = "C1E", 258 .desc = "MWAIT 0x01", 259 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 260 .exit_latency = 10, 261 .target_residency = 20, 262 .enter = &intel_idle, 263 .enter_s2idle = intel_idle_s2idle, }, 264 { 265 .name = "C3", 266 .desc = "MWAIT 0x10", 267 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 268 .exit_latency = 20, 269 .target_residency = 80, 270 .enter = &intel_idle, 271 .enter_s2idle = intel_idle_s2idle, }, 272 { 273 .name = "C6", 274 .desc = "MWAIT 0x20", 275 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 276 .exit_latency = 200, 277 .target_residency = 800, 278 .enter = &intel_idle, 279 .enter_s2idle = intel_idle_s2idle, }, 280 { 281 .enter = NULL } 282 }; 283 284 static struct cpuidle_state snb_cstates[] __initdata = { 285 { 286 .name = "C1", 287 .desc = "MWAIT 0x00", 288 .flags = MWAIT2flg(0x00), 289 .exit_latency = 2, 290 .target_residency = 2, 291 .enter = &intel_idle, 292 .enter_s2idle = intel_idle_s2idle, }, 293 { 294 .name = "C1E", 295 .desc = "MWAIT 0x01", 296 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 297 .exit_latency = 10, 298 .target_residency = 20, 299 .enter = &intel_idle, 300 .enter_s2idle = intel_idle_s2idle, }, 301 { 302 .name = "C3", 303 .desc = "MWAIT 0x10", 304 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 305 .exit_latency = 80, 306 .target_residency = 211, 307 .enter = &intel_idle, 308 .enter_s2idle = intel_idle_s2idle, }, 309 { 310 .name = "C6", 311 .desc = "MWAIT 0x20", 312 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 313 .exit_latency = 104, 314 .target_residency = 345, 315 .enter = &intel_idle, 316 .enter_s2idle = intel_idle_s2idle, }, 317 { 318 .name = "C7", 319 .desc = "MWAIT 0x30", 320 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 321 .exit_latency = 109, 322 .target_residency = 345, 323 .enter = &intel_idle, 324 .enter_s2idle = intel_idle_s2idle, }, 325 { 326 .enter = NULL } 327 }; 328 329 static struct cpuidle_state byt_cstates[] __initdata = { 330 { 331 .name = "C1", 332 .desc = "MWAIT 0x00", 333 .flags = MWAIT2flg(0x00), 334 .exit_latency = 1, 335 .target_residency = 1, 336 .enter = &intel_idle, 337 .enter_s2idle = intel_idle_s2idle, }, 338 { 339 .name = "C6N", 340 .desc = "MWAIT 0x58", 341 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 342 .exit_latency = 300, 343 .target_residency = 275, 344 .enter = &intel_idle, 345 .enter_s2idle = intel_idle_s2idle, }, 346 { 347 .name = "C6S", 348 .desc = "MWAIT 0x52", 349 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 350 .exit_latency = 500, 351 .target_residency = 560, 352 .enter = &intel_idle, 353 .enter_s2idle = intel_idle_s2idle, }, 354 { 355 .name = "C7", 356 .desc = "MWAIT 0x60", 357 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 358 .exit_latency = 1200, 359 .target_residency = 4000, 360 .enter = &intel_idle, 361 .enter_s2idle = intel_idle_s2idle, }, 362 { 363 .name = "C7S", 364 .desc = "MWAIT 0x64", 365 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 366 .exit_latency = 10000, 367 .target_residency = 20000, 368 .enter = &intel_idle, 369 .enter_s2idle = intel_idle_s2idle, }, 370 { 371 .enter = NULL } 372 }; 373 374 static struct cpuidle_state cht_cstates[] __initdata = { 375 { 376 .name = "C1", 377 .desc = "MWAIT 0x00", 378 .flags = MWAIT2flg(0x00), 379 .exit_latency = 1, 380 .target_residency = 1, 381 .enter = &intel_idle, 382 .enter_s2idle = intel_idle_s2idle, }, 383 { 384 .name = "C6N", 385 .desc = "MWAIT 0x58", 386 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 387 .exit_latency = 80, 388 .target_residency = 275, 389 .enter = &intel_idle, 390 .enter_s2idle = intel_idle_s2idle, }, 391 { 392 .name = "C6S", 393 .desc = "MWAIT 0x52", 394 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 395 .exit_latency = 200, 396 .target_residency = 560, 397 .enter = &intel_idle, 398 .enter_s2idle = intel_idle_s2idle, }, 399 { 400 .name = "C7", 401 .desc = "MWAIT 0x60", 402 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 403 .exit_latency = 1200, 404 .target_residency = 4000, 405 .enter = &intel_idle, 406 .enter_s2idle = intel_idle_s2idle, }, 407 { 408 .name = "C7S", 409 .desc = "MWAIT 0x64", 410 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 411 .exit_latency = 10000, 412 .target_residency = 20000, 413 .enter = &intel_idle, 414 .enter_s2idle = intel_idle_s2idle, }, 415 { 416 .enter = NULL } 417 }; 418 419 static struct cpuidle_state ivb_cstates[] __initdata = { 420 { 421 .name = "C1", 422 .desc = "MWAIT 0x00", 423 .flags = MWAIT2flg(0x00), 424 .exit_latency = 1, 425 .target_residency = 1, 426 .enter = &intel_idle, 427 .enter_s2idle = intel_idle_s2idle, }, 428 { 429 .name = "C1E", 430 .desc = "MWAIT 0x01", 431 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 432 .exit_latency = 10, 433 .target_residency = 20, 434 .enter = &intel_idle, 435 .enter_s2idle = intel_idle_s2idle, }, 436 { 437 .name = "C3", 438 .desc = "MWAIT 0x10", 439 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 440 .exit_latency = 59, 441 .target_residency = 156, 442 .enter = &intel_idle, 443 .enter_s2idle = intel_idle_s2idle, }, 444 { 445 .name = "C6", 446 .desc = "MWAIT 0x20", 447 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 448 .exit_latency = 80, 449 .target_residency = 300, 450 .enter = &intel_idle, 451 .enter_s2idle = intel_idle_s2idle, }, 452 { 453 .name = "C7", 454 .desc = "MWAIT 0x30", 455 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 456 .exit_latency = 87, 457 .target_residency = 300, 458 .enter = &intel_idle, 459 .enter_s2idle = intel_idle_s2idle, }, 460 { 461 .enter = NULL } 462 }; 463 464 static struct cpuidle_state ivt_cstates[] __initdata = { 465 { 466 .name = "C1", 467 .desc = "MWAIT 0x00", 468 .flags = MWAIT2flg(0x00), 469 .exit_latency = 1, 470 .target_residency = 1, 471 .enter = &intel_idle, 472 .enter_s2idle = intel_idle_s2idle, }, 473 { 474 .name = "C1E", 475 .desc = "MWAIT 0x01", 476 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 477 .exit_latency = 10, 478 .target_residency = 80, 479 .enter = &intel_idle, 480 .enter_s2idle = intel_idle_s2idle, }, 481 { 482 .name = "C3", 483 .desc = "MWAIT 0x10", 484 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 485 .exit_latency = 59, 486 .target_residency = 156, 487 .enter = &intel_idle, 488 .enter_s2idle = intel_idle_s2idle, }, 489 { 490 .name = "C6", 491 .desc = "MWAIT 0x20", 492 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 493 .exit_latency = 82, 494 .target_residency = 300, 495 .enter = &intel_idle, 496 .enter_s2idle = intel_idle_s2idle, }, 497 { 498 .enter = NULL } 499 }; 500 501 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 502 { 503 .name = "C1", 504 .desc = "MWAIT 0x00", 505 .flags = MWAIT2flg(0x00), 506 .exit_latency = 1, 507 .target_residency = 1, 508 .enter = &intel_idle, 509 .enter_s2idle = intel_idle_s2idle, }, 510 { 511 .name = "C1E", 512 .desc = "MWAIT 0x01", 513 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 514 .exit_latency = 10, 515 .target_residency = 250, 516 .enter = &intel_idle, 517 .enter_s2idle = intel_idle_s2idle, }, 518 { 519 .name = "C3", 520 .desc = "MWAIT 0x10", 521 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 522 .exit_latency = 59, 523 .target_residency = 300, 524 .enter = &intel_idle, 525 .enter_s2idle = intel_idle_s2idle, }, 526 { 527 .name = "C6", 528 .desc = "MWAIT 0x20", 529 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 530 .exit_latency = 84, 531 .target_residency = 400, 532 .enter = &intel_idle, 533 .enter_s2idle = intel_idle_s2idle, }, 534 { 535 .enter = NULL } 536 }; 537 538 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 539 { 540 .name = "C1", 541 .desc = "MWAIT 0x00", 542 .flags = MWAIT2flg(0x00), 543 .exit_latency = 1, 544 .target_residency = 1, 545 .enter = &intel_idle, 546 .enter_s2idle = intel_idle_s2idle, }, 547 { 548 .name = "C1E", 549 .desc = "MWAIT 0x01", 550 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 551 .exit_latency = 10, 552 .target_residency = 500, 553 .enter = &intel_idle, 554 .enter_s2idle = intel_idle_s2idle, }, 555 { 556 .name = "C3", 557 .desc = "MWAIT 0x10", 558 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 559 .exit_latency = 59, 560 .target_residency = 600, 561 .enter = &intel_idle, 562 .enter_s2idle = intel_idle_s2idle, }, 563 { 564 .name = "C6", 565 .desc = "MWAIT 0x20", 566 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 567 .exit_latency = 88, 568 .target_residency = 700, 569 .enter = &intel_idle, 570 .enter_s2idle = intel_idle_s2idle, }, 571 { 572 .enter = NULL } 573 }; 574 575 static struct cpuidle_state hsw_cstates[] __initdata = { 576 { 577 .name = "C1", 578 .desc = "MWAIT 0x00", 579 .flags = MWAIT2flg(0x00), 580 .exit_latency = 2, 581 .target_residency = 2, 582 .enter = &intel_idle, 583 .enter_s2idle = intel_idle_s2idle, }, 584 { 585 .name = "C1E", 586 .desc = "MWAIT 0x01", 587 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 588 .exit_latency = 10, 589 .target_residency = 20, 590 .enter = &intel_idle, 591 .enter_s2idle = intel_idle_s2idle, }, 592 { 593 .name = "C3", 594 .desc = "MWAIT 0x10", 595 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 596 .exit_latency = 33, 597 .target_residency = 100, 598 .enter = &intel_idle, 599 .enter_s2idle = intel_idle_s2idle, }, 600 { 601 .name = "C6", 602 .desc = "MWAIT 0x20", 603 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 604 .exit_latency = 133, 605 .target_residency = 400, 606 .enter = &intel_idle, 607 .enter_s2idle = intel_idle_s2idle, }, 608 { 609 .name = "C7s", 610 .desc = "MWAIT 0x32", 611 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 612 .exit_latency = 166, 613 .target_residency = 500, 614 .enter = &intel_idle, 615 .enter_s2idle = intel_idle_s2idle, }, 616 { 617 .name = "C8", 618 .desc = "MWAIT 0x40", 619 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 620 .exit_latency = 300, 621 .target_residency = 900, 622 .enter = &intel_idle, 623 .enter_s2idle = intel_idle_s2idle, }, 624 { 625 .name = "C9", 626 .desc = "MWAIT 0x50", 627 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 628 .exit_latency = 600, 629 .target_residency = 1800, 630 .enter = &intel_idle, 631 .enter_s2idle = intel_idle_s2idle, }, 632 { 633 .name = "C10", 634 .desc = "MWAIT 0x60", 635 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 636 .exit_latency = 2600, 637 .target_residency = 7700, 638 .enter = &intel_idle, 639 .enter_s2idle = intel_idle_s2idle, }, 640 { 641 .enter = NULL } 642 }; 643 static struct cpuidle_state bdw_cstates[] __initdata = { 644 { 645 .name = "C1", 646 .desc = "MWAIT 0x00", 647 .flags = MWAIT2flg(0x00), 648 .exit_latency = 2, 649 .target_residency = 2, 650 .enter = &intel_idle, 651 .enter_s2idle = intel_idle_s2idle, }, 652 { 653 .name = "C1E", 654 .desc = "MWAIT 0x01", 655 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 656 .exit_latency = 10, 657 .target_residency = 20, 658 .enter = &intel_idle, 659 .enter_s2idle = intel_idle_s2idle, }, 660 { 661 .name = "C3", 662 .desc = "MWAIT 0x10", 663 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 664 .exit_latency = 40, 665 .target_residency = 100, 666 .enter = &intel_idle, 667 .enter_s2idle = intel_idle_s2idle, }, 668 { 669 .name = "C6", 670 .desc = "MWAIT 0x20", 671 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 672 .exit_latency = 133, 673 .target_residency = 400, 674 .enter = &intel_idle, 675 .enter_s2idle = intel_idle_s2idle, }, 676 { 677 .name = "C7s", 678 .desc = "MWAIT 0x32", 679 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 680 .exit_latency = 166, 681 .target_residency = 500, 682 .enter = &intel_idle, 683 .enter_s2idle = intel_idle_s2idle, }, 684 { 685 .name = "C8", 686 .desc = "MWAIT 0x40", 687 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 688 .exit_latency = 300, 689 .target_residency = 900, 690 .enter = &intel_idle, 691 .enter_s2idle = intel_idle_s2idle, }, 692 { 693 .name = "C9", 694 .desc = "MWAIT 0x50", 695 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 696 .exit_latency = 600, 697 .target_residency = 1800, 698 .enter = &intel_idle, 699 .enter_s2idle = intel_idle_s2idle, }, 700 { 701 .name = "C10", 702 .desc = "MWAIT 0x60", 703 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 704 .exit_latency = 2600, 705 .target_residency = 7700, 706 .enter = &intel_idle, 707 .enter_s2idle = intel_idle_s2idle, }, 708 { 709 .enter = NULL } 710 }; 711 712 static struct cpuidle_state skl_cstates[] __initdata = { 713 { 714 .name = "C1", 715 .desc = "MWAIT 0x00", 716 .flags = MWAIT2flg(0x00), 717 .exit_latency = 2, 718 .target_residency = 2, 719 .enter = &intel_idle, 720 .enter_s2idle = intel_idle_s2idle, }, 721 { 722 .name = "C1E", 723 .desc = "MWAIT 0x01", 724 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 725 .exit_latency = 10, 726 .target_residency = 20, 727 .enter = &intel_idle, 728 .enter_s2idle = intel_idle_s2idle, }, 729 { 730 .name = "C3", 731 .desc = "MWAIT 0x10", 732 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 733 .exit_latency = 70, 734 .target_residency = 100, 735 .enter = &intel_idle, 736 .enter_s2idle = intel_idle_s2idle, }, 737 { 738 .name = "C6", 739 .desc = "MWAIT 0x20", 740 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 741 .exit_latency = 85, 742 .target_residency = 200, 743 .enter = &intel_idle, 744 .enter_s2idle = intel_idle_s2idle, }, 745 { 746 .name = "C7s", 747 .desc = "MWAIT 0x33", 748 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 749 .exit_latency = 124, 750 .target_residency = 800, 751 .enter = &intel_idle, 752 .enter_s2idle = intel_idle_s2idle, }, 753 { 754 .name = "C8", 755 .desc = "MWAIT 0x40", 756 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 757 .exit_latency = 200, 758 .target_residency = 800, 759 .enter = &intel_idle, 760 .enter_s2idle = intel_idle_s2idle, }, 761 { 762 .name = "C9", 763 .desc = "MWAIT 0x50", 764 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 765 .exit_latency = 480, 766 .target_residency = 5000, 767 .enter = &intel_idle, 768 .enter_s2idle = intel_idle_s2idle, }, 769 { 770 .name = "C10", 771 .desc = "MWAIT 0x60", 772 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 773 .exit_latency = 890, 774 .target_residency = 5000, 775 .enter = &intel_idle, 776 .enter_s2idle = intel_idle_s2idle, }, 777 { 778 .enter = NULL } 779 }; 780 781 static struct cpuidle_state skx_cstates[] __initdata = { 782 { 783 .name = "C1", 784 .desc = "MWAIT 0x00", 785 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 786 .exit_latency = 2, 787 .target_residency = 2, 788 .enter = &intel_idle, 789 .enter_s2idle = intel_idle_s2idle, }, 790 { 791 .name = "C1E", 792 .desc = "MWAIT 0x01", 793 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 794 .exit_latency = 10, 795 .target_residency = 20, 796 .enter = &intel_idle, 797 .enter_s2idle = intel_idle_s2idle, }, 798 { 799 .name = "C6", 800 .desc = "MWAIT 0x20", 801 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 802 .exit_latency = 133, 803 .target_residency = 600, 804 .enter = &intel_idle, 805 .enter_s2idle = intel_idle_s2idle, }, 806 { 807 .enter = NULL } 808 }; 809 810 static struct cpuidle_state icx_cstates[] __initdata = { 811 { 812 .name = "C1", 813 .desc = "MWAIT 0x00", 814 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 815 .exit_latency = 1, 816 .target_residency = 1, 817 .enter = &intel_idle, 818 .enter_s2idle = intel_idle_s2idle, }, 819 { 820 .name = "C1E", 821 .desc = "MWAIT 0x01", 822 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 823 .exit_latency = 4, 824 .target_residency = 4, 825 .enter = &intel_idle, 826 .enter_s2idle = intel_idle_s2idle, }, 827 { 828 .name = "C6", 829 .desc = "MWAIT 0x20", 830 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 831 .exit_latency = 170, 832 .target_residency = 600, 833 .enter = &intel_idle, 834 .enter_s2idle = intel_idle_s2idle, }, 835 { 836 .enter = NULL } 837 }; 838 839 /* 840 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 841 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 842 * But in this case there is effectively no C1, because C1 requests are 843 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 844 * and C1E requests end up with C1, so there is effectively no C1E. 845 * 846 * By default we enable C1E and disable C1 by marking it with 847 * 'CPUIDLE_FLAG_UNUSABLE'. 848 */ 849 static struct cpuidle_state adl_cstates[] __initdata = { 850 { 851 .name = "C1", 852 .desc = "MWAIT 0x00", 853 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 854 .exit_latency = 1, 855 .target_residency = 1, 856 .enter = &intel_idle, 857 .enter_s2idle = intel_idle_s2idle, }, 858 { 859 .name = "C1E", 860 .desc = "MWAIT 0x01", 861 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 862 .exit_latency = 2, 863 .target_residency = 4, 864 .enter = &intel_idle, 865 .enter_s2idle = intel_idle_s2idle, }, 866 { 867 .name = "C6", 868 .desc = "MWAIT 0x20", 869 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 870 .exit_latency = 220, 871 .target_residency = 600, 872 .enter = &intel_idle, 873 .enter_s2idle = intel_idle_s2idle, }, 874 { 875 .name = "C8", 876 .desc = "MWAIT 0x40", 877 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 878 .exit_latency = 280, 879 .target_residency = 800, 880 .enter = &intel_idle, 881 .enter_s2idle = intel_idle_s2idle, }, 882 { 883 .name = "C10", 884 .desc = "MWAIT 0x60", 885 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 886 .exit_latency = 680, 887 .target_residency = 2000, 888 .enter = &intel_idle, 889 .enter_s2idle = intel_idle_s2idle, }, 890 { 891 .enter = NULL } 892 }; 893 894 static struct cpuidle_state adl_l_cstates[] __initdata = { 895 { 896 .name = "C1", 897 .desc = "MWAIT 0x00", 898 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 899 .exit_latency = 1, 900 .target_residency = 1, 901 .enter = &intel_idle, 902 .enter_s2idle = intel_idle_s2idle, }, 903 { 904 .name = "C1E", 905 .desc = "MWAIT 0x01", 906 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 907 .exit_latency = 2, 908 .target_residency = 4, 909 .enter = &intel_idle, 910 .enter_s2idle = intel_idle_s2idle, }, 911 { 912 .name = "C6", 913 .desc = "MWAIT 0x20", 914 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 915 .exit_latency = 170, 916 .target_residency = 500, 917 .enter = &intel_idle, 918 .enter_s2idle = intel_idle_s2idle, }, 919 { 920 .name = "C8", 921 .desc = "MWAIT 0x40", 922 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 923 .exit_latency = 200, 924 .target_residency = 600, 925 .enter = &intel_idle, 926 .enter_s2idle = intel_idle_s2idle, }, 927 { 928 .name = "C10", 929 .desc = "MWAIT 0x60", 930 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 931 .exit_latency = 230, 932 .target_residency = 700, 933 .enter = &intel_idle, 934 .enter_s2idle = intel_idle_s2idle, }, 935 { 936 .enter = NULL } 937 }; 938 939 static struct cpuidle_state mtl_l_cstates[] __initdata = { 940 { 941 .name = "C1E", 942 .desc = "MWAIT 0x01", 943 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 944 .exit_latency = 1, 945 .target_residency = 1, 946 .enter = &intel_idle, 947 .enter_s2idle = intel_idle_s2idle, }, 948 { 949 .name = "C6", 950 .desc = "MWAIT 0x20", 951 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 952 .exit_latency = 140, 953 .target_residency = 420, 954 .enter = &intel_idle, 955 .enter_s2idle = intel_idle_s2idle, }, 956 { 957 .name = "C10", 958 .desc = "MWAIT 0x60", 959 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 960 .exit_latency = 310, 961 .target_residency = 930, 962 .enter = &intel_idle, 963 .enter_s2idle = intel_idle_s2idle, }, 964 { 965 .enter = NULL } 966 }; 967 968 static struct cpuidle_state gmt_cstates[] __initdata = { 969 { 970 .name = "C1", 971 .desc = "MWAIT 0x00", 972 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 973 .exit_latency = 1, 974 .target_residency = 1, 975 .enter = &intel_idle, 976 .enter_s2idle = intel_idle_s2idle, }, 977 { 978 .name = "C1E", 979 .desc = "MWAIT 0x01", 980 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 981 .exit_latency = 2, 982 .target_residency = 4, 983 .enter = &intel_idle, 984 .enter_s2idle = intel_idle_s2idle, }, 985 { 986 .name = "C6", 987 .desc = "MWAIT 0x20", 988 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 989 .exit_latency = 195, 990 .target_residency = 585, 991 .enter = &intel_idle, 992 .enter_s2idle = intel_idle_s2idle, }, 993 { 994 .name = "C8", 995 .desc = "MWAIT 0x40", 996 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 997 .exit_latency = 260, 998 .target_residency = 1040, 999 .enter = &intel_idle, 1000 .enter_s2idle = intel_idle_s2idle, }, 1001 { 1002 .name = "C10", 1003 .desc = "MWAIT 0x60", 1004 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1005 .exit_latency = 660, 1006 .target_residency = 1980, 1007 .enter = &intel_idle, 1008 .enter_s2idle = intel_idle_s2idle, }, 1009 { 1010 .enter = NULL } 1011 }; 1012 1013 static struct cpuidle_state spr_cstates[] __initdata = { 1014 { 1015 .name = "C1", 1016 .desc = "MWAIT 0x00", 1017 .flags = MWAIT2flg(0x00), 1018 .exit_latency = 1, 1019 .target_residency = 1, 1020 .enter = &intel_idle, 1021 .enter_s2idle = intel_idle_s2idle, }, 1022 { 1023 .name = "C1E", 1024 .desc = "MWAIT 0x01", 1025 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1026 .exit_latency = 2, 1027 .target_residency = 4, 1028 .enter = &intel_idle, 1029 .enter_s2idle = intel_idle_s2idle, }, 1030 { 1031 .name = "C6", 1032 .desc = "MWAIT 0x20", 1033 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1034 CPUIDLE_FLAG_INIT_XSTATE, 1035 .exit_latency = 290, 1036 .target_residency = 800, 1037 .enter = &intel_idle, 1038 .enter_s2idle = intel_idle_s2idle, }, 1039 { 1040 .enter = NULL } 1041 }; 1042 1043 static struct cpuidle_state gnr_cstates[] __initdata = { 1044 { 1045 .name = "C1", 1046 .desc = "MWAIT 0x00", 1047 .flags = MWAIT2flg(0x00), 1048 .exit_latency = 1, 1049 .target_residency = 1, 1050 .enter = &intel_idle, 1051 .enter_s2idle = intel_idle_s2idle, }, 1052 { 1053 .name = "C1E", 1054 .desc = "MWAIT 0x01", 1055 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1056 .exit_latency = 4, 1057 .target_residency = 4, 1058 .enter = &intel_idle, 1059 .enter_s2idle = intel_idle_s2idle, }, 1060 { 1061 .name = "C6", 1062 .desc = "MWAIT 0x20", 1063 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1064 CPUIDLE_FLAG_INIT_XSTATE | 1065 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1066 .exit_latency = 170, 1067 .target_residency = 650, 1068 .enter = &intel_idle, 1069 .enter_s2idle = intel_idle_s2idle, }, 1070 { 1071 .name = "C6P", 1072 .desc = "MWAIT 0x21", 1073 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1074 CPUIDLE_FLAG_INIT_XSTATE | 1075 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1076 .exit_latency = 210, 1077 .target_residency = 1000, 1078 .enter = &intel_idle, 1079 .enter_s2idle = intel_idle_s2idle, }, 1080 { 1081 .enter = NULL } 1082 }; 1083 1084 static struct cpuidle_state gnrd_cstates[] __initdata = { 1085 { 1086 .name = "C1", 1087 .desc = "MWAIT 0x00", 1088 .flags = MWAIT2flg(0x00), 1089 .exit_latency = 1, 1090 .target_residency = 1, 1091 .enter = &intel_idle, 1092 .enter_s2idle = intel_idle_s2idle, }, 1093 { 1094 .name = "C1E", 1095 .desc = "MWAIT 0x01", 1096 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1097 .exit_latency = 4, 1098 .target_residency = 4, 1099 .enter = &intel_idle, 1100 .enter_s2idle = intel_idle_s2idle, }, 1101 { 1102 .name = "C6", 1103 .desc = "MWAIT 0x20", 1104 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1105 CPUIDLE_FLAG_INIT_XSTATE | 1106 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1107 .exit_latency = 220, 1108 .target_residency = 650, 1109 .enter = &intel_idle, 1110 .enter_s2idle = intel_idle_s2idle, }, 1111 { 1112 .name = "C6P", 1113 .desc = "MWAIT 0x21", 1114 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1115 CPUIDLE_FLAG_INIT_XSTATE | 1116 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1117 .exit_latency = 240, 1118 .target_residency = 750, 1119 .enter = &intel_idle, 1120 .enter_s2idle = intel_idle_s2idle, }, 1121 { 1122 .enter = NULL } 1123 }; 1124 1125 static struct cpuidle_state atom_cstates[] __initdata = { 1126 { 1127 .name = "C1E", 1128 .desc = "MWAIT 0x00", 1129 .flags = MWAIT2flg(0x00), 1130 .exit_latency = 10, 1131 .target_residency = 20, 1132 .enter = &intel_idle, 1133 .enter_s2idle = intel_idle_s2idle, }, 1134 { 1135 .name = "C2", 1136 .desc = "MWAIT 0x10", 1137 .flags = MWAIT2flg(0x10), 1138 .exit_latency = 20, 1139 .target_residency = 80, 1140 .enter = &intel_idle, 1141 .enter_s2idle = intel_idle_s2idle, }, 1142 { 1143 .name = "C4", 1144 .desc = "MWAIT 0x30", 1145 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1146 .exit_latency = 100, 1147 .target_residency = 400, 1148 .enter = &intel_idle, 1149 .enter_s2idle = intel_idle_s2idle, }, 1150 { 1151 .name = "C6", 1152 .desc = "MWAIT 0x52", 1153 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1154 .exit_latency = 140, 1155 .target_residency = 560, 1156 .enter = &intel_idle, 1157 .enter_s2idle = intel_idle_s2idle, }, 1158 { 1159 .enter = NULL } 1160 }; 1161 static struct cpuidle_state tangier_cstates[] __initdata = { 1162 { 1163 .name = "C1", 1164 .desc = "MWAIT 0x00", 1165 .flags = MWAIT2flg(0x00), 1166 .exit_latency = 1, 1167 .target_residency = 4, 1168 .enter = &intel_idle, 1169 .enter_s2idle = intel_idle_s2idle, }, 1170 { 1171 .name = "C4", 1172 .desc = "MWAIT 0x30", 1173 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1174 .exit_latency = 100, 1175 .target_residency = 400, 1176 .enter = &intel_idle, 1177 .enter_s2idle = intel_idle_s2idle, }, 1178 { 1179 .name = "C6", 1180 .desc = "MWAIT 0x52", 1181 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1182 .exit_latency = 140, 1183 .target_residency = 560, 1184 .enter = &intel_idle, 1185 .enter_s2idle = intel_idle_s2idle, }, 1186 { 1187 .name = "C7", 1188 .desc = "MWAIT 0x60", 1189 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1190 .exit_latency = 1200, 1191 .target_residency = 4000, 1192 .enter = &intel_idle, 1193 .enter_s2idle = intel_idle_s2idle, }, 1194 { 1195 .name = "C9", 1196 .desc = "MWAIT 0x64", 1197 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1198 .exit_latency = 10000, 1199 .target_residency = 20000, 1200 .enter = &intel_idle, 1201 .enter_s2idle = intel_idle_s2idle, }, 1202 { 1203 .enter = NULL } 1204 }; 1205 static struct cpuidle_state avn_cstates[] __initdata = { 1206 { 1207 .name = "C1", 1208 .desc = "MWAIT 0x00", 1209 .flags = MWAIT2flg(0x00), 1210 .exit_latency = 2, 1211 .target_residency = 2, 1212 .enter = &intel_idle, 1213 .enter_s2idle = intel_idle_s2idle, }, 1214 { 1215 .name = "C6", 1216 .desc = "MWAIT 0x51", 1217 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1218 .exit_latency = 15, 1219 .target_residency = 45, 1220 .enter = &intel_idle, 1221 .enter_s2idle = intel_idle_s2idle, }, 1222 { 1223 .enter = NULL } 1224 }; 1225 static struct cpuidle_state knl_cstates[] __initdata = { 1226 { 1227 .name = "C1", 1228 .desc = "MWAIT 0x00", 1229 .flags = MWAIT2flg(0x00), 1230 .exit_latency = 1, 1231 .target_residency = 2, 1232 .enter = &intel_idle, 1233 .enter_s2idle = intel_idle_s2idle }, 1234 { 1235 .name = "C6", 1236 .desc = "MWAIT 0x10", 1237 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1238 .exit_latency = 120, 1239 .target_residency = 500, 1240 .enter = &intel_idle, 1241 .enter_s2idle = intel_idle_s2idle }, 1242 { 1243 .enter = NULL } 1244 }; 1245 1246 static struct cpuidle_state bxt_cstates[] __initdata = { 1247 { 1248 .name = "C1", 1249 .desc = "MWAIT 0x00", 1250 .flags = MWAIT2flg(0x00), 1251 .exit_latency = 2, 1252 .target_residency = 2, 1253 .enter = &intel_idle, 1254 .enter_s2idle = intel_idle_s2idle, }, 1255 { 1256 .name = "C1E", 1257 .desc = "MWAIT 0x01", 1258 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1259 .exit_latency = 10, 1260 .target_residency = 20, 1261 .enter = &intel_idle, 1262 .enter_s2idle = intel_idle_s2idle, }, 1263 { 1264 .name = "C6", 1265 .desc = "MWAIT 0x20", 1266 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1267 .exit_latency = 133, 1268 .target_residency = 133, 1269 .enter = &intel_idle, 1270 .enter_s2idle = intel_idle_s2idle, }, 1271 { 1272 .name = "C7s", 1273 .desc = "MWAIT 0x31", 1274 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1275 .exit_latency = 155, 1276 .target_residency = 155, 1277 .enter = &intel_idle, 1278 .enter_s2idle = intel_idle_s2idle, }, 1279 { 1280 .name = "C8", 1281 .desc = "MWAIT 0x40", 1282 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1283 .exit_latency = 1000, 1284 .target_residency = 1000, 1285 .enter = &intel_idle, 1286 .enter_s2idle = intel_idle_s2idle, }, 1287 { 1288 .name = "C9", 1289 .desc = "MWAIT 0x50", 1290 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1291 .exit_latency = 2000, 1292 .target_residency = 2000, 1293 .enter = &intel_idle, 1294 .enter_s2idle = intel_idle_s2idle, }, 1295 { 1296 .name = "C10", 1297 .desc = "MWAIT 0x60", 1298 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1299 .exit_latency = 10000, 1300 .target_residency = 10000, 1301 .enter = &intel_idle, 1302 .enter_s2idle = intel_idle_s2idle, }, 1303 { 1304 .enter = NULL } 1305 }; 1306 1307 static struct cpuidle_state dnv_cstates[] __initdata = { 1308 { 1309 .name = "C1", 1310 .desc = "MWAIT 0x00", 1311 .flags = MWAIT2flg(0x00), 1312 .exit_latency = 2, 1313 .target_residency = 2, 1314 .enter = &intel_idle, 1315 .enter_s2idle = intel_idle_s2idle, }, 1316 { 1317 .name = "C1E", 1318 .desc = "MWAIT 0x01", 1319 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1320 .exit_latency = 10, 1321 .target_residency = 20, 1322 .enter = &intel_idle, 1323 .enter_s2idle = intel_idle_s2idle, }, 1324 { 1325 .name = "C6", 1326 .desc = "MWAIT 0x20", 1327 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1328 .exit_latency = 50, 1329 .target_residency = 500, 1330 .enter = &intel_idle, 1331 .enter_s2idle = intel_idle_s2idle, }, 1332 { 1333 .enter = NULL } 1334 }; 1335 1336 /* 1337 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1338 * C6, and this is indicated in the CPUID mwait leaf. 1339 */ 1340 static struct cpuidle_state snr_cstates[] __initdata = { 1341 { 1342 .name = "C1", 1343 .desc = "MWAIT 0x00", 1344 .flags = MWAIT2flg(0x00), 1345 .exit_latency = 2, 1346 .target_residency = 2, 1347 .enter = &intel_idle, 1348 .enter_s2idle = intel_idle_s2idle, }, 1349 { 1350 .name = "C1E", 1351 .desc = "MWAIT 0x01", 1352 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1353 .exit_latency = 15, 1354 .target_residency = 25, 1355 .enter = &intel_idle, 1356 .enter_s2idle = intel_idle_s2idle, }, 1357 { 1358 .name = "C6", 1359 .desc = "MWAIT 0x20", 1360 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1361 .exit_latency = 130, 1362 .target_residency = 500, 1363 .enter = &intel_idle, 1364 .enter_s2idle = intel_idle_s2idle, }, 1365 { 1366 .enter = NULL } 1367 }; 1368 1369 static struct cpuidle_state grr_cstates[] __initdata = { 1370 { 1371 .name = "C1", 1372 .desc = "MWAIT 0x00", 1373 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1374 .exit_latency = 1, 1375 .target_residency = 1, 1376 .enter = &intel_idle, 1377 .enter_s2idle = intel_idle_s2idle, }, 1378 { 1379 .name = "C1E", 1380 .desc = "MWAIT 0x01", 1381 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1382 .exit_latency = 2, 1383 .target_residency = 10, 1384 .enter = &intel_idle, 1385 .enter_s2idle = intel_idle_s2idle, }, 1386 { 1387 .name = "C6S", 1388 .desc = "MWAIT 0x22", 1389 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1390 .exit_latency = 140, 1391 .target_residency = 500, 1392 .enter = &intel_idle, 1393 .enter_s2idle = intel_idle_s2idle, }, 1394 { 1395 .enter = NULL } 1396 }; 1397 1398 static struct cpuidle_state srf_cstates[] __initdata = { 1399 { 1400 .name = "C1", 1401 .desc = "MWAIT 0x00", 1402 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1403 .exit_latency = 1, 1404 .target_residency = 1, 1405 .enter = &intel_idle, 1406 .enter_s2idle = intel_idle_s2idle, }, 1407 { 1408 .name = "C1E", 1409 .desc = "MWAIT 0x01", 1410 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1411 .exit_latency = 2, 1412 .target_residency = 10, 1413 .enter = &intel_idle, 1414 .enter_s2idle = intel_idle_s2idle, }, 1415 { 1416 .name = "C6S", 1417 .desc = "MWAIT 0x22", 1418 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1419 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1420 .exit_latency = 270, 1421 .target_residency = 700, 1422 .enter = &intel_idle, 1423 .enter_s2idle = intel_idle_s2idle, }, 1424 { 1425 .name = "C6SP", 1426 .desc = "MWAIT 0x23", 1427 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1428 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1429 .exit_latency = 310, 1430 .target_residency = 900, 1431 .enter = &intel_idle, 1432 .enter_s2idle = intel_idle_s2idle, }, 1433 { 1434 .enter = NULL } 1435 }; 1436 1437 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1438 .state_table = nehalem_cstates, 1439 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1440 .disable_promotion_to_c1e = true, 1441 }; 1442 1443 static const struct idle_cpu idle_cpu_nhx __initconst = { 1444 .state_table = nehalem_cstates, 1445 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1446 .disable_promotion_to_c1e = true, 1447 .use_acpi = true, 1448 }; 1449 1450 static const struct idle_cpu idle_cpu_atom __initconst = { 1451 .state_table = atom_cstates, 1452 }; 1453 1454 static const struct idle_cpu idle_cpu_tangier __initconst = { 1455 .state_table = tangier_cstates, 1456 }; 1457 1458 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1459 .state_table = atom_cstates, 1460 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1461 }; 1462 1463 static const struct idle_cpu idle_cpu_snb __initconst = { 1464 .state_table = snb_cstates, 1465 .disable_promotion_to_c1e = true, 1466 }; 1467 1468 static const struct idle_cpu idle_cpu_snx __initconst = { 1469 .state_table = snb_cstates, 1470 .disable_promotion_to_c1e = true, 1471 .use_acpi = true, 1472 }; 1473 1474 static const struct idle_cpu idle_cpu_byt __initconst = { 1475 .state_table = byt_cstates, 1476 .disable_promotion_to_c1e = true, 1477 .byt_auto_demotion_disable_flag = true, 1478 }; 1479 1480 static const struct idle_cpu idle_cpu_cht __initconst = { 1481 .state_table = cht_cstates, 1482 .disable_promotion_to_c1e = true, 1483 .byt_auto_demotion_disable_flag = true, 1484 }; 1485 1486 static const struct idle_cpu idle_cpu_ivb __initconst = { 1487 .state_table = ivb_cstates, 1488 .disable_promotion_to_c1e = true, 1489 }; 1490 1491 static const struct idle_cpu idle_cpu_ivt __initconst = { 1492 .state_table = ivt_cstates, 1493 .disable_promotion_to_c1e = true, 1494 .use_acpi = true, 1495 }; 1496 1497 static const struct idle_cpu idle_cpu_hsw __initconst = { 1498 .state_table = hsw_cstates, 1499 .disable_promotion_to_c1e = true, 1500 }; 1501 1502 static const struct idle_cpu idle_cpu_hsx __initconst = { 1503 .state_table = hsw_cstates, 1504 .disable_promotion_to_c1e = true, 1505 .use_acpi = true, 1506 }; 1507 1508 static const struct idle_cpu idle_cpu_bdw __initconst = { 1509 .state_table = bdw_cstates, 1510 .disable_promotion_to_c1e = true, 1511 }; 1512 1513 static const struct idle_cpu idle_cpu_bdx __initconst = { 1514 .state_table = bdw_cstates, 1515 .disable_promotion_to_c1e = true, 1516 .use_acpi = true, 1517 }; 1518 1519 static const struct idle_cpu idle_cpu_skl __initconst = { 1520 .state_table = skl_cstates, 1521 .disable_promotion_to_c1e = true, 1522 }; 1523 1524 static const struct idle_cpu idle_cpu_skx __initconst = { 1525 .state_table = skx_cstates, 1526 .disable_promotion_to_c1e = true, 1527 .use_acpi = true, 1528 }; 1529 1530 static const struct idle_cpu idle_cpu_icx __initconst = { 1531 .state_table = icx_cstates, 1532 .disable_promotion_to_c1e = true, 1533 .use_acpi = true, 1534 }; 1535 1536 static const struct idle_cpu idle_cpu_adl __initconst = { 1537 .state_table = adl_cstates, 1538 }; 1539 1540 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1541 .state_table = adl_l_cstates, 1542 }; 1543 1544 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1545 .state_table = mtl_l_cstates, 1546 }; 1547 1548 static const struct idle_cpu idle_cpu_gmt __initconst = { 1549 .state_table = gmt_cstates, 1550 }; 1551 1552 static const struct idle_cpu idle_cpu_spr __initconst = { 1553 .state_table = spr_cstates, 1554 .disable_promotion_to_c1e = true, 1555 .use_acpi = true, 1556 }; 1557 1558 static const struct idle_cpu idle_cpu_gnr __initconst = { 1559 .state_table = gnr_cstates, 1560 .disable_promotion_to_c1e = true, 1561 .use_acpi = true, 1562 }; 1563 1564 static const struct idle_cpu idle_cpu_gnrd __initconst = { 1565 .state_table = gnrd_cstates, 1566 .disable_promotion_to_c1e = true, 1567 .use_acpi = true, 1568 }; 1569 1570 static const struct idle_cpu idle_cpu_avn __initconst = { 1571 .state_table = avn_cstates, 1572 .disable_promotion_to_c1e = true, 1573 .use_acpi = true, 1574 }; 1575 1576 static const struct idle_cpu idle_cpu_knl __initconst = { 1577 .state_table = knl_cstates, 1578 .use_acpi = true, 1579 }; 1580 1581 static const struct idle_cpu idle_cpu_bxt __initconst = { 1582 .state_table = bxt_cstates, 1583 .disable_promotion_to_c1e = true, 1584 }; 1585 1586 static const struct idle_cpu idle_cpu_dnv __initconst = { 1587 .state_table = dnv_cstates, 1588 .disable_promotion_to_c1e = true, 1589 .use_acpi = true, 1590 }; 1591 1592 static const struct idle_cpu idle_cpu_tmt __initconst = { 1593 .disable_promotion_to_c1e = true, 1594 }; 1595 1596 static const struct idle_cpu idle_cpu_snr __initconst = { 1597 .state_table = snr_cstates, 1598 .disable_promotion_to_c1e = true, 1599 .use_acpi = true, 1600 }; 1601 1602 static const struct idle_cpu idle_cpu_grr __initconst = { 1603 .state_table = grr_cstates, 1604 .disable_promotion_to_c1e = true, 1605 .use_acpi = true, 1606 }; 1607 1608 static const struct idle_cpu idle_cpu_srf __initconst = { 1609 .state_table = srf_cstates, 1610 .disable_promotion_to_c1e = true, 1611 .use_acpi = true, 1612 }; 1613 1614 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1615 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1616 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1617 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1618 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1619 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1620 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1621 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1622 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1623 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1624 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1625 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1626 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1627 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1628 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1629 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1630 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1631 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1632 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1633 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1634 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1635 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1636 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1637 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1638 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1639 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1640 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1641 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1642 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1643 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1644 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1645 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1646 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1647 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1648 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1649 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1650 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1651 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1652 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1653 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1654 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1655 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), 1656 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1657 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1658 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1659 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1660 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1661 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1662 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1663 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1664 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1665 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1666 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf), 1667 {} 1668 }; 1669 1670 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1671 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1672 {} 1673 }; 1674 1675 static bool __init intel_idle_max_cstate_reached(int cstate) 1676 { 1677 if (cstate + 1 > max_cstate) { 1678 pr_info("max_cstate %d reached\n", max_cstate); 1679 return true; 1680 } 1681 return false; 1682 } 1683 1684 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1685 { 1686 unsigned long eax = flg2MWAIT(state->flags); 1687 1688 if (boot_cpu_has(X86_FEATURE_ARAT)) 1689 return false; 1690 1691 /* 1692 * Switch over to one-shot tick broadcast if the target C-state 1693 * is deeper than C1. 1694 */ 1695 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1696 } 1697 1698 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1699 #include <acpi/processor.h> 1700 1701 static bool no_acpi __read_mostly; 1702 module_param(no_acpi, bool, 0444); 1703 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1704 1705 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1706 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1707 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1708 1709 static struct acpi_processor_power acpi_state_table __initdata; 1710 1711 /** 1712 * intel_idle_cst_usable - Check if the _CST information can be used. 1713 * 1714 * Check if all of the C-states listed by _CST in the max_cstate range are 1715 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1716 */ 1717 static bool __init intel_idle_cst_usable(void) 1718 { 1719 int cstate, limit; 1720 1721 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1722 acpi_state_table.count); 1723 1724 for (cstate = 1; cstate < limit; cstate++) { 1725 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1726 1727 if (cx->entry_method != ACPI_CSTATE_FFH) 1728 return false; 1729 } 1730 1731 return true; 1732 } 1733 1734 static bool __init intel_idle_acpi_cst_extract(void) 1735 { 1736 unsigned int cpu; 1737 1738 if (no_acpi) { 1739 pr_debug("Not allowed to use ACPI _CST\n"); 1740 return false; 1741 } 1742 1743 for_each_possible_cpu(cpu) { 1744 struct acpi_processor *pr = per_cpu(processors, cpu); 1745 1746 if (!pr) 1747 continue; 1748 1749 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1750 continue; 1751 1752 acpi_state_table.count++; 1753 1754 if (!intel_idle_cst_usable()) 1755 continue; 1756 1757 if (!acpi_processor_claim_cst_control()) 1758 break; 1759 1760 return true; 1761 } 1762 1763 acpi_state_table.count = 0; 1764 pr_debug("ACPI _CST not found or not usable\n"); 1765 return false; 1766 } 1767 1768 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1769 { 1770 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1771 1772 /* 1773 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1774 * the interesting states are ACPI_CSTATE_FFH. 1775 */ 1776 for (cstate = 1; cstate < limit; cstate++) { 1777 struct acpi_processor_cx *cx; 1778 struct cpuidle_state *state; 1779 1780 if (intel_idle_max_cstate_reached(cstate - 1)) 1781 break; 1782 1783 cx = &acpi_state_table.states[cstate]; 1784 1785 state = &drv->states[drv->state_count++]; 1786 1787 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1788 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1789 state->exit_latency = cx->latency; 1790 /* 1791 * For C1-type C-states use the same number for both the exit 1792 * latency and target residency, because that is the case for 1793 * C1 in the majority of the static C-states tables above. 1794 * For the other types of C-states, however, set the target 1795 * residency to 3 times the exit latency which should lead to 1796 * a reasonable balance between energy-efficiency and 1797 * performance in the majority of interesting cases. 1798 */ 1799 state->target_residency = cx->latency; 1800 if (cx->type > ACPI_STATE_C1) 1801 state->target_residency *= 3; 1802 1803 state->flags = MWAIT2flg(cx->address); 1804 if (cx->type > ACPI_STATE_C2) 1805 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1806 1807 if (disabled_states_mask & BIT(cstate)) 1808 state->flags |= CPUIDLE_FLAG_OFF; 1809 1810 if (intel_idle_state_needs_timer_stop(state)) 1811 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1812 1813 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1814 mark_tsc_unstable("TSC halts in idle"); 1815 1816 state->enter = intel_idle; 1817 state->enter_dead = intel_idle_enter_dead; 1818 state->enter_s2idle = intel_idle_s2idle; 1819 } 1820 } 1821 1822 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1823 { 1824 int cstate, limit; 1825 1826 /* 1827 * If there are no _CST C-states, do not disable any C-states by 1828 * default. 1829 */ 1830 if (!acpi_state_table.count) 1831 return false; 1832 1833 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1834 /* 1835 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1836 * the interesting states are ACPI_CSTATE_FFH. 1837 */ 1838 for (cstate = 1; cstate < limit; cstate++) { 1839 u32 acpi_hint = acpi_state_table.states[cstate].address; 1840 u32 table_hint = mwait_hint; 1841 1842 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 1843 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 1844 table_hint &= ~MWAIT_SUBSTATE_MASK; 1845 } 1846 1847 if (acpi_hint == table_hint) 1848 return false; 1849 } 1850 return true; 1851 } 1852 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1853 #define force_use_acpi (false) 1854 1855 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1856 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1857 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1858 { 1859 return false; 1860 } 1861 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1862 1863 /** 1864 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1865 * 1866 * Tune IVT multi-socket targets. 1867 * Assumption: num_sockets == (max_package_num + 1). 1868 */ 1869 static void __init ivt_idle_state_table_update(void) 1870 { 1871 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1872 int cpu, package_num, num_sockets = 1; 1873 1874 for_each_online_cpu(cpu) { 1875 package_num = topology_physical_package_id(cpu); 1876 if (package_num + 1 > num_sockets) { 1877 num_sockets = package_num + 1; 1878 1879 if (num_sockets > 4) { 1880 cpuidle_state_table = ivt_cstates_8s; 1881 return; 1882 } 1883 } 1884 } 1885 1886 if (num_sockets > 2) 1887 cpuidle_state_table = ivt_cstates_4s; 1888 1889 /* else, 1 and 2 socket systems use default ivt_cstates */ 1890 } 1891 1892 /** 1893 * irtl_2_usec - IRTL to microseconds conversion. 1894 * @irtl: IRTL MSR value. 1895 * 1896 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1897 */ 1898 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1899 { 1900 static const unsigned int irtl_ns_units[] __initconst = { 1901 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1902 }; 1903 unsigned long long ns; 1904 1905 if (!irtl) 1906 return 0; 1907 1908 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1909 1910 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1911 } 1912 1913 /** 1914 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1915 * 1916 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1917 * definitive maximum latency and use the same value for target_residency. 1918 */ 1919 static void __init bxt_idle_state_table_update(void) 1920 { 1921 unsigned long long msr; 1922 unsigned int usec; 1923 1924 rdmsrl(MSR_PKGC6_IRTL, msr); 1925 usec = irtl_2_usec(msr); 1926 if (usec) { 1927 bxt_cstates[2].exit_latency = usec; 1928 bxt_cstates[2].target_residency = usec; 1929 } 1930 1931 rdmsrl(MSR_PKGC7_IRTL, msr); 1932 usec = irtl_2_usec(msr); 1933 if (usec) { 1934 bxt_cstates[3].exit_latency = usec; 1935 bxt_cstates[3].target_residency = usec; 1936 } 1937 1938 rdmsrl(MSR_PKGC8_IRTL, msr); 1939 usec = irtl_2_usec(msr); 1940 if (usec) { 1941 bxt_cstates[4].exit_latency = usec; 1942 bxt_cstates[4].target_residency = usec; 1943 } 1944 1945 rdmsrl(MSR_PKGC9_IRTL, msr); 1946 usec = irtl_2_usec(msr); 1947 if (usec) { 1948 bxt_cstates[5].exit_latency = usec; 1949 bxt_cstates[5].target_residency = usec; 1950 } 1951 1952 rdmsrl(MSR_PKGC10_IRTL, msr); 1953 usec = irtl_2_usec(msr); 1954 if (usec) { 1955 bxt_cstates[6].exit_latency = usec; 1956 bxt_cstates[6].target_residency = usec; 1957 } 1958 1959 } 1960 1961 /** 1962 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1963 * 1964 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1965 */ 1966 static void __init sklh_idle_state_table_update(void) 1967 { 1968 unsigned long long msr; 1969 unsigned int eax, ebx, ecx, edx; 1970 1971 1972 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1973 if (max_cstate <= 7) 1974 return; 1975 1976 /* if PC10 not present in CPUID.MWAIT.EDX */ 1977 if ((mwait_substates & (0xF << 28)) == 0) 1978 return; 1979 1980 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1981 1982 /* PC10 is not enabled in PKG C-state limit */ 1983 if ((msr & 0xF) != 8) 1984 return; 1985 1986 ecx = 0; 1987 cpuid(7, &eax, &ebx, &ecx, &edx); 1988 1989 /* if SGX is present */ 1990 if (ebx & (1 << 2)) { 1991 1992 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1993 1994 /* if SGX is enabled */ 1995 if (msr & (1 << 18)) 1996 return; 1997 } 1998 1999 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 2000 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 2001 } 2002 2003 /** 2004 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 2005 * idle states table. 2006 */ 2007 static void __init skx_idle_state_table_update(void) 2008 { 2009 unsigned long long msr; 2010 2011 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2012 2013 /* 2014 * 000b: C0/C1 (no package C-state support) 2015 * 001b: C2 2016 * 010b: C6 (non-retention) 2017 * 011b: C6 (retention) 2018 * 111b: No Package C state limits. 2019 */ 2020 if ((msr & 0x7) < 2) { 2021 /* 2022 * Uses the CC6 + PC0 latency and 3 times of 2023 * latency for target_residency if the PC6 2024 * is disabled in BIOS. This is consistent 2025 * with how intel_idle driver uses _CST 2026 * to set the target_residency. 2027 */ 2028 skx_cstates[2].exit_latency = 92; 2029 skx_cstates[2].target_residency = 276; 2030 } 2031 } 2032 2033 /** 2034 * adl_idle_state_table_update - Adjust AlderLake idle states table. 2035 */ 2036 static void __init adl_idle_state_table_update(void) 2037 { 2038 /* Check if user prefers C1 over C1E. */ 2039 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 2040 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 2041 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 2042 2043 /* Disable C1E by clearing the "C1E promotion" bit. */ 2044 c1e_promotion = C1E_PROMOTION_DISABLE; 2045 return; 2046 } 2047 2048 /* Make sure C1E is enabled by default */ 2049 c1e_promotion = C1E_PROMOTION_ENABLE; 2050 } 2051 2052 /** 2053 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 2054 */ 2055 static void __init spr_idle_state_table_update(void) 2056 { 2057 unsigned long long msr; 2058 2059 /* 2060 * By default, the C6 state assumes the worst-case scenario of package 2061 * C6. However, if PC6 is disabled, we update the numbers to match 2062 * core C6. 2063 */ 2064 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2065 2066 /* Limit value 2 and above allow for PC6. */ 2067 if ((msr & 0x7) < 2) { 2068 spr_cstates[2].exit_latency = 190; 2069 spr_cstates[2].target_residency = 600; 2070 } 2071 } 2072 2073 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2074 { 2075 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2076 MWAIT_CSTATE_MASK; 2077 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2078 MWAIT_SUBSTATE_MASK; 2079 2080 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2081 if (num_substates == 0) 2082 return false; 2083 2084 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2085 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2086 2087 return true; 2088 } 2089 2090 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2091 { 2092 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2093 /* 2094 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2095 * is not currently supported but this driver. 2096 */ 2097 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2098 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2099 state->enter = intel_idle_xstate; 2100 return; 2101 } 2102 2103 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2104 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2105 /* 2106 * IBRS mitigation requires that C-states are entered 2107 * with interrupts disabled. 2108 */ 2109 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2110 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2111 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2112 state->enter = intel_idle_ibrs; 2113 return; 2114 } 2115 2116 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2117 state->enter = intel_idle_irq; 2118 return; 2119 } 2120 2121 if (force_irq_on) { 2122 pr_info("forced intel_idle_irq for state %d\n", cstate); 2123 state->enter = intel_idle_irq; 2124 } 2125 } 2126 2127 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2128 { 2129 int cstate; 2130 2131 switch (boot_cpu_data.x86_vfm) { 2132 case INTEL_IVYBRIDGE_X: 2133 ivt_idle_state_table_update(); 2134 break; 2135 case INTEL_ATOM_GOLDMONT: 2136 case INTEL_ATOM_GOLDMONT_PLUS: 2137 bxt_idle_state_table_update(); 2138 break; 2139 case INTEL_SKYLAKE: 2140 sklh_idle_state_table_update(); 2141 break; 2142 case INTEL_SKYLAKE_X: 2143 skx_idle_state_table_update(); 2144 break; 2145 case INTEL_SAPPHIRERAPIDS_X: 2146 case INTEL_EMERALDRAPIDS_X: 2147 spr_idle_state_table_update(); 2148 break; 2149 case INTEL_ALDERLAKE: 2150 case INTEL_ALDERLAKE_L: 2151 case INTEL_ATOM_GRACEMONT: 2152 adl_idle_state_table_update(); 2153 break; 2154 } 2155 2156 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2157 struct cpuidle_state *state; 2158 unsigned int mwait_hint; 2159 2160 if (intel_idle_max_cstate_reached(cstate)) 2161 break; 2162 2163 if (!cpuidle_state_table[cstate].enter && 2164 !cpuidle_state_table[cstate].enter_s2idle) 2165 break; 2166 2167 if (!cpuidle_state_table[cstate].enter_dead) 2168 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead; 2169 2170 /* If marked as unusable, skip this state. */ 2171 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2172 pr_debug("state %s is disabled\n", 2173 cpuidle_state_table[cstate].name); 2174 continue; 2175 } 2176 2177 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2178 if (!intel_idle_verify_cstate(mwait_hint)) 2179 continue; 2180 2181 /* Structure copy. */ 2182 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2183 state = &drv->states[drv->state_count]; 2184 2185 state_update_enter_method(state, cstate); 2186 2187 2188 if ((disabled_states_mask & BIT(drv->state_count)) || 2189 ((icpu->use_acpi || force_use_acpi) && 2190 intel_idle_off_by_default(state->flags, mwait_hint) && 2191 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2192 state->flags |= CPUIDLE_FLAG_OFF; 2193 2194 if (intel_idle_state_needs_timer_stop(state)) 2195 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2196 2197 drv->state_count++; 2198 } 2199 2200 if (icpu->byt_auto_demotion_disable_flag) { 2201 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2202 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2203 } 2204 } 2205 2206 /** 2207 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2208 * @drv: cpuidle driver structure to initialize. 2209 */ 2210 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2211 { 2212 cpuidle_poll_state_init(drv); 2213 2214 if (disabled_states_mask & BIT(0)) 2215 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2216 2217 drv->state_count = 1; 2218 2219 if (icpu && icpu->state_table) 2220 intel_idle_init_cstates_icpu(drv); 2221 else 2222 intel_idle_init_cstates_acpi(drv); 2223 } 2224 2225 static void auto_demotion_disable(void) 2226 { 2227 unsigned long long msr_bits; 2228 2229 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2230 msr_bits &= ~auto_demotion_disable_flags; 2231 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2232 } 2233 2234 static void c1e_promotion_enable(void) 2235 { 2236 unsigned long long msr_bits; 2237 2238 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2239 msr_bits |= 0x2; 2240 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2241 } 2242 2243 static void c1e_promotion_disable(void) 2244 { 2245 unsigned long long msr_bits; 2246 2247 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2248 msr_bits &= ~0x2; 2249 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2250 } 2251 2252 /** 2253 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2254 * @cpu: CPU to initialize. 2255 * 2256 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2257 * with the processor model flags. 2258 */ 2259 static int intel_idle_cpu_init(unsigned int cpu) 2260 { 2261 struct cpuidle_device *dev; 2262 2263 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2264 dev->cpu = cpu; 2265 2266 if (cpuidle_register_device(dev)) { 2267 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2268 return -EIO; 2269 } 2270 2271 if (auto_demotion_disable_flags) 2272 auto_demotion_disable(); 2273 2274 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2275 c1e_promotion_enable(); 2276 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2277 c1e_promotion_disable(); 2278 2279 return 0; 2280 } 2281 2282 static int intel_idle_cpu_online(unsigned int cpu) 2283 { 2284 struct cpuidle_device *dev; 2285 2286 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2287 tick_broadcast_enable(); 2288 2289 /* 2290 * Some systems can hotplug a cpu at runtime after 2291 * the kernel has booted, we have to initialize the 2292 * driver in this case 2293 */ 2294 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2295 if (!dev->registered) 2296 return intel_idle_cpu_init(cpu); 2297 2298 return 0; 2299 } 2300 2301 /** 2302 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2303 */ 2304 static void __init intel_idle_cpuidle_devices_uninit(void) 2305 { 2306 int i; 2307 2308 for_each_online_cpu(i) 2309 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2310 } 2311 2312 static int __init intel_idle_init(void) 2313 { 2314 const struct x86_cpu_id *id; 2315 unsigned int eax, ebx, ecx; 2316 int retval; 2317 2318 /* Do not load intel_idle at all for now if idle= is passed */ 2319 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2320 return -ENODEV; 2321 2322 if (max_cstate == 0) { 2323 pr_debug("disabled\n"); 2324 return -EPERM; 2325 } 2326 2327 id = x86_match_cpu(intel_idle_ids); 2328 if (id) { 2329 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2330 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2331 return -ENODEV; 2332 } 2333 } else { 2334 id = x86_match_cpu(intel_mwait_ids); 2335 if (!id) 2336 return -ENODEV; 2337 } 2338 2339 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates); 2340 2341 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2342 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2343 !mwait_substates) 2344 return -ENODEV; 2345 2346 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2347 2348 icpu = (const struct idle_cpu *)id->driver_data; 2349 if (icpu) { 2350 if (icpu->state_table) 2351 cpuidle_state_table = icpu->state_table; 2352 else if (!intel_idle_acpi_cst_extract()) 2353 return -ENODEV; 2354 2355 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2356 if (icpu->disable_promotion_to_c1e) 2357 c1e_promotion = C1E_PROMOTION_DISABLE; 2358 if (icpu->use_acpi || force_use_acpi) 2359 intel_idle_acpi_cst_extract(); 2360 } else if (!intel_idle_acpi_cst_extract()) { 2361 return -ENODEV; 2362 } 2363 2364 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2365 boot_cpu_data.x86_model); 2366 2367 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2368 if (!intel_idle_cpuidle_devices) 2369 return -ENOMEM; 2370 2371 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2372 2373 retval = cpuidle_register_driver(&intel_idle_driver); 2374 if (retval) { 2375 struct cpuidle_driver *drv = cpuidle_get_driver(); 2376 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2377 drv ? drv->name : "none"); 2378 goto init_driver_fail; 2379 } 2380 2381 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2382 intel_idle_cpu_online, NULL); 2383 if (retval < 0) 2384 goto hp_setup_fail; 2385 2386 pr_debug("Local APIC timer is reliable in %s\n", 2387 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2388 2389 return 0; 2390 2391 hp_setup_fail: 2392 intel_idle_cpuidle_devices_uninit(); 2393 cpuidle_unregister_driver(&intel_idle_driver); 2394 init_driver_fail: 2395 free_percpu(intel_idle_cpuidle_devices); 2396 return retval; 2397 2398 } 2399 device_initcall(intel_idle_init); 2400 2401 /* 2402 * We are not really modular, but we used to support that. Meaning we also 2403 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2404 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2405 * is the easiest way (currently) to continue doing that. 2406 */ 2407 module_param(max_cstate, int, 0444); 2408 /* 2409 * The positions of the bits that are set in this number are the indices of the 2410 * idle states to be disabled by default (as reflected by the names of the 2411 * corresponding idle state directories in sysfs, "state0", "state1" ... 2412 * "state<i>" ..., where <i> is the index of the given state). 2413 */ 2414 module_param_named(states_off, disabled_states_mask, uint, 0444); 2415 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2416 /* 2417 * Some platforms come with mutually exclusive C-states, so that if one is 2418 * enabled, the other C-states must not be used. Example: C1 and C1E on 2419 * Sapphire Rapids platform. This parameter allows for selecting the 2420 * preferred C-states among the groups of mutually exclusive C-states - the 2421 * selected C-states will be registered, the other C-states from the mutually 2422 * exclusive group won't be registered. If the platform has no mutually 2423 * exclusive C-states, this parameter has no effect. 2424 */ 2425 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2426 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2427 /* 2428 * Debugging option that forces the driver to enter all C-states with 2429 * interrupts enabled. Does not apply to C-states with 2430 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2431 */ 2432 module_param(force_irq_on, bool, 0444); 2433 /* 2434 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2435 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2436 */ 2437 module_param(ibrs_off, bool, 0444); 2438 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2439