1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/sched/smt.h> 51 #include <linux/notifier.h> 52 #include <linux/cpu.h> 53 #include <linux/moduleparam.h> 54 #include <asm/cpuid.h> 55 #include <asm/cpu_device_id.h> 56 #include <asm/intel-family.h> 57 #include <asm/mwait.h> 58 #include <asm/spec-ctrl.h> 59 #include <asm/tsc.h> 60 #include <asm/fpu/api.h> 61 62 #define INTEL_IDLE_VERSION "0.5.1" 63 64 static struct cpuidle_driver intel_idle_driver = { 65 .name = "intel_idle", 66 .owner = THIS_MODULE, 67 }; 68 /* intel_idle.max_cstate=0 disables driver */ 69 static int max_cstate = CPUIDLE_STATE_MAX - 1; 70 static unsigned int disabled_states_mask __read_mostly; 71 static unsigned int preferred_states_mask __read_mostly; 72 static bool force_irq_on __read_mostly; 73 static bool ibrs_off __read_mostly; 74 75 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 76 77 static unsigned long auto_demotion_disable_flags; 78 79 static enum { 80 C1E_PROMOTION_PRESERVE, 81 C1E_PROMOTION_ENABLE, 82 C1E_PROMOTION_DISABLE 83 } c1e_promotion = C1E_PROMOTION_PRESERVE; 84 85 struct idle_cpu { 86 struct cpuidle_state *state_table; 87 88 /* 89 * Hardware C-state auto-demotion may not always be optimal. 90 * Indicate which enable bits to clear here. 91 */ 92 unsigned long auto_demotion_disable_flags; 93 bool byt_auto_demotion_disable_flag; 94 bool disable_promotion_to_c1e; 95 bool use_acpi; 96 }; 97 98 static const struct idle_cpu *icpu __initdata; 99 static struct cpuidle_state *cpuidle_state_table __initdata; 100 101 static unsigned int mwait_substates __initdata; 102 103 /* 104 * Enable interrupts before entering the C-state. On some platforms and for 105 * some C-states, this may measurably decrease interrupt latency. 106 */ 107 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 108 109 /* 110 * Enable this state by default even if the ACPI _CST does not list it. 111 */ 112 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 113 114 /* 115 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 116 * above. 117 */ 118 #define CPUIDLE_FLAG_IBRS BIT(16) 119 120 /* 121 * Initialize large xstate for the C6-state entrance. 122 */ 123 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 124 125 /* 126 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 127 * custom tables. 128 */ 129 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 130 131 /* 132 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 133 * the C-state (top nibble) and sub-state (bottom nibble) 134 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 135 * 136 * We store the hint at the top of our "flags" for each state. 137 */ 138 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 139 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 140 141 static __always_inline int __intel_idle(struct cpuidle_device *dev, 142 struct cpuidle_driver *drv, 143 int index, bool irqoff) 144 { 145 struct cpuidle_state *state = &drv->states[index]; 146 unsigned long eax = flg2MWAIT(state->flags); 147 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 148 149 mwait_idle_with_hints(eax, ecx); 150 151 return index; 152 } 153 154 /** 155 * intel_idle - Ask the processor to enter the given idle state. 156 * @dev: cpuidle device of the target CPU. 157 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 158 * @index: Target idle state index. 159 * 160 * Use the MWAIT instruction to notify the processor that the CPU represented by 161 * @dev is idle and it can try to enter the idle state corresponding to @index. 162 * 163 * If the local APIC timer is not known to be reliable in the target idle state, 164 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 165 * 166 * Must be called under local_irq_disable(). 167 */ 168 static __cpuidle int intel_idle(struct cpuidle_device *dev, 169 struct cpuidle_driver *drv, int index) 170 { 171 return __intel_idle(dev, drv, index, true); 172 } 173 174 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 175 struct cpuidle_driver *drv, int index) 176 { 177 return __intel_idle(dev, drv, index, false); 178 } 179 180 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 181 struct cpuidle_driver *drv, int index) 182 { 183 bool smt_active = sched_smt_active(); 184 u64 spec_ctrl = spec_ctrl_current(); 185 int ret; 186 187 if (smt_active) 188 __update_spec_ctrl(0); 189 190 ret = __intel_idle(dev, drv, index, true); 191 192 if (smt_active) 193 __update_spec_ctrl(spec_ctrl); 194 195 return ret; 196 } 197 198 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 199 struct cpuidle_driver *drv, int index) 200 { 201 fpu_idle_fpregs(); 202 return __intel_idle(dev, drv, index, true); 203 } 204 205 /** 206 * intel_idle_s2idle - Ask the processor to enter the given idle state. 207 * @dev: cpuidle device of the target CPU. 208 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 209 * @index: Target idle state index. 210 * 211 * Use the MWAIT instruction to notify the processor that the CPU represented by 212 * @dev is idle and it can try to enter the idle state corresponding to @index. 213 * 214 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 215 * scheduler tick and suspended scheduler clock on the target CPU. 216 */ 217 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 218 struct cpuidle_driver *drv, int index) 219 { 220 unsigned long ecx = 1; /* break on interrupt flag */ 221 struct cpuidle_state *state = &drv->states[index]; 222 unsigned long eax = flg2MWAIT(state->flags); 223 224 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 225 fpu_idle_fpregs(); 226 227 mwait_idle_with_hints(eax, ecx); 228 229 return 0; 230 } 231 232 /* 233 * States are indexed by the cstate number, 234 * which is also the index into the MWAIT hint array. 235 * Thus C0 is a dummy. 236 */ 237 static struct cpuidle_state nehalem_cstates[] __initdata = { 238 { 239 .name = "C1", 240 .desc = "MWAIT 0x00", 241 .flags = MWAIT2flg(0x00), 242 .exit_latency = 3, 243 .target_residency = 6, 244 .enter = &intel_idle, 245 .enter_s2idle = intel_idle_s2idle, }, 246 { 247 .name = "C1E", 248 .desc = "MWAIT 0x01", 249 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 250 .exit_latency = 10, 251 .target_residency = 20, 252 .enter = &intel_idle, 253 .enter_s2idle = intel_idle_s2idle, }, 254 { 255 .name = "C3", 256 .desc = "MWAIT 0x10", 257 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 258 .exit_latency = 20, 259 .target_residency = 80, 260 .enter = &intel_idle, 261 .enter_s2idle = intel_idle_s2idle, }, 262 { 263 .name = "C6", 264 .desc = "MWAIT 0x20", 265 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 266 .exit_latency = 200, 267 .target_residency = 800, 268 .enter = &intel_idle, 269 .enter_s2idle = intel_idle_s2idle, }, 270 { 271 .enter = NULL } 272 }; 273 274 static struct cpuidle_state snb_cstates[] __initdata = { 275 { 276 .name = "C1", 277 .desc = "MWAIT 0x00", 278 .flags = MWAIT2flg(0x00), 279 .exit_latency = 2, 280 .target_residency = 2, 281 .enter = &intel_idle, 282 .enter_s2idle = intel_idle_s2idle, }, 283 { 284 .name = "C1E", 285 .desc = "MWAIT 0x01", 286 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 287 .exit_latency = 10, 288 .target_residency = 20, 289 .enter = &intel_idle, 290 .enter_s2idle = intel_idle_s2idle, }, 291 { 292 .name = "C3", 293 .desc = "MWAIT 0x10", 294 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 295 .exit_latency = 80, 296 .target_residency = 211, 297 .enter = &intel_idle, 298 .enter_s2idle = intel_idle_s2idle, }, 299 { 300 .name = "C6", 301 .desc = "MWAIT 0x20", 302 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 303 .exit_latency = 104, 304 .target_residency = 345, 305 .enter = &intel_idle, 306 .enter_s2idle = intel_idle_s2idle, }, 307 { 308 .name = "C7", 309 .desc = "MWAIT 0x30", 310 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 311 .exit_latency = 109, 312 .target_residency = 345, 313 .enter = &intel_idle, 314 .enter_s2idle = intel_idle_s2idle, }, 315 { 316 .enter = NULL } 317 }; 318 319 static struct cpuidle_state byt_cstates[] __initdata = { 320 { 321 .name = "C1", 322 .desc = "MWAIT 0x00", 323 .flags = MWAIT2flg(0x00), 324 .exit_latency = 1, 325 .target_residency = 1, 326 .enter = &intel_idle, 327 .enter_s2idle = intel_idle_s2idle, }, 328 { 329 .name = "C6N", 330 .desc = "MWAIT 0x58", 331 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 332 .exit_latency = 300, 333 .target_residency = 275, 334 .enter = &intel_idle, 335 .enter_s2idle = intel_idle_s2idle, }, 336 { 337 .name = "C6S", 338 .desc = "MWAIT 0x52", 339 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 340 .exit_latency = 500, 341 .target_residency = 560, 342 .enter = &intel_idle, 343 .enter_s2idle = intel_idle_s2idle, }, 344 { 345 .name = "C7", 346 .desc = "MWAIT 0x60", 347 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 348 .exit_latency = 1200, 349 .target_residency = 4000, 350 .enter = &intel_idle, 351 .enter_s2idle = intel_idle_s2idle, }, 352 { 353 .name = "C7S", 354 .desc = "MWAIT 0x64", 355 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 356 .exit_latency = 10000, 357 .target_residency = 20000, 358 .enter = &intel_idle, 359 .enter_s2idle = intel_idle_s2idle, }, 360 { 361 .enter = NULL } 362 }; 363 364 static struct cpuidle_state cht_cstates[] __initdata = { 365 { 366 .name = "C1", 367 .desc = "MWAIT 0x00", 368 .flags = MWAIT2flg(0x00), 369 .exit_latency = 1, 370 .target_residency = 1, 371 .enter = &intel_idle, 372 .enter_s2idle = intel_idle_s2idle, }, 373 { 374 .name = "C6N", 375 .desc = "MWAIT 0x58", 376 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 377 .exit_latency = 80, 378 .target_residency = 275, 379 .enter = &intel_idle, 380 .enter_s2idle = intel_idle_s2idle, }, 381 { 382 .name = "C6S", 383 .desc = "MWAIT 0x52", 384 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 385 .exit_latency = 200, 386 .target_residency = 560, 387 .enter = &intel_idle, 388 .enter_s2idle = intel_idle_s2idle, }, 389 { 390 .name = "C7", 391 .desc = "MWAIT 0x60", 392 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 393 .exit_latency = 1200, 394 .target_residency = 4000, 395 .enter = &intel_idle, 396 .enter_s2idle = intel_idle_s2idle, }, 397 { 398 .name = "C7S", 399 .desc = "MWAIT 0x64", 400 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 401 .exit_latency = 10000, 402 .target_residency = 20000, 403 .enter = &intel_idle, 404 .enter_s2idle = intel_idle_s2idle, }, 405 { 406 .enter = NULL } 407 }; 408 409 static struct cpuidle_state ivb_cstates[] __initdata = { 410 { 411 .name = "C1", 412 .desc = "MWAIT 0x00", 413 .flags = MWAIT2flg(0x00), 414 .exit_latency = 1, 415 .target_residency = 1, 416 .enter = &intel_idle, 417 .enter_s2idle = intel_idle_s2idle, }, 418 { 419 .name = "C1E", 420 .desc = "MWAIT 0x01", 421 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 422 .exit_latency = 10, 423 .target_residency = 20, 424 .enter = &intel_idle, 425 .enter_s2idle = intel_idle_s2idle, }, 426 { 427 .name = "C3", 428 .desc = "MWAIT 0x10", 429 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 430 .exit_latency = 59, 431 .target_residency = 156, 432 .enter = &intel_idle, 433 .enter_s2idle = intel_idle_s2idle, }, 434 { 435 .name = "C6", 436 .desc = "MWAIT 0x20", 437 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 438 .exit_latency = 80, 439 .target_residency = 300, 440 .enter = &intel_idle, 441 .enter_s2idle = intel_idle_s2idle, }, 442 { 443 .name = "C7", 444 .desc = "MWAIT 0x30", 445 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 446 .exit_latency = 87, 447 .target_residency = 300, 448 .enter = &intel_idle, 449 .enter_s2idle = intel_idle_s2idle, }, 450 { 451 .enter = NULL } 452 }; 453 454 static struct cpuidle_state ivt_cstates[] __initdata = { 455 { 456 .name = "C1", 457 .desc = "MWAIT 0x00", 458 .flags = MWAIT2flg(0x00), 459 .exit_latency = 1, 460 .target_residency = 1, 461 .enter = &intel_idle, 462 .enter_s2idle = intel_idle_s2idle, }, 463 { 464 .name = "C1E", 465 .desc = "MWAIT 0x01", 466 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 467 .exit_latency = 10, 468 .target_residency = 80, 469 .enter = &intel_idle, 470 .enter_s2idle = intel_idle_s2idle, }, 471 { 472 .name = "C3", 473 .desc = "MWAIT 0x10", 474 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 475 .exit_latency = 59, 476 .target_residency = 156, 477 .enter = &intel_idle, 478 .enter_s2idle = intel_idle_s2idle, }, 479 { 480 .name = "C6", 481 .desc = "MWAIT 0x20", 482 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 483 .exit_latency = 82, 484 .target_residency = 300, 485 .enter = &intel_idle, 486 .enter_s2idle = intel_idle_s2idle, }, 487 { 488 .enter = NULL } 489 }; 490 491 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 492 { 493 .name = "C1", 494 .desc = "MWAIT 0x00", 495 .flags = MWAIT2flg(0x00), 496 .exit_latency = 1, 497 .target_residency = 1, 498 .enter = &intel_idle, 499 .enter_s2idle = intel_idle_s2idle, }, 500 { 501 .name = "C1E", 502 .desc = "MWAIT 0x01", 503 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 504 .exit_latency = 10, 505 .target_residency = 250, 506 .enter = &intel_idle, 507 .enter_s2idle = intel_idle_s2idle, }, 508 { 509 .name = "C3", 510 .desc = "MWAIT 0x10", 511 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 512 .exit_latency = 59, 513 .target_residency = 300, 514 .enter = &intel_idle, 515 .enter_s2idle = intel_idle_s2idle, }, 516 { 517 .name = "C6", 518 .desc = "MWAIT 0x20", 519 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 520 .exit_latency = 84, 521 .target_residency = 400, 522 .enter = &intel_idle, 523 .enter_s2idle = intel_idle_s2idle, }, 524 { 525 .enter = NULL } 526 }; 527 528 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 529 { 530 .name = "C1", 531 .desc = "MWAIT 0x00", 532 .flags = MWAIT2flg(0x00), 533 .exit_latency = 1, 534 .target_residency = 1, 535 .enter = &intel_idle, 536 .enter_s2idle = intel_idle_s2idle, }, 537 { 538 .name = "C1E", 539 .desc = "MWAIT 0x01", 540 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 541 .exit_latency = 10, 542 .target_residency = 500, 543 .enter = &intel_idle, 544 .enter_s2idle = intel_idle_s2idle, }, 545 { 546 .name = "C3", 547 .desc = "MWAIT 0x10", 548 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 549 .exit_latency = 59, 550 .target_residency = 600, 551 .enter = &intel_idle, 552 .enter_s2idle = intel_idle_s2idle, }, 553 { 554 .name = "C6", 555 .desc = "MWAIT 0x20", 556 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 557 .exit_latency = 88, 558 .target_residency = 700, 559 .enter = &intel_idle, 560 .enter_s2idle = intel_idle_s2idle, }, 561 { 562 .enter = NULL } 563 }; 564 565 static struct cpuidle_state hsw_cstates[] __initdata = { 566 { 567 .name = "C1", 568 .desc = "MWAIT 0x00", 569 .flags = MWAIT2flg(0x00), 570 .exit_latency = 2, 571 .target_residency = 2, 572 .enter = &intel_idle, 573 .enter_s2idle = intel_idle_s2idle, }, 574 { 575 .name = "C1E", 576 .desc = "MWAIT 0x01", 577 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 578 .exit_latency = 10, 579 .target_residency = 20, 580 .enter = &intel_idle, 581 .enter_s2idle = intel_idle_s2idle, }, 582 { 583 .name = "C3", 584 .desc = "MWAIT 0x10", 585 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 586 .exit_latency = 33, 587 .target_residency = 100, 588 .enter = &intel_idle, 589 .enter_s2idle = intel_idle_s2idle, }, 590 { 591 .name = "C6", 592 .desc = "MWAIT 0x20", 593 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 594 .exit_latency = 133, 595 .target_residency = 400, 596 .enter = &intel_idle, 597 .enter_s2idle = intel_idle_s2idle, }, 598 { 599 .name = "C7s", 600 .desc = "MWAIT 0x32", 601 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 602 .exit_latency = 166, 603 .target_residency = 500, 604 .enter = &intel_idle, 605 .enter_s2idle = intel_idle_s2idle, }, 606 { 607 .name = "C8", 608 .desc = "MWAIT 0x40", 609 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 610 .exit_latency = 300, 611 .target_residency = 900, 612 .enter = &intel_idle, 613 .enter_s2idle = intel_idle_s2idle, }, 614 { 615 .name = "C9", 616 .desc = "MWAIT 0x50", 617 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 618 .exit_latency = 600, 619 .target_residency = 1800, 620 .enter = &intel_idle, 621 .enter_s2idle = intel_idle_s2idle, }, 622 { 623 .name = "C10", 624 .desc = "MWAIT 0x60", 625 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 626 .exit_latency = 2600, 627 .target_residency = 7700, 628 .enter = &intel_idle, 629 .enter_s2idle = intel_idle_s2idle, }, 630 { 631 .enter = NULL } 632 }; 633 static struct cpuidle_state bdw_cstates[] __initdata = { 634 { 635 .name = "C1", 636 .desc = "MWAIT 0x00", 637 .flags = MWAIT2flg(0x00), 638 .exit_latency = 2, 639 .target_residency = 2, 640 .enter = &intel_idle, 641 .enter_s2idle = intel_idle_s2idle, }, 642 { 643 .name = "C1E", 644 .desc = "MWAIT 0x01", 645 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 646 .exit_latency = 10, 647 .target_residency = 20, 648 .enter = &intel_idle, 649 .enter_s2idle = intel_idle_s2idle, }, 650 { 651 .name = "C3", 652 .desc = "MWAIT 0x10", 653 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 654 .exit_latency = 40, 655 .target_residency = 100, 656 .enter = &intel_idle, 657 .enter_s2idle = intel_idle_s2idle, }, 658 { 659 .name = "C6", 660 .desc = "MWAIT 0x20", 661 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 662 .exit_latency = 133, 663 .target_residency = 400, 664 .enter = &intel_idle, 665 .enter_s2idle = intel_idle_s2idle, }, 666 { 667 .name = "C7s", 668 .desc = "MWAIT 0x32", 669 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 670 .exit_latency = 166, 671 .target_residency = 500, 672 .enter = &intel_idle, 673 .enter_s2idle = intel_idle_s2idle, }, 674 { 675 .name = "C8", 676 .desc = "MWAIT 0x40", 677 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 678 .exit_latency = 300, 679 .target_residency = 900, 680 .enter = &intel_idle, 681 .enter_s2idle = intel_idle_s2idle, }, 682 { 683 .name = "C9", 684 .desc = "MWAIT 0x50", 685 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 686 .exit_latency = 600, 687 .target_residency = 1800, 688 .enter = &intel_idle, 689 .enter_s2idle = intel_idle_s2idle, }, 690 { 691 .name = "C10", 692 .desc = "MWAIT 0x60", 693 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 694 .exit_latency = 2600, 695 .target_residency = 7700, 696 .enter = &intel_idle, 697 .enter_s2idle = intel_idle_s2idle, }, 698 { 699 .enter = NULL } 700 }; 701 702 static struct cpuidle_state skl_cstates[] __initdata = { 703 { 704 .name = "C1", 705 .desc = "MWAIT 0x00", 706 .flags = MWAIT2flg(0x00), 707 .exit_latency = 2, 708 .target_residency = 2, 709 .enter = &intel_idle, 710 .enter_s2idle = intel_idle_s2idle, }, 711 { 712 .name = "C1E", 713 .desc = "MWAIT 0x01", 714 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 715 .exit_latency = 10, 716 .target_residency = 20, 717 .enter = &intel_idle, 718 .enter_s2idle = intel_idle_s2idle, }, 719 { 720 .name = "C3", 721 .desc = "MWAIT 0x10", 722 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 723 .exit_latency = 70, 724 .target_residency = 100, 725 .enter = &intel_idle, 726 .enter_s2idle = intel_idle_s2idle, }, 727 { 728 .name = "C6", 729 .desc = "MWAIT 0x20", 730 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 731 .exit_latency = 85, 732 .target_residency = 200, 733 .enter = &intel_idle, 734 .enter_s2idle = intel_idle_s2idle, }, 735 { 736 .name = "C7s", 737 .desc = "MWAIT 0x33", 738 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 739 .exit_latency = 124, 740 .target_residency = 800, 741 .enter = &intel_idle, 742 .enter_s2idle = intel_idle_s2idle, }, 743 { 744 .name = "C8", 745 .desc = "MWAIT 0x40", 746 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 747 .exit_latency = 200, 748 .target_residency = 800, 749 .enter = &intel_idle, 750 .enter_s2idle = intel_idle_s2idle, }, 751 { 752 .name = "C9", 753 .desc = "MWAIT 0x50", 754 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 755 .exit_latency = 480, 756 .target_residency = 5000, 757 .enter = &intel_idle, 758 .enter_s2idle = intel_idle_s2idle, }, 759 { 760 .name = "C10", 761 .desc = "MWAIT 0x60", 762 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 763 .exit_latency = 890, 764 .target_residency = 5000, 765 .enter = &intel_idle, 766 .enter_s2idle = intel_idle_s2idle, }, 767 { 768 .enter = NULL } 769 }; 770 771 static struct cpuidle_state skx_cstates[] __initdata = { 772 { 773 .name = "C1", 774 .desc = "MWAIT 0x00", 775 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 776 .exit_latency = 2, 777 .target_residency = 2, 778 .enter = &intel_idle, 779 .enter_s2idle = intel_idle_s2idle, }, 780 { 781 .name = "C1E", 782 .desc = "MWAIT 0x01", 783 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 784 .exit_latency = 10, 785 .target_residency = 20, 786 .enter = &intel_idle, 787 .enter_s2idle = intel_idle_s2idle, }, 788 { 789 .name = "C6", 790 .desc = "MWAIT 0x20", 791 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 792 .exit_latency = 133, 793 .target_residency = 600, 794 .enter = &intel_idle, 795 .enter_s2idle = intel_idle_s2idle, }, 796 { 797 .enter = NULL } 798 }; 799 800 static struct cpuidle_state icx_cstates[] __initdata = { 801 { 802 .name = "C1", 803 .desc = "MWAIT 0x00", 804 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 805 .exit_latency = 1, 806 .target_residency = 1, 807 .enter = &intel_idle, 808 .enter_s2idle = intel_idle_s2idle, }, 809 { 810 .name = "C1E", 811 .desc = "MWAIT 0x01", 812 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 813 .exit_latency = 4, 814 .target_residency = 4, 815 .enter = &intel_idle, 816 .enter_s2idle = intel_idle_s2idle, }, 817 { 818 .name = "C6", 819 .desc = "MWAIT 0x20", 820 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 821 .exit_latency = 170, 822 .target_residency = 600, 823 .enter = &intel_idle, 824 .enter_s2idle = intel_idle_s2idle, }, 825 { 826 .enter = NULL } 827 }; 828 829 /* 830 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 831 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 832 * But in this case there is effectively no C1, because C1 requests are 833 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 834 * and C1E requests end up with C1, so there is effectively no C1E. 835 * 836 * By default we enable C1E and disable C1 by marking it with 837 * 'CPUIDLE_FLAG_UNUSABLE'. 838 */ 839 static struct cpuidle_state adl_cstates[] __initdata = { 840 { 841 .name = "C1", 842 .desc = "MWAIT 0x00", 843 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 844 .exit_latency = 1, 845 .target_residency = 1, 846 .enter = &intel_idle, 847 .enter_s2idle = intel_idle_s2idle, }, 848 { 849 .name = "C1E", 850 .desc = "MWAIT 0x01", 851 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 852 .exit_latency = 2, 853 .target_residency = 4, 854 .enter = &intel_idle, 855 .enter_s2idle = intel_idle_s2idle, }, 856 { 857 .name = "C6", 858 .desc = "MWAIT 0x20", 859 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 860 .exit_latency = 220, 861 .target_residency = 600, 862 .enter = &intel_idle, 863 .enter_s2idle = intel_idle_s2idle, }, 864 { 865 .name = "C8", 866 .desc = "MWAIT 0x40", 867 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 868 .exit_latency = 280, 869 .target_residency = 800, 870 .enter = &intel_idle, 871 .enter_s2idle = intel_idle_s2idle, }, 872 { 873 .name = "C10", 874 .desc = "MWAIT 0x60", 875 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 876 .exit_latency = 680, 877 .target_residency = 2000, 878 .enter = &intel_idle, 879 .enter_s2idle = intel_idle_s2idle, }, 880 { 881 .enter = NULL } 882 }; 883 884 static struct cpuidle_state adl_l_cstates[] __initdata = { 885 { 886 .name = "C1", 887 .desc = "MWAIT 0x00", 888 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 889 .exit_latency = 1, 890 .target_residency = 1, 891 .enter = &intel_idle, 892 .enter_s2idle = intel_idle_s2idle, }, 893 { 894 .name = "C1E", 895 .desc = "MWAIT 0x01", 896 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 897 .exit_latency = 2, 898 .target_residency = 4, 899 .enter = &intel_idle, 900 .enter_s2idle = intel_idle_s2idle, }, 901 { 902 .name = "C6", 903 .desc = "MWAIT 0x20", 904 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 905 .exit_latency = 170, 906 .target_residency = 500, 907 .enter = &intel_idle, 908 .enter_s2idle = intel_idle_s2idle, }, 909 { 910 .name = "C8", 911 .desc = "MWAIT 0x40", 912 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 913 .exit_latency = 200, 914 .target_residency = 600, 915 .enter = &intel_idle, 916 .enter_s2idle = intel_idle_s2idle, }, 917 { 918 .name = "C10", 919 .desc = "MWAIT 0x60", 920 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 921 .exit_latency = 230, 922 .target_residency = 700, 923 .enter = &intel_idle, 924 .enter_s2idle = intel_idle_s2idle, }, 925 { 926 .enter = NULL } 927 }; 928 929 static struct cpuidle_state mtl_l_cstates[] __initdata = { 930 { 931 .name = "C1E", 932 .desc = "MWAIT 0x01", 933 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 934 .exit_latency = 1, 935 .target_residency = 1, 936 .enter = &intel_idle, 937 .enter_s2idle = intel_idle_s2idle, }, 938 { 939 .name = "C6", 940 .desc = "MWAIT 0x20", 941 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 942 .exit_latency = 140, 943 .target_residency = 420, 944 .enter = &intel_idle, 945 .enter_s2idle = intel_idle_s2idle, }, 946 { 947 .name = "C10", 948 .desc = "MWAIT 0x60", 949 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 950 .exit_latency = 310, 951 .target_residency = 930, 952 .enter = &intel_idle, 953 .enter_s2idle = intel_idle_s2idle, }, 954 { 955 .enter = NULL } 956 }; 957 958 static struct cpuidle_state gmt_cstates[] __initdata = { 959 { 960 .name = "C1", 961 .desc = "MWAIT 0x00", 962 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 963 .exit_latency = 1, 964 .target_residency = 1, 965 .enter = &intel_idle, 966 .enter_s2idle = intel_idle_s2idle, }, 967 { 968 .name = "C1E", 969 .desc = "MWAIT 0x01", 970 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 971 .exit_latency = 2, 972 .target_residency = 4, 973 .enter = &intel_idle, 974 .enter_s2idle = intel_idle_s2idle, }, 975 { 976 .name = "C6", 977 .desc = "MWAIT 0x20", 978 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 979 .exit_latency = 195, 980 .target_residency = 585, 981 .enter = &intel_idle, 982 .enter_s2idle = intel_idle_s2idle, }, 983 { 984 .name = "C8", 985 .desc = "MWAIT 0x40", 986 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 987 .exit_latency = 260, 988 .target_residency = 1040, 989 .enter = &intel_idle, 990 .enter_s2idle = intel_idle_s2idle, }, 991 { 992 .name = "C10", 993 .desc = "MWAIT 0x60", 994 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 995 .exit_latency = 660, 996 .target_residency = 1980, 997 .enter = &intel_idle, 998 .enter_s2idle = intel_idle_s2idle, }, 999 { 1000 .enter = NULL } 1001 }; 1002 1003 static struct cpuidle_state spr_cstates[] __initdata = { 1004 { 1005 .name = "C1", 1006 .desc = "MWAIT 0x00", 1007 .flags = MWAIT2flg(0x00), 1008 .exit_latency = 1, 1009 .target_residency = 1, 1010 .enter = &intel_idle, 1011 .enter_s2idle = intel_idle_s2idle, }, 1012 { 1013 .name = "C1E", 1014 .desc = "MWAIT 0x01", 1015 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1016 .exit_latency = 2, 1017 .target_residency = 4, 1018 .enter = &intel_idle, 1019 .enter_s2idle = intel_idle_s2idle, }, 1020 { 1021 .name = "C6", 1022 .desc = "MWAIT 0x20", 1023 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1024 CPUIDLE_FLAG_INIT_XSTATE, 1025 .exit_latency = 290, 1026 .target_residency = 800, 1027 .enter = &intel_idle, 1028 .enter_s2idle = intel_idle_s2idle, }, 1029 { 1030 .enter = NULL } 1031 }; 1032 1033 static struct cpuidle_state gnr_cstates[] __initdata = { 1034 { 1035 .name = "C1", 1036 .desc = "MWAIT 0x00", 1037 .flags = MWAIT2flg(0x00), 1038 .exit_latency = 1, 1039 .target_residency = 1, 1040 .enter = &intel_idle, 1041 .enter_s2idle = intel_idle_s2idle, }, 1042 { 1043 .name = "C1E", 1044 .desc = "MWAIT 0x01", 1045 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1046 .exit_latency = 4, 1047 .target_residency = 4, 1048 .enter = &intel_idle, 1049 .enter_s2idle = intel_idle_s2idle, }, 1050 { 1051 .name = "C6", 1052 .desc = "MWAIT 0x20", 1053 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1054 CPUIDLE_FLAG_INIT_XSTATE | 1055 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1056 .exit_latency = 170, 1057 .target_residency = 650, 1058 .enter = &intel_idle, 1059 .enter_s2idle = intel_idle_s2idle, }, 1060 { 1061 .name = "C6P", 1062 .desc = "MWAIT 0x21", 1063 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1064 CPUIDLE_FLAG_INIT_XSTATE | 1065 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1066 .exit_latency = 210, 1067 .target_residency = 1000, 1068 .enter = &intel_idle, 1069 .enter_s2idle = intel_idle_s2idle, }, 1070 { 1071 .enter = NULL } 1072 }; 1073 1074 static struct cpuidle_state gnrd_cstates[] __initdata = { 1075 { 1076 .name = "C1", 1077 .desc = "MWAIT 0x00", 1078 .flags = MWAIT2flg(0x00), 1079 .exit_latency = 1, 1080 .target_residency = 1, 1081 .enter = &intel_idle, 1082 .enter_s2idle = intel_idle_s2idle, }, 1083 { 1084 .name = "C1E", 1085 .desc = "MWAIT 0x01", 1086 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1087 .exit_latency = 4, 1088 .target_residency = 4, 1089 .enter = &intel_idle, 1090 .enter_s2idle = intel_idle_s2idle, }, 1091 { 1092 .name = "C6", 1093 .desc = "MWAIT 0x20", 1094 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1095 CPUIDLE_FLAG_INIT_XSTATE | 1096 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1097 .exit_latency = 220, 1098 .target_residency = 650, 1099 .enter = &intel_idle, 1100 .enter_s2idle = intel_idle_s2idle, }, 1101 { 1102 .name = "C6P", 1103 .desc = "MWAIT 0x21", 1104 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1105 CPUIDLE_FLAG_INIT_XSTATE | 1106 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1107 .exit_latency = 240, 1108 .target_residency = 750, 1109 .enter = &intel_idle, 1110 .enter_s2idle = intel_idle_s2idle, }, 1111 { 1112 .enter = NULL } 1113 }; 1114 1115 static struct cpuidle_state atom_cstates[] __initdata = { 1116 { 1117 .name = "C1E", 1118 .desc = "MWAIT 0x00", 1119 .flags = MWAIT2flg(0x00), 1120 .exit_latency = 10, 1121 .target_residency = 20, 1122 .enter = &intel_idle, 1123 .enter_s2idle = intel_idle_s2idle, }, 1124 { 1125 .name = "C2", 1126 .desc = "MWAIT 0x10", 1127 .flags = MWAIT2flg(0x10), 1128 .exit_latency = 20, 1129 .target_residency = 80, 1130 .enter = &intel_idle, 1131 .enter_s2idle = intel_idle_s2idle, }, 1132 { 1133 .name = "C4", 1134 .desc = "MWAIT 0x30", 1135 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1136 .exit_latency = 100, 1137 .target_residency = 400, 1138 .enter = &intel_idle, 1139 .enter_s2idle = intel_idle_s2idle, }, 1140 { 1141 .name = "C6", 1142 .desc = "MWAIT 0x52", 1143 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1144 .exit_latency = 140, 1145 .target_residency = 560, 1146 .enter = &intel_idle, 1147 .enter_s2idle = intel_idle_s2idle, }, 1148 { 1149 .enter = NULL } 1150 }; 1151 static struct cpuidle_state tangier_cstates[] __initdata = { 1152 { 1153 .name = "C1", 1154 .desc = "MWAIT 0x00", 1155 .flags = MWAIT2flg(0x00), 1156 .exit_latency = 1, 1157 .target_residency = 4, 1158 .enter = &intel_idle, 1159 .enter_s2idle = intel_idle_s2idle, }, 1160 { 1161 .name = "C4", 1162 .desc = "MWAIT 0x30", 1163 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1164 .exit_latency = 100, 1165 .target_residency = 400, 1166 .enter = &intel_idle, 1167 .enter_s2idle = intel_idle_s2idle, }, 1168 { 1169 .name = "C6", 1170 .desc = "MWAIT 0x52", 1171 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1172 .exit_latency = 140, 1173 .target_residency = 560, 1174 .enter = &intel_idle, 1175 .enter_s2idle = intel_idle_s2idle, }, 1176 { 1177 .name = "C7", 1178 .desc = "MWAIT 0x60", 1179 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1180 .exit_latency = 1200, 1181 .target_residency = 4000, 1182 .enter = &intel_idle, 1183 .enter_s2idle = intel_idle_s2idle, }, 1184 { 1185 .name = "C9", 1186 .desc = "MWAIT 0x64", 1187 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1188 .exit_latency = 10000, 1189 .target_residency = 20000, 1190 .enter = &intel_idle, 1191 .enter_s2idle = intel_idle_s2idle, }, 1192 { 1193 .enter = NULL } 1194 }; 1195 static struct cpuidle_state avn_cstates[] __initdata = { 1196 { 1197 .name = "C1", 1198 .desc = "MWAIT 0x00", 1199 .flags = MWAIT2flg(0x00), 1200 .exit_latency = 2, 1201 .target_residency = 2, 1202 .enter = &intel_idle, 1203 .enter_s2idle = intel_idle_s2idle, }, 1204 { 1205 .name = "C6", 1206 .desc = "MWAIT 0x51", 1207 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1208 .exit_latency = 15, 1209 .target_residency = 45, 1210 .enter = &intel_idle, 1211 .enter_s2idle = intel_idle_s2idle, }, 1212 { 1213 .enter = NULL } 1214 }; 1215 static struct cpuidle_state knl_cstates[] __initdata = { 1216 { 1217 .name = "C1", 1218 .desc = "MWAIT 0x00", 1219 .flags = MWAIT2flg(0x00), 1220 .exit_latency = 1, 1221 .target_residency = 2, 1222 .enter = &intel_idle, 1223 .enter_s2idle = intel_idle_s2idle }, 1224 { 1225 .name = "C6", 1226 .desc = "MWAIT 0x10", 1227 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1228 .exit_latency = 120, 1229 .target_residency = 500, 1230 .enter = &intel_idle, 1231 .enter_s2idle = intel_idle_s2idle }, 1232 { 1233 .enter = NULL } 1234 }; 1235 1236 static struct cpuidle_state bxt_cstates[] __initdata = { 1237 { 1238 .name = "C1", 1239 .desc = "MWAIT 0x00", 1240 .flags = MWAIT2flg(0x00), 1241 .exit_latency = 2, 1242 .target_residency = 2, 1243 .enter = &intel_idle, 1244 .enter_s2idle = intel_idle_s2idle, }, 1245 { 1246 .name = "C1E", 1247 .desc = "MWAIT 0x01", 1248 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1249 .exit_latency = 10, 1250 .target_residency = 20, 1251 .enter = &intel_idle, 1252 .enter_s2idle = intel_idle_s2idle, }, 1253 { 1254 .name = "C6", 1255 .desc = "MWAIT 0x20", 1256 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1257 .exit_latency = 133, 1258 .target_residency = 133, 1259 .enter = &intel_idle, 1260 .enter_s2idle = intel_idle_s2idle, }, 1261 { 1262 .name = "C7s", 1263 .desc = "MWAIT 0x31", 1264 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1265 .exit_latency = 155, 1266 .target_residency = 155, 1267 .enter = &intel_idle, 1268 .enter_s2idle = intel_idle_s2idle, }, 1269 { 1270 .name = "C8", 1271 .desc = "MWAIT 0x40", 1272 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1273 .exit_latency = 1000, 1274 .target_residency = 1000, 1275 .enter = &intel_idle, 1276 .enter_s2idle = intel_idle_s2idle, }, 1277 { 1278 .name = "C9", 1279 .desc = "MWAIT 0x50", 1280 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1281 .exit_latency = 2000, 1282 .target_residency = 2000, 1283 .enter = &intel_idle, 1284 .enter_s2idle = intel_idle_s2idle, }, 1285 { 1286 .name = "C10", 1287 .desc = "MWAIT 0x60", 1288 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1289 .exit_latency = 10000, 1290 .target_residency = 10000, 1291 .enter = &intel_idle, 1292 .enter_s2idle = intel_idle_s2idle, }, 1293 { 1294 .enter = NULL } 1295 }; 1296 1297 static struct cpuidle_state dnv_cstates[] __initdata = { 1298 { 1299 .name = "C1", 1300 .desc = "MWAIT 0x00", 1301 .flags = MWAIT2flg(0x00), 1302 .exit_latency = 2, 1303 .target_residency = 2, 1304 .enter = &intel_idle, 1305 .enter_s2idle = intel_idle_s2idle, }, 1306 { 1307 .name = "C1E", 1308 .desc = "MWAIT 0x01", 1309 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1310 .exit_latency = 10, 1311 .target_residency = 20, 1312 .enter = &intel_idle, 1313 .enter_s2idle = intel_idle_s2idle, }, 1314 { 1315 .name = "C6", 1316 .desc = "MWAIT 0x20", 1317 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1318 .exit_latency = 50, 1319 .target_residency = 500, 1320 .enter = &intel_idle, 1321 .enter_s2idle = intel_idle_s2idle, }, 1322 { 1323 .enter = NULL } 1324 }; 1325 1326 /* 1327 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1328 * C6, and this is indicated in the CPUID mwait leaf. 1329 */ 1330 static struct cpuidle_state snr_cstates[] __initdata = { 1331 { 1332 .name = "C1", 1333 .desc = "MWAIT 0x00", 1334 .flags = MWAIT2flg(0x00), 1335 .exit_latency = 2, 1336 .target_residency = 2, 1337 .enter = &intel_idle, 1338 .enter_s2idle = intel_idle_s2idle, }, 1339 { 1340 .name = "C1E", 1341 .desc = "MWAIT 0x01", 1342 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1343 .exit_latency = 15, 1344 .target_residency = 25, 1345 .enter = &intel_idle, 1346 .enter_s2idle = intel_idle_s2idle, }, 1347 { 1348 .name = "C6", 1349 .desc = "MWAIT 0x20", 1350 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1351 .exit_latency = 130, 1352 .target_residency = 500, 1353 .enter = &intel_idle, 1354 .enter_s2idle = intel_idle_s2idle, }, 1355 { 1356 .enter = NULL } 1357 }; 1358 1359 static struct cpuidle_state grr_cstates[] __initdata = { 1360 { 1361 .name = "C1", 1362 .desc = "MWAIT 0x00", 1363 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1364 .exit_latency = 1, 1365 .target_residency = 1, 1366 .enter = &intel_idle, 1367 .enter_s2idle = intel_idle_s2idle, }, 1368 { 1369 .name = "C1E", 1370 .desc = "MWAIT 0x01", 1371 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1372 .exit_latency = 2, 1373 .target_residency = 10, 1374 .enter = &intel_idle, 1375 .enter_s2idle = intel_idle_s2idle, }, 1376 { 1377 .name = "C6S", 1378 .desc = "MWAIT 0x22", 1379 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1380 .exit_latency = 140, 1381 .target_residency = 500, 1382 .enter = &intel_idle, 1383 .enter_s2idle = intel_idle_s2idle, }, 1384 { 1385 .enter = NULL } 1386 }; 1387 1388 static struct cpuidle_state srf_cstates[] __initdata = { 1389 { 1390 .name = "C1", 1391 .desc = "MWAIT 0x00", 1392 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1393 .exit_latency = 1, 1394 .target_residency = 1, 1395 .enter = &intel_idle, 1396 .enter_s2idle = intel_idle_s2idle, }, 1397 { 1398 .name = "C1E", 1399 .desc = "MWAIT 0x01", 1400 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1401 .exit_latency = 2, 1402 .target_residency = 10, 1403 .enter = &intel_idle, 1404 .enter_s2idle = intel_idle_s2idle, }, 1405 { 1406 .name = "C6S", 1407 .desc = "MWAIT 0x22", 1408 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1409 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1410 .exit_latency = 270, 1411 .target_residency = 700, 1412 .enter = &intel_idle, 1413 .enter_s2idle = intel_idle_s2idle, }, 1414 { 1415 .name = "C6SP", 1416 .desc = "MWAIT 0x23", 1417 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1418 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1419 .exit_latency = 310, 1420 .target_residency = 900, 1421 .enter = &intel_idle, 1422 .enter_s2idle = intel_idle_s2idle, }, 1423 { 1424 .enter = NULL } 1425 }; 1426 1427 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1428 .state_table = nehalem_cstates, 1429 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1430 .disable_promotion_to_c1e = true, 1431 }; 1432 1433 static const struct idle_cpu idle_cpu_nhx __initconst = { 1434 .state_table = nehalem_cstates, 1435 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1436 .disable_promotion_to_c1e = true, 1437 .use_acpi = true, 1438 }; 1439 1440 static const struct idle_cpu idle_cpu_atom __initconst = { 1441 .state_table = atom_cstates, 1442 }; 1443 1444 static const struct idle_cpu idle_cpu_tangier __initconst = { 1445 .state_table = tangier_cstates, 1446 }; 1447 1448 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1449 .state_table = atom_cstates, 1450 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1451 }; 1452 1453 static const struct idle_cpu idle_cpu_snb __initconst = { 1454 .state_table = snb_cstates, 1455 .disable_promotion_to_c1e = true, 1456 }; 1457 1458 static const struct idle_cpu idle_cpu_snx __initconst = { 1459 .state_table = snb_cstates, 1460 .disable_promotion_to_c1e = true, 1461 .use_acpi = true, 1462 }; 1463 1464 static const struct idle_cpu idle_cpu_byt __initconst = { 1465 .state_table = byt_cstates, 1466 .disable_promotion_to_c1e = true, 1467 .byt_auto_demotion_disable_flag = true, 1468 }; 1469 1470 static const struct idle_cpu idle_cpu_cht __initconst = { 1471 .state_table = cht_cstates, 1472 .disable_promotion_to_c1e = true, 1473 .byt_auto_demotion_disable_flag = true, 1474 }; 1475 1476 static const struct idle_cpu idle_cpu_ivb __initconst = { 1477 .state_table = ivb_cstates, 1478 .disable_promotion_to_c1e = true, 1479 }; 1480 1481 static const struct idle_cpu idle_cpu_ivt __initconst = { 1482 .state_table = ivt_cstates, 1483 .disable_promotion_to_c1e = true, 1484 .use_acpi = true, 1485 }; 1486 1487 static const struct idle_cpu idle_cpu_hsw __initconst = { 1488 .state_table = hsw_cstates, 1489 .disable_promotion_to_c1e = true, 1490 }; 1491 1492 static const struct idle_cpu idle_cpu_hsx __initconst = { 1493 .state_table = hsw_cstates, 1494 .disable_promotion_to_c1e = true, 1495 .use_acpi = true, 1496 }; 1497 1498 static const struct idle_cpu idle_cpu_bdw __initconst = { 1499 .state_table = bdw_cstates, 1500 .disable_promotion_to_c1e = true, 1501 }; 1502 1503 static const struct idle_cpu idle_cpu_bdx __initconst = { 1504 .state_table = bdw_cstates, 1505 .disable_promotion_to_c1e = true, 1506 .use_acpi = true, 1507 }; 1508 1509 static const struct idle_cpu idle_cpu_skl __initconst = { 1510 .state_table = skl_cstates, 1511 .disable_promotion_to_c1e = true, 1512 }; 1513 1514 static const struct idle_cpu idle_cpu_skx __initconst = { 1515 .state_table = skx_cstates, 1516 .disable_promotion_to_c1e = true, 1517 .use_acpi = true, 1518 }; 1519 1520 static const struct idle_cpu idle_cpu_icx __initconst = { 1521 .state_table = icx_cstates, 1522 .disable_promotion_to_c1e = true, 1523 .use_acpi = true, 1524 }; 1525 1526 static const struct idle_cpu idle_cpu_adl __initconst = { 1527 .state_table = adl_cstates, 1528 }; 1529 1530 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1531 .state_table = adl_l_cstates, 1532 }; 1533 1534 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1535 .state_table = mtl_l_cstates, 1536 }; 1537 1538 static const struct idle_cpu idle_cpu_gmt __initconst = { 1539 .state_table = gmt_cstates, 1540 }; 1541 1542 static const struct idle_cpu idle_cpu_spr __initconst = { 1543 .state_table = spr_cstates, 1544 .disable_promotion_to_c1e = true, 1545 .use_acpi = true, 1546 }; 1547 1548 static const struct idle_cpu idle_cpu_gnr __initconst = { 1549 .state_table = gnr_cstates, 1550 .disable_promotion_to_c1e = true, 1551 .use_acpi = true, 1552 }; 1553 1554 static const struct idle_cpu idle_cpu_gnrd __initconst = { 1555 .state_table = gnrd_cstates, 1556 .disable_promotion_to_c1e = true, 1557 .use_acpi = true, 1558 }; 1559 1560 static const struct idle_cpu idle_cpu_avn __initconst = { 1561 .state_table = avn_cstates, 1562 .disable_promotion_to_c1e = true, 1563 .use_acpi = true, 1564 }; 1565 1566 static const struct idle_cpu idle_cpu_knl __initconst = { 1567 .state_table = knl_cstates, 1568 .use_acpi = true, 1569 }; 1570 1571 static const struct idle_cpu idle_cpu_bxt __initconst = { 1572 .state_table = bxt_cstates, 1573 .disable_promotion_to_c1e = true, 1574 }; 1575 1576 static const struct idle_cpu idle_cpu_dnv __initconst = { 1577 .state_table = dnv_cstates, 1578 .disable_promotion_to_c1e = true, 1579 .use_acpi = true, 1580 }; 1581 1582 static const struct idle_cpu idle_cpu_tmt __initconst = { 1583 .disable_promotion_to_c1e = true, 1584 }; 1585 1586 static const struct idle_cpu idle_cpu_snr __initconst = { 1587 .state_table = snr_cstates, 1588 .disable_promotion_to_c1e = true, 1589 .use_acpi = true, 1590 }; 1591 1592 static const struct idle_cpu idle_cpu_grr __initconst = { 1593 .state_table = grr_cstates, 1594 .disable_promotion_to_c1e = true, 1595 .use_acpi = true, 1596 }; 1597 1598 static const struct idle_cpu idle_cpu_srf __initconst = { 1599 .state_table = srf_cstates, 1600 .disable_promotion_to_c1e = true, 1601 .use_acpi = true, 1602 }; 1603 1604 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1605 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1606 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1607 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1608 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1609 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1610 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1611 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1612 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1613 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1614 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1615 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1616 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1617 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1618 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1619 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1620 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1621 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1622 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1623 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1624 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1625 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1626 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1627 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1628 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1629 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1630 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1631 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1632 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1633 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1634 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1635 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1636 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1637 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1638 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1639 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1640 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1641 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1642 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1643 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1644 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1645 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), 1646 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1647 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1648 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1649 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1650 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1651 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1652 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1653 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1654 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1655 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1656 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf), 1657 {} 1658 }; 1659 1660 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1661 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1662 {} 1663 }; 1664 1665 static bool __init intel_idle_max_cstate_reached(int cstate) 1666 { 1667 if (cstate + 1 > max_cstate) { 1668 pr_info("max_cstate %d reached\n", max_cstate); 1669 return true; 1670 } 1671 return false; 1672 } 1673 1674 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1675 { 1676 unsigned long eax = flg2MWAIT(state->flags); 1677 1678 if (boot_cpu_has(X86_FEATURE_ARAT)) 1679 return false; 1680 1681 /* 1682 * Switch over to one-shot tick broadcast if the target C-state 1683 * is deeper than C1. 1684 */ 1685 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1686 } 1687 1688 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1689 #include <acpi/processor.h> 1690 1691 static bool no_acpi __read_mostly; 1692 module_param(no_acpi, bool, 0444); 1693 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1694 1695 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1696 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1697 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1698 1699 static struct acpi_processor_power acpi_state_table __initdata; 1700 1701 /** 1702 * intel_idle_cst_usable - Check if the _CST information can be used. 1703 * 1704 * Check if all of the C-states listed by _CST in the max_cstate range are 1705 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1706 */ 1707 static bool __init intel_idle_cst_usable(void) 1708 { 1709 int cstate, limit; 1710 1711 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1712 acpi_state_table.count); 1713 1714 for (cstate = 1; cstate < limit; cstate++) { 1715 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1716 1717 if (cx->entry_method != ACPI_CSTATE_FFH) 1718 return false; 1719 } 1720 1721 return true; 1722 } 1723 1724 static bool __init intel_idle_acpi_cst_extract(void) 1725 { 1726 unsigned int cpu; 1727 1728 if (no_acpi) { 1729 pr_debug("Not allowed to use ACPI _CST\n"); 1730 return false; 1731 } 1732 1733 for_each_possible_cpu(cpu) { 1734 struct acpi_processor *pr = per_cpu(processors, cpu); 1735 1736 if (!pr) 1737 continue; 1738 1739 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1740 continue; 1741 1742 acpi_state_table.count++; 1743 1744 if (!intel_idle_cst_usable()) 1745 continue; 1746 1747 if (!acpi_processor_claim_cst_control()) 1748 break; 1749 1750 return true; 1751 } 1752 1753 acpi_state_table.count = 0; 1754 pr_debug("ACPI _CST not found or not usable\n"); 1755 return false; 1756 } 1757 1758 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1759 { 1760 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1761 1762 /* 1763 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1764 * the interesting states are ACPI_CSTATE_FFH. 1765 */ 1766 for (cstate = 1; cstate < limit; cstate++) { 1767 struct acpi_processor_cx *cx; 1768 struct cpuidle_state *state; 1769 1770 if (intel_idle_max_cstate_reached(cstate - 1)) 1771 break; 1772 1773 cx = &acpi_state_table.states[cstate]; 1774 1775 state = &drv->states[drv->state_count++]; 1776 1777 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1778 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1779 state->exit_latency = cx->latency; 1780 /* 1781 * For C1-type C-states use the same number for both the exit 1782 * latency and target residency, because that is the case for 1783 * C1 in the majority of the static C-states tables above. 1784 * For the other types of C-states, however, set the target 1785 * residency to 3 times the exit latency which should lead to 1786 * a reasonable balance between energy-efficiency and 1787 * performance in the majority of interesting cases. 1788 */ 1789 state->target_residency = cx->latency; 1790 if (cx->type > ACPI_STATE_C1) 1791 state->target_residency *= 3; 1792 1793 state->flags = MWAIT2flg(cx->address); 1794 if (cx->type > ACPI_STATE_C2) 1795 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1796 1797 if (disabled_states_mask & BIT(cstate)) 1798 state->flags |= CPUIDLE_FLAG_OFF; 1799 1800 if (intel_idle_state_needs_timer_stop(state)) 1801 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1802 1803 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1804 mark_tsc_unstable("TSC halts in idle"); 1805 1806 state->enter = intel_idle; 1807 state->enter_s2idle = intel_idle_s2idle; 1808 } 1809 } 1810 1811 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1812 { 1813 int cstate, limit; 1814 1815 /* 1816 * If there are no _CST C-states, do not disable any C-states by 1817 * default. 1818 */ 1819 if (!acpi_state_table.count) 1820 return false; 1821 1822 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1823 /* 1824 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1825 * the interesting states are ACPI_CSTATE_FFH. 1826 */ 1827 for (cstate = 1; cstate < limit; cstate++) { 1828 u32 acpi_hint = acpi_state_table.states[cstate].address; 1829 u32 table_hint = mwait_hint; 1830 1831 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 1832 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 1833 table_hint &= ~MWAIT_SUBSTATE_MASK; 1834 } 1835 1836 if (acpi_hint == table_hint) 1837 return false; 1838 } 1839 return true; 1840 } 1841 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1842 #define force_use_acpi (false) 1843 1844 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1845 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1846 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1847 { 1848 return false; 1849 } 1850 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1851 1852 /** 1853 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1854 * 1855 * Tune IVT multi-socket targets. 1856 * Assumption: num_sockets == (max_package_num + 1). 1857 */ 1858 static void __init ivt_idle_state_table_update(void) 1859 { 1860 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1861 int cpu, package_num, num_sockets = 1; 1862 1863 for_each_online_cpu(cpu) { 1864 package_num = topology_physical_package_id(cpu); 1865 if (package_num + 1 > num_sockets) { 1866 num_sockets = package_num + 1; 1867 1868 if (num_sockets > 4) { 1869 cpuidle_state_table = ivt_cstates_8s; 1870 return; 1871 } 1872 } 1873 } 1874 1875 if (num_sockets > 2) 1876 cpuidle_state_table = ivt_cstates_4s; 1877 1878 /* else, 1 and 2 socket systems use default ivt_cstates */ 1879 } 1880 1881 /** 1882 * irtl_2_usec - IRTL to microseconds conversion. 1883 * @irtl: IRTL MSR value. 1884 * 1885 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1886 */ 1887 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1888 { 1889 static const unsigned int irtl_ns_units[] __initconst = { 1890 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1891 }; 1892 unsigned long long ns; 1893 1894 if (!irtl) 1895 return 0; 1896 1897 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1898 1899 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1900 } 1901 1902 /** 1903 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1904 * 1905 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1906 * definitive maximum latency and use the same value for target_residency. 1907 */ 1908 static void __init bxt_idle_state_table_update(void) 1909 { 1910 unsigned long long msr; 1911 unsigned int usec; 1912 1913 rdmsrl(MSR_PKGC6_IRTL, msr); 1914 usec = irtl_2_usec(msr); 1915 if (usec) { 1916 bxt_cstates[2].exit_latency = usec; 1917 bxt_cstates[2].target_residency = usec; 1918 } 1919 1920 rdmsrl(MSR_PKGC7_IRTL, msr); 1921 usec = irtl_2_usec(msr); 1922 if (usec) { 1923 bxt_cstates[3].exit_latency = usec; 1924 bxt_cstates[3].target_residency = usec; 1925 } 1926 1927 rdmsrl(MSR_PKGC8_IRTL, msr); 1928 usec = irtl_2_usec(msr); 1929 if (usec) { 1930 bxt_cstates[4].exit_latency = usec; 1931 bxt_cstates[4].target_residency = usec; 1932 } 1933 1934 rdmsrl(MSR_PKGC9_IRTL, msr); 1935 usec = irtl_2_usec(msr); 1936 if (usec) { 1937 bxt_cstates[5].exit_latency = usec; 1938 bxt_cstates[5].target_residency = usec; 1939 } 1940 1941 rdmsrl(MSR_PKGC10_IRTL, msr); 1942 usec = irtl_2_usec(msr); 1943 if (usec) { 1944 bxt_cstates[6].exit_latency = usec; 1945 bxt_cstates[6].target_residency = usec; 1946 } 1947 1948 } 1949 1950 /** 1951 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1952 * 1953 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1954 */ 1955 static void __init sklh_idle_state_table_update(void) 1956 { 1957 unsigned long long msr; 1958 unsigned int eax, ebx, ecx, edx; 1959 1960 1961 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1962 if (max_cstate <= 7) 1963 return; 1964 1965 /* if PC10 not present in CPUID.MWAIT.EDX */ 1966 if ((mwait_substates & (0xF << 28)) == 0) 1967 return; 1968 1969 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1970 1971 /* PC10 is not enabled in PKG C-state limit */ 1972 if ((msr & 0xF) != 8) 1973 return; 1974 1975 ecx = 0; 1976 cpuid(7, &eax, &ebx, &ecx, &edx); 1977 1978 /* if SGX is present */ 1979 if (ebx & (1 << 2)) { 1980 1981 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1982 1983 /* if SGX is enabled */ 1984 if (msr & (1 << 18)) 1985 return; 1986 } 1987 1988 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1989 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1990 } 1991 1992 /** 1993 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1994 * idle states table. 1995 */ 1996 static void __init skx_idle_state_table_update(void) 1997 { 1998 unsigned long long msr; 1999 2000 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2001 2002 /* 2003 * 000b: C0/C1 (no package C-state support) 2004 * 001b: C2 2005 * 010b: C6 (non-retention) 2006 * 011b: C6 (retention) 2007 * 111b: No Package C state limits. 2008 */ 2009 if ((msr & 0x7) < 2) { 2010 /* 2011 * Uses the CC6 + PC0 latency and 3 times of 2012 * latency for target_residency if the PC6 2013 * is disabled in BIOS. This is consistent 2014 * with how intel_idle driver uses _CST 2015 * to set the target_residency. 2016 */ 2017 skx_cstates[2].exit_latency = 92; 2018 skx_cstates[2].target_residency = 276; 2019 } 2020 } 2021 2022 /** 2023 * adl_idle_state_table_update - Adjust AlderLake idle states table. 2024 */ 2025 static void __init adl_idle_state_table_update(void) 2026 { 2027 /* Check if user prefers C1 over C1E. */ 2028 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 2029 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 2030 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 2031 2032 /* Disable C1E by clearing the "C1E promotion" bit. */ 2033 c1e_promotion = C1E_PROMOTION_DISABLE; 2034 return; 2035 } 2036 2037 /* Make sure C1E is enabled by default */ 2038 c1e_promotion = C1E_PROMOTION_ENABLE; 2039 } 2040 2041 /** 2042 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 2043 */ 2044 static void __init spr_idle_state_table_update(void) 2045 { 2046 unsigned long long msr; 2047 2048 /* 2049 * By default, the C6 state assumes the worst-case scenario of package 2050 * C6. However, if PC6 is disabled, we update the numbers to match 2051 * core C6. 2052 */ 2053 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2054 2055 /* Limit value 2 and above allow for PC6. */ 2056 if ((msr & 0x7) < 2) { 2057 spr_cstates[2].exit_latency = 190; 2058 spr_cstates[2].target_residency = 600; 2059 } 2060 } 2061 2062 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2063 { 2064 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2065 MWAIT_CSTATE_MASK; 2066 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2067 MWAIT_SUBSTATE_MASK; 2068 2069 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2070 if (num_substates == 0) 2071 return false; 2072 2073 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2074 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2075 2076 return true; 2077 } 2078 2079 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2080 { 2081 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2082 /* 2083 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2084 * is not currently supported but this driver. 2085 */ 2086 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2087 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2088 state->enter = intel_idle_xstate; 2089 return; 2090 } 2091 2092 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2093 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2094 /* 2095 * IBRS mitigation requires that C-states are entered 2096 * with interrupts disabled. 2097 */ 2098 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2099 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2100 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2101 state->enter = intel_idle_ibrs; 2102 return; 2103 } 2104 2105 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2106 state->enter = intel_idle_irq; 2107 return; 2108 } 2109 2110 if (force_irq_on) { 2111 pr_info("forced intel_idle_irq for state %d\n", cstate); 2112 state->enter = intel_idle_irq; 2113 } 2114 } 2115 2116 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2117 { 2118 int cstate; 2119 2120 switch (boot_cpu_data.x86_vfm) { 2121 case INTEL_IVYBRIDGE_X: 2122 ivt_idle_state_table_update(); 2123 break; 2124 case INTEL_ATOM_GOLDMONT: 2125 case INTEL_ATOM_GOLDMONT_PLUS: 2126 bxt_idle_state_table_update(); 2127 break; 2128 case INTEL_SKYLAKE: 2129 sklh_idle_state_table_update(); 2130 break; 2131 case INTEL_SKYLAKE_X: 2132 skx_idle_state_table_update(); 2133 break; 2134 case INTEL_SAPPHIRERAPIDS_X: 2135 case INTEL_EMERALDRAPIDS_X: 2136 spr_idle_state_table_update(); 2137 break; 2138 case INTEL_ALDERLAKE: 2139 case INTEL_ALDERLAKE_L: 2140 case INTEL_ATOM_GRACEMONT: 2141 adl_idle_state_table_update(); 2142 break; 2143 } 2144 2145 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2146 struct cpuidle_state *state; 2147 unsigned int mwait_hint; 2148 2149 if (intel_idle_max_cstate_reached(cstate)) 2150 break; 2151 2152 if (!cpuidle_state_table[cstate].enter && 2153 !cpuidle_state_table[cstate].enter_s2idle) 2154 break; 2155 2156 /* If marked as unusable, skip this state. */ 2157 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2158 pr_debug("state %s is disabled\n", 2159 cpuidle_state_table[cstate].name); 2160 continue; 2161 } 2162 2163 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2164 if (!intel_idle_verify_cstate(mwait_hint)) 2165 continue; 2166 2167 /* Structure copy. */ 2168 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2169 state = &drv->states[drv->state_count]; 2170 2171 state_update_enter_method(state, cstate); 2172 2173 2174 if ((disabled_states_mask & BIT(drv->state_count)) || 2175 ((icpu->use_acpi || force_use_acpi) && 2176 intel_idle_off_by_default(state->flags, mwait_hint) && 2177 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2178 state->flags |= CPUIDLE_FLAG_OFF; 2179 2180 if (intel_idle_state_needs_timer_stop(state)) 2181 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2182 2183 drv->state_count++; 2184 } 2185 2186 if (icpu->byt_auto_demotion_disable_flag) { 2187 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2188 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2189 } 2190 } 2191 2192 /** 2193 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2194 * @drv: cpuidle driver structure to initialize. 2195 */ 2196 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2197 { 2198 cpuidle_poll_state_init(drv); 2199 2200 if (disabled_states_mask & BIT(0)) 2201 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2202 2203 drv->state_count = 1; 2204 2205 if (icpu && icpu->state_table) 2206 intel_idle_init_cstates_icpu(drv); 2207 else 2208 intel_idle_init_cstates_acpi(drv); 2209 } 2210 2211 static void auto_demotion_disable(void) 2212 { 2213 unsigned long long msr_bits; 2214 2215 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2216 msr_bits &= ~auto_demotion_disable_flags; 2217 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2218 } 2219 2220 static void c1e_promotion_enable(void) 2221 { 2222 unsigned long long msr_bits; 2223 2224 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2225 msr_bits |= 0x2; 2226 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2227 } 2228 2229 static void c1e_promotion_disable(void) 2230 { 2231 unsigned long long msr_bits; 2232 2233 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2234 msr_bits &= ~0x2; 2235 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2236 } 2237 2238 /** 2239 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2240 * @cpu: CPU to initialize. 2241 * 2242 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2243 * with the processor model flags. 2244 */ 2245 static int intel_idle_cpu_init(unsigned int cpu) 2246 { 2247 struct cpuidle_device *dev; 2248 2249 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2250 dev->cpu = cpu; 2251 2252 if (cpuidle_register_device(dev)) { 2253 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2254 return -EIO; 2255 } 2256 2257 if (auto_demotion_disable_flags) 2258 auto_demotion_disable(); 2259 2260 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2261 c1e_promotion_enable(); 2262 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2263 c1e_promotion_disable(); 2264 2265 return 0; 2266 } 2267 2268 static int intel_idle_cpu_online(unsigned int cpu) 2269 { 2270 struct cpuidle_device *dev; 2271 2272 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2273 tick_broadcast_enable(); 2274 2275 /* 2276 * Some systems can hotplug a cpu at runtime after 2277 * the kernel has booted, we have to initialize the 2278 * driver in this case 2279 */ 2280 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2281 if (!dev->registered) 2282 return intel_idle_cpu_init(cpu); 2283 2284 return 0; 2285 } 2286 2287 /** 2288 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2289 */ 2290 static void __init intel_idle_cpuidle_devices_uninit(void) 2291 { 2292 int i; 2293 2294 for_each_online_cpu(i) 2295 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2296 } 2297 2298 static int __init intel_idle_init(void) 2299 { 2300 const struct x86_cpu_id *id; 2301 unsigned int eax, ebx, ecx; 2302 int retval; 2303 2304 /* Do not load intel_idle at all for now if idle= is passed */ 2305 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2306 return -ENODEV; 2307 2308 if (max_cstate == 0) { 2309 pr_debug("disabled\n"); 2310 return -EPERM; 2311 } 2312 2313 id = x86_match_cpu(intel_idle_ids); 2314 if (id) { 2315 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2316 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2317 return -ENODEV; 2318 } 2319 } else { 2320 id = x86_match_cpu(intel_mwait_ids); 2321 if (!id) 2322 return -ENODEV; 2323 } 2324 2325 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates); 2326 2327 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2328 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2329 !mwait_substates) 2330 return -ENODEV; 2331 2332 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2333 2334 icpu = (const struct idle_cpu *)id->driver_data; 2335 if (icpu) { 2336 if (icpu->state_table) 2337 cpuidle_state_table = icpu->state_table; 2338 else if (!intel_idle_acpi_cst_extract()) 2339 return -ENODEV; 2340 2341 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2342 if (icpu->disable_promotion_to_c1e) 2343 c1e_promotion = C1E_PROMOTION_DISABLE; 2344 if (icpu->use_acpi || force_use_acpi) 2345 intel_idle_acpi_cst_extract(); 2346 } else if (!intel_idle_acpi_cst_extract()) { 2347 return -ENODEV; 2348 } 2349 2350 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2351 boot_cpu_data.x86_model); 2352 2353 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2354 if (!intel_idle_cpuidle_devices) 2355 return -ENOMEM; 2356 2357 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2358 2359 retval = cpuidle_register_driver(&intel_idle_driver); 2360 if (retval) { 2361 struct cpuidle_driver *drv = cpuidle_get_driver(); 2362 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2363 drv ? drv->name : "none"); 2364 goto init_driver_fail; 2365 } 2366 2367 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2368 intel_idle_cpu_online, NULL); 2369 if (retval < 0) 2370 goto hp_setup_fail; 2371 2372 pr_debug("Local APIC timer is reliable in %s\n", 2373 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2374 2375 return 0; 2376 2377 hp_setup_fail: 2378 intel_idle_cpuidle_devices_uninit(); 2379 cpuidle_unregister_driver(&intel_idle_driver); 2380 init_driver_fail: 2381 free_percpu(intel_idle_cpuidle_devices); 2382 return retval; 2383 2384 } 2385 device_initcall(intel_idle_init); 2386 2387 /* 2388 * We are not really modular, but we used to support that. Meaning we also 2389 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2390 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2391 * is the easiest way (currently) to continue doing that. 2392 */ 2393 module_param(max_cstate, int, 0444); 2394 /* 2395 * The positions of the bits that are set in this number are the indices of the 2396 * idle states to be disabled by default (as reflected by the names of the 2397 * corresponding idle state directories in sysfs, "state0", "state1" ... 2398 * "state<i>" ..., where <i> is the index of the given state). 2399 */ 2400 module_param_named(states_off, disabled_states_mask, uint, 0444); 2401 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2402 /* 2403 * Some platforms come with mutually exclusive C-states, so that if one is 2404 * enabled, the other C-states must not be used. Example: C1 and C1E on 2405 * Sapphire Rapids platform. This parameter allows for selecting the 2406 * preferred C-states among the groups of mutually exclusive C-states - the 2407 * selected C-states will be registered, the other C-states from the mutually 2408 * exclusive group won't be registered. If the platform has no mutually 2409 * exclusive C-states, this parameter has no effect. 2410 */ 2411 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2412 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2413 /* 2414 * Debugging option that forces the driver to enter all C-states with 2415 * interrupts enabled. Does not apply to C-states with 2416 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2417 */ 2418 module_param(force_irq_on, bool, 0444); 2419 /* 2420 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2421 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2422 */ 2423 module_param(ibrs_off, bool, 0444); 2424 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2425