1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/sched/smt.h> 51 #include <linux/notifier.h> 52 #include <linux/cpu.h> 53 #include <linux/moduleparam.h> 54 #include <asm/cpu_device_id.h> 55 #include <asm/intel-family.h> 56 #include <asm/mwait.h> 57 #include <asm/spec-ctrl.h> 58 #include <asm/fpu/api.h> 59 60 #define INTEL_IDLE_VERSION "0.5.1" 61 62 static struct cpuidle_driver intel_idle_driver = { 63 .name = "intel_idle", 64 .owner = THIS_MODULE, 65 }; 66 /* intel_idle.max_cstate=0 disables driver */ 67 static int max_cstate = CPUIDLE_STATE_MAX - 1; 68 static unsigned int disabled_states_mask __read_mostly; 69 static unsigned int preferred_states_mask __read_mostly; 70 static bool force_irq_on __read_mostly; 71 static bool ibrs_off __read_mostly; 72 73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 74 75 static unsigned long auto_demotion_disable_flags; 76 77 static enum { 78 C1E_PROMOTION_PRESERVE, 79 C1E_PROMOTION_ENABLE, 80 C1E_PROMOTION_DISABLE 81 } c1e_promotion = C1E_PROMOTION_PRESERVE; 82 83 struct idle_cpu { 84 struct cpuidle_state *state_table; 85 86 /* 87 * Hardware C-state auto-demotion may not always be optimal. 88 * Indicate which enable bits to clear here. 89 */ 90 unsigned long auto_demotion_disable_flags; 91 bool byt_auto_demotion_disable_flag; 92 bool disable_promotion_to_c1e; 93 bool use_acpi; 94 }; 95 96 static const struct idle_cpu *icpu __initdata; 97 static struct cpuidle_state *cpuidle_state_table __initdata; 98 99 static unsigned int mwait_substates __initdata; 100 101 /* 102 * Enable interrupts before entering the C-state. On some platforms and for 103 * some C-states, this may measurably decrease interrupt latency. 104 */ 105 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 106 107 /* 108 * Enable this state by default even if the ACPI _CST does not list it. 109 */ 110 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 111 112 /* 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 114 * above. 115 */ 116 #define CPUIDLE_FLAG_IBRS BIT(16) 117 118 /* 119 * Initialize large xstate for the C6-state entrance. 120 */ 121 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 122 123 /* 124 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 125 * custom tables. 126 */ 127 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 128 129 /* 130 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 131 * the C-state (top nibble) and sub-state (bottom nibble) 132 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 133 * 134 * We store the hint at the top of our "flags" for each state. 135 */ 136 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 137 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 138 139 static __always_inline int __intel_idle(struct cpuidle_device *dev, 140 struct cpuidle_driver *drv, 141 int index, bool irqoff) 142 { 143 struct cpuidle_state *state = &drv->states[index]; 144 unsigned long eax = flg2MWAIT(state->flags); 145 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 146 147 mwait_idle_with_hints(eax, ecx); 148 149 return index; 150 } 151 152 /** 153 * intel_idle - Ask the processor to enter the given idle state. 154 * @dev: cpuidle device of the target CPU. 155 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 156 * @index: Target idle state index. 157 * 158 * Use the MWAIT instruction to notify the processor that the CPU represented by 159 * @dev is idle and it can try to enter the idle state corresponding to @index. 160 * 161 * If the local APIC timer is not known to be reliable in the target idle state, 162 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 163 * 164 * Must be called under local_irq_disable(). 165 */ 166 static __cpuidle int intel_idle(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, int index) 168 { 169 return __intel_idle(dev, drv, index, true); 170 } 171 172 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 173 struct cpuidle_driver *drv, int index) 174 { 175 return __intel_idle(dev, drv, index, false); 176 } 177 178 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 179 struct cpuidle_driver *drv, int index) 180 { 181 bool smt_active = sched_smt_active(); 182 u64 spec_ctrl = spec_ctrl_current(); 183 int ret; 184 185 if (smt_active) 186 __update_spec_ctrl(0); 187 188 ret = __intel_idle(dev, drv, index, true); 189 190 if (smt_active) 191 __update_spec_ctrl(spec_ctrl); 192 193 return ret; 194 } 195 196 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 197 struct cpuidle_driver *drv, int index) 198 { 199 fpu_idle_fpregs(); 200 return __intel_idle(dev, drv, index, true); 201 } 202 203 /** 204 * intel_idle_s2idle - Ask the processor to enter the given idle state. 205 * @dev: cpuidle device of the target CPU. 206 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 207 * @index: Target idle state index. 208 * 209 * Use the MWAIT instruction to notify the processor that the CPU represented by 210 * @dev is idle and it can try to enter the idle state corresponding to @index. 211 * 212 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 213 * scheduler tick and suspended scheduler clock on the target CPU. 214 */ 215 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 216 struct cpuidle_driver *drv, int index) 217 { 218 unsigned long ecx = 1; /* break on interrupt flag */ 219 struct cpuidle_state *state = &drv->states[index]; 220 unsigned long eax = flg2MWAIT(state->flags); 221 222 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 223 fpu_idle_fpregs(); 224 225 mwait_idle_with_hints(eax, ecx); 226 227 return 0; 228 } 229 230 /* 231 * States are indexed by the cstate number, 232 * which is also the index into the MWAIT hint array. 233 * Thus C0 is a dummy. 234 */ 235 static struct cpuidle_state nehalem_cstates[] __initdata = { 236 { 237 .name = "C1", 238 .desc = "MWAIT 0x00", 239 .flags = MWAIT2flg(0x00), 240 .exit_latency = 3, 241 .target_residency = 6, 242 .enter = &intel_idle, 243 .enter_s2idle = intel_idle_s2idle, }, 244 { 245 .name = "C1E", 246 .desc = "MWAIT 0x01", 247 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 248 .exit_latency = 10, 249 .target_residency = 20, 250 .enter = &intel_idle, 251 .enter_s2idle = intel_idle_s2idle, }, 252 { 253 .name = "C3", 254 .desc = "MWAIT 0x10", 255 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 256 .exit_latency = 20, 257 .target_residency = 80, 258 .enter = &intel_idle, 259 .enter_s2idle = intel_idle_s2idle, }, 260 { 261 .name = "C6", 262 .desc = "MWAIT 0x20", 263 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 264 .exit_latency = 200, 265 .target_residency = 800, 266 .enter = &intel_idle, 267 .enter_s2idle = intel_idle_s2idle, }, 268 { 269 .enter = NULL } 270 }; 271 272 static struct cpuidle_state snb_cstates[] __initdata = { 273 { 274 .name = "C1", 275 .desc = "MWAIT 0x00", 276 .flags = MWAIT2flg(0x00), 277 .exit_latency = 2, 278 .target_residency = 2, 279 .enter = &intel_idle, 280 .enter_s2idle = intel_idle_s2idle, }, 281 { 282 .name = "C1E", 283 .desc = "MWAIT 0x01", 284 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 285 .exit_latency = 10, 286 .target_residency = 20, 287 .enter = &intel_idle, 288 .enter_s2idle = intel_idle_s2idle, }, 289 { 290 .name = "C3", 291 .desc = "MWAIT 0x10", 292 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 293 .exit_latency = 80, 294 .target_residency = 211, 295 .enter = &intel_idle, 296 .enter_s2idle = intel_idle_s2idle, }, 297 { 298 .name = "C6", 299 .desc = "MWAIT 0x20", 300 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 301 .exit_latency = 104, 302 .target_residency = 345, 303 .enter = &intel_idle, 304 .enter_s2idle = intel_idle_s2idle, }, 305 { 306 .name = "C7", 307 .desc = "MWAIT 0x30", 308 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 309 .exit_latency = 109, 310 .target_residency = 345, 311 .enter = &intel_idle, 312 .enter_s2idle = intel_idle_s2idle, }, 313 { 314 .enter = NULL } 315 }; 316 317 static struct cpuidle_state byt_cstates[] __initdata = { 318 { 319 .name = "C1", 320 .desc = "MWAIT 0x00", 321 .flags = MWAIT2flg(0x00), 322 .exit_latency = 1, 323 .target_residency = 1, 324 .enter = &intel_idle, 325 .enter_s2idle = intel_idle_s2idle, }, 326 { 327 .name = "C6N", 328 .desc = "MWAIT 0x58", 329 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 330 .exit_latency = 300, 331 .target_residency = 275, 332 .enter = &intel_idle, 333 .enter_s2idle = intel_idle_s2idle, }, 334 { 335 .name = "C6S", 336 .desc = "MWAIT 0x52", 337 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 338 .exit_latency = 500, 339 .target_residency = 560, 340 .enter = &intel_idle, 341 .enter_s2idle = intel_idle_s2idle, }, 342 { 343 .name = "C7", 344 .desc = "MWAIT 0x60", 345 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 346 .exit_latency = 1200, 347 .target_residency = 4000, 348 .enter = &intel_idle, 349 .enter_s2idle = intel_idle_s2idle, }, 350 { 351 .name = "C7S", 352 .desc = "MWAIT 0x64", 353 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 354 .exit_latency = 10000, 355 .target_residency = 20000, 356 .enter = &intel_idle, 357 .enter_s2idle = intel_idle_s2idle, }, 358 { 359 .enter = NULL } 360 }; 361 362 static struct cpuidle_state cht_cstates[] __initdata = { 363 { 364 .name = "C1", 365 .desc = "MWAIT 0x00", 366 .flags = MWAIT2flg(0x00), 367 .exit_latency = 1, 368 .target_residency = 1, 369 .enter = &intel_idle, 370 .enter_s2idle = intel_idle_s2idle, }, 371 { 372 .name = "C6N", 373 .desc = "MWAIT 0x58", 374 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 375 .exit_latency = 80, 376 .target_residency = 275, 377 .enter = &intel_idle, 378 .enter_s2idle = intel_idle_s2idle, }, 379 { 380 .name = "C6S", 381 .desc = "MWAIT 0x52", 382 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 383 .exit_latency = 200, 384 .target_residency = 560, 385 .enter = &intel_idle, 386 .enter_s2idle = intel_idle_s2idle, }, 387 { 388 .name = "C7", 389 .desc = "MWAIT 0x60", 390 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 391 .exit_latency = 1200, 392 .target_residency = 4000, 393 .enter = &intel_idle, 394 .enter_s2idle = intel_idle_s2idle, }, 395 { 396 .name = "C7S", 397 .desc = "MWAIT 0x64", 398 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 399 .exit_latency = 10000, 400 .target_residency = 20000, 401 .enter = &intel_idle, 402 .enter_s2idle = intel_idle_s2idle, }, 403 { 404 .enter = NULL } 405 }; 406 407 static struct cpuidle_state ivb_cstates[] __initdata = { 408 { 409 .name = "C1", 410 .desc = "MWAIT 0x00", 411 .flags = MWAIT2flg(0x00), 412 .exit_latency = 1, 413 .target_residency = 1, 414 .enter = &intel_idle, 415 .enter_s2idle = intel_idle_s2idle, }, 416 { 417 .name = "C1E", 418 .desc = "MWAIT 0x01", 419 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 420 .exit_latency = 10, 421 .target_residency = 20, 422 .enter = &intel_idle, 423 .enter_s2idle = intel_idle_s2idle, }, 424 { 425 .name = "C3", 426 .desc = "MWAIT 0x10", 427 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 428 .exit_latency = 59, 429 .target_residency = 156, 430 .enter = &intel_idle, 431 .enter_s2idle = intel_idle_s2idle, }, 432 { 433 .name = "C6", 434 .desc = "MWAIT 0x20", 435 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 436 .exit_latency = 80, 437 .target_residency = 300, 438 .enter = &intel_idle, 439 .enter_s2idle = intel_idle_s2idle, }, 440 { 441 .name = "C7", 442 .desc = "MWAIT 0x30", 443 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 444 .exit_latency = 87, 445 .target_residency = 300, 446 .enter = &intel_idle, 447 .enter_s2idle = intel_idle_s2idle, }, 448 { 449 .enter = NULL } 450 }; 451 452 static struct cpuidle_state ivt_cstates[] __initdata = { 453 { 454 .name = "C1", 455 .desc = "MWAIT 0x00", 456 .flags = MWAIT2flg(0x00), 457 .exit_latency = 1, 458 .target_residency = 1, 459 .enter = &intel_idle, 460 .enter_s2idle = intel_idle_s2idle, }, 461 { 462 .name = "C1E", 463 .desc = "MWAIT 0x01", 464 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 465 .exit_latency = 10, 466 .target_residency = 80, 467 .enter = &intel_idle, 468 .enter_s2idle = intel_idle_s2idle, }, 469 { 470 .name = "C3", 471 .desc = "MWAIT 0x10", 472 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 473 .exit_latency = 59, 474 .target_residency = 156, 475 .enter = &intel_idle, 476 .enter_s2idle = intel_idle_s2idle, }, 477 { 478 .name = "C6", 479 .desc = "MWAIT 0x20", 480 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 481 .exit_latency = 82, 482 .target_residency = 300, 483 .enter = &intel_idle, 484 .enter_s2idle = intel_idle_s2idle, }, 485 { 486 .enter = NULL } 487 }; 488 489 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 490 { 491 .name = "C1", 492 .desc = "MWAIT 0x00", 493 .flags = MWAIT2flg(0x00), 494 .exit_latency = 1, 495 .target_residency = 1, 496 .enter = &intel_idle, 497 .enter_s2idle = intel_idle_s2idle, }, 498 { 499 .name = "C1E", 500 .desc = "MWAIT 0x01", 501 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 502 .exit_latency = 10, 503 .target_residency = 250, 504 .enter = &intel_idle, 505 .enter_s2idle = intel_idle_s2idle, }, 506 { 507 .name = "C3", 508 .desc = "MWAIT 0x10", 509 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 510 .exit_latency = 59, 511 .target_residency = 300, 512 .enter = &intel_idle, 513 .enter_s2idle = intel_idle_s2idle, }, 514 { 515 .name = "C6", 516 .desc = "MWAIT 0x20", 517 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 518 .exit_latency = 84, 519 .target_residency = 400, 520 .enter = &intel_idle, 521 .enter_s2idle = intel_idle_s2idle, }, 522 { 523 .enter = NULL } 524 }; 525 526 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 527 { 528 .name = "C1", 529 .desc = "MWAIT 0x00", 530 .flags = MWAIT2flg(0x00), 531 .exit_latency = 1, 532 .target_residency = 1, 533 .enter = &intel_idle, 534 .enter_s2idle = intel_idle_s2idle, }, 535 { 536 .name = "C1E", 537 .desc = "MWAIT 0x01", 538 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 539 .exit_latency = 10, 540 .target_residency = 500, 541 .enter = &intel_idle, 542 .enter_s2idle = intel_idle_s2idle, }, 543 { 544 .name = "C3", 545 .desc = "MWAIT 0x10", 546 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 547 .exit_latency = 59, 548 .target_residency = 600, 549 .enter = &intel_idle, 550 .enter_s2idle = intel_idle_s2idle, }, 551 { 552 .name = "C6", 553 .desc = "MWAIT 0x20", 554 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 555 .exit_latency = 88, 556 .target_residency = 700, 557 .enter = &intel_idle, 558 .enter_s2idle = intel_idle_s2idle, }, 559 { 560 .enter = NULL } 561 }; 562 563 static struct cpuidle_state hsw_cstates[] __initdata = { 564 { 565 .name = "C1", 566 .desc = "MWAIT 0x00", 567 .flags = MWAIT2flg(0x00), 568 .exit_latency = 2, 569 .target_residency = 2, 570 .enter = &intel_idle, 571 .enter_s2idle = intel_idle_s2idle, }, 572 { 573 .name = "C1E", 574 .desc = "MWAIT 0x01", 575 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 576 .exit_latency = 10, 577 .target_residency = 20, 578 .enter = &intel_idle, 579 .enter_s2idle = intel_idle_s2idle, }, 580 { 581 .name = "C3", 582 .desc = "MWAIT 0x10", 583 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 584 .exit_latency = 33, 585 .target_residency = 100, 586 .enter = &intel_idle, 587 .enter_s2idle = intel_idle_s2idle, }, 588 { 589 .name = "C6", 590 .desc = "MWAIT 0x20", 591 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 592 .exit_latency = 133, 593 .target_residency = 400, 594 .enter = &intel_idle, 595 .enter_s2idle = intel_idle_s2idle, }, 596 { 597 .name = "C7s", 598 .desc = "MWAIT 0x32", 599 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 600 .exit_latency = 166, 601 .target_residency = 500, 602 .enter = &intel_idle, 603 .enter_s2idle = intel_idle_s2idle, }, 604 { 605 .name = "C8", 606 .desc = "MWAIT 0x40", 607 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 608 .exit_latency = 300, 609 .target_residency = 900, 610 .enter = &intel_idle, 611 .enter_s2idle = intel_idle_s2idle, }, 612 { 613 .name = "C9", 614 .desc = "MWAIT 0x50", 615 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 616 .exit_latency = 600, 617 .target_residency = 1800, 618 .enter = &intel_idle, 619 .enter_s2idle = intel_idle_s2idle, }, 620 { 621 .name = "C10", 622 .desc = "MWAIT 0x60", 623 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 624 .exit_latency = 2600, 625 .target_residency = 7700, 626 .enter = &intel_idle, 627 .enter_s2idle = intel_idle_s2idle, }, 628 { 629 .enter = NULL } 630 }; 631 static struct cpuidle_state bdw_cstates[] __initdata = { 632 { 633 .name = "C1", 634 .desc = "MWAIT 0x00", 635 .flags = MWAIT2flg(0x00), 636 .exit_latency = 2, 637 .target_residency = 2, 638 .enter = &intel_idle, 639 .enter_s2idle = intel_idle_s2idle, }, 640 { 641 .name = "C1E", 642 .desc = "MWAIT 0x01", 643 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 644 .exit_latency = 10, 645 .target_residency = 20, 646 .enter = &intel_idle, 647 .enter_s2idle = intel_idle_s2idle, }, 648 { 649 .name = "C3", 650 .desc = "MWAIT 0x10", 651 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 652 .exit_latency = 40, 653 .target_residency = 100, 654 .enter = &intel_idle, 655 .enter_s2idle = intel_idle_s2idle, }, 656 { 657 .name = "C6", 658 .desc = "MWAIT 0x20", 659 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 660 .exit_latency = 133, 661 .target_residency = 400, 662 .enter = &intel_idle, 663 .enter_s2idle = intel_idle_s2idle, }, 664 { 665 .name = "C7s", 666 .desc = "MWAIT 0x32", 667 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 668 .exit_latency = 166, 669 .target_residency = 500, 670 .enter = &intel_idle, 671 .enter_s2idle = intel_idle_s2idle, }, 672 { 673 .name = "C8", 674 .desc = "MWAIT 0x40", 675 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 676 .exit_latency = 300, 677 .target_residency = 900, 678 .enter = &intel_idle, 679 .enter_s2idle = intel_idle_s2idle, }, 680 { 681 .name = "C9", 682 .desc = "MWAIT 0x50", 683 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 684 .exit_latency = 600, 685 .target_residency = 1800, 686 .enter = &intel_idle, 687 .enter_s2idle = intel_idle_s2idle, }, 688 { 689 .name = "C10", 690 .desc = "MWAIT 0x60", 691 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 692 .exit_latency = 2600, 693 .target_residency = 7700, 694 .enter = &intel_idle, 695 .enter_s2idle = intel_idle_s2idle, }, 696 { 697 .enter = NULL } 698 }; 699 700 static struct cpuidle_state skl_cstates[] __initdata = { 701 { 702 .name = "C1", 703 .desc = "MWAIT 0x00", 704 .flags = MWAIT2flg(0x00), 705 .exit_latency = 2, 706 .target_residency = 2, 707 .enter = &intel_idle, 708 .enter_s2idle = intel_idle_s2idle, }, 709 { 710 .name = "C1E", 711 .desc = "MWAIT 0x01", 712 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 713 .exit_latency = 10, 714 .target_residency = 20, 715 .enter = &intel_idle, 716 .enter_s2idle = intel_idle_s2idle, }, 717 { 718 .name = "C3", 719 .desc = "MWAIT 0x10", 720 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 721 .exit_latency = 70, 722 .target_residency = 100, 723 .enter = &intel_idle, 724 .enter_s2idle = intel_idle_s2idle, }, 725 { 726 .name = "C6", 727 .desc = "MWAIT 0x20", 728 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 729 .exit_latency = 85, 730 .target_residency = 200, 731 .enter = &intel_idle, 732 .enter_s2idle = intel_idle_s2idle, }, 733 { 734 .name = "C7s", 735 .desc = "MWAIT 0x33", 736 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 737 .exit_latency = 124, 738 .target_residency = 800, 739 .enter = &intel_idle, 740 .enter_s2idle = intel_idle_s2idle, }, 741 { 742 .name = "C8", 743 .desc = "MWAIT 0x40", 744 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 745 .exit_latency = 200, 746 .target_residency = 800, 747 .enter = &intel_idle, 748 .enter_s2idle = intel_idle_s2idle, }, 749 { 750 .name = "C9", 751 .desc = "MWAIT 0x50", 752 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 753 .exit_latency = 480, 754 .target_residency = 5000, 755 .enter = &intel_idle, 756 .enter_s2idle = intel_idle_s2idle, }, 757 { 758 .name = "C10", 759 .desc = "MWAIT 0x60", 760 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 761 .exit_latency = 890, 762 .target_residency = 5000, 763 .enter = &intel_idle, 764 .enter_s2idle = intel_idle_s2idle, }, 765 { 766 .enter = NULL } 767 }; 768 769 static struct cpuidle_state skx_cstates[] __initdata = { 770 { 771 .name = "C1", 772 .desc = "MWAIT 0x00", 773 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 774 .exit_latency = 2, 775 .target_residency = 2, 776 .enter = &intel_idle, 777 .enter_s2idle = intel_idle_s2idle, }, 778 { 779 .name = "C1E", 780 .desc = "MWAIT 0x01", 781 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 782 .exit_latency = 10, 783 .target_residency = 20, 784 .enter = &intel_idle, 785 .enter_s2idle = intel_idle_s2idle, }, 786 { 787 .name = "C6", 788 .desc = "MWAIT 0x20", 789 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 790 .exit_latency = 133, 791 .target_residency = 600, 792 .enter = &intel_idle, 793 .enter_s2idle = intel_idle_s2idle, }, 794 { 795 .enter = NULL } 796 }; 797 798 static struct cpuidle_state icx_cstates[] __initdata = { 799 { 800 .name = "C1", 801 .desc = "MWAIT 0x00", 802 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 803 .exit_latency = 1, 804 .target_residency = 1, 805 .enter = &intel_idle, 806 .enter_s2idle = intel_idle_s2idle, }, 807 { 808 .name = "C1E", 809 .desc = "MWAIT 0x01", 810 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 811 .exit_latency = 4, 812 .target_residency = 4, 813 .enter = &intel_idle, 814 .enter_s2idle = intel_idle_s2idle, }, 815 { 816 .name = "C6", 817 .desc = "MWAIT 0x20", 818 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 819 .exit_latency = 170, 820 .target_residency = 600, 821 .enter = &intel_idle, 822 .enter_s2idle = intel_idle_s2idle, }, 823 { 824 .enter = NULL } 825 }; 826 827 /* 828 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 829 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 830 * But in this case there is effectively no C1, because C1 requests are 831 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 832 * and C1E requests end up with C1, so there is effectively no C1E. 833 * 834 * By default we enable C1E and disable C1 by marking it with 835 * 'CPUIDLE_FLAG_UNUSABLE'. 836 */ 837 static struct cpuidle_state adl_cstates[] __initdata = { 838 { 839 .name = "C1", 840 .desc = "MWAIT 0x00", 841 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 842 .exit_latency = 1, 843 .target_residency = 1, 844 .enter = &intel_idle, 845 .enter_s2idle = intel_idle_s2idle, }, 846 { 847 .name = "C1E", 848 .desc = "MWAIT 0x01", 849 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 850 .exit_latency = 2, 851 .target_residency = 4, 852 .enter = &intel_idle, 853 .enter_s2idle = intel_idle_s2idle, }, 854 { 855 .name = "C6", 856 .desc = "MWAIT 0x20", 857 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 858 .exit_latency = 220, 859 .target_residency = 600, 860 .enter = &intel_idle, 861 .enter_s2idle = intel_idle_s2idle, }, 862 { 863 .name = "C8", 864 .desc = "MWAIT 0x40", 865 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 866 .exit_latency = 280, 867 .target_residency = 800, 868 .enter = &intel_idle, 869 .enter_s2idle = intel_idle_s2idle, }, 870 { 871 .name = "C10", 872 .desc = "MWAIT 0x60", 873 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 874 .exit_latency = 680, 875 .target_residency = 2000, 876 .enter = &intel_idle, 877 .enter_s2idle = intel_idle_s2idle, }, 878 { 879 .enter = NULL } 880 }; 881 882 static struct cpuidle_state adl_l_cstates[] __initdata = { 883 { 884 .name = "C1", 885 .desc = "MWAIT 0x00", 886 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 887 .exit_latency = 1, 888 .target_residency = 1, 889 .enter = &intel_idle, 890 .enter_s2idle = intel_idle_s2idle, }, 891 { 892 .name = "C1E", 893 .desc = "MWAIT 0x01", 894 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 895 .exit_latency = 2, 896 .target_residency = 4, 897 .enter = &intel_idle, 898 .enter_s2idle = intel_idle_s2idle, }, 899 { 900 .name = "C6", 901 .desc = "MWAIT 0x20", 902 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 903 .exit_latency = 170, 904 .target_residency = 500, 905 .enter = &intel_idle, 906 .enter_s2idle = intel_idle_s2idle, }, 907 { 908 .name = "C8", 909 .desc = "MWAIT 0x40", 910 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 911 .exit_latency = 200, 912 .target_residency = 600, 913 .enter = &intel_idle, 914 .enter_s2idle = intel_idle_s2idle, }, 915 { 916 .name = "C10", 917 .desc = "MWAIT 0x60", 918 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 919 .exit_latency = 230, 920 .target_residency = 700, 921 .enter = &intel_idle, 922 .enter_s2idle = intel_idle_s2idle, }, 923 { 924 .enter = NULL } 925 }; 926 927 static struct cpuidle_state mtl_l_cstates[] __initdata = { 928 { 929 .name = "C1E", 930 .desc = "MWAIT 0x01", 931 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 932 .exit_latency = 1, 933 .target_residency = 1, 934 .enter = &intel_idle, 935 .enter_s2idle = intel_idle_s2idle, }, 936 { 937 .name = "C6", 938 .desc = "MWAIT 0x20", 939 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 940 .exit_latency = 140, 941 .target_residency = 420, 942 .enter = &intel_idle, 943 .enter_s2idle = intel_idle_s2idle, }, 944 { 945 .name = "C10", 946 .desc = "MWAIT 0x60", 947 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 948 .exit_latency = 310, 949 .target_residency = 930, 950 .enter = &intel_idle, 951 .enter_s2idle = intel_idle_s2idle, }, 952 { 953 .enter = NULL } 954 }; 955 956 static struct cpuidle_state gmt_cstates[] __initdata = { 957 { 958 .name = "C1", 959 .desc = "MWAIT 0x00", 960 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 961 .exit_latency = 1, 962 .target_residency = 1, 963 .enter = &intel_idle, 964 .enter_s2idle = intel_idle_s2idle, }, 965 { 966 .name = "C1E", 967 .desc = "MWAIT 0x01", 968 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 969 .exit_latency = 2, 970 .target_residency = 4, 971 .enter = &intel_idle, 972 .enter_s2idle = intel_idle_s2idle, }, 973 { 974 .name = "C6", 975 .desc = "MWAIT 0x20", 976 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 977 .exit_latency = 195, 978 .target_residency = 585, 979 .enter = &intel_idle, 980 .enter_s2idle = intel_idle_s2idle, }, 981 { 982 .name = "C8", 983 .desc = "MWAIT 0x40", 984 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 985 .exit_latency = 260, 986 .target_residency = 1040, 987 .enter = &intel_idle, 988 .enter_s2idle = intel_idle_s2idle, }, 989 { 990 .name = "C10", 991 .desc = "MWAIT 0x60", 992 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 993 .exit_latency = 660, 994 .target_residency = 1980, 995 .enter = &intel_idle, 996 .enter_s2idle = intel_idle_s2idle, }, 997 { 998 .enter = NULL } 999 }; 1000 1001 static struct cpuidle_state spr_cstates[] __initdata = { 1002 { 1003 .name = "C1", 1004 .desc = "MWAIT 0x00", 1005 .flags = MWAIT2flg(0x00), 1006 .exit_latency = 1, 1007 .target_residency = 1, 1008 .enter = &intel_idle, 1009 .enter_s2idle = intel_idle_s2idle, }, 1010 { 1011 .name = "C1E", 1012 .desc = "MWAIT 0x01", 1013 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1014 .exit_latency = 2, 1015 .target_residency = 4, 1016 .enter = &intel_idle, 1017 .enter_s2idle = intel_idle_s2idle, }, 1018 { 1019 .name = "C6", 1020 .desc = "MWAIT 0x20", 1021 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1022 CPUIDLE_FLAG_INIT_XSTATE, 1023 .exit_latency = 290, 1024 .target_residency = 800, 1025 .enter = &intel_idle, 1026 .enter_s2idle = intel_idle_s2idle, }, 1027 { 1028 .enter = NULL } 1029 }; 1030 1031 static struct cpuidle_state gnr_cstates[] __initdata = { 1032 { 1033 .name = "C1", 1034 .desc = "MWAIT 0x00", 1035 .flags = MWAIT2flg(0x00), 1036 .exit_latency = 1, 1037 .target_residency = 1, 1038 .enter = &intel_idle, 1039 .enter_s2idle = intel_idle_s2idle, }, 1040 { 1041 .name = "C1E", 1042 .desc = "MWAIT 0x01", 1043 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1044 .exit_latency = 4, 1045 .target_residency = 4, 1046 .enter = &intel_idle, 1047 .enter_s2idle = intel_idle_s2idle, }, 1048 { 1049 .name = "C6", 1050 .desc = "MWAIT 0x20", 1051 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1052 CPUIDLE_FLAG_INIT_XSTATE | 1053 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1054 .exit_latency = 170, 1055 .target_residency = 650, 1056 .enter = &intel_idle, 1057 .enter_s2idle = intel_idle_s2idle, }, 1058 { 1059 .name = "C6P", 1060 .desc = "MWAIT 0x21", 1061 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1062 CPUIDLE_FLAG_INIT_XSTATE | 1063 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1064 .exit_latency = 210, 1065 .target_residency = 1000, 1066 .enter = &intel_idle, 1067 .enter_s2idle = intel_idle_s2idle, }, 1068 { 1069 .enter = NULL } 1070 }; 1071 1072 static struct cpuidle_state gnrd_cstates[] __initdata = { 1073 { 1074 .name = "C1", 1075 .desc = "MWAIT 0x00", 1076 .flags = MWAIT2flg(0x00), 1077 .exit_latency = 1, 1078 .target_residency = 1, 1079 .enter = &intel_idle, 1080 .enter_s2idle = intel_idle_s2idle, }, 1081 { 1082 .name = "C1E", 1083 .desc = "MWAIT 0x01", 1084 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1085 .exit_latency = 4, 1086 .target_residency = 4, 1087 .enter = &intel_idle, 1088 .enter_s2idle = intel_idle_s2idle, }, 1089 { 1090 .name = "C6", 1091 .desc = "MWAIT 0x20", 1092 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1093 CPUIDLE_FLAG_INIT_XSTATE | 1094 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1095 .exit_latency = 220, 1096 .target_residency = 650, 1097 .enter = &intel_idle, 1098 .enter_s2idle = intel_idle_s2idle, }, 1099 { 1100 .name = "C6P", 1101 .desc = "MWAIT 0x21", 1102 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1103 CPUIDLE_FLAG_INIT_XSTATE | 1104 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1105 .exit_latency = 240, 1106 .target_residency = 750, 1107 .enter = &intel_idle, 1108 .enter_s2idle = intel_idle_s2idle, }, 1109 { 1110 .enter = NULL } 1111 }; 1112 1113 static struct cpuidle_state atom_cstates[] __initdata = { 1114 { 1115 .name = "C1E", 1116 .desc = "MWAIT 0x00", 1117 .flags = MWAIT2flg(0x00), 1118 .exit_latency = 10, 1119 .target_residency = 20, 1120 .enter = &intel_idle, 1121 .enter_s2idle = intel_idle_s2idle, }, 1122 { 1123 .name = "C2", 1124 .desc = "MWAIT 0x10", 1125 .flags = MWAIT2flg(0x10), 1126 .exit_latency = 20, 1127 .target_residency = 80, 1128 .enter = &intel_idle, 1129 .enter_s2idle = intel_idle_s2idle, }, 1130 { 1131 .name = "C4", 1132 .desc = "MWAIT 0x30", 1133 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1134 .exit_latency = 100, 1135 .target_residency = 400, 1136 .enter = &intel_idle, 1137 .enter_s2idle = intel_idle_s2idle, }, 1138 { 1139 .name = "C6", 1140 .desc = "MWAIT 0x52", 1141 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1142 .exit_latency = 140, 1143 .target_residency = 560, 1144 .enter = &intel_idle, 1145 .enter_s2idle = intel_idle_s2idle, }, 1146 { 1147 .enter = NULL } 1148 }; 1149 static struct cpuidle_state tangier_cstates[] __initdata = { 1150 { 1151 .name = "C1", 1152 .desc = "MWAIT 0x00", 1153 .flags = MWAIT2flg(0x00), 1154 .exit_latency = 1, 1155 .target_residency = 4, 1156 .enter = &intel_idle, 1157 .enter_s2idle = intel_idle_s2idle, }, 1158 { 1159 .name = "C4", 1160 .desc = "MWAIT 0x30", 1161 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1162 .exit_latency = 100, 1163 .target_residency = 400, 1164 .enter = &intel_idle, 1165 .enter_s2idle = intel_idle_s2idle, }, 1166 { 1167 .name = "C6", 1168 .desc = "MWAIT 0x52", 1169 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1170 .exit_latency = 140, 1171 .target_residency = 560, 1172 .enter = &intel_idle, 1173 .enter_s2idle = intel_idle_s2idle, }, 1174 { 1175 .name = "C7", 1176 .desc = "MWAIT 0x60", 1177 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1178 .exit_latency = 1200, 1179 .target_residency = 4000, 1180 .enter = &intel_idle, 1181 .enter_s2idle = intel_idle_s2idle, }, 1182 { 1183 .name = "C9", 1184 .desc = "MWAIT 0x64", 1185 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1186 .exit_latency = 10000, 1187 .target_residency = 20000, 1188 .enter = &intel_idle, 1189 .enter_s2idle = intel_idle_s2idle, }, 1190 { 1191 .enter = NULL } 1192 }; 1193 static struct cpuidle_state avn_cstates[] __initdata = { 1194 { 1195 .name = "C1", 1196 .desc = "MWAIT 0x00", 1197 .flags = MWAIT2flg(0x00), 1198 .exit_latency = 2, 1199 .target_residency = 2, 1200 .enter = &intel_idle, 1201 .enter_s2idle = intel_idle_s2idle, }, 1202 { 1203 .name = "C6", 1204 .desc = "MWAIT 0x51", 1205 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1206 .exit_latency = 15, 1207 .target_residency = 45, 1208 .enter = &intel_idle, 1209 .enter_s2idle = intel_idle_s2idle, }, 1210 { 1211 .enter = NULL } 1212 }; 1213 static struct cpuidle_state knl_cstates[] __initdata = { 1214 { 1215 .name = "C1", 1216 .desc = "MWAIT 0x00", 1217 .flags = MWAIT2flg(0x00), 1218 .exit_latency = 1, 1219 .target_residency = 2, 1220 .enter = &intel_idle, 1221 .enter_s2idle = intel_idle_s2idle }, 1222 { 1223 .name = "C6", 1224 .desc = "MWAIT 0x10", 1225 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1226 .exit_latency = 120, 1227 .target_residency = 500, 1228 .enter = &intel_idle, 1229 .enter_s2idle = intel_idle_s2idle }, 1230 { 1231 .enter = NULL } 1232 }; 1233 1234 static struct cpuidle_state bxt_cstates[] __initdata = { 1235 { 1236 .name = "C1", 1237 .desc = "MWAIT 0x00", 1238 .flags = MWAIT2flg(0x00), 1239 .exit_latency = 2, 1240 .target_residency = 2, 1241 .enter = &intel_idle, 1242 .enter_s2idle = intel_idle_s2idle, }, 1243 { 1244 .name = "C1E", 1245 .desc = "MWAIT 0x01", 1246 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1247 .exit_latency = 10, 1248 .target_residency = 20, 1249 .enter = &intel_idle, 1250 .enter_s2idle = intel_idle_s2idle, }, 1251 { 1252 .name = "C6", 1253 .desc = "MWAIT 0x20", 1254 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1255 .exit_latency = 133, 1256 .target_residency = 133, 1257 .enter = &intel_idle, 1258 .enter_s2idle = intel_idle_s2idle, }, 1259 { 1260 .name = "C7s", 1261 .desc = "MWAIT 0x31", 1262 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1263 .exit_latency = 155, 1264 .target_residency = 155, 1265 .enter = &intel_idle, 1266 .enter_s2idle = intel_idle_s2idle, }, 1267 { 1268 .name = "C8", 1269 .desc = "MWAIT 0x40", 1270 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1271 .exit_latency = 1000, 1272 .target_residency = 1000, 1273 .enter = &intel_idle, 1274 .enter_s2idle = intel_idle_s2idle, }, 1275 { 1276 .name = "C9", 1277 .desc = "MWAIT 0x50", 1278 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1279 .exit_latency = 2000, 1280 .target_residency = 2000, 1281 .enter = &intel_idle, 1282 .enter_s2idle = intel_idle_s2idle, }, 1283 { 1284 .name = "C10", 1285 .desc = "MWAIT 0x60", 1286 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1287 .exit_latency = 10000, 1288 .target_residency = 10000, 1289 .enter = &intel_idle, 1290 .enter_s2idle = intel_idle_s2idle, }, 1291 { 1292 .enter = NULL } 1293 }; 1294 1295 static struct cpuidle_state dnv_cstates[] __initdata = { 1296 { 1297 .name = "C1", 1298 .desc = "MWAIT 0x00", 1299 .flags = MWAIT2flg(0x00), 1300 .exit_latency = 2, 1301 .target_residency = 2, 1302 .enter = &intel_idle, 1303 .enter_s2idle = intel_idle_s2idle, }, 1304 { 1305 .name = "C1E", 1306 .desc = "MWAIT 0x01", 1307 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1308 .exit_latency = 10, 1309 .target_residency = 20, 1310 .enter = &intel_idle, 1311 .enter_s2idle = intel_idle_s2idle, }, 1312 { 1313 .name = "C6", 1314 .desc = "MWAIT 0x20", 1315 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1316 .exit_latency = 50, 1317 .target_residency = 500, 1318 .enter = &intel_idle, 1319 .enter_s2idle = intel_idle_s2idle, }, 1320 { 1321 .enter = NULL } 1322 }; 1323 1324 /* 1325 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1326 * C6, and this is indicated in the CPUID mwait leaf. 1327 */ 1328 static struct cpuidle_state snr_cstates[] __initdata = { 1329 { 1330 .name = "C1", 1331 .desc = "MWAIT 0x00", 1332 .flags = MWAIT2flg(0x00), 1333 .exit_latency = 2, 1334 .target_residency = 2, 1335 .enter = &intel_idle, 1336 .enter_s2idle = intel_idle_s2idle, }, 1337 { 1338 .name = "C1E", 1339 .desc = "MWAIT 0x01", 1340 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1341 .exit_latency = 15, 1342 .target_residency = 25, 1343 .enter = &intel_idle, 1344 .enter_s2idle = intel_idle_s2idle, }, 1345 { 1346 .name = "C6", 1347 .desc = "MWAIT 0x20", 1348 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1349 .exit_latency = 130, 1350 .target_residency = 500, 1351 .enter = &intel_idle, 1352 .enter_s2idle = intel_idle_s2idle, }, 1353 { 1354 .enter = NULL } 1355 }; 1356 1357 static struct cpuidle_state grr_cstates[] __initdata = { 1358 { 1359 .name = "C1", 1360 .desc = "MWAIT 0x00", 1361 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1362 .exit_latency = 1, 1363 .target_residency = 1, 1364 .enter = &intel_idle, 1365 .enter_s2idle = intel_idle_s2idle, }, 1366 { 1367 .name = "C1E", 1368 .desc = "MWAIT 0x01", 1369 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1370 .exit_latency = 2, 1371 .target_residency = 10, 1372 .enter = &intel_idle, 1373 .enter_s2idle = intel_idle_s2idle, }, 1374 { 1375 .name = "C6S", 1376 .desc = "MWAIT 0x22", 1377 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1378 .exit_latency = 140, 1379 .target_residency = 500, 1380 .enter = &intel_idle, 1381 .enter_s2idle = intel_idle_s2idle, }, 1382 { 1383 .enter = NULL } 1384 }; 1385 1386 static struct cpuidle_state srf_cstates[] __initdata = { 1387 { 1388 .name = "C1", 1389 .desc = "MWAIT 0x00", 1390 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1391 .exit_latency = 1, 1392 .target_residency = 1, 1393 .enter = &intel_idle, 1394 .enter_s2idle = intel_idle_s2idle, }, 1395 { 1396 .name = "C1E", 1397 .desc = "MWAIT 0x01", 1398 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1399 .exit_latency = 2, 1400 .target_residency = 10, 1401 .enter = &intel_idle, 1402 .enter_s2idle = intel_idle_s2idle, }, 1403 { 1404 .name = "C6S", 1405 .desc = "MWAIT 0x22", 1406 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1407 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1408 .exit_latency = 270, 1409 .target_residency = 700, 1410 .enter = &intel_idle, 1411 .enter_s2idle = intel_idle_s2idle, }, 1412 { 1413 .name = "C6SP", 1414 .desc = "MWAIT 0x23", 1415 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1416 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1417 .exit_latency = 310, 1418 .target_residency = 900, 1419 .enter = &intel_idle, 1420 .enter_s2idle = intel_idle_s2idle, }, 1421 { 1422 .enter = NULL } 1423 }; 1424 1425 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1426 .state_table = nehalem_cstates, 1427 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1428 .disable_promotion_to_c1e = true, 1429 }; 1430 1431 static const struct idle_cpu idle_cpu_nhx __initconst = { 1432 .state_table = nehalem_cstates, 1433 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1434 .disable_promotion_to_c1e = true, 1435 .use_acpi = true, 1436 }; 1437 1438 static const struct idle_cpu idle_cpu_atom __initconst = { 1439 .state_table = atom_cstates, 1440 }; 1441 1442 static const struct idle_cpu idle_cpu_tangier __initconst = { 1443 .state_table = tangier_cstates, 1444 }; 1445 1446 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1447 .state_table = atom_cstates, 1448 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1449 }; 1450 1451 static const struct idle_cpu idle_cpu_snb __initconst = { 1452 .state_table = snb_cstates, 1453 .disable_promotion_to_c1e = true, 1454 }; 1455 1456 static const struct idle_cpu idle_cpu_snx __initconst = { 1457 .state_table = snb_cstates, 1458 .disable_promotion_to_c1e = true, 1459 .use_acpi = true, 1460 }; 1461 1462 static const struct idle_cpu idle_cpu_byt __initconst = { 1463 .state_table = byt_cstates, 1464 .disable_promotion_to_c1e = true, 1465 .byt_auto_demotion_disable_flag = true, 1466 }; 1467 1468 static const struct idle_cpu idle_cpu_cht __initconst = { 1469 .state_table = cht_cstates, 1470 .disable_promotion_to_c1e = true, 1471 .byt_auto_demotion_disable_flag = true, 1472 }; 1473 1474 static const struct idle_cpu idle_cpu_ivb __initconst = { 1475 .state_table = ivb_cstates, 1476 .disable_promotion_to_c1e = true, 1477 }; 1478 1479 static const struct idle_cpu idle_cpu_ivt __initconst = { 1480 .state_table = ivt_cstates, 1481 .disable_promotion_to_c1e = true, 1482 .use_acpi = true, 1483 }; 1484 1485 static const struct idle_cpu idle_cpu_hsw __initconst = { 1486 .state_table = hsw_cstates, 1487 .disable_promotion_to_c1e = true, 1488 }; 1489 1490 static const struct idle_cpu idle_cpu_hsx __initconst = { 1491 .state_table = hsw_cstates, 1492 .disable_promotion_to_c1e = true, 1493 .use_acpi = true, 1494 }; 1495 1496 static const struct idle_cpu idle_cpu_bdw __initconst = { 1497 .state_table = bdw_cstates, 1498 .disable_promotion_to_c1e = true, 1499 }; 1500 1501 static const struct idle_cpu idle_cpu_bdx __initconst = { 1502 .state_table = bdw_cstates, 1503 .disable_promotion_to_c1e = true, 1504 .use_acpi = true, 1505 }; 1506 1507 static const struct idle_cpu idle_cpu_skl __initconst = { 1508 .state_table = skl_cstates, 1509 .disable_promotion_to_c1e = true, 1510 }; 1511 1512 static const struct idle_cpu idle_cpu_skx __initconst = { 1513 .state_table = skx_cstates, 1514 .disable_promotion_to_c1e = true, 1515 .use_acpi = true, 1516 }; 1517 1518 static const struct idle_cpu idle_cpu_icx __initconst = { 1519 .state_table = icx_cstates, 1520 .disable_promotion_to_c1e = true, 1521 .use_acpi = true, 1522 }; 1523 1524 static const struct idle_cpu idle_cpu_adl __initconst = { 1525 .state_table = adl_cstates, 1526 }; 1527 1528 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1529 .state_table = adl_l_cstates, 1530 }; 1531 1532 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1533 .state_table = mtl_l_cstates, 1534 }; 1535 1536 static const struct idle_cpu idle_cpu_gmt __initconst = { 1537 .state_table = gmt_cstates, 1538 }; 1539 1540 static const struct idle_cpu idle_cpu_spr __initconst = { 1541 .state_table = spr_cstates, 1542 .disable_promotion_to_c1e = true, 1543 .use_acpi = true, 1544 }; 1545 1546 static const struct idle_cpu idle_cpu_gnr __initconst = { 1547 .state_table = gnr_cstates, 1548 .disable_promotion_to_c1e = true, 1549 .use_acpi = true, 1550 }; 1551 1552 static const struct idle_cpu idle_cpu_gnrd __initconst = { 1553 .state_table = gnrd_cstates, 1554 .disable_promotion_to_c1e = true, 1555 .use_acpi = true, 1556 }; 1557 1558 static const struct idle_cpu idle_cpu_avn __initconst = { 1559 .state_table = avn_cstates, 1560 .disable_promotion_to_c1e = true, 1561 .use_acpi = true, 1562 }; 1563 1564 static const struct idle_cpu idle_cpu_knl __initconst = { 1565 .state_table = knl_cstates, 1566 .use_acpi = true, 1567 }; 1568 1569 static const struct idle_cpu idle_cpu_bxt __initconst = { 1570 .state_table = bxt_cstates, 1571 .disable_promotion_to_c1e = true, 1572 }; 1573 1574 static const struct idle_cpu idle_cpu_dnv __initconst = { 1575 .state_table = dnv_cstates, 1576 .disable_promotion_to_c1e = true, 1577 .use_acpi = true, 1578 }; 1579 1580 static const struct idle_cpu idle_cpu_tmt __initconst = { 1581 .disable_promotion_to_c1e = true, 1582 }; 1583 1584 static const struct idle_cpu idle_cpu_snr __initconst = { 1585 .state_table = snr_cstates, 1586 .disable_promotion_to_c1e = true, 1587 .use_acpi = true, 1588 }; 1589 1590 static const struct idle_cpu idle_cpu_grr __initconst = { 1591 .state_table = grr_cstates, 1592 .disable_promotion_to_c1e = true, 1593 .use_acpi = true, 1594 }; 1595 1596 static const struct idle_cpu idle_cpu_srf __initconst = { 1597 .state_table = srf_cstates, 1598 .disable_promotion_to_c1e = true, 1599 .use_acpi = true, 1600 }; 1601 1602 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1603 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1604 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1605 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1606 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1607 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1608 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1609 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1610 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1611 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1612 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1613 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1614 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1615 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1616 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1617 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1618 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1619 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1620 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1621 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1622 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1623 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1624 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1625 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1626 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1627 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1628 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1629 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1630 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1631 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1632 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1633 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1634 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1635 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1636 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1637 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1638 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1639 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1640 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1641 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1642 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1643 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), 1644 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1645 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1646 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1647 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1648 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1649 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1650 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1651 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1652 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1653 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1654 {} 1655 }; 1656 1657 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1658 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1659 {} 1660 }; 1661 1662 static bool __init intel_idle_max_cstate_reached(int cstate) 1663 { 1664 if (cstate + 1 > max_cstate) { 1665 pr_info("max_cstate %d reached\n", max_cstate); 1666 return true; 1667 } 1668 return false; 1669 } 1670 1671 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1672 { 1673 unsigned long eax = flg2MWAIT(state->flags); 1674 1675 if (boot_cpu_has(X86_FEATURE_ARAT)) 1676 return false; 1677 1678 /* 1679 * Switch over to one-shot tick broadcast if the target C-state 1680 * is deeper than C1. 1681 */ 1682 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1683 } 1684 1685 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1686 #include <acpi/processor.h> 1687 1688 static bool no_acpi __read_mostly; 1689 module_param(no_acpi, bool, 0444); 1690 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1691 1692 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1693 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1694 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1695 1696 static struct acpi_processor_power acpi_state_table __initdata; 1697 1698 /** 1699 * intel_idle_cst_usable - Check if the _CST information can be used. 1700 * 1701 * Check if all of the C-states listed by _CST in the max_cstate range are 1702 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1703 */ 1704 static bool __init intel_idle_cst_usable(void) 1705 { 1706 int cstate, limit; 1707 1708 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1709 acpi_state_table.count); 1710 1711 for (cstate = 1; cstate < limit; cstate++) { 1712 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1713 1714 if (cx->entry_method != ACPI_CSTATE_FFH) 1715 return false; 1716 } 1717 1718 return true; 1719 } 1720 1721 static bool __init intel_idle_acpi_cst_extract(void) 1722 { 1723 unsigned int cpu; 1724 1725 if (no_acpi) { 1726 pr_debug("Not allowed to use ACPI _CST\n"); 1727 return false; 1728 } 1729 1730 for_each_possible_cpu(cpu) { 1731 struct acpi_processor *pr = per_cpu(processors, cpu); 1732 1733 if (!pr) 1734 continue; 1735 1736 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1737 continue; 1738 1739 acpi_state_table.count++; 1740 1741 if (!intel_idle_cst_usable()) 1742 continue; 1743 1744 if (!acpi_processor_claim_cst_control()) 1745 break; 1746 1747 return true; 1748 } 1749 1750 acpi_state_table.count = 0; 1751 pr_debug("ACPI _CST not found or not usable\n"); 1752 return false; 1753 } 1754 1755 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1756 { 1757 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1758 1759 /* 1760 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1761 * the interesting states are ACPI_CSTATE_FFH. 1762 */ 1763 for (cstate = 1; cstate < limit; cstate++) { 1764 struct acpi_processor_cx *cx; 1765 struct cpuidle_state *state; 1766 1767 if (intel_idle_max_cstate_reached(cstate - 1)) 1768 break; 1769 1770 cx = &acpi_state_table.states[cstate]; 1771 1772 state = &drv->states[drv->state_count++]; 1773 1774 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1775 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1776 state->exit_latency = cx->latency; 1777 /* 1778 * For C1-type C-states use the same number for both the exit 1779 * latency and target residency, because that is the case for 1780 * C1 in the majority of the static C-states tables above. 1781 * For the other types of C-states, however, set the target 1782 * residency to 3 times the exit latency which should lead to 1783 * a reasonable balance between energy-efficiency and 1784 * performance in the majority of interesting cases. 1785 */ 1786 state->target_residency = cx->latency; 1787 if (cx->type > ACPI_STATE_C1) 1788 state->target_residency *= 3; 1789 1790 state->flags = MWAIT2flg(cx->address); 1791 if (cx->type > ACPI_STATE_C2) 1792 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1793 1794 if (disabled_states_mask & BIT(cstate)) 1795 state->flags |= CPUIDLE_FLAG_OFF; 1796 1797 if (intel_idle_state_needs_timer_stop(state)) 1798 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1799 1800 state->enter = intel_idle; 1801 state->enter_s2idle = intel_idle_s2idle; 1802 } 1803 } 1804 1805 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1806 { 1807 int cstate, limit; 1808 1809 /* 1810 * If there are no _CST C-states, do not disable any C-states by 1811 * default. 1812 */ 1813 if (!acpi_state_table.count) 1814 return false; 1815 1816 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1817 /* 1818 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1819 * the interesting states are ACPI_CSTATE_FFH. 1820 */ 1821 for (cstate = 1; cstate < limit; cstate++) { 1822 u32 acpi_hint = acpi_state_table.states[cstate].address; 1823 u32 table_hint = mwait_hint; 1824 1825 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 1826 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 1827 table_hint &= ~MWAIT_SUBSTATE_MASK; 1828 } 1829 1830 if (acpi_hint == table_hint) 1831 return false; 1832 } 1833 return true; 1834 } 1835 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1836 #define force_use_acpi (false) 1837 1838 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1839 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1840 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1841 { 1842 return false; 1843 } 1844 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1845 1846 /** 1847 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1848 * 1849 * Tune IVT multi-socket targets. 1850 * Assumption: num_sockets == (max_package_num + 1). 1851 */ 1852 static void __init ivt_idle_state_table_update(void) 1853 { 1854 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1855 int cpu, package_num, num_sockets = 1; 1856 1857 for_each_online_cpu(cpu) { 1858 package_num = topology_physical_package_id(cpu); 1859 if (package_num + 1 > num_sockets) { 1860 num_sockets = package_num + 1; 1861 1862 if (num_sockets > 4) { 1863 cpuidle_state_table = ivt_cstates_8s; 1864 return; 1865 } 1866 } 1867 } 1868 1869 if (num_sockets > 2) 1870 cpuidle_state_table = ivt_cstates_4s; 1871 1872 /* else, 1 and 2 socket systems use default ivt_cstates */ 1873 } 1874 1875 /** 1876 * irtl_2_usec - IRTL to microseconds conversion. 1877 * @irtl: IRTL MSR value. 1878 * 1879 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1880 */ 1881 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1882 { 1883 static const unsigned int irtl_ns_units[] __initconst = { 1884 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1885 }; 1886 unsigned long long ns; 1887 1888 if (!irtl) 1889 return 0; 1890 1891 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1892 1893 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1894 } 1895 1896 /** 1897 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1898 * 1899 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1900 * definitive maximum latency and use the same value for target_residency. 1901 */ 1902 static void __init bxt_idle_state_table_update(void) 1903 { 1904 unsigned long long msr; 1905 unsigned int usec; 1906 1907 rdmsrl(MSR_PKGC6_IRTL, msr); 1908 usec = irtl_2_usec(msr); 1909 if (usec) { 1910 bxt_cstates[2].exit_latency = usec; 1911 bxt_cstates[2].target_residency = usec; 1912 } 1913 1914 rdmsrl(MSR_PKGC7_IRTL, msr); 1915 usec = irtl_2_usec(msr); 1916 if (usec) { 1917 bxt_cstates[3].exit_latency = usec; 1918 bxt_cstates[3].target_residency = usec; 1919 } 1920 1921 rdmsrl(MSR_PKGC8_IRTL, msr); 1922 usec = irtl_2_usec(msr); 1923 if (usec) { 1924 bxt_cstates[4].exit_latency = usec; 1925 bxt_cstates[4].target_residency = usec; 1926 } 1927 1928 rdmsrl(MSR_PKGC9_IRTL, msr); 1929 usec = irtl_2_usec(msr); 1930 if (usec) { 1931 bxt_cstates[5].exit_latency = usec; 1932 bxt_cstates[5].target_residency = usec; 1933 } 1934 1935 rdmsrl(MSR_PKGC10_IRTL, msr); 1936 usec = irtl_2_usec(msr); 1937 if (usec) { 1938 bxt_cstates[6].exit_latency = usec; 1939 bxt_cstates[6].target_residency = usec; 1940 } 1941 1942 } 1943 1944 /** 1945 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1946 * 1947 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1948 */ 1949 static void __init sklh_idle_state_table_update(void) 1950 { 1951 unsigned long long msr; 1952 unsigned int eax, ebx, ecx, edx; 1953 1954 1955 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1956 if (max_cstate <= 7) 1957 return; 1958 1959 /* if PC10 not present in CPUID.MWAIT.EDX */ 1960 if ((mwait_substates & (0xF << 28)) == 0) 1961 return; 1962 1963 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1964 1965 /* PC10 is not enabled in PKG C-state limit */ 1966 if ((msr & 0xF) != 8) 1967 return; 1968 1969 ecx = 0; 1970 cpuid(7, &eax, &ebx, &ecx, &edx); 1971 1972 /* if SGX is present */ 1973 if (ebx & (1 << 2)) { 1974 1975 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1976 1977 /* if SGX is enabled */ 1978 if (msr & (1 << 18)) 1979 return; 1980 } 1981 1982 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1983 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1984 } 1985 1986 /** 1987 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1988 * idle states table. 1989 */ 1990 static void __init skx_idle_state_table_update(void) 1991 { 1992 unsigned long long msr; 1993 1994 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1995 1996 /* 1997 * 000b: C0/C1 (no package C-state support) 1998 * 001b: C2 1999 * 010b: C6 (non-retention) 2000 * 011b: C6 (retention) 2001 * 111b: No Package C state limits. 2002 */ 2003 if ((msr & 0x7) < 2) { 2004 /* 2005 * Uses the CC6 + PC0 latency and 3 times of 2006 * latency for target_residency if the PC6 2007 * is disabled in BIOS. This is consistent 2008 * with how intel_idle driver uses _CST 2009 * to set the target_residency. 2010 */ 2011 skx_cstates[2].exit_latency = 92; 2012 skx_cstates[2].target_residency = 276; 2013 } 2014 } 2015 2016 /** 2017 * adl_idle_state_table_update - Adjust AlderLake idle states table. 2018 */ 2019 static void __init adl_idle_state_table_update(void) 2020 { 2021 /* Check if user prefers C1 over C1E. */ 2022 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 2023 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 2024 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 2025 2026 /* Disable C1E by clearing the "C1E promotion" bit. */ 2027 c1e_promotion = C1E_PROMOTION_DISABLE; 2028 return; 2029 } 2030 2031 /* Make sure C1E is enabled by default */ 2032 c1e_promotion = C1E_PROMOTION_ENABLE; 2033 } 2034 2035 /** 2036 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 2037 */ 2038 static void __init spr_idle_state_table_update(void) 2039 { 2040 unsigned long long msr; 2041 2042 /* 2043 * By default, the C6 state assumes the worst-case scenario of package 2044 * C6. However, if PC6 is disabled, we update the numbers to match 2045 * core C6. 2046 */ 2047 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2048 2049 /* Limit value 2 and above allow for PC6. */ 2050 if ((msr & 0x7) < 2) { 2051 spr_cstates[2].exit_latency = 190; 2052 spr_cstates[2].target_residency = 600; 2053 } 2054 } 2055 2056 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2057 { 2058 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2059 MWAIT_CSTATE_MASK; 2060 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2061 MWAIT_SUBSTATE_MASK; 2062 2063 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2064 if (num_substates == 0) 2065 return false; 2066 2067 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2068 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2069 2070 return true; 2071 } 2072 2073 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2074 { 2075 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2076 /* 2077 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2078 * is not currently supported but this driver. 2079 */ 2080 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2081 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2082 state->enter = intel_idle_xstate; 2083 return; 2084 } 2085 2086 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2087 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2088 /* 2089 * IBRS mitigation requires that C-states are entered 2090 * with interrupts disabled. 2091 */ 2092 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2093 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2094 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2095 state->enter = intel_idle_ibrs; 2096 return; 2097 } 2098 2099 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2100 state->enter = intel_idle_irq; 2101 return; 2102 } 2103 2104 if (force_irq_on) { 2105 pr_info("forced intel_idle_irq for state %d\n", cstate); 2106 state->enter = intel_idle_irq; 2107 } 2108 } 2109 2110 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2111 { 2112 int cstate; 2113 2114 switch (boot_cpu_data.x86_vfm) { 2115 case INTEL_IVYBRIDGE_X: 2116 ivt_idle_state_table_update(); 2117 break; 2118 case INTEL_ATOM_GOLDMONT: 2119 case INTEL_ATOM_GOLDMONT_PLUS: 2120 bxt_idle_state_table_update(); 2121 break; 2122 case INTEL_SKYLAKE: 2123 sklh_idle_state_table_update(); 2124 break; 2125 case INTEL_SKYLAKE_X: 2126 skx_idle_state_table_update(); 2127 break; 2128 case INTEL_SAPPHIRERAPIDS_X: 2129 case INTEL_EMERALDRAPIDS_X: 2130 spr_idle_state_table_update(); 2131 break; 2132 case INTEL_ALDERLAKE: 2133 case INTEL_ALDERLAKE_L: 2134 case INTEL_ATOM_GRACEMONT: 2135 adl_idle_state_table_update(); 2136 break; 2137 } 2138 2139 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2140 struct cpuidle_state *state; 2141 unsigned int mwait_hint; 2142 2143 if (intel_idle_max_cstate_reached(cstate)) 2144 break; 2145 2146 if (!cpuidle_state_table[cstate].enter && 2147 !cpuidle_state_table[cstate].enter_s2idle) 2148 break; 2149 2150 /* If marked as unusable, skip this state. */ 2151 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2152 pr_debug("state %s is disabled\n", 2153 cpuidle_state_table[cstate].name); 2154 continue; 2155 } 2156 2157 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2158 if (!intel_idle_verify_cstate(mwait_hint)) 2159 continue; 2160 2161 /* Structure copy. */ 2162 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2163 state = &drv->states[drv->state_count]; 2164 2165 state_update_enter_method(state, cstate); 2166 2167 2168 if ((disabled_states_mask & BIT(drv->state_count)) || 2169 ((icpu->use_acpi || force_use_acpi) && 2170 intel_idle_off_by_default(state->flags, mwait_hint) && 2171 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2172 state->flags |= CPUIDLE_FLAG_OFF; 2173 2174 if (intel_idle_state_needs_timer_stop(state)) 2175 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2176 2177 drv->state_count++; 2178 } 2179 2180 if (icpu->byt_auto_demotion_disable_flag) { 2181 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2182 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2183 } 2184 } 2185 2186 /** 2187 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2188 * @drv: cpuidle driver structure to initialize. 2189 */ 2190 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2191 { 2192 cpuidle_poll_state_init(drv); 2193 2194 if (disabled_states_mask & BIT(0)) 2195 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2196 2197 drv->state_count = 1; 2198 2199 if (icpu && icpu->state_table) 2200 intel_idle_init_cstates_icpu(drv); 2201 else 2202 intel_idle_init_cstates_acpi(drv); 2203 } 2204 2205 static void auto_demotion_disable(void) 2206 { 2207 unsigned long long msr_bits; 2208 2209 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2210 msr_bits &= ~auto_demotion_disable_flags; 2211 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2212 } 2213 2214 static void c1e_promotion_enable(void) 2215 { 2216 unsigned long long msr_bits; 2217 2218 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2219 msr_bits |= 0x2; 2220 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2221 } 2222 2223 static void c1e_promotion_disable(void) 2224 { 2225 unsigned long long msr_bits; 2226 2227 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2228 msr_bits &= ~0x2; 2229 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2230 } 2231 2232 /** 2233 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2234 * @cpu: CPU to initialize. 2235 * 2236 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2237 * with the processor model flags. 2238 */ 2239 static int intel_idle_cpu_init(unsigned int cpu) 2240 { 2241 struct cpuidle_device *dev; 2242 2243 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2244 dev->cpu = cpu; 2245 2246 if (cpuidle_register_device(dev)) { 2247 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2248 return -EIO; 2249 } 2250 2251 if (auto_demotion_disable_flags) 2252 auto_demotion_disable(); 2253 2254 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2255 c1e_promotion_enable(); 2256 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2257 c1e_promotion_disable(); 2258 2259 return 0; 2260 } 2261 2262 static int intel_idle_cpu_online(unsigned int cpu) 2263 { 2264 struct cpuidle_device *dev; 2265 2266 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2267 tick_broadcast_enable(); 2268 2269 /* 2270 * Some systems can hotplug a cpu at runtime after 2271 * the kernel has booted, we have to initialize the 2272 * driver in this case 2273 */ 2274 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2275 if (!dev->registered) 2276 return intel_idle_cpu_init(cpu); 2277 2278 return 0; 2279 } 2280 2281 /** 2282 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2283 */ 2284 static void __init intel_idle_cpuidle_devices_uninit(void) 2285 { 2286 int i; 2287 2288 for_each_online_cpu(i) 2289 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2290 } 2291 2292 static int __init intel_idle_init(void) 2293 { 2294 const struct x86_cpu_id *id; 2295 unsigned int eax, ebx, ecx; 2296 int retval; 2297 2298 /* Do not load intel_idle at all for now if idle= is passed */ 2299 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2300 return -ENODEV; 2301 2302 if (max_cstate == 0) { 2303 pr_debug("disabled\n"); 2304 return -EPERM; 2305 } 2306 2307 id = x86_match_cpu(intel_idle_ids); 2308 if (id) { 2309 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2310 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2311 return -ENODEV; 2312 } 2313 } else { 2314 id = x86_match_cpu(intel_mwait_ids); 2315 if (!id) 2316 return -ENODEV; 2317 } 2318 2319 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 2320 return -ENODEV; 2321 2322 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 2323 2324 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2325 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2326 !mwait_substates) 2327 return -ENODEV; 2328 2329 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2330 2331 icpu = (const struct idle_cpu *)id->driver_data; 2332 if (icpu) { 2333 if (icpu->state_table) 2334 cpuidle_state_table = icpu->state_table; 2335 else if (!intel_idle_acpi_cst_extract()) 2336 return -ENODEV; 2337 2338 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2339 if (icpu->disable_promotion_to_c1e) 2340 c1e_promotion = C1E_PROMOTION_DISABLE; 2341 if (icpu->use_acpi || force_use_acpi) 2342 intel_idle_acpi_cst_extract(); 2343 } else if (!intel_idle_acpi_cst_extract()) { 2344 return -ENODEV; 2345 } 2346 2347 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2348 boot_cpu_data.x86_model); 2349 2350 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2351 if (!intel_idle_cpuidle_devices) 2352 return -ENOMEM; 2353 2354 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2355 2356 retval = cpuidle_register_driver(&intel_idle_driver); 2357 if (retval) { 2358 struct cpuidle_driver *drv = cpuidle_get_driver(); 2359 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2360 drv ? drv->name : "none"); 2361 goto init_driver_fail; 2362 } 2363 2364 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2365 intel_idle_cpu_online, NULL); 2366 if (retval < 0) 2367 goto hp_setup_fail; 2368 2369 pr_debug("Local APIC timer is reliable in %s\n", 2370 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2371 2372 return 0; 2373 2374 hp_setup_fail: 2375 intel_idle_cpuidle_devices_uninit(); 2376 cpuidle_unregister_driver(&intel_idle_driver); 2377 init_driver_fail: 2378 free_percpu(intel_idle_cpuidle_devices); 2379 return retval; 2380 2381 } 2382 device_initcall(intel_idle_init); 2383 2384 /* 2385 * We are not really modular, but we used to support that. Meaning we also 2386 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2387 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2388 * is the easiest way (currently) to continue doing that. 2389 */ 2390 module_param(max_cstate, int, 0444); 2391 /* 2392 * The positions of the bits that are set in this number are the indices of the 2393 * idle states to be disabled by default (as reflected by the names of the 2394 * corresponding idle state directories in sysfs, "state0", "state1" ... 2395 * "state<i>" ..., where <i> is the index of the given state). 2396 */ 2397 module_param_named(states_off, disabled_states_mask, uint, 0444); 2398 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2399 /* 2400 * Some platforms come with mutually exclusive C-states, so that if one is 2401 * enabled, the other C-states must not be used. Example: C1 and C1E on 2402 * Sapphire Rapids platform. This parameter allows for selecting the 2403 * preferred C-states among the groups of mutually exclusive C-states - the 2404 * selected C-states will be registered, the other C-states from the mutually 2405 * exclusive group won't be registered. If the platform has no mutually 2406 * exclusive C-states, this parameter has no effect. 2407 */ 2408 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2409 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2410 /* 2411 * Debugging option that forces the driver to enter all C-states with 2412 * interrupts enabled. Does not apply to C-states with 2413 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2414 */ 2415 module_param(force_irq_on, bool, 0444); 2416 /* 2417 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2418 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2419 */ 2420 module_param(ibrs_off, bool, 0444); 2421 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2422