1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/sched/smt.h> 51 #include <linux/notifier.h> 52 #include <linux/cpu.h> 53 #include <linux/moduleparam.h> 54 #include <asm/cpu_device_id.h> 55 #include <asm/intel-family.h> 56 #include <asm/mwait.h> 57 #include <asm/spec-ctrl.h> 58 #include <asm/fpu/api.h> 59 60 #define INTEL_IDLE_VERSION "0.5.1" 61 62 static struct cpuidle_driver intel_idle_driver = { 63 .name = "intel_idle", 64 .owner = THIS_MODULE, 65 }; 66 /* intel_idle.max_cstate=0 disables driver */ 67 static int max_cstate = CPUIDLE_STATE_MAX - 1; 68 static unsigned int disabled_states_mask __read_mostly; 69 static unsigned int preferred_states_mask __read_mostly; 70 static bool force_irq_on __read_mostly; 71 static bool ibrs_off __read_mostly; 72 73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 74 75 static unsigned long auto_demotion_disable_flags; 76 77 static enum { 78 C1E_PROMOTION_PRESERVE, 79 C1E_PROMOTION_ENABLE, 80 C1E_PROMOTION_DISABLE 81 } c1e_promotion = C1E_PROMOTION_PRESERVE; 82 83 struct idle_cpu { 84 struct cpuidle_state *state_table; 85 86 /* 87 * Hardware C-state auto-demotion may not always be optimal. 88 * Indicate which enable bits to clear here. 89 */ 90 unsigned long auto_demotion_disable_flags; 91 bool byt_auto_demotion_disable_flag; 92 bool disable_promotion_to_c1e; 93 bool use_acpi; 94 }; 95 96 static const struct idle_cpu *icpu __initdata; 97 static struct cpuidle_state *cpuidle_state_table __initdata; 98 99 static unsigned int mwait_substates __initdata; 100 101 /* 102 * Enable interrupts before entering the C-state. On some platforms and for 103 * some C-states, this may measurably decrease interrupt latency. 104 */ 105 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 106 107 /* 108 * Enable this state by default even if the ACPI _CST does not list it. 109 */ 110 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 111 112 /* 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 114 * above. 115 */ 116 #define CPUIDLE_FLAG_IBRS BIT(16) 117 118 /* 119 * Initialize large xstate for the C6-state entrance. 120 */ 121 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 122 123 /* 124 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 125 * custom tables. 126 */ 127 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 128 129 /* 130 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 131 * the C-state (top nibble) and sub-state (bottom nibble) 132 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 133 * 134 * We store the hint at the top of our "flags" for each state. 135 */ 136 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 137 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 138 139 static __always_inline int __intel_idle(struct cpuidle_device *dev, 140 struct cpuidle_driver *drv, 141 int index, bool irqoff) 142 { 143 struct cpuidle_state *state = &drv->states[index]; 144 unsigned long eax = flg2MWAIT(state->flags); 145 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 146 147 mwait_idle_with_hints(eax, ecx); 148 149 return index; 150 } 151 152 /** 153 * intel_idle - Ask the processor to enter the given idle state. 154 * @dev: cpuidle device of the target CPU. 155 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 156 * @index: Target idle state index. 157 * 158 * Use the MWAIT instruction to notify the processor that the CPU represented by 159 * @dev is idle and it can try to enter the idle state corresponding to @index. 160 * 161 * If the local APIC timer is not known to be reliable in the target idle state, 162 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 163 * 164 * Must be called under local_irq_disable(). 165 */ 166 static __cpuidle int intel_idle(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, int index) 168 { 169 return __intel_idle(dev, drv, index, true); 170 } 171 172 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 173 struct cpuidle_driver *drv, int index) 174 { 175 return __intel_idle(dev, drv, index, false); 176 } 177 178 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 179 struct cpuidle_driver *drv, int index) 180 { 181 bool smt_active = sched_smt_active(); 182 u64 spec_ctrl = spec_ctrl_current(); 183 int ret; 184 185 if (smt_active) 186 __update_spec_ctrl(0); 187 188 ret = __intel_idle(dev, drv, index, true); 189 190 if (smt_active) 191 __update_spec_ctrl(spec_ctrl); 192 193 return ret; 194 } 195 196 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 197 struct cpuidle_driver *drv, int index) 198 { 199 fpu_idle_fpregs(); 200 return __intel_idle(dev, drv, index, true); 201 } 202 203 /** 204 * intel_idle_s2idle - Ask the processor to enter the given idle state. 205 * @dev: cpuidle device of the target CPU. 206 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 207 * @index: Target idle state index. 208 * 209 * Use the MWAIT instruction to notify the processor that the CPU represented by 210 * @dev is idle and it can try to enter the idle state corresponding to @index. 211 * 212 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 213 * scheduler tick and suspended scheduler clock on the target CPU. 214 */ 215 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 216 struct cpuidle_driver *drv, int index) 217 { 218 unsigned long ecx = 1; /* break on interrupt flag */ 219 struct cpuidle_state *state = &drv->states[index]; 220 unsigned long eax = flg2MWAIT(state->flags); 221 222 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 223 fpu_idle_fpregs(); 224 225 mwait_idle_with_hints(eax, ecx); 226 227 return 0; 228 } 229 230 /* 231 * States are indexed by the cstate number, 232 * which is also the index into the MWAIT hint array. 233 * Thus C0 is a dummy. 234 */ 235 static struct cpuidle_state nehalem_cstates[] __initdata = { 236 { 237 .name = "C1", 238 .desc = "MWAIT 0x00", 239 .flags = MWAIT2flg(0x00), 240 .exit_latency = 3, 241 .target_residency = 6, 242 .enter = &intel_idle, 243 .enter_s2idle = intel_idle_s2idle, }, 244 { 245 .name = "C1E", 246 .desc = "MWAIT 0x01", 247 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 248 .exit_latency = 10, 249 .target_residency = 20, 250 .enter = &intel_idle, 251 .enter_s2idle = intel_idle_s2idle, }, 252 { 253 .name = "C3", 254 .desc = "MWAIT 0x10", 255 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 256 .exit_latency = 20, 257 .target_residency = 80, 258 .enter = &intel_idle, 259 .enter_s2idle = intel_idle_s2idle, }, 260 { 261 .name = "C6", 262 .desc = "MWAIT 0x20", 263 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 264 .exit_latency = 200, 265 .target_residency = 800, 266 .enter = &intel_idle, 267 .enter_s2idle = intel_idle_s2idle, }, 268 { 269 .enter = NULL } 270 }; 271 272 static struct cpuidle_state snb_cstates[] __initdata = { 273 { 274 .name = "C1", 275 .desc = "MWAIT 0x00", 276 .flags = MWAIT2flg(0x00), 277 .exit_latency = 2, 278 .target_residency = 2, 279 .enter = &intel_idle, 280 .enter_s2idle = intel_idle_s2idle, }, 281 { 282 .name = "C1E", 283 .desc = "MWAIT 0x01", 284 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 285 .exit_latency = 10, 286 .target_residency = 20, 287 .enter = &intel_idle, 288 .enter_s2idle = intel_idle_s2idle, }, 289 { 290 .name = "C3", 291 .desc = "MWAIT 0x10", 292 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 293 .exit_latency = 80, 294 .target_residency = 211, 295 .enter = &intel_idle, 296 .enter_s2idle = intel_idle_s2idle, }, 297 { 298 .name = "C6", 299 .desc = "MWAIT 0x20", 300 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 301 .exit_latency = 104, 302 .target_residency = 345, 303 .enter = &intel_idle, 304 .enter_s2idle = intel_idle_s2idle, }, 305 { 306 .name = "C7", 307 .desc = "MWAIT 0x30", 308 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 309 .exit_latency = 109, 310 .target_residency = 345, 311 .enter = &intel_idle, 312 .enter_s2idle = intel_idle_s2idle, }, 313 { 314 .enter = NULL } 315 }; 316 317 static struct cpuidle_state byt_cstates[] __initdata = { 318 { 319 .name = "C1", 320 .desc = "MWAIT 0x00", 321 .flags = MWAIT2flg(0x00), 322 .exit_latency = 1, 323 .target_residency = 1, 324 .enter = &intel_idle, 325 .enter_s2idle = intel_idle_s2idle, }, 326 { 327 .name = "C6N", 328 .desc = "MWAIT 0x58", 329 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 330 .exit_latency = 300, 331 .target_residency = 275, 332 .enter = &intel_idle, 333 .enter_s2idle = intel_idle_s2idle, }, 334 { 335 .name = "C6S", 336 .desc = "MWAIT 0x52", 337 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 338 .exit_latency = 500, 339 .target_residency = 560, 340 .enter = &intel_idle, 341 .enter_s2idle = intel_idle_s2idle, }, 342 { 343 .name = "C7", 344 .desc = "MWAIT 0x60", 345 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 346 .exit_latency = 1200, 347 .target_residency = 4000, 348 .enter = &intel_idle, 349 .enter_s2idle = intel_idle_s2idle, }, 350 { 351 .name = "C7S", 352 .desc = "MWAIT 0x64", 353 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 354 .exit_latency = 10000, 355 .target_residency = 20000, 356 .enter = &intel_idle, 357 .enter_s2idle = intel_idle_s2idle, }, 358 { 359 .enter = NULL } 360 }; 361 362 static struct cpuidle_state cht_cstates[] __initdata = { 363 { 364 .name = "C1", 365 .desc = "MWAIT 0x00", 366 .flags = MWAIT2flg(0x00), 367 .exit_latency = 1, 368 .target_residency = 1, 369 .enter = &intel_idle, 370 .enter_s2idle = intel_idle_s2idle, }, 371 { 372 .name = "C6N", 373 .desc = "MWAIT 0x58", 374 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 375 .exit_latency = 80, 376 .target_residency = 275, 377 .enter = &intel_idle, 378 .enter_s2idle = intel_idle_s2idle, }, 379 { 380 .name = "C6S", 381 .desc = "MWAIT 0x52", 382 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 383 .exit_latency = 200, 384 .target_residency = 560, 385 .enter = &intel_idle, 386 .enter_s2idle = intel_idle_s2idle, }, 387 { 388 .name = "C7", 389 .desc = "MWAIT 0x60", 390 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 391 .exit_latency = 1200, 392 .target_residency = 4000, 393 .enter = &intel_idle, 394 .enter_s2idle = intel_idle_s2idle, }, 395 { 396 .name = "C7S", 397 .desc = "MWAIT 0x64", 398 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 399 .exit_latency = 10000, 400 .target_residency = 20000, 401 .enter = &intel_idle, 402 .enter_s2idle = intel_idle_s2idle, }, 403 { 404 .enter = NULL } 405 }; 406 407 static struct cpuidle_state ivb_cstates[] __initdata = { 408 { 409 .name = "C1", 410 .desc = "MWAIT 0x00", 411 .flags = MWAIT2flg(0x00), 412 .exit_latency = 1, 413 .target_residency = 1, 414 .enter = &intel_idle, 415 .enter_s2idle = intel_idle_s2idle, }, 416 { 417 .name = "C1E", 418 .desc = "MWAIT 0x01", 419 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 420 .exit_latency = 10, 421 .target_residency = 20, 422 .enter = &intel_idle, 423 .enter_s2idle = intel_idle_s2idle, }, 424 { 425 .name = "C3", 426 .desc = "MWAIT 0x10", 427 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 428 .exit_latency = 59, 429 .target_residency = 156, 430 .enter = &intel_idle, 431 .enter_s2idle = intel_idle_s2idle, }, 432 { 433 .name = "C6", 434 .desc = "MWAIT 0x20", 435 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 436 .exit_latency = 80, 437 .target_residency = 300, 438 .enter = &intel_idle, 439 .enter_s2idle = intel_idle_s2idle, }, 440 { 441 .name = "C7", 442 .desc = "MWAIT 0x30", 443 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 444 .exit_latency = 87, 445 .target_residency = 300, 446 .enter = &intel_idle, 447 .enter_s2idle = intel_idle_s2idle, }, 448 { 449 .enter = NULL } 450 }; 451 452 static struct cpuidle_state ivt_cstates[] __initdata = { 453 { 454 .name = "C1", 455 .desc = "MWAIT 0x00", 456 .flags = MWAIT2flg(0x00), 457 .exit_latency = 1, 458 .target_residency = 1, 459 .enter = &intel_idle, 460 .enter_s2idle = intel_idle_s2idle, }, 461 { 462 .name = "C1E", 463 .desc = "MWAIT 0x01", 464 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 465 .exit_latency = 10, 466 .target_residency = 80, 467 .enter = &intel_idle, 468 .enter_s2idle = intel_idle_s2idle, }, 469 { 470 .name = "C3", 471 .desc = "MWAIT 0x10", 472 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 473 .exit_latency = 59, 474 .target_residency = 156, 475 .enter = &intel_idle, 476 .enter_s2idle = intel_idle_s2idle, }, 477 { 478 .name = "C6", 479 .desc = "MWAIT 0x20", 480 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 481 .exit_latency = 82, 482 .target_residency = 300, 483 .enter = &intel_idle, 484 .enter_s2idle = intel_idle_s2idle, }, 485 { 486 .enter = NULL } 487 }; 488 489 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 490 { 491 .name = "C1", 492 .desc = "MWAIT 0x00", 493 .flags = MWAIT2flg(0x00), 494 .exit_latency = 1, 495 .target_residency = 1, 496 .enter = &intel_idle, 497 .enter_s2idle = intel_idle_s2idle, }, 498 { 499 .name = "C1E", 500 .desc = "MWAIT 0x01", 501 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 502 .exit_latency = 10, 503 .target_residency = 250, 504 .enter = &intel_idle, 505 .enter_s2idle = intel_idle_s2idle, }, 506 { 507 .name = "C3", 508 .desc = "MWAIT 0x10", 509 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 510 .exit_latency = 59, 511 .target_residency = 300, 512 .enter = &intel_idle, 513 .enter_s2idle = intel_idle_s2idle, }, 514 { 515 .name = "C6", 516 .desc = "MWAIT 0x20", 517 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 518 .exit_latency = 84, 519 .target_residency = 400, 520 .enter = &intel_idle, 521 .enter_s2idle = intel_idle_s2idle, }, 522 { 523 .enter = NULL } 524 }; 525 526 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 527 { 528 .name = "C1", 529 .desc = "MWAIT 0x00", 530 .flags = MWAIT2flg(0x00), 531 .exit_latency = 1, 532 .target_residency = 1, 533 .enter = &intel_idle, 534 .enter_s2idle = intel_idle_s2idle, }, 535 { 536 .name = "C1E", 537 .desc = "MWAIT 0x01", 538 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 539 .exit_latency = 10, 540 .target_residency = 500, 541 .enter = &intel_idle, 542 .enter_s2idle = intel_idle_s2idle, }, 543 { 544 .name = "C3", 545 .desc = "MWAIT 0x10", 546 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 547 .exit_latency = 59, 548 .target_residency = 600, 549 .enter = &intel_idle, 550 .enter_s2idle = intel_idle_s2idle, }, 551 { 552 .name = "C6", 553 .desc = "MWAIT 0x20", 554 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 555 .exit_latency = 88, 556 .target_residency = 700, 557 .enter = &intel_idle, 558 .enter_s2idle = intel_idle_s2idle, }, 559 { 560 .enter = NULL } 561 }; 562 563 static struct cpuidle_state hsw_cstates[] __initdata = { 564 { 565 .name = "C1", 566 .desc = "MWAIT 0x00", 567 .flags = MWAIT2flg(0x00), 568 .exit_latency = 2, 569 .target_residency = 2, 570 .enter = &intel_idle, 571 .enter_s2idle = intel_idle_s2idle, }, 572 { 573 .name = "C1E", 574 .desc = "MWAIT 0x01", 575 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 576 .exit_latency = 10, 577 .target_residency = 20, 578 .enter = &intel_idle, 579 .enter_s2idle = intel_idle_s2idle, }, 580 { 581 .name = "C3", 582 .desc = "MWAIT 0x10", 583 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 584 .exit_latency = 33, 585 .target_residency = 100, 586 .enter = &intel_idle, 587 .enter_s2idle = intel_idle_s2idle, }, 588 { 589 .name = "C6", 590 .desc = "MWAIT 0x20", 591 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 592 .exit_latency = 133, 593 .target_residency = 400, 594 .enter = &intel_idle, 595 .enter_s2idle = intel_idle_s2idle, }, 596 { 597 .name = "C7s", 598 .desc = "MWAIT 0x32", 599 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 600 .exit_latency = 166, 601 .target_residency = 500, 602 .enter = &intel_idle, 603 .enter_s2idle = intel_idle_s2idle, }, 604 { 605 .name = "C8", 606 .desc = "MWAIT 0x40", 607 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 608 .exit_latency = 300, 609 .target_residency = 900, 610 .enter = &intel_idle, 611 .enter_s2idle = intel_idle_s2idle, }, 612 { 613 .name = "C9", 614 .desc = "MWAIT 0x50", 615 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 616 .exit_latency = 600, 617 .target_residency = 1800, 618 .enter = &intel_idle, 619 .enter_s2idle = intel_idle_s2idle, }, 620 { 621 .name = "C10", 622 .desc = "MWAIT 0x60", 623 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 624 .exit_latency = 2600, 625 .target_residency = 7700, 626 .enter = &intel_idle, 627 .enter_s2idle = intel_idle_s2idle, }, 628 { 629 .enter = NULL } 630 }; 631 static struct cpuidle_state bdw_cstates[] __initdata = { 632 { 633 .name = "C1", 634 .desc = "MWAIT 0x00", 635 .flags = MWAIT2flg(0x00), 636 .exit_latency = 2, 637 .target_residency = 2, 638 .enter = &intel_idle, 639 .enter_s2idle = intel_idle_s2idle, }, 640 { 641 .name = "C1E", 642 .desc = "MWAIT 0x01", 643 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 644 .exit_latency = 10, 645 .target_residency = 20, 646 .enter = &intel_idle, 647 .enter_s2idle = intel_idle_s2idle, }, 648 { 649 .name = "C3", 650 .desc = "MWAIT 0x10", 651 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 652 .exit_latency = 40, 653 .target_residency = 100, 654 .enter = &intel_idle, 655 .enter_s2idle = intel_idle_s2idle, }, 656 { 657 .name = "C6", 658 .desc = "MWAIT 0x20", 659 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 660 .exit_latency = 133, 661 .target_residency = 400, 662 .enter = &intel_idle, 663 .enter_s2idle = intel_idle_s2idle, }, 664 { 665 .name = "C7s", 666 .desc = "MWAIT 0x32", 667 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 668 .exit_latency = 166, 669 .target_residency = 500, 670 .enter = &intel_idle, 671 .enter_s2idle = intel_idle_s2idle, }, 672 { 673 .name = "C8", 674 .desc = "MWAIT 0x40", 675 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 676 .exit_latency = 300, 677 .target_residency = 900, 678 .enter = &intel_idle, 679 .enter_s2idle = intel_idle_s2idle, }, 680 { 681 .name = "C9", 682 .desc = "MWAIT 0x50", 683 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 684 .exit_latency = 600, 685 .target_residency = 1800, 686 .enter = &intel_idle, 687 .enter_s2idle = intel_idle_s2idle, }, 688 { 689 .name = "C10", 690 .desc = "MWAIT 0x60", 691 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 692 .exit_latency = 2600, 693 .target_residency = 7700, 694 .enter = &intel_idle, 695 .enter_s2idle = intel_idle_s2idle, }, 696 { 697 .enter = NULL } 698 }; 699 700 static struct cpuidle_state skl_cstates[] __initdata = { 701 { 702 .name = "C1", 703 .desc = "MWAIT 0x00", 704 .flags = MWAIT2flg(0x00), 705 .exit_latency = 2, 706 .target_residency = 2, 707 .enter = &intel_idle, 708 .enter_s2idle = intel_idle_s2idle, }, 709 { 710 .name = "C1E", 711 .desc = "MWAIT 0x01", 712 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 713 .exit_latency = 10, 714 .target_residency = 20, 715 .enter = &intel_idle, 716 .enter_s2idle = intel_idle_s2idle, }, 717 { 718 .name = "C3", 719 .desc = "MWAIT 0x10", 720 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 721 .exit_latency = 70, 722 .target_residency = 100, 723 .enter = &intel_idle, 724 .enter_s2idle = intel_idle_s2idle, }, 725 { 726 .name = "C6", 727 .desc = "MWAIT 0x20", 728 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 729 .exit_latency = 85, 730 .target_residency = 200, 731 .enter = &intel_idle, 732 .enter_s2idle = intel_idle_s2idle, }, 733 { 734 .name = "C7s", 735 .desc = "MWAIT 0x33", 736 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 737 .exit_latency = 124, 738 .target_residency = 800, 739 .enter = &intel_idle, 740 .enter_s2idle = intel_idle_s2idle, }, 741 { 742 .name = "C8", 743 .desc = "MWAIT 0x40", 744 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 745 .exit_latency = 200, 746 .target_residency = 800, 747 .enter = &intel_idle, 748 .enter_s2idle = intel_idle_s2idle, }, 749 { 750 .name = "C9", 751 .desc = "MWAIT 0x50", 752 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 753 .exit_latency = 480, 754 .target_residency = 5000, 755 .enter = &intel_idle, 756 .enter_s2idle = intel_idle_s2idle, }, 757 { 758 .name = "C10", 759 .desc = "MWAIT 0x60", 760 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 761 .exit_latency = 890, 762 .target_residency = 5000, 763 .enter = &intel_idle, 764 .enter_s2idle = intel_idle_s2idle, }, 765 { 766 .enter = NULL } 767 }; 768 769 static struct cpuidle_state skx_cstates[] __initdata = { 770 { 771 .name = "C1", 772 .desc = "MWAIT 0x00", 773 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 774 .exit_latency = 2, 775 .target_residency = 2, 776 .enter = &intel_idle, 777 .enter_s2idle = intel_idle_s2idle, }, 778 { 779 .name = "C1E", 780 .desc = "MWAIT 0x01", 781 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 782 .exit_latency = 10, 783 .target_residency = 20, 784 .enter = &intel_idle, 785 .enter_s2idle = intel_idle_s2idle, }, 786 { 787 .name = "C6", 788 .desc = "MWAIT 0x20", 789 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 790 .exit_latency = 133, 791 .target_residency = 600, 792 .enter = &intel_idle, 793 .enter_s2idle = intel_idle_s2idle, }, 794 { 795 .enter = NULL } 796 }; 797 798 static struct cpuidle_state icx_cstates[] __initdata = { 799 { 800 .name = "C1", 801 .desc = "MWAIT 0x00", 802 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 803 .exit_latency = 1, 804 .target_residency = 1, 805 .enter = &intel_idle, 806 .enter_s2idle = intel_idle_s2idle, }, 807 { 808 .name = "C1E", 809 .desc = "MWAIT 0x01", 810 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 811 .exit_latency = 4, 812 .target_residency = 4, 813 .enter = &intel_idle, 814 .enter_s2idle = intel_idle_s2idle, }, 815 { 816 .name = "C6", 817 .desc = "MWAIT 0x20", 818 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 819 .exit_latency = 170, 820 .target_residency = 600, 821 .enter = &intel_idle, 822 .enter_s2idle = intel_idle_s2idle, }, 823 { 824 .enter = NULL } 825 }; 826 827 /* 828 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 829 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 830 * But in this case there is effectively no C1, because C1 requests are 831 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 832 * and C1E requests end up with C1, so there is effectively no C1E. 833 * 834 * By default we enable C1E and disable C1 by marking it with 835 * 'CPUIDLE_FLAG_UNUSABLE'. 836 */ 837 static struct cpuidle_state adl_cstates[] __initdata = { 838 { 839 .name = "C1", 840 .desc = "MWAIT 0x00", 841 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 842 .exit_latency = 1, 843 .target_residency = 1, 844 .enter = &intel_idle, 845 .enter_s2idle = intel_idle_s2idle, }, 846 { 847 .name = "C1E", 848 .desc = "MWAIT 0x01", 849 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 850 .exit_latency = 2, 851 .target_residency = 4, 852 .enter = &intel_idle, 853 .enter_s2idle = intel_idle_s2idle, }, 854 { 855 .name = "C6", 856 .desc = "MWAIT 0x20", 857 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 858 .exit_latency = 220, 859 .target_residency = 600, 860 .enter = &intel_idle, 861 .enter_s2idle = intel_idle_s2idle, }, 862 { 863 .name = "C8", 864 .desc = "MWAIT 0x40", 865 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 866 .exit_latency = 280, 867 .target_residency = 800, 868 .enter = &intel_idle, 869 .enter_s2idle = intel_idle_s2idle, }, 870 { 871 .name = "C10", 872 .desc = "MWAIT 0x60", 873 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 874 .exit_latency = 680, 875 .target_residency = 2000, 876 .enter = &intel_idle, 877 .enter_s2idle = intel_idle_s2idle, }, 878 { 879 .enter = NULL } 880 }; 881 882 static struct cpuidle_state adl_l_cstates[] __initdata = { 883 { 884 .name = "C1", 885 .desc = "MWAIT 0x00", 886 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 887 .exit_latency = 1, 888 .target_residency = 1, 889 .enter = &intel_idle, 890 .enter_s2idle = intel_idle_s2idle, }, 891 { 892 .name = "C1E", 893 .desc = "MWAIT 0x01", 894 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 895 .exit_latency = 2, 896 .target_residency = 4, 897 .enter = &intel_idle, 898 .enter_s2idle = intel_idle_s2idle, }, 899 { 900 .name = "C6", 901 .desc = "MWAIT 0x20", 902 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 903 .exit_latency = 170, 904 .target_residency = 500, 905 .enter = &intel_idle, 906 .enter_s2idle = intel_idle_s2idle, }, 907 { 908 .name = "C8", 909 .desc = "MWAIT 0x40", 910 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 911 .exit_latency = 200, 912 .target_residency = 600, 913 .enter = &intel_idle, 914 .enter_s2idle = intel_idle_s2idle, }, 915 { 916 .name = "C10", 917 .desc = "MWAIT 0x60", 918 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 919 .exit_latency = 230, 920 .target_residency = 700, 921 .enter = &intel_idle, 922 .enter_s2idle = intel_idle_s2idle, }, 923 { 924 .enter = NULL } 925 }; 926 927 static struct cpuidle_state mtl_l_cstates[] __initdata = { 928 { 929 .name = "C1E", 930 .desc = "MWAIT 0x01", 931 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 932 .exit_latency = 1, 933 .target_residency = 1, 934 .enter = &intel_idle, 935 .enter_s2idle = intel_idle_s2idle, }, 936 { 937 .name = "C6", 938 .desc = "MWAIT 0x20", 939 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 940 .exit_latency = 140, 941 .target_residency = 420, 942 .enter = &intel_idle, 943 .enter_s2idle = intel_idle_s2idle, }, 944 { 945 .name = "C10", 946 .desc = "MWAIT 0x60", 947 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 948 .exit_latency = 310, 949 .target_residency = 930, 950 .enter = &intel_idle, 951 .enter_s2idle = intel_idle_s2idle, }, 952 { 953 .enter = NULL } 954 }; 955 956 static struct cpuidle_state gmt_cstates[] __initdata = { 957 { 958 .name = "C1", 959 .desc = "MWAIT 0x00", 960 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 961 .exit_latency = 1, 962 .target_residency = 1, 963 .enter = &intel_idle, 964 .enter_s2idle = intel_idle_s2idle, }, 965 { 966 .name = "C1E", 967 .desc = "MWAIT 0x01", 968 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 969 .exit_latency = 2, 970 .target_residency = 4, 971 .enter = &intel_idle, 972 .enter_s2idle = intel_idle_s2idle, }, 973 { 974 .name = "C6", 975 .desc = "MWAIT 0x20", 976 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 977 .exit_latency = 195, 978 .target_residency = 585, 979 .enter = &intel_idle, 980 .enter_s2idle = intel_idle_s2idle, }, 981 { 982 .name = "C8", 983 .desc = "MWAIT 0x40", 984 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 985 .exit_latency = 260, 986 .target_residency = 1040, 987 .enter = &intel_idle, 988 .enter_s2idle = intel_idle_s2idle, }, 989 { 990 .name = "C10", 991 .desc = "MWAIT 0x60", 992 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 993 .exit_latency = 660, 994 .target_residency = 1980, 995 .enter = &intel_idle, 996 .enter_s2idle = intel_idle_s2idle, }, 997 { 998 .enter = NULL } 999 }; 1000 1001 static struct cpuidle_state spr_cstates[] __initdata = { 1002 { 1003 .name = "C1", 1004 .desc = "MWAIT 0x00", 1005 .flags = MWAIT2flg(0x00), 1006 .exit_latency = 1, 1007 .target_residency = 1, 1008 .enter = &intel_idle, 1009 .enter_s2idle = intel_idle_s2idle, }, 1010 { 1011 .name = "C1E", 1012 .desc = "MWAIT 0x01", 1013 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1014 .exit_latency = 2, 1015 .target_residency = 4, 1016 .enter = &intel_idle, 1017 .enter_s2idle = intel_idle_s2idle, }, 1018 { 1019 .name = "C6", 1020 .desc = "MWAIT 0x20", 1021 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1022 CPUIDLE_FLAG_INIT_XSTATE, 1023 .exit_latency = 290, 1024 .target_residency = 800, 1025 .enter = &intel_idle, 1026 .enter_s2idle = intel_idle_s2idle, }, 1027 { 1028 .enter = NULL } 1029 }; 1030 1031 static struct cpuidle_state gnr_cstates[] __initdata = { 1032 { 1033 .name = "C1", 1034 .desc = "MWAIT 0x00", 1035 .flags = MWAIT2flg(0x00), 1036 .exit_latency = 1, 1037 .target_residency = 1, 1038 .enter = &intel_idle, 1039 .enter_s2idle = intel_idle_s2idle, }, 1040 { 1041 .name = "C1E", 1042 .desc = "MWAIT 0x01", 1043 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1044 .exit_latency = 4, 1045 .target_residency = 4, 1046 .enter = &intel_idle, 1047 .enter_s2idle = intel_idle_s2idle, }, 1048 { 1049 .name = "C6", 1050 .desc = "MWAIT 0x20", 1051 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1052 CPUIDLE_FLAG_INIT_XSTATE | 1053 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1054 .exit_latency = 170, 1055 .target_residency = 650, 1056 .enter = &intel_idle, 1057 .enter_s2idle = intel_idle_s2idle, }, 1058 { 1059 .name = "C6P", 1060 .desc = "MWAIT 0x21", 1061 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1062 CPUIDLE_FLAG_INIT_XSTATE | 1063 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1064 .exit_latency = 210, 1065 .target_residency = 1000, 1066 .enter = &intel_idle, 1067 .enter_s2idle = intel_idle_s2idle, }, 1068 { 1069 .enter = NULL } 1070 }; 1071 1072 static struct cpuidle_state atom_cstates[] __initdata = { 1073 { 1074 .name = "C1E", 1075 .desc = "MWAIT 0x00", 1076 .flags = MWAIT2flg(0x00), 1077 .exit_latency = 10, 1078 .target_residency = 20, 1079 .enter = &intel_idle, 1080 .enter_s2idle = intel_idle_s2idle, }, 1081 { 1082 .name = "C2", 1083 .desc = "MWAIT 0x10", 1084 .flags = MWAIT2flg(0x10), 1085 .exit_latency = 20, 1086 .target_residency = 80, 1087 .enter = &intel_idle, 1088 .enter_s2idle = intel_idle_s2idle, }, 1089 { 1090 .name = "C4", 1091 .desc = "MWAIT 0x30", 1092 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1093 .exit_latency = 100, 1094 .target_residency = 400, 1095 .enter = &intel_idle, 1096 .enter_s2idle = intel_idle_s2idle, }, 1097 { 1098 .name = "C6", 1099 .desc = "MWAIT 0x52", 1100 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1101 .exit_latency = 140, 1102 .target_residency = 560, 1103 .enter = &intel_idle, 1104 .enter_s2idle = intel_idle_s2idle, }, 1105 { 1106 .enter = NULL } 1107 }; 1108 static struct cpuidle_state tangier_cstates[] __initdata = { 1109 { 1110 .name = "C1", 1111 .desc = "MWAIT 0x00", 1112 .flags = MWAIT2flg(0x00), 1113 .exit_latency = 1, 1114 .target_residency = 4, 1115 .enter = &intel_idle, 1116 .enter_s2idle = intel_idle_s2idle, }, 1117 { 1118 .name = "C4", 1119 .desc = "MWAIT 0x30", 1120 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1121 .exit_latency = 100, 1122 .target_residency = 400, 1123 .enter = &intel_idle, 1124 .enter_s2idle = intel_idle_s2idle, }, 1125 { 1126 .name = "C6", 1127 .desc = "MWAIT 0x52", 1128 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1129 .exit_latency = 140, 1130 .target_residency = 560, 1131 .enter = &intel_idle, 1132 .enter_s2idle = intel_idle_s2idle, }, 1133 { 1134 .name = "C7", 1135 .desc = "MWAIT 0x60", 1136 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1137 .exit_latency = 1200, 1138 .target_residency = 4000, 1139 .enter = &intel_idle, 1140 .enter_s2idle = intel_idle_s2idle, }, 1141 { 1142 .name = "C9", 1143 .desc = "MWAIT 0x64", 1144 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1145 .exit_latency = 10000, 1146 .target_residency = 20000, 1147 .enter = &intel_idle, 1148 .enter_s2idle = intel_idle_s2idle, }, 1149 { 1150 .enter = NULL } 1151 }; 1152 static struct cpuidle_state avn_cstates[] __initdata = { 1153 { 1154 .name = "C1", 1155 .desc = "MWAIT 0x00", 1156 .flags = MWAIT2flg(0x00), 1157 .exit_latency = 2, 1158 .target_residency = 2, 1159 .enter = &intel_idle, 1160 .enter_s2idle = intel_idle_s2idle, }, 1161 { 1162 .name = "C6", 1163 .desc = "MWAIT 0x51", 1164 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1165 .exit_latency = 15, 1166 .target_residency = 45, 1167 .enter = &intel_idle, 1168 .enter_s2idle = intel_idle_s2idle, }, 1169 { 1170 .enter = NULL } 1171 }; 1172 static struct cpuidle_state knl_cstates[] __initdata = { 1173 { 1174 .name = "C1", 1175 .desc = "MWAIT 0x00", 1176 .flags = MWAIT2flg(0x00), 1177 .exit_latency = 1, 1178 .target_residency = 2, 1179 .enter = &intel_idle, 1180 .enter_s2idle = intel_idle_s2idle }, 1181 { 1182 .name = "C6", 1183 .desc = "MWAIT 0x10", 1184 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1185 .exit_latency = 120, 1186 .target_residency = 500, 1187 .enter = &intel_idle, 1188 .enter_s2idle = intel_idle_s2idle }, 1189 { 1190 .enter = NULL } 1191 }; 1192 1193 static struct cpuidle_state bxt_cstates[] __initdata = { 1194 { 1195 .name = "C1", 1196 .desc = "MWAIT 0x00", 1197 .flags = MWAIT2flg(0x00), 1198 .exit_latency = 2, 1199 .target_residency = 2, 1200 .enter = &intel_idle, 1201 .enter_s2idle = intel_idle_s2idle, }, 1202 { 1203 .name = "C1E", 1204 .desc = "MWAIT 0x01", 1205 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1206 .exit_latency = 10, 1207 .target_residency = 20, 1208 .enter = &intel_idle, 1209 .enter_s2idle = intel_idle_s2idle, }, 1210 { 1211 .name = "C6", 1212 .desc = "MWAIT 0x20", 1213 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1214 .exit_latency = 133, 1215 .target_residency = 133, 1216 .enter = &intel_idle, 1217 .enter_s2idle = intel_idle_s2idle, }, 1218 { 1219 .name = "C7s", 1220 .desc = "MWAIT 0x31", 1221 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1222 .exit_latency = 155, 1223 .target_residency = 155, 1224 .enter = &intel_idle, 1225 .enter_s2idle = intel_idle_s2idle, }, 1226 { 1227 .name = "C8", 1228 .desc = "MWAIT 0x40", 1229 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1230 .exit_latency = 1000, 1231 .target_residency = 1000, 1232 .enter = &intel_idle, 1233 .enter_s2idle = intel_idle_s2idle, }, 1234 { 1235 .name = "C9", 1236 .desc = "MWAIT 0x50", 1237 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1238 .exit_latency = 2000, 1239 .target_residency = 2000, 1240 .enter = &intel_idle, 1241 .enter_s2idle = intel_idle_s2idle, }, 1242 { 1243 .name = "C10", 1244 .desc = "MWAIT 0x60", 1245 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1246 .exit_latency = 10000, 1247 .target_residency = 10000, 1248 .enter = &intel_idle, 1249 .enter_s2idle = intel_idle_s2idle, }, 1250 { 1251 .enter = NULL } 1252 }; 1253 1254 static struct cpuidle_state dnv_cstates[] __initdata = { 1255 { 1256 .name = "C1", 1257 .desc = "MWAIT 0x00", 1258 .flags = MWAIT2flg(0x00), 1259 .exit_latency = 2, 1260 .target_residency = 2, 1261 .enter = &intel_idle, 1262 .enter_s2idle = intel_idle_s2idle, }, 1263 { 1264 .name = "C1E", 1265 .desc = "MWAIT 0x01", 1266 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1267 .exit_latency = 10, 1268 .target_residency = 20, 1269 .enter = &intel_idle, 1270 .enter_s2idle = intel_idle_s2idle, }, 1271 { 1272 .name = "C6", 1273 .desc = "MWAIT 0x20", 1274 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1275 .exit_latency = 50, 1276 .target_residency = 500, 1277 .enter = &intel_idle, 1278 .enter_s2idle = intel_idle_s2idle, }, 1279 { 1280 .enter = NULL } 1281 }; 1282 1283 /* 1284 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1285 * C6, and this is indicated in the CPUID mwait leaf. 1286 */ 1287 static struct cpuidle_state snr_cstates[] __initdata = { 1288 { 1289 .name = "C1", 1290 .desc = "MWAIT 0x00", 1291 .flags = MWAIT2flg(0x00), 1292 .exit_latency = 2, 1293 .target_residency = 2, 1294 .enter = &intel_idle, 1295 .enter_s2idle = intel_idle_s2idle, }, 1296 { 1297 .name = "C1E", 1298 .desc = "MWAIT 0x01", 1299 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1300 .exit_latency = 15, 1301 .target_residency = 25, 1302 .enter = &intel_idle, 1303 .enter_s2idle = intel_idle_s2idle, }, 1304 { 1305 .name = "C6", 1306 .desc = "MWAIT 0x20", 1307 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1308 .exit_latency = 130, 1309 .target_residency = 500, 1310 .enter = &intel_idle, 1311 .enter_s2idle = intel_idle_s2idle, }, 1312 { 1313 .enter = NULL } 1314 }; 1315 1316 static struct cpuidle_state grr_cstates[] __initdata = { 1317 { 1318 .name = "C1", 1319 .desc = "MWAIT 0x00", 1320 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1321 .exit_latency = 1, 1322 .target_residency = 1, 1323 .enter = &intel_idle, 1324 .enter_s2idle = intel_idle_s2idle, }, 1325 { 1326 .name = "C1E", 1327 .desc = "MWAIT 0x01", 1328 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1329 .exit_latency = 2, 1330 .target_residency = 10, 1331 .enter = &intel_idle, 1332 .enter_s2idle = intel_idle_s2idle, }, 1333 { 1334 .name = "C6S", 1335 .desc = "MWAIT 0x22", 1336 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1337 .exit_latency = 140, 1338 .target_residency = 500, 1339 .enter = &intel_idle, 1340 .enter_s2idle = intel_idle_s2idle, }, 1341 { 1342 .enter = NULL } 1343 }; 1344 1345 static struct cpuidle_state srf_cstates[] __initdata = { 1346 { 1347 .name = "C1", 1348 .desc = "MWAIT 0x00", 1349 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1350 .exit_latency = 1, 1351 .target_residency = 1, 1352 .enter = &intel_idle, 1353 .enter_s2idle = intel_idle_s2idle, }, 1354 { 1355 .name = "C1E", 1356 .desc = "MWAIT 0x01", 1357 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1358 .exit_latency = 2, 1359 .target_residency = 10, 1360 .enter = &intel_idle, 1361 .enter_s2idle = intel_idle_s2idle, }, 1362 { 1363 .name = "C6S", 1364 .desc = "MWAIT 0x22", 1365 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1366 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1367 .exit_latency = 270, 1368 .target_residency = 700, 1369 .enter = &intel_idle, 1370 .enter_s2idle = intel_idle_s2idle, }, 1371 { 1372 .name = "C6SP", 1373 .desc = "MWAIT 0x23", 1374 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1375 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1376 .exit_latency = 310, 1377 .target_residency = 900, 1378 .enter = &intel_idle, 1379 .enter_s2idle = intel_idle_s2idle, }, 1380 { 1381 .enter = NULL } 1382 }; 1383 1384 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1385 .state_table = nehalem_cstates, 1386 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1387 .disable_promotion_to_c1e = true, 1388 }; 1389 1390 static const struct idle_cpu idle_cpu_nhx __initconst = { 1391 .state_table = nehalem_cstates, 1392 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1393 .disable_promotion_to_c1e = true, 1394 .use_acpi = true, 1395 }; 1396 1397 static const struct idle_cpu idle_cpu_atom __initconst = { 1398 .state_table = atom_cstates, 1399 }; 1400 1401 static const struct idle_cpu idle_cpu_tangier __initconst = { 1402 .state_table = tangier_cstates, 1403 }; 1404 1405 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1406 .state_table = atom_cstates, 1407 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1408 }; 1409 1410 static const struct idle_cpu idle_cpu_snb __initconst = { 1411 .state_table = snb_cstates, 1412 .disable_promotion_to_c1e = true, 1413 }; 1414 1415 static const struct idle_cpu idle_cpu_snx __initconst = { 1416 .state_table = snb_cstates, 1417 .disable_promotion_to_c1e = true, 1418 .use_acpi = true, 1419 }; 1420 1421 static const struct idle_cpu idle_cpu_byt __initconst = { 1422 .state_table = byt_cstates, 1423 .disable_promotion_to_c1e = true, 1424 .byt_auto_demotion_disable_flag = true, 1425 }; 1426 1427 static const struct idle_cpu idle_cpu_cht __initconst = { 1428 .state_table = cht_cstates, 1429 .disable_promotion_to_c1e = true, 1430 .byt_auto_demotion_disable_flag = true, 1431 }; 1432 1433 static const struct idle_cpu idle_cpu_ivb __initconst = { 1434 .state_table = ivb_cstates, 1435 .disable_promotion_to_c1e = true, 1436 }; 1437 1438 static const struct idle_cpu idle_cpu_ivt __initconst = { 1439 .state_table = ivt_cstates, 1440 .disable_promotion_to_c1e = true, 1441 .use_acpi = true, 1442 }; 1443 1444 static const struct idle_cpu idle_cpu_hsw __initconst = { 1445 .state_table = hsw_cstates, 1446 .disable_promotion_to_c1e = true, 1447 }; 1448 1449 static const struct idle_cpu idle_cpu_hsx __initconst = { 1450 .state_table = hsw_cstates, 1451 .disable_promotion_to_c1e = true, 1452 .use_acpi = true, 1453 }; 1454 1455 static const struct idle_cpu idle_cpu_bdw __initconst = { 1456 .state_table = bdw_cstates, 1457 .disable_promotion_to_c1e = true, 1458 }; 1459 1460 static const struct idle_cpu idle_cpu_bdx __initconst = { 1461 .state_table = bdw_cstates, 1462 .disable_promotion_to_c1e = true, 1463 .use_acpi = true, 1464 }; 1465 1466 static const struct idle_cpu idle_cpu_skl __initconst = { 1467 .state_table = skl_cstates, 1468 .disable_promotion_to_c1e = true, 1469 }; 1470 1471 static const struct idle_cpu idle_cpu_skx __initconst = { 1472 .state_table = skx_cstates, 1473 .disable_promotion_to_c1e = true, 1474 .use_acpi = true, 1475 }; 1476 1477 static const struct idle_cpu idle_cpu_icx __initconst = { 1478 .state_table = icx_cstates, 1479 .disable_promotion_to_c1e = true, 1480 .use_acpi = true, 1481 }; 1482 1483 static const struct idle_cpu idle_cpu_adl __initconst = { 1484 .state_table = adl_cstates, 1485 }; 1486 1487 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1488 .state_table = adl_l_cstates, 1489 }; 1490 1491 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1492 .state_table = mtl_l_cstates, 1493 }; 1494 1495 static const struct idle_cpu idle_cpu_gmt __initconst = { 1496 .state_table = gmt_cstates, 1497 }; 1498 1499 static const struct idle_cpu idle_cpu_spr __initconst = { 1500 .state_table = spr_cstates, 1501 .disable_promotion_to_c1e = true, 1502 .use_acpi = true, 1503 }; 1504 1505 static const struct idle_cpu idle_cpu_gnr __initconst = { 1506 .state_table = gnr_cstates, 1507 .disable_promotion_to_c1e = true, 1508 .use_acpi = true, 1509 }; 1510 1511 static const struct idle_cpu idle_cpu_avn __initconst = { 1512 .state_table = avn_cstates, 1513 .disable_promotion_to_c1e = true, 1514 .use_acpi = true, 1515 }; 1516 1517 static const struct idle_cpu idle_cpu_knl __initconst = { 1518 .state_table = knl_cstates, 1519 .use_acpi = true, 1520 }; 1521 1522 static const struct idle_cpu idle_cpu_bxt __initconst = { 1523 .state_table = bxt_cstates, 1524 .disable_promotion_to_c1e = true, 1525 }; 1526 1527 static const struct idle_cpu idle_cpu_dnv __initconst = { 1528 .state_table = dnv_cstates, 1529 .disable_promotion_to_c1e = true, 1530 .use_acpi = true, 1531 }; 1532 1533 static const struct idle_cpu idle_cpu_tmt __initconst = { 1534 .disable_promotion_to_c1e = true, 1535 }; 1536 1537 static const struct idle_cpu idle_cpu_snr __initconst = { 1538 .state_table = snr_cstates, 1539 .disable_promotion_to_c1e = true, 1540 .use_acpi = true, 1541 }; 1542 1543 static const struct idle_cpu idle_cpu_grr __initconst = { 1544 .state_table = grr_cstates, 1545 .disable_promotion_to_c1e = true, 1546 .use_acpi = true, 1547 }; 1548 1549 static const struct idle_cpu idle_cpu_srf __initconst = { 1550 .state_table = srf_cstates, 1551 .disable_promotion_to_c1e = true, 1552 .use_acpi = true, 1553 }; 1554 1555 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1556 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1557 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1558 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1559 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1560 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1561 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1562 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1563 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1564 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1565 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1566 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1567 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1568 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1569 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1570 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1571 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1572 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1573 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1574 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1575 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1576 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1577 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1578 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1579 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1580 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1581 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1582 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1583 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1584 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1585 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1586 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1587 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1588 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1589 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1590 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1591 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1592 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1593 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1594 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1595 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1596 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1597 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1598 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1599 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1600 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1601 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1602 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1603 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1604 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1605 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1606 {} 1607 }; 1608 1609 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1610 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1611 {} 1612 }; 1613 1614 static bool __init intel_idle_max_cstate_reached(int cstate) 1615 { 1616 if (cstate + 1 > max_cstate) { 1617 pr_info("max_cstate %d reached\n", max_cstate); 1618 return true; 1619 } 1620 return false; 1621 } 1622 1623 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1624 { 1625 unsigned long eax = flg2MWAIT(state->flags); 1626 1627 if (boot_cpu_has(X86_FEATURE_ARAT)) 1628 return false; 1629 1630 /* 1631 * Switch over to one-shot tick broadcast if the target C-state 1632 * is deeper than C1. 1633 */ 1634 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1635 } 1636 1637 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1638 #include <acpi/processor.h> 1639 1640 static bool no_acpi __read_mostly; 1641 module_param(no_acpi, bool, 0444); 1642 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1643 1644 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1645 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1646 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1647 1648 static struct acpi_processor_power acpi_state_table __initdata; 1649 1650 /** 1651 * intel_idle_cst_usable - Check if the _CST information can be used. 1652 * 1653 * Check if all of the C-states listed by _CST in the max_cstate range are 1654 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1655 */ 1656 static bool __init intel_idle_cst_usable(void) 1657 { 1658 int cstate, limit; 1659 1660 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1661 acpi_state_table.count); 1662 1663 for (cstate = 1; cstate < limit; cstate++) { 1664 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1665 1666 if (cx->entry_method != ACPI_CSTATE_FFH) 1667 return false; 1668 } 1669 1670 return true; 1671 } 1672 1673 static bool __init intel_idle_acpi_cst_extract(void) 1674 { 1675 unsigned int cpu; 1676 1677 if (no_acpi) { 1678 pr_debug("Not allowed to use ACPI _CST\n"); 1679 return false; 1680 } 1681 1682 for_each_possible_cpu(cpu) { 1683 struct acpi_processor *pr = per_cpu(processors, cpu); 1684 1685 if (!pr) 1686 continue; 1687 1688 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1689 continue; 1690 1691 acpi_state_table.count++; 1692 1693 if (!intel_idle_cst_usable()) 1694 continue; 1695 1696 if (!acpi_processor_claim_cst_control()) 1697 break; 1698 1699 return true; 1700 } 1701 1702 acpi_state_table.count = 0; 1703 pr_debug("ACPI _CST not found or not usable\n"); 1704 return false; 1705 } 1706 1707 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1708 { 1709 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1710 1711 /* 1712 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1713 * the interesting states are ACPI_CSTATE_FFH. 1714 */ 1715 for (cstate = 1; cstate < limit; cstate++) { 1716 struct acpi_processor_cx *cx; 1717 struct cpuidle_state *state; 1718 1719 if (intel_idle_max_cstate_reached(cstate - 1)) 1720 break; 1721 1722 cx = &acpi_state_table.states[cstate]; 1723 1724 state = &drv->states[drv->state_count++]; 1725 1726 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1727 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1728 state->exit_latency = cx->latency; 1729 /* 1730 * For C1-type C-states use the same number for both the exit 1731 * latency and target residency, because that is the case for 1732 * C1 in the majority of the static C-states tables above. 1733 * For the other types of C-states, however, set the target 1734 * residency to 3 times the exit latency which should lead to 1735 * a reasonable balance between energy-efficiency and 1736 * performance in the majority of interesting cases. 1737 */ 1738 state->target_residency = cx->latency; 1739 if (cx->type > ACPI_STATE_C1) 1740 state->target_residency *= 3; 1741 1742 state->flags = MWAIT2flg(cx->address); 1743 if (cx->type > ACPI_STATE_C2) 1744 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1745 1746 if (disabled_states_mask & BIT(cstate)) 1747 state->flags |= CPUIDLE_FLAG_OFF; 1748 1749 if (intel_idle_state_needs_timer_stop(state)) 1750 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1751 1752 state->enter = intel_idle; 1753 state->enter_s2idle = intel_idle_s2idle; 1754 } 1755 } 1756 1757 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1758 { 1759 int cstate, limit; 1760 1761 /* 1762 * If there are no _CST C-states, do not disable any C-states by 1763 * default. 1764 */ 1765 if (!acpi_state_table.count) 1766 return false; 1767 1768 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1769 /* 1770 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1771 * the interesting states are ACPI_CSTATE_FFH. 1772 */ 1773 for (cstate = 1; cstate < limit; cstate++) { 1774 u32 acpi_hint = acpi_state_table.states[cstate].address; 1775 u32 table_hint = mwait_hint; 1776 1777 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 1778 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 1779 table_hint &= ~MWAIT_SUBSTATE_MASK; 1780 } 1781 1782 if (acpi_hint == table_hint) 1783 return false; 1784 } 1785 return true; 1786 } 1787 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1788 #define force_use_acpi (false) 1789 1790 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1791 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1792 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1793 { 1794 return false; 1795 } 1796 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1797 1798 /** 1799 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1800 * 1801 * Tune IVT multi-socket targets. 1802 * Assumption: num_sockets == (max_package_num + 1). 1803 */ 1804 static void __init ivt_idle_state_table_update(void) 1805 { 1806 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1807 int cpu, package_num, num_sockets = 1; 1808 1809 for_each_online_cpu(cpu) { 1810 package_num = topology_physical_package_id(cpu); 1811 if (package_num + 1 > num_sockets) { 1812 num_sockets = package_num + 1; 1813 1814 if (num_sockets > 4) { 1815 cpuidle_state_table = ivt_cstates_8s; 1816 return; 1817 } 1818 } 1819 } 1820 1821 if (num_sockets > 2) 1822 cpuidle_state_table = ivt_cstates_4s; 1823 1824 /* else, 1 and 2 socket systems use default ivt_cstates */ 1825 } 1826 1827 /** 1828 * irtl_2_usec - IRTL to microseconds conversion. 1829 * @irtl: IRTL MSR value. 1830 * 1831 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1832 */ 1833 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1834 { 1835 static const unsigned int irtl_ns_units[] __initconst = { 1836 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1837 }; 1838 unsigned long long ns; 1839 1840 if (!irtl) 1841 return 0; 1842 1843 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1844 1845 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1846 } 1847 1848 /** 1849 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1850 * 1851 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1852 * definitive maximum latency and use the same value for target_residency. 1853 */ 1854 static void __init bxt_idle_state_table_update(void) 1855 { 1856 unsigned long long msr; 1857 unsigned int usec; 1858 1859 rdmsrl(MSR_PKGC6_IRTL, msr); 1860 usec = irtl_2_usec(msr); 1861 if (usec) { 1862 bxt_cstates[2].exit_latency = usec; 1863 bxt_cstates[2].target_residency = usec; 1864 } 1865 1866 rdmsrl(MSR_PKGC7_IRTL, msr); 1867 usec = irtl_2_usec(msr); 1868 if (usec) { 1869 bxt_cstates[3].exit_latency = usec; 1870 bxt_cstates[3].target_residency = usec; 1871 } 1872 1873 rdmsrl(MSR_PKGC8_IRTL, msr); 1874 usec = irtl_2_usec(msr); 1875 if (usec) { 1876 bxt_cstates[4].exit_latency = usec; 1877 bxt_cstates[4].target_residency = usec; 1878 } 1879 1880 rdmsrl(MSR_PKGC9_IRTL, msr); 1881 usec = irtl_2_usec(msr); 1882 if (usec) { 1883 bxt_cstates[5].exit_latency = usec; 1884 bxt_cstates[5].target_residency = usec; 1885 } 1886 1887 rdmsrl(MSR_PKGC10_IRTL, msr); 1888 usec = irtl_2_usec(msr); 1889 if (usec) { 1890 bxt_cstates[6].exit_latency = usec; 1891 bxt_cstates[6].target_residency = usec; 1892 } 1893 1894 } 1895 1896 /** 1897 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1898 * 1899 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1900 */ 1901 static void __init sklh_idle_state_table_update(void) 1902 { 1903 unsigned long long msr; 1904 unsigned int eax, ebx, ecx, edx; 1905 1906 1907 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1908 if (max_cstate <= 7) 1909 return; 1910 1911 /* if PC10 not present in CPUID.MWAIT.EDX */ 1912 if ((mwait_substates & (0xF << 28)) == 0) 1913 return; 1914 1915 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1916 1917 /* PC10 is not enabled in PKG C-state limit */ 1918 if ((msr & 0xF) != 8) 1919 return; 1920 1921 ecx = 0; 1922 cpuid(7, &eax, &ebx, &ecx, &edx); 1923 1924 /* if SGX is present */ 1925 if (ebx & (1 << 2)) { 1926 1927 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1928 1929 /* if SGX is enabled */ 1930 if (msr & (1 << 18)) 1931 return; 1932 } 1933 1934 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1935 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1936 } 1937 1938 /** 1939 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1940 * idle states table. 1941 */ 1942 static void __init skx_idle_state_table_update(void) 1943 { 1944 unsigned long long msr; 1945 1946 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1947 1948 /* 1949 * 000b: C0/C1 (no package C-state support) 1950 * 001b: C2 1951 * 010b: C6 (non-retention) 1952 * 011b: C6 (retention) 1953 * 111b: No Package C state limits. 1954 */ 1955 if ((msr & 0x7) < 2) { 1956 /* 1957 * Uses the CC6 + PC0 latency and 3 times of 1958 * latency for target_residency if the PC6 1959 * is disabled in BIOS. This is consistent 1960 * with how intel_idle driver uses _CST 1961 * to set the target_residency. 1962 */ 1963 skx_cstates[2].exit_latency = 92; 1964 skx_cstates[2].target_residency = 276; 1965 } 1966 } 1967 1968 /** 1969 * adl_idle_state_table_update - Adjust AlderLake idle states table. 1970 */ 1971 static void __init adl_idle_state_table_update(void) 1972 { 1973 /* Check if user prefers C1 over C1E. */ 1974 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 1975 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1976 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 1977 1978 /* Disable C1E by clearing the "C1E promotion" bit. */ 1979 c1e_promotion = C1E_PROMOTION_DISABLE; 1980 return; 1981 } 1982 1983 /* Make sure C1E is enabled by default */ 1984 c1e_promotion = C1E_PROMOTION_ENABLE; 1985 } 1986 1987 /** 1988 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1989 */ 1990 static void __init spr_idle_state_table_update(void) 1991 { 1992 unsigned long long msr; 1993 1994 /* 1995 * By default, the C6 state assumes the worst-case scenario of package 1996 * C6. However, if PC6 is disabled, we update the numbers to match 1997 * core C6. 1998 */ 1999 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 2000 2001 /* Limit value 2 and above allow for PC6. */ 2002 if ((msr & 0x7) < 2) { 2003 spr_cstates[2].exit_latency = 190; 2004 spr_cstates[2].target_residency = 600; 2005 } 2006 } 2007 2008 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2009 { 2010 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2011 MWAIT_CSTATE_MASK; 2012 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2013 MWAIT_SUBSTATE_MASK; 2014 2015 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2016 if (num_substates == 0) 2017 return false; 2018 2019 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2020 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2021 2022 return true; 2023 } 2024 2025 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2026 { 2027 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2028 /* 2029 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2030 * is not currently supported but this driver. 2031 */ 2032 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2033 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2034 state->enter = intel_idle_xstate; 2035 return; 2036 } 2037 2038 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2039 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2040 /* 2041 * IBRS mitigation requires that C-states are entered 2042 * with interrupts disabled. 2043 */ 2044 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2045 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2046 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2047 state->enter = intel_idle_ibrs; 2048 return; 2049 } 2050 2051 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2052 state->enter = intel_idle_irq; 2053 return; 2054 } 2055 2056 if (force_irq_on) { 2057 pr_info("forced intel_idle_irq for state %d\n", cstate); 2058 state->enter = intel_idle_irq; 2059 } 2060 } 2061 2062 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2063 { 2064 int cstate; 2065 2066 switch (boot_cpu_data.x86_vfm) { 2067 case INTEL_IVYBRIDGE_X: 2068 ivt_idle_state_table_update(); 2069 break; 2070 case INTEL_ATOM_GOLDMONT: 2071 case INTEL_ATOM_GOLDMONT_PLUS: 2072 bxt_idle_state_table_update(); 2073 break; 2074 case INTEL_SKYLAKE: 2075 sklh_idle_state_table_update(); 2076 break; 2077 case INTEL_SKYLAKE_X: 2078 skx_idle_state_table_update(); 2079 break; 2080 case INTEL_SAPPHIRERAPIDS_X: 2081 case INTEL_EMERALDRAPIDS_X: 2082 spr_idle_state_table_update(); 2083 break; 2084 case INTEL_ALDERLAKE: 2085 case INTEL_ALDERLAKE_L: 2086 case INTEL_ATOM_GRACEMONT: 2087 adl_idle_state_table_update(); 2088 break; 2089 } 2090 2091 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2092 struct cpuidle_state *state; 2093 unsigned int mwait_hint; 2094 2095 if (intel_idle_max_cstate_reached(cstate)) 2096 break; 2097 2098 if (!cpuidle_state_table[cstate].enter && 2099 !cpuidle_state_table[cstate].enter_s2idle) 2100 break; 2101 2102 /* If marked as unusable, skip this state. */ 2103 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2104 pr_debug("state %s is disabled\n", 2105 cpuidle_state_table[cstate].name); 2106 continue; 2107 } 2108 2109 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2110 if (!intel_idle_verify_cstate(mwait_hint)) 2111 continue; 2112 2113 /* Structure copy. */ 2114 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2115 state = &drv->states[drv->state_count]; 2116 2117 state_update_enter_method(state, cstate); 2118 2119 2120 if ((disabled_states_mask & BIT(drv->state_count)) || 2121 ((icpu->use_acpi || force_use_acpi) && 2122 intel_idle_off_by_default(state->flags, mwait_hint) && 2123 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2124 state->flags |= CPUIDLE_FLAG_OFF; 2125 2126 if (intel_idle_state_needs_timer_stop(state)) 2127 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2128 2129 drv->state_count++; 2130 } 2131 2132 if (icpu->byt_auto_demotion_disable_flag) { 2133 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2134 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2135 } 2136 } 2137 2138 /** 2139 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2140 * @drv: cpuidle driver structure to initialize. 2141 */ 2142 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2143 { 2144 cpuidle_poll_state_init(drv); 2145 2146 if (disabled_states_mask & BIT(0)) 2147 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2148 2149 drv->state_count = 1; 2150 2151 if (icpu && icpu->state_table) 2152 intel_idle_init_cstates_icpu(drv); 2153 else 2154 intel_idle_init_cstates_acpi(drv); 2155 } 2156 2157 static void auto_demotion_disable(void) 2158 { 2159 unsigned long long msr_bits; 2160 2161 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2162 msr_bits &= ~auto_demotion_disable_flags; 2163 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2164 } 2165 2166 static void c1e_promotion_enable(void) 2167 { 2168 unsigned long long msr_bits; 2169 2170 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2171 msr_bits |= 0x2; 2172 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2173 } 2174 2175 static void c1e_promotion_disable(void) 2176 { 2177 unsigned long long msr_bits; 2178 2179 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2180 msr_bits &= ~0x2; 2181 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2182 } 2183 2184 /** 2185 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2186 * @cpu: CPU to initialize. 2187 * 2188 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2189 * with the processor model flags. 2190 */ 2191 static int intel_idle_cpu_init(unsigned int cpu) 2192 { 2193 struct cpuidle_device *dev; 2194 2195 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2196 dev->cpu = cpu; 2197 2198 if (cpuidle_register_device(dev)) { 2199 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2200 return -EIO; 2201 } 2202 2203 if (auto_demotion_disable_flags) 2204 auto_demotion_disable(); 2205 2206 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2207 c1e_promotion_enable(); 2208 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2209 c1e_promotion_disable(); 2210 2211 return 0; 2212 } 2213 2214 static int intel_idle_cpu_online(unsigned int cpu) 2215 { 2216 struct cpuidle_device *dev; 2217 2218 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2219 tick_broadcast_enable(); 2220 2221 /* 2222 * Some systems can hotplug a cpu at runtime after 2223 * the kernel has booted, we have to initialize the 2224 * driver in this case 2225 */ 2226 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2227 if (!dev->registered) 2228 return intel_idle_cpu_init(cpu); 2229 2230 return 0; 2231 } 2232 2233 /** 2234 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2235 */ 2236 static void __init intel_idle_cpuidle_devices_uninit(void) 2237 { 2238 int i; 2239 2240 for_each_online_cpu(i) 2241 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2242 } 2243 2244 static int __init intel_idle_init(void) 2245 { 2246 const struct x86_cpu_id *id; 2247 unsigned int eax, ebx, ecx; 2248 int retval; 2249 2250 /* Do not load intel_idle at all for now if idle= is passed */ 2251 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2252 return -ENODEV; 2253 2254 if (max_cstate == 0) { 2255 pr_debug("disabled\n"); 2256 return -EPERM; 2257 } 2258 2259 id = x86_match_cpu(intel_idle_ids); 2260 if (id) { 2261 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2262 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2263 return -ENODEV; 2264 } 2265 } else { 2266 id = x86_match_cpu(intel_mwait_ids); 2267 if (!id) 2268 return -ENODEV; 2269 } 2270 2271 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 2272 return -ENODEV; 2273 2274 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 2275 2276 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2277 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2278 !mwait_substates) 2279 return -ENODEV; 2280 2281 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2282 2283 icpu = (const struct idle_cpu *)id->driver_data; 2284 if (icpu) { 2285 if (icpu->state_table) 2286 cpuidle_state_table = icpu->state_table; 2287 else if (!intel_idle_acpi_cst_extract()) 2288 return -ENODEV; 2289 2290 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2291 if (icpu->disable_promotion_to_c1e) 2292 c1e_promotion = C1E_PROMOTION_DISABLE; 2293 if (icpu->use_acpi || force_use_acpi) 2294 intel_idle_acpi_cst_extract(); 2295 } else if (!intel_idle_acpi_cst_extract()) { 2296 return -ENODEV; 2297 } 2298 2299 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2300 boot_cpu_data.x86_model); 2301 2302 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2303 if (!intel_idle_cpuidle_devices) 2304 return -ENOMEM; 2305 2306 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2307 2308 retval = cpuidle_register_driver(&intel_idle_driver); 2309 if (retval) { 2310 struct cpuidle_driver *drv = cpuidle_get_driver(); 2311 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2312 drv ? drv->name : "none"); 2313 goto init_driver_fail; 2314 } 2315 2316 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2317 intel_idle_cpu_online, NULL); 2318 if (retval < 0) 2319 goto hp_setup_fail; 2320 2321 pr_debug("Local APIC timer is reliable in %s\n", 2322 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2323 2324 return 0; 2325 2326 hp_setup_fail: 2327 intel_idle_cpuidle_devices_uninit(); 2328 cpuidle_unregister_driver(&intel_idle_driver); 2329 init_driver_fail: 2330 free_percpu(intel_idle_cpuidle_devices); 2331 return retval; 2332 2333 } 2334 device_initcall(intel_idle_init); 2335 2336 /* 2337 * We are not really modular, but we used to support that. Meaning we also 2338 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2339 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2340 * is the easiest way (currently) to continue doing that. 2341 */ 2342 module_param(max_cstate, int, 0444); 2343 /* 2344 * The positions of the bits that are set in this number are the indices of the 2345 * idle states to be disabled by default (as reflected by the names of the 2346 * corresponding idle state directories in sysfs, "state0", "state1" ... 2347 * "state<i>" ..., where <i> is the index of the given state). 2348 */ 2349 module_param_named(states_off, disabled_states_mask, uint, 0444); 2350 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2351 /* 2352 * Some platforms come with mutually exclusive C-states, so that if one is 2353 * enabled, the other C-states must not be used. Example: C1 and C1E on 2354 * Sapphire Rapids platform. This parameter allows for selecting the 2355 * preferred C-states among the groups of mutually exclusive C-states - the 2356 * selected C-states will be registered, the other C-states from the mutually 2357 * exclusive group won't be registered. If the platform has no mutually 2358 * exclusive C-states, this parameter has no effect. 2359 */ 2360 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2361 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2362 /* 2363 * Debugging option that forces the driver to enter all C-states with 2364 * interrupts enabled. Does not apply to C-states with 2365 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2366 */ 2367 module_param(force_irq_on, bool, 0444); 2368 /* 2369 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2370 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2371 */ 2372 module_param(ibrs_off, bool, 0444); 2373 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2374