1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <linux/time64.h> 49 #include <trace/events/power.h> 50 #include <linux/sched.h> 51 #include <linux/sched/smt.h> 52 #include <linux/mutex.h> 53 #include <linux/notifier.h> 54 #include <linux/cpu.h> 55 #include <linux/moduleparam.h> 56 #include <linux/pm_qos.h> 57 #include <linux/sysfs.h> 58 #include <asm/cpuid/api.h> 59 #include <asm/cpu_device_id.h> 60 #include <asm/intel-family.h> 61 #include <asm/mwait.h> 62 #include <asm/spec-ctrl.h> 63 #include <asm/msr.h> 64 #include <asm/tsc.h> 65 #include <asm/fpu/api.h> 66 #include <asm/smp.h> 67 68 static struct cpuidle_driver intel_idle_driver = { 69 .name = "intel_idle", 70 .owner = THIS_MODULE, 71 }; 72 /* intel_idle.max_cstate=0 disables driver */ 73 static int max_cstate = CPUIDLE_STATE_MAX - 1; 74 static unsigned int disabled_states_mask __read_mostly; 75 static bool force_irq_on __read_mostly; 76 static bool ibrs_off __read_mostly; 77 78 /* The maximum allowed length for the 'table' module parameter */ 79 #define MAX_CMDLINE_TABLE_LEN 256 80 /* Maximum allowed C-state latency */ 81 #define MAX_CMDLINE_LATENCY_US (5 * USEC_PER_MSEC) 82 /* Maximum allowed C-state target residency */ 83 #define MAX_CMDLINE_RESIDENCY_US (100 * USEC_PER_MSEC) 84 85 /* The Package C-State Limit bits in MSR_PKG_CST_CONFIG_CONTROL */ 86 #define SKX_PKG_CST_LIMIT_MASK GENMASK(2, 0) 87 /* PC6 is enabled when Package C-State Limit >= this value */ 88 #define SKX_PKG_CST_LIMIT_PC6 2 89 90 static char cmdline_table_str[MAX_CMDLINE_TABLE_LEN] __read_mostly; 91 92 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 93 94 static unsigned long auto_demotion_disable_flags; 95 96 static enum { 97 C1E_PROMOTION_PRESERVE, 98 C1E_PROMOTION_ENABLE, 99 C1E_PROMOTION_DISABLE 100 } c1e_promotion = C1E_PROMOTION_PRESERVE; 101 102 struct idle_cpu { 103 struct cpuidle_state *state_table; 104 105 /* 106 * Hardware C-state auto-demotion may not always be optimal. 107 * Indicate which enable bits to clear here. 108 */ 109 unsigned long auto_demotion_disable_flags; 110 bool disable_promotion_to_c1e; 111 bool c1_demotion_supported; 112 bool use_acpi; 113 }; 114 115 static bool c1_demotion_supported; 116 static DEFINE_MUTEX(c1_demotion_mutex); 117 118 static struct device *sysfs_root __initdata; 119 120 static const struct idle_cpu *icpu __initdata; 121 static struct cpuidle_state *cpuidle_state_table __initdata; 122 123 /* C-states data from the 'intel_idle.table' cmdline parameter */ 124 static struct cpuidle_state cmdline_states[CPUIDLE_STATE_MAX] __initdata; 125 126 static unsigned int mwait_substates __initdata; 127 128 /* 129 * Enable interrupts before entering the C-state. On some platforms and for 130 * some C-states, this may measurably decrease interrupt latency. 131 */ 132 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 133 134 /* 135 * Enable this state by default even if the ACPI _CST does not list it. 136 */ 137 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 138 139 /* 140 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 141 * above. 142 */ 143 #define CPUIDLE_FLAG_IBRS BIT(16) 144 145 /* 146 * Initialize large xstate for the C6-state entrance. 147 */ 148 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 149 150 /* 151 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 152 * custom tables. 153 */ 154 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 155 156 /* 157 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 158 * the C-state (top nibble) and sub-state (bottom nibble) 159 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 160 * 161 * We store the hint at the top of our "flags" for each state. 162 */ 163 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 164 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 165 166 static __always_inline int __intel_idle(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, 168 int index, bool irqoff) 169 { 170 struct cpuidle_state *state = &drv->states[index]; 171 unsigned int eax = flg2MWAIT(state->flags); 172 unsigned int ecx = 1*irqoff; /* break on interrupt flag */ 173 174 mwait_idle_with_hints(eax, ecx); 175 176 return index; 177 } 178 179 /** 180 * intel_idle - Ask the processor to enter the given idle state. 181 * @dev: cpuidle device of the target CPU. 182 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 183 * @index: Target idle state index. 184 * 185 * Use the MWAIT instruction to notify the processor that the CPU represented by 186 * @dev is idle and it can try to enter the idle state corresponding to @index. 187 * 188 * If the local APIC timer is not known to be reliable in the target idle state, 189 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 190 * 191 * Must be called under local_irq_disable(). 192 */ 193 static __cpuidle int intel_idle(struct cpuidle_device *dev, 194 struct cpuidle_driver *drv, int index) 195 { 196 return __intel_idle(dev, drv, index, true); 197 } 198 199 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 200 struct cpuidle_driver *drv, int index) 201 { 202 return __intel_idle(dev, drv, index, false); 203 } 204 205 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 206 struct cpuidle_driver *drv, int index) 207 { 208 bool smt_active = sched_smt_active(); 209 u64 spec_ctrl = spec_ctrl_current(); 210 int ret; 211 212 if (smt_active) 213 __update_spec_ctrl(0); 214 215 ret = __intel_idle(dev, drv, index, true); 216 217 if (smt_active) 218 __update_spec_ctrl(spec_ctrl); 219 220 return ret; 221 } 222 223 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 224 struct cpuidle_driver *drv, int index) 225 { 226 fpu_idle_fpregs(); 227 return __intel_idle(dev, drv, index, true); 228 } 229 230 /** 231 * intel_idle_s2idle - Ask the processor to enter the given idle state. 232 * @dev: cpuidle device of the target CPU. 233 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 234 * @index: Target idle state index. 235 * 236 * Use the MWAIT instruction to notify the processor that the CPU represented by 237 * @dev is idle and it can try to enter the idle state corresponding to @index. 238 * 239 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 240 * scheduler tick and suspended scheduler clock on the target CPU. 241 */ 242 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 243 struct cpuidle_driver *drv, int index) 244 { 245 struct cpuidle_state *state = &drv->states[index]; 246 unsigned int eax = flg2MWAIT(state->flags); 247 unsigned int ecx = 1; /* break on interrupt flag */ 248 249 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 250 fpu_idle_fpregs(); 251 252 mwait_idle_with_hints(eax, ecx); 253 254 return 0; 255 } 256 257 static void intel_idle_enter_dead(struct cpuidle_device *dev, int index) 258 { 259 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 260 struct cpuidle_state *state = &drv->states[index]; 261 unsigned long eax = flg2MWAIT(state->flags); 262 263 mwait_play_dead(eax); 264 } 265 266 /* 267 * States are indexed by the cstate number, 268 * which is also the index into the MWAIT hint array. 269 * Thus C0 is a dummy. 270 */ 271 static struct cpuidle_state nehalem_cstates[] __initdata = { 272 { 273 .name = "C1", 274 .desc = "MWAIT 0x00", 275 .flags = MWAIT2flg(0x00), 276 .exit_latency = 3, 277 .target_residency = 6, 278 .enter = intel_idle, 279 .enter_s2idle = intel_idle_s2idle, }, 280 { 281 .name = "C1E", 282 .desc = "MWAIT 0x01", 283 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 284 .exit_latency = 10, 285 .target_residency = 20, 286 .enter = intel_idle, 287 .enter_s2idle = intel_idle_s2idle, }, 288 { 289 .name = "C3", 290 .desc = "MWAIT 0x10", 291 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 292 .exit_latency = 20, 293 .target_residency = 80, 294 .enter = intel_idle, 295 .enter_s2idle = intel_idle_s2idle, }, 296 { 297 .name = "C6", 298 .desc = "MWAIT 0x20", 299 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 300 .exit_latency = 200, 301 .target_residency = 800, 302 .enter = intel_idle, 303 .enter_s2idle = intel_idle_s2idle, }, 304 { 305 .enter = NULL } 306 }; 307 308 static struct cpuidle_state snb_cstates[] __initdata = { 309 { 310 .name = "C1", 311 .desc = "MWAIT 0x00", 312 .flags = MWAIT2flg(0x00), 313 .exit_latency = 2, 314 .target_residency = 2, 315 .enter = intel_idle, 316 .enter_s2idle = intel_idle_s2idle, }, 317 { 318 .name = "C1E", 319 .desc = "MWAIT 0x01", 320 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 321 .exit_latency = 10, 322 .target_residency = 20, 323 .enter = intel_idle, 324 .enter_s2idle = intel_idle_s2idle, }, 325 { 326 .name = "C3", 327 .desc = "MWAIT 0x10", 328 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 329 .exit_latency = 80, 330 .target_residency = 211, 331 .enter = intel_idle, 332 .enter_s2idle = intel_idle_s2idle, }, 333 { 334 .name = "C6", 335 .desc = "MWAIT 0x20", 336 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 337 .exit_latency = 104, 338 .target_residency = 345, 339 .enter = intel_idle, 340 .enter_s2idle = intel_idle_s2idle, }, 341 { 342 .name = "C7", 343 .desc = "MWAIT 0x30", 344 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 345 .exit_latency = 109, 346 .target_residency = 345, 347 .enter = intel_idle, 348 .enter_s2idle = intel_idle_s2idle, }, 349 { 350 .enter = NULL } 351 }; 352 353 static struct cpuidle_state byt_cstates[] __initdata = { 354 { 355 .name = "C1", 356 .desc = "MWAIT 0x00", 357 .flags = MWAIT2flg(0x00), 358 .exit_latency = 1, 359 .target_residency = 1, 360 .enter = intel_idle, 361 .enter_s2idle = intel_idle_s2idle, }, 362 { 363 .name = "C6N", 364 .desc = "MWAIT 0x58", 365 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 366 .exit_latency = 300, 367 .target_residency = 275, 368 .enter = intel_idle, 369 .enter_s2idle = intel_idle_s2idle, }, 370 { 371 .name = "C6S", 372 .desc = "MWAIT 0x52", 373 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 374 .exit_latency = 500, 375 .target_residency = 560, 376 .enter = intel_idle, 377 .enter_s2idle = intel_idle_s2idle, }, 378 { 379 .name = "C7", 380 .desc = "MWAIT 0x60", 381 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 382 .exit_latency = 1200, 383 .target_residency = 4000, 384 .enter = intel_idle, 385 .enter_s2idle = intel_idle_s2idle, }, 386 { 387 .name = "C7S", 388 .desc = "MWAIT 0x64", 389 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 390 .exit_latency = 10000, 391 .target_residency = 20000, 392 .enter = intel_idle, 393 .enter_s2idle = intel_idle_s2idle, }, 394 { 395 .enter = NULL } 396 }; 397 398 static struct cpuidle_state cht_cstates[] __initdata = { 399 { 400 .name = "C1", 401 .desc = "MWAIT 0x00", 402 .flags = MWAIT2flg(0x00), 403 .exit_latency = 1, 404 .target_residency = 1, 405 .enter = intel_idle, 406 .enter_s2idle = intel_idle_s2idle, }, 407 { 408 .name = "C6N", 409 .desc = "MWAIT 0x58", 410 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 411 .exit_latency = 80, 412 .target_residency = 275, 413 .enter = intel_idle, 414 .enter_s2idle = intel_idle_s2idle, }, 415 { 416 .name = "C6S", 417 .desc = "MWAIT 0x52", 418 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 419 .exit_latency = 200, 420 .target_residency = 560, 421 .enter = intel_idle, 422 .enter_s2idle = intel_idle_s2idle, }, 423 { 424 .name = "C7", 425 .desc = "MWAIT 0x60", 426 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 427 .exit_latency = 1200, 428 .target_residency = 4000, 429 .enter = intel_idle, 430 .enter_s2idle = intel_idle_s2idle, }, 431 { 432 .name = "C7S", 433 .desc = "MWAIT 0x64", 434 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 435 .exit_latency = 10000, 436 .target_residency = 20000, 437 .enter = intel_idle, 438 .enter_s2idle = intel_idle_s2idle, }, 439 { 440 .enter = NULL } 441 }; 442 443 static struct cpuidle_state ivb_cstates[] __initdata = { 444 { 445 .name = "C1", 446 .desc = "MWAIT 0x00", 447 .flags = MWAIT2flg(0x00), 448 .exit_latency = 1, 449 .target_residency = 1, 450 .enter = intel_idle, 451 .enter_s2idle = intel_idle_s2idle, }, 452 { 453 .name = "C1E", 454 .desc = "MWAIT 0x01", 455 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 456 .exit_latency = 10, 457 .target_residency = 20, 458 .enter = intel_idle, 459 .enter_s2idle = intel_idle_s2idle, }, 460 { 461 .name = "C3", 462 .desc = "MWAIT 0x10", 463 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 464 .exit_latency = 59, 465 .target_residency = 156, 466 .enter = intel_idle, 467 .enter_s2idle = intel_idle_s2idle, }, 468 { 469 .name = "C6", 470 .desc = "MWAIT 0x20", 471 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 472 .exit_latency = 80, 473 .target_residency = 300, 474 .enter = intel_idle, 475 .enter_s2idle = intel_idle_s2idle, }, 476 { 477 .name = "C7", 478 .desc = "MWAIT 0x30", 479 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 480 .exit_latency = 87, 481 .target_residency = 300, 482 .enter = intel_idle, 483 .enter_s2idle = intel_idle_s2idle, }, 484 { 485 .enter = NULL } 486 }; 487 488 static struct cpuidle_state ivt_cstates[] __initdata = { 489 { 490 .name = "C1", 491 .desc = "MWAIT 0x00", 492 .flags = MWAIT2flg(0x00), 493 .exit_latency = 1, 494 .target_residency = 1, 495 .enter = intel_idle, 496 .enter_s2idle = intel_idle_s2idle, }, 497 { 498 .name = "C1E", 499 .desc = "MWAIT 0x01", 500 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 501 .exit_latency = 10, 502 .target_residency = 80, 503 .enter = intel_idle, 504 .enter_s2idle = intel_idle_s2idle, }, 505 { 506 .name = "C3", 507 .desc = "MWAIT 0x10", 508 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 509 .exit_latency = 59, 510 .target_residency = 156, 511 .enter = intel_idle, 512 .enter_s2idle = intel_idle_s2idle, }, 513 { 514 .name = "C6", 515 .desc = "MWAIT 0x20", 516 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 517 .exit_latency = 82, 518 .target_residency = 300, 519 .enter = intel_idle, 520 .enter_s2idle = intel_idle_s2idle, }, 521 { 522 .enter = NULL } 523 }; 524 525 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 526 { 527 .name = "C1", 528 .desc = "MWAIT 0x00", 529 .flags = MWAIT2flg(0x00), 530 .exit_latency = 1, 531 .target_residency = 1, 532 .enter = intel_idle, 533 .enter_s2idle = intel_idle_s2idle, }, 534 { 535 .name = "C1E", 536 .desc = "MWAIT 0x01", 537 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 538 .exit_latency = 10, 539 .target_residency = 250, 540 .enter = intel_idle, 541 .enter_s2idle = intel_idle_s2idle, }, 542 { 543 .name = "C3", 544 .desc = "MWAIT 0x10", 545 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 546 .exit_latency = 59, 547 .target_residency = 300, 548 .enter = intel_idle, 549 .enter_s2idle = intel_idle_s2idle, }, 550 { 551 .name = "C6", 552 .desc = "MWAIT 0x20", 553 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 554 .exit_latency = 84, 555 .target_residency = 400, 556 .enter = intel_idle, 557 .enter_s2idle = intel_idle_s2idle, }, 558 { 559 .enter = NULL } 560 }; 561 562 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 563 { 564 .name = "C1", 565 .desc = "MWAIT 0x00", 566 .flags = MWAIT2flg(0x00), 567 .exit_latency = 1, 568 .target_residency = 1, 569 .enter = intel_idle, 570 .enter_s2idle = intel_idle_s2idle, }, 571 { 572 .name = "C1E", 573 .desc = "MWAIT 0x01", 574 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 575 .exit_latency = 10, 576 .target_residency = 500, 577 .enter = intel_idle, 578 .enter_s2idle = intel_idle_s2idle, }, 579 { 580 .name = "C3", 581 .desc = "MWAIT 0x10", 582 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 583 .exit_latency = 59, 584 .target_residency = 600, 585 .enter = intel_idle, 586 .enter_s2idle = intel_idle_s2idle, }, 587 { 588 .name = "C6", 589 .desc = "MWAIT 0x20", 590 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 591 .exit_latency = 88, 592 .target_residency = 700, 593 .enter = intel_idle, 594 .enter_s2idle = intel_idle_s2idle, }, 595 { 596 .enter = NULL } 597 }; 598 599 static struct cpuidle_state hsw_cstates[] __initdata = { 600 { 601 .name = "C1", 602 .desc = "MWAIT 0x00", 603 .flags = MWAIT2flg(0x00), 604 .exit_latency = 2, 605 .target_residency = 2, 606 .enter = intel_idle, 607 .enter_s2idle = intel_idle_s2idle, }, 608 { 609 .name = "C1E", 610 .desc = "MWAIT 0x01", 611 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 612 .exit_latency = 10, 613 .target_residency = 20, 614 .enter = intel_idle, 615 .enter_s2idle = intel_idle_s2idle, }, 616 { 617 .name = "C3", 618 .desc = "MWAIT 0x10", 619 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 620 .exit_latency = 33, 621 .target_residency = 100, 622 .enter = intel_idle, 623 .enter_s2idle = intel_idle_s2idle, }, 624 { 625 .name = "C6", 626 .desc = "MWAIT 0x20", 627 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 628 .exit_latency = 133, 629 .target_residency = 400, 630 .enter = intel_idle, 631 .enter_s2idle = intel_idle_s2idle, }, 632 { 633 .name = "C7s", 634 .desc = "MWAIT 0x32", 635 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 636 .exit_latency = 166, 637 .target_residency = 500, 638 .enter = intel_idle, 639 .enter_s2idle = intel_idle_s2idle, }, 640 { 641 .name = "C8", 642 .desc = "MWAIT 0x40", 643 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 644 .exit_latency = 300, 645 .target_residency = 900, 646 .enter = intel_idle, 647 .enter_s2idle = intel_idle_s2idle, }, 648 { 649 .name = "C9", 650 .desc = "MWAIT 0x50", 651 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 652 .exit_latency = 600, 653 .target_residency = 1800, 654 .enter = intel_idle, 655 .enter_s2idle = intel_idle_s2idle, }, 656 { 657 .name = "C10", 658 .desc = "MWAIT 0x60", 659 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 660 .exit_latency = 2600, 661 .target_residency = 7700, 662 .enter = intel_idle, 663 .enter_s2idle = intel_idle_s2idle, }, 664 { 665 .enter = NULL } 666 }; 667 static struct cpuidle_state bdw_cstates[] __initdata = { 668 { 669 .name = "C1", 670 .desc = "MWAIT 0x00", 671 .flags = MWAIT2flg(0x00), 672 .exit_latency = 2, 673 .target_residency = 2, 674 .enter = intel_idle, 675 .enter_s2idle = intel_idle_s2idle, }, 676 { 677 .name = "C1E", 678 .desc = "MWAIT 0x01", 679 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 680 .exit_latency = 10, 681 .target_residency = 20, 682 .enter = intel_idle, 683 .enter_s2idle = intel_idle_s2idle, }, 684 { 685 .name = "C3", 686 .desc = "MWAIT 0x10", 687 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 688 .exit_latency = 40, 689 .target_residency = 100, 690 .enter = intel_idle, 691 .enter_s2idle = intel_idle_s2idle, }, 692 { 693 .name = "C6", 694 .desc = "MWAIT 0x20", 695 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 696 .exit_latency = 133, 697 .target_residency = 400, 698 .enter = intel_idle, 699 .enter_s2idle = intel_idle_s2idle, }, 700 { 701 .name = "C7s", 702 .desc = "MWAIT 0x32", 703 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 704 .exit_latency = 166, 705 .target_residency = 500, 706 .enter = intel_idle, 707 .enter_s2idle = intel_idle_s2idle, }, 708 { 709 .name = "C8", 710 .desc = "MWAIT 0x40", 711 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 712 .exit_latency = 300, 713 .target_residency = 900, 714 .enter = intel_idle, 715 .enter_s2idle = intel_idle_s2idle, }, 716 { 717 .name = "C9", 718 .desc = "MWAIT 0x50", 719 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 720 .exit_latency = 600, 721 .target_residency = 1800, 722 .enter = intel_idle, 723 .enter_s2idle = intel_idle_s2idle, }, 724 { 725 .name = "C10", 726 .desc = "MWAIT 0x60", 727 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 728 .exit_latency = 2600, 729 .target_residency = 7700, 730 .enter = intel_idle, 731 .enter_s2idle = intel_idle_s2idle, }, 732 { 733 .enter = NULL } 734 }; 735 736 static struct cpuidle_state skl_cstates[] __initdata = { 737 { 738 .name = "C1", 739 .desc = "MWAIT 0x00", 740 .flags = MWAIT2flg(0x00), 741 .exit_latency = 2, 742 .target_residency = 2, 743 .enter = intel_idle, 744 .enter_s2idle = intel_idle_s2idle, }, 745 { 746 .name = "C1E", 747 .desc = "MWAIT 0x01", 748 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 749 .exit_latency = 10, 750 .target_residency = 20, 751 .enter = intel_idle, 752 .enter_s2idle = intel_idle_s2idle, }, 753 { 754 .name = "C3", 755 .desc = "MWAIT 0x10", 756 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 757 .exit_latency = 70, 758 .target_residency = 100, 759 .enter = intel_idle, 760 .enter_s2idle = intel_idle_s2idle, }, 761 { 762 .name = "C6", 763 .desc = "MWAIT 0x20", 764 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 765 .exit_latency = 85, 766 .target_residency = 200, 767 .enter = intel_idle, 768 .enter_s2idle = intel_idle_s2idle, }, 769 { 770 .name = "C7s", 771 .desc = "MWAIT 0x33", 772 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 773 .exit_latency = 124, 774 .target_residency = 800, 775 .enter = intel_idle, 776 .enter_s2idle = intel_idle_s2idle, }, 777 { 778 .name = "C8", 779 .desc = "MWAIT 0x40", 780 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 781 .exit_latency = 200, 782 .target_residency = 800, 783 .enter = intel_idle, 784 .enter_s2idle = intel_idle_s2idle, }, 785 { 786 .name = "C9", 787 .desc = "MWAIT 0x50", 788 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 789 .exit_latency = 480, 790 .target_residency = 5000, 791 .enter = intel_idle, 792 .enter_s2idle = intel_idle_s2idle, }, 793 { 794 .name = "C10", 795 .desc = "MWAIT 0x60", 796 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 797 .exit_latency = 890, 798 .target_residency = 5000, 799 .enter = intel_idle, 800 .enter_s2idle = intel_idle_s2idle, }, 801 { 802 .enter = NULL } 803 }; 804 805 static struct cpuidle_state skx_cstates[] __initdata = { 806 { 807 .name = "C1", 808 .desc = "MWAIT 0x00", 809 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 810 .exit_latency = 2, 811 .target_residency = 2, 812 .enter = intel_idle, 813 .enter_s2idle = intel_idle_s2idle, }, 814 { 815 .name = "C1E", 816 .desc = "MWAIT 0x01", 817 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 818 .exit_latency = 10, 819 .target_residency = 20, 820 .enter = intel_idle, 821 .enter_s2idle = intel_idle_s2idle, }, 822 { 823 .name = "C6", 824 .desc = "MWAIT 0x20", 825 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 826 .exit_latency = 133, 827 .target_residency = 600, 828 .enter = intel_idle, 829 .enter_s2idle = intel_idle_s2idle, }, 830 { 831 .enter = NULL } 832 }; 833 834 static struct cpuidle_state icx_cstates[] __initdata = { 835 { 836 .name = "C1", 837 .desc = "MWAIT 0x00", 838 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 839 .exit_latency = 1, 840 .target_residency = 1, 841 .enter = intel_idle, 842 .enter_s2idle = intel_idle_s2idle, }, 843 { 844 .name = "C1E", 845 .desc = "MWAIT 0x01", 846 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 847 .exit_latency = 4, 848 .target_residency = 4, 849 .enter = intel_idle, 850 .enter_s2idle = intel_idle_s2idle, }, 851 { 852 .name = "C6", 853 .desc = "MWAIT 0x20", 854 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 855 .exit_latency = 170, 856 .target_residency = 600, 857 .enter = intel_idle, 858 .enter_s2idle = intel_idle_s2idle, }, 859 { 860 .enter = NULL } 861 }; 862 863 /* 864 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 865 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 866 * But in this case there is effectively no C1, because C1 requests are 867 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 868 * and C1E requests end up with C1, so there is effectively no C1E. 869 * 870 * By default we enable C1E and disable C1 by marking it with 871 * 'CPUIDLE_FLAG_UNUSABLE'. 872 */ 873 static struct cpuidle_state adl_cstates[] __initdata = { 874 { 875 .name = "C1", 876 .desc = "MWAIT 0x00", 877 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 878 .exit_latency = 1, 879 .target_residency = 1, 880 .enter = intel_idle, 881 .enter_s2idle = intel_idle_s2idle, }, 882 { 883 .name = "C1E", 884 .desc = "MWAIT 0x01", 885 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 886 .exit_latency = 2, 887 .target_residency = 4, 888 .enter = intel_idle, 889 .enter_s2idle = intel_idle_s2idle, }, 890 { 891 .name = "C6", 892 .desc = "MWAIT 0x20", 893 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 894 .exit_latency = 220, 895 .target_residency = 600, 896 .enter = intel_idle, 897 .enter_s2idle = intel_idle_s2idle, }, 898 { 899 .name = "C8", 900 .desc = "MWAIT 0x40", 901 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 902 .exit_latency = 280, 903 .target_residency = 800, 904 .enter = intel_idle, 905 .enter_s2idle = intel_idle_s2idle, }, 906 { 907 .name = "C10", 908 .desc = "MWAIT 0x60", 909 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 910 .exit_latency = 680, 911 .target_residency = 2000, 912 .enter = intel_idle, 913 .enter_s2idle = intel_idle_s2idle, }, 914 { 915 .enter = NULL } 916 }; 917 918 static struct cpuidle_state adl_l_cstates[] __initdata = { 919 { 920 .name = "C1", 921 .desc = "MWAIT 0x00", 922 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 923 .exit_latency = 1, 924 .target_residency = 1, 925 .enter = intel_idle, 926 .enter_s2idle = intel_idle_s2idle, }, 927 { 928 .name = "C1E", 929 .desc = "MWAIT 0x01", 930 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 931 .exit_latency = 2, 932 .target_residency = 4, 933 .enter = intel_idle, 934 .enter_s2idle = intel_idle_s2idle, }, 935 { 936 .name = "C6", 937 .desc = "MWAIT 0x20", 938 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 939 .exit_latency = 170, 940 .target_residency = 500, 941 .enter = intel_idle, 942 .enter_s2idle = intel_idle_s2idle, }, 943 { 944 .name = "C8", 945 .desc = "MWAIT 0x40", 946 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 947 .exit_latency = 200, 948 .target_residency = 600, 949 .enter = intel_idle, 950 .enter_s2idle = intel_idle_s2idle, }, 951 { 952 .name = "C10", 953 .desc = "MWAIT 0x60", 954 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 955 .exit_latency = 230, 956 .target_residency = 700, 957 .enter = intel_idle, 958 .enter_s2idle = intel_idle_s2idle, }, 959 { 960 .enter = NULL } 961 }; 962 963 static struct cpuidle_state mtl_l_cstates[] __initdata = { 964 { 965 .name = "C1E", 966 .desc = "MWAIT 0x01", 967 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 968 .exit_latency = 1, 969 .target_residency = 1, 970 .enter = intel_idle, 971 .enter_s2idle = intel_idle_s2idle, }, 972 { 973 .name = "C6", 974 .desc = "MWAIT 0x20", 975 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 976 .exit_latency = 140, 977 .target_residency = 420, 978 .enter = intel_idle, 979 .enter_s2idle = intel_idle_s2idle, }, 980 { 981 .name = "C10", 982 .desc = "MWAIT 0x60", 983 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 984 .exit_latency = 310, 985 .target_residency = 930, 986 .enter = intel_idle, 987 .enter_s2idle = intel_idle_s2idle, }, 988 { 989 .enter = NULL } 990 }; 991 992 static struct cpuidle_state ptl_cstates[] __initdata = { 993 { 994 .name = "C1", 995 .desc = "MWAIT 0x00", 996 .flags = MWAIT2flg(0x00), 997 .exit_latency = 1, 998 .target_residency = 1, 999 .enter = &intel_idle, 1000 .enter_s2idle = intel_idle_s2idle, }, 1001 { 1002 .name = "C1E", 1003 .desc = "MWAIT 0x01", 1004 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1005 .exit_latency = 10, 1006 .target_residency = 10, 1007 .enter = &intel_idle, 1008 .enter_s2idle = intel_idle_s2idle, }, 1009 { 1010 .name = "C6S", 1011 .desc = "MWAIT 0x21", 1012 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED, 1013 .exit_latency = 300, 1014 .target_residency = 300, 1015 .enter = &intel_idle, 1016 .enter_s2idle = intel_idle_s2idle, }, 1017 { 1018 .name = "C10", 1019 .desc = "MWAIT 0x60", 1020 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1021 .exit_latency = 370, 1022 .target_residency = 2500, 1023 .enter = &intel_idle, 1024 .enter_s2idle = intel_idle_s2idle, }, 1025 { 1026 .enter = NULL } 1027 }; 1028 1029 static struct cpuidle_state gmt_cstates[] __initdata = { 1030 { 1031 .name = "C1", 1032 .desc = "MWAIT 0x00", 1033 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 1034 .exit_latency = 1, 1035 .target_residency = 1, 1036 .enter = intel_idle, 1037 .enter_s2idle = intel_idle_s2idle, }, 1038 { 1039 .name = "C1E", 1040 .desc = "MWAIT 0x01", 1041 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1042 .exit_latency = 2, 1043 .target_residency = 4, 1044 .enter = intel_idle, 1045 .enter_s2idle = intel_idle_s2idle, }, 1046 { 1047 .name = "C6", 1048 .desc = "MWAIT 0x20", 1049 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1050 .exit_latency = 195, 1051 .target_residency = 585, 1052 .enter = intel_idle, 1053 .enter_s2idle = intel_idle_s2idle, }, 1054 { 1055 .name = "C8", 1056 .desc = "MWAIT 0x40", 1057 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1058 .exit_latency = 260, 1059 .target_residency = 1040, 1060 .enter = intel_idle, 1061 .enter_s2idle = intel_idle_s2idle, }, 1062 { 1063 .name = "C10", 1064 .desc = "MWAIT 0x60", 1065 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1066 .exit_latency = 660, 1067 .target_residency = 1980, 1068 .enter = intel_idle, 1069 .enter_s2idle = intel_idle_s2idle, }, 1070 { 1071 .enter = NULL } 1072 }; 1073 1074 static struct cpuidle_state spr_cstates[] __initdata = { 1075 { 1076 .name = "C1", 1077 .desc = "MWAIT 0x00", 1078 .flags = MWAIT2flg(0x00), 1079 .exit_latency = 1, 1080 .target_residency = 1, 1081 .enter = intel_idle, 1082 .enter_s2idle = intel_idle_s2idle, }, 1083 { 1084 .name = "C1E", 1085 .desc = "MWAIT 0x01", 1086 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1087 .exit_latency = 2, 1088 .target_residency = 4, 1089 .enter = intel_idle, 1090 .enter_s2idle = intel_idle_s2idle, }, 1091 { 1092 .name = "C6", 1093 .desc = "MWAIT 0x20", 1094 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1095 CPUIDLE_FLAG_INIT_XSTATE, 1096 .exit_latency = 290, 1097 .target_residency = 800, 1098 .enter = intel_idle, 1099 .enter_s2idle = intel_idle_s2idle, }, 1100 { 1101 .enter = NULL } 1102 }; 1103 1104 static struct cpuidle_state gnr_cstates[] __initdata = { 1105 { 1106 .name = "C1", 1107 .desc = "MWAIT 0x00", 1108 .flags = MWAIT2flg(0x00), 1109 .exit_latency = 1, 1110 .target_residency = 1, 1111 .enter = intel_idle, 1112 .enter_s2idle = intel_idle_s2idle, }, 1113 { 1114 .name = "C1E", 1115 .desc = "MWAIT 0x01", 1116 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1117 .exit_latency = 4, 1118 .target_residency = 4, 1119 .enter = intel_idle, 1120 .enter_s2idle = intel_idle_s2idle, }, 1121 { 1122 .name = "C6", 1123 .desc = "MWAIT 0x20", 1124 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1125 CPUIDLE_FLAG_INIT_XSTATE | 1126 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1127 .exit_latency = 170, 1128 .target_residency = 650, 1129 .enter = intel_idle, 1130 .enter_s2idle = intel_idle_s2idle, }, 1131 { 1132 .name = "C6P", 1133 .desc = "MWAIT 0x21", 1134 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1135 CPUIDLE_FLAG_INIT_XSTATE | 1136 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1137 .exit_latency = 210, 1138 .target_residency = 1000, 1139 .enter = intel_idle, 1140 .enter_s2idle = intel_idle_s2idle, }, 1141 { 1142 .enter = NULL } 1143 }; 1144 1145 static struct cpuidle_state gnrd_cstates[] __initdata = { 1146 { 1147 .name = "C1", 1148 .desc = "MWAIT 0x00", 1149 .flags = MWAIT2flg(0x00), 1150 .exit_latency = 1, 1151 .target_residency = 1, 1152 .enter = intel_idle, 1153 .enter_s2idle = intel_idle_s2idle, }, 1154 { 1155 .name = "C1E", 1156 .desc = "MWAIT 0x01", 1157 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1158 .exit_latency = 4, 1159 .target_residency = 4, 1160 .enter = intel_idle, 1161 .enter_s2idle = intel_idle_s2idle, }, 1162 { 1163 .name = "C6", 1164 .desc = "MWAIT 0x20", 1165 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1166 CPUIDLE_FLAG_INIT_XSTATE | 1167 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1168 .exit_latency = 220, 1169 .target_residency = 650, 1170 .enter = intel_idle, 1171 .enter_s2idle = intel_idle_s2idle, }, 1172 { 1173 .name = "C6P", 1174 .desc = "MWAIT 0x21", 1175 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1176 CPUIDLE_FLAG_INIT_XSTATE | 1177 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1178 .exit_latency = 240, 1179 .target_residency = 750, 1180 .enter = intel_idle, 1181 .enter_s2idle = intel_idle_s2idle, }, 1182 { 1183 .enter = NULL } 1184 }; 1185 1186 static struct cpuidle_state atom_cstates[] __initdata = { 1187 { 1188 .name = "C1E", 1189 .desc = "MWAIT 0x00", 1190 .flags = MWAIT2flg(0x00), 1191 .exit_latency = 10, 1192 .target_residency = 20, 1193 .enter = intel_idle, 1194 .enter_s2idle = intel_idle_s2idle, }, 1195 { 1196 .name = "C2", 1197 .desc = "MWAIT 0x10", 1198 .flags = MWAIT2flg(0x10), 1199 .exit_latency = 20, 1200 .target_residency = 80, 1201 .enter = intel_idle, 1202 .enter_s2idle = intel_idle_s2idle, }, 1203 { 1204 .name = "C4", 1205 .desc = "MWAIT 0x30", 1206 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1207 .exit_latency = 100, 1208 .target_residency = 400, 1209 .enter = intel_idle, 1210 .enter_s2idle = intel_idle_s2idle, }, 1211 { 1212 .name = "C6", 1213 .desc = "MWAIT 0x52", 1214 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1215 .exit_latency = 140, 1216 .target_residency = 560, 1217 .enter = intel_idle, 1218 .enter_s2idle = intel_idle_s2idle, }, 1219 { 1220 .enter = NULL } 1221 }; 1222 static struct cpuidle_state tangier_cstates[] __initdata = { 1223 { 1224 .name = "C1", 1225 .desc = "MWAIT 0x00", 1226 .flags = MWAIT2flg(0x00), 1227 .exit_latency = 1, 1228 .target_residency = 4, 1229 .enter = intel_idle, 1230 .enter_s2idle = intel_idle_s2idle, }, 1231 { 1232 .name = "C4", 1233 .desc = "MWAIT 0x30", 1234 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1235 .exit_latency = 100, 1236 .target_residency = 400, 1237 .enter = intel_idle, 1238 .enter_s2idle = intel_idle_s2idle, }, 1239 { 1240 .name = "C6", 1241 .desc = "MWAIT 0x52", 1242 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1243 .exit_latency = 140, 1244 .target_residency = 560, 1245 .enter = intel_idle, 1246 .enter_s2idle = intel_idle_s2idle, }, 1247 { 1248 .name = "C7", 1249 .desc = "MWAIT 0x60", 1250 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1251 .exit_latency = 1200, 1252 .target_residency = 4000, 1253 .enter = intel_idle, 1254 .enter_s2idle = intel_idle_s2idle, }, 1255 { 1256 .name = "C9", 1257 .desc = "MWAIT 0x64", 1258 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1259 .exit_latency = 10000, 1260 .target_residency = 20000, 1261 .enter = intel_idle, 1262 .enter_s2idle = intel_idle_s2idle, }, 1263 { 1264 .enter = NULL } 1265 }; 1266 static struct cpuidle_state avn_cstates[] __initdata = { 1267 { 1268 .name = "C1", 1269 .desc = "MWAIT 0x00", 1270 .flags = MWAIT2flg(0x00), 1271 .exit_latency = 2, 1272 .target_residency = 2, 1273 .enter = intel_idle, 1274 .enter_s2idle = intel_idle_s2idle, }, 1275 { 1276 .name = "C6", 1277 .desc = "MWAIT 0x51", 1278 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1279 .exit_latency = 15, 1280 .target_residency = 45, 1281 .enter = intel_idle, 1282 .enter_s2idle = intel_idle_s2idle, }, 1283 { 1284 .enter = NULL } 1285 }; 1286 static struct cpuidle_state knl_cstates[] __initdata = { 1287 { 1288 .name = "C1", 1289 .desc = "MWAIT 0x00", 1290 .flags = MWAIT2flg(0x00), 1291 .exit_latency = 1, 1292 .target_residency = 2, 1293 .enter = intel_idle, 1294 .enter_s2idle = intel_idle_s2idle }, 1295 { 1296 .name = "C6", 1297 .desc = "MWAIT 0x10", 1298 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1299 .exit_latency = 120, 1300 .target_residency = 500, 1301 .enter = intel_idle, 1302 .enter_s2idle = intel_idle_s2idle }, 1303 { 1304 .enter = NULL } 1305 }; 1306 1307 static struct cpuidle_state bxt_cstates[] __initdata = { 1308 { 1309 .name = "C1", 1310 .desc = "MWAIT 0x00", 1311 .flags = MWAIT2flg(0x00), 1312 .exit_latency = 2, 1313 .target_residency = 2, 1314 .enter = intel_idle, 1315 .enter_s2idle = intel_idle_s2idle, }, 1316 { 1317 .name = "C1E", 1318 .desc = "MWAIT 0x01", 1319 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1320 .exit_latency = 10, 1321 .target_residency = 20, 1322 .enter = intel_idle, 1323 .enter_s2idle = intel_idle_s2idle, }, 1324 { 1325 .name = "C6", 1326 .desc = "MWAIT 0x20", 1327 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1328 .exit_latency = 133, 1329 .target_residency = 133, 1330 .enter = intel_idle, 1331 .enter_s2idle = intel_idle_s2idle, }, 1332 { 1333 .name = "C7s", 1334 .desc = "MWAIT 0x31", 1335 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1336 .exit_latency = 155, 1337 .target_residency = 155, 1338 .enter = intel_idle, 1339 .enter_s2idle = intel_idle_s2idle, }, 1340 { 1341 .name = "C8", 1342 .desc = "MWAIT 0x40", 1343 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1344 .exit_latency = 1000, 1345 .target_residency = 1000, 1346 .enter = intel_idle, 1347 .enter_s2idle = intel_idle_s2idle, }, 1348 { 1349 .name = "C9", 1350 .desc = "MWAIT 0x50", 1351 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1352 .exit_latency = 2000, 1353 .target_residency = 2000, 1354 .enter = intel_idle, 1355 .enter_s2idle = intel_idle_s2idle, }, 1356 { 1357 .name = "C10", 1358 .desc = "MWAIT 0x60", 1359 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1360 .exit_latency = 10000, 1361 .target_residency = 10000, 1362 .enter = intel_idle, 1363 .enter_s2idle = intel_idle_s2idle, }, 1364 { 1365 .enter = NULL } 1366 }; 1367 1368 static struct cpuidle_state dnv_cstates[] __initdata = { 1369 { 1370 .name = "C1", 1371 .desc = "MWAIT 0x00", 1372 .flags = MWAIT2flg(0x00), 1373 .exit_latency = 2, 1374 .target_residency = 2, 1375 .enter = intel_idle, 1376 .enter_s2idle = intel_idle_s2idle, }, 1377 { 1378 .name = "C1E", 1379 .desc = "MWAIT 0x01", 1380 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1381 .exit_latency = 10, 1382 .target_residency = 20, 1383 .enter = intel_idle, 1384 .enter_s2idle = intel_idle_s2idle, }, 1385 { 1386 .name = "C6", 1387 .desc = "MWAIT 0x20", 1388 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1389 .exit_latency = 50, 1390 .target_residency = 500, 1391 .enter = intel_idle, 1392 .enter_s2idle = intel_idle_s2idle, }, 1393 { 1394 .enter = NULL } 1395 }; 1396 1397 /* 1398 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1399 * C6, and this is indicated in the CPUID mwait leaf. 1400 */ 1401 static struct cpuidle_state snr_cstates[] __initdata = { 1402 { 1403 .name = "C1", 1404 .desc = "MWAIT 0x00", 1405 .flags = MWAIT2flg(0x00), 1406 .exit_latency = 2, 1407 .target_residency = 2, 1408 .enter = intel_idle, 1409 .enter_s2idle = intel_idle_s2idle, }, 1410 { 1411 .name = "C1E", 1412 .desc = "MWAIT 0x01", 1413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1414 .exit_latency = 15, 1415 .target_residency = 25, 1416 .enter = intel_idle, 1417 .enter_s2idle = intel_idle_s2idle, }, 1418 { 1419 .name = "C6", 1420 .desc = "MWAIT 0x20", 1421 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1422 .exit_latency = 130, 1423 .target_residency = 500, 1424 .enter = intel_idle, 1425 .enter_s2idle = intel_idle_s2idle, }, 1426 { 1427 .enter = NULL } 1428 }; 1429 1430 static struct cpuidle_state grr_cstates[] __initdata = { 1431 { 1432 .name = "C1", 1433 .desc = "MWAIT 0x00", 1434 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1435 .exit_latency = 1, 1436 .target_residency = 1, 1437 .enter = intel_idle, 1438 .enter_s2idle = intel_idle_s2idle, }, 1439 { 1440 .name = "C1E", 1441 .desc = "MWAIT 0x01", 1442 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1443 .exit_latency = 2, 1444 .target_residency = 10, 1445 .enter = intel_idle, 1446 .enter_s2idle = intel_idle_s2idle, }, 1447 { 1448 .name = "C6S", 1449 .desc = "MWAIT 0x22", 1450 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1451 .exit_latency = 140, 1452 .target_residency = 500, 1453 .enter = intel_idle, 1454 .enter_s2idle = intel_idle_s2idle, }, 1455 { 1456 .enter = NULL } 1457 }; 1458 1459 static struct cpuidle_state srf_cstates[] __initdata = { 1460 { 1461 .name = "C1", 1462 .desc = "MWAIT 0x00", 1463 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1464 .exit_latency = 1, 1465 .target_residency = 1, 1466 .enter = intel_idle, 1467 .enter_s2idle = intel_idle_s2idle, }, 1468 { 1469 .name = "C1E", 1470 .desc = "MWAIT 0x01", 1471 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1472 .exit_latency = 2, 1473 .target_residency = 10, 1474 .enter = intel_idle, 1475 .enter_s2idle = intel_idle_s2idle, }, 1476 { 1477 .name = "C6S", 1478 .desc = "MWAIT 0x22", 1479 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1480 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1481 .exit_latency = 270, 1482 .target_residency = 700, 1483 .enter = intel_idle, 1484 .enter_s2idle = intel_idle_s2idle, }, 1485 { 1486 .name = "C6SP", 1487 .desc = "MWAIT 0x23", 1488 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1489 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1490 .exit_latency = 310, 1491 .target_residency = 900, 1492 .enter = intel_idle, 1493 .enter_s2idle = intel_idle_s2idle, }, 1494 { 1495 .enter = NULL } 1496 }; 1497 1498 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1499 .state_table = nehalem_cstates, 1500 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1501 .disable_promotion_to_c1e = true, 1502 }; 1503 1504 static const struct idle_cpu idle_cpu_nhx __initconst = { 1505 .state_table = nehalem_cstates, 1506 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1507 .disable_promotion_to_c1e = true, 1508 .use_acpi = true, 1509 }; 1510 1511 static const struct idle_cpu idle_cpu_atom __initconst = { 1512 .state_table = atom_cstates, 1513 }; 1514 1515 static const struct idle_cpu idle_cpu_tangier __initconst = { 1516 .state_table = tangier_cstates, 1517 }; 1518 1519 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1520 .state_table = atom_cstates, 1521 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1522 }; 1523 1524 static const struct idle_cpu idle_cpu_snb __initconst = { 1525 .state_table = snb_cstates, 1526 .disable_promotion_to_c1e = true, 1527 }; 1528 1529 static const struct idle_cpu idle_cpu_snx __initconst = { 1530 .state_table = snb_cstates, 1531 .disable_promotion_to_c1e = true, 1532 .use_acpi = true, 1533 }; 1534 1535 static const struct idle_cpu idle_cpu_byt __initconst = { 1536 .state_table = byt_cstates, 1537 .disable_promotion_to_c1e = true, 1538 }; 1539 1540 static const struct idle_cpu idle_cpu_cht __initconst = { 1541 .state_table = cht_cstates, 1542 .disable_promotion_to_c1e = true, 1543 }; 1544 1545 static const struct idle_cpu idle_cpu_ivb __initconst = { 1546 .state_table = ivb_cstates, 1547 .disable_promotion_to_c1e = true, 1548 }; 1549 1550 static const struct idle_cpu idle_cpu_ivt __initconst = { 1551 .state_table = ivt_cstates, 1552 .disable_promotion_to_c1e = true, 1553 .use_acpi = true, 1554 }; 1555 1556 static const struct idle_cpu idle_cpu_hsw __initconst = { 1557 .state_table = hsw_cstates, 1558 .disable_promotion_to_c1e = true, 1559 }; 1560 1561 static const struct idle_cpu idle_cpu_hsx __initconst = { 1562 .state_table = hsw_cstates, 1563 .disable_promotion_to_c1e = true, 1564 .use_acpi = true, 1565 }; 1566 1567 static const struct idle_cpu idle_cpu_bdw __initconst = { 1568 .state_table = bdw_cstates, 1569 .disable_promotion_to_c1e = true, 1570 }; 1571 1572 static const struct idle_cpu idle_cpu_bdx __initconst = { 1573 .state_table = bdw_cstates, 1574 .disable_promotion_to_c1e = true, 1575 .use_acpi = true, 1576 }; 1577 1578 static const struct idle_cpu idle_cpu_skl __initconst = { 1579 .state_table = skl_cstates, 1580 .disable_promotion_to_c1e = true, 1581 }; 1582 1583 static const struct idle_cpu idle_cpu_skx __initconst = { 1584 .state_table = skx_cstates, 1585 .disable_promotion_to_c1e = true, 1586 .use_acpi = true, 1587 }; 1588 1589 static const struct idle_cpu idle_cpu_icx __initconst = { 1590 .state_table = icx_cstates, 1591 .disable_promotion_to_c1e = true, 1592 .use_acpi = true, 1593 }; 1594 1595 static const struct idle_cpu idle_cpu_adl __initconst = { 1596 .state_table = adl_cstates, 1597 }; 1598 1599 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1600 .state_table = adl_l_cstates, 1601 }; 1602 1603 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1604 .state_table = mtl_l_cstates, 1605 }; 1606 1607 static const struct idle_cpu idle_cpu_ptl __initconst = { 1608 .state_table = ptl_cstates, 1609 }; 1610 1611 static const struct idle_cpu idle_cpu_gmt __initconst = { 1612 .state_table = gmt_cstates, 1613 }; 1614 1615 static const struct idle_cpu idle_cpu_spr __initconst = { 1616 .state_table = spr_cstates, 1617 .disable_promotion_to_c1e = true, 1618 .c1_demotion_supported = true, 1619 .use_acpi = true, 1620 }; 1621 1622 static const struct idle_cpu idle_cpu_gnr __initconst = { 1623 .state_table = gnr_cstates, 1624 .disable_promotion_to_c1e = true, 1625 .c1_demotion_supported = true, 1626 .use_acpi = true, 1627 }; 1628 1629 static const struct idle_cpu idle_cpu_gnrd __initconst = { 1630 .state_table = gnrd_cstates, 1631 .disable_promotion_to_c1e = true, 1632 .c1_demotion_supported = true, 1633 .use_acpi = true, 1634 }; 1635 1636 static const struct idle_cpu idle_cpu_avn __initconst = { 1637 .state_table = avn_cstates, 1638 .disable_promotion_to_c1e = true, 1639 .use_acpi = true, 1640 }; 1641 1642 static const struct idle_cpu idle_cpu_knl __initconst = { 1643 .state_table = knl_cstates, 1644 .use_acpi = true, 1645 }; 1646 1647 static const struct idle_cpu idle_cpu_bxt __initconst = { 1648 .state_table = bxt_cstates, 1649 .disable_promotion_to_c1e = true, 1650 }; 1651 1652 static const struct idle_cpu idle_cpu_dnv __initconst = { 1653 .state_table = dnv_cstates, 1654 .disable_promotion_to_c1e = true, 1655 .use_acpi = true, 1656 }; 1657 1658 static const struct idle_cpu idle_cpu_tmt __initconst = { 1659 .disable_promotion_to_c1e = true, 1660 }; 1661 1662 static const struct idle_cpu idle_cpu_snr __initconst = { 1663 .state_table = snr_cstates, 1664 .disable_promotion_to_c1e = true, 1665 .use_acpi = true, 1666 }; 1667 1668 static const struct idle_cpu idle_cpu_grr __initconst = { 1669 .state_table = grr_cstates, 1670 .disable_promotion_to_c1e = true, 1671 .c1_demotion_supported = true, 1672 .use_acpi = true, 1673 }; 1674 1675 static const struct idle_cpu idle_cpu_srf __initconst = { 1676 .state_table = srf_cstates, 1677 .disable_promotion_to_c1e = true, 1678 .c1_demotion_supported = true, 1679 .use_acpi = true, 1680 }; 1681 1682 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1683 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1684 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1685 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1686 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1687 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1688 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1689 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1690 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1691 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1692 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1693 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1694 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1695 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1696 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1697 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1698 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1699 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1700 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1701 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1702 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1703 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1704 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1705 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1706 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1707 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1708 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1709 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1710 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1711 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1712 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1713 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1714 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1715 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1716 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1717 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1718 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1719 X86_MATCH_VFM(INTEL_PANTHERLAKE_L, &idle_cpu_ptl), 1720 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1721 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1722 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1723 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1724 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), 1725 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1726 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1727 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1728 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1729 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1730 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1731 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1732 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1733 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1734 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1735 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf), 1736 {} 1737 }; 1738 1739 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1740 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, X86_FAMILY_ANY, X86_FEATURE_MWAIT, NULL), 1741 {} 1742 }; 1743 1744 static bool __init intel_idle_max_cstate_reached(int cstate) 1745 { 1746 if (cstate + 1 > max_cstate) { 1747 pr_info("max_cstate %d reached\n", max_cstate); 1748 return true; 1749 } 1750 return false; 1751 } 1752 1753 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1754 { 1755 unsigned long eax = flg2MWAIT(state->flags); 1756 1757 if (boot_cpu_has(X86_FEATURE_ARAT)) 1758 return false; 1759 1760 /* 1761 * Switch over to one-shot tick broadcast if the target C-state 1762 * is deeper than C1. 1763 */ 1764 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1765 } 1766 1767 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1768 #include <acpi/processor.h> 1769 1770 static bool no_acpi __read_mostly; 1771 module_param(no_acpi, bool, 0444); 1772 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1773 1774 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1775 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1776 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1777 1778 static bool no_native __read_mostly; /* No effect if no_acpi is set. */ 1779 module_param_named(no_native, no_native, bool, 0444); 1780 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states"); 1781 1782 static struct acpi_processor_power acpi_state_table __initdata; 1783 static bool acpi_lpi_available __initdata; 1784 1785 /** 1786 * intel_idle_cst_usable - Check if the _CST information can be used. 1787 * 1788 * Check if all of the C-states listed by _CST in the max_cstate range are 1789 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1790 */ 1791 static bool __init intel_idle_cst_usable(void) 1792 { 1793 int cstate, limit; 1794 1795 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1796 acpi_state_table.count); 1797 1798 for (cstate = 1; cstate < limit; cstate++) { 1799 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1800 1801 if (cx->entry_method != ACPI_CSTATE_FFH) 1802 return false; 1803 } 1804 1805 return true; 1806 } 1807 1808 static bool __init intel_idle_acpi_extract_lpi_cstates(void) 1809 { 1810 unsigned int cpu; 1811 1812 for_each_possible_cpu(cpu) { 1813 struct acpi_processor *pr; 1814 1815 pr = per_cpu(processors, cpu); 1816 if (!pr) 1817 continue; 1818 1819 if (acpi_processor_extract_lpi_info(pr->handle, 1820 &acpi_state_table, true)) 1821 continue; 1822 1823 acpi_lpi_available = true; 1824 return true; 1825 } 1826 1827 pr_debug("No ACPI _LPI idle states\n"); 1828 return false; 1829 } 1830 1831 static bool __init intel_idle_acpi_extract_cst_cstates(void) 1832 { 1833 unsigned int cpu; 1834 1835 for_each_possible_cpu(cpu) { 1836 struct acpi_processor *pr; 1837 1838 pr = per_cpu(processors, cpu); 1839 if (!pr) 1840 continue; 1841 1842 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1843 continue; 1844 1845 acpi_state_table.count++; 1846 1847 if (!intel_idle_cst_usable()) 1848 continue; 1849 1850 return true; 1851 } 1852 1853 pr_debug("ACPI _CST not found or not usable\n"); 1854 return false; 1855 } 1856 1857 static bool __init intel_idle_acpi_extract_cstates(void) 1858 { 1859 if (intel_idle_acpi_extract_lpi_cstates()) 1860 return true; 1861 1862 if (intel_idle_acpi_extract_cst_cstates()) 1863 return true; 1864 1865 return false; 1866 } 1867 1868 static bool __init intel_idle_acpi_probe(void) 1869 { 1870 if (no_acpi) { 1871 pr_debug("Not allowed to use ACPI for C-states extraction\n"); 1872 return false; 1873 } 1874 1875 if (intel_idle_acpi_extract_cstates() && 1876 acpi_processor_claim_cst_control()) 1877 return true; 1878 1879 acpi_state_table.count = 0; 1880 return false; 1881 } 1882 1883 static void __init intel_idle_complete_state_init(struct cpuidle_state *state) 1884 { 1885 if (intel_idle_state_needs_timer_stop(state)) 1886 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1887 1888 state->enter = intel_idle; 1889 state->enter_dead = intel_idle_enter_dead; 1890 state->enter_s2idle = intel_idle_s2idle; 1891 } 1892 1893 static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv) 1894 { 1895 int index; 1896 1897 for (index = 0; index < acpi_state_table.count; index++) { 1898 struct acpi_lpi_state *lpi_state; 1899 struct cpuidle_state *state; 1900 1901 if (intel_idle_max_cstate_reached(index)) 1902 break; 1903 1904 lpi_state = &acpi_state_table.lpi_states[index]; 1905 1906 state = &drv->states[drv->state_count++]; 1907 1908 scnprintf(state->name, CPUIDLE_NAME_LEN, "C%d_LPI", index + 1); 1909 strscpy(state->desc, lpi_state->desc, CPUIDLE_DESC_LEN); 1910 state->exit_latency = lpi_state->wake_latency; 1911 state->target_residency = lpi_state->min_residency; 1912 state->flags = MWAIT2flg(lpi_state->address); 1913 /* 1914 * Assume that entering any of the idle states extracted from 1915 * _LPI except for the first two will cause the TLB to be 1916 * flushed and let the core call leave_mm() for them upfront 1917 * to avoid unnecessary wakeups due to TLB shootdowns. 1918 */ 1919 if (index > 1) 1920 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1921 1922 if (disabled_states_mask & BIT(index + 1)) 1923 state->flags |= CPUIDLE_FLAG_OFF; 1924 1925 intel_idle_complete_state_init(state); 1926 1927 pr_info("%s: MWAIT hint 0x%x\n", state->name, flg2MWAIT(state->flags)); 1928 } 1929 1930 /* 1931 * Assume the first idle state in the table to be C1 and if any deeper 1932 * idle states are exposed while X86_FEATURE_NONSTOP_TSC is unset, mark 1933 * the TSC as unstable. 1934 */ 1935 if (index > 1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1936 mark_tsc_unstable("TSC halts in idle"); 1937 } 1938 1939 static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv) 1940 { 1941 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1942 1943 /* 1944 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1945 * the interesting states are ACPI_CSTATE_FFH. 1946 */ 1947 for (cstate = 1; cstate < limit; cstate++) { 1948 struct acpi_processor_cx *cx; 1949 struct cpuidle_state *state; 1950 1951 if (intel_idle_max_cstate_reached(cstate - 1)) 1952 break; 1953 1954 cx = &acpi_state_table.states[cstate]; 1955 1956 state = &drv->states[drv->state_count++]; 1957 1958 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1959 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1960 state->exit_latency = cx->latency; 1961 /* 1962 * For C1-type C-states use the same number for both the exit 1963 * latency and target residency, because that is the case for 1964 * C1 in the majority of the static C-states tables above. 1965 * For the other types of C-states, however, set the target 1966 * residency to 3 times the exit latency which should lead to 1967 * a reasonable balance between energy-efficiency and 1968 * performance in the majority of interesting cases. 1969 */ 1970 state->target_residency = cx->latency; 1971 if (cx->type > ACPI_STATE_C1) 1972 state->target_residency *= 3; 1973 1974 state->flags = MWAIT2flg(cx->address); 1975 if (cx->type > ACPI_STATE_C2) 1976 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1977 1978 if (disabled_states_mask & BIT(cstate)) 1979 state->flags |= CPUIDLE_FLAG_OFF; 1980 1981 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1982 mark_tsc_unstable("TSC halts in idle"); 1983 1984 intel_idle_complete_state_init(state); 1985 } 1986 } 1987 1988 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1989 { 1990 if (acpi_lpi_available) 1991 intel_idle_init_cstates_acpi_lpi(drv); 1992 else 1993 intel_idle_init_cstates_acpi_cst(drv); 1994 } 1995 1996 static bool __init intel_idle_acpi_hint_match(unsigned int flags, u32 acpi_hint, 1997 u32 table_hint) 1998 { 1999 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 2000 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 2001 table_hint &= ~MWAIT_SUBSTATE_MASK; 2002 } 2003 return acpi_hint == table_hint; 2004 } 2005 2006 static bool __init intel_idle_off_by_default_lpi(unsigned int flags, u32 mwait_hint) 2007 { 2008 int index; 2009 2010 for (index = 0; index < acpi_state_table.count; index++) { 2011 u32 acpi_hint = acpi_state_table.lpi_states[index].address; 2012 2013 if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint)) 2014 return false; 2015 } 2016 return true; 2017 } 2018 2019 static bool __init intel_idle_off_by_default_cst(unsigned int flags, u32 mwait_hint) 2020 { 2021 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 2022 2023 /* 2024 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 2025 * the interesting states are ACPI_CSTATE_FFH. 2026 */ 2027 for (cstate = 1; cstate < limit; cstate++) { 2028 u32 acpi_hint = acpi_state_table.states[cstate].address; 2029 2030 if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint)) 2031 return false; 2032 } 2033 return true; 2034 } 2035 2036 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 2037 { 2038 /* 2039 * If there is no C-states information in the ACPI tables, do not 2040 * disable any C-states by default. 2041 */ 2042 if (!acpi_state_table.count) 2043 return false; 2044 2045 if (acpi_lpi_available) 2046 return intel_idle_off_by_default_lpi(flags, mwait_hint); 2047 2048 return intel_idle_off_by_default_cst(flags, mwait_hint); 2049 } 2050 2051 static inline bool ignore_native(void) 2052 { 2053 return no_native && !no_acpi; 2054 } 2055 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 2056 #define force_use_acpi (false) 2057 2058 static inline bool intel_idle_acpi_probe(void) { return false; } 2059 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 2060 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 2061 { 2062 return false; 2063 } 2064 static inline bool ignore_native(void) { return false; } 2065 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 2066 2067 /** 2068 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 2069 * 2070 * Tune IVT multi-socket targets. 2071 * Assumption: num_sockets == (max_package_num + 1). 2072 */ 2073 static void __init ivt_idle_state_table_update(void) 2074 { 2075 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 2076 int cpu, package_num, num_sockets = 1; 2077 2078 for_each_online_cpu(cpu) { 2079 package_num = topology_physical_package_id(cpu); 2080 if (package_num + 1 > num_sockets) { 2081 num_sockets = package_num + 1; 2082 2083 if (num_sockets > 4) { 2084 cpuidle_state_table = ivt_cstates_8s; 2085 return; 2086 } 2087 } 2088 } 2089 2090 if (num_sockets > 2) 2091 cpuidle_state_table = ivt_cstates_4s; 2092 2093 /* else, 1 and 2 socket systems use default ivt_cstates */ 2094 } 2095 2096 /** 2097 * irtl_2_usec - IRTL to microseconds conversion. 2098 * @irtl: IRTL MSR value. 2099 * 2100 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 2101 */ 2102 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 2103 { 2104 static const unsigned int irtl_ns_units[] __initconst = { 2105 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 2106 }; 2107 unsigned long long ns; 2108 2109 if (!irtl) 2110 return 0; 2111 2112 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 2113 2114 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 2115 } 2116 2117 /** 2118 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 2119 * 2120 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 2121 * definitive maximum latency and use the same value for target_residency. 2122 */ 2123 static void __init bxt_idle_state_table_update(void) 2124 { 2125 unsigned long long msr; 2126 unsigned int usec; 2127 2128 rdmsrq(MSR_PKGC6_IRTL, msr); 2129 usec = irtl_2_usec(msr); 2130 if (usec) { 2131 bxt_cstates[2].exit_latency = usec; 2132 bxt_cstates[2].target_residency = usec; 2133 } 2134 2135 rdmsrq(MSR_PKGC7_IRTL, msr); 2136 usec = irtl_2_usec(msr); 2137 if (usec) { 2138 bxt_cstates[3].exit_latency = usec; 2139 bxt_cstates[3].target_residency = usec; 2140 } 2141 2142 rdmsrq(MSR_PKGC8_IRTL, msr); 2143 usec = irtl_2_usec(msr); 2144 if (usec) { 2145 bxt_cstates[4].exit_latency = usec; 2146 bxt_cstates[4].target_residency = usec; 2147 } 2148 2149 rdmsrq(MSR_PKGC9_IRTL, msr); 2150 usec = irtl_2_usec(msr); 2151 if (usec) { 2152 bxt_cstates[5].exit_latency = usec; 2153 bxt_cstates[5].target_residency = usec; 2154 } 2155 2156 rdmsrq(MSR_PKGC10_IRTL, msr); 2157 usec = irtl_2_usec(msr); 2158 if (usec) { 2159 bxt_cstates[6].exit_latency = usec; 2160 bxt_cstates[6].target_residency = usec; 2161 } 2162 2163 } 2164 2165 /** 2166 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 2167 * 2168 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 2169 */ 2170 static void __init sklh_idle_state_table_update(void) 2171 { 2172 unsigned long long msr; 2173 unsigned int eax, ebx, ecx, edx; 2174 2175 2176 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 2177 if (max_cstate <= 7) 2178 return; 2179 2180 /* if PC10 not present in CPUID.MWAIT.EDX */ 2181 if ((mwait_substates & (0xF << 28)) == 0) 2182 return; 2183 2184 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); 2185 2186 /* PC10 is not enabled in PKG C-state limit */ 2187 if ((msr & 0xF) != 8) 2188 return; 2189 2190 ecx = 0; 2191 cpuid(7, &eax, &ebx, &ecx, &edx); 2192 2193 /* if SGX is present */ 2194 if (ebx & (1 << 2)) { 2195 2196 rdmsrq(MSR_IA32_FEAT_CTL, msr); 2197 2198 /* if SGX is enabled */ 2199 if (msr & (1 << 18)) 2200 return; 2201 } 2202 2203 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 2204 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 2205 } 2206 2207 /** 2208 * skx_is_pc6_disabled() - Check if PC6 is disabled in BIOS. 2209 * 2210 * Return: %true if PC6 is disabled, %false otherwise. 2211 */ 2212 static bool __init skx_is_pc6_disabled(void) 2213 { 2214 u64 msr; 2215 2216 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); 2217 2218 /* 2219 * 000b: C0/C1 (no package C-state support) 2220 * 001b: C2 2221 * 010b: C6 (non-retention) 2222 * 011b: C6 (retention) 2223 * 111b: No Package C state limits. 2224 */ 2225 return (msr & SKX_PKG_CST_LIMIT_MASK) < SKX_PKG_CST_LIMIT_PC6; 2226 } 2227 2228 /** 2229 * skx_idle_state_table_update - Adjust the SKX/CLX idle states table. 2230 * 2231 * Adjust Sky Lake or Cascade Lake Xeon idle states if PC6 is disabled in BIOS. 2232 * Use the CC6 + PC0 latency and 3 times of that latency for target_residency. 2233 * This is consistent with how the intel_idle driver uses _CST to set the 2234 * target_residency. 2235 */ 2236 static void __init skx_idle_state_table_update(void) 2237 { 2238 if (skx_is_pc6_disabled()) { 2239 skx_cstates[2].exit_latency = 92; 2240 skx_cstates[2].target_residency = 276; 2241 } 2242 } 2243 2244 /** 2245 * spr_idle_state_table_update - Adjust Sapphire Rapids Xeon idle states table. 2246 * 2247 * By default, the C6 state assumes the worst-case scenario of package C6. 2248 * However, if PC6 is disabled in BIOS, update the numbers to match core C6. 2249 */ 2250 static void __init spr_idle_state_table_update(void) 2251 { 2252 if (skx_is_pc6_disabled()) { 2253 spr_cstates[2].exit_latency = 190; 2254 spr_cstates[2].target_residency = 600; 2255 } 2256 } 2257 2258 /** 2259 * drop_pc6_redundant_cstates() - Drop C-states redundant when PC6 is disabled. 2260 * @states: Idle states table to modify. 2261 * 2262 * When PC6 is disabled in BIOS, C-states that exist solely to enable PC6 2263 * entry (such as C6P or C6SP) become identical to shallower C-states like 2264 * C6, and are therefore redundant. Should be called only on systems with 2265 * multiple C6 flavors. 2266 */ 2267 static void __init drop_pc6_redundant_cstates(struct cpuidle_state *states) 2268 { 2269 int count; 2270 2271 if (!skx_is_pc6_disabled()) 2272 /* PC6 is not disabled, nothing to do */ 2273 return; 2274 2275 for (count = 0; states[count].enter; count++) 2276 continue; 2277 2278 if (count < 2) { 2279 pr_debug("Too few idle states to drop PC6-redundant states\n"); 2280 return; 2281 } 2282 2283 /* 2284 * Sanity check: At this point all platforms with multiple C6 flavors 2285 * use the CPUIDLE_FLAG_PARTIAL_HINT_MATCH flag. And the last state in 2286 * the table is the one that becomes redundant when PC6 is disabled. 2287 */ 2288 if (!(states[count - 1].flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH)) { 2289 pr_debug("Can't drop PC6-redundant states: unexpected flags\n"); 2290 return; 2291 } 2292 2293 /* 2294 * On all current platforms with multiple C6 flavors, there is only one 2295 * C-state that becomes redundant when PC6 is disabled. This state is 2296 * the last one in the table. Drop it by marking it with 2297 * CPUIDLE_FLAG_UNUSABLE so that cpuidle excludes it when registering 2298 * idle states. 2299 */ 2300 pr_info("Dropping idle state %s because PC6 is disabled\n", 2301 states[count - 1].name); 2302 states[count - 1].flags |= CPUIDLE_FLAG_UNUSABLE; 2303 } 2304 2305 /** 2306 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion. 2307 */ 2308 static void __init byt_cht_auto_demotion_disable(void) 2309 { 2310 wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2311 wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2312 } 2313 2314 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2315 { 2316 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2317 MWAIT_CSTATE_MASK; 2318 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2319 MWAIT_SUBSTATE_MASK; 2320 2321 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2322 if (num_substates == 0) 2323 return false; 2324 2325 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2326 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2327 2328 return true; 2329 } 2330 2331 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2332 { 2333 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2334 /* 2335 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2336 * is not currently supported but this driver. 2337 */ 2338 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2339 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2340 state->enter = intel_idle_xstate; 2341 return; 2342 } 2343 2344 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2345 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2346 /* 2347 * IBRS mitigation requires that C-states are entered 2348 * with interrupts disabled. 2349 */ 2350 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2351 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2352 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2353 state->enter = intel_idle_ibrs; 2354 return; 2355 } 2356 2357 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2358 state->enter = intel_idle_irq; 2359 return; 2360 } 2361 2362 if (force_irq_on) { 2363 pr_info("forced intel_idle_irq for state %d\n", cstate); 2364 state->enter = intel_idle_irq; 2365 } 2366 } 2367 2368 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2369 { 2370 int cstate; 2371 2372 switch (boot_cpu_data.x86_vfm) { 2373 case INTEL_IVYBRIDGE_X: 2374 ivt_idle_state_table_update(); 2375 break; 2376 case INTEL_ATOM_GOLDMONT: 2377 case INTEL_ATOM_GOLDMONT_PLUS: 2378 bxt_idle_state_table_update(); 2379 break; 2380 case INTEL_SKYLAKE: 2381 sklh_idle_state_table_update(); 2382 break; 2383 case INTEL_SKYLAKE_X: 2384 skx_idle_state_table_update(); 2385 break; 2386 case INTEL_SAPPHIRERAPIDS_X: 2387 case INTEL_EMERALDRAPIDS_X: 2388 spr_idle_state_table_update(); 2389 break; 2390 case INTEL_ATOM_SILVERMONT: 2391 case INTEL_ATOM_AIRMONT: 2392 byt_cht_auto_demotion_disable(); 2393 break; 2394 case INTEL_GRANITERAPIDS_D: 2395 case INTEL_GRANITERAPIDS_X: 2396 case INTEL_ATOM_CRESTMONT_X: 2397 case INTEL_ATOM_DARKMONT_X: 2398 drop_pc6_redundant_cstates(cpuidle_state_table); 2399 break; 2400 } 2401 2402 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2403 struct cpuidle_state *state; 2404 unsigned int mwait_hint; 2405 2406 if (intel_idle_max_cstate_reached(cstate)) 2407 break; 2408 2409 if (!cpuidle_state_table[cstate].enter && 2410 !cpuidle_state_table[cstate].enter_s2idle) 2411 break; 2412 2413 if (!cpuidle_state_table[cstate].enter_dead) 2414 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead; 2415 2416 /* If marked as unusable, skip this state. */ 2417 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2418 pr_debug("state %s is disabled\n", 2419 cpuidle_state_table[cstate].name); 2420 continue; 2421 } 2422 2423 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2424 if (!intel_idle_verify_cstate(mwait_hint)) 2425 continue; 2426 2427 /* Structure copy. */ 2428 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2429 state = &drv->states[drv->state_count]; 2430 2431 state_update_enter_method(state, cstate); 2432 2433 2434 if ((disabled_states_mask & BIT(drv->state_count)) || 2435 ((icpu->use_acpi || force_use_acpi) && 2436 intel_idle_off_by_default(state->flags, mwait_hint) && 2437 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2438 state->flags |= CPUIDLE_FLAG_OFF; 2439 2440 if (intel_idle_state_needs_timer_stop(state)) 2441 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2442 2443 drv->state_count++; 2444 } 2445 } 2446 2447 /** 2448 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2449 * @drv: cpuidle driver structure to initialize. 2450 */ 2451 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2452 { 2453 cpuidle_poll_state_init(drv); 2454 2455 if (disabled_states_mask & BIT(0)) 2456 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2457 2458 drv->state_count = 1; 2459 2460 if (icpu && icpu->state_table) 2461 intel_idle_init_cstates_icpu(drv); 2462 else 2463 intel_idle_init_cstates_acpi(drv); 2464 } 2465 2466 static void auto_demotion_disable(void) 2467 { 2468 unsigned long long msr_bits; 2469 2470 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2471 msr_bits &= ~auto_demotion_disable_flags; 2472 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2473 } 2474 2475 static void c1e_promotion_enable(void) 2476 { 2477 unsigned long long msr_bits; 2478 2479 rdmsrq(MSR_IA32_POWER_CTL, msr_bits); 2480 msr_bits |= 0x2; 2481 wrmsrq(MSR_IA32_POWER_CTL, msr_bits); 2482 } 2483 2484 static void c1e_promotion_disable(void) 2485 { 2486 unsigned long long msr_bits; 2487 2488 rdmsrq(MSR_IA32_POWER_CTL, msr_bits); 2489 msr_bits &= ~0x2; 2490 wrmsrq(MSR_IA32_POWER_CTL, msr_bits); 2491 } 2492 2493 /** 2494 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2495 * @cpu: CPU to initialize. 2496 * 2497 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2498 * with the processor model flags. 2499 */ 2500 static int intel_idle_cpu_init(unsigned int cpu) 2501 { 2502 struct cpuidle_device *dev; 2503 2504 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2505 dev->cpu = cpu; 2506 2507 if (cpuidle_register_device(dev)) { 2508 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2509 return -EIO; 2510 } 2511 2512 if (auto_demotion_disable_flags) 2513 auto_demotion_disable(); 2514 2515 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2516 c1e_promotion_enable(); 2517 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2518 c1e_promotion_disable(); 2519 2520 return 0; 2521 } 2522 2523 static int intel_idle_cpu_online(unsigned int cpu) 2524 { 2525 struct cpuidle_device *dev; 2526 2527 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2528 tick_broadcast_enable(); 2529 2530 /* 2531 * Some systems can hotplug a cpu at runtime after 2532 * the kernel has booted, we have to initialize the 2533 * driver in this case 2534 */ 2535 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2536 if (!dev->registered) 2537 return intel_idle_cpu_init(cpu); 2538 2539 return 0; 2540 } 2541 2542 /** 2543 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2544 */ 2545 static void __init intel_idle_cpuidle_devices_uninit(void) 2546 { 2547 int i; 2548 2549 for_each_online_cpu(i) 2550 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2551 } 2552 2553 static void intel_c1_demotion_toggle(void *enable) 2554 { 2555 unsigned long long msr_val; 2556 2557 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2558 /* 2559 * Enable/disable C1 undemotion along with C1 demotion, as this is the 2560 * most sensible configuration in general. 2561 */ 2562 if (enable) 2563 msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE; 2564 else 2565 msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE); 2566 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2567 } 2568 2569 static ssize_t intel_c1_demotion_store(struct device *dev, 2570 struct device_attribute *attr, 2571 const char *buf, size_t count) 2572 { 2573 bool enable; 2574 int err; 2575 2576 err = kstrtobool(buf, &enable); 2577 if (err) 2578 return err; 2579 2580 mutex_lock(&c1_demotion_mutex); 2581 /* Enable/disable C1 demotion on all CPUs */ 2582 on_each_cpu(intel_c1_demotion_toggle, (void *)enable, 1); 2583 mutex_unlock(&c1_demotion_mutex); 2584 2585 return count; 2586 } 2587 2588 static ssize_t intel_c1_demotion_show(struct device *dev, 2589 struct device_attribute *attr, char *buf) 2590 { 2591 unsigned long long msr_val; 2592 2593 /* 2594 * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other 2595 * configuration would be a BIOS bug. 2596 */ 2597 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2598 return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE)); 2599 } 2600 static DEVICE_ATTR_RW(intel_c1_demotion); 2601 2602 static int __init intel_idle_sysfs_init(void) 2603 { 2604 int err; 2605 2606 if (!c1_demotion_supported) 2607 return 0; 2608 2609 sysfs_root = bus_get_dev_root(&cpu_subsys); 2610 if (!sysfs_root) 2611 return 0; 2612 2613 err = sysfs_add_file_to_group(&sysfs_root->kobj, 2614 &dev_attr_intel_c1_demotion.attr, 2615 "cpuidle"); 2616 if (err) { 2617 put_device(sysfs_root); 2618 return err; 2619 } 2620 2621 return 0; 2622 } 2623 2624 static void __init intel_idle_sysfs_uninit(void) 2625 { 2626 if (!sysfs_root) 2627 return; 2628 2629 sysfs_remove_file_from_group(&sysfs_root->kobj, 2630 &dev_attr_intel_c1_demotion.attr, 2631 "cpuidle"); 2632 put_device(sysfs_root); 2633 } 2634 2635 /** 2636 * get_cmdline_field - Get the current field from a cmdline string. 2637 * @args: The cmdline string to get the current field from. 2638 * @field: Pointer to the current field upon return. 2639 * @sep: The fields separator character. 2640 * 2641 * Examples: 2642 * Input: args="C1:1:1,C1E:2:10", sep=':' 2643 * Output: field="C1", return "1:1,C1E:2:10" 2644 * Input: args="C1:1:1,C1E:2:10", sep=',' 2645 * Output: field="C1:1:1", return "C1E:2:10" 2646 * Ipnut: args="::", sep=':' 2647 * Output: field="", return ":" 2648 * 2649 * Return: The continuation of the cmdline string after the field or NULL. 2650 */ 2651 static char *get_cmdline_field(char *args, char **field, char sep) 2652 { 2653 unsigned int i; 2654 2655 for (i = 0; args[i] && !isspace(args[i]); i++) { 2656 if (args[i] == sep) 2657 break; 2658 } 2659 2660 *field = args; 2661 2662 if (args[i] != sep) 2663 return NULL; 2664 2665 args[i] = '\0'; 2666 return args + i + 1; 2667 } 2668 2669 /** 2670 * validate_cmdline_cstate - Validate a C-state from cmdline. 2671 * @state: The C-state to validate. 2672 * @prev_state: The previous C-state in the table or NULL. 2673 * 2674 * Return: 0 if the C-state is valid or -EINVAL otherwise. 2675 */ 2676 static int validate_cmdline_cstate(struct cpuidle_state *state, 2677 struct cpuidle_state *prev_state) 2678 { 2679 if (state->exit_latency == 0) 2680 /* Exit latency 0 can only be used for the POLL state */ 2681 return -EINVAL; 2682 2683 if (state->exit_latency > MAX_CMDLINE_LATENCY_US) 2684 return -EINVAL; 2685 2686 if (state->target_residency > MAX_CMDLINE_RESIDENCY_US) 2687 return -EINVAL; 2688 2689 if (state->target_residency < state->exit_latency) 2690 return -EINVAL; 2691 2692 if (!prev_state) 2693 return 0; 2694 2695 if (state->exit_latency <= prev_state->exit_latency) 2696 return -EINVAL; 2697 2698 if (state->target_residency <= prev_state->target_residency) 2699 return -EINVAL; 2700 2701 return 0; 2702 } 2703 2704 /** 2705 * cmdline_table_adjust - Adjust the C-states table with data from cmdline. 2706 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 2707 * 2708 * Adjust the C-states table with data from the 'intel_idle.table' module 2709 * parameter (if specified). 2710 */ 2711 static void __init cmdline_table_adjust(struct cpuidle_driver *drv) 2712 { 2713 char *args = cmdline_table_str; 2714 struct cpuidle_state *state; 2715 int i; 2716 2717 if (args[0] == '\0') 2718 /* The 'intel_idle.table' module parameter was not specified */ 2719 return; 2720 2721 /* Create a copy of the C-states table */ 2722 for (i = 0; i < drv->state_count; i++) 2723 cmdline_states[i] = drv->states[i]; 2724 2725 /* 2726 * Adjust the C-states table copy with data from the 'intel_idle.table' 2727 * module parameter. 2728 */ 2729 while (args) { 2730 char *fields, *name, *val; 2731 2732 /* 2733 * Get the next C-state definition, which is expected to be 2734 * '<name>:<latency_us>:<target_residency_us>'. Treat "empty" 2735 * fields as unchanged. For example, 2736 * '<name>::<target_residency_us>' leaves the latency unchanged. 2737 */ 2738 args = get_cmdline_field(args, &fields, ','); 2739 2740 /* name */ 2741 fields = get_cmdline_field(fields, &name, ':'); 2742 if (!fields) 2743 goto error; 2744 2745 if (!strcmp(name, "POLL")) { 2746 pr_err("Cannot adjust POLL\n"); 2747 continue; 2748 } 2749 2750 /* Find the C-state by its name */ 2751 state = NULL; 2752 for (i = 0; i < drv->state_count; i++) { 2753 if (!strcmp(name, drv->states[i].name)) { 2754 state = &cmdline_states[i]; 2755 break; 2756 } 2757 } 2758 2759 if (!state) { 2760 pr_err("C-state '%s' was not found\n", name); 2761 continue; 2762 } 2763 2764 /* Latency */ 2765 fields = get_cmdline_field(fields, &val, ':'); 2766 if (!fields) 2767 goto error; 2768 2769 if (*val) { 2770 if (kstrtouint(val, 0, &state->exit_latency)) 2771 goto error; 2772 } 2773 2774 /* Target residency */ 2775 fields = get_cmdline_field(fields, &val, ':'); 2776 2777 if (*val) { 2778 if (kstrtouint(val, 0, &state->target_residency)) 2779 goto error; 2780 } 2781 2782 /* 2783 * Allow for 3 more fields, but ignore them. Helps to make 2784 * possible future extensions of the cmdline format backward 2785 * compatible. 2786 */ 2787 for (i = 0; fields && i < 3; i++) { 2788 fields = get_cmdline_field(fields, &val, ':'); 2789 if (!fields) 2790 break; 2791 } 2792 2793 if (fields) { 2794 pr_err("Too many fields for C-state '%s'\n", state->name); 2795 goto error; 2796 } 2797 2798 pr_info("C-state from cmdline: name=%s, latency=%u, residency=%u\n", 2799 state->name, state->exit_latency, state->target_residency); 2800 } 2801 2802 /* Validate the adjusted C-states, start with index 1 to skip POLL */ 2803 for (i = 1; i < drv->state_count; i++) { 2804 struct cpuidle_state *prev_state; 2805 2806 state = &cmdline_states[i]; 2807 prev_state = &cmdline_states[i - 1]; 2808 2809 if (validate_cmdline_cstate(state, prev_state)) { 2810 pr_err("C-state '%s' validation failed\n", state->name); 2811 goto error; 2812 } 2813 } 2814 2815 /* Copy the adjusted C-states table back */ 2816 for (i = 1; i < drv->state_count; i++) 2817 drv->states[i] = cmdline_states[i]; 2818 2819 pr_info("Adjusted C-states with data from 'intel_idle.table'\n"); 2820 return; 2821 2822 error: 2823 pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n"); 2824 } 2825 2826 #define INTEL_IDLE_INIT_QOS 20 2827 static struct pm_qos_request qos_req __initdata; 2828 2829 static int __init intel_idle_init(void) 2830 { 2831 const struct x86_cpu_id *id; 2832 unsigned int eax, ebx, ecx; 2833 int retval; 2834 2835 /* Do not load intel_idle at all for now if idle= is passed */ 2836 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2837 return -ENODEV; 2838 2839 if (max_cstate == 0) { 2840 pr_debug("disabled\n"); 2841 return -EPERM; 2842 } 2843 2844 id = x86_match_cpu(intel_idle_ids); 2845 if (id) { 2846 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2847 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2848 return -ENODEV; 2849 } 2850 } else { 2851 id = x86_match_cpu(intel_mwait_ids); 2852 if (!id) 2853 return -ENODEV; 2854 } 2855 2856 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates); 2857 2858 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2859 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2860 !mwait_substates) 2861 return -ENODEV; 2862 2863 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2864 2865 icpu = (const struct idle_cpu *)id->driver_data; 2866 if (icpu && ignore_native()) { 2867 pr_debug("ignoring native CPU idle states\n"); 2868 icpu = NULL; 2869 } 2870 if (icpu) { 2871 if (icpu->state_table) 2872 cpuidle_state_table = icpu->state_table; 2873 else if (!intel_idle_acpi_probe()) 2874 return -ENODEV; 2875 2876 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2877 if (icpu->disable_promotion_to_c1e) 2878 c1e_promotion = C1E_PROMOTION_DISABLE; 2879 if (icpu->c1_demotion_supported) 2880 c1_demotion_supported = true; 2881 if (icpu->use_acpi || force_use_acpi) 2882 intel_idle_acpi_probe(); 2883 } else if (!intel_idle_acpi_probe()) { 2884 return -ENODEV; 2885 } 2886 2887 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2888 if (!intel_idle_cpuidle_devices) 2889 return -ENOMEM; 2890 2891 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2892 cmdline_table_adjust(&intel_idle_driver); 2893 2894 retval = intel_idle_sysfs_init(); 2895 if (retval) 2896 pr_warn("failed to initialized sysfs"); 2897 2898 /* 2899 * Some platforms, in particular the Intel S1200BTL motherboard, have a 2900 * problem with using package idle states too early, so prevent that 2901 * from taking place until the device_initcall() phase is over. 2902 */ 2903 cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS); 2904 2905 retval = cpuidle_register_driver(&intel_idle_driver); 2906 if (retval) { 2907 struct cpuidle_driver *drv = cpuidle_get_driver(); 2908 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2909 drv ? drv->name : "none"); 2910 goto init_driver_fail; 2911 } 2912 2913 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2914 intel_idle_cpu_online, NULL); 2915 if (retval < 0) 2916 goto hp_setup_fail; 2917 2918 pr_debug("Local APIC timer is reliable in %s\n", 2919 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2920 2921 arch_cpu_rescan_dead_smt_siblings(); 2922 2923 return 0; 2924 2925 hp_setup_fail: 2926 intel_idle_cpuidle_devices_uninit(); 2927 cpuidle_unregister_driver(&intel_idle_driver); 2928 init_driver_fail: 2929 if (cpu_latency_qos_request_active((&qos_req))) 2930 cpu_latency_qos_remove_request(&qos_req); 2931 2932 intel_idle_sysfs_uninit(); 2933 free_percpu(intel_idle_cpuidle_devices); 2934 return retval; 2935 2936 } 2937 subsys_initcall_sync(intel_idle_init); 2938 2939 static int __init intel_idle_init_complete(void) 2940 { 2941 if (cpu_latency_qos_request_active((&qos_req))) 2942 cpu_latency_qos_remove_request(&qos_req); 2943 2944 return 0; 2945 } 2946 device_initcall_sync(intel_idle_init_complete); 2947 2948 /* 2949 * We are not really modular, but we used to support that. Meaning we also 2950 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2951 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2952 * is the easiest way (currently) to continue doing that. 2953 */ 2954 module_param(max_cstate, int, 0444); 2955 /* 2956 * The positions of the bits that are set in this number are the indices of the 2957 * idle states to be disabled by default (as reflected by the names of the 2958 * corresponding idle state directories in sysfs, "state0", "state1" ... 2959 * "state<i>" ..., where <i> is the index of the given state). 2960 */ 2961 module_param_named(states_off, disabled_states_mask, uint, 0444); 2962 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2963 /* 2964 * Debugging option that forces the driver to enter all C-states with 2965 * interrupts enabled. Does not apply to C-states with 2966 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2967 */ 2968 module_param(force_irq_on, bool, 0444); 2969 /* 2970 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2971 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2972 */ 2973 module_param(ibrs_off, bool, 0444); 2974 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2975 2976 /* 2977 * Define the C-states table from a user input string. Expected format is 2978 * 'name:latency:residency', where: 2979 * - name: The C-state name. 2980 * - latency: The C-state exit latency in us. 2981 * - residency: The C-state target residency in us. 2982 * 2983 * Multiple C-states can be defined by separating them with commas: 2984 * 'name1:latency1:residency1,name2:latency2:residency2' 2985 * 2986 * Example: intel_idle.table=C1:1:1,C1E:5:10,C6:100:600 2987 * 2988 * To leave latency or residency unchanged, use an empty field, for example: 2989 * 'C1:1:1,C1E::10' - leaves C1E latency unchanged. 2990 */ 2991 module_param_string(table, cmdline_table_str, MAX_CMDLINE_TABLE_LEN, 0444); 2992 MODULE_PARM_DESC(table, "Build the C-states table from a user input string"); 2993