1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <linux/time64.h> 49 #include <trace/events/power.h> 50 #include <linux/sched.h> 51 #include <linux/sched/smt.h> 52 #include <linux/mutex.h> 53 #include <linux/notifier.h> 54 #include <linux/cpu.h> 55 #include <linux/moduleparam.h> 56 #include <linux/sysfs.h> 57 #include <asm/cpuid/api.h> 58 #include <asm/cpu_device_id.h> 59 #include <asm/intel-family.h> 60 #include <asm/mwait.h> 61 #include <asm/spec-ctrl.h> 62 #include <asm/msr.h> 63 #include <asm/tsc.h> 64 #include <asm/fpu/api.h> 65 #include <asm/smp.h> 66 67 static struct cpuidle_driver intel_idle_driver = { 68 .name = "intel_idle", 69 .owner = THIS_MODULE, 70 }; 71 /* intel_idle.max_cstate=0 disables driver */ 72 static int max_cstate = CPUIDLE_STATE_MAX - 1; 73 static unsigned int disabled_states_mask __read_mostly; 74 static bool force_irq_on __read_mostly; 75 static bool ibrs_off __read_mostly; 76 77 /* The maximum allowed length for the 'table' module parameter */ 78 #define MAX_CMDLINE_TABLE_LEN 256 79 /* Maximum allowed C-state latency */ 80 #define MAX_CMDLINE_LATENCY_US (5 * USEC_PER_MSEC) 81 /* Maximum allowed C-state target residency */ 82 #define MAX_CMDLINE_RESIDENCY_US (100 * USEC_PER_MSEC) 83 84 static char cmdline_table_str[MAX_CMDLINE_TABLE_LEN] __read_mostly; 85 86 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 87 88 static unsigned long auto_demotion_disable_flags; 89 90 static enum { 91 C1E_PROMOTION_PRESERVE, 92 C1E_PROMOTION_ENABLE, 93 C1E_PROMOTION_DISABLE 94 } c1e_promotion = C1E_PROMOTION_PRESERVE; 95 96 struct idle_cpu { 97 struct cpuidle_state *state_table; 98 99 /* 100 * Hardware C-state auto-demotion may not always be optimal. 101 * Indicate which enable bits to clear here. 102 */ 103 unsigned long auto_demotion_disable_flags; 104 bool disable_promotion_to_c1e; 105 bool c1_demotion_supported; 106 bool use_acpi; 107 }; 108 109 static bool c1_demotion_supported; 110 static DEFINE_MUTEX(c1_demotion_mutex); 111 112 static struct device *sysfs_root __initdata; 113 114 static const struct idle_cpu *icpu __initdata; 115 static struct cpuidle_state *cpuidle_state_table __initdata; 116 117 /* C-states data from the 'intel_idle.table' cmdline parameter */ 118 static struct cpuidle_state cmdline_states[CPUIDLE_STATE_MAX] __initdata; 119 120 static unsigned int mwait_substates __initdata; 121 122 /* 123 * Enable interrupts before entering the C-state. On some platforms and for 124 * some C-states, this may measurably decrease interrupt latency. 125 */ 126 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 127 128 /* 129 * Enable this state by default even if the ACPI _CST does not list it. 130 */ 131 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 132 133 /* 134 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 135 * above. 136 */ 137 #define CPUIDLE_FLAG_IBRS BIT(16) 138 139 /* 140 * Initialize large xstate for the C6-state entrance. 141 */ 142 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 143 144 /* 145 * Ignore the sub-state when matching mwait hints between the ACPI _CST and 146 * custom tables. 147 */ 148 #define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18) 149 150 /* 151 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 152 * the C-state (top nibble) and sub-state (bottom nibble) 153 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 154 * 155 * We store the hint at the top of our "flags" for each state. 156 */ 157 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 158 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 159 160 static __always_inline int __intel_idle(struct cpuidle_device *dev, 161 struct cpuidle_driver *drv, 162 int index, bool irqoff) 163 { 164 struct cpuidle_state *state = &drv->states[index]; 165 unsigned int eax = flg2MWAIT(state->flags); 166 unsigned int ecx = 1*irqoff; /* break on interrupt flag */ 167 168 mwait_idle_with_hints(eax, ecx); 169 170 return index; 171 } 172 173 /** 174 * intel_idle - Ask the processor to enter the given idle state. 175 * @dev: cpuidle device of the target CPU. 176 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 177 * @index: Target idle state index. 178 * 179 * Use the MWAIT instruction to notify the processor that the CPU represented by 180 * @dev is idle and it can try to enter the idle state corresponding to @index. 181 * 182 * If the local APIC timer is not known to be reliable in the target idle state, 183 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 184 * 185 * Must be called under local_irq_disable(). 186 */ 187 static __cpuidle int intel_idle(struct cpuidle_device *dev, 188 struct cpuidle_driver *drv, int index) 189 { 190 return __intel_idle(dev, drv, index, true); 191 } 192 193 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 194 struct cpuidle_driver *drv, int index) 195 { 196 return __intel_idle(dev, drv, index, false); 197 } 198 199 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 200 struct cpuidle_driver *drv, int index) 201 { 202 bool smt_active = sched_smt_active(); 203 u64 spec_ctrl = spec_ctrl_current(); 204 int ret; 205 206 if (smt_active) 207 __update_spec_ctrl(0); 208 209 ret = __intel_idle(dev, drv, index, true); 210 211 if (smt_active) 212 __update_spec_ctrl(spec_ctrl); 213 214 return ret; 215 } 216 217 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 218 struct cpuidle_driver *drv, int index) 219 { 220 fpu_idle_fpregs(); 221 return __intel_idle(dev, drv, index, true); 222 } 223 224 /** 225 * intel_idle_s2idle - Ask the processor to enter the given idle state. 226 * @dev: cpuidle device of the target CPU. 227 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 228 * @index: Target idle state index. 229 * 230 * Use the MWAIT instruction to notify the processor that the CPU represented by 231 * @dev is idle and it can try to enter the idle state corresponding to @index. 232 * 233 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 234 * scheduler tick and suspended scheduler clock on the target CPU. 235 */ 236 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 237 struct cpuidle_driver *drv, int index) 238 { 239 struct cpuidle_state *state = &drv->states[index]; 240 unsigned int eax = flg2MWAIT(state->flags); 241 unsigned int ecx = 1; /* break on interrupt flag */ 242 243 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 244 fpu_idle_fpregs(); 245 246 mwait_idle_with_hints(eax, ecx); 247 248 return 0; 249 } 250 251 static void intel_idle_enter_dead(struct cpuidle_device *dev, int index) 252 { 253 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); 254 struct cpuidle_state *state = &drv->states[index]; 255 unsigned long eax = flg2MWAIT(state->flags); 256 257 mwait_play_dead(eax); 258 } 259 260 /* 261 * States are indexed by the cstate number, 262 * which is also the index into the MWAIT hint array. 263 * Thus C0 is a dummy. 264 */ 265 static struct cpuidle_state nehalem_cstates[] __initdata = { 266 { 267 .name = "C1", 268 .desc = "MWAIT 0x00", 269 .flags = MWAIT2flg(0x00), 270 .exit_latency = 3, 271 .target_residency = 6, 272 .enter = intel_idle, 273 .enter_s2idle = intel_idle_s2idle, }, 274 { 275 .name = "C1E", 276 .desc = "MWAIT 0x01", 277 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 278 .exit_latency = 10, 279 .target_residency = 20, 280 .enter = intel_idle, 281 .enter_s2idle = intel_idle_s2idle, }, 282 { 283 .name = "C3", 284 .desc = "MWAIT 0x10", 285 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 286 .exit_latency = 20, 287 .target_residency = 80, 288 .enter = intel_idle, 289 .enter_s2idle = intel_idle_s2idle, }, 290 { 291 .name = "C6", 292 .desc = "MWAIT 0x20", 293 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 294 .exit_latency = 200, 295 .target_residency = 800, 296 .enter = intel_idle, 297 .enter_s2idle = intel_idle_s2idle, }, 298 { 299 .enter = NULL } 300 }; 301 302 static struct cpuidle_state snb_cstates[] __initdata = { 303 { 304 .name = "C1", 305 .desc = "MWAIT 0x00", 306 .flags = MWAIT2flg(0x00), 307 .exit_latency = 2, 308 .target_residency = 2, 309 .enter = intel_idle, 310 .enter_s2idle = intel_idle_s2idle, }, 311 { 312 .name = "C1E", 313 .desc = "MWAIT 0x01", 314 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 315 .exit_latency = 10, 316 .target_residency = 20, 317 .enter = intel_idle, 318 .enter_s2idle = intel_idle_s2idle, }, 319 { 320 .name = "C3", 321 .desc = "MWAIT 0x10", 322 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 323 .exit_latency = 80, 324 .target_residency = 211, 325 .enter = intel_idle, 326 .enter_s2idle = intel_idle_s2idle, }, 327 { 328 .name = "C6", 329 .desc = "MWAIT 0x20", 330 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 331 .exit_latency = 104, 332 .target_residency = 345, 333 .enter = intel_idle, 334 .enter_s2idle = intel_idle_s2idle, }, 335 { 336 .name = "C7", 337 .desc = "MWAIT 0x30", 338 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 339 .exit_latency = 109, 340 .target_residency = 345, 341 .enter = intel_idle, 342 .enter_s2idle = intel_idle_s2idle, }, 343 { 344 .enter = NULL } 345 }; 346 347 static struct cpuidle_state byt_cstates[] __initdata = { 348 { 349 .name = "C1", 350 .desc = "MWAIT 0x00", 351 .flags = MWAIT2flg(0x00), 352 .exit_latency = 1, 353 .target_residency = 1, 354 .enter = intel_idle, 355 .enter_s2idle = intel_idle_s2idle, }, 356 { 357 .name = "C6N", 358 .desc = "MWAIT 0x58", 359 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 360 .exit_latency = 300, 361 .target_residency = 275, 362 .enter = intel_idle, 363 .enter_s2idle = intel_idle_s2idle, }, 364 { 365 .name = "C6S", 366 .desc = "MWAIT 0x52", 367 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 368 .exit_latency = 500, 369 .target_residency = 560, 370 .enter = intel_idle, 371 .enter_s2idle = intel_idle_s2idle, }, 372 { 373 .name = "C7", 374 .desc = "MWAIT 0x60", 375 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 376 .exit_latency = 1200, 377 .target_residency = 4000, 378 .enter = intel_idle, 379 .enter_s2idle = intel_idle_s2idle, }, 380 { 381 .name = "C7S", 382 .desc = "MWAIT 0x64", 383 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 384 .exit_latency = 10000, 385 .target_residency = 20000, 386 .enter = intel_idle, 387 .enter_s2idle = intel_idle_s2idle, }, 388 { 389 .enter = NULL } 390 }; 391 392 static struct cpuidle_state cht_cstates[] __initdata = { 393 { 394 .name = "C1", 395 .desc = "MWAIT 0x00", 396 .flags = MWAIT2flg(0x00), 397 .exit_latency = 1, 398 .target_residency = 1, 399 .enter = intel_idle, 400 .enter_s2idle = intel_idle_s2idle, }, 401 { 402 .name = "C6N", 403 .desc = "MWAIT 0x58", 404 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 405 .exit_latency = 80, 406 .target_residency = 275, 407 .enter = intel_idle, 408 .enter_s2idle = intel_idle_s2idle, }, 409 { 410 .name = "C6S", 411 .desc = "MWAIT 0x52", 412 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 413 .exit_latency = 200, 414 .target_residency = 560, 415 .enter = intel_idle, 416 .enter_s2idle = intel_idle_s2idle, }, 417 { 418 .name = "C7", 419 .desc = "MWAIT 0x60", 420 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 421 .exit_latency = 1200, 422 .target_residency = 4000, 423 .enter = intel_idle, 424 .enter_s2idle = intel_idle_s2idle, }, 425 { 426 .name = "C7S", 427 .desc = "MWAIT 0x64", 428 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 429 .exit_latency = 10000, 430 .target_residency = 20000, 431 .enter = intel_idle, 432 .enter_s2idle = intel_idle_s2idle, }, 433 { 434 .enter = NULL } 435 }; 436 437 static struct cpuidle_state ivb_cstates[] __initdata = { 438 { 439 .name = "C1", 440 .desc = "MWAIT 0x00", 441 .flags = MWAIT2flg(0x00), 442 .exit_latency = 1, 443 .target_residency = 1, 444 .enter = intel_idle, 445 .enter_s2idle = intel_idle_s2idle, }, 446 { 447 .name = "C1E", 448 .desc = "MWAIT 0x01", 449 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 450 .exit_latency = 10, 451 .target_residency = 20, 452 .enter = intel_idle, 453 .enter_s2idle = intel_idle_s2idle, }, 454 { 455 .name = "C3", 456 .desc = "MWAIT 0x10", 457 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 458 .exit_latency = 59, 459 .target_residency = 156, 460 .enter = intel_idle, 461 .enter_s2idle = intel_idle_s2idle, }, 462 { 463 .name = "C6", 464 .desc = "MWAIT 0x20", 465 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 466 .exit_latency = 80, 467 .target_residency = 300, 468 .enter = intel_idle, 469 .enter_s2idle = intel_idle_s2idle, }, 470 { 471 .name = "C7", 472 .desc = "MWAIT 0x30", 473 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 474 .exit_latency = 87, 475 .target_residency = 300, 476 .enter = intel_idle, 477 .enter_s2idle = intel_idle_s2idle, }, 478 { 479 .enter = NULL } 480 }; 481 482 static struct cpuidle_state ivt_cstates[] __initdata = { 483 { 484 .name = "C1", 485 .desc = "MWAIT 0x00", 486 .flags = MWAIT2flg(0x00), 487 .exit_latency = 1, 488 .target_residency = 1, 489 .enter = intel_idle, 490 .enter_s2idle = intel_idle_s2idle, }, 491 { 492 .name = "C1E", 493 .desc = "MWAIT 0x01", 494 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 495 .exit_latency = 10, 496 .target_residency = 80, 497 .enter = intel_idle, 498 .enter_s2idle = intel_idle_s2idle, }, 499 { 500 .name = "C3", 501 .desc = "MWAIT 0x10", 502 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 503 .exit_latency = 59, 504 .target_residency = 156, 505 .enter = intel_idle, 506 .enter_s2idle = intel_idle_s2idle, }, 507 { 508 .name = "C6", 509 .desc = "MWAIT 0x20", 510 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 511 .exit_latency = 82, 512 .target_residency = 300, 513 .enter = intel_idle, 514 .enter_s2idle = intel_idle_s2idle, }, 515 { 516 .enter = NULL } 517 }; 518 519 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 520 { 521 .name = "C1", 522 .desc = "MWAIT 0x00", 523 .flags = MWAIT2flg(0x00), 524 .exit_latency = 1, 525 .target_residency = 1, 526 .enter = intel_idle, 527 .enter_s2idle = intel_idle_s2idle, }, 528 { 529 .name = "C1E", 530 .desc = "MWAIT 0x01", 531 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 532 .exit_latency = 10, 533 .target_residency = 250, 534 .enter = intel_idle, 535 .enter_s2idle = intel_idle_s2idle, }, 536 { 537 .name = "C3", 538 .desc = "MWAIT 0x10", 539 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 540 .exit_latency = 59, 541 .target_residency = 300, 542 .enter = intel_idle, 543 .enter_s2idle = intel_idle_s2idle, }, 544 { 545 .name = "C6", 546 .desc = "MWAIT 0x20", 547 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 548 .exit_latency = 84, 549 .target_residency = 400, 550 .enter = intel_idle, 551 .enter_s2idle = intel_idle_s2idle, }, 552 { 553 .enter = NULL } 554 }; 555 556 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 557 { 558 .name = "C1", 559 .desc = "MWAIT 0x00", 560 .flags = MWAIT2flg(0x00), 561 .exit_latency = 1, 562 .target_residency = 1, 563 .enter = intel_idle, 564 .enter_s2idle = intel_idle_s2idle, }, 565 { 566 .name = "C1E", 567 .desc = "MWAIT 0x01", 568 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 569 .exit_latency = 10, 570 .target_residency = 500, 571 .enter = intel_idle, 572 .enter_s2idle = intel_idle_s2idle, }, 573 { 574 .name = "C3", 575 .desc = "MWAIT 0x10", 576 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 577 .exit_latency = 59, 578 .target_residency = 600, 579 .enter = intel_idle, 580 .enter_s2idle = intel_idle_s2idle, }, 581 { 582 .name = "C6", 583 .desc = "MWAIT 0x20", 584 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 585 .exit_latency = 88, 586 .target_residency = 700, 587 .enter = intel_idle, 588 .enter_s2idle = intel_idle_s2idle, }, 589 { 590 .enter = NULL } 591 }; 592 593 static struct cpuidle_state hsw_cstates[] __initdata = { 594 { 595 .name = "C1", 596 .desc = "MWAIT 0x00", 597 .flags = MWAIT2flg(0x00), 598 .exit_latency = 2, 599 .target_residency = 2, 600 .enter = intel_idle, 601 .enter_s2idle = intel_idle_s2idle, }, 602 { 603 .name = "C1E", 604 .desc = "MWAIT 0x01", 605 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 606 .exit_latency = 10, 607 .target_residency = 20, 608 .enter = intel_idle, 609 .enter_s2idle = intel_idle_s2idle, }, 610 { 611 .name = "C3", 612 .desc = "MWAIT 0x10", 613 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 614 .exit_latency = 33, 615 .target_residency = 100, 616 .enter = intel_idle, 617 .enter_s2idle = intel_idle_s2idle, }, 618 { 619 .name = "C6", 620 .desc = "MWAIT 0x20", 621 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 622 .exit_latency = 133, 623 .target_residency = 400, 624 .enter = intel_idle, 625 .enter_s2idle = intel_idle_s2idle, }, 626 { 627 .name = "C7s", 628 .desc = "MWAIT 0x32", 629 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 630 .exit_latency = 166, 631 .target_residency = 500, 632 .enter = intel_idle, 633 .enter_s2idle = intel_idle_s2idle, }, 634 { 635 .name = "C8", 636 .desc = "MWAIT 0x40", 637 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 638 .exit_latency = 300, 639 .target_residency = 900, 640 .enter = intel_idle, 641 .enter_s2idle = intel_idle_s2idle, }, 642 { 643 .name = "C9", 644 .desc = "MWAIT 0x50", 645 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 646 .exit_latency = 600, 647 .target_residency = 1800, 648 .enter = intel_idle, 649 .enter_s2idle = intel_idle_s2idle, }, 650 { 651 .name = "C10", 652 .desc = "MWAIT 0x60", 653 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 654 .exit_latency = 2600, 655 .target_residency = 7700, 656 .enter = intel_idle, 657 .enter_s2idle = intel_idle_s2idle, }, 658 { 659 .enter = NULL } 660 }; 661 static struct cpuidle_state bdw_cstates[] __initdata = { 662 { 663 .name = "C1", 664 .desc = "MWAIT 0x00", 665 .flags = MWAIT2flg(0x00), 666 .exit_latency = 2, 667 .target_residency = 2, 668 .enter = intel_idle, 669 .enter_s2idle = intel_idle_s2idle, }, 670 { 671 .name = "C1E", 672 .desc = "MWAIT 0x01", 673 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 674 .exit_latency = 10, 675 .target_residency = 20, 676 .enter = intel_idle, 677 .enter_s2idle = intel_idle_s2idle, }, 678 { 679 .name = "C3", 680 .desc = "MWAIT 0x10", 681 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 682 .exit_latency = 40, 683 .target_residency = 100, 684 .enter = intel_idle, 685 .enter_s2idle = intel_idle_s2idle, }, 686 { 687 .name = "C6", 688 .desc = "MWAIT 0x20", 689 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 690 .exit_latency = 133, 691 .target_residency = 400, 692 .enter = intel_idle, 693 .enter_s2idle = intel_idle_s2idle, }, 694 { 695 .name = "C7s", 696 .desc = "MWAIT 0x32", 697 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 698 .exit_latency = 166, 699 .target_residency = 500, 700 .enter = intel_idle, 701 .enter_s2idle = intel_idle_s2idle, }, 702 { 703 .name = "C8", 704 .desc = "MWAIT 0x40", 705 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 706 .exit_latency = 300, 707 .target_residency = 900, 708 .enter = intel_idle, 709 .enter_s2idle = intel_idle_s2idle, }, 710 { 711 .name = "C9", 712 .desc = "MWAIT 0x50", 713 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 714 .exit_latency = 600, 715 .target_residency = 1800, 716 .enter = intel_idle, 717 .enter_s2idle = intel_idle_s2idle, }, 718 { 719 .name = "C10", 720 .desc = "MWAIT 0x60", 721 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 722 .exit_latency = 2600, 723 .target_residency = 7700, 724 .enter = intel_idle, 725 .enter_s2idle = intel_idle_s2idle, }, 726 { 727 .enter = NULL } 728 }; 729 730 static struct cpuidle_state skl_cstates[] __initdata = { 731 { 732 .name = "C1", 733 .desc = "MWAIT 0x00", 734 .flags = MWAIT2flg(0x00), 735 .exit_latency = 2, 736 .target_residency = 2, 737 .enter = intel_idle, 738 .enter_s2idle = intel_idle_s2idle, }, 739 { 740 .name = "C1E", 741 .desc = "MWAIT 0x01", 742 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 743 .exit_latency = 10, 744 .target_residency = 20, 745 .enter = intel_idle, 746 .enter_s2idle = intel_idle_s2idle, }, 747 { 748 .name = "C3", 749 .desc = "MWAIT 0x10", 750 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 751 .exit_latency = 70, 752 .target_residency = 100, 753 .enter = intel_idle, 754 .enter_s2idle = intel_idle_s2idle, }, 755 { 756 .name = "C6", 757 .desc = "MWAIT 0x20", 758 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 759 .exit_latency = 85, 760 .target_residency = 200, 761 .enter = intel_idle, 762 .enter_s2idle = intel_idle_s2idle, }, 763 { 764 .name = "C7s", 765 .desc = "MWAIT 0x33", 766 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 767 .exit_latency = 124, 768 .target_residency = 800, 769 .enter = intel_idle, 770 .enter_s2idle = intel_idle_s2idle, }, 771 { 772 .name = "C8", 773 .desc = "MWAIT 0x40", 774 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 775 .exit_latency = 200, 776 .target_residency = 800, 777 .enter = intel_idle, 778 .enter_s2idle = intel_idle_s2idle, }, 779 { 780 .name = "C9", 781 .desc = "MWAIT 0x50", 782 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 783 .exit_latency = 480, 784 .target_residency = 5000, 785 .enter = intel_idle, 786 .enter_s2idle = intel_idle_s2idle, }, 787 { 788 .name = "C10", 789 .desc = "MWAIT 0x60", 790 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 791 .exit_latency = 890, 792 .target_residency = 5000, 793 .enter = intel_idle, 794 .enter_s2idle = intel_idle_s2idle, }, 795 { 796 .enter = NULL } 797 }; 798 799 static struct cpuidle_state skx_cstates[] __initdata = { 800 { 801 .name = "C1", 802 .desc = "MWAIT 0x00", 803 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 804 .exit_latency = 2, 805 .target_residency = 2, 806 .enter = intel_idle, 807 .enter_s2idle = intel_idle_s2idle, }, 808 { 809 .name = "C1E", 810 .desc = "MWAIT 0x01", 811 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 812 .exit_latency = 10, 813 .target_residency = 20, 814 .enter = intel_idle, 815 .enter_s2idle = intel_idle_s2idle, }, 816 { 817 .name = "C6", 818 .desc = "MWAIT 0x20", 819 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 820 .exit_latency = 133, 821 .target_residency = 600, 822 .enter = intel_idle, 823 .enter_s2idle = intel_idle_s2idle, }, 824 { 825 .enter = NULL } 826 }; 827 828 static struct cpuidle_state icx_cstates[] __initdata = { 829 { 830 .name = "C1", 831 .desc = "MWAIT 0x00", 832 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 833 .exit_latency = 1, 834 .target_residency = 1, 835 .enter = intel_idle, 836 .enter_s2idle = intel_idle_s2idle, }, 837 { 838 .name = "C1E", 839 .desc = "MWAIT 0x01", 840 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 841 .exit_latency = 4, 842 .target_residency = 4, 843 .enter = intel_idle, 844 .enter_s2idle = intel_idle_s2idle, }, 845 { 846 .name = "C6", 847 .desc = "MWAIT 0x20", 848 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 849 .exit_latency = 170, 850 .target_residency = 600, 851 .enter = intel_idle, 852 .enter_s2idle = intel_idle_s2idle, }, 853 { 854 .enter = NULL } 855 }; 856 857 /* 858 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 859 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 860 * But in this case there is effectively no C1, because C1 requests are 861 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 862 * and C1E requests end up with C1, so there is effectively no C1E. 863 * 864 * By default we enable C1E and disable C1 by marking it with 865 * 'CPUIDLE_FLAG_UNUSABLE'. 866 */ 867 static struct cpuidle_state adl_cstates[] __initdata = { 868 { 869 .name = "C1", 870 .desc = "MWAIT 0x00", 871 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 872 .exit_latency = 1, 873 .target_residency = 1, 874 .enter = intel_idle, 875 .enter_s2idle = intel_idle_s2idle, }, 876 { 877 .name = "C1E", 878 .desc = "MWAIT 0x01", 879 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 880 .exit_latency = 2, 881 .target_residency = 4, 882 .enter = intel_idle, 883 .enter_s2idle = intel_idle_s2idle, }, 884 { 885 .name = "C6", 886 .desc = "MWAIT 0x20", 887 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 888 .exit_latency = 220, 889 .target_residency = 600, 890 .enter = intel_idle, 891 .enter_s2idle = intel_idle_s2idle, }, 892 { 893 .name = "C8", 894 .desc = "MWAIT 0x40", 895 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 896 .exit_latency = 280, 897 .target_residency = 800, 898 .enter = intel_idle, 899 .enter_s2idle = intel_idle_s2idle, }, 900 { 901 .name = "C10", 902 .desc = "MWAIT 0x60", 903 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 904 .exit_latency = 680, 905 .target_residency = 2000, 906 .enter = intel_idle, 907 .enter_s2idle = intel_idle_s2idle, }, 908 { 909 .enter = NULL } 910 }; 911 912 static struct cpuidle_state adl_l_cstates[] __initdata = { 913 { 914 .name = "C1", 915 .desc = "MWAIT 0x00", 916 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 917 .exit_latency = 1, 918 .target_residency = 1, 919 .enter = intel_idle, 920 .enter_s2idle = intel_idle_s2idle, }, 921 { 922 .name = "C1E", 923 .desc = "MWAIT 0x01", 924 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 925 .exit_latency = 2, 926 .target_residency = 4, 927 .enter = intel_idle, 928 .enter_s2idle = intel_idle_s2idle, }, 929 { 930 .name = "C6", 931 .desc = "MWAIT 0x20", 932 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 933 .exit_latency = 170, 934 .target_residency = 500, 935 .enter = intel_idle, 936 .enter_s2idle = intel_idle_s2idle, }, 937 { 938 .name = "C8", 939 .desc = "MWAIT 0x40", 940 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 941 .exit_latency = 200, 942 .target_residency = 600, 943 .enter = intel_idle, 944 .enter_s2idle = intel_idle_s2idle, }, 945 { 946 .name = "C10", 947 .desc = "MWAIT 0x60", 948 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 949 .exit_latency = 230, 950 .target_residency = 700, 951 .enter = intel_idle, 952 .enter_s2idle = intel_idle_s2idle, }, 953 { 954 .enter = NULL } 955 }; 956 957 static struct cpuidle_state mtl_l_cstates[] __initdata = { 958 { 959 .name = "C1E", 960 .desc = "MWAIT 0x01", 961 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 962 .exit_latency = 1, 963 .target_residency = 1, 964 .enter = intel_idle, 965 .enter_s2idle = intel_idle_s2idle, }, 966 { 967 .name = "C6", 968 .desc = "MWAIT 0x20", 969 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 970 .exit_latency = 140, 971 .target_residency = 420, 972 .enter = intel_idle, 973 .enter_s2idle = intel_idle_s2idle, }, 974 { 975 .name = "C10", 976 .desc = "MWAIT 0x60", 977 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 978 .exit_latency = 310, 979 .target_residency = 930, 980 .enter = intel_idle, 981 .enter_s2idle = intel_idle_s2idle, }, 982 { 983 .enter = NULL } 984 }; 985 986 static struct cpuidle_state gmt_cstates[] __initdata = { 987 { 988 .name = "C1", 989 .desc = "MWAIT 0x00", 990 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 991 .exit_latency = 1, 992 .target_residency = 1, 993 .enter = intel_idle, 994 .enter_s2idle = intel_idle_s2idle, }, 995 { 996 .name = "C1E", 997 .desc = "MWAIT 0x01", 998 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 999 .exit_latency = 2, 1000 .target_residency = 4, 1001 .enter = intel_idle, 1002 .enter_s2idle = intel_idle_s2idle, }, 1003 { 1004 .name = "C6", 1005 .desc = "MWAIT 0x20", 1006 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1007 .exit_latency = 195, 1008 .target_residency = 585, 1009 .enter = intel_idle, 1010 .enter_s2idle = intel_idle_s2idle, }, 1011 { 1012 .name = "C8", 1013 .desc = "MWAIT 0x40", 1014 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1015 .exit_latency = 260, 1016 .target_residency = 1040, 1017 .enter = intel_idle, 1018 .enter_s2idle = intel_idle_s2idle, }, 1019 { 1020 .name = "C10", 1021 .desc = "MWAIT 0x60", 1022 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1023 .exit_latency = 660, 1024 .target_residency = 1980, 1025 .enter = intel_idle, 1026 .enter_s2idle = intel_idle_s2idle, }, 1027 { 1028 .enter = NULL } 1029 }; 1030 1031 static struct cpuidle_state spr_cstates[] __initdata = { 1032 { 1033 .name = "C1", 1034 .desc = "MWAIT 0x00", 1035 .flags = MWAIT2flg(0x00), 1036 .exit_latency = 1, 1037 .target_residency = 1, 1038 .enter = intel_idle, 1039 .enter_s2idle = intel_idle_s2idle, }, 1040 { 1041 .name = "C1E", 1042 .desc = "MWAIT 0x01", 1043 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1044 .exit_latency = 2, 1045 .target_residency = 4, 1046 .enter = intel_idle, 1047 .enter_s2idle = intel_idle_s2idle, }, 1048 { 1049 .name = "C6", 1050 .desc = "MWAIT 0x20", 1051 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1052 CPUIDLE_FLAG_INIT_XSTATE, 1053 .exit_latency = 290, 1054 .target_residency = 800, 1055 .enter = intel_idle, 1056 .enter_s2idle = intel_idle_s2idle, }, 1057 { 1058 .enter = NULL } 1059 }; 1060 1061 static struct cpuidle_state gnr_cstates[] __initdata = { 1062 { 1063 .name = "C1", 1064 .desc = "MWAIT 0x00", 1065 .flags = MWAIT2flg(0x00), 1066 .exit_latency = 1, 1067 .target_residency = 1, 1068 .enter = intel_idle, 1069 .enter_s2idle = intel_idle_s2idle, }, 1070 { 1071 .name = "C1E", 1072 .desc = "MWAIT 0x01", 1073 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1074 .exit_latency = 4, 1075 .target_residency = 4, 1076 .enter = intel_idle, 1077 .enter_s2idle = intel_idle_s2idle, }, 1078 { 1079 .name = "C6", 1080 .desc = "MWAIT 0x20", 1081 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1082 CPUIDLE_FLAG_INIT_XSTATE | 1083 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1084 .exit_latency = 170, 1085 .target_residency = 650, 1086 .enter = intel_idle, 1087 .enter_s2idle = intel_idle_s2idle, }, 1088 { 1089 .name = "C6P", 1090 .desc = "MWAIT 0x21", 1091 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1092 CPUIDLE_FLAG_INIT_XSTATE | 1093 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1094 .exit_latency = 210, 1095 .target_residency = 1000, 1096 .enter = intel_idle, 1097 .enter_s2idle = intel_idle_s2idle, }, 1098 { 1099 .enter = NULL } 1100 }; 1101 1102 static struct cpuidle_state gnrd_cstates[] __initdata = { 1103 { 1104 .name = "C1", 1105 .desc = "MWAIT 0x00", 1106 .flags = MWAIT2flg(0x00), 1107 .exit_latency = 1, 1108 .target_residency = 1, 1109 .enter = intel_idle, 1110 .enter_s2idle = intel_idle_s2idle, }, 1111 { 1112 .name = "C1E", 1113 .desc = "MWAIT 0x01", 1114 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1115 .exit_latency = 4, 1116 .target_residency = 4, 1117 .enter = intel_idle, 1118 .enter_s2idle = intel_idle_s2idle, }, 1119 { 1120 .name = "C6", 1121 .desc = "MWAIT 0x20", 1122 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1123 CPUIDLE_FLAG_INIT_XSTATE | 1124 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1125 .exit_latency = 220, 1126 .target_residency = 650, 1127 .enter = intel_idle, 1128 .enter_s2idle = intel_idle_s2idle, }, 1129 { 1130 .name = "C6P", 1131 .desc = "MWAIT 0x21", 1132 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1133 CPUIDLE_FLAG_INIT_XSTATE | 1134 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1135 .exit_latency = 240, 1136 .target_residency = 750, 1137 .enter = intel_idle, 1138 .enter_s2idle = intel_idle_s2idle, }, 1139 { 1140 .enter = NULL } 1141 }; 1142 1143 static struct cpuidle_state atom_cstates[] __initdata = { 1144 { 1145 .name = "C1E", 1146 .desc = "MWAIT 0x00", 1147 .flags = MWAIT2flg(0x00), 1148 .exit_latency = 10, 1149 .target_residency = 20, 1150 .enter = intel_idle, 1151 .enter_s2idle = intel_idle_s2idle, }, 1152 { 1153 .name = "C2", 1154 .desc = "MWAIT 0x10", 1155 .flags = MWAIT2flg(0x10), 1156 .exit_latency = 20, 1157 .target_residency = 80, 1158 .enter = intel_idle, 1159 .enter_s2idle = intel_idle_s2idle, }, 1160 { 1161 .name = "C4", 1162 .desc = "MWAIT 0x30", 1163 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1164 .exit_latency = 100, 1165 .target_residency = 400, 1166 .enter = intel_idle, 1167 .enter_s2idle = intel_idle_s2idle, }, 1168 { 1169 .name = "C6", 1170 .desc = "MWAIT 0x52", 1171 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1172 .exit_latency = 140, 1173 .target_residency = 560, 1174 .enter = intel_idle, 1175 .enter_s2idle = intel_idle_s2idle, }, 1176 { 1177 .enter = NULL } 1178 }; 1179 static struct cpuidle_state tangier_cstates[] __initdata = { 1180 { 1181 .name = "C1", 1182 .desc = "MWAIT 0x00", 1183 .flags = MWAIT2flg(0x00), 1184 .exit_latency = 1, 1185 .target_residency = 4, 1186 .enter = intel_idle, 1187 .enter_s2idle = intel_idle_s2idle, }, 1188 { 1189 .name = "C4", 1190 .desc = "MWAIT 0x30", 1191 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1192 .exit_latency = 100, 1193 .target_residency = 400, 1194 .enter = intel_idle, 1195 .enter_s2idle = intel_idle_s2idle, }, 1196 { 1197 .name = "C6", 1198 .desc = "MWAIT 0x52", 1199 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1200 .exit_latency = 140, 1201 .target_residency = 560, 1202 .enter = intel_idle, 1203 .enter_s2idle = intel_idle_s2idle, }, 1204 { 1205 .name = "C7", 1206 .desc = "MWAIT 0x60", 1207 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1208 .exit_latency = 1200, 1209 .target_residency = 4000, 1210 .enter = intel_idle, 1211 .enter_s2idle = intel_idle_s2idle, }, 1212 { 1213 .name = "C9", 1214 .desc = "MWAIT 0x64", 1215 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1216 .exit_latency = 10000, 1217 .target_residency = 20000, 1218 .enter = intel_idle, 1219 .enter_s2idle = intel_idle_s2idle, }, 1220 { 1221 .enter = NULL } 1222 }; 1223 static struct cpuidle_state avn_cstates[] __initdata = { 1224 { 1225 .name = "C1", 1226 .desc = "MWAIT 0x00", 1227 .flags = MWAIT2flg(0x00), 1228 .exit_latency = 2, 1229 .target_residency = 2, 1230 .enter = intel_idle, 1231 .enter_s2idle = intel_idle_s2idle, }, 1232 { 1233 .name = "C6", 1234 .desc = "MWAIT 0x51", 1235 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1236 .exit_latency = 15, 1237 .target_residency = 45, 1238 .enter = intel_idle, 1239 .enter_s2idle = intel_idle_s2idle, }, 1240 { 1241 .enter = NULL } 1242 }; 1243 static struct cpuidle_state knl_cstates[] __initdata = { 1244 { 1245 .name = "C1", 1246 .desc = "MWAIT 0x00", 1247 .flags = MWAIT2flg(0x00), 1248 .exit_latency = 1, 1249 .target_residency = 2, 1250 .enter = intel_idle, 1251 .enter_s2idle = intel_idle_s2idle }, 1252 { 1253 .name = "C6", 1254 .desc = "MWAIT 0x10", 1255 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1256 .exit_latency = 120, 1257 .target_residency = 500, 1258 .enter = intel_idle, 1259 .enter_s2idle = intel_idle_s2idle }, 1260 { 1261 .enter = NULL } 1262 }; 1263 1264 static struct cpuidle_state bxt_cstates[] __initdata = { 1265 { 1266 .name = "C1", 1267 .desc = "MWAIT 0x00", 1268 .flags = MWAIT2flg(0x00), 1269 .exit_latency = 2, 1270 .target_residency = 2, 1271 .enter = intel_idle, 1272 .enter_s2idle = intel_idle_s2idle, }, 1273 { 1274 .name = "C1E", 1275 .desc = "MWAIT 0x01", 1276 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1277 .exit_latency = 10, 1278 .target_residency = 20, 1279 .enter = intel_idle, 1280 .enter_s2idle = intel_idle_s2idle, }, 1281 { 1282 .name = "C6", 1283 .desc = "MWAIT 0x20", 1284 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1285 .exit_latency = 133, 1286 .target_residency = 133, 1287 .enter = intel_idle, 1288 .enter_s2idle = intel_idle_s2idle, }, 1289 { 1290 .name = "C7s", 1291 .desc = "MWAIT 0x31", 1292 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1293 .exit_latency = 155, 1294 .target_residency = 155, 1295 .enter = intel_idle, 1296 .enter_s2idle = intel_idle_s2idle, }, 1297 { 1298 .name = "C8", 1299 .desc = "MWAIT 0x40", 1300 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1301 .exit_latency = 1000, 1302 .target_residency = 1000, 1303 .enter = intel_idle, 1304 .enter_s2idle = intel_idle_s2idle, }, 1305 { 1306 .name = "C9", 1307 .desc = "MWAIT 0x50", 1308 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1309 .exit_latency = 2000, 1310 .target_residency = 2000, 1311 .enter = intel_idle, 1312 .enter_s2idle = intel_idle_s2idle, }, 1313 { 1314 .name = "C10", 1315 .desc = "MWAIT 0x60", 1316 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1317 .exit_latency = 10000, 1318 .target_residency = 10000, 1319 .enter = intel_idle, 1320 .enter_s2idle = intel_idle_s2idle, }, 1321 { 1322 .enter = NULL } 1323 }; 1324 1325 static struct cpuidle_state dnv_cstates[] __initdata = { 1326 { 1327 .name = "C1", 1328 .desc = "MWAIT 0x00", 1329 .flags = MWAIT2flg(0x00), 1330 .exit_latency = 2, 1331 .target_residency = 2, 1332 .enter = intel_idle, 1333 .enter_s2idle = intel_idle_s2idle, }, 1334 { 1335 .name = "C1E", 1336 .desc = "MWAIT 0x01", 1337 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1338 .exit_latency = 10, 1339 .target_residency = 20, 1340 .enter = intel_idle, 1341 .enter_s2idle = intel_idle_s2idle, }, 1342 { 1343 .name = "C6", 1344 .desc = "MWAIT 0x20", 1345 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1346 .exit_latency = 50, 1347 .target_residency = 500, 1348 .enter = intel_idle, 1349 .enter_s2idle = intel_idle_s2idle, }, 1350 { 1351 .enter = NULL } 1352 }; 1353 1354 /* 1355 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1356 * C6, and this is indicated in the CPUID mwait leaf. 1357 */ 1358 static struct cpuidle_state snr_cstates[] __initdata = { 1359 { 1360 .name = "C1", 1361 .desc = "MWAIT 0x00", 1362 .flags = MWAIT2flg(0x00), 1363 .exit_latency = 2, 1364 .target_residency = 2, 1365 .enter = intel_idle, 1366 .enter_s2idle = intel_idle_s2idle, }, 1367 { 1368 .name = "C1E", 1369 .desc = "MWAIT 0x01", 1370 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1371 .exit_latency = 15, 1372 .target_residency = 25, 1373 .enter = intel_idle, 1374 .enter_s2idle = intel_idle_s2idle, }, 1375 { 1376 .name = "C6", 1377 .desc = "MWAIT 0x20", 1378 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1379 .exit_latency = 130, 1380 .target_residency = 500, 1381 .enter = intel_idle, 1382 .enter_s2idle = intel_idle_s2idle, }, 1383 { 1384 .enter = NULL } 1385 }; 1386 1387 static struct cpuidle_state grr_cstates[] __initdata = { 1388 { 1389 .name = "C1", 1390 .desc = "MWAIT 0x00", 1391 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1392 .exit_latency = 1, 1393 .target_residency = 1, 1394 .enter = intel_idle, 1395 .enter_s2idle = intel_idle_s2idle, }, 1396 { 1397 .name = "C1E", 1398 .desc = "MWAIT 0x01", 1399 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1400 .exit_latency = 2, 1401 .target_residency = 10, 1402 .enter = intel_idle, 1403 .enter_s2idle = intel_idle_s2idle, }, 1404 { 1405 .name = "C6S", 1406 .desc = "MWAIT 0x22", 1407 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1408 .exit_latency = 140, 1409 .target_residency = 500, 1410 .enter = intel_idle, 1411 .enter_s2idle = intel_idle_s2idle, }, 1412 { 1413 .enter = NULL } 1414 }; 1415 1416 static struct cpuidle_state srf_cstates[] __initdata = { 1417 { 1418 .name = "C1", 1419 .desc = "MWAIT 0x00", 1420 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1421 .exit_latency = 1, 1422 .target_residency = 1, 1423 .enter = intel_idle, 1424 .enter_s2idle = intel_idle_s2idle, }, 1425 { 1426 .name = "C1E", 1427 .desc = "MWAIT 0x01", 1428 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1429 .exit_latency = 2, 1430 .target_residency = 10, 1431 .enter = intel_idle, 1432 .enter_s2idle = intel_idle_s2idle, }, 1433 { 1434 .name = "C6S", 1435 .desc = "MWAIT 0x22", 1436 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED | 1437 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1438 .exit_latency = 270, 1439 .target_residency = 700, 1440 .enter = intel_idle, 1441 .enter_s2idle = intel_idle_s2idle, }, 1442 { 1443 .name = "C6SP", 1444 .desc = "MWAIT 0x23", 1445 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED | 1446 CPUIDLE_FLAG_PARTIAL_HINT_MATCH, 1447 .exit_latency = 310, 1448 .target_residency = 900, 1449 .enter = intel_idle, 1450 .enter_s2idle = intel_idle_s2idle, }, 1451 { 1452 .enter = NULL } 1453 }; 1454 1455 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1456 .state_table = nehalem_cstates, 1457 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1458 .disable_promotion_to_c1e = true, 1459 }; 1460 1461 static const struct idle_cpu idle_cpu_nhx __initconst = { 1462 .state_table = nehalem_cstates, 1463 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1464 .disable_promotion_to_c1e = true, 1465 .use_acpi = true, 1466 }; 1467 1468 static const struct idle_cpu idle_cpu_atom __initconst = { 1469 .state_table = atom_cstates, 1470 }; 1471 1472 static const struct idle_cpu idle_cpu_tangier __initconst = { 1473 .state_table = tangier_cstates, 1474 }; 1475 1476 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1477 .state_table = atom_cstates, 1478 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1479 }; 1480 1481 static const struct idle_cpu idle_cpu_snb __initconst = { 1482 .state_table = snb_cstates, 1483 .disable_promotion_to_c1e = true, 1484 }; 1485 1486 static const struct idle_cpu idle_cpu_snx __initconst = { 1487 .state_table = snb_cstates, 1488 .disable_promotion_to_c1e = true, 1489 .use_acpi = true, 1490 }; 1491 1492 static const struct idle_cpu idle_cpu_byt __initconst = { 1493 .state_table = byt_cstates, 1494 .disable_promotion_to_c1e = true, 1495 }; 1496 1497 static const struct idle_cpu idle_cpu_cht __initconst = { 1498 .state_table = cht_cstates, 1499 .disable_promotion_to_c1e = true, 1500 }; 1501 1502 static const struct idle_cpu idle_cpu_ivb __initconst = { 1503 .state_table = ivb_cstates, 1504 .disable_promotion_to_c1e = true, 1505 }; 1506 1507 static const struct idle_cpu idle_cpu_ivt __initconst = { 1508 .state_table = ivt_cstates, 1509 .disable_promotion_to_c1e = true, 1510 .use_acpi = true, 1511 }; 1512 1513 static const struct idle_cpu idle_cpu_hsw __initconst = { 1514 .state_table = hsw_cstates, 1515 .disable_promotion_to_c1e = true, 1516 }; 1517 1518 static const struct idle_cpu idle_cpu_hsx __initconst = { 1519 .state_table = hsw_cstates, 1520 .disable_promotion_to_c1e = true, 1521 .use_acpi = true, 1522 }; 1523 1524 static const struct idle_cpu idle_cpu_bdw __initconst = { 1525 .state_table = bdw_cstates, 1526 .disable_promotion_to_c1e = true, 1527 }; 1528 1529 static const struct idle_cpu idle_cpu_bdx __initconst = { 1530 .state_table = bdw_cstates, 1531 .disable_promotion_to_c1e = true, 1532 .use_acpi = true, 1533 }; 1534 1535 static const struct idle_cpu idle_cpu_skl __initconst = { 1536 .state_table = skl_cstates, 1537 .disable_promotion_to_c1e = true, 1538 }; 1539 1540 static const struct idle_cpu idle_cpu_skx __initconst = { 1541 .state_table = skx_cstates, 1542 .disable_promotion_to_c1e = true, 1543 .use_acpi = true, 1544 }; 1545 1546 static const struct idle_cpu idle_cpu_icx __initconst = { 1547 .state_table = icx_cstates, 1548 .disable_promotion_to_c1e = true, 1549 .use_acpi = true, 1550 }; 1551 1552 static const struct idle_cpu idle_cpu_adl __initconst = { 1553 .state_table = adl_cstates, 1554 }; 1555 1556 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1557 .state_table = adl_l_cstates, 1558 }; 1559 1560 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1561 .state_table = mtl_l_cstates, 1562 }; 1563 1564 static const struct idle_cpu idle_cpu_gmt __initconst = { 1565 .state_table = gmt_cstates, 1566 }; 1567 1568 static const struct idle_cpu idle_cpu_spr __initconst = { 1569 .state_table = spr_cstates, 1570 .disable_promotion_to_c1e = true, 1571 .c1_demotion_supported = true, 1572 .use_acpi = true, 1573 }; 1574 1575 static const struct idle_cpu idle_cpu_gnr __initconst = { 1576 .state_table = gnr_cstates, 1577 .disable_promotion_to_c1e = true, 1578 .c1_demotion_supported = true, 1579 .use_acpi = true, 1580 }; 1581 1582 static const struct idle_cpu idle_cpu_gnrd __initconst = { 1583 .state_table = gnrd_cstates, 1584 .disable_promotion_to_c1e = true, 1585 .c1_demotion_supported = true, 1586 .use_acpi = true, 1587 }; 1588 1589 static const struct idle_cpu idle_cpu_avn __initconst = { 1590 .state_table = avn_cstates, 1591 .disable_promotion_to_c1e = true, 1592 .use_acpi = true, 1593 }; 1594 1595 static const struct idle_cpu idle_cpu_knl __initconst = { 1596 .state_table = knl_cstates, 1597 .use_acpi = true, 1598 }; 1599 1600 static const struct idle_cpu idle_cpu_bxt __initconst = { 1601 .state_table = bxt_cstates, 1602 .disable_promotion_to_c1e = true, 1603 }; 1604 1605 static const struct idle_cpu idle_cpu_dnv __initconst = { 1606 .state_table = dnv_cstates, 1607 .disable_promotion_to_c1e = true, 1608 .use_acpi = true, 1609 }; 1610 1611 static const struct idle_cpu idle_cpu_tmt __initconst = { 1612 .disable_promotion_to_c1e = true, 1613 }; 1614 1615 static const struct idle_cpu idle_cpu_snr __initconst = { 1616 .state_table = snr_cstates, 1617 .disable_promotion_to_c1e = true, 1618 .use_acpi = true, 1619 }; 1620 1621 static const struct idle_cpu idle_cpu_grr __initconst = { 1622 .state_table = grr_cstates, 1623 .disable_promotion_to_c1e = true, 1624 .c1_demotion_supported = true, 1625 .use_acpi = true, 1626 }; 1627 1628 static const struct idle_cpu idle_cpu_srf __initconst = { 1629 .state_table = srf_cstates, 1630 .disable_promotion_to_c1e = true, 1631 .c1_demotion_supported = true, 1632 .use_acpi = true, 1633 }; 1634 1635 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1636 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1637 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1638 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1639 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1640 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1641 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1642 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1643 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1644 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1645 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1646 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1647 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1648 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1649 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1650 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1651 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1652 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1653 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1654 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1655 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1656 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1657 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1658 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1659 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1660 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1661 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1662 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1663 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1664 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1665 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1666 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1667 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1668 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1669 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1670 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1671 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1672 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1673 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1674 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1675 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1676 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd), 1677 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1678 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1679 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1680 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1681 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1682 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1683 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1684 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1685 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1686 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1687 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf), 1688 {} 1689 }; 1690 1691 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1692 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, X86_FAMILY_ANY, X86_FEATURE_MWAIT, NULL), 1693 {} 1694 }; 1695 1696 static bool __init intel_idle_max_cstate_reached(int cstate) 1697 { 1698 if (cstate + 1 > max_cstate) { 1699 pr_info("max_cstate %d reached\n", max_cstate); 1700 return true; 1701 } 1702 return false; 1703 } 1704 1705 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1706 { 1707 unsigned long eax = flg2MWAIT(state->flags); 1708 1709 if (boot_cpu_has(X86_FEATURE_ARAT)) 1710 return false; 1711 1712 /* 1713 * Switch over to one-shot tick broadcast if the target C-state 1714 * is deeper than C1. 1715 */ 1716 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1717 } 1718 1719 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1720 #include <acpi/processor.h> 1721 1722 static bool no_acpi __read_mostly; 1723 module_param(no_acpi, bool, 0444); 1724 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1725 1726 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1727 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1728 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1729 1730 static bool no_native __read_mostly; /* No effect if no_acpi is set. */ 1731 module_param_named(no_native, no_native, bool, 0444); 1732 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states"); 1733 1734 static struct acpi_processor_power acpi_state_table __initdata; 1735 1736 /** 1737 * intel_idle_cst_usable - Check if the _CST information can be used. 1738 * 1739 * Check if all of the C-states listed by _CST in the max_cstate range are 1740 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1741 */ 1742 static bool __init intel_idle_cst_usable(void) 1743 { 1744 int cstate, limit; 1745 1746 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1747 acpi_state_table.count); 1748 1749 for (cstate = 1; cstate < limit; cstate++) { 1750 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1751 1752 if (cx->entry_method != ACPI_CSTATE_FFH) 1753 return false; 1754 } 1755 1756 return true; 1757 } 1758 1759 static bool __init intel_idle_acpi_cst_extract(void) 1760 { 1761 unsigned int cpu; 1762 1763 if (no_acpi) { 1764 pr_debug("Not allowed to use ACPI _CST\n"); 1765 return false; 1766 } 1767 1768 for_each_possible_cpu(cpu) { 1769 struct acpi_processor *pr = per_cpu(processors, cpu); 1770 1771 if (!pr) 1772 continue; 1773 1774 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1775 continue; 1776 1777 acpi_state_table.count++; 1778 1779 if (!intel_idle_cst_usable()) 1780 continue; 1781 1782 if (!acpi_processor_claim_cst_control()) 1783 break; 1784 1785 return true; 1786 } 1787 1788 acpi_state_table.count = 0; 1789 pr_debug("ACPI _CST not found or not usable\n"); 1790 return false; 1791 } 1792 1793 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1794 { 1795 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1796 1797 /* 1798 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1799 * the interesting states are ACPI_CSTATE_FFH. 1800 */ 1801 for (cstate = 1; cstate < limit; cstate++) { 1802 struct acpi_processor_cx *cx; 1803 struct cpuidle_state *state; 1804 1805 if (intel_idle_max_cstate_reached(cstate - 1)) 1806 break; 1807 1808 cx = &acpi_state_table.states[cstate]; 1809 1810 state = &drv->states[drv->state_count++]; 1811 1812 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1813 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1814 state->exit_latency = cx->latency; 1815 /* 1816 * For C1-type C-states use the same number for both the exit 1817 * latency and target residency, because that is the case for 1818 * C1 in the majority of the static C-states tables above. 1819 * For the other types of C-states, however, set the target 1820 * residency to 3 times the exit latency which should lead to 1821 * a reasonable balance between energy-efficiency and 1822 * performance in the majority of interesting cases. 1823 */ 1824 state->target_residency = cx->latency; 1825 if (cx->type > ACPI_STATE_C1) 1826 state->target_residency *= 3; 1827 1828 state->flags = MWAIT2flg(cx->address); 1829 if (cx->type > ACPI_STATE_C2) 1830 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1831 1832 if (disabled_states_mask & BIT(cstate)) 1833 state->flags |= CPUIDLE_FLAG_OFF; 1834 1835 if (intel_idle_state_needs_timer_stop(state)) 1836 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1837 1838 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1839 mark_tsc_unstable("TSC halts in idle"); 1840 1841 state->enter = intel_idle; 1842 state->enter_dead = intel_idle_enter_dead; 1843 state->enter_s2idle = intel_idle_s2idle; 1844 } 1845 } 1846 1847 static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1848 { 1849 int cstate, limit; 1850 1851 /* 1852 * If there are no _CST C-states, do not disable any C-states by 1853 * default. 1854 */ 1855 if (!acpi_state_table.count) 1856 return false; 1857 1858 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1859 /* 1860 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1861 * the interesting states are ACPI_CSTATE_FFH. 1862 */ 1863 for (cstate = 1; cstate < limit; cstate++) { 1864 u32 acpi_hint = acpi_state_table.states[cstate].address; 1865 u32 table_hint = mwait_hint; 1866 1867 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) { 1868 acpi_hint &= ~MWAIT_SUBSTATE_MASK; 1869 table_hint &= ~MWAIT_SUBSTATE_MASK; 1870 } 1871 1872 if (acpi_hint == table_hint) 1873 return false; 1874 } 1875 return true; 1876 } 1877 1878 static inline bool ignore_native(void) 1879 { 1880 return no_native && !no_acpi; 1881 } 1882 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1883 #define force_use_acpi (false) 1884 1885 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1886 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1887 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint) 1888 { 1889 return false; 1890 } 1891 static inline bool ignore_native(void) { return false; } 1892 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1893 1894 /** 1895 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1896 * 1897 * Tune IVT multi-socket targets. 1898 * Assumption: num_sockets == (max_package_num + 1). 1899 */ 1900 static void __init ivt_idle_state_table_update(void) 1901 { 1902 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1903 int cpu, package_num, num_sockets = 1; 1904 1905 for_each_online_cpu(cpu) { 1906 package_num = topology_physical_package_id(cpu); 1907 if (package_num + 1 > num_sockets) { 1908 num_sockets = package_num + 1; 1909 1910 if (num_sockets > 4) { 1911 cpuidle_state_table = ivt_cstates_8s; 1912 return; 1913 } 1914 } 1915 } 1916 1917 if (num_sockets > 2) 1918 cpuidle_state_table = ivt_cstates_4s; 1919 1920 /* else, 1 and 2 socket systems use default ivt_cstates */ 1921 } 1922 1923 /** 1924 * irtl_2_usec - IRTL to microseconds conversion. 1925 * @irtl: IRTL MSR value. 1926 * 1927 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1928 */ 1929 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1930 { 1931 static const unsigned int irtl_ns_units[] __initconst = { 1932 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1933 }; 1934 unsigned long long ns; 1935 1936 if (!irtl) 1937 return 0; 1938 1939 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1940 1941 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1942 } 1943 1944 /** 1945 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1946 * 1947 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1948 * definitive maximum latency and use the same value for target_residency. 1949 */ 1950 static void __init bxt_idle_state_table_update(void) 1951 { 1952 unsigned long long msr; 1953 unsigned int usec; 1954 1955 rdmsrq(MSR_PKGC6_IRTL, msr); 1956 usec = irtl_2_usec(msr); 1957 if (usec) { 1958 bxt_cstates[2].exit_latency = usec; 1959 bxt_cstates[2].target_residency = usec; 1960 } 1961 1962 rdmsrq(MSR_PKGC7_IRTL, msr); 1963 usec = irtl_2_usec(msr); 1964 if (usec) { 1965 bxt_cstates[3].exit_latency = usec; 1966 bxt_cstates[3].target_residency = usec; 1967 } 1968 1969 rdmsrq(MSR_PKGC8_IRTL, msr); 1970 usec = irtl_2_usec(msr); 1971 if (usec) { 1972 bxt_cstates[4].exit_latency = usec; 1973 bxt_cstates[4].target_residency = usec; 1974 } 1975 1976 rdmsrq(MSR_PKGC9_IRTL, msr); 1977 usec = irtl_2_usec(msr); 1978 if (usec) { 1979 bxt_cstates[5].exit_latency = usec; 1980 bxt_cstates[5].target_residency = usec; 1981 } 1982 1983 rdmsrq(MSR_PKGC10_IRTL, msr); 1984 usec = irtl_2_usec(msr); 1985 if (usec) { 1986 bxt_cstates[6].exit_latency = usec; 1987 bxt_cstates[6].target_residency = usec; 1988 } 1989 1990 } 1991 1992 /** 1993 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1994 * 1995 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1996 */ 1997 static void __init sklh_idle_state_table_update(void) 1998 { 1999 unsigned long long msr; 2000 unsigned int eax, ebx, ecx, edx; 2001 2002 2003 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 2004 if (max_cstate <= 7) 2005 return; 2006 2007 /* if PC10 not present in CPUID.MWAIT.EDX */ 2008 if ((mwait_substates & (0xF << 28)) == 0) 2009 return; 2010 2011 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); 2012 2013 /* PC10 is not enabled in PKG C-state limit */ 2014 if ((msr & 0xF) != 8) 2015 return; 2016 2017 ecx = 0; 2018 cpuid(7, &eax, &ebx, &ecx, &edx); 2019 2020 /* if SGX is present */ 2021 if (ebx & (1 << 2)) { 2022 2023 rdmsrq(MSR_IA32_FEAT_CTL, msr); 2024 2025 /* if SGX is enabled */ 2026 if (msr & (1 << 18)) 2027 return; 2028 } 2029 2030 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 2031 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 2032 } 2033 2034 /** 2035 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 2036 * idle states table. 2037 */ 2038 static void __init skx_idle_state_table_update(void) 2039 { 2040 unsigned long long msr; 2041 2042 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); 2043 2044 /* 2045 * 000b: C0/C1 (no package C-state support) 2046 * 001b: C2 2047 * 010b: C6 (non-retention) 2048 * 011b: C6 (retention) 2049 * 111b: No Package C state limits. 2050 */ 2051 if ((msr & 0x7) < 2) { 2052 /* 2053 * Uses the CC6 + PC0 latency and 3 times of 2054 * latency for target_residency if the PC6 2055 * is disabled in BIOS. This is consistent 2056 * with how intel_idle driver uses _CST 2057 * to set the target_residency. 2058 */ 2059 skx_cstates[2].exit_latency = 92; 2060 skx_cstates[2].target_residency = 276; 2061 } 2062 } 2063 2064 /** 2065 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 2066 */ 2067 static void __init spr_idle_state_table_update(void) 2068 { 2069 unsigned long long msr; 2070 2071 /* 2072 * By default, the C6 state assumes the worst-case scenario of package 2073 * C6. However, if PC6 is disabled, we update the numbers to match 2074 * core C6. 2075 */ 2076 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr); 2077 2078 /* Limit value 2 and above allow for PC6. */ 2079 if ((msr & 0x7) < 2) { 2080 spr_cstates[2].exit_latency = 190; 2081 spr_cstates[2].target_residency = 600; 2082 } 2083 } 2084 2085 /** 2086 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion. 2087 */ 2088 static void __init byt_cht_auto_demotion_disable(void) 2089 { 2090 wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2091 wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2092 } 2093 2094 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 2095 { 2096 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 2097 MWAIT_CSTATE_MASK; 2098 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 2099 MWAIT_SUBSTATE_MASK; 2100 2101 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 2102 if (num_substates == 0) 2103 return false; 2104 2105 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 2106 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2107 2108 return true; 2109 } 2110 2111 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2112 { 2113 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2114 /* 2115 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2116 * is not currently supported but this driver. 2117 */ 2118 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2119 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2120 state->enter = intel_idle_xstate; 2121 return; 2122 } 2123 2124 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2125 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2126 /* 2127 * IBRS mitigation requires that C-states are entered 2128 * with interrupts disabled. 2129 */ 2130 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2131 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2132 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2133 state->enter = intel_idle_ibrs; 2134 return; 2135 } 2136 2137 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2138 state->enter = intel_idle_irq; 2139 return; 2140 } 2141 2142 if (force_irq_on) { 2143 pr_info("forced intel_idle_irq for state %d\n", cstate); 2144 state->enter = intel_idle_irq; 2145 } 2146 } 2147 2148 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2149 { 2150 int cstate; 2151 2152 switch (boot_cpu_data.x86_vfm) { 2153 case INTEL_IVYBRIDGE_X: 2154 ivt_idle_state_table_update(); 2155 break; 2156 case INTEL_ATOM_GOLDMONT: 2157 case INTEL_ATOM_GOLDMONT_PLUS: 2158 bxt_idle_state_table_update(); 2159 break; 2160 case INTEL_SKYLAKE: 2161 sklh_idle_state_table_update(); 2162 break; 2163 case INTEL_SKYLAKE_X: 2164 skx_idle_state_table_update(); 2165 break; 2166 case INTEL_SAPPHIRERAPIDS_X: 2167 case INTEL_EMERALDRAPIDS_X: 2168 spr_idle_state_table_update(); 2169 break; 2170 case INTEL_ATOM_SILVERMONT: 2171 case INTEL_ATOM_AIRMONT: 2172 byt_cht_auto_demotion_disable(); 2173 break; 2174 } 2175 2176 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2177 struct cpuidle_state *state; 2178 unsigned int mwait_hint; 2179 2180 if (intel_idle_max_cstate_reached(cstate)) 2181 break; 2182 2183 if (!cpuidle_state_table[cstate].enter && 2184 !cpuidle_state_table[cstate].enter_s2idle) 2185 break; 2186 2187 if (!cpuidle_state_table[cstate].enter_dead) 2188 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead; 2189 2190 /* If marked as unusable, skip this state. */ 2191 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2192 pr_debug("state %s is disabled\n", 2193 cpuidle_state_table[cstate].name); 2194 continue; 2195 } 2196 2197 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2198 if (!intel_idle_verify_cstate(mwait_hint)) 2199 continue; 2200 2201 /* Structure copy. */ 2202 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2203 state = &drv->states[drv->state_count]; 2204 2205 state_update_enter_method(state, cstate); 2206 2207 2208 if ((disabled_states_mask & BIT(drv->state_count)) || 2209 ((icpu->use_acpi || force_use_acpi) && 2210 intel_idle_off_by_default(state->flags, mwait_hint) && 2211 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2212 state->flags |= CPUIDLE_FLAG_OFF; 2213 2214 if (intel_idle_state_needs_timer_stop(state)) 2215 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2216 2217 drv->state_count++; 2218 } 2219 } 2220 2221 /** 2222 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2223 * @drv: cpuidle driver structure to initialize. 2224 */ 2225 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2226 { 2227 cpuidle_poll_state_init(drv); 2228 2229 if (disabled_states_mask & BIT(0)) 2230 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2231 2232 drv->state_count = 1; 2233 2234 if (icpu && icpu->state_table) 2235 intel_idle_init_cstates_icpu(drv); 2236 else 2237 intel_idle_init_cstates_acpi(drv); 2238 } 2239 2240 static void auto_demotion_disable(void) 2241 { 2242 unsigned long long msr_bits; 2243 2244 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2245 msr_bits &= ~auto_demotion_disable_flags; 2246 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2247 } 2248 2249 static void c1e_promotion_enable(void) 2250 { 2251 unsigned long long msr_bits; 2252 2253 rdmsrq(MSR_IA32_POWER_CTL, msr_bits); 2254 msr_bits |= 0x2; 2255 wrmsrq(MSR_IA32_POWER_CTL, msr_bits); 2256 } 2257 2258 static void c1e_promotion_disable(void) 2259 { 2260 unsigned long long msr_bits; 2261 2262 rdmsrq(MSR_IA32_POWER_CTL, msr_bits); 2263 msr_bits &= ~0x2; 2264 wrmsrq(MSR_IA32_POWER_CTL, msr_bits); 2265 } 2266 2267 /** 2268 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2269 * @cpu: CPU to initialize. 2270 * 2271 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2272 * with the processor model flags. 2273 */ 2274 static int intel_idle_cpu_init(unsigned int cpu) 2275 { 2276 struct cpuidle_device *dev; 2277 2278 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2279 dev->cpu = cpu; 2280 2281 if (cpuidle_register_device(dev)) { 2282 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2283 return -EIO; 2284 } 2285 2286 if (auto_demotion_disable_flags) 2287 auto_demotion_disable(); 2288 2289 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2290 c1e_promotion_enable(); 2291 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2292 c1e_promotion_disable(); 2293 2294 return 0; 2295 } 2296 2297 static int intel_idle_cpu_online(unsigned int cpu) 2298 { 2299 struct cpuidle_device *dev; 2300 2301 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2302 tick_broadcast_enable(); 2303 2304 /* 2305 * Some systems can hotplug a cpu at runtime after 2306 * the kernel has booted, we have to initialize the 2307 * driver in this case 2308 */ 2309 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2310 if (!dev->registered) 2311 return intel_idle_cpu_init(cpu); 2312 2313 return 0; 2314 } 2315 2316 /** 2317 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2318 */ 2319 static void __init intel_idle_cpuidle_devices_uninit(void) 2320 { 2321 int i; 2322 2323 for_each_online_cpu(i) 2324 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2325 } 2326 2327 static void intel_c1_demotion_toggle(void *enable) 2328 { 2329 unsigned long long msr_val; 2330 2331 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2332 /* 2333 * Enable/disable C1 undemotion along with C1 demotion, as this is the 2334 * most sensible configuration in general. 2335 */ 2336 if (enable) 2337 msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE; 2338 else 2339 msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE); 2340 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2341 } 2342 2343 static ssize_t intel_c1_demotion_store(struct device *dev, 2344 struct device_attribute *attr, 2345 const char *buf, size_t count) 2346 { 2347 bool enable; 2348 int err; 2349 2350 err = kstrtobool(buf, &enable); 2351 if (err) 2352 return err; 2353 2354 mutex_lock(&c1_demotion_mutex); 2355 /* Enable/disable C1 demotion on all CPUs */ 2356 on_each_cpu(intel_c1_demotion_toggle, (void *)enable, 1); 2357 mutex_unlock(&c1_demotion_mutex); 2358 2359 return count; 2360 } 2361 2362 static ssize_t intel_c1_demotion_show(struct device *dev, 2363 struct device_attribute *attr, char *buf) 2364 { 2365 unsigned long long msr_val; 2366 2367 /* 2368 * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other 2369 * configuration would be a BIOS bug. 2370 */ 2371 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val); 2372 return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE)); 2373 } 2374 static DEVICE_ATTR_RW(intel_c1_demotion); 2375 2376 static int __init intel_idle_sysfs_init(void) 2377 { 2378 int err; 2379 2380 if (!c1_demotion_supported) 2381 return 0; 2382 2383 sysfs_root = bus_get_dev_root(&cpu_subsys); 2384 if (!sysfs_root) 2385 return 0; 2386 2387 err = sysfs_add_file_to_group(&sysfs_root->kobj, 2388 &dev_attr_intel_c1_demotion.attr, 2389 "cpuidle"); 2390 if (err) { 2391 put_device(sysfs_root); 2392 return err; 2393 } 2394 2395 return 0; 2396 } 2397 2398 static void __init intel_idle_sysfs_uninit(void) 2399 { 2400 if (!sysfs_root) 2401 return; 2402 2403 sysfs_remove_file_from_group(&sysfs_root->kobj, 2404 &dev_attr_intel_c1_demotion.attr, 2405 "cpuidle"); 2406 put_device(sysfs_root); 2407 } 2408 2409 /** 2410 * get_cmdline_field - Get the current field from a cmdline string. 2411 * @args: The cmdline string to get the current field from. 2412 * @field: Pointer to the current field upon return. 2413 * @sep: The fields separator character. 2414 * 2415 * Examples: 2416 * Input: args="C1:1:1,C1E:2:10", sep=':' 2417 * Output: field="C1", return "1:1,C1E:2:10" 2418 * Input: args="C1:1:1,C1E:2:10", sep=',' 2419 * Output: field="C1:1:1", return "C1E:2:10" 2420 * Ipnut: args="::", sep=':' 2421 * Output: field="", return ":" 2422 * 2423 * Return: The continuation of the cmdline string after the field or NULL. 2424 */ 2425 static char *get_cmdline_field(char *args, char **field, char sep) 2426 { 2427 unsigned int i; 2428 2429 for (i = 0; args[i] && !isspace(args[i]); i++) { 2430 if (args[i] == sep) 2431 break; 2432 } 2433 2434 *field = args; 2435 2436 if (args[i] != sep) 2437 return NULL; 2438 2439 args[i] = '\0'; 2440 return args + i + 1; 2441 } 2442 2443 /** 2444 * validate_cmdline_cstate - Validate a C-state from cmdline. 2445 * @state: The C-state to validate. 2446 * @prev_state: The previous C-state in the table or NULL. 2447 * 2448 * Return: 0 if the C-state is valid or -EINVAL otherwise. 2449 */ 2450 static int validate_cmdline_cstate(struct cpuidle_state *state, 2451 struct cpuidle_state *prev_state) 2452 { 2453 if (state->exit_latency == 0) 2454 /* Exit latency 0 can only be used for the POLL state */ 2455 return -EINVAL; 2456 2457 if (state->exit_latency > MAX_CMDLINE_LATENCY_US) 2458 return -EINVAL; 2459 2460 if (state->target_residency > MAX_CMDLINE_RESIDENCY_US) 2461 return -EINVAL; 2462 2463 if (state->target_residency < state->exit_latency) 2464 return -EINVAL; 2465 2466 if (!prev_state) 2467 return 0; 2468 2469 if (state->exit_latency <= prev_state->exit_latency) 2470 return -EINVAL; 2471 2472 if (state->target_residency <= prev_state->target_residency) 2473 return -EINVAL; 2474 2475 return 0; 2476 } 2477 2478 /** 2479 * cmdline_table_adjust - Adjust the C-states table with data from cmdline. 2480 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 2481 * 2482 * Adjust the C-states table with data from the 'intel_idle.table' module 2483 * parameter (if specified). 2484 */ 2485 static void __init cmdline_table_adjust(struct cpuidle_driver *drv) 2486 { 2487 char *args = cmdline_table_str; 2488 struct cpuidle_state *state; 2489 int i; 2490 2491 if (args[0] == '\0') 2492 /* The 'intel_idle.table' module parameter was not specified */ 2493 return; 2494 2495 /* Create a copy of the C-states table */ 2496 for (i = 0; i < drv->state_count; i++) 2497 cmdline_states[i] = drv->states[i]; 2498 2499 /* 2500 * Adjust the C-states table copy with data from the 'intel_idle.table' 2501 * module parameter. 2502 */ 2503 while (args) { 2504 char *fields, *name, *val; 2505 2506 /* 2507 * Get the next C-state definition, which is expected to be 2508 * '<name>:<latency_us>:<target_residency_us>'. Treat "empty" 2509 * fields as unchanged. For example, 2510 * '<name>::<target_residency_us>' leaves the latency unchanged. 2511 */ 2512 args = get_cmdline_field(args, &fields, ','); 2513 2514 /* name */ 2515 fields = get_cmdline_field(fields, &name, ':'); 2516 if (!fields) 2517 goto error; 2518 2519 if (!strcmp(name, "POLL")) { 2520 pr_err("Cannot adjust POLL\n"); 2521 continue; 2522 } 2523 2524 /* Find the C-state by its name */ 2525 state = NULL; 2526 for (i = 0; i < drv->state_count; i++) { 2527 if (!strcmp(name, drv->states[i].name)) { 2528 state = &cmdline_states[i]; 2529 break; 2530 } 2531 } 2532 2533 if (!state) { 2534 pr_err("C-state '%s' was not found\n", name); 2535 continue; 2536 } 2537 2538 /* Latency */ 2539 fields = get_cmdline_field(fields, &val, ':'); 2540 if (!fields) 2541 goto error; 2542 2543 if (*val) { 2544 if (kstrtouint(val, 0, &state->exit_latency)) 2545 goto error; 2546 } 2547 2548 /* Target residency */ 2549 fields = get_cmdline_field(fields, &val, ':'); 2550 2551 if (*val) { 2552 if (kstrtouint(val, 0, &state->target_residency)) 2553 goto error; 2554 } 2555 2556 /* 2557 * Allow for 3 more fields, but ignore them. Helps to make 2558 * possible future extensions of the cmdline format backward 2559 * compatible. 2560 */ 2561 for (i = 0; fields && i < 3; i++) { 2562 fields = get_cmdline_field(fields, &val, ':'); 2563 if (!fields) 2564 break; 2565 } 2566 2567 if (fields) { 2568 pr_err("Too many fields for C-state '%s'\n", state->name); 2569 goto error; 2570 } 2571 2572 pr_info("C-state from cmdline: name=%s, latency=%u, residency=%u\n", 2573 state->name, state->exit_latency, state->target_residency); 2574 } 2575 2576 /* Validate the adjusted C-states, start with index 1 to skip POLL */ 2577 for (i = 1; i < drv->state_count; i++) { 2578 struct cpuidle_state *prev_state; 2579 2580 state = &cmdline_states[i]; 2581 prev_state = &cmdline_states[i - 1]; 2582 2583 if (validate_cmdline_cstate(state, prev_state)) { 2584 pr_err("C-state '%s' validation failed\n", state->name); 2585 goto error; 2586 } 2587 } 2588 2589 /* Copy the adjusted C-states table back */ 2590 for (i = 1; i < drv->state_count; i++) 2591 drv->states[i] = cmdline_states[i]; 2592 2593 pr_info("Adjusted C-states with data from 'intel_idle.table'\n"); 2594 return; 2595 2596 error: 2597 pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n"); 2598 } 2599 2600 static int __init intel_idle_init(void) 2601 { 2602 const struct x86_cpu_id *id; 2603 unsigned int eax, ebx, ecx; 2604 int retval; 2605 2606 /* Do not load intel_idle at all for now if idle= is passed */ 2607 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2608 return -ENODEV; 2609 2610 if (max_cstate == 0) { 2611 pr_debug("disabled\n"); 2612 return -EPERM; 2613 } 2614 2615 id = x86_match_cpu(intel_idle_ids); 2616 if (id) { 2617 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2618 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2619 return -ENODEV; 2620 } 2621 } else { 2622 id = x86_match_cpu(intel_mwait_ids); 2623 if (!id) 2624 return -ENODEV; 2625 } 2626 2627 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates); 2628 2629 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2630 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2631 !mwait_substates) 2632 return -ENODEV; 2633 2634 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2635 2636 icpu = (const struct idle_cpu *)id->driver_data; 2637 if (icpu && ignore_native()) { 2638 pr_debug("ignoring native CPU idle states\n"); 2639 icpu = NULL; 2640 } 2641 if (icpu) { 2642 if (icpu->state_table) 2643 cpuidle_state_table = icpu->state_table; 2644 else if (!intel_idle_acpi_cst_extract()) 2645 return -ENODEV; 2646 2647 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2648 if (icpu->disable_promotion_to_c1e) 2649 c1e_promotion = C1E_PROMOTION_DISABLE; 2650 if (icpu->c1_demotion_supported) 2651 c1_demotion_supported = true; 2652 if (icpu->use_acpi || force_use_acpi) 2653 intel_idle_acpi_cst_extract(); 2654 } else if (!intel_idle_acpi_cst_extract()) { 2655 return -ENODEV; 2656 } 2657 2658 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2659 if (!intel_idle_cpuidle_devices) 2660 return -ENOMEM; 2661 2662 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2663 cmdline_table_adjust(&intel_idle_driver); 2664 2665 retval = intel_idle_sysfs_init(); 2666 if (retval) 2667 pr_warn("failed to initialized sysfs"); 2668 2669 retval = cpuidle_register_driver(&intel_idle_driver); 2670 if (retval) { 2671 struct cpuidle_driver *drv = cpuidle_get_driver(); 2672 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2673 drv ? drv->name : "none"); 2674 goto init_driver_fail; 2675 } 2676 2677 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2678 intel_idle_cpu_online, NULL); 2679 if (retval < 0) 2680 goto hp_setup_fail; 2681 2682 pr_debug("Local APIC timer is reliable in %s\n", 2683 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2684 2685 arch_cpu_rescan_dead_smt_siblings(); 2686 2687 return 0; 2688 2689 hp_setup_fail: 2690 intel_idle_cpuidle_devices_uninit(); 2691 cpuidle_unregister_driver(&intel_idle_driver); 2692 init_driver_fail: 2693 intel_idle_sysfs_uninit(); 2694 free_percpu(intel_idle_cpuidle_devices); 2695 return retval; 2696 2697 } 2698 subsys_initcall_sync(intel_idle_init); 2699 2700 /* 2701 * We are not really modular, but we used to support that. Meaning we also 2702 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2703 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2704 * is the easiest way (currently) to continue doing that. 2705 */ 2706 module_param(max_cstate, int, 0444); 2707 /* 2708 * The positions of the bits that are set in this number are the indices of the 2709 * idle states to be disabled by default (as reflected by the names of the 2710 * corresponding idle state directories in sysfs, "state0", "state1" ... 2711 * "state<i>" ..., where <i> is the index of the given state). 2712 */ 2713 module_param_named(states_off, disabled_states_mask, uint, 0444); 2714 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2715 /* 2716 * Debugging option that forces the driver to enter all C-states with 2717 * interrupts enabled. Does not apply to C-states with 2718 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2719 */ 2720 module_param(force_irq_on, bool, 0444); 2721 /* 2722 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2723 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2724 */ 2725 module_param(ibrs_off, bool, 0444); 2726 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2727 2728 /* 2729 * Define the C-states table from a user input string. Expected format is 2730 * 'name:latency:residency', where: 2731 * - name: The C-state name. 2732 * - latency: The C-state exit latency in us. 2733 * - residency: The C-state target residency in us. 2734 * 2735 * Multiple C-states can be defined by separating them with commas: 2736 * 'name1:latency1:residency1,name2:latency2:residency2' 2737 * 2738 * Example: intel_idle.table=C1:1:1,C1E:5:10,C6:100:600 2739 * 2740 * To leave latency or residency unchanged, use an empty field, for example: 2741 * 'C1:1:1,C1E::10' - leaves C1E latency unchanged. 2742 */ 2743 module_param_string(table, cmdline_table_str, MAX_CMDLINE_TABLE_LEN, 0444); 2744 MODULE_PARM_DESC(table, "Build the C-states table from a user input string"); 2745