1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10 /* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17 /* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30 /* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39 /* un-comment DEBUG to enable pr_debug() statements */ 40 /* #define DEBUG */ 41 42 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44 #include <linux/acpi.h> 45 #include <linux/kernel.h> 46 #include <linux/cpuidle.h> 47 #include <linux/tick.h> 48 #include <trace/events/power.h> 49 #include <linux/sched.h> 50 #include <linux/sched/smt.h> 51 #include <linux/notifier.h> 52 #include <linux/cpu.h> 53 #include <linux/moduleparam.h> 54 #include <asm/cpu_device_id.h> 55 #include <asm/intel-family.h> 56 #include <asm/mwait.h> 57 #include <asm/spec-ctrl.h> 58 #include <asm/fpu/api.h> 59 60 #define INTEL_IDLE_VERSION "0.5.1" 61 62 static struct cpuidle_driver intel_idle_driver = { 63 .name = "intel_idle", 64 .owner = THIS_MODULE, 65 }; 66 /* intel_idle.max_cstate=0 disables driver */ 67 static int max_cstate = CPUIDLE_STATE_MAX - 1; 68 static unsigned int disabled_states_mask __read_mostly; 69 static unsigned int preferred_states_mask __read_mostly; 70 static bool force_irq_on __read_mostly; 71 static bool ibrs_off __read_mostly; 72 73 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 74 75 static unsigned long auto_demotion_disable_flags; 76 77 static enum { 78 C1E_PROMOTION_PRESERVE, 79 C1E_PROMOTION_ENABLE, 80 C1E_PROMOTION_DISABLE 81 } c1e_promotion = C1E_PROMOTION_PRESERVE; 82 83 struct idle_cpu { 84 struct cpuidle_state *state_table; 85 86 /* 87 * Hardware C-state auto-demotion may not always be optimal. 88 * Indicate which enable bits to clear here. 89 */ 90 unsigned long auto_demotion_disable_flags; 91 bool byt_auto_demotion_disable_flag; 92 bool disable_promotion_to_c1e; 93 bool use_acpi; 94 }; 95 96 static const struct idle_cpu *icpu __initdata; 97 static struct cpuidle_state *cpuidle_state_table __initdata; 98 99 static unsigned int mwait_substates __initdata; 100 101 /* 102 * Enable interrupts before entering the C-state. On some platforms and for 103 * some C-states, this may measurably decrease interrupt latency. 104 */ 105 #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 106 107 /* 108 * Enable this state by default even if the ACPI _CST does not list it. 109 */ 110 #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 111 112 /* 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 114 * above. 115 */ 116 #define CPUIDLE_FLAG_IBRS BIT(16) 117 118 /* 119 * Initialize large xstate for the C6-state entrance. 120 */ 121 #define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 122 123 /* 124 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 125 * the C-state (top nibble) and sub-state (bottom nibble) 126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 127 * 128 * We store the hint at the top of our "flags" for each state. 129 */ 130 #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 131 #define MWAIT2flg(eax) ((eax & 0xFF) << 24) 132 133 static __always_inline int __intel_idle(struct cpuidle_device *dev, 134 struct cpuidle_driver *drv, 135 int index, bool irqoff) 136 { 137 struct cpuidle_state *state = &drv->states[index]; 138 unsigned long eax = flg2MWAIT(state->flags); 139 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 140 141 mwait_idle_with_hints(eax, ecx); 142 143 return index; 144 } 145 146 /** 147 * intel_idle - Ask the processor to enter the given idle state. 148 * @dev: cpuidle device of the target CPU. 149 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 150 * @index: Target idle state index. 151 * 152 * Use the MWAIT instruction to notify the processor that the CPU represented by 153 * @dev is idle and it can try to enter the idle state corresponding to @index. 154 * 155 * If the local APIC timer is not known to be reliable in the target idle state, 156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 157 * 158 * Must be called under local_irq_disable(). 159 */ 160 static __cpuidle int intel_idle(struct cpuidle_device *dev, 161 struct cpuidle_driver *drv, int index) 162 { 163 return __intel_idle(dev, drv, index, true); 164 } 165 166 static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, int index) 168 { 169 return __intel_idle(dev, drv, index, false); 170 } 171 172 static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 173 struct cpuidle_driver *drv, int index) 174 { 175 bool smt_active = sched_smt_active(); 176 u64 spec_ctrl = spec_ctrl_current(); 177 int ret; 178 179 if (smt_active) 180 __update_spec_ctrl(0); 181 182 ret = __intel_idle(dev, drv, index, true); 183 184 if (smt_active) 185 __update_spec_ctrl(spec_ctrl); 186 187 return ret; 188 } 189 190 static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 191 struct cpuidle_driver *drv, int index) 192 { 193 fpu_idle_fpregs(); 194 return __intel_idle(dev, drv, index, true); 195 } 196 197 /** 198 * intel_idle_s2idle - Ask the processor to enter the given idle state. 199 * @dev: cpuidle device of the target CPU. 200 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 201 * @index: Target idle state index. 202 * 203 * Use the MWAIT instruction to notify the processor that the CPU represented by 204 * @dev is idle and it can try to enter the idle state corresponding to @index. 205 * 206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 207 * scheduler tick and suspended scheduler clock on the target CPU. 208 */ 209 static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 210 struct cpuidle_driver *drv, int index) 211 { 212 unsigned long ecx = 1; /* break on interrupt flag */ 213 struct cpuidle_state *state = &drv->states[index]; 214 unsigned long eax = flg2MWAIT(state->flags); 215 216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 217 fpu_idle_fpregs(); 218 219 mwait_idle_with_hints(eax, ecx); 220 221 return 0; 222 } 223 224 /* 225 * States are indexed by the cstate number, 226 * which is also the index into the MWAIT hint array. 227 * Thus C0 is a dummy. 228 */ 229 static struct cpuidle_state nehalem_cstates[] __initdata = { 230 { 231 .name = "C1", 232 .desc = "MWAIT 0x00", 233 .flags = MWAIT2flg(0x00), 234 .exit_latency = 3, 235 .target_residency = 6, 236 .enter = &intel_idle, 237 .enter_s2idle = intel_idle_s2idle, }, 238 { 239 .name = "C1E", 240 .desc = "MWAIT 0x01", 241 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 242 .exit_latency = 10, 243 .target_residency = 20, 244 .enter = &intel_idle, 245 .enter_s2idle = intel_idle_s2idle, }, 246 { 247 .name = "C3", 248 .desc = "MWAIT 0x10", 249 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 250 .exit_latency = 20, 251 .target_residency = 80, 252 .enter = &intel_idle, 253 .enter_s2idle = intel_idle_s2idle, }, 254 { 255 .name = "C6", 256 .desc = "MWAIT 0x20", 257 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 258 .exit_latency = 200, 259 .target_residency = 800, 260 .enter = &intel_idle, 261 .enter_s2idle = intel_idle_s2idle, }, 262 { 263 .enter = NULL } 264 }; 265 266 static struct cpuidle_state snb_cstates[] __initdata = { 267 { 268 .name = "C1", 269 .desc = "MWAIT 0x00", 270 .flags = MWAIT2flg(0x00), 271 .exit_latency = 2, 272 .target_residency = 2, 273 .enter = &intel_idle, 274 .enter_s2idle = intel_idle_s2idle, }, 275 { 276 .name = "C1E", 277 .desc = "MWAIT 0x01", 278 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 279 .exit_latency = 10, 280 .target_residency = 20, 281 .enter = &intel_idle, 282 .enter_s2idle = intel_idle_s2idle, }, 283 { 284 .name = "C3", 285 .desc = "MWAIT 0x10", 286 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 287 .exit_latency = 80, 288 .target_residency = 211, 289 .enter = &intel_idle, 290 .enter_s2idle = intel_idle_s2idle, }, 291 { 292 .name = "C6", 293 .desc = "MWAIT 0x20", 294 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 295 .exit_latency = 104, 296 .target_residency = 345, 297 .enter = &intel_idle, 298 .enter_s2idle = intel_idle_s2idle, }, 299 { 300 .name = "C7", 301 .desc = "MWAIT 0x30", 302 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 303 .exit_latency = 109, 304 .target_residency = 345, 305 .enter = &intel_idle, 306 .enter_s2idle = intel_idle_s2idle, }, 307 { 308 .enter = NULL } 309 }; 310 311 static struct cpuidle_state byt_cstates[] __initdata = { 312 { 313 .name = "C1", 314 .desc = "MWAIT 0x00", 315 .flags = MWAIT2flg(0x00), 316 .exit_latency = 1, 317 .target_residency = 1, 318 .enter = &intel_idle, 319 .enter_s2idle = intel_idle_s2idle, }, 320 { 321 .name = "C6N", 322 .desc = "MWAIT 0x58", 323 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 324 .exit_latency = 300, 325 .target_residency = 275, 326 .enter = &intel_idle, 327 .enter_s2idle = intel_idle_s2idle, }, 328 { 329 .name = "C6S", 330 .desc = "MWAIT 0x52", 331 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 332 .exit_latency = 500, 333 .target_residency = 560, 334 .enter = &intel_idle, 335 .enter_s2idle = intel_idle_s2idle, }, 336 { 337 .name = "C7", 338 .desc = "MWAIT 0x60", 339 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 340 .exit_latency = 1200, 341 .target_residency = 4000, 342 .enter = &intel_idle, 343 .enter_s2idle = intel_idle_s2idle, }, 344 { 345 .name = "C7S", 346 .desc = "MWAIT 0x64", 347 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 348 .exit_latency = 10000, 349 .target_residency = 20000, 350 .enter = &intel_idle, 351 .enter_s2idle = intel_idle_s2idle, }, 352 { 353 .enter = NULL } 354 }; 355 356 static struct cpuidle_state cht_cstates[] __initdata = { 357 { 358 .name = "C1", 359 .desc = "MWAIT 0x00", 360 .flags = MWAIT2flg(0x00), 361 .exit_latency = 1, 362 .target_residency = 1, 363 .enter = &intel_idle, 364 .enter_s2idle = intel_idle_s2idle, }, 365 { 366 .name = "C6N", 367 .desc = "MWAIT 0x58", 368 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 369 .exit_latency = 80, 370 .target_residency = 275, 371 .enter = &intel_idle, 372 .enter_s2idle = intel_idle_s2idle, }, 373 { 374 .name = "C6S", 375 .desc = "MWAIT 0x52", 376 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 377 .exit_latency = 200, 378 .target_residency = 560, 379 .enter = &intel_idle, 380 .enter_s2idle = intel_idle_s2idle, }, 381 { 382 .name = "C7", 383 .desc = "MWAIT 0x60", 384 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 385 .exit_latency = 1200, 386 .target_residency = 4000, 387 .enter = &intel_idle, 388 .enter_s2idle = intel_idle_s2idle, }, 389 { 390 .name = "C7S", 391 .desc = "MWAIT 0x64", 392 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 393 .exit_latency = 10000, 394 .target_residency = 20000, 395 .enter = &intel_idle, 396 .enter_s2idle = intel_idle_s2idle, }, 397 { 398 .enter = NULL } 399 }; 400 401 static struct cpuidle_state ivb_cstates[] __initdata = { 402 { 403 .name = "C1", 404 .desc = "MWAIT 0x00", 405 .flags = MWAIT2flg(0x00), 406 .exit_latency = 1, 407 .target_residency = 1, 408 .enter = &intel_idle, 409 .enter_s2idle = intel_idle_s2idle, }, 410 { 411 .name = "C1E", 412 .desc = "MWAIT 0x01", 413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 414 .exit_latency = 10, 415 .target_residency = 20, 416 .enter = &intel_idle, 417 .enter_s2idle = intel_idle_s2idle, }, 418 { 419 .name = "C3", 420 .desc = "MWAIT 0x10", 421 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 422 .exit_latency = 59, 423 .target_residency = 156, 424 .enter = &intel_idle, 425 .enter_s2idle = intel_idle_s2idle, }, 426 { 427 .name = "C6", 428 .desc = "MWAIT 0x20", 429 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 430 .exit_latency = 80, 431 .target_residency = 300, 432 .enter = &intel_idle, 433 .enter_s2idle = intel_idle_s2idle, }, 434 { 435 .name = "C7", 436 .desc = "MWAIT 0x30", 437 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 438 .exit_latency = 87, 439 .target_residency = 300, 440 .enter = &intel_idle, 441 .enter_s2idle = intel_idle_s2idle, }, 442 { 443 .enter = NULL } 444 }; 445 446 static struct cpuidle_state ivt_cstates[] __initdata = { 447 { 448 .name = "C1", 449 .desc = "MWAIT 0x00", 450 .flags = MWAIT2flg(0x00), 451 .exit_latency = 1, 452 .target_residency = 1, 453 .enter = &intel_idle, 454 .enter_s2idle = intel_idle_s2idle, }, 455 { 456 .name = "C1E", 457 .desc = "MWAIT 0x01", 458 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 459 .exit_latency = 10, 460 .target_residency = 80, 461 .enter = &intel_idle, 462 .enter_s2idle = intel_idle_s2idle, }, 463 { 464 .name = "C3", 465 .desc = "MWAIT 0x10", 466 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 467 .exit_latency = 59, 468 .target_residency = 156, 469 .enter = &intel_idle, 470 .enter_s2idle = intel_idle_s2idle, }, 471 { 472 .name = "C6", 473 .desc = "MWAIT 0x20", 474 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 475 .exit_latency = 82, 476 .target_residency = 300, 477 .enter = &intel_idle, 478 .enter_s2idle = intel_idle_s2idle, }, 479 { 480 .enter = NULL } 481 }; 482 483 static struct cpuidle_state ivt_cstates_4s[] __initdata = { 484 { 485 .name = "C1", 486 .desc = "MWAIT 0x00", 487 .flags = MWAIT2flg(0x00), 488 .exit_latency = 1, 489 .target_residency = 1, 490 .enter = &intel_idle, 491 .enter_s2idle = intel_idle_s2idle, }, 492 { 493 .name = "C1E", 494 .desc = "MWAIT 0x01", 495 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 496 .exit_latency = 10, 497 .target_residency = 250, 498 .enter = &intel_idle, 499 .enter_s2idle = intel_idle_s2idle, }, 500 { 501 .name = "C3", 502 .desc = "MWAIT 0x10", 503 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 504 .exit_latency = 59, 505 .target_residency = 300, 506 .enter = &intel_idle, 507 .enter_s2idle = intel_idle_s2idle, }, 508 { 509 .name = "C6", 510 .desc = "MWAIT 0x20", 511 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 512 .exit_latency = 84, 513 .target_residency = 400, 514 .enter = &intel_idle, 515 .enter_s2idle = intel_idle_s2idle, }, 516 { 517 .enter = NULL } 518 }; 519 520 static struct cpuidle_state ivt_cstates_8s[] __initdata = { 521 { 522 .name = "C1", 523 .desc = "MWAIT 0x00", 524 .flags = MWAIT2flg(0x00), 525 .exit_latency = 1, 526 .target_residency = 1, 527 .enter = &intel_idle, 528 .enter_s2idle = intel_idle_s2idle, }, 529 { 530 .name = "C1E", 531 .desc = "MWAIT 0x01", 532 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 533 .exit_latency = 10, 534 .target_residency = 500, 535 .enter = &intel_idle, 536 .enter_s2idle = intel_idle_s2idle, }, 537 { 538 .name = "C3", 539 .desc = "MWAIT 0x10", 540 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 541 .exit_latency = 59, 542 .target_residency = 600, 543 .enter = &intel_idle, 544 .enter_s2idle = intel_idle_s2idle, }, 545 { 546 .name = "C6", 547 .desc = "MWAIT 0x20", 548 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 549 .exit_latency = 88, 550 .target_residency = 700, 551 .enter = &intel_idle, 552 .enter_s2idle = intel_idle_s2idle, }, 553 { 554 .enter = NULL } 555 }; 556 557 static struct cpuidle_state hsw_cstates[] __initdata = { 558 { 559 .name = "C1", 560 .desc = "MWAIT 0x00", 561 .flags = MWAIT2flg(0x00), 562 .exit_latency = 2, 563 .target_residency = 2, 564 .enter = &intel_idle, 565 .enter_s2idle = intel_idle_s2idle, }, 566 { 567 .name = "C1E", 568 .desc = "MWAIT 0x01", 569 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 570 .exit_latency = 10, 571 .target_residency = 20, 572 .enter = &intel_idle, 573 .enter_s2idle = intel_idle_s2idle, }, 574 { 575 .name = "C3", 576 .desc = "MWAIT 0x10", 577 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 578 .exit_latency = 33, 579 .target_residency = 100, 580 .enter = &intel_idle, 581 .enter_s2idle = intel_idle_s2idle, }, 582 { 583 .name = "C6", 584 .desc = "MWAIT 0x20", 585 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 586 .exit_latency = 133, 587 .target_residency = 400, 588 .enter = &intel_idle, 589 .enter_s2idle = intel_idle_s2idle, }, 590 { 591 .name = "C7s", 592 .desc = "MWAIT 0x32", 593 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 594 .exit_latency = 166, 595 .target_residency = 500, 596 .enter = &intel_idle, 597 .enter_s2idle = intel_idle_s2idle, }, 598 { 599 .name = "C8", 600 .desc = "MWAIT 0x40", 601 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 602 .exit_latency = 300, 603 .target_residency = 900, 604 .enter = &intel_idle, 605 .enter_s2idle = intel_idle_s2idle, }, 606 { 607 .name = "C9", 608 .desc = "MWAIT 0x50", 609 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 610 .exit_latency = 600, 611 .target_residency = 1800, 612 .enter = &intel_idle, 613 .enter_s2idle = intel_idle_s2idle, }, 614 { 615 .name = "C10", 616 .desc = "MWAIT 0x60", 617 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 618 .exit_latency = 2600, 619 .target_residency = 7700, 620 .enter = &intel_idle, 621 .enter_s2idle = intel_idle_s2idle, }, 622 { 623 .enter = NULL } 624 }; 625 static struct cpuidle_state bdw_cstates[] __initdata = { 626 { 627 .name = "C1", 628 .desc = "MWAIT 0x00", 629 .flags = MWAIT2flg(0x00), 630 .exit_latency = 2, 631 .target_residency = 2, 632 .enter = &intel_idle, 633 .enter_s2idle = intel_idle_s2idle, }, 634 { 635 .name = "C1E", 636 .desc = "MWAIT 0x01", 637 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 638 .exit_latency = 10, 639 .target_residency = 20, 640 .enter = &intel_idle, 641 .enter_s2idle = intel_idle_s2idle, }, 642 { 643 .name = "C3", 644 .desc = "MWAIT 0x10", 645 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 646 .exit_latency = 40, 647 .target_residency = 100, 648 .enter = &intel_idle, 649 .enter_s2idle = intel_idle_s2idle, }, 650 { 651 .name = "C6", 652 .desc = "MWAIT 0x20", 653 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 654 .exit_latency = 133, 655 .target_residency = 400, 656 .enter = &intel_idle, 657 .enter_s2idle = intel_idle_s2idle, }, 658 { 659 .name = "C7s", 660 .desc = "MWAIT 0x32", 661 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 662 .exit_latency = 166, 663 .target_residency = 500, 664 .enter = &intel_idle, 665 .enter_s2idle = intel_idle_s2idle, }, 666 { 667 .name = "C8", 668 .desc = "MWAIT 0x40", 669 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 670 .exit_latency = 300, 671 .target_residency = 900, 672 .enter = &intel_idle, 673 .enter_s2idle = intel_idle_s2idle, }, 674 { 675 .name = "C9", 676 .desc = "MWAIT 0x50", 677 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 678 .exit_latency = 600, 679 .target_residency = 1800, 680 .enter = &intel_idle, 681 .enter_s2idle = intel_idle_s2idle, }, 682 { 683 .name = "C10", 684 .desc = "MWAIT 0x60", 685 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 686 .exit_latency = 2600, 687 .target_residency = 7700, 688 .enter = &intel_idle, 689 .enter_s2idle = intel_idle_s2idle, }, 690 { 691 .enter = NULL } 692 }; 693 694 static struct cpuidle_state skl_cstates[] __initdata = { 695 { 696 .name = "C1", 697 .desc = "MWAIT 0x00", 698 .flags = MWAIT2flg(0x00), 699 .exit_latency = 2, 700 .target_residency = 2, 701 .enter = &intel_idle, 702 .enter_s2idle = intel_idle_s2idle, }, 703 { 704 .name = "C1E", 705 .desc = "MWAIT 0x01", 706 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 707 .exit_latency = 10, 708 .target_residency = 20, 709 .enter = &intel_idle, 710 .enter_s2idle = intel_idle_s2idle, }, 711 { 712 .name = "C3", 713 .desc = "MWAIT 0x10", 714 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 715 .exit_latency = 70, 716 .target_residency = 100, 717 .enter = &intel_idle, 718 .enter_s2idle = intel_idle_s2idle, }, 719 { 720 .name = "C6", 721 .desc = "MWAIT 0x20", 722 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 723 .exit_latency = 85, 724 .target_residency = 200, 725 .enter = &intel_idle, 726 .enter_s2idle = intel_idle_s2idle, }, 727 { 728 .name = "C7s", 729 .desc = "MWAIT 0x33", 730 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 731 .exit_latency = 124, 732 .target_residency = 800, 733 .enter = &intel_idle, 734 .enter_s2idle = intel_idle_s2idle, }, 735 { 736 .name = "C8", 737 .desc = "MWAIT 0x40", 738 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 739 .exit_latency = 200, 740 .target_residency = 800, 741 .enter = &intel_idle, 742 .enter_s2idle = intel_idle_s2idle, }, 743 { 744 .name = "C9", 745 .desc = "MWAIT 0x50", 746 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 747 .exit_latency = 480, 748 .target_residency = 5000, 749 .enter = &intel_idle, 750 .enter_s2idle = intel_idle_s2idle, }, 751 { 752 .name = "C10", 753 .desc = "MWAIT 0x60", 754 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 755 .exit_latency = 890, 756 .target_residency = 5000, 757 .enter = &intel_idle, 758 .enter_s2idle = intel_idle_s2idle, }, 759 { 760 .enter = NULL } 761 }; 762 763 static struct cpuidle_state skx_cstates[] __initdata = { 764 { 765 .name = "C1", 766 .desc = "MWAIT 0x00", 767 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 768 .exit_latency = 2, 769 .target_residency = 2, 770 .enter = &intel_idle, 771 .enter_s2idle = intel_idle_s2idle, }, 772 { 773 .name = "C1E", 774 .desc = "MWAIT 0x01", 775 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 776 .exit_latency = 10, 777 .target_residency = 20, 778 .enter = &intel_idle, 779 .enter_s2idle = intel_idle_s2idle, }, 780 { 781 .name = "C6", 782 .desc = "MWAIT 0x20", 783 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 784 .exit_latency = 133, 785 .target_residency = 600, 786 .enter = &intel_idle, 787 .enter_s2idle = intel_idle_s2idle, }, 788 { 789 .enter = NULL } 790 }; 791 792 static struct cpuidle_state icx_cstates[] __initdata = { 793 { 794 .name = "C1", 795 .desc = "MWAIT 0x00", 796 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 797 .exit_latency = 1, 798 .target_residency = 1, 799 .enter = &intel_idle, 800 .enter_s2idle = intel_idle_s2idle, }, 801 { 802 .name = "C1E", 803 .desc = "MWAIT 0x01", 804 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 805 .exit_latency = 4, 806 .target_residency = 4, 807 .enter = &intel_idle, 808 .enter_s2idle = intel_idle_s2idle, }, 809 { 810 .name = "C6", 811 .desc = "MWAIT 0x20", 812 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 813 .exit_latency = 170, 814 .target_residency = 600, 815 .enter = &intel_idle, 816 .enter_s2idle = intel_idle_s2idle, }, 817 { 818 .enter = NULL } 819 }; 820 821 /* 822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 824 * But in this case there is effectively no C1, because C1 requests are 825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 826 * and C1E requests end up with C1, so there is effectively no C1E. 827 * 828 * By default we enable C1E and disable C1 by marking it with 829 * 'CPUIDLE_FLAG_UNUSABLE'. 830 */ 831 static struct cpuidle_state adl_cstates[] __initdata = { 832 { 833 .name = "C1", 834 .desc = "MWAIT 0x00", 835 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 836 .exit_latency = 1, 837 .target_residency = 1, 838 .enter = &intel_idle, 839 .enter_s2idle = intel_idle_s2idle, }, 840 { 841 .name = "C1E", 842 .desc = "MWAIT 0x01", 843 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 844 .exit_latency = 2, 845 .target_residency = 4, 846 .enter = &intel_idle, 847 .enter_s2idle = intel_idle_s2idle, }, 848 { 849 .name = "C6", 850 .desc = "MWAIT 0x20", 851 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 852 .exit_latency = 220, 853 .target_residency = 600, 854 .enter = &intel_idle, 855 .enter_s2idle = intel_idle_s2idle, }, 856 { 857 .name = "C8", 858 .desc = "MWAIT 0x40", 859 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 860 .exit_latency = 280, 861 .target_residency = 800, 862 .enter = &intel_idle, 863 .enter_s2idle = intel_idle_s2idle, }, 864 { 865 .name = "C10", 866 .desc = "MWAIT 0x60", 867 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 868 .exit_latency = 680, 869 .target_residency = 2000, 870 .enter = &intel_idle, 871 .enter_s2idle = intel_idle_s2idle, }, 872 { 873 .enter = NULL } 874 }; 875 876 static struct cpuidle_state adl_l_cstates[] __initdata = { 877 { 878 .name = "C1", 879 .desc = "MWAIT 0x00", 880 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 881 .exit_latency = 1, 882 .target_residency = 1, 883 .enter = &intel_idle, 884 .enter_s2idle = intel_idle_s2idle, }, 885 { 886 .name = "C1E", 887 .desc = "MWAIT 0x01", 888 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 889 .exit_latency = 2, 890 .target_residency = 4, 891 .enter = &intel_idle, 892 .enter_s2idle = intel_idle_s2idle, }, 893 { 894 .name = "C6", 895 .desc = "MWAIT 0x20", 896 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 897 .exit_latency = 170, 898 .target_residency = 500, 899 .enter = &intel_idle, 900 .enter_s2idle = intel_idle_s2idle, }, 901 { 902 .name = "C8", 903 .desc = "MWAIT 0x40", 904 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 905 .exit_latency = 200, 906 .target_residency = 600, 907 .enter = &intel_idle, 908 .enter_s2idle = intel_idle_s2idle, }, 909 { 910 .name = "C10", 911 .desc = "MWAIT 0x60", 912 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 913 .exit_latency = 230, 914 .target_residency = 700, 915 .enter = &intel_idle, 916 .enter_s2idle = intel_idle_s2idle, }, 917 { 918 .enter = NULL } 919 }; 920 921 static struct cpuidle_state mtl_l_cstates[] __initdata = { 922 { 923 .name = "C1E", 924 .desc = "MWAIT 0x01", 925 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 926 .exit_latency = 1, 927 .target_residency = 1, 928 .enter = &intel_idle, 929 .enter_s2idle = intel_idle_s2idle, }, 930 { 931 .name = "C6", 932 .desc = "MWAIT 0x20", 933 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 934 .exit_latency = 140, 935 .target_residency = 420, 936 .enter = &intel_idle, 937 .enter_s2idle = intel_idle_s2idle, }, 938 { 939 .name = "C10", 940 .desc = "MWAIT 0x60", 941 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 942 .exit_latency = 310, 943 .target_residency = 930, 944 .enter = &intel_idle, 945 .enter_s2idle = intel_idle_s2idle, }, 946 { 947 .enter = NULL } 948 }; 949 950 static struct cpuidle_state gmt_cstates[] __initdata = { 951 { 952 .name = "C1", 953 .desc = "MWAIT 0x00", 954 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 955 .exit_latency = 1, 956 .target_residency = 1, 957 .enter = &intel_idle, 958 .enter_s2idle = intel_idle_s2idle, }, 959 { 960 .name = "C1E", 961 .desc = "MWAIT 0x01", 962 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 963 .exit_latency = 2, 964 .target_residency = 4, 965 .enter = &intel_idle, 966 .enter_s2idle = intel_idle_s2idle, }, 967 { 968 .name = "C6", 969 .desc = "MWAIT 0x20", 970 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 971 .exit_latency = 195, 972 .target_residency = 585, 973 .enter = &intel_idle, 974 .enter_s2idle = intel_idle_s2idle, }, 975 { 976 .name = "C8", 977 .desc = "MWAIT 0x40", 978 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 979 .exit_latency = 260, 980 .target_residency = 1040, 981 .enter = &intel_idle, 982 .enter_s2idle = intel_idle_s2idle, }, 983 { 984 .name = "C10", 985 .desc = "MWAIT 0x60", 986 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 987 .exit_latency = 660, 988 .target_residency = 1980, 989 .enter = &intel_idle, 990 .enter_s2idle = intel_idle_s2idle, }, 991 { 992 .enter = NULL } 993 }; 994 995 static struct cpuidle_state spr_cstates[] __initdata = { 996 { 997 .name = "C1", 998 .desc = "MWAIT 0x00", 999 .flags = MWAIT2flg(0x00), 1000 .exit_latency = 1, 1001 .target_residency = 1, 1002 .enter = &intel_idle, 1003 .enter_s2idle = intel_idle_s2idle, }, 1004 { 1005 .name = "C1E", 1006 .desc = "MWAIT 0x01", 1007 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1008 .exit_latency = 2, 1009 .target_residency = 4, 1010 .enter = &intel_idle, 1011 .enter_s2idle = intel_idle_s2idle, }, 1012 { 1013 .name = "C6", 1014 .desc = "MWAIT 0x20", 1015 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1016 CPUIDLE_FLAG_INIT_XSTATE, 1017 .exit_latency = 290, 1018 .target_residency = 800, 1019 .enter = &intel_idle, 1020 .enter_s2idle = intel_idle_s2idle, }, 1021 { 1022 .enter = NULL } 1023 }; 1024 1025 static struct cpuidle_state gnr_cstates[] __initdata = { 1026 { 1027 .name = "C1", 1028 .desc = "MWAIT 0x00", 1029 .flags = MWAIT2flg(0x00), 1030 .exit_latency = 1, 1031 .target_residency = 1, 1032 .enter = &intel_idle, 1033 .enter_s2idle = intel_idle_s2idle, }, 1034 { 1035 .name = "C1E", 1036 .desc = "MWAIT 0x01", 1037 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1038 .exit_latency = 4, 1039 .target_residency = 4, 1040 .enter = &intel_idle, 1041 .enter_s2idle = intel_idle_s2idle, }, 1042 { 1043 .name = "C6", 1044 .desc = "MWAIT 0x20", 1045 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1046 CPUIDLE_FLAG_INIT_XSTATE, 1047 .exit_latency = 170, 1048 .target_residency = 650, 1049 .enter = &intel_idle, 1050 .enter_s2idle = intel_idle_s2idle, }, 1051 { 1052 .name = "C6P", 1053 .desc = "MWAIT 0x21", 1054 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED | 1055 CPUIDLE_FLAG_INIT_XSTATE, 1056 .exit_latency = 210, 1057 .target_residency = 1000, 1058 .enter = &intel_idle, 1059 .enter_s2idle = intel_idle_s2idle, }, 1060 { 1061 .enter = NULL } 1062 }; 1063 1064 static struct cpuidle_state atom_cstates[] __initdata = { 1065 { 1066 .name = "C1E", 1067 .desc = "MWAIT 0x00", 1068 .flags = MWAIT2flg(0x00), 1069 .exit_latency = 10, 1070 .target_residency = 20, 1071 .enter = &intel_idle, 1072 .enter_s2idle = intel_idle_s2idle, }, 1073 { 1074 .name = "C2", 1075 .desc = "MWAIT 0x10", 1076 .flags = MWAIT2flg(0x10), 1077 .exit_latency = 20, 1078 .target_residency = 80, 1079 .enter = &intel_idle, 1080 .enter_s2idle = intel_idle_s2idle, }, 1081 { 1082 .name = "C4", 1083 .desc = "MWAIT 0x30", 1084 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1085 .exit_latency = 100, 1086 .target_residency = 400, 1087 .enter = &intel_idle, 1088 .enter_s2idle = intel_idle_s2idle, }, 1089 { 1090 .name = "C6", 1091 .desc = "MWAIT 0x52", 1092 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1093 .exit_latency = 140, 1094 .target_residency = 560, 1095 .enter = &intel_idle, 1096 .enter_s2idle = intel_idle_s2idle, }, 1097 { 1098 .enter = NULL } 1099 }; 1100 static struct cpuidle_state tangier_cstates[] __initdata = { 1101 { 1102 .name = "C1", 1103 .desc = "MWAIT 0x00", 1104 .flags = MWAIT2flg(0x00), 1105 .exit_latency = 1, 1106 .target_residency = 4, 1107 .enter = &intel_idle, 1108 .enter_s2idle = intel_idle_s2idle, }, 1109 { 1110 .name = "C4", 1111 .desc = "MWAIT 0x30", 1112 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1113 .exit_latency = 100, 1114 .target_residency = 400, 1115 .enter = &intel_idle, 1116 .enter_s2idle = intel_idle_s2idle, }, 1117 { 1118 .name = "C6", 1119 .desc = "MWAIT 0x52", 1120 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1121 .exit_latency = 140, 1122 .target_residency = 560, 1123 .enter = &intel_idle, 1124 .enter_s2idle = intel_idle_s2idle, }, 1125 { 1126 .name = "C7", 1127 .desc = "MWAIT 0x60", 1128 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1129 .exit_latency = 1200, 1130 .target_residency = 4000, 1131 .enter = &intel_idle, 1132 .enter_s2idle = intel_idle_s2idle, }, 1133 { 1134 .name = "C9", 1135 .desc = "MWAIT 0x64", 1136 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1137 .exit_latency = 10000, 1138 .target_residency = 20000, 1139 .enter = &intel_idle, 1140 .enter_s2idle = intel_idle_s2idle, }, 1141 { 1142 .enter = NULL } 1143 }; 1144 static struct cpuidle_state avn_cstates[] __initdata = { 1145 { 1146 .name = "C1", 1147 .desc = "MWAIT 0x00", 1148 .flags = MWAIT2flg(0x00), 1149 .exit_latency = 2, 1150 .target_residency = 2, 1151 .enter = &intel_idle, 1152 .enter_s2idle = intel_idle_s2idle, }, 1153 { 1154 .name = "C6", 1155 .desc = "MWAIT 0x51", 1156 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1157 .exit_latency = 15, 1158 .target_residency = 45, 1159 .enter = &intel_idle, 1160 .enter_s2idle = intel_idle_s2idle, }, 1161 { 1162 .enter = NULL } 1163 }; 1164 static struct cpuidle_state knl_cstates[] __initdata = { 1165 { 1166 .name = "C1", 1167 .desc = "MWAIT 0x00", 1168 .flags = MWAIT2flg(0x00), 1169 .exit_latency = 1, 1170 .target_residency = 2, 1171 .enter = &intel_idle, 1172 .enter_s2idle = intel_idle_s2idle }, 1173 { 1174 .name = "C6", 1175 .desc = "MWAIT 0x10", 1176 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1177 .exit_latency = 120, 1178 .target_residency = 500, 1179 .enter = &intel_idle, 1180 .enter_s2idle = intel_idle_s2idle }, 1181 { 1182 .enter = NULL } 1183 }; 1184 1185 static struct cpuidle_state bxt_cstates[] __initdata = { 1186 { 1187 .name = "C1", 1188 .desc = "MWAIT 0x00", 1189 .flags = MWAIT2flg(0x00), 1190 .exit_latency = 2, 1191 .target_residency = 2, 1192 .enter = &intel_idle, 1193 .enter_s2idle = intel_idle_s2idle, }, 1194 { 1195 .name = "C1E", 1196 .desc = "MWAIT 0x01", 1197 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1198 .exit_latency = 10, 1199 .target_residency = 20, 1200 .enter = &intel_idle, 1201 .enter_s2idle = intel_idle_s2idle, }, 1202 { 1203 .name = "C6", 1204 .desc = "MWAIT 0x20", 1205 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1206 .exit_latency = 133, 1207 .target_residency = 133, 1208 .enter = &intel_idle, 1209 .enter_s2idle = intel_idle_s2idle, }, 1210 { 1211 .name = "C7s", 1212 .desc = "MWAIT 0x31", 1213 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1214 .exit_latency = 155, 1215 .target_residency = 155, 1216 .enter = &intel_idle, 1217 .enter_s2idle = intel_idle_s2idle, }, 1218 { 1219 .name = "C8", 1220 .desc = "MWAIT 0x40", 1221 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1222 .exit_latency = 1000, 1223 .target_residency = 1000, 1224 .enter = &intel_idle, 1225 .enter_s2idle = intel_idle_s2idle, }, 1226 { 1227 .name = "C9", 1228 .desc = "MWAIT 0x50", 1229 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1230 .exit_latency = 2000, 1231 .target_residency = 2000, 1232 .enter = &intel_idle, 1233 .enter_s2idle = intel_idle_s2idle, }, 1234 { 1235 .name = "C10", 1236 .desc = "MWAIT 0x60", 1237 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1238 .exit_latency = 10000, 1239 .target_residency = 10000, 1240 .enter = &intel_idle, 1241 .enter_s2idle = intel_idle_s2idle, }, 1242 { 1243 .enter = NULL } 1244 }; 1245 1246 static struct cpuidle_state dnv_cstates[] __initdata = { 1247 { 1248 .name = "C1", 1249 .desc = "MWAIT 0x00", 1250 .flags = MWAIT2flg(0x00), 1251 .exit_latency = 2, 1252 .target_residency = 2, 1253 .enter = &intel_idle, 1254 .enter_s2idle = intel_idle_s2idle, }, 1255 { 1256 .name = "C1E", 1257 .desc = "MWAIT 0x01", 1258 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1259 .exit_latency = 10, 1260 .target_residency = 20, 1261 .enter = &intel_idle, 1262 .enter_s2idle = intel_idle_s2idle, }, 1263 { 1264 .name = "C6", 1265 .desc = "MWAIT 0x20", 1266 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1267 .exit_latency = 50, 1268 .target_residency = 500, 1269 .enter = &intel_idle, 1270 .enter_s2idle = intel_idle_s2idle, }, 1271 { 1272 .enter = NULL } 1273 }; 1274 1275 /* 1276 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1277 * C6, and this is indicated in the CPUID mwait leaf. 1278 */ 1279 static struct cpuidle_state snr_cstates[] __initdata = { 1280 { 1281 .name = "C1", 1282 .desc = "MWAIT 0x00", 1283 .flags = MWAIT2flg(0x00), 1284 .exit_latency = 2, 1285 .target_residency = 2, 1286 .enter = &intel_idle, 1287 .enter_s2idle = intel_idle_s2idle, }, 1288 { 1289 .name = "C1E", 1290 .desc = "MWAIT 0x01", 1291 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1292 .exit_latency = 15, 1293 .target_residency = 25, 1294 .enter = &intel_idle, 1295 .enter_s2idle = intel_idle_s2idle, }, 1296 { 1297 .name = "C6", 1298 .desc = "MWAIT 0x20", 1299 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1300 .exit_latency = 130, 1301 .target_residency = 500, 1302 .enter = &intel_idle, 1303 .enter_s2idle = intel_idle_s2idle, }, 1304 { 1305 .enter = NULL } 1306 }; 1307 1308 static struct cpuidle_state grr_cstates[] __initdata = { 1309 { 1310 .name = "C1", 1311 .desc = "MWAIT 0x00", 1312 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1313 .exit_latency = 1, 1314 .target_residency = 1, 1315 .enter = &intel_idle, 1316 .enter_s2idle = intel_idle_s2idle, }, 1317 { 1318 .name = "C1E", 1319 .desc = "MWAIT 0x01", 1320 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1321 .exit_latency = 2, 1322 .target_residency = 10, 1323 .enter = &intel_idle, 1324 .enter_s2idle = intel_idle_s2idle, }, 1325 { 1326 .name = "C6S", 1327 .desc = "MWAIT 0x22", 1328 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1329 .exit_latency = 140, 1330 .target_residency = 500, 1331 .enter = &intel_idle, 1332 .enter_s2idle = intel_idle_s2idle, }, 1333 { 1334 .enter = NULL } 1335 }; 1336 1337 static struct cpuidle_state srf_cstates[] __initdata = { 1338 { 1339 .name = "C1", 1340 .desc = "MWAIT 0x00", 1341 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1342 .exit_latency = 1, 1343 .target_residency = 1, 1344 .enter = &intel_idle, 1345 .enter_s2idle = intel_idle_s2idle, }, 1346 { 1347 .name = "C1E", 1348 .desc = "MWAIT 0x01", 1349 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1350 .exit_latency = 2, 1351 .target_residency = 10, 1352 .enter = &intel_idle, 1353 .enter_s2idle = intel_idle_s2idle, }, 1354 { 1355 .name = "C6S", 1356 .desc = "MWAIT 0x22", 1357 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1358 .exit_latency = 270, 1359 .target_residency = 700, 1360 .enter = &intel_idle, 1361 .enter_s2idle = intel_idle_s2idle, }, 1362 { 1363 .name = "C6SP", 1364 .desc = "MWAIT 0x23", 1365 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED, 1366 .exit_latency = 310, 1367 .target_residency = 900, 1368 .enter = &intel_idle, 1369 .enter_s2idle = intel_idle_s2idle, }, 1370 { 1371 .enter = NULL } 1372 }; 1373 1374 static const struct idle_cpu idle_cpu_nehalem __initconst = { 1375 .state_table = nehalem_cstates, 1376 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1377 .disable_promotion_to_c1e = true, 1378 }; 1379 1380 static const struct idle_cpu idle_cpu_nhx __initconst = { 1381 .state_table = nehalem_cstates, 1382 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1383 .disable_promotion_to_c1e = true, 1384 .use_acpi = true, 1385 }; 1386 1387 static const struct idle_cpu idle_cpu_atom __initconst = { 1388 .state_table = atom_cstates, 1389 }; 1390 1391 static const struct idle_cpu idle_cpu_tangier __initconst = { 1392 .state_table = tangier_cstates, 1393 }; 1394 1395 static const struct idle_cpu idle_cpu_lincroft __initconst = { 1396 .state_table = atom_cstates, 1397 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1398 }; 1399 1400 static const struct idle_cpu idle_cpu_snb __initconst = { 1401 .state_table = snb_cstates, 1402 .disable_promotion_to_c1e = true, 1403 }; 1404 1405 static const struct idle_cpu idle_cpu_snx __initconst = { 1406 .state_table = snb_cstates, 1407 .disable_promotion_to_c1e = true, 1408 .use_acpi = true, 1409 }; 1410 1411 static const struct idle_cpu idle_cpu_byt __initconst = { 1412 .state_table = byt_cstates, 1413 .disable_promotion_to_c1e = true, 1414 .byt_auto_demotion_disable_flag = true, 1415 }; 1416 1417 static const struct idle_cpu idle_cpu_cht __initconst = { 1418 .state_table = cht_cstates, 1419 .disable_promotion_to_c1e = true, 1420 .byt_auto_demotion_disable_flag = true, 1421 }; 1422 1423 static const struct idle_cpu idle_cpu_ivb __initconst = { 1424 .state_table = ivb_cstates, 1425 .disable_promotion_to_c1e = true, 1426 }; 1427 1428 static const struct idle_cpu idle_cpu_ivt __initconst = { 1429 .state_table = ivt_cstates, 1430 .disable_promotion_to_c1e = true, 1431 .use_acpi = true, 1432 }; 1433 1434 static const struct idle_cpu idle_cpu_hsw __initconst = { 1435 .state_table = hsw_cstates, 1436 .disable_promotion_to_c1e = true, 1437 }; 1438 1439 static const struct idle_cpu idle_cpu_hsx __initconst = { 1440 .state_table = hsw_cstates, 1441 .disable_promotion_to_c1e = true, 1442 .use_acpi = true, 1443 }; 1444 1445 static const struct idle_cpu idle_cpu_bdw __initconst = { 1446 .state_table = bdw_cstates, 1447 .disable_promotion_to_c1e = true, 1448 }; 1449 1450 static const struct idle_cpu idle_cpu_bdx __initconst = { 1451 .state_table = bdw_cstates, 1452 .disable_promotion_to_c1e = true, 1453 .use_acpi = true, 1454 }; 1455 1456 static const struct idle_cpu idle_cpu_skl __initconst = { 1457 .state_table = skl_cstates, 1458 .disable_promotion_to_c1e = true, 1459 }; 1460 1461 static const struct idle_cpu idle_cpu_skx __initconst = { 1462 .state_table = skx_cstates, 1463 .disable_promotion_to_c1e = true, 1464 .use_acpi = true, 1465 }; 1466 1467 static const struct idle_cpu idle_cpu_icx __initconst = { 1468 .state_table = icx_cstates, 1469 .disable_promotion_to_c1e = true, 1470 .use_acpi = true, 1471 }; 1472 1473 static const struct idle_cpu idle_cpu_adl __initconst = { 1474 .state_table = adl_cstates, 1475 }; 1476 1477 static const struct idle_cpu idle_cpu_adl_l __initconst = { 1478 .state_table = adl_l_cstates, 1479 }; 1480 1481 static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1482 .state_table = mtl_l_cstates, 1483 }; 1484 1485 static const struct idle_cpu idle_cpu_gmt __initconst = { 1486 .state_table = gmt_cstates, 1487 }; 1488 1489 static const struct idle_cpu idle_cpu_spr __initconst = { 1490 .state_table = spr_cstates, 1491 .disable_promotion_to_c1e = true, 1492 .use_acpi = true, 1493 }; 1494 1495 static const struct idle_cpu idle_cpu_gnr __initconst = { 1496 .state_table = gnr_cstates, 1497 .disable_promotion_to_c1e = true, 1498 .use_acpi = true, 1499 }; 1500 1501 static const struct idle_cpu idle_cpu_avn __initconst = { 1502 .state_table = avn_cstates, 1503 .disable_promotion_to_c1e = true, 1504 .use_acpi = true, 1505 }; 1506 1507 static const struct idle_cpu idle_cpu_knl __initconst = { 1508 .state_table = knl_cstates, 1509 .use_acpi = true, 1510 }; 1511 1512 static const struct idle_cpu idle_cpu_bxt __initconst = { 1513 .state_table = bxt_cstates, 1514 .disable_promotion_to_c1e = true, 1515 }; 1516 1517 static const struct idle_cpu idle_cpu_dnv __initconst = { 1518 .state_table = dnv_cstates, 1519 .disable_promotion_to_c1e = true, 1520 .use_acpi = true, 1521 }; 1522 1523 static const struct idle_cpu idle_cpu_tmt __initconst = { 1524 .disable_promotion_to_c1e = true, 1525 }; 1526 1527 static const struct idle_cpu idle_cpu_snr __initconst = { 1528 .state_table = snr_cstates, 1529 .disable_promotion_to_c1e = true, 1530 .use_acpi = true, 1531 }; 1532 1533 static const struct idle_cpu idle_cpu_grr __initconst = { 1534 .state_table = grr_cstates, 1535 .disable_promotion_to_c1e = true, 1536 .use_acpi = true, 1537 }; 1538 1539 static const struct idle_cpu idle_cpu_srf __initconst = { 1540 .state_table = srf_cstates, 1541 .disable_promotion_to_c1e = true, 1542 .use_acpi = true, 1543 }; 1544 1545 static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1546 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx), 1547 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem), 1548 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem), 1549 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem), 1550 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx), 1551 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx), 1552 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom), 1553 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft), 1554 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx), 1555 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb), 1556 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx), 1557 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom), 1558 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt), 1559 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1560 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht), 1561 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb), 1562 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt), 1563 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw), 1564 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx), 1565 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw), 1566 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw), 1567 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn), 1568 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw), 1569 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw), 1570 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx), 1571 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx), 1572 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl), 1573 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl), 1574 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl), 1575 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl), 1576 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx), 1577 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx), 1578 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx), 1579 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl), 1580 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l), 1581 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l), 1582 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt), 1583 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr), 1584 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr), 1585 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr), 1586 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl), 1587 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl), 1588 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt), 1589 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1590 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv), 1591 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt), 1592 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt), 1593 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr), 1594 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr), 1595 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf), 1596 {} 1597 }; 1598 1599 static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1600 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1601 {} 1602 }; 1603 1604 static bool __init intel_idle_max_cstate_reached(int cstate) 1605 { 1606 if (cstate + 1 > max_cstate) { 1607 pr_info("max_cstate %d reached\n", max_cstate); 1608 return true; 1609 } 1610 return false; 1611 } 1612 1613 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1614 { 1615 unsigned long eax = flg2MWAIT(state->flags); 1616 1617 if (boot_cpu_has(X86_FEATURE_ARAT)) 1618 return false; 1619 1620 /* 1621 * Switch over to one-shot tick broadcast if the target C-state 1622 * is deeper than C1. 1623 */ 1624 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1625 } 1626 1627 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1628 #include <acpi/processor.h> 1629 1630 static bool no_acpi __read_mostly; 1631 module_param(no_acpi, bool, 0444); 1632 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1633 1634 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1635 module_param_named(use_acpi, force_use_acpi, bool, 0444); 1636 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1637 1638 static struct acpi_processor_power acpi_state_table __initdata; 1639 1640 /** 1641 * intel_idle_cst_usable - Check if the _CST information can be used. 1642 * 1643 * Check if all of the C-states listed by _CST in the max_cstate range are 1644 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1645 */ 1646 static bool __init intel_idle_cst_usable(void) 1647 { 1648 int cstate, limit; 1649 1650 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1651 acpi_state_table.count); 1652 1653 for (cstate = 1; cstate < limit; cstate++) { 1654 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1655 1656 if (cx->entry_method != ACPI_CSTATE_FFH) 1657 return false; 1658 } 1659 1660 return true; 1661 } 1662 1663 static bool __init intel_idle_acpi_cst_extract(void) 1664 { 1665 unsigned int cpu; 1666 1667 if (no_acpi) { 1668 pr_debug("Not allowed to use ACPI _CST\n"); 1669 return false; 1670 } 1671 1672 for_each_possible_cpu(cpu) { 1673 struct acpi_processor *pr = per_cpu(processors, cpu); 1674 1675 if (!pr) 1676 continue; 1677 1678 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1679 continue; 1680 1681 acpi_state_table.count++; 1682 1683 if (!intel_idle_cst_usable()) 1684 continue; 1685 1686 if (!acpi_processor_claim_cst_control()) 1687 break; 1688 1689 return true; 1690 } 1691 1692 acpi_state_table.count = 0; 1693 pr_debug("ACPI _CST not found or not usable\n"); 1694 return false; 1695 } 1696 1697 static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1698 { 1699 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1700 1701 /* 1702 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1703 * the interesting states are ACPI_CSTATE_FFH. 1704 */ 1705 for (cstate = 1; cstate < limit; cstate++) { 1706 struct acpi_processor_cx *cx; 1707 struct cpuidle_state *state; 1708 1709 if (intel_idle_max_cstate_reached(cstate - 1)) 1710 break; 1711 1712 cx = &acpi_state_table.states[cstate]; 1713 1714 state = &drv->states[drv->state_count++]; 1715 1716 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1717 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1718 state->exit_latency = cx->latency; 1719 /* 1720 * For C1-type C-states use the same number for both the exit 1721 * latency and target residency, because that is the case for 1722 * C1 in the majority of the static C-states tables above. 1723 * For the other types of C-states, however, set the target 1724 * residency to 3 times the exit latency which should lead to 1725 * a reasonable balance between energy-efficiency and 1726 * performance in the majority of interesting cases. 1727 */ 1728 state->target_residency = cx->latency; 1729 if (cx->type > ACPI_STATE_C1) 1730 state->target_residency *= 3; 1731 1732 state->flags = MWAIT2flg(cx->address); 1733 if (cx->type > ACPI_STATE_C2) 1734 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1735 1736 if (disabled_states_mask & BIT(cstate)) 1737 state->flags |= CPUIDLE_FLAG_OFF; 1738 1739 if (intel_idle_state_needs_timer_stop(state)) 1740 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1741 1742 state->enter = intel_idle; 1743 state->enter_s2idle = intel_idle_s2idle; 1744 } 1745 } 1746 1747 static bool __init intel_idle_off_by_default(u32 mwait_hint) 1748 { 1749 int cstate, limit; 1750 1751 /* 1752 * If there are no _CST C-states, do not disable any C-states by 1753 * default. 1754 */ 1755 if (!acpi_state_table.count) 1756 return false; 1757 1758 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1759 /* 1760 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1761 * the interesting states are ACPI_CSTATE_FFH. 1762 */ 1763 for (cstate = 1; cstate < limit; cstate++) { 1764 if (acpi_state_table.states[cstate].address == mwait_hint) 1765 return false; 1766 } 1767 return true; 1768 } 1769 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1770 #define force_use_acpi (false) 1771 1772 static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1773 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1774 static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 1775 #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1776 1777 /** 1778 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1779 * 1780 * Tune IVT multi-socket targets. 1781 * Assumption: num_sockets == (max_package_num + 1). 1782 */ 1783 static void __init ivt_idle_state_table_update(void) 1784 { 1785 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1786 int cpu, package_num, num_sockets = 1; 1787 1788 for_each_online_cpu(cpu) { 1789 package_num = topology_physical_package_id(cpu); 1790 if (package_num + 1 > num_sockets) { 1791 num_sockets = package_num + 1; 1792 1793 if (num_sockets > 4) { 1794 cpuidle_state_table = ivt_cstates_8s; 1795 return; 1796 } 1797 } 1798 } 1799 1800 if (num_sockets > 2) 1801 cpuidle_state_table = ivt_cstates_4s; 1802 1803 /* else, 1 and 2 socket systems use default ivt_cstates */ 1804 } 1805 1806 /** 1807 * irtl_2_usec - IRTL to microseconds conversion. 1808 * @irtl: IRTL MSR value. 1809 * 1810 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1811 */ 1812 static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1813 { 1814 static const unsigned int irtl_ns_units[] __initconst = { 1815 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1816 }; 1817 unsigned long long ns; 1818 1819 if (!irtl) 1820 return 0; 1821 1822 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1823 1824 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1825 } 1826 1827 /** 1828 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1829 * 1830 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1831 * definitive maximum latency and use the same value for target_residency. 1832 */ 1833 static void __init bxt_idle_state_table_update(void) 1834 { 1835 unsigned long long msr; 1836 unsigned int usec; 1837 1838 rdmsrl(MSR_PKGC6_IRTL, msr); 1839 usec = irtl_2_usec(msr); 1840 if (usec) { 1841 bxt_cstates[2].exit_latency = usec; 1842 bxt_cstates[2].target_residency = usec; 1843 } 1844 1845 rdmsrl(MSR_PKGC7_IRTL, msr); 1846 usec = irtl_2_usec(msr); 1847 if (usec) { 1848 bxt_cstates[3].exit_latency = usec; 1849 bxt_cstates[3].target_residency = usec; 1850 } 1851 1852 rdmsrl(MSR_PKGC8_IRTL, msr); 1853 usec = irtl_2_usec(msr); 1854 if (usec) { 1855 bxt_cstates[4].exit_latency = usec; 1856 bxt_cstates[4].target_residency = usec; 1857 } 1858 1859 rdmsrl(MSR_PKGC9_IRTL, msr); 1860 usec = irtl_2_usec(msr); 1861 if (usec) { 1862 bxt_cstates[5].exit_latency = usec; 1863 bxt_cstates[5].target_residency = usec; 1864 } 1865 1866 rdmsrl(MSR_PKGC10_IRTL, msr); 1867 usec = irtl_2_usec(msr); 1868 if (usec) { 1869 bxt_cstates[6].exit_latency = usec; 1870 bxt_cstates[6].target_residency = usec; 1871 } 1872 1873 } 1874 1875 /** 1876 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1877 * 1878 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1879 */ 1880 static void __init sklh_idle_state_table_update(void) 1881 { 1882 unsigned long long msr; 1883 unsigned int eax, ebx, ecx, edx; 1884 1885 1886 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1887 if (max_cstate <= 7) 1888 return; 1889 1890 /* if PC10 not present in CPUID.MWAIT.EDX */ 1891 if ((mwait_substates & (0xF << 28)) == 0) 1892 return; 1893 1894 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1895 1896 /* PC10 is not enabled in PKG C-state limit */ 1897 if ((msr & 0xF) != 8) 1898 return; 1899 1900 ecx = 0; 1901 cpuid(7, &eax, &ebx, &ecx, &edx); 1902 1903 /* if SGX is present */ 1904 if (ebx & (1 << 2)) { 1905 1906 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1907 1908 /* if SGX is enabled */ 1909 if (msr & (1 << 18)) 1910 return; 1911 } 1912 1913 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1914 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1915 } 1916 1917 /** 1918 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1919 * idle states table. 1920 */ 1921 static void __init skx_idle_state_table_update(void) 1922 { 1923 unsigned long long msr; 1924 1925 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1926 1927 /* 1928 * 000b: C0/C1 (no package C-state support) 1929 * 001b: C2 1930 * 010b: C6 (non-retention) 1931 * 011b: C6 (retention) 1932 * 111b: No Package C state limits. 1933 */ 1934 if ((msr & 0x7) < 2) { 1935 /* 1936 * Uses the CC6 + PC0 latency and 3 times of 1937 * latency for target_residency if the PC6 1938 * is disabled in BIOS. This is consistent 1939 * with how intel_idle driver uses _CST 1940 * to set the target_residency. 1941 */ 1942 skx_cstates[2].exit_latency = 92; 1943 skx_cstates[2].target_residency = 276; 1944 } 1945 } 1946 1947 /** 1948 * adl_idle_state_table_update - Adjust AlderLake idle states table. 1949 */ 1950 static void __init adl_idle_state_table_update(void) 1951 { 1952 /* Check if user prefers C1 over C1E. */ 1953 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 1954 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1955 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 1956 1957 /* Disable C1E by clearing the "C1E promotion" bit. */ 1958 c1e_promotion = C1E_PROMOTION_DISABLE; 1959 return; 1960 } 1961 1962 /* Make sure C1E is enabled by default */ 1963 c1e_promotion = C1E_PROMOTION_ENABLE; 1964 } 1965 1966 /** 1967 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1968 */ 1969 static void __init spr_idle_state_table_update(void) 1970 { 1971 unsigned long long msr; 1972 1973 /* 1974 * By default, the C6 state assumes the worst-case scenario of package 1975 * C6. However, if PC6 is disabled, we update the numbers to match 1976 * core C6. 1977 */ 1978 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1979 1980 /* Limit value 2 and above allow for PC6. */ 1981 if ((msr & 0x7) < 2) { 1982 spr_cstates[2].exit_latency = 190; 1983 spr_cstates[2].target_residency = 600; 1984 } 1985 } 1986 1987 static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 1988 { 1989 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 1990 MWAIT_CSTATE_MASK; 1991 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 1992 MWAIT_SUBSTATE_MASK; 1993 1994 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 1995 if (num_substates == 0) 1996 return false; 1997 1998 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1999 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 2000 2001 return true; 2002 } 2003 2004 static void state_update_enter_method(struct cpuidle_state *state, int cstate) 2005 { 2006 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 2007 /* 2008 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 2009 * is not currently supported but this driver. 2010 */ 2011 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 2012 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2013 state->enter = intel_idle_xstate; 2014 return; 2015 } 2016 2017 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 2018 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 2019 /* 2020 * IBRS mitigation requires that C-states are entered 2021 * with interrupts disabled. 2022 */ 2023 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 2024 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 2025 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 2026 state->enter = intel_idle_ibrs; 2027 return; 2028 } 2029 2030 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 2031 state->enter = intel_idle_irq; 2032 return; 2033 } 2034 2035 if (force_irq_on) { 2036 pr_info("forced intel_idle_irq for state %d\n", cstate); 2037 state->enter = intel_idle_irq; 2038 } 2039 } 2040 2041 static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 2042 { 2043 int cstate; 2044 2045 switch (boot_cpu_data.x86_vfm) { 2046 case INTEL_IVYBRIDGE_X: 2047 ivt_idle_state_table_update(); 2048 break; 2049 case INTEL_ATOM_GOLDMONT: 2050 case INTEL_ATOM_GOLDMONT_PLUS: 2051 bxt_idle_state_table_update(); 2052 break; 2053 case INTEL_SKYLAKE: 2054 sklh_idle_state_table_update(); 2055 break; 2056 case INTEL_SKYLAKE_X: 2057 skx_idle_state_table_update(); 2058 break; 2059 case INTEL_SAPPHIRERAPIDS_X: 2060 case INTEL_EMERALDRAPIDS_X: 2061 spr_idle_state_table_update(); 2062 break; 2063 case INTEL_ALDERLAKE: 2064 case INTEL_ALDERLAKE_L: 2065 case INTEL_ATOM_GRACEMONT: 2066 adl_idle_state_table_update(); 2067 break; 2068 } 2069 2070 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2071 struct cpuidle_state *state; 2072 unsigned int mwait_hint; 2073 2074 if (intel_idle_max_cstate_reached(cstate)) 2075 break; 2076 2077 if (!cpuidle_state_table[cstate].enter && 2078 !cpuidle_state_table[cstate].enter_s2idle) 2079 break; 2080 2081 /* If marked as unusable, skip this state. */ 2082 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2083 pr_debug("state %s is disabled\n", 2084 cpuidle_state_table[cstate].name); 2085 continue; 2086 } 2087 2088 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2089 if (!intel_idle_verify_cstate(mwait_hint)) 2090 continue; 2091 2092 /* Structure copy. */ 2093 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2094 state = &drv->states[drv->state_count]; 2095 2096 state_update_enter_method(state, cstate); 2097 2098 2099 if ((disabled_states_mask & BIT(drv->state_count)) || 2100 ((icpu->use_acpi || force_use_acpi) && 2101 intel_idle_off_by_default(mwait_hint) && 2102 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2103 state->flags |= CPUIDLE_FLAG_OFF; 2104 2105 if (intel_idle_state_needs_timer_stop(state)) 2106 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2107 2108 drv->state_count++; 2109 } 2110 2111 if (icpu->byt_auto_demotion_disable_flag) { 2112 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2113 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2114 } 2115 } 2116 2117 /** 2118 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2119 * @drv: cpuidle driver structure to initialize. 2120 */ 2121 static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2122 { 2123 cpuidle_poll_state_init(drv); 2124 2125 if (disabled_states_mask & BIT(0)) 2126 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2127 2128 drv->state_count = 1; 2129 2130 if (icpu && icpu->state_table) 2131 intel_idle_init_cstates_icpu(drv); 2132 else 2133 intel_idle_init_cstates_acpi(drv); 2134 } 2135 2136 static void auto_demotion_disable(void) 2137 { 2138 unsigned long long msr_bits; 2139 2140 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2141 msr_bits &= ~auto_demotion_disable_flags; 2142 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2143 } 2144 2145 static void c1e_promotion_enable(void) 2146 { 2147 unsigned long long msr_bits; 2148 2149 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2150 msr_bits |= 0x2; 2151 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2152 } 2153 2154 static void c1e_promotion_disable(void) 2155 { 2156 unsigned long long msr_bits; 2157 2158 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2159 msr_bits &= ~0x2; 2160 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2161 } 2162 2163 /** 2164 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2165 * @cpu: CPU to initialize. 2166 * 2167 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2168 * with the processor model flags. 2169 */ 2170 static int intel_idle_cpu_init(unsigned int cpu) 2171 { 2172 struct cpuidle_device *dev; 2173 2174 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2175 dev->cpu = cpu; 2176 2177 if (cpuidle_register_device(dev)) { 2178 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2179 return -EIO; 2180 } 2181 2182 if (auto_demotion_disable_flags) 2183 auto_demotion_disable(); 2184 2185 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2186 c1e_promotion_enable(); 2187 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2188 c1e_promotion_disable(); 2189 2190 return 0; 2191 } 2192 2193 static int intel_idle_cpu_online(unsigned int cpu) 2194 { 2195 struct cpuidle_device *dev; 2196 2197 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2198 tick_broadcast_enable(); 2199 2200 /* 2201 * Some systems can hotplug a cpu at runtime after 2202 * the kernel has booted, we have to initialize the 2203 * driver in this case 2204 */ 2205 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2206 if (!dev->registered) 2207 return intel_idle_cpu_init(cpu); 2208 2209 return 0; 2210 } 2211 2212 /** 2213 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2214 */ 2215 static void __init intel_idle_cpuidle_devices_uninit(void) 2216 { 2217 int i; 2218 2219 for_each_online_cpu(i) 2220 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2221 } 2222 2223 static int __init intel_idle_init(void) 2224 { 2225 const struct x86_cpu_id *id; 2226 unsigned int eax, ebx, ecx; 2227 int retval; 2228 2229 /* Do not load intel_idle at all for now if idle= is passed */ 2230 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2231 return -ENODEV; 2232 2233 if (max_cstate == 0) { 2234 pr_debug("disabled\n"); 2235 return -EPERM; 2236 } 2237 2238 id = x86_match_cpu(intel_idle_ids); 2239 if (id) { 2240 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2241 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2242 return -ENODEV; 2243 } 2244 } else { 2245 id = x86_match_cpu(intel_mwait_ids); 2246 if (!id) 2247 return -ENODEV; 2248 } 2249 2250 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 2251 return -ENODEV; 2252 2253 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 2254 2255 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2256 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2257 !mwait_substates) 2258 return -ENODEV; 2259 2260 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2261 2262 icpu = (const struct idle_cpu *)id->driver_data; 2263 if (icpu) { 2264 if (icpu->state_table) 2265 cpuidle_state_table = icpu->state_table; 2266 else if (!intel_idle_acpi_cst_extract()) 2267 return -ENODEV; 2268 2269 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2270 if (icpu->disable_promotion_to_c1e) 2271 c1e_promotion = C1E_PROMOTION_DISABLE; 2272 if (icpu->use_acpi || force_use_acpi) 2273 intel_idle_acpi_cst_extract(); 2274 } else if (!intel_idle_acpi_cst_extract()) { 2275 return -ENODEV; 2276 } 2277 2278 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2279 boot_cpu_data.x86_model); 2280 2281 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2282 if (!intel_idle_cpuidle_devices) 2283 return -ENOMEM; 2284 2285 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2286 2287 retval = cpuidle_register_driver(&intel_idle_driver); 2288 if (retval) { 2289 struct cpuidle_driver *drv = cpuidle_get_driver(); 2290 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2291 drv ? drv->name : "none"); 2292 goto init_driver_fail; 2293 } 2294 2295 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2296 intel_idle_cpu_online, NULL); 2297 if (retval < 0) 2298 goto hp_setup_fail; 2299 2300 pr_debug("Local APIC timer is reliable in %s\n", 2301 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2302 2303 return 0; 2304 2305 hp_setup_fail: 2306 intel_idle_cpuidle_devices_uninit(); 2307 cpuidle_unregister_driver(&intel_idle_driver); 2308 init_driver_fail: 2309 free_percpu(intel_idle_cpuidle_devices); 2310 return retval; 2311 2312 } 2313 device_initcall(intel_idle_init); 2314 2315 /* 2316 * We are not really modular, but we used to support that. Meaning we also 2317 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2318 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2319 * is the easiest way (currently) to continue doing that. 2320 */ 2321 module_param(max_cstate, int, 0444); 2322 /* 2323 * The positions of the bits that are set in this number are the indices of the 2324 * idle states to be disabled by default (as reflected by the names of the 2325 * corresponding idle state directories in sysfs, "state0", "state1" ... 2326 * "state<i>" ..., where <i> is the index of the given state). 2327 */ 2328 module_param_named(states_off, disabled_states_mask, uint, 0444); 2329 MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2330 /* 2331 * Some platforms come with mutually exclusive C-states, so that if one is 2332 * enabled, the other C-states must not be used. Example: C1 and C1E on 2333 * Sapphire Rapids platform. This parameter allows for selecting the 2334 * preferred C-states among the groups of mutually exclusive C-states - the 2335 * selected C-states will be registered, the other C-states from the mutually 2336 * exclusive group won't be registered. If the platform has no mutually 2337 * exclusive C-states, this parameter has no effect. 2338 */ 2339 module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2340 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2341 /* 2342 * Debugging option that forces the driver to enter all C-states with 2343 * interrupts enabled. Does not apply to C-states with 2344 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2345 */ 2346 module_param(force_irq_on, bool, 0444); 2347 /* 2348 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2349 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2350 */ 2351 module_param(ibrs_off, bool, 0444); 2352 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2353