1 // SPDX-License-Identifier: GPL-2.0-only
3  * intel_idle.c - native hardware idle loop for modern Intel processors
5  * Copyright (c) 2013 - 2020, Intel Corporation.
23  *	for preventing entry into deep C-states
25  * CPU will flush caches as needed when entering a C-state via MWAIT
33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
39 /* un-comment DEBUG to enable pr_debug() statements */
58 #include <asm/intel-family.h>
60 #include <asm/spec-ctrl.h>
73 static int max_cstate = CPUIDLE_STATE_MAX - 1;
93 	 * Hardware C-state auto-demotion may not always be optimal.
113  * Enable interrupts before entering the C-state. On some platforms and for
114  * some C-states, this may measurably decrease interrupt latency.
130  * Initialize large xstate for the C6-state entrance.
135  * Ignore the sub-state when matching mwait hints between the ACPI _CST and
141  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
142  * the C-state (top nibble) and sub-state (bottom nibble)
154 	struct cpuidle_state *state = &drv->states[index];  in __intel_idle()
155 	unsigned int eax = flg2MWAIT(state->flags);  in __intel_idle()
156 	unsigned int ecx = 1*irqoff; /* break on interrupt flag */  in __intel_idle()  local
158 	mwait_idle_with_hints(eax, ecx);  in __intel_idle()
164  * intel_idle - Ask the processor to enter the given idle state.
173  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
215  * intel_idle_s2idle - Ask the processor to enter the given idle state.
223  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
229 	struct cpuidle_state *state = &drv->states[index];  in intel_idle_s2idle()
230 	unsigned int eax = flg2MWAIT(state->flags);  in intel_idle_s2idle()
231 	unsigned int ecx = 1; /* break on interrupt flag */  in intel_idle_s2idle()  local
233 	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)  in intel_idle_s2idle()
236 	mwait_idle_with_hints(eax, ecx);  in intel_idle_s2idle()
244 	struct cpuidle_state *state = &drv->states[index];  in intel_idle_enter_dead()
245 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle_enter_dead()
895 		.target_residency = 2000,
1299 		.exit_latency = 2000,
1300 		.target_residency = 2000,
1697 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle_state_needs_timer_stop()
1703 	 * Switch over to one-shot tick broadcast if the target C-state  in intel_idle_state_needs_timer_stop()
1727  * intel_idle_cst_usable - Check if the _CST information can be used.
1729  * Check if all of the C-states listed by _CST in the max_cstate range are
1742 		if (cx->entry_method != ACPI_CSTATE_FFH)  in intel_idle_cst_usable()
1764 		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))  in intel_idle_acpi_cst_extract()
1795 		if (intel_idle_max_cstate_reached(cstate - 1))  in intel_idle_init_cstates_acpi()
1800 		state = &drv->states[drv->state_count++];  in intel_idle_init_cstates_acpi()
1802 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);  in intel_idle_init_cstates_acpi()
1803 		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);  in intel_idle_init_cstates_acpi()
1804 		state->exit_latency = cx->latency;  in intel_idle_init_cstates_acpi()
1806 		 * For C1-type C-states use the same number for both the exit  in intel_idle_init_cstates_acpi()
1808 		 * C1 in the majority of the static C-states tables above.  in intel_idle_init_cstates_acpi()
1809 		 * For the other types of C-states, however, set the target  in intel_idle_init_cstates_acpi()
1811 		 * a reasonable balance between energy-efficiency and  in intel_idle_init_cstates_acpi()
1814 		state->target_residency = cx->latency;  in intel_idle_init_cstates_acpi()
1815 		if (cx->type > ACPI_STATE_C1)  in intel_idle_init_cstates_acpi()
1816 			state->target_residency *= 3;  in intel_idle_init_cstates_acpi()
1818 		state->flags = MWAIT2flg(cx->address);  in intel_idle_init_cstates_acpi()
1819 		if (cx->type > ACPI_STATE_C2)  in intel_idle_init_cstates_acpi()
1820 			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;  in intel_idle_init_cstates_acpi()
1823 			state->flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_acpi()
1826 			state->flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_acpi()
1828 		if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))  in intel_idle_init_cstates_acpi()
1831 		state->enter = intel_idle;  in intel_idle_init_cstates_acpi()
1832 		state->enter_dead = intel_idle_enter_dead;  in intel_idle_init_cstates_acpi()
1833 		state->enter_s2idle = intel_idle_s2idle;  in intel_idle_init_cstates_acpi()
1842 	 * If there are no _CST C-states, do not disable any C-states by  in intel_idle_off_by_default()
1885  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1887  * Tune IVT multi-socket targets.
1892 	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */  in ivt_idle_state_table_update()
1914  * irtl_2_usec - IRTL to microseconds conversion.
1935  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1983  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1985  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1990 	unsigned int eax, ebx, ecx, edx;  in sklh_idle_state_table_update()  local
2003 	/* PC10 is not enabled in PKG C-state limit */  in sklh_idle_state_table_update()
2007 	ecx = 0;  in sklh_idle_state_table_update()
2008 	cpuid(7, &eax, &ebx, &ecx, &edx);  in sklh_idle_state_table_update()
2020 	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */  in sklh_idle_state_table_update()
2021 	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */  in sklh_idle_state_table_update()
2025  * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
2035 	 * 000b: C0/C1 (no package C-state support)  in skx_idle_state_table_update()
2037 	 * 010b: C6 (non-retention)  in skx_idle_state_table_update()
2055  * adl_idle_state_table_update - Adjust AlderLake idle states table.
2074  * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
2081 	 * By default, the C6 state assumes the worst-case scenario of package  in spr_idle_state_table_update()
2095  * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2110 	/* Ignore the C-state if there are NO sub-states in CPUID for it. */  in intel_idle_verify_cstate()
2122 	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {  in state_update_enter_method()
2127 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);  in state_update_enter_method()
2128 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);  in state_update_enter_method()
2129 		state->enter = intel_idle_xstate;  in state_update_enter_method()
2134 			((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {  in state_update_enter_method()
2136 		 * IBRS mitigation requires that C-states are entered  in state_update_enter_method()
2139 		if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))  in state_update_enter_method()
2140 			state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;  in state_update_enter_method()
2141 		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);  in state_update_enter_method()
2142 		state->enter = intel_idle_ibrs;  in state_update_enter_method()
2146 	if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {  in state_update_enter_method()
2147 		state->enter = intel_idle_irq;  in state_update_enter_method()
2153 		state->enter = intel_idle_irq;  in state_update_enter_method()
2216 		drv->states[drv->state_count] = cpuidle_state_table[cstate];  in intel_idle_init_cstates_icpu()
2217 		state = &drv->states[drv->state_count];  in intel_idle_init_cstates_icpu()
2222 		if ((disabled_states_mask & BIT(drv->state_count)) ||  in intel_idle_init_cstates_icpu()
2223 		    ((icpu->use_acpi || force_use_acpi) &&  in intel_idle_init_cstates_icpu()
2224 		     intel_idle_off_by_default(state->flags, mwait_hint) &&  in intel_idle_init_cstates_icpu()
2225 		     !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))  in intel_idle_init_cstates_icpu()
2226 			state->flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_icpu()
2229 			state->flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_icpu()
2231 		drv->state_count++;  in intel_idle_init_cstates_icpu()
2236  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2244 		drv->states[0].flags |= CPUIDLE_FLAG_OFF;  in intel_idle_cpuidle_driver_init()
2246 	drv->state_count = 1;  in intel_idle_cpuidle_driver_init()
2248 	if (icpu && icpu->state_table)  in intel_idle_cpuidle_driver_init()
2282  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2293 	dev->cpu = cpu;  in intel_idle_cpu_init()
2297 		return -EIO;  in intel_idle_cpu_init()
2324 	if (!dev->registered)  in intel_idle_cpu_online()
2331  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2401 	err = sysfs_add_file_to_group(&sysfs_root->kobj,  in intel_idle_sysfs_init()
2417 	sysfs_remove_file_from_group(&sysfs_root->kobj,  in intel_idle_sysfs_uninit()
2426 	unsigned int eax, ebx, ecx;  in intel_idle_init()  local
2431 		return -ENODEV;  in intel_idle_init()
2435 		return -EPERM;  in intel_idle_init()
2442 			return -ENODEV;  in intel_idle_init()
2447 			return -ENODEV;  in intel_idle_init()
2450 	cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);  in intel_idle_init()
2452 	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||  in intel_idle_init()
2453 	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||  in intel_idle_init()
2455 			return -ENODEV;  in intel_idle_init()
2459 	icpu = (const struct idle_cpu *)id->driver_data;  in intel_idle_init()
2465 		if (icpu->state_table)  in intel_idle_init()
2466 			cpuidle_state_table = icpu->state_table;  in intel_idle_init()
2468 			return -ENODEV;  in intel_idle_init()
2470 		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;  in intel_idle_init()
2471 		if (icpu->disable_promotion_to_c1e)  in intel_idle_init()
2473 		if (icpu->c1_demotion_supported)  in intel_idle_init()
2475 		if (icpu->use_acpi || force_use_acpi)  in intel_idle_init()
2478 		return -ENODEV;  in intel_idle_init()
2486 		return -ENOMEM;  in intel_idle_init()
2498 		       drv ? drv->name : "none");  in intel_idle_init()
2508 		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");  in intel_idle_init()
2527  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2528  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2541  * Some platforms come with mutually exclusive C-states, so that if one is
2542  * enabled, the other C-states must not be used. Example: C1 and C1E on
2544  * preferred C-states among the groups of mutually exclusive C-states - the
2545  * selected C-states will be registered, the other C-states from the mutually
2547  * exclusive C-states, this parameter has no effect.
2552  * Debugging option that forces the driver to enter all C-states with
2553  * interrupts enabled. Does not apply to C-states with