2  * cpuidle.c - core cpuidle infrastructure
4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
11 #include "linux/percpu-defs.h"
17 #include <linux/sched/idle.h>
55 	return off || !initialized || !drv || !dev || !dev->enabled;  in cpuidle_not_available()
59  * cpuidle_play_dead - cpu off-lining
70 		return -ENODEV;  in cpuidle_play_dead()
72 	/* Find lowest-power state that supports long-term idle */  in cpuidle_play_dead()
73 	for (i = drv->state_count - 1; i >= 0; i--)  in cpuidle_play_dead()
74 		if (drv->states[i].enter_dead)  in cpuidle_play_dead()
75 			return drv->states[i].enter_dead(dev, i);  in cpuidle_play_dead()
77 	return -ENODEV;  in cpuidle_play_dead()
89 	for (i = 1; i < drv->state_count; i++) {  in find_deepest_state()
90 		struct cpuidle_state *s = &drv->states[i];  in find_deepest_state()
92 		if (dev->states_usage[i].disable ||  in find_deepest_state()
93 		    s->exit_latency_ns <= latency_req ||  in find_deepest_state()
94 		    s->exit_latency_ns > max_latency_ns ||  in find_deepest_state()
95 		    (s->flags & forbidden_flags) ||  in find_deepest_state()
96 		    (s2idle && !s->enter_s2idle))  in find_deepest_state()
99 		latency_req = s->exit_latency_ns;  in find_deepest_state()
106  * cpuidle_use_deepest_state - Set/unset governor override mode.
107  * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
109  * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle
110  * state with exit latency within @latency_limit_ns (override governors going
120 		dev->forced_idle_latency_limit_ns = latency_limit_ns;  in cpuidle_use_deepest_state()
125  * cpuidle_find_deepest_state - Find the deepest available idle state.
128  * @latency_limit_ns: Idle state exit latency limit
130  * Return: the index of the deepest available idle state.
143 	struct cpuidle_state *target_state = &drv->states[index];  in enter_s2idle_proper()
152 	 * The state used here cannot be a "coupled" one, because the "coupled"  in enter_s2idle_proper()
157 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in enter_s2idle_proper()
162 	target_state->enter_s2idle(dev, drv, index);  in enter_s2idle_proper()
165 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in enter_s2idle_proper()
174 	dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);  in enter_s2idle_proper()
175 	dev->states_usage[index].s2idle_usage++;  in enter_s2idle_proper()
180  * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
184  * If there are states with the ->enter_s2idle callback, find the deepest of
192 	 * Find the deepest state with ->enter_s2idle present, which guarantees  in cpuidle_enter_s2idle()
206  * cpuidle_enter_state - enter the state and update stats
209  * @index: index into the states table in @drv of the state to enter
217 	struct cpuidle_state *target_state = &drv->states[index];  in cpuidle_enter_state()
218 	bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);  in cpuidle_enter_state()
229 		index = find_deepest_state(drv, dev, target_state->exit_latency_ns,  in cpuidle_enter_state()
232 		target_state = &drv->states[index];  in cpuidle_enter_state()
236 	if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)  in cpuidle_enter_state()
239 	/* Take note of the planned idle state. */  in cpuidle_enter_state()
242 	trace_cpu_idle(index, dev->cpu);  in cpuidle_enter_state()
246 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in cpuidle_enter_state()
262 	 * functions called within the RCU-idle region.  in cpuidle_enter_state()
264 	entered_state = target_state->enter(dev, drv, index);  in cpuidle_enter_state()
266 	if (WARN_ONCE(!irqs_disabled(), "%ps leaked IRQ state", target_state->enter))  in cpuidle_enter_state()
269 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE)) {  in cpuidle_enter_state()
277 	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);  in cpuidle_enter_state()
279 	/* The cpu is no longer idle or about to enter idle. */  in cpuidle_enter_state()
289 		s64 diff, delay = drv->states[entered_state].exit_latency_ns;  in cpuidle_enter_state()
299 		dev->last_residency_ns = diff;  in cpuidle_enter_state()
300 		dev->states_usage[entered_state].time_ns += diff;  in cpuidle_enter_state()
301 		dev->states_usage[entered_state].usage++;  in cpuidle_enter_state()
303 		if (diff < drv->states[entered_state].target_residency_ns) {  in cpuidle_enter_state()
304 			for (i = entered_state - 1; i >= 0; i--) {  in cpuidle_enter_state()
305 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
309 				dev->states_usage[entered_state].above++;  in cpuidle_enter_state()
310 				trace_cpu_idle_miss(dev->cpu, entered_state, false);  in cpuidle_enter_state()
314 			for (i = entered_state + 1; i < drv->state_count; i++) {  in cpuidle_enter_state()
315 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
319 				 * Update if a deeper state would have been a  in cpuidle_enter_state()
320 				 * better match for the observed idle duration.  in cpuidle_enter_state()
322 				if (diff - delay >= drv->states[i].target_residency_ns) {  in cpuidle_enter_state()
323 					dev->states_usage[entered_state].below++;  in cpuidle_enter_state()
324 					trace_cpu_idle_miss(dev->cpu, entered_state, true);  in cpuidle_enter_state()
331 		dev->last_residency_ns = 0;  in cpuidle_enter_state()
332 		dev->states_usage[index].rejected++;  in cpuidle_enter_state()
341  * cpuidle_select - ask the cpuidle framework to choose an idle state
347  * Returns the index of the idle state.  The return value must not be negative.
351  * entering the returned state.
356 	return cpuidle_curr_governor->select(drv, dev, stop_tick);  in cpuidle_select()
360  * cpuidle_enter - enter into the specified idle state
364  * @index: the index in the idle state table
366  * Returns the index in the idle state, < 0 in case of error.
378 	 * ->select() callback have decided, whether to stop the tick or not.  in cpuidle_enter()
380 	WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());  in cpuidle_enter()
387 	WRITE_ONCE(dev->next_hrtimer, 0);  in cpuidle_enter()
392  * cpuidle_reflect - tell the underlying governor what was the state
396  * @index: the index in the idle state table
401 	if (cpuidle_curr_governor->reflect && index >= 0)  in cpuidle_reflect()
402 		cpuidle_curr_governor->reflect(dev, index);  in cpuidle_reflect()
407  * for most users, the time for a single ping-pong workload like
411  * perf bench sched pipe -l 10000
419  * cpuidle_poll_time - return amount of time to poll for,
420  * governors can override dev->poll_limit_ns if necessary
434 	if (dev->poll_limit_ns)  in cpuidle_poll_time()
435 		return dev->poll_limit_ns;  in cpuidle_poll_time()
438 	for (i = 1; i < drv->state_count; i++) {  in cpuidle_poll_time()
441 		if (dev->states_usage[i].disable)  in cpuidle_poll_time()
444 		state_limit = drv->states[i].target_residency_ns;  in cpuidle_poll_time()
452 	dev->poll_limit_ns = limit_ns;  in cpuidle_poll_time()
454 	return dev->poll_limit_ns;  in cpuidle_poll_time()
458  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
463 		/* Make sure all changes finished before we switch to new idle */  in cpuidle_install_idle_handler()
470  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
481 	 * are done looking at pointed idle states.  in cpuidle_uninstall_idle_handler()
487  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
498  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
525  * cpuidle_enable_device - enables idle PM for a CPU
537 		return -EINVAL;  in cpuidle_enable_device()
539 	if (dev->enabled)  in cpuidle_enable_device()
543 		return -EIO;  in cpuidle_enable_device()
548 		return -EIO;  in cpuidle_enable_device()
550 	if (!dev->registered)  in cpuidle_enable_device()
551 		return -EINVAL;  in cpuidle_enable_device()
557 	if (cpuidle_curr_governor->enable) {  in cpuidle_enable_device()
558 		ret = cpuidle_curr_governor->enable(drv, dev);  in cpuidle_enable_device()
565 	dev->enabled = 1;  in cpuidle_enable_device()
579  * cpuidle_disable_device - disables idle PM for a CPU
589 	if (!dev || !dev->enabled)  in cpuidle_disable_device()
595 	dev->enabled = 0;  in cpuidle_disable_device()
597 	if (cpuidle_curr_governor->disable)  in cpuidle_disable_device()
598 		cpuidle_curr_governor->disable(drv, dev);  in cpuidle_disable_device()
601 	enabled_devices--;  in cpuidle_disable_device()
610 	list_del(&dev->device_list);  in __cpuidle_unregister_device()
611 	per_cpu(cpuidle_devices, dev->cpu) = NULL;  in __cpuidle_unregister_device()
612 	module_put(drv->owner);  in __cpuidle_unregister_device()
614 	dev->registered = 0;  in __cpuidle_unregister_device()
619 	memset(dev->states_usage, 0, sizeof(dev->states_usage));  in __cpuidle_device_init()
620 	dev->last_residency_ns = 0;  in __cpuidle_device_init()
621 	dev->next_hrtimer = 0;  in __cpuidle_device_init()
625  * __cpuidle_register_device - internal register function called before register
636 	if (!try_module_get(drv->owner))  in __cpuidle_register_device()
637 		return -EINVAL;  in __cpuidle_register_device()
639 	for (i = 0; i < drv->state_count; i++) {  in __cpuidle_register_device()
640 		if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE)  in __cpuidle_register_device()
641 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;  in __cpuidle_register_device()
643 		if (drv->states[i].flags & CPUIDLE_FLAG_OFF)  in __cpuidle_register_device()
644 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;  in __cpuidle_register_device()
647 	per_cpu(cpuidle_devices, dev->cpu) = dev;  in __cpuidle_register_device()
648 	list_add(&dev->device_list, &cpuidle_detected_devices);  in __cpuidle_register_device()
654 		dev->registered = 1;  in __cpuidle_register_device()
660  * cpuidle_register_device - registers a CPU's idle PM feature
665 	int ret = -EBUSY;  in cpuidle_register_device()
668 		return -EINVAL;  in cpuidle_register_device()
672 	if (dev->registered)  in cpuidle_register_device()
706  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
711 	if (!dev || dev->registered == 0)  in cpuidle_unregister_device()
741 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_unregister()
773 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_register()
775 		device->cpu = cpu;  in cpuidle_register()
779 		 * On multiplatform for ARM, the coupled idle states could be  in cpuidle_register()
784 			device->coupled_cpus = *coupled_cpus;  in cpuidle_register()
801  * cpuidle_init - core initializer
806 		return -ENODEV;  in cpuidle_init()