11 #include <linux/cpu.h>
25  * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
26  * power down), or due to HW bugs (on OMAP4460, a cpu powering up
27  * will corrupt the gic state unless the other cpu runs a work
28  * around).  Each cpu has a power state that it can enter without
29  * coordinating with the other cpu (usually Wait For Interrupt, or
35  * This file implements a solution, where each cpu will wait in the
45  * each cpu will increment the ready counter, and continue once the
47  * cpu exits idle, the other cpus will decrement their counter and
50  * requested_state stores the deepest coupled idle state each cpu
54  * variable is not locked.  It is only written from the cpu that
55  * it stores (or by the on/offlining cpu if that cpu is offline),
71  *    set in the struct cpuidle_device for each cpu.
82  *    should ensure that the cpus all abort together if any cpu tries
95  * @prevent: flag to prevent coupled idle while a cpu is hotplugging
118  * __smp_call_function_single with the per cpu call_single_data_t struct already
125  * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
133  * @dev: cpuidle_device of the calling cpu
141  * The atomic variable must be initialized to 0 before any cpu calls
142  * this function, will be reset to 0 before any cpu returns from this function.
203  * cpuidle_coupled_set_ready - mark a cpu as ready
204  * @coupled: the struct coupled that contains the current cpu
212  * cpuidle_coupled_set_not_ready - mark a cpu as not ready
213  * @coupled: the struct coupled that contains the current cpu
216  * is equal to the number of online cpus.  Prevents a race where one cpu
218  * cpu has decremented its ready counter, leading to the ready counter going
240  * @coupled: the struct coupled that contains the current cpu
252  * @coupled: the struct coupled that contains the current cpu
264  * @coupled: the struct coupled that contains the current cpu
276  * @coupled: the struct coupled that contains the current cpu
288  * @dev: struct cpuidle_device for this cpu
289  * @coupled: the struct coupled that contains the current cpu
315 	int cpu = (unsigned long)info;  in cpuidle_coupled_handle_poke()  local
316 	cpumask_set_cpu(cpu, &cpuidle_coupled_poked);  in cpuidle_coupled_handle_poke()
317 	cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);  in cpuidle_coupled_handle_poke()
321  * cpuidle_coupled_poke - wake up a cpu that may be waiting
322  * @cpu: target cpu
324  * Ensures that the target cpu exits it's waiting idle state (if it is in it)
328  * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
332 static void cpuidle_coupled_poke(int cpu)  in cpuidle_coupled_poke()  argument
334 	call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);  in cpuidle_coupled_poke()
336 	if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))  in cpuidle_coupled_poke()
337 		smp_call_function_single_async(cpu, csd);  in cpuidle_coupled_poke()
342  * @this_cpu: target cpu
343  * @coupled: the struct coupled that contains the current cpu
350 	int cpu;  in cpuidle_coupled_poke_others()  local
352 	for_each_cpu(cpu, &coupled->coupled_cpus)  in cpuidle_coupled_poke_others()
353 		if (cpu != this_cpu && cpu_online(cpu))  in cpuidle_coupled_poke_others()
354 			cpuidle_coupled_poke(cpu);  in cpuidle_coupled_poke_others()
358  * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
359  * @cpu: target cpu
360  * @coupled: the struct coupled that contains the current cpu
361  * @next_state: the index in drv->states of the requested state for this cpu
366 static int cpuidle_coupled_set_waiting(int cpu,  in cpuidle_coupled_set_waiting()  argument
369 	coupled->requested_state[cpu] = next_state;  in cpuidle_coupled_set_waiting()
379  * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
380  * @cpu: target cpu
381  * @coupled: the struct coupled that contains the current cpu
385 static void cpuidle_coupled_set_not_waiting(int cpu,  in cpuidle_coupled_set_not_waiting()  argument
392 	 * notice that this cpu has cleared it's requested_state.  in cpuidle_coupled_set_not_waiting()
396 	coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;  in cpuidle_coupled_set_not_waiting()
400  * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
401  * @cpu: the current cpu
402  * @coupled: the struct coupled that contains the current cpu
404  * Marks this cpu as no longer in the ready and waiting loops.  Decrements
405  * the waiting count first to prevent another cpu looping back in and seeing
406  * this cpu as waiting just before it exits idle.
408 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)  in cpuidle_coupled_set_done()  argument
410 	cpuidle_coupled_set_not_waiting(cpu, coupled);  in cpuidle_coupled_set_done()
416  * @cpu: this cpu
423  * the interrupt didn't schedule work that should take the cpu out of idle.
427 static int cpuidle_coupled_clear_pokes(int cpu)  in cpuidle_coupled_clear_pokes()  argument
429 	if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))  in cpuidle_coupled_clear_pokes()
433 	while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))  in cpuidle_coupled_clear_pokes()
448  * @dev: struct cpuidle_device for the current cpu
458  * the second stage will start.  Each coupled cpu will spin until all cpus have
476 		cpuidle_coupled_clear_pokes(dev->cpu);  in cpuidle_enter_state_coupled()
490 	cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);  in cpuidle_enter_state_coupled()
492 	w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);  in cpuidle_enter_state_coupled()
494 	 * If this is the last cpu to enter the waiting state, poke  in cpuidle_enter_state_coupled()
501 		cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);  in cpuidle_enter_state_coupled()
502 		cpuidle_coupled_poke_others(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
508 	 * allowed for a single cpu.  If this was not the poking cpu, wait  in cpuidle_enter_state_coupled()
515 			!cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {  in cpuidle_enter_state_coupled()
516 		if (cpuidle_coupled_clear_pokes(dev->cpu))  in cpuidle_enter_state_coupled()
520 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
525 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
534 	cpuidle_coupled_clear_pokes(dev->cpu);  in cpuidle_enter_state_coupled()
536 		cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
541 	 * Make sure final poke status for this cpu is visible before setting  in cpuidle_enter_state_coupled()
542 	 * cpu as ready.  in cpuidle_enter_state_coupled()
550 	 * cpu has incremented the ready counter, it cannot abort idle and must  in cpuidle_enter_state_coupled()
552 	 * another cpu leaves idle and decrements the waiting counter.  in cpuidle_enter_state_coupled()
571 	 * There is a small chance that a cpu left and reentered idle after this  in cpuidle_enter_state_coupled()
572 	 * cpu saw that all cpus were waiting.  The cpu that reentered idle will  in cpuidle_enter_state_coupled()
573 	 * have sent this cpu a poke, which will still be pending after the  in cpuidle_enter_state_coupled()
581 		cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
592 	cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
597 	 * That leads to an inefficiency where a cpu receiving an interrupt  in cpuidle_enter_state_coupled()
600 	 * other cpus will need to spin waiting for the cpu that is processing  in cpuidle_enter_state_coupled()
613 	 * a cpu exits and re-enters the ready state because this cpu has  in cpuidle_enter_state_coupled()
629  * @dev: struct cpuidle_device for the current cpu
637 	int cpu;  in cpuidle_coupled_register_device()  local
645 	for_each_cpu(cpu, &dev->coupled_cpus) {  in cpuidle_coupled_register_device()
646 		other_dev = per_cpu(cpuidle_devices, cpu);  in cpuidle_coupled_register_device()
669 	csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);  in cpuidle_coupled_register_device()
670 	INIT_CSD(csd, cpuidle_coupled_handle_poke, (void *)(unsigned long)dev->cpu);  in cpuidle_coupled_register_device()
677  * @dev: struct cpuidle_device for the current cpu
680  * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
681  * this was the last cpu in the set.
697  * @coupled: the struct coupled that contains the cpu that is changing state
704 	int cpu = get_cpu();  in cpuidle_coupled_prevent_idle()  local
708 	cpuidle_coupled_poke_others(cpu, coupled);  in cpuidle_coupled_prevent_idle()
716  * @coupled: the struct coupled that contains the cpu that is changing state
723 	int cpu = get_cpu();  in cpuidle_coupled_allow_idle()  local
732 	cpuidle_coupled_poke_others(cpu, coupled);  in cpuidle_coupled_allow_idle()
736 static int coupled_cpu_online(unsigned int cpu)  in coupled_cpu_online()  argument
742 	dev = per_cpu(cpuidle_devices, cpu);  in coupled_cpu_online()
752 static int coupled_cpu_up_prepare(unsigned int cpu)  in coupled_cpu_up_prepare()  argument
758 	dev = per_cpu(cpuidle_devices, cpu);  in coupled_cpu_up_prepare()