1 // SPDX-License-Identifier: GPL-2.0
3  * This file contains functions which emulate a local clock-event
6  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
7  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
8  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
20 #include "tick-internal.h"
23  * Broadcast support for broken x86 hardware, where the local apic
24  * timer stops in C3 state.
89 	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||  in tick_check_broadcast_device()
90 	    (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||  in tick_check_broadcast_device()
91 	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_check_broadcast_device()
95 	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_check_broadcast_device()
98 	return !curdev || newdev->rating > curdev->rating;  in tick_check_broadcast_device()
124 	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||  in tick_set_oneshot_wakeup_device()
125 	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_set_oneshot_wakeup_device()
128 	if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) ||  in tick_set_oneshot_wakeup_device()
129 	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_set_oneshot_wakeup_device()
132 	if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu)))  in tick_set_oneshot_wakeup_device()
135 	if (curdev && newdev->rating <= curdev->rating)  in tick_set_oneshot_wakeup_device()
138 	if (!try_module_get(newdev->owner))  in tick_set_oneshot_wakeup_device()
141 	newdev->event_handler = tick_oneshot_wakeup_handler;  in tick_set_oneshot_wakeup_device()
173 	if (!try_module_get(dev->owner))  in tick_install_broadcast_device()
178 		cur->event_handler = clockevents_handle_noop;  in tick_install_broadcast_device()
183 	if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_install_broadcast_device()
216 	int ret = -ENODEV;  in tick_broadcast_update_freq()
229 	pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");  in err_broadcast()
234 	if (!dev->broadcast)  in tick_device_setup_broadcast_func()
235 		dev->broadcast = tick_broadcast;  in tick_device_setup_broadcast_func()
236 	if (!dev->broadcast) {  in tick_device_setup_broadcast_func()
238 			     dev->name);  in tick_device_setup_broadcast_func()
239 		dev->broadcast = err_broadcast;  in tick_device_setup_broadcast_func()
259 	 * the cpu local device.  in tick_device_uses_broadcast()
262 		dev->event_handler = tick_handle_periodic;  in tick_device_uses_broadcast()
275 		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_device_uses_broadcast()
294 			 * state affected device to stop. Let the  in tick_device_uses_broadcast()
317 			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))  in tick_device_uses_broadcast()
331 	struct clock_event_device *evt = td->evtdev;  in tick_receive_broadcast()
334 		return -ENODEV;  in tick_receive_broadcast()
336 	if (!evt->event_handler)  in tick_receive_broadcast()
337 		return -EINVAL;  in tick_receive_broadcast()
339 	evt->event_handler(evt);  in tick_receive_broadcast()
350 	bool local = false;  in tick_do_broadcast()  local
360 		 * We only run the local handler, if the broadcast  in tick_do_broadcast()
364 		 * local timer_interrupt()  in tick_do_broadcast()
371 		local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);  in tick_do_broadcast()
382 		td->evtdev->broadcast(mask);  in tick_do_broadcast()
384 	return local;  in tick_do_broadcast()
389  * - invoke the broadcast handlers
416 		ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC);  in tick_handle_periodic_broadcast()
423 	 * We run the handler of the local cpu after dropping  in tick_handle_periodic_broadcast()
428 		td->evtdev->event_handler(td->evtdev);  in tick_handle_periodic_broadcast()
432  * tick_broadcast_control - Enable/disable or force broadcast mode
436  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
445 	/* Protects also the local clockevent device. */  in tick_broadcast_control()
448 	dev = td->evtdev;  in tick_broadcast_control()
453 	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_broadcast_control()
471 			 * Only shutdown the cpu local device, if:  in tick_broadcast_control()
473 			 * - the broadcast device exists  in tick_broadcast_control()
474 			 * - the broadcast device is not a hrtimer based one  in tick_broadcast_control()
475 			 * - the broadcast device is in periodic mode to  in tick_broadcast_control()
478 			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&  in tick_broadcast_control()
518 		dev->event_handler = tick_handle_periodic;  in tick_set_periodic_handler()
520 		dev->event_handler = tick_handle_periodic_broadcast;  in tick_set_periodic_handler()
641 	if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))  in tick_broadcast_set_affinity()
644 	if (cpumask_equal(bc->cpumask, cpumask))  in tick_broadcast_set_affinity()
647 	bc->cpumask = cpumask;  in tick_broadcast_set_affinity()
648 	irq_set_affinity(bc->irq, bc->cpumask);  in tick_broadcast_set_affinity()
680 		if (td->mode == TICKDEV_MODE_ONESHOT) {  in tick_check_oneshot_broadcast_this_cpu()
681 			clockevents_switch_state(td->evtdev,  in tick_check_oneshot_broadcast_this_cpu()
698 	dev->next_event = KTIME_MAX;  in tick_handle_oneshot_broadcast()
713 		if (td->evtdev->next_event <= now) {  in tick_handle_oneshot_broadcast()
717 			 * it can avoid reprogramming the cpu local  in tick_handle_oneshot_broadcast()
718 			 * timer in tick_broadcast_oneshot_control().  in tick_handle_oneshot_broadcast()
721 		} else if (td->evtdev->next_event < next_event) {  in tick_handle_oneshot_broadcast()
722 			next_event = td->evtdev->next_event;  in tick_handle_oneshot_broadcast()
752 	 * - The global event did not expire any CPU local  in tick_handle_oneshot_broadcast()
756 	 * - There are pending events on sleeping CPUs which were not  in tick_handle_oneshot_broadcast()
766 		td->evtdev->event_handler(td->evtdev);  in tick_handle_oneshot_broadcast()
772 	if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))  in broadcast_needs_cpu()
774 	if (bc->next_event == KTIME_MAX)  in broadcast_needs_cpu()
776 	return bc->bound_on == cpu ? -EBUSY : 0;  in broadcast_needs_cpu()
784 	 * local device if our own event is the first one to expire or  in broadcast_shutdown_local()
785 	 * if we own the broadcast timer.  in broadcast_shutdown_local()
787 	if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {  in broadcast_shutdown_local()
790 		if (dev->next_event < bc->next_event)  in broadcast_shutdown_local()
800 	struct clock_event_device *bc, *dev = td->evtdev;  in ___tick_broadcast_oneshot_control()
812 		 * through the EXIT path as the local timer is not  in ___tick_broadcast_oneshot_control()
825 			if (bc->features & CLOCK_EVT_FEAT_HRTIMER)  in ___tick_broadcast_oneshot_control()
826 				ret = -EBUSY;  in ___tick_broadcast_oneshot_control()
833 			/* Conditionally shut down the local timer. */  in ___tick_broadcast_oneshot_control()
837 			 * We only reprogram the broadcast timer if we  in ___tick_broadcast_oneshot_control()
839 			 * if the cpu local event is earlier than the  in ___tick_broadcast_oneshot_control()
847 				ret = -EBUSY;  in ___tick_broadcast_oneshot_control()
848 			} else if (dev->next_event < bc->next_event) {  in ___tick_broadcast_oneshot_control()
849 				tick_broadcast_set_event(bc, cpu, dev->next_event);  in ___tick_broadcast_oneshot_control()
853 				 * timer to this cpu. If yes, remove  in ___tick_broadcast_oneshot_control()
869 			 * timer marked this cpu in the broadcast  in ___tick_broadcast_oneshot_control()
873 			 * handler. No need to reprogram the timer  in ___tick_broadcast_oneshot_control()
883 			if (dev->next_event == KTIME_MAX)  in ___tick_broadcast_oneshot_control()
891 			 * if the cpu local expiry time is already  in ___tick_broadcast_oneshot_control()
892 			 * reached, we would reprogram the cpu local  in ___tick_broadcast_oneshot_control()
893 			 * timer with an already expired event.  in ___tick_broadcast_oneshot_control()
895 			 * This can lead to a ping-pong when we return  in ___tick_broadcast_oneshot_control()
897 			 * timer before the cpu local timer was able  in ___tick_broadcast_oneshot_control()
902 			 * enough out that the ping-pong starts.  in ___tick_broadcast_oneshot_control()
904 			 * If the cpu local next_event has expired  in ___tick_broadcast_oneshot_control()
905 			 * then we know that the broadcast timer  in ___tick_broadcast_oneshot_control()
910 			 * will invoke the cpu local handler.  in ___tick_broadcast_oneshot_control()
918 			if (dev->next_event <= now) {  in ___tick_broadcast_oneshot_control()
924 			 * the cpu local timer device.  in ___tick_broadcast_oneshot_control()
926 			tick_program_event(dev->next_event, 1);  in ___tick_broadcast_oneshot_control()
940 	dev = td->evtdev;  in tick_oneshot_wakeup_control()
941 	if (td->mode != TICKDEV_MODE_ONESHOT)  in tick_oneshot_wakeup_control()
942 		return -EINVAL;  in tick_oneshot_wakeup_control()
946 		return -ENODEV;  in tick_oneshot_wakeup_control()
952 		clockevents_program_event(wd, dev->next_event, 1);  in tick_oneshot_wakeup_control()
957 			return -ENODEV;  in tick_oneshot_wakeup_control()
978 	return -EBUSY;  in __tick_broadcast_oneshot_control()
1000 		if (td->evtdev)  in tick_broadcast_init_next_event()
1001 			td->evtdev->next_event = expires;  in tick_broadcast_init_next_event()
1022  * tick_broadcast_setup_oneshot - setup the broadcast device
1036 	 * code via hrtimer_run_queues() -> tick_check_oneshot_change()  in tick_broadcast_setup_oneshot()
1039 	if (bc->event_handler == tick_handle_oneshot_broadcast) {  in tick_broadcast_setup_oneshot()
1047 		 * Clear the bit for the local CPU as the set bit would  in tick_broadcast_setup_oneshot()
1063 	bc->event_handler = tick_handle_oneshot_broadcast;  in tick_broadcast_setup_oneshot()
1064 	bc->next_event = KTIME_MAX;  in tick_broadcast_setup_oneshot()
1069 	 * get their wake-up at the next tick.  This is achieved by ORing  in tick_broadcast_setup_oneshot()
1080 		/* Remove the local CPU as it is obviously not idle */  in tick_broadcast_setup_oneshot()
1107 	 * immediately in order to reevaluate the next expiring timer.  in tick_broadcast_setup_oneshot()
1153 		 * then the current CPU has not yet reprogrammed the local  in hotplug_cpu__broadcast_tick_pull()
1154 		 * timer device to avoid a ping-pong race. See  in hotplug_cpu__broadcast_tick_pull()
1159 		 * effect because the local clockevent device is not  in hotplug_cpu__broadcast_tick_pull()
1161 		 * is not earlier than the pending event of the local clock  in hotplug_cpu__broadcast_tick_pull()
1165 		 * Detect this condition and reprogram the cpu local timer  in hotplug_cpu__broadcast_tick_pull()
1172 			tick_program_event(td->evtdev->next_event, 1);  in hotplug_cpu__broadcast_tick_pull()
1176 		clockevents_program_event(bc, bc->next_event, 1);  in hotplug_cpu__broadcast_tick_pull()
1190 	 * Clear the broadcast masks for the dead cpu, but do not stop  in tick_broadcast_oneshot_offline()
1214 	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;  in tick_broadcast_oneshot_available()
1222 	if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))  in __tick_broadcast_oneshot_control()
1223 		return -EBUSY;  in __tick_broadcast_oneshot_control()