1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef _LINUX_HRTIMER_REARM_H 3 #define _LINUX_HRTIMER_REARM_H 4 5 #include <linux/types.h> 6 7 #ifdef CONFIG_HRTIMER_REARM_DEFERRED 8 #include <linux/irqflags.h> 9 #include <linux/lockdep.h> 10 #include <linux/preempt.h> 11 #include <linux/thread_info.h> 12 13 void __hrtimer_rearm_deferred(void); 14 15 /* 16 * This is purely CPU local, so check the TIF bit first to avoid the overhead of 17 * the atomic test_and_clear_bit() operation for the common case where the bit 18 * is not set. 19 */ hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)20static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work) 21 { 22 lockdep_assert_irqs_disabled(); 23 24 if (unlikely(tif_work & _TIF_HRTIMER_REARM)) { 25 clear_thread_flag(TIF_HRTIMER_REARM); 26 return true; 27 } 28 return false; 29 } 30 31 #define TIF_REARM_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM) 32 33 /* Invoked from the exit to user before invoking exit_to_user_mode_loop() */ 34 static __always_inline bool hrtimer_rearm_deferred_user_irq(unsigned long * tif_work,const unsigned long tif_mask)35hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) 36 { 37 /* Help the compiler to optimize the function out for syscall returns */ 38 if (!(tif_mask & _TIF_HRTIMER_REARM)) 39 return false; 40 /* 41 * Rearm the timer if none of the resched flags is set before going into 42 * the loop which re-enables interrupts. 43 */ 44 if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) { 45 clear_thread_flag(TIF_HRTIMER_REARM); 46 __hrtimer_rearm_deferred(); 47 /* Don't go into the loop if HRTIMER_REARM was the only flag */ 48 *tif_work &= ~_TIF_HRTIMER_REARM; 49 return !*tif_work; 50 } 51 return false; 52 } 53 54 /* Invoked from the time slice extension decision function */ hrtimer_rearm_deferred_tif(unsigned long tif_work)55static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) 56 { 57 if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work)) 58 __hrtimer_rearm_deferred(); 59 } 60 61 /* 62 * This is to be called on all irqentry_exit() paths that will enable 63 * interrupts. 64 */ hrtimer_rearm_deferred(void)65static __always_inline void hrtimer_rearm_deferred(void) 66 { 67 hrtimer_rearm_deferred_tif(read_thread_flags()); 68 } 69 70 /* 71 * Invoked from the scheduler on entry to __schedule() so it can defer 72 * rearming after the load balancing callbacks which might change hrtick. 73 */ hrtimer_test_and_clear_rearm_deferred(void)74static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) 75 { 76 return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags()); 77 } 78 79 #else /* CONFIG_HRTIMER_REARM_DEFERRED */ __hrtimer_rearm_deferred(void)80static __always_inline void __hrtimer_rearm_deferred(void) { } hrtimer_rearm_deferred(void)81static __always_inline void hrtimer_rearm_deferred(void) { } hrtimer_rearm_deferred_tif(unsigned long tif_work)82static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { } 83 static __always_inline bool hrtimer_rearm_deferred_user_irq(unsigned long * tif_work,const unsigned long tif_mask)84hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; } hrtimer_test_and_clear_rearm_deferred(void)85static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; } 86 #endif /* !CONFIG_HRTIMER_REARM_DEFERRED */ 87 88 #endif 89