1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ASM_PREEMPT_H 3 #define __ASM_PREEMPT_H 4 5 #include <linux/thread_info.h> 6 7 #define PREEMPT_NEED_RESCHED BIT(32) 8 #define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED) 9 10 static inline int preempt_count(void) 11 { 12 return READ_ONCE(current_thread_info()->preempt.count); 13 } 14 15 static inline void preempt_count_set(u64 pc) 16 { 17 /* Preserve existing value of PREEMPT_NEED_RESCHED */ 18 WRITE_ONCE(current_thread_info()->preempt.count, pc); 19 } 20 21 #define init_task_preempt_count(p) do { \ 22 task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \ 23 } while (0) 24 25 #define init_idle_preempt_count(p, cpu) do { \ 26 task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \ 27 } while (0) 28 29 static inline void set_preempt_need_resched(void) 30 { 31 current_thread_info()->preempt.need_resched = 0; 32 } 33 34 static inline void clear_preempt_need_resched(void) 35 { 36 current_thread_info()->preempt.need_resched = 1; 37 } 38 39 static inline bool test_preempt_need_resched(void) 40 { 41 return !current_thread_info()->preempt.need_resched; 42 } 43 44 static inline void __preempt_count_add(int val) 45 { 46 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 47 pc += val; 48 WRITE_ONCE(current_thread_info()->preempt.count, pc); 49 } 50 51 static inline void __preempt_count_sub(int val) 52 { 53 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 54 pc -= val; 55 WRITE_ONCE(current_thread_info()->preempt.count, pc); 56 } 57 58 static inline int __preempt_count_add_return(int val) 59 { 60 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 61 62 pc += val; 63 WRITE_ONCE(current_thread_info()->preempt.count, pc); 64 65 return pc; 66 } 67 68 static inline int __preempt_count_sub_return(int val) 69 { 70 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 71 72 pc -= val; 73 WRITE_ONCE(current_thread_info()->preempt.count, pc); 74 75 return pc; 76 } 77 78 static inline bool should_resched(int preempt_offset) 79 { 80 u64 pc = READ_ONCE(current_thread_info()->preempt_count); 81 return unlikely(pc == preempt_offset); 82 } 83 84 static inline bool __preempt_count_dec_and_test(void) 85 { 86 /* 87 * We must load the combined 'preempt_count' after decrementing 88 * 'preempt.count' as an interrupt could modify 'need_resched' before 89 * __preempt_count_sub() writes back to 'preempt.count'. 90 */ 91 __preempt_count_sub(1); 92 return should_resched(0); 93 } 94 95 #ifdef CONFIG_PREEMPTION 96 97 void preempt_schedule(void); 98 void preempt_schedule_notrace(void); 99 100 #define __preempt_schedule() preempt_schedule() 101 #define __preempt_schedule_notrace() preempt_schedule_notrace() 102 103 #endif /* CONFIG_PREEMPTION */ 104 105 #endif /* __ASM_PREEMPT_H */ 106