1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ASM_PREEMPT_H 3 #define __ASM_PREEMPT_H 4 5 #include <linux/thread_info.h> 6 7 #define PREEMPT_NEED_RESCHED BIT(32) 8 #define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED) 9 10 static inline int preempt_count(void) 11 { 12 return READ_ONCE(current_thread_info()->preempt.count); 13 } 14 15 static inline void preempt_count_set(u64 pc) 16 { 17 /* Preserve existing value of PREEMPT_NEED_RESCHED */ 18 WRITE_ONCE(current_thread_info()->preempt.count, pc); 19 } 20 21 #define init_task_preempt_count(p) do { \ 22 task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \ 23 } while (0) 24 25 #define init_idle_preempt_count(p, cpu) do { \ 26 task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \ 27 } while (0) 28 29 static inline void set_preempt_need_resched(void) 30 { 31 current_thread_info()->preempt.need_resched = 0; 32 } 33 34 static inline void clear_preempt_need_resched(void) 35 { 36 current_thread_info()->preempt.need_resched = 1; 37 } 38 39 static inline bool test_preempt_need_resched(void) 40 { 41 return !current_thread_info()->preempt.need_resched; 42 } 43 44 static inline void __preempt_count_add(int val) 45 { 46 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 47 pc += val; 48 WRITE_ONCE(current_thread_info()->preempt.count, pc); 49 } 50 51 static inline void __preempt_count_sub(int val) 52 { 53 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 54 pc -= val; 55 WRITE_ONCE(current_thread_info()->preempt.count, pc); 56 } 57 58 static inline int __preempt_count_add_return(int val) 59 { 60 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 61 62 pc += val; 63 WRITE_ONCE(current_thread_info()->preempt.count, pc); 64 65 return pc; 66 } 67 68 static inline int __preempt_count_sub_return(int val) 69 { 70 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 71 72 pc -= val; 73 WRITE_ONCE(current_thread_info()->preempt.count, pc); 74 75 return pc; 76 } 77 78 static inline bool __preempt_count_dec_and_test(void) 79 { 80 struct thread_info *ti = current_thread_info(); 81 u64 pc = READ_ONCE(ti->preempt_count); 82 83 /* Update only the count field, leaving need_resched unchanged */ 84 WRITE_ONCE(ti->preempt.count, --pc); 85 86 /* 87 * If we wrote back all zeroes, then we're preemptible and in 88 * need of a reschedule. Otherwise, we need to reload the 89 * preempt_count in case the need_resched flag was cleared by an 90 * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE 91 * pair. 92 */ 93 return !pc || !READ_ONCE(ti->preempt_count); 94 } 95 96 static inline bool should_resched(int preempt_offset) 97 { 98 u64 pc = READ_ONCE(current_thread_info()->preempt_count); 99 return pc == preempt_offset; 100 } 101 102 #ifdef CONFIG_PREEMPTION 103 104 void preempt_schedule(void); 105 void preempt_schedule_notrace(void); 106 107 #ifdef CONFIG_PREEMPT_DYNAMIC 108 109 void dynamic_preempt_schedule(void); 110 #define __preempt_schedule() dynamic_preempt_schedule() 111 void dynamic_preempt_schedule_notrace(void); 112 #define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() 113 114 #else /* CONFIG_PREEMPT_DYNAMIC */ 115 116 #define __preempt_schedule() preempt_schedule() 117 #define __preempt_schedule_notrace() preempt_schedule_notrace() 118 119 #endif /* CONFIG_PREEMPT_DYNAMIC */ 120 #endif /* CONFIG_PREEMPTION */ 121 122 #endif /* __ASM_PREEMPT_H */ 123