1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/irq-entry-common.h> 4 #include <linux/resume_user_mode.h> 5 #include <linux/highmem.h> 6 #include <linux/jump_label.h> 7 #include <linux/kmsan.h> 8 #include <linux/livepatch.h> 9 #include <linux/tick.h> 10 11 /* Workaround to allow gradual conversion of architecture code */ 12 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { } 13 14 #ifdef CONFIG_HAVE_GENERIC_TIF_BITS 15 #define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK & ~_TIF_RSEQ) 16 #else 17 #define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK) 18 #endif 19 20 /* TIF bits, which prevent a time slice extension. */ 21 #ifdef CONFIG_PREEMPT_RT 22 /* 23 * Since rseq slice ext has a direct correlation to the worst case 24 * scheduling latency (schedule is delayed after all), only have it affect 25 * LAZY reschedules on PREEMPT_RT for now. 26 * 27 * However, since this delay is only applicable to userspace, a value 28 * for rseq_slice_extension_nsec that is strictly less than the worst case 29 * kernel space preempt_disable() region, should mean the scheduling latency 30 * is not affected, even for !LAZY. 31 * 32 * However, since this value depends on the hardware at hand, it cannot be 33 * pre-determined in any sensible way. Hence punt on this problem for now. 34 */ 35 # define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED_LAZY) 36 #else 37 # define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) 38 #endif 39 #define TIF_SLICE_EXT_DENY (EXIT_TO_USER_MODE_WORK & ~TIF_SLICE_EXT_SCHED) 40 41 static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs, 42 unsigned long ti_work) 43 { 44 /* 45 * Before returning to user space ensure that all pending work 46 * items have been completed. 47 */ 48 while (ti_work & EXIT_TO_USER_MODE_WORK_LOOP) { 49 50 local_irq_enable(); 51 52 if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) { 53 if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY)) 54 schedule(); 55 } 56 57 if (ti_work & _TIF_UPROBE) 58 uprobe_notify_resume(regs); 59 60 if (ti_work & _TIF_PATCH_PENDING) 61 klp_update_patch_state(current); 62 63 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) 64 arch_do_signal_or_restart(regs); 65 66 if (ti_work & _TIF_NOTIFY_RESUME) 67 resume_user_mode_work(regs); 68 69 /* Architecture specific TIF work */ 70 arch_exit_to_user_mode_work(regs, ti_work); 71 72 /* 73 * Disable interrupts and reevaluate the work flags as they 74 * might have changed while interrupts and preemption was 75 * enabled above. 76 */ 77 local_irq_disable(); 78 79 /* Check if any of the above work has queued a deferred wakeup */ 80 tick_nohz_user_enter_prepare(); 81 82 ti_work = read_thread_flags(); 83 } 84 85 /* Return the latest work state for arch_exit_to_user_mode() */ 86 return ti_work; 87 } 88 89 /** 90 * exit_to_user_mode_loop - do any pending work before leaving to user space 91 * @regs: Pointer to pt_regs on entry stack 92 * @ti_work: TIF work flags as read by the caller 93 */ 94 __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 95 unsigned long ti_work) 96 { 97 for (;;) { 98 ti_work = __exit_to_user_mode_loop(regs, ti_work); 99 100 if (likely(!rseq_exit_to_user_mode_restart(regs, ti_work))) 101 return ti_work; 102 ti_work = read_thread_flags(); 103 } 104 } 105 106 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) 107 { 108 if (user_mode(regs)) { 109 irqentry_state_t ret = { 110 .exit_rcu = false, 111 }; 112 113 irqentry_enter_from_user_mode(regs); 114 return ret; 115 } 116 117 return irqentry_enter_from_kernel_mode(regs); 118 } 119 120 /** 121 * arch_irqentry_exit_need_resched - Architecture specific need resched function 122 * 123 * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed. 124 * Defaults return true. 125 * 126 * The main purpose is to permit arch to avoid preemption of a task from an IRQ. 127 */ 128 static inline bool arch_irqentry_exit_need_resched(void); 129 130 #ifndef arch_irqentry_exit_need_resched 131 static inline bool arch_irqentry_exit_need_resched(void) { return true; } 132 #endif 133 134 void raw_irqentry_exit_cond_resched(void) 135 { 136 if (!preempt_count()) { 137 /* Sanity check RCU and thread stack */ 138 rcu_irq_exit_check_preempt(); 139 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) 140 WARN_ON_ONCE(!on_thread_stack()); 141 if (need_resched() && arch_irqentry_exit_need_resched()) 142 preempt_schedule_irq(); 143 } 144 } 145 #ifdef CONFIG_PREEMPT_DYNAMIC 146 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 147 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 148 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 149 DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 150 void dynamic_irqentry_exit_cond_resched(void) 151 { 152 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) 153 return; 154 raw_irqentry_exit_cond_resched(); 155 } 156 #endif 157 #endif 158 159 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) 160 { 161 if (user_mode(regs)) 162 irqentry_exit_to_user_mode(regs); 163 else 164 irqentry_exit_to_kernel_mode(regs, state); 165 } 166 167 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) 168 { 169 irqentry_state_t irq_state; 170 171 irq_state.lockdep = lockdep_hardirqs_enabled(); 172 173 __nmi_enter(); 174 lockdep_hardirqs_off(CALLER_ADDR0); 175 lockdep_hardirq_enter(); 176 ct_nmi_enter(); 177 178 instrumentation_begin(); 179 kmsan_unpoison_entry_regs(regs); 180 trace_hardirqs_off_finish(); 181 ftrace_nmi_enter(); 182 instrumentation_end(); 183 184 return irq_state; 185 } 186 187 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) 188 { 189 instrumentation_begin(); 190 ftrace_nmi_exit(); 191 if (irq_state.lockdep) { 192 trace_hardirqs_on_prepare(); 193 lockdep_hardirqs_on_prepare(); 194 } 195 instrumentation_end(); 196 197 ct_nmi_exit(); 198 lockdep_hardirq_exit(); 199 if (irq_state.lockdep) 200 lockdep_hardirqs_on(CALLER_ADDR0); 201 __nmi_exit(); 202 } 203