1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_IRQENTRYCOMMON_H 3 #define __LINUX_IRQENTRYCOMMON_H 4 5 #include <linux/static_call_types.h> 6 #include <linux/syscalls.h> 7 #include <linux/context_tracking.h> 8 #include <linux/tick.h> 9 #include <linux/kmsan.h> 10 11 #include <asm/entry-common.h> 12 13 /* 14 * Define dummy _TIF work flags if not defined by the architecture or for 15 * disabled functionality. 16 */ 17 #ifndef _TIF_PATCH_PENDING 18 # define _TIF_PATCH_PENDING (0) 19 #endif 20 21 /* 22 * TIF flags handled in exit_to_user_mode_loop() 23 */ 24 #ifndef ARCH_EXIT_TO_USER_MODE_WORK 25 # define ARCH_EXIT_TO_USER_MODE_WORK (0) 26 #endif 27 28 #define EXIT_TO_USER_MODE_WORK \ 29 (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ 30 _TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | \ 31 _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ 32 ARCH_EXIT_TO_USER_MODE_WORK) 33 34 /** 35 * arch_enter_from_user_mode - Architecture specific sanity check for user mode regs 36 * @regs: Pointer to currents pt_regs 37 * 38 * Defaults to an empty implementation. Can be replaced by architecture 39 * specific code. 40 * 41 * Invoked from syscall_enter_from_user_mode() in the non-instrumentable 42 * section. Use __always_inline so the compiler cannot push it out of line 43 * and make it instrumentable. 44 */ 45 static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs); 46 47 #ifndef arch_enter_from_user_mode 48 static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs) {} 49 #endif 50 51 /** 52 * arch_in_rcu_eqs - Architecture specific check for RCU extended quiescent 53 * states. 54 * 55 * Returns: true if the CPU is potentially in an RCU EQS, false otherwise. 56 * 57 * Architectures only need to define this if threads other than the idle thread 58 * may have an interruptible EQS. This does not need to handle idle threads. It 59 * is safe to over-estimate at the cost of redundant RCU management work. 60 * 61 * Invoked from irqentry_enter() 62 */ 63 #ifndef arch_in_rcu_eqs 64 static __always_inline bool arch_in_rcu_eqs(void) { return false; } 65 #endif 66 67 /** 68 * enter_from_user_mode - Establish state when coming from user mode 69 * 70 * Syscall/interrupt entry disables interrupts, but user mode is traced as 71 * interrupts enabled. Also with NO_HZ_FULL RCU might be idle. 72 * 73 * 1) Tell lockdep that interrupts are disabled 74 * 2) Invoke context tracking if enabled to reactivate RCU 75 * 3) Trace interrupts off state 76 * 77 * Invoked from architecture specific syscall entry code with interrupts 78 * disabled. The calling code has to be non-instrumentable. When the 79 * function returns all state is correct and interrupts are still 80 * disabled. The subsequent functions can be instrumented. 81 * 82 * This is invoked when there is architecture specific functionality to be 83 * done between establishing state and enabling interrupts. The caller must 84 * enable interrupts before invoking syscall_enter_from_user_mode_work(). 85 */ 86 static __always_inline void enter_from_user_mode(struct pt_regs *regs) 87 { 88 arch_enter_from_user_mode(regs); 89 lockdep_hardirqs_off(CALLER_ADDR0); 90 91 CT_WARN_ON(__ct_state() != CT_STATE_USER); 92 user_exit_irqoff(); 93 94 instrumentation_begin(); 95 kmsan_unpoison_entry_regs(regs); 96 trace_hardirqs_off_finish(); 97 instrumentation_end(); 98 } 99 100 /** 101 * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() 102 * @ti_work: Cached TIF flags gathered with interrupts disabled 103 * 104 * Defaults to local_irq_enable(). Can be supplied by architecture specific 105 * code. 106 */ 107 static inline void local_irq_enable_exit_to_user(unsigned long ti_work); 108 109 #ifndef local_irq_enable_exit_to_user 110 static inline void local_irq_enable_exit_to_user(unsigned long ti_work) 111 { 112 local_irq_enable(); 113 } 114 #endif 115 116 /** 117 * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() 118 * 119 * Defaults to local_irq_disable(). Can be supplied by architecture specific 120 * code. 121 */ 122 static inline void local_irq_disable_exit_to_user(void); 123 124 #ifndef local_irq_disable_exit_to_user 125 static inline void local_irq_disable_exit_to_user(void) 126 { 127 local_irq_disable(); 128 } 129 #endif 130 131 /** 132 * arch_exit_to_user_mode_work - Architecture specific TIF work for exit 133 * to user mode. 134 * @regs: Pointer to currents pt_regs 135 * @ti_work: Cached TIF flags gathered with interrupts disabled 136 * 137 * Invoked from exit_to_user_mode_loop() with interrupt enabled 138 * 139 * Defaults to NOOP. Can be supplied by architecture specific code. 140 */ 141 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 142 unsigned long ti_work); 143 144 #ifndef arch_exit_to_user_mode_work 145 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 146 unsigned long ti_work) 147 { 148 } 149 #endif 150 151 /** 152 * arch_exit_to_user_mode_prepare - Architecture specific preparation for 153 * exit to user mode. 154 * @regs: Pointer to currents pt_regs 155 * @ti_work: Cached TIF flags gathered with interrupts disabled 156 * 157 * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last 158 * function before return. Defaults to NOOP. 159 */ 160 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 161 unsigned long ti_work); 162 163 #ifndef arch_exit_to_user_mode_prepare 164 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 165 unsigned long ti_work) 166 { 167 } 168 #endif 169 170 /** 171 * arch_exit_to_user_mode - Architecture specific final work before 172 * exit to user mode. 173 * 174 * Invoked from exit_to_user_mode() with interrupt disabled as the last 175 * function before return. Defaults to NOOP. 176 * 177 * This needs to be __always_inline because it is non-instrumentable code 178 * invoked after context tracking switched to user mode. 179 * 180 * An architecture implementation must not do anything complex, no locking 181 * etc. The main purpose is for speculation mitigations. 182 */ 183 static __always_inline void arch_exit_to_user_mode(void); 184 185 #ifndef arch_exit_to_user_mode 186 static __always_inline void arch_exit_to_user_mode(void) { } 187 #endif 188 189 /** 190 * arch_do_signal_or_restart - Architecture specific signal delivery function 191 * @regs: Pointer to currents pt_regs 192 * 193 * Invoked from exit_to_user_mode_loop(). 194 */ 195 void arch_do_signal_or_restart(struct pt_regs *regs); 196 197 /** 198 * exit_to_user_mode_loop - do any pending work before leaving to user space 199 */ 200 unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 201 unsigned long ti_work); 202 203 /** 204 * exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required 205 * @regs: Pointer to pt_regs on entry stack 206 * 207 * 1) check that interrupts are disabled 208 * 2) call tick_nohz_user_enter_prepare() 209 * 3) call exit_to_user_mode_loop() if any flags from 210 * EXIT_TO_USER_MODE_WORK are set 211 * 4) check that interrupts are still disabled 212 */ 213 static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) 214 { 215 unsigned long ti_work; 216 217 lockdep_assert_irqs_disabled(); 218 219 /* Flush pending rcuog wakeup before the last need_resched() check */ 220 tick_nohz_user_enter_prepare(); 221 222 ti_work = read_thread_flags(); 223 if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) 224 ti_work = exit_to_user_mode_loop(regs, ti_work); 225 226 arch_exit_to_user_mode_prepare(regs, ti_work); 227 228 /* Ensure that kernel state is sane for a return to userspace */ 229 kmap_assert_nomap(); 230 lockdep_assert_irqs_disabled(); 231 lockdep_sys_exit(); 232 } 233 234 /** 235 * exit_to_user_mode - Fixup state when exiting to user mode 236 * 237 * Syscall/interrupt exit enables interrupts, but the kernel state is 238 * interrupts disabled when this is invoked. Also tell RCU about it. 239 * 240 * 1) Trace interrupts on state 241 * 2) Invoke context tracking if enabled to adjust RCU state 242 * 3) Invoke architecture specific last minute exit code, e.g. speculation 243 * mitigations, etc.: arch_exit_to_user_mode() 244 * 4) Tell lockdep that interrupts are enabled 245 * 246 * Invoked from architecture specific code when syscall_exit_to_user_mode() 247 * is not suitable as the last step before returning to userspace. Must be 248 * invoked with interrupts disabled and the caller must be 249 * non-instrumentable. 250 * The caller has to invoke syscall_exit_to_user_mode_work() before this. 251 */ 252 static __always_inline void exit_to_user_mode(void) 253 { 254 instrumentation_begin(); 255 trace_hardirqs_on_prepare(); 256 lockdep_hardirqs_on_prepare(); 257 instrumentation_end(); 258 259 user_enter_irqoff(); 260 arch_exit_to_user_mode(); 261 lockdep_hardirqs_on(CALLER_ADDR0); 262 } 263 264 /** 265 * irqentry_enter_from_user_mode - Establish state before invoking the irq handler 266 * @regs: Pointer to currents pt_regs 267 * 268 * Invoked from architecture specific entry code with interrupts disabled. 269 * Can only be called when the interrupt entry came from user mode. The 270 * calling code must be non-instrumentable. When the function returns all 271 * state is correct and the subsequent functions can be instrumented. 272 * 273 * The function establishes state (lockdep, RCU (context tracking), tracing) 274 */ 275 void irqentry_enter_from_user_mode(struct pt_regs *regs); 276 277 /** 278 * irqentry_exit_to_user_mode - Interrupt exit work 279 * @regs: Pointer to current's pt_regs 280 * 281 * Invoked with interrupts disabled and fully valid regs. Returns with all 282 * work handled, interrupts disabled such that the caller can immediately 283 * switch to user mode. Called from architecture specific interrupt 284 * handling code. 285 * 286 * The call order is #2 and #3 as described in syscall_exit_to_user_mode(). 287 * Interrupt exit is not invoking #1 which is the syscall specific one time 288 * work. 289 */ 290 void irqentry_exit_to_user_mode(struct pt_regs *regs); 291 292 #ifndef irqentry_state 293 /** 294 * struct irqentry_state - Opaque object for exception state storage 295 * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the 296 * exit path has to invoke ct_irq_exit(). 297 * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that 298 * lockdep state is restored correctly on exit from nmi. 299 * 300 * This opaque object is filled in by the irqentry_*_enter() functions and 301 * must be passed back into the corresponding irqentry_*_exit() functions 302 * when the exception is complete. 303 * 304 * Callers of irqentry_*_[enter|exit]() must consider this structure opaque 305 * and all members private. Descriptions of the members are provided to aid in 306 * the maintenance of the irqentry_*() functions. 307 */ 308 typedef struct irqentry_state { 309 union { 310 bool exit_rcu; 311 bool lockdep; 312 }; 313 } irqentry_state_t; 314 #endif 315 316 /** 317 * irqentry_enter - Handle state tracking on ordinary interrupt entries 318 * @regs: Pointer to pt_regs of interrupted context 319 * 320 * Invokes: 321 * - lockdep irqflag state tracking as low level ASM entry disabled 322 * interrupts. 323 * 324 * - Context tracking if the exception hit user mode. 325 * 326 * - The hardirq tracer to keep the state consistent as low level ASM 327 * entry disabled interrupts. 328 * 329 * As a precondition, this requires that the entry came from user mode, 330 * idle, or a kernel context in which RCU is watching. 331 * 332 * For kernel mode entries RCU handling is done conditional. If RCU is 333 * watching then the only RCU requirement is to check whether the tick has 334 * to be restarted. If RCU is not watching then ct_irq_enter() has to be 335 * invoked on entry and ct_irq_exit() on exit. 336 * 337 * Avoiding the ct_irq_enter/exit() calls is an optimization but also 338 * solves the problem of kernel mode pagefaults which can schedule, which 339 * is not possible after invoking ct_irq_enter() without undoing it. 340 * 341 * For user mode entries irqentry_enter_from_user_mode() is invoked to 342 * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit 343 * would not be possible. 344 * 345 * Returns: An opaque object that must be passed to idtentry_exit() 346 */ 347 irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); 348 349 /** 350 * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt 351 * 352 * Conditional reschedule with additional sanity checks. 353 */ 354 void raw_irqentry_exit_cond_resched(void); 355 #ifdef CONFIG_PREEMPT_DYNAMIC 356 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 357 #define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched 358 #define irqentry_exit_cond_resched_dynamic_disabled NULL 359 DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 360 #define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() 361 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 362 DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 363 void dynamic_irqentry_exit_cond_resched(void); 364 #define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() 365 #endif 366 #else /* CONFIG_PREEMPT_DYNAMIC */ 367 #define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() 368 #endif /* CONFIG_PREEMPT_DYNAMIC */ 369 370 /** 371 * irqentry_exit - Handle return from exception that used irqentry_enter() 372 * @regs: Pointer to pt_regs (exception entry regs) 373 * @state: Return value from matching call to irqentry_enter() 374 * 375 * Depending on the return target (kernel/user) this runs the necessary 376 * preemption and work checks if possible and required and returns to 377 * the caller with interrupts disabled and no further work pending. 378 * 379 * This is the last action before returning to the low level ASM code which 380 * just needs to return to the appropriate context. 381 * 382 * Counterpart to irqentry_enter(). 383 */ 384 void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); 385 386 /** 387 * irqentry_nmi_enter - Handle NMI entry 388 * @regs: Pointer to currents pt_regs 389 * 390 * Similar to irqentry_enter() but taking care of the NMI constraints. 391 */ 392 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs); 393 394 /** 395 * irqentry_nmi_exit - Handle return from NMI handling 396 * @regs: Pointer to pt_regs (NMI entry regs) 397 * @irq_state: Return value from matching call to irqentry_nmi_enter() 398 * 399 * Last action before returning to the low level assembly code. 400 * 401 * Counterpart to irqentry_nmi_enter(). 402 */ 403 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state); 404 405 #endif 406