1142781e1SThomas Gleixner /* SPDX-License-Identifier: GPL-2.0 */ 2142781e1SThomas Gleixner #ifndef __LINUX_ENTRYCOMMON_H 3142781e1SThomas Gleixner #define __LINUX_ENTRYCOMMON_H 4142781e1SThomas Gleixner 5142781e1SThomas Gleixner #include <linux/tracehook.h> 6142781e1SThomas Gleixner #include <linux/syscalls.h> 7142781e1SThomas Gleixner #include <linux/seccomp.h> 8142781e1SThomas Gleixner #include <linux/sched.h> 9142781e1SThomas Gleixner 10142781e1SThomas Gleixner #include <asm/entry-common.h> 11142781e1SThomas Gleixner 12142781e1SThomas Gleixner /* 13142781e1SThomas Gleixner * Define dummy _TIF work flags if not defined by the architecture or for 14142781e1SThomas Gleixner * disabled functionality. 15142781e1SThomas Gleixner */ 16142781e1SThomas Gleixner #ifndef _TIF_SYSCALL_EMU 17142781e1SThomas Gleixner # define _TIF_SYSCALL_EMU (0) 18142781e1SThomas Gleixner #endif 19142781e1SThomas Gleixner 20142781e1SThomas Gleixner #ifndef _TIF_SYSCALL_TRACEPOINT 21142781e1SThomas Gleixner # define _TIF_SYSCALL_TRACEPOINT (0) 22142781e1SThomas Gleixner #endif 23142781e1SThomas Gleixner 24142781e1SThomas Gleixner #ifndef _TIF_SECCOMP 25142781e1SThomas Gleixner # define _TIF_SECCOMP (0) 26142781e1SThomas Gleixner #endif 27142781e1SThomas Gleixner 28142781e1SThomas Gleixner #ifndef _TIF_SYSCALL_AUDIT 29142781e1SThomas Gleixner # define _TIF_SYSCALL_AUDIT (0) 30142781e1SThomas Gleixner #endif 31142781e1SThomas Gleixner 32a9f3a74aSThomas Gleixner #ifndef _TIF_PATCH_PENDING 33a9f3a74aSThomas Gleixner # define _TIF_PATCH_PENDING (0) 34a9f3a74aSThomas Gleixner #endif 35a9f3a74aSThomas Gleixner 36a9f3a74aSThomas Gleixner #ifndef _TIF_UPROBE 37a9f3a74aSThomas Gleixner # define _TIF_UPROBE (0) 38a9f3a74aSThomas Gleixner #endif 39a9f3a74aSThomas Gleixner 4012db8b69SJens Axboe #ifndef _TIF_NOTIFY_SIGNAL 4112db8b69SJens Axboe # define _TIF_NOTIFY_SIGNAL (0) 4212db8b69SJens Axboe #endif 4312db8b69SJens Axboe 44142781e1SThomas Gleixner /* 45900ffe39SKees Cook * TIF flags handled in syscall_enter_from_user_mode() 46142781e1SThomas Gleixner */ 47142781e1SThomas Gleixner #ifndef ARCH_SYSCALL_ENTER_WORK 48142781e1SThomas Gleixner # define ARCH_SYSCALL_ENTER_WORK (0) 49142781e1SThomas Gleixner #endif 50142781e1SThomas Gleixner 51142781e1SThomas Gleixner #define SYSCALL_ENTER_WORK \ 52142781e1SThomas Gleixner (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 53142781e1SThomas Gleixner _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_EMU | \ 54142781e1SThomas Gleixner ARCH_SYSCALL_ENTER_WORK) 55142781e1SThomas Gleixner 56a9f3a74aSThomas Gleixner /* 57a9f3a74aSThomas Gleixner * TIF flags handled in syscall_exit_to_user_mode() 58a9f3a74aSThomas Gleixner */ 59a9f3a74aSThomas Gleixner #ifndef ARCH_SYSCALL_EXIT_WORK 60a9f3a74aSThomas Gleixner # define ARCH_SYSCALL_EXIT_WORK (0) 61a9f3a74aSThomas Gleixner #endif 62a9f3a74aSThomas Gleixner 63a9f3a74aSThomas Gleixner #define SYSCALL_EXIT_WORK \ 64a9f3a74aSThomas Gleixner (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ 65a9f3a74aSThomas Gleixner _TIF_SYSCALL_TRACEPOINT | ARCH_SYSCALL_EXIT_WORK) 66a9f3a74aSThomas Gleixner 67a9f3a74aSThomas Gleixner /* 68a9f3a74aSThomas Gleixner * TIF flags handled in exit_to_user_mode_loop() 69a9f3a74aSThomas Gleixner */ 70a9f3a74aSThomas Gleixner #ifndef ARCH_EXIT_TO_USER_MODE_WORK 71a9f3a74aSThomas Gleixner # define ARCH_EXIT_TO_USER_MODE_WORK (0) 72a9f3a74aSThomas Gleixner #endif 73a9f3a74aSThomas Gleixner 74a9f3a74aSThomas Gleixner #define EXIT_TO_USER_MODE_WORK \ 75a9f3a74aSThomas Gleixner (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ 7612db8b69SJens Axboe _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ 77a9f3a74aSThomas Gleixner ARCH_EXIT_TO_USER_MODE_WORK) 78a9f3a74aSThomas Gleixner 79142781e1SThomas Gleixner /** 80142781e1SThomas Gleixner * arch_check_user_regs - Architecture specific sanity check for user mode regs 81142781e1SThomas Gleixner * @regs: Pointer to currents pt_regs 82142781e1SThomas Gleixner * 83142781e1SThomas Gleixner * Defaults to an empty implementation. Can be replaced by architecture 84142781e1SThomas Gleixner * specific code. 85142781e1SThomas Gleixner * 86142781e1SThomas Gleixner * Invoked from syscall_enter_from_user_mode() in the non-instrumentable 87142781e1SThomas Gleixner * section. Use __always_inline so the compiler cannot push it out of line 88142781e1SThomas Gleixner * and make it instrumentable. 89142781e1SThomas Gleixner */ 90142781e1SThomas Gleixner static __always_inline void arch_check_user_regs(struct pt_regs *regs); 91142781e1SThomas Gleixner 92142781e1SThomas Gleixner #ifndef arch_check_user_regs 93142781e1SThomas Gleixner static __always_inline void arch_check_user_regs(struct pt_regs *regs) {} 94142781e1SThomas Gleixner #endif 95142781e1SThomas Gleixner 96142781e1SThomas Gleixner /** 97142781e1SThomas Gleixner * arch_syscall_enter_tracehook - Wrapper around tracehook_report_syscall_entry() 98142781e1SThomas Gleixner * @regs: Pointer to currents pt_regs 99142781e1SThomas Gleixner * 100142781e1SThomas Gleixner * Returns: 0 on success or an error code to skip the syscall. 101142781e1SThomas Gleixner * 102142781e1SThomas Gleixner * Defaults to tracehook_report_syscall_entry(). Can be replaced by 103142781e1SThomas Gleixner * architecture specific code. 104142781e1SThomas Gleixner * 105142781e1SThomas Gleixner * Invoked from syscall_enter_from_user_mode() 106142781e1SThomas Gleixner */ 107142781e1SThomas Gleixner static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs); 108142781e1SThomas Gleixner 109142781e1SThomas Gleixner #ifndef arch_syscall_enter_tracehook 110142781e1SThomas Gleixner static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs) 111142781e1SThomas Gleixner { 112142781e1SThomas Gleixner return tracehook_report_syscall_entry(regs); 113142781e1SThomas Gleixner } 114142781e1SThomas Gleixner #endif 115142781e1SThomas Gleixner 116142781e1SThomas Gleixner /** 1174facb95bSThomas Gleixner * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts 1184facb95bSThomas Gleixner * @regs: Pointer to currents pt_regs 1194facb95bSThomas Gleixner * 1204facb95bSThomas Gleixner * Invoked from architecture specific syscall entry code with interrupts 1214facb95bSThomas Gleixner * disabled. The calling code has to be non-instrumentable. When the 1224facb95bSThomas Gleixner * function returns all state is correct, interrupts are enabled and the 1234facb95bSThomas Gleixner * subsequent functions can be instrumented. 1244facb95bSThomas Gleixner * 1254facb95bSThomas Gleixner * This handles lockdep, RCU (context tracking) and tracing state. 1264facb95bSThomas Gleixner * 1274facb95bSThomas Gleixner * This is invoked when there is extra architecture specific functionality 1284facb95bSThomas Gleixner * to be done between establishing state and handling user mode entry work. 1294facb95bSThomas Gleixner */ 1304facb95bSThomas Gleixner void syscall_enter_from_user_mode_prepare(struct pt_regs *regs); 1314facb95bSThomas Gleixner 1324facb95bSThomas Gleixner /** 1334facb95bSThomas Gleixner * syscall_enter_from_user_mode_work - Check and handle work before invoking 134142781e1SThomas Gleixner * a syscall 135142781e1SThomas Gleixner * @regs: Pointer to currents pt_regs 136142781e1SThomas Gleixner * @syscall: The syscall number 137142781e1SThomas Gleixner * 138142781e1SThomas Gleixner * Invoked from architecture specific syscall entry code with interrupts 1394facb95bSThomas Gleixner * enabled after invoking syscall_enter_from_user_mode_prepare() and extra 1404facb95bSThomas Gleixner * architecture specific work. 141142781e1SThomas Gleixner * 142142781e1SThomas Gleixner * Returns: The original or a modified syscall number 143142781e1SThomas Gleixner * 144142781e1SThomas Gleixner * If the returned syscall number is -1 then the syscall should be 145142781e1SThomas Gleixner * skipped. In this case the caller may invoke syscall_set_error() or 146142781e1SThomas Gleixner * syscall_set_return_value() first. If neither of those are called and -1 147142781e1SThomas Gleixner * is returned, then the syscall will fail with ENOSYS. 148142781e1SThomas Gleixner * 1494facb95bSThomas Gleixner * It handles the following work items: 150142781e1SThomas Gleixner * 1514facb95bSThomas Gleixner * 1) TIF flag dependent invocations of arch_syscall_enter_tracehook(), 152142781e1SThomas Gleixner * __secure_computing(), trace_sys_enter() 1534facb95bSThomas Gleixner * 2) Invocation of audit_syscall_entry() 1544facb95bSThomas Gleixner */ 1554facb95bSThomas Gleixner long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall); 1564facb95bSThomas Gleixner 1574facb95bSThomas Gleixner /** 1584facb95bSThomas Gleixner * syscall_enter_from_user_mode - Establish state and check and handle work 1594facb95bSThomas Gleixner * before invoking a syscall 1604facb95bSThomas Gleixner * @regs: Pointer to currents pt_regs 1614facb95bSThomas Gleixner * @syscall: The syscall number 1624facb95bSThomas Gleixner * 1634facb95bSThomas Gleixner * Invoked from architecture specific syscall entry code with interrupts 1644facb95bSThomas Gleixner * disabled. The calling code has to be non-instrumentable. When the 1654facb95bSThomas Gleixner * function returns all state is correct, interrupts are enabled and the 1664facb95bSThomas Gleixner * subsequent functions can be instrumented. 1674facb95bSThomas Gleixner * 1684facb95bSThomas Gleixner * This is combination of syscall_enter_from_user_mode_prepare() and 1694facb95bSThomas Gleixner * syscall_enter_from_user_mode_work(). 1704facb95bSThomas Gleixner * 1714facb95bSThomas Gleixner * Returns: The original or a modified syscall number. See 1724facb95bSThomas Gleixner * syscall_enter_from_user_mode_work() for further explanation. 173142781e1SThomas Gleixner */ 174142781e1SThomas Gleixner long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall); 175142781e1SThomas Gleixner 176142781e1SThomas Gleixner /** 177a9f3a74aSThomas Gleixner * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() 178a9f3a74aSThomas Gleixner * @ti_work: Cached TIF flags gathered with interrupts disabled 179a9f3a74aSThomas Gleixner * 180a9f3a74aSThomas Gleixner * Defaults to local_irq_enable(). Can be supplied by architecture specific 181a9f3a74aSThomas Gleixner * code. 182a9f3a74aSThomas Gleixner */ 183a9f3a74aSThomas Gleixner static inline void local_irq_enable_exit_to_user(unsigned long ti_work); 184a9f3a74aSThomas Gleixner 185a9f3a74aSThomas Gleixner #ifndef local_irq_enable_exit_to_user 186a9f3a74aSThomas Gleixner static inline void local_irq_enable_exit_to_user(unsigned long ti_work) 187a9f3a74aSThomas Gleixner { 188a9f3a74aSThomas Gleixner local_irq_enable(); 189a9f3a74aSThomas Gleixner } 190a9f3a74aSThomas Gleixner #endif 191a9f3a74aSThomas Gleixner 192a9f3a74aSThomas Gleixner /** 193a9f3a74aSThomas Gleixner * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() 194a9f3a74aSThomas Gleixner * 195a9f3a74aSThomas Gleixner * Defaults to local_irq_disable(). Can be supplied by architecture specific 196a9f3a74aSThomas Gleixner * code. 197a9f3a74aSThomas Gleixner */ 198a9f3a74aSThomas Gleixner static inline void local_irq_disable_exit_to_user(void); 199a9f3a74aSThomas Gleixner 200a9f3a74aSThomas Gleixner #ifndef local_irq_disable_exit_to_user 201a9f3a74aSThomas Gleixner static inline void local_irq_disable_exit_to_user(void) 202a9f3a74aSThomas Gleixner { 203a9f3a74aSThomas Gleixner local_irq_disable(); 204a9f3a74aSThomas Gleixner } 205a9f3a74aSThomas Gleixner #endif 206a9f3a74aSThomas Gleixner 207a9f3a74aSThomas Gleixner /** 208a9f3a74aSThomas Gleixner * arch_exit_to_user_mode_work - Architecture specific TIF work for exit 209a9f3a74aSThomas Gleixner * to user mode. 210a9f3a74aSThomas Gleixner * @regs: Pointer to currents pt_regs 211a9f3a74aSThomas Gleixner * @ti_work: Cached TIF flags gathered with interrupts disabled 212a9f3a74aSThomas Gleixner * 213a9f3a74aSThomas Gleixner * Invoked from exit_to_user_mode_loop() with interrupt enabled 214a9f3a74aSThomas Gleixner * 215a9f3a74aSThomas Gleixner * Defaults to NOOP. Can be supplied by architecture specific code. 216a9f3a74aSThomas Gleixner */ 217a9f3a74aSThomas Gleixner static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 218a9f3a74aSThomas Gleixner unsigned long ti_work); 219a9f3a74aSThomas Gleixner 220a9f3a74aSThomas Gleixner #ifndef arch_exit_to_user_mode_work 221a9f3a74aSThomas Gleixner static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 222a9f3a74aSThomas Gleixner unsigned long ti_work) 223a9f3a74aSThomas Gleixner { 224a9f3a74aSThomas Gleixner } 225a9f3a74aSThomas Gleixner #endif 226a9f3a74aSThomas Gleixner 227a9f3a74aSThomas Gleixner /** 228a9f3a74aSThomas Gleixner * arch_exit_to_user_mode_prepare - Architecture specific preparation for 229a9f3a74aSThomas Gleixner * exit to user mode. 230a9f3a74aSThomas Gleixner * @regs: Pointer to currents pt_regs 231a9f3a74aSThomas Gleixner * @ti_work: Cached TIF flags gathered with interrupts disabled 232a9f3a74aSThomas Gleixner * 233a9f3a74aSThomas Gleixner * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last 234a9f3a74aSThomas Gleixner * function before return. Defaults to NOOP. 235a9f3a74aSThomas Gleixner */ 236a9f3a74aSThomas Gleixner static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 237a9f3a74aSThomas Gleixner unsigned long ti_work); 238a9f3a74aSThomas Gleixner 239a9f3a74aSThomas Gleixner #ifndef arch_exit_to_user_mode_prepare 240a9f3a74aSThomas Gleixner static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 241a9f3a74aSThomas Gleixner unsigned long ti_work) 242a9f3a74aSThomas Gleixner { 243a9f3a74aSThomas Gleixner } 244a9f3a74aSThomas Gleixner #endif 245a9f3a74aSThomas Gleixner 246a9f3a74aSThomas Gleixner /** 247a9f3a74aSThomas Gleixner * arch_exit_to_user_mode - Architecture specific final work before 248a9f3a74aSThomas Gleixner * exit to user mode. 249a9f3a74aSThomas Gleixner * 250a9f3a74aSThomas Gleixner * Invoked from exit_to_user_mode() with interrupt disabled as the last 251a9f3a74aSThomas Gleixner * function before return. Defaults to NOOP. 252a9f3a74aSThomas Gleixner * 253a9f3a74aSThomas Gleixner * This needs to be __always_inline because it is non-instrumentable code 254a9f3a74aSThomas Gleixner * invoked after context tracking switched to user mode. 255a9f3a74aSThomas Gleixner * 256a9f3a74aSThomas Gleixner * An architecture implementation must not do anything complex, no locking 257a9f3a74aSThomas Gleixner * etc. The main purpose is for speculation mitigations. 258a9f3a74aSThomas Gleixner */ 259a9f3a74aSThomas Gleixner static __always_inline void arch_exit_to_user_mode(void); 260a9f3a74aSThomas Gleixner 261a9f3a74aSThomas Gleixner #ifndef arch_exit_to_user_mode 262a9f3a74aSThomas Gleixner static __always_inline void arch_exit_to_user_mode(void) { } 263a9f3a74aSThomas Gleixner #endif 264a9f3a74aSThomas Gleixner 265a9f3a74aSThomas Gleixner /** 26612db8b69SJens Axboe * arch_do_signal_or_restart - Architecture specific signal delivery function 267a9f3a74aSThomas Gleixner * @regs: Pointer to currents pt_regs 26812db8b69SJens Axboe * @has_signal: actual signal to handle 269a9f3a74aSThomas Gleixner * 270a9f3a74aSThomas Gleixner * Invoked from exit_to_user_mode_loop(). 271a9f3a74aSThomas Gleixner */ 27212db8b69SJens Axboe void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal); 273a9f3a74aSThomas Gleixner 274a9f3a74aSThomas Gleixner /** 275a9f3a74aSThomas Gleixner * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() 276a9f3a74aSThomas Gleixner * @regs: Pointer to currents pt_regs 277a9f3a74aSThomas Gleixner * @step: Indicator for single step 278a9f3a74aSThomas Gleixner * 279a9f3a74aSThomas Gleixner * Defaults to tracehook_report_syscall_exit(). Can be replaced by 280a9f3a74aSThomas Gleixner * architecture specific code. 281a9f3a74aSThomas Gleixner * 282a9f3a74aSThomas Gleixner * Invoked from syscall_exit_to_user_mode() 283a9f3a74aSThomas Gleixner */ 284a9f3a74aSThomas Gleixner static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step); 285a9f3a74aSThomas Gleixner 286a9f3a74aSThomas Gleixner #ifndef arch_syscall_exit_tracehook 287a9f3a74aSThomas Gleixner static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step) 288a9f3a74aSThomas Gleixner { 289a9f3a74aSThomas Gleixner tracehook_report_syscall_exit(regs, step); 290a9f3a74aSThomas Gleixner } 291a9f3a74aSThomas Gleixner #endif 292a9f3a74aSThomas Gleixner 293a9f3a74aSThomas Gleixner /** 294a9f3a74aSThomas Gleixner * syscall_exit_to_user_mode - Handle work before returning to user mode 295a9f3a74aSThomas Gleixner * @regs: Pointer to currents pt_regs 296a9f3a74aSThomas Gleixner * 297a9f3a74aSThomas Gleixner * Invoked with interrupts enabled and fully valid regs. Returns with all 298a9f3a74aSThomas Gleixner * work handled, interrupts disabled such that the caller can immediately 299a9f3a74aSThomas Gleixner * switch to user mode. Called from architecture specific syscall and ret 300a9f3a74aSThomas Gleixner * from fork code. 301a9f3a74aSThomas Gleixner * 302a9f3a74aSThomas Gleixner * The call order is: 303a9f3a74aSThomas Gleixner * 1) One-time syscall exit work: 304a9f3a74aSThomas Gleixner * - rseq syscall exit 305a9f3a74aSThomas Gleixner * - audit 306a9f3a74aSThomas Gleixner * - syscall tracing 307a9f3a74aSThomas Gleixner * - tracehook (single stepping) 308a9f3a74aSThomas Gleixner * 309a9f3a74aSThomas Gleixner * 2) Preparatory work 310a9f3a74aSThomas Gleixner * - Exit to user mode loop (common TIF handling). Invokes 311a9f3a74aSThomas Gleixner * arch_exit_to_user_mode_work() for architecture specific TIF work 312a9f3a74aSThomas Gleixner * - Architecture specific one time work arch_exit_to_user_mode_prepare() 313a9f3a74aSThomas Gleixner * - Address limit and lockdep checks 314a9f3a74aSThomas Gleixner * 315a9f3a74aSThomas Gleixner * 3) Final transition (lockdep, tracing, context tracking, RCU). Invokes 316a9f3a74aSThomas Gleixner * arch_exit_to_user_mode() to handle e.g. speculation mitigations 317a9f3a74aSThomas Gleixner */ 318a9f3a74aSThomas Gleixner void syscall_exit_to_user_mode(struct pt_regs *regs); 319a9f3a74aSThomas Gleixner 320a9f3a74aSThomas Gleixner /** 321142781e1SThomas Gleixner * irqentry_enter_from_user_mode - Establish state before invoking the irq handler 322142781e1SThomas Gleixner * @regs: Pointer to currents pt_regs 323142781e1SThomas Gleixner * 324142781e1SThomas Gleixner * Invoked from architecture specific entry code with interrupts disabled. 325142781e1SThomas Gleixner * Can only be called when the interrupt entry came from user mode. The 326142781e1SThomas Gleixner * calling code must be non-instrumentable. When the function returns all 327142781e1SThomas Gleixner * state is correct and the subsequent functions can be instrumented. 328142781e1SThomas Gleixner * 329142781e1SThomas Gleixner * The function establishes state (lockdep, RCU (context tracking), tracing) 330142781e1SThomas Gleixner */ 331142781e1SThomas Gleixner void irqentry_enter_from_user_mode(struct pt_regs *regs); 332142781e1SThomas Gleixner 333a9f3a74aSThomas Gleixner /** 334a9f3a74aSThomas Gleixner * irqentry_exit_to_user_mode - Interrupt exit work 335a9f3a74aSThomas Gleixner * @regs: Pointer to current's pt_regs 336a9f3a74aSThomas Gleixner * 337a9f3a74aSThomas Gleixner * Invoked with interrupts disbled and fully valid regs. Returns with all 338a9f3a74aSThomas Gleixner * work handled, interrupts disabled such that the caller can immediately 339a9f3a74aSThomas Gleixner * switch to user mode. Called from architecture specific interrupt 340a9f3a74aSThomas Gleixner * handling code. 341a9f3a74aSThomas Gleixner * 342a9f3a74aSThomas Gleixner * The call order is #2 and #3 as described in syscall_exit_to_user_mode(). 343a9f3a74aSThomas Gleixner * Interrupt exit is not invoking #1 which is the syscall specific one time 344a9f3a74aSThomas Gleixner * work. 345a9f3a74aSThomas Gleixner */ 346a9f3a74aSThomas Gleixner void irqentry_exit_to_user_mode(struct pt_regs *regs); 347a9f3a74aSThomas Gleixner 348a5497babSThomas Gleixner #ifndef irqentry_state 349*b6be002bSThomas Gleixner /** 350*b6be002bSThomas Gleixner * struct irqentry_state - Opaque object for exception state storage 351*b6be002bSThomas Gleixner * @exit_rcu: Used exclusively in the irqentry_*() calls; signals whether the 352*b6be002bSThomas Gleixner * exit path has to invoke rcu_irq_exit(). 353*b6be002bSThomas Gleixner * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that 354*b6be002bSThomas Gleixner * lockdep state is restored correctly on exit from nmi. 355*b6be002bSThomas Gleixner * 356*b6be002bSThomas Gleixner * This opaque object is filled in by the irqentry_*_enter() functions and 357*b6be002bSThomas Gleixner * must be passed back into the corresponding irqentry_*_exit() functions 358*b6be002bSThomas Gleixner * when the exception is complete. 359*b6be002bSThomas Gleixner * 360*b6be002bSThomas Gleixner * Callers of irqentry_*_[enter|exit]() must consider this structure opaque 361*b6be002bSThomas Gleixner * and all members private. Descriptions of the members are provided to aid in 362*b6be002bSThomas Gleixner * the maintenance of the irqentry_*() functions. 363*b6be002bSThomas Gleixner */ 364a5497babSThomas Gleixner typedef struct irqentry_state { 365*b6be002bSThomas Gleixner union { 366a5497babSThomas Gleixner bool exit_rcu; 367*b6be002bSThomas Gleixner bool lockdep; 368*b6be002bSThomas Gleixner }; 369a5497babSThomas Gleixner } irqentry_state_t; 370a5497babSThomas Gleixner #endif 371a5497babSThomas Gleixner 372a5497babSThomas Gleixner /** 373a5497babSThomas Gleixner * irqentry_enter - Handle state tracking on ordinary interrupt entries 374a5497babSThomas Gleixner * @regs: Pointer to pt_regs of interrupted context 375a5497babSThomas Gleixner * 376a5497babSThomas Gleixner * Invokes: 377a5497babSThomas Gleixner * - lockdep irqflag state tracking as low level ASM entry disabled 378a5497babSThomas Gleixner * interrupts. 379a5497babSThomas Gleixner * 380a5497babSThomas Gleixner * - Context tracking if the exception hit user mode. 381a5497babSThomas Gleixner * 382a5497babSThomas Gleixner * - The hardirq tracer to keep the state consistent as low level ASM 383a5497babSThomas Gleixner * entry disabled interrupts. 384a5497babSThomas Gleixner * 385a5497babSThomas Gleixner * As a precondition, this requires that the entry came from user mode, 386a5497babSThomas Gleixner * idle, or a kernel context in which RCU is watching. 387a5497babSThomas Gleixner * 388a5497babSThomas Gleixner * For kernel mode entries RCU handling is done conditional. If RCU is 389a5497babSThomas Gleixner * watching then the only RCU requirement is to check whether the tick has 390a5497babSThomas Gleixner * to be restarted. If RCU is not watching then rcu_irq_enter() has to be 391a5497babSThomas Gleixner * invoked on entry and rcu_irq_exit() on exit. 392a5497babSThomas Gleixner * 393a5497babSThomas Gleixner * Avoiding the rcu_irq_enter/exit() calls is an optimization but also 394a5497babSThomas Gleixner * solves the problem of kernel mode pagefaults which can schedule, which 395a5497babSThomas Gleixner * is not possible after invoking rcu_irq_enter() without undoing it. 396a5497babSThomas Gleixner * 397a5497babSThomas Gleixner * For user mode entries irqentry_enter_from_user_mode() is invoked to 398a5497babSThomas Gleixner * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit 399a5497babSThomas Gleixner * would not be possible. 400a5497babSThomas Gleixner * 401a5497babSThomas Gleixner * Returns: An opaque object that must be passed to idtentry_exit() 402a5497babSThomas Gleixner */ 403a5497babSThomas Gleixner irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); 404a5497babSThomas Gleixner 405a5497babSThomas Gleixner /** 406a5497babSThomas Gleixner * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt 407a5497babSThomas Gleixner * 408a5497babSThomas Gleixner * Conditional reschedule with additional sanity checks. 409a5497babSThomas Gleixner */ 410a5497babSThomas Gleixner void irqentry_exit_cond_resched(void); 411a5497babSThomas Gleixner 412a5497babSThomas Gleixner /** 413a5497babSThomas Gleixner * irqentry_exit - Handle return from exception that used irqentry_enter() 414a5497babSThomas Gleixner * @regs: Pointer to pt_regs (exception entry regs) 415a5497babSThomas Gleixner * @state: Return value from matching call to irqentry_enter() 416a5497babSThomas Gleixner * 417a5497babSThomas Gleixner * Depending on the return target (kernel/user) this runs the necessary 418a5497babSThomas Gleixner * preemption and work checks if possible and reguired and returns to 419a5497babSThomas Gleixner * the caller with interrupts disabled and no further work pending. 420a5497babSThomas Gleixner * 421a5497babSThomas Gleixner * This is the last action before returning to the low level ASM code which 422a5497babSThomas Gleixner * just needs to return to the appropriate context. 423a5497babSThomas Gleixner * 424a5497babSThomas Gleixner * Counterpart to irqentry_enter(). 425a5497babSThomas Gleixner */ 426a5497babSThomas Gleixner void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); 427a5497babSThomas Gleixner 428*b6be002bSThomas Gleixner /** 429*b6be002bSThomas Gleixner * irqentry_nmi_enter - Handle NMI entry 430*b6be002bSThomas Gleixner * @regs: Pointer to currents pt_regs 431*b6be002bSThomas Gleixner * 432*b6be002bSThomas Gleixner * Similar to irqentry_enter() but taking care of the NMI constraints. 433*b6be002bSThomas Gleixner */ 434*b6be002bSThomas Gleixner irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs); 435*b6be002bSThomas Gleixner 436*b6be002bSThomas Gleixner /** 437*b6be002bSThomas Gleixner * irqentry_nmi_exit - Handle return from NMI handling 438*b6be002bSThomas Gleixner * @regs: Pointer to pt_regs (NMI entry regs) 439*b6be002bSThomas Gleixner * @irq_state: Return value from matching call to irqentry_nmi_enter() 440*b6be002bSThomas Gleixner * 441*b6be002bSThomas Gleixner * Last action before returning to the low level assmenbly code. 442*b6be002bSThomas Gleixner * 443*b6be002bSThomas Gleixner * Counterpart to irqentry_nmi_enter(). 444*b6be002bSThomas Gleixner */ 445*b6be002bSThomas Gleixner void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state); 446*b6be002bSThomas Gleixner 447142781e1SThomas Gleixner #endif 448