1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_ENTRYCOMMON_H 3 #define __LINUX_ENTRYCOMMON_H 4 5 #include <linux/tracehook.h> 6 #include <linux/syscalls.h> 7 #include <linux/seccomp.h> 8 #include <linux/sched.h> 9 10 #include <asm/entry-common.h> 11 12 /* 13 * Define dummy _TIF work flags if not defined by the architecture or for 14 * disabled functionality. 15 */ 16 #ifndef _TIF_SYSCALL_EMU 17 # define _TIF_SYSCALL_EMU (0) 18 #endif 19 20 #ifndef _TIF_SYSCALL_TRACEPOINT 21 # define _TIF_SYSCALL_TRACEPOINT (0) 22 #endif 23 24 #ifndef _TIF_SECCOMP 25 # define _TIF_SECCOMP (0) 26 #endif 27 28 #ifndef _TIF_SYSCALL_AUDIT 29 # define _TIF_SYSCALL_AUDIT (0) 30 #endif 31 32 #ifndef _TIF_PATCH_PENDING 33 # define _TIF_PATCH_PENDING (0) 34 #endif 35 36 #ifndef _TIF_UPROBE 37 # define _TIF_UPROBE (0) 38 #endif 39 40 /* 41 * TIF flags handled in syscall_enter_from_user_mode() 42 */ 43 #ifndef ARCH_SYSCALL_ENTER_WORK 44 # define ARCH_SYSCALL_ENTER_WORK (0) 45 #endif 46 47 #define SYSCALL_ENTER_WORK \ 48 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ 49 _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_EMU | \ 50 ARCH_SYSCALL_ENTER_WORK) 51 52 /* 53 * TIF flags handled in syscall_exit_to_user_mode() 54 */ 55 #ifndef ARCH_SYSCALL_EXIT_WORK 56 # define ARCH_SYSCALL_EXIT_WORK (0) 57 #endif 58 59 #define SYSCALL_EXIT_WORK \ 60 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ 61 _TIF_SYSCALL_TRACEPOINT | ARCH_SYSCALL_EXIT_WORK) 62 63 /* 64 * TIF flags handled in exit_to_user_mode_loop() 65 */ 66 #ifndef ARCH_EXIT_TO_USER_MODE_WORK 67 # define ARCH_EXIT_TO_USER_MODE_WORK (0) 68 #endif 69 70 #define EXIT_TO_USER_MODE_WORK \ 71 (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ 72 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | \ 73 ARCH_EXIT_TO_USER_MODE_WORK) 74 75 /** 76 * arch_check_user_regs - Architecture specific sanity check for user mode regs 77 * @regs: Pointer to currents pt_regs 78 * 79 * Defaults to an empty implementation. Can be replaced by architecture 80 * specific code. 81 * 82 * Invoked from syscall_enter_from_user_mode() in the non-instrumentable 83 * section. Use __always_inline so the compiler cannot push it out of line 84 * and make it instrumentable. 85 */ 86 static __always_inline void arch_check_user_regs(struct pt_regs *regs); 87 88 #ifndef arch_check_user_regs 89 static __always_inline void arch_check_user_regs(struct pt_regs *regs) {} 90 #endif 91 92 /** 93 * arch_syscall_enter_tracehook - Wrapper around tracehook_report_syscall_entry() 94 * @regs: Pointer to currents pt_regs 95 * 96 * Returns: 0 on success or an error code to skip the syscall. 97 * 98 * Defaults to tracehook_report_syscall_entry(). Can be replaced by 99 * architecture specific code. 100 * 101 * Invoked from syscall_enter_from_user_mode() 102 */ 103 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs); 104 105 #ifndef arch_syscall_enter_tracehook 106 static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs) 107 { 108 return tracehook_report_syscall_entry(regs); 109 } 110 #endif 111 112 /** 113 * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts 114 * @regs: Pointer to currents pt_regs 115 * 116 * Invoked from architecture specific syscall entry code with interrupts 117 * disabled. The calling code has to be non-instrumentable. When the 118 * function returns all state is correct, interrupts are enabled and the 119 * subsequent functions can be instrumented. 120 * 121 * This handles lockdep, RCU (context tracking) and tracing state. 122 * 123 * This is invoked when there is extra architecture specific functionality 124 * to be done between establishing state and handling user mode entry work. 125 */ 126 void syscall_enter_from_user_mode_prepare(struct pt_regs *regs); 127 128 /** 129 * syscall_enter_from_user_mode_work - Check and handle work before invoking 130 * a syscall 131 * @regs: Pointer to currents pt_regs 132 * @syscall: The syscall number 133 * 134 * Invoked from architecture specific syscall entry code with interrupts 135 * enabled after invoking syscall_enter_from_user_mode_prepare() and extra 136 * architecture specific work. 137 * 138 * Returns: The original or a modified syscall number 139 * 140 * If the returned syscall number is -1 then the syscall should be 141 * skipped. In this case the caller may invoke syscall_set_error() or 142 * syscall_set_return_value() first. If neither of those are called and -1 143 * is returned, then the syscall will fail with ENOSYS. 144 * 145 * It handles the following work items: 146 * 147 * 1) TIF flag dependent invocations of arch_syscall_enter_tracehook(), 148 * __secure_computing(), trace_sys_enter() 149 * 2) Invocation of audit_syscall_entry() 150 */ 151 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall); 152 153 /** 154 * syscall_enter_from_user_mode - Establish state and check and handle work 155 * before invoking a syscall 156 * @regs: Pointer to currents pt_regs 157 * @syscall: The syscall number 158 * 159 * Invoked from architecture specific syscall entry code with interrupts 160 * disabled. The calling code has to be non-instrumentable. When the 161 * function returns all state is correct, interrupts are enabled and the 162 * subsequent functions can be instrumented. 163 * 164 * This is combination of syscall_enter_from_user_mode_prepare() and 165 * syscall_enter_from_user_mode_work(). 166 * 167 * Returns: The original or a modified syscall number. See 168 * syscall_enter_from_user_mode_work() for further explanation. 169 */ 170 long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall); 171 172 /** 173 * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() 174 * @ti_work: Cached TIF flags gathered with interrupts disabled 175 * 176 * Defaults to local_irq_enable(). Can be supplied by architecture specific 177 * code. 178 */ 179 static inline void local_irq_enable_exit_to_user(unsigned long ti_work); 180 181 #ifndef local_irq_enable_exit_to_user 182 static inline void local_irq_enable_exit_to_user(unsigned long ti_work) 183 { 184 local_irq_enable(); 185 } 186 #endif 187 188 /** 189 * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() 190 * 191 * Defaults to local_irq_disable(). Can be supplied by architecture specific 192 * code. 193 */ 194 static inline void local_irq_disable_exit_to_user(void); 195 196 #ifndef local_irq_disable_exit_to_user 197 static inline void local_irq_disable_exit_to_user(void) 198 { 199 local_irq_disable(); 200 } 201 #endif 202 203 /** 204 * arch_exit_to_user_mode_work - Architecture specific TIF work for exit 205 * to user mode. 206 * @regs: Pointer to currents pt_regs 207 * @ti_work: Cached TIF flags gathered with interrupts disabled 208 * 209 * Invoked from exit_to_user_mode_loop() with interrupt enabled 210 * 211 * Defaults to NOOP. Can be supplied by architecture specific code. 212 */ 213 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 214 unsigned long ti_work); 215 216 #ifndef arch_exit_to_user_mode_work 217 static inline void arch_exit_to_user_mode_work(struct pt_regs *regs, 218 unsigned long ti_work) 219 { 220 } 221 #endif 222 223 /** 224 * arch_exit_to_user_mode_prepare - Architecture specific preparation for 225 * exit to user mode. 226 * @regs: Pointer to currents pt_regs 227 * @ti_work: Cached TIF flags gathered with interrupts disabled 228 * 229 * Invoked from exit_to_user_mode_prepare() with interrupt disabled as the last 230 * function before return. Defaults to NOOP. 231 */ 232 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 233 unsigned long ti_work); 234 235 #ifndef arch_exit_to_user_mode_prepare 236 static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, 237 unsigned long ti_work) 238 { 239 } 240 #endif 241 242 /** 243 * arch_exit_to_user_mode - Architecture specific final work before 244 * exit to user mode. 245 * 246 * Invoked from exit_to_user_mode() with interrupt disabled as the last 247 * function before return. Defaults to NOOP. 248 * 249 * This needs to be __always_inline because it is non-instrumentable code 250 * invoked after context tracking switched to user mode. 251 * 252 * An architecture implementation must not do anything complex, no locking 253 * etc. The main purpose is for speculation mitigations. 254 */ 255 static __always_inline void arch_exit_to_user_mode(void); 256 257 #ifndef arch_exit_to_user_mode 258 static __always_inline void arch_exit_to_user_mode(void) { } 259 #endif 260 261 /** 262 * arch_do_signal - Architecture specific signal delivery function 263 * @regs: Pointer to currents pt_regs 264 * 265 * Invoked from exit_to_user_mode_loop(). 266 */ 267 void arch_do_signal(struct pt_regs *regs); 268 269 /** 270 * arch_syscall_exit_tracehook - Wrapper around tracehook_report_syscall_exit() 271 * @regs: Pointer to currents pt_regs 272 * @step: Indicator for single step 273 * 274 * Defaults to tracehook_report_syscall_exit(). Can be replaced by 275 * architecture specific code. 276 * 277 * Invoked from syscall_exit_to_user_mode() 278 */ 279 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step); 280 281 #ifndef arch_syscall_exit_tracehook 282 static inline void arch_syscall_exit_tracehook(struct pt_regs *regs, bool step) 283 { 284 tracehook_report_syscall_exit(regs, step); 285 } 286 #endif 287 288 /** 289 * syscall_exit_to_user_mode - Handle work before returning to user mode 290 * @regs: Pointer to currents pt_regs 291 * 292 * Invoked with interrupts enabled and fully valid regs. Returns with all 293 * work handled, interrupts disabled such that the caller can immediately 294 * switch to user mode. Called from architecture specific syscall and ret 295 * from fork code. 296 * 297 * The call order is: 298 * 1) One-time syscall exit work: 299 * - rseq syscall exit 300 * - audit 301 * - syscall tracing 302 * - tracehook (single stepping) 303 * 304 * 2) Preparatory work 305 * - Exit to user mode loop (common TIF handling). Invokes 306 * arch_exit_to_user_mode_work() for architecture specific TIF work 307 * - Architecture specific one time work arch_exit_to_user_mode_prepare() 308 * - Address limit and lockdep checks 309 * 310 * 3) Final transition (lockdep, tracing, context tracking, RCU). Invokes 311 * arch_exit_to_user_mode() to handle e.g. speculation mitigations 312 */ 313 void syscall_exit_to_user_mode(struct pt_regs *regs); 314 315 /** 316 * irqentry_enter_from_user_mode - Establish state before invoking the irq handler 317 * @regs: Pointer to currents pt_regs 318 * 319 * Invoked from architecture specific entry code with interrupts disabled. 320 * Can only be called when the interrupt entry came from user mode. The 321 * calling code must be non-instrumentable. When the function returns all 322 * state is correct and the subsequent functions can be instrumented. 323 * 324 * The function establishes state (lockdep, RCU (context tracking), tracing) 325 */ 326 void irqentry_enter_from_user_mode(struct pt_regs *regs); 327 328 /** 329 * irqentry_exit_to_user_mode - Interrupt exit work 330 * @regs: Pointer to current's pt_regs 331 * 332 * Invoked with interrupts disbled and fully valid regs. Returns with all 333 * work handled, interrupts disabled such that the caller can immediately 334 * switch to user mode. Called from architecture specific interrupt 335 * handling code. 336 * 337 * The call order is #2 and #3 as described in syscall_exit_to_user_mode(). 338 * Interrupt exit is not invoking #1 which is the syscall specific one time 339 * work. 340 */ 341 void irqentry_exit_to_user_mode(struct pt_regs *regs); 342 343 #ifndef irqentry_state 344 typedef struct irqentry_state { 345 bool exit_rcu; 346 } irqentry_state_t; 347 #endif 348 349 /** 350 * irqentry_enter - Handle state tracking on ordinary interrupt entries 351 * @regs: Pointer to pt_regs of interrupted context 352 * 353 * Invokes: 354 * - lockdep irqflag state tracking as low level ASM entry disabled 355 * interrupts. 356 * 357 * - Context tracking if the exception hit user mode. 358 * 359 * - The hardirq tracer to keep the state consistent as low level ASM 360 * entry disabled interrupts. 361 * 362 * As a precondition, this requires that the entry came from user mode, 363 * idle, or a kernel context in which RCU is watching. 364 * 365 * For kernel mode entries RCU handling is done conditional. If RCU is 366 * watching then the only RCU requirement is to check whether the tick has 367 * to be restarted. If RCU is not watching then rcu_irq_enter() has to be 368 * invoked on entry and rcu_irq_exit() on exit. 369 * 370 * Avoiding the rcu_irq_enter/exit() calls is an optimization but also 371 * solves the problem of kernel mode pagefaults which can schedule, which 372 * is not possible after invoking rcu_irq_enter() without undoing it. 373 * 374 * For user mode entries irqentry_enter_from_user_mode() is invoked to 375 * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit 376 * would not be possible. 377 * 378 * Returns: An opaque object that must be passed to idtentry_exit() 379 */ 380 irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); 381 382 /** 383 * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt 384 * 385 * Conditional reschedule with additional sanity checks. 386 */ 387 void irqentry_exit_cond_resched(void); 388 389 /** 390 * irqentry_exit - Handle return from exception that used irqentry_enter() 391 * @regs: Pointer to pt_regs (exception entry regs) 392 * @state: Return value from matching call to irqentry_enter() 393 * 394 * Depending on the return target (kernel/user) this runs the necessary 395 * preemption and work checks if possible and reguired and returns to 396 * the caller with interrupts disabled and no further work pending. 397 * 398 * This is the last action before returning to the low level ASM code which 399 * just needs to return to the appropriate context. 400 * 401 * Counterpart to irqentry_enter(). 402 */ 403 void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state); 404 405 #endif 406