1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_ENTRYCOMMON_H 3 #define __LINUX_ENTRYCOMMON_H 4 5 #include <linux/audit.h> 6 #include <linux/irq-entry-common.h> 7 #include <linux/livepatch.h> 8 #include <linux/ptrace.h> 9 #include <linux/randomize_kstack.h> 10 #include <linux/resume_user_mode.h> 11 #include <linux/seccomp.h> 12 #include <linux/sched.h> 13 #include <linux/syscall_user_dispatch.h> 14 15 #include <asm/entry-common.h> 16 #include <asm/syscall.h> 17 18 #ifndef _TIF_UPROBE 19 # define _TIF_UPROBE (0) 20 #endif 21 22 /* 23 * SYSCALL_WORK flags handled in syscall_enter_from_user_mode_work() 24 */ 25 #define SYSCALL_WORK_ENTER (SYSCALL_WORK_SECCOMP | \ 26 SYSCALL_WORK_SYSCALL_TRACEPOINT | \ 27 SYSCALL_WORK_SYSCALL_TRACE | \ 28 SYSCALL_WORK_SYSCALL_EMU | \ 29 SYSCALL_WORK_SYSCALL_AUDIT | \ 30 SYSCALL_WORK_SYSCALL_USER_DISPATCH | \ 31 SYSCALL_WORK_SYSCALL_RSEQ_SLICE) 32 /* 33 * SYSCALL_WORK flags handled in syscall_exit_to_user_mode() 34 */ 35 #define SYSCALL_WORK_EXIT (SYSCALL_WORK_SYSCALL_TRACEPOINT | \ 36 SYSCALL_WORK_SYSCALL_TRACE | \ 37 SYSCALL_WORK_SYSCALL_AUDIT | \ 38 SYSCALL_WORK_SYSCALL_USER_DISPATCH | \ 39 SYSCALL_WORK_SYSCALL_EXIT_TRAP) 40 41 /** 42 * arch_ptrace_report_syscall_permit_entry - Architecture specific wrapper for 43 * ptrace_report_syscall_permit_entry() 44 * @regs: Pointer to the register state at syscall entry 45 * 46 * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_permit_entry(). 47 * 48 * This allows architecture specific ptrace_report_syscall_permit_entry() 49 * implementations. If not defined by the architecture this falls back to 50 * to ptrace_report_syscall_permit_entry(). 51 */ 52 static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs); 53 54 #ifndef arch_ptrace_report_syscall_permit_entry 55 static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs) 56 { 57 return ptrace_report_syscall_permit_entry(regs); 58 } 59 #endif 60 61 void trace_syscall_enter(struct pt_regs *regs); 62 void trace_syscall_exit(struct pt_regs *regs, long ret); 63 void syscall_enter_audit(struct pt_regs *regs); 64 65 static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work, 66 long syscall) 67 { 68 /* 69 * Handle Syscall User Dispatch. This must comes first, since 70 * the ABI here can be something that doesn't make sense for 71 * other syscall_work features. 72 */ 73 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 74 if (syscall_user_dispatch(regs)) 75 return false; 76 } 77 78 /* 79 * User space got a time slice extension granted and relinquishes 80 * the CPU. The work stops the slice timer to avoid an extra round 81 * through hrtimer_interrupt(). 82 */ 83 if (work & SYSCALL_WORK_SYSCALL_RSEQ_SLICE) 84 rseq_syscall_enter_work(syscall); 85 86 /* Handle ptrace */ 87 if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) { 88 if (!arch_ptrace_report_syscall_permit_entry(regs) || 89 (work & SYSCALL_WORK_SYSCALL_EMU)) 90 return false; 91 92 /* ptrace might have changed work flags */ 93 work = READ_ONCE(current_thread_info()->syscall_work); 94 } 95 96 /* Do seccomp after ptrace, to catch any tracer changes. */ 97 if (work & SYSCALL_WORK_SECCOMP) { 98 if (!__seccomp_permit_syscall()) 99 return false; 100 } 101 102 if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT)) 103 trace_syscall_enter(regs); 104 105 if (unlikely(audit_context())) 106 syscall_enter_audit(regs); 107 108 return true; 109 } 110 111 /** 112 * syscall_enter_from_user_mode_work - Check and handle work before invoking 113 * a syscall 114 * @regs: Pointer to currents pt_regs 115 * @syscall: The syscall number 116 * 117 * Invoked from architecture specific syscall entry code with interrupts enabled 118 * after invoking enter_from_user_mode(), enabling interrupts and extra 119 * architecture specific work with the syscall return value preset to -ENOSYS. 120 * 121 * Returns: True if the syscall should be invoked, False otherwise. 122 * 123 * If the return value is false, the caller must skip the syscall and leave the 124 * syscall return value unmodified as it might have been set by one of the entry 125 * work functions. 126 * 127 * It handles the following work items: 128 * 129 * 1) syscall_work flag dependent invocations of 130 * ptrace_report_syscall_permit_entry(), __seccomp_permit_syscall(), trace_sys_enter() 131 * 2) Invocation of audit_syscall_entry() 132 */ 133 static __always_inline bool syscall_enter_from_user_mode_work(struct pt_regs *regs, long *syscall) 134 { 135 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 136 137 if (!(work & SYSCALL_WORK_ENTER)) 138 return true; 139 140 if (unlikely(!syscall_trace_enter(regs, work, *syscall))) 141 return false; 142 143 /* Reread the syscall number as it might have been modified */ 144 *syscall = syscall_get_nr(current, regs); 145 146 return true; 147 } 148 149 /** 150 * enter_from_user_mode_randomize_stack - Establish state and add stack randomization 151 * before invoking syscall_enter_from_user_mode_work() 152 * @regs: Pointer to currents pt_regs 153 * 154 * Invoked from architecture specific syscall entry code with interrupts 155 * disabled. The calling code has to be non-instrumentable. When the function 156 * returns all state is correct, interrupts are still disabled and the 157 * subsequent functions can be instrumented. 158 * 159 * Implemented as a macro so that the stack randomization is effective 160 * throughout the function in which it is invoked. An inline would only make it 161 * effective in the scope of the inline function. 162 */ 163 #define enter_from_user_mode_randomize_stack(regs) \ 164 do { \ 165 enter_from_user_mode(regs); \ 166 instrumentation_begin(); \ 167 add_random_kstack_offset_irqsoff(); \ 168 instrumentation_end(); \ 169 } while (0) 170 171 /** 172 * syscall_enter_from_user_mode_randomize_stack - Establish state and check and handle work 173 * before invoking a syscall 174 * @regs: Pointer to currents pt_regs 175 * @syscall: The syscall number 176 * 177 * Invoked from architecture specific syscall entry code with interrupts 178 * disabled. The calling code has to be non-instrumentable. When the 179 * function returns all state is correct, interrupts are enabled and the 180 * subsequent functions can be instrumented. 181 * 182 * This is the combination of enter_from_user_mode_randomize_stack() and 183 * syscall_enter_from_user_mode_work() to be used when there is no 184 * architecture specific work to be done between the two. 185 * 186 * Returns: The original or a modified syscall number. See 187 * syscall_enter_from_user_mode_work() for further explanation. 188 * 189 * Implemented as a macro to make stack randomization effective in the calling 190 * scope. 191 */ 192 #define syscall_enter_from_user_mode_randomize_stack(regs, syscall) \ 193 ({ \ 194 enter_from_user_mode_randomize_stack(regs); \ 195 \ 196 instrumentation_begin(); \ 197 local_irq_enable(); \ 198 long _ret = syscall_enter_from_user_mode_work(regs, syscall); \ 199 instrumentation_end(); \ 200 \ 201 _ret; \ 202 }) 203 204 /* 205 * If SYSCALL_EMU is set, then the only reason to report is when SINGLESTEP is 206 * set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall instruction has been 207 * already reported in syscall_enter_from_user_mode_work(). 208 */ 209 static __always_inline bool report_single_step(unsigned long work) 210 { 211 if (work & SYSCALL_WORK_SYSCALL_EMU) 212 return false; 213 214 return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; 215 } 216 217 /** 218 * arch_ptrace_report_syscall_exit - Architecture specific ptrace_report_syscall_exit() 219 * @regs: Pointer to the register state at syscall exit 220 * @step: Indicates a single-step exit rather than a normal syscall exit 221 * 222 * This allows architecture specific ptrace_report_syscall_exit() 223 * implementations. If not defined by the architecture this falls back to 224 * to ptrace_report_syscall_exit(). 225 */ 226 static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, 227 int step); 228 229 #ifndef arch_ptrace_report_syscall_exit 230 static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs, 231 int step) 232 { 233 ptrace_report_syscall_exit(regs, step); 234 } 235 #endif 236 237 /** 238 * syscall_exit_work - Handle work before returning to user mode 239 * @regs: Pointer to current pt_regs 240 * @work: Current thread syscall work 241 * 242 * Do one-time syscall specific work. 243 */ 244 static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned long work) 245 { 246 bool step; 247 248 /* 249 * If the syscall was rolled back due to syscall user dispatching, 250 * then the tracers below are not invoked for the same reason as 251 * the entry side was not invoked in syscall_trace_enter(): The ABI 252 * of these syscalls is unknown. 253 */ 254 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 255 if (syscall_user_dispatch_clear_on_dispatch()) 256 return; 257 } 258 259 audit_syscall_exit(regs); 260 261 if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) 262 trace_syscall_exit(regs, syscall_get_return_value(current, regs)); 263 264 step = report_single_step(work); 265 if (step || work & SYSCALL_WORK_SYSCALL_TRACE) 266 arch_ptrace_report_syscall_exit(regs, step); 267 } 268 269 /** 270 * syscall_exit_to_user_mode_work - Handle one time work before returning to user mode 271 * @regs: Pointer to currents pt_regs 272 * 273 * Step 1 of syscall_exit_to_user_mode() with the same calling convention. 274 * 275 * The caller must invoke steps 2-3 of syscall_exit_to_user_mode() afterwards. 276 */ 277 static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs) 278 { 279 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 280 unsigned long nr = syscall_get_nr(current, regs); 281 282 CT_WARN_ON(ct_state() != CT_STATE_KERNEL); 283 284 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 285 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) 286 local_irq_enable(); 287 } 288 289 rseq_debug_syscall_return(regs); 290 291 /* 292 * Do one-time syscall specific work. If these work items are 293 * enabled, we want to run them exactly once per syscall exit with 294 * interrupts enabled. 295 */ 296 if (unlikely(work & SYSCALL_WORK_EXIT)) 297 syscall_exit_work(regs, work); 298 } 299 300 /** 301 * syscall_exit_to_user_mode - Handle work before returning to user mode 302 * @regs: Pointer to currents pt_regs 303 * 304 * Invoked with interrupts enabled and fully valid @regs. Returns with all 305 * work handled, interrupts disabled such that the caller can immediately 306 * switch to user mode. Called from architecture specific syscall and ret 307 * from fork code. 308 * 309 * The call order is: 310 * 1) One-time syscall exit work: 311 * - rseq syscall exit 312 * - audit 313 * - syscall tracing 314 * - ptrace (single stepping) 315 * 316 * 2) Preparatory work 317 * - Disable interrupts 318 * - Exit to user mode loop (common TIF handling). Invokes 319 * arch_exit_to_user_mode_work() for architecture specific TIF work 320 * - Architecture specific one time work arch_exit_to_user_mode_prepare() 321 * - Address limit and lockdep checks 322 * 323 * 3) Final transition (lockdep, tracing, context tracking, RCU), i.e. the 324 * functionality in exit_to_user_mode(). 325 * 326 * This is a combination of syscall_exit_to_user_mode_work() (1), disabling 327 * interrupts followed by syscall_exit_to_user_mode_prepare() (2) and 328 * exit_to_user_mode() (3). This function is preferred unless there is a 329 * compelling architectural reason to invoke the functions separately. 330 */ 331 static __always_inline void syscall_exit_to_user_mode(struct pt_regs *regs) 332 { 333 instrumentation_begin(); 334 syscall_exit_to_user_mode_work(regs); 335 local_irq_disable(); 336 syscall_exit_to_user_mode_prepare(regs); 337 instrumentation_end(); 338 exit_to_user_mode(); 339 } 340 341 #endif 342