1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_ENTRYCOMMON_H
3 #define __LINUX_ENTRYCOMMON_H
4
5 #include <linux/audit.h>
6 #include <linux/irq-entry-common.h>
7 #include <linux/livepatch.h>
8 #include <linux/ptrace.h>
9 #include <linux/randomize_kstack.h>
10 #include <linux/resume_user_mode.h>
11 #include <linux/seccomp.h>
12 #include <linux/sched.h>
13 #include <linux/syscall_user_dispatch.h>
14
15 #include <asm/entry-common.h>
16 #include <asm/syscall.h>
17
18 #ifndef _TIF_UPROBE
19 # define _TIF_UPROBE (0)
20 #endif
21
22 /*
23 * SYSCALL_WORK flags handled in syscall_enter_from_user_mode_work()
24 */
25 #define SYSCALL_WORK_ENTER (SYSCALL_WORK_SECCOMP | \
26 SYSCALL_WORK_SYSCALL_TRACEPOINT | \
27 SYSCALL_WORK_SYSCALL_TRACE | \
28 SYSCALL_WORK_SYSCALL_EMU | \
29 SYSCALL_WORK_SYSCALL_AUDIT | \
30 SYSCALL_WORK_SYSCALL_USER_DISPATCH | \
31 SYSCALL_WORK_SYSCALL_RSEQ_SLICE)
32 /*
33 * SYSCALL_WORK flags handled in syscall_exit_to_user_mode()
34 */
35 #define SYSCALL_WORK_EXIT (SYSCALL_WORK_SYSCALL_TRACEPOINT | \
36 SYSCALL_WORK_SYSCALL_TRACE | \
37 SYSCALL_WORK_SYSCALL_AUDIT | \
38 SYSCALL_WORK_SYSCALL_USER_DISPATCH | \
39 SYSCALL_WORK_SYSCALL_EXIT_TRAP)
40
41 /**
42 * arch_ptrace_report_syscall_permit_entry - Architecture specific wrapper for
43 * ptrace_report_syscall_permit_entry()
44 * @regs: Pointer to the register state at syscall entry
45 *
46 * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_permit_entry().
47 *
48 * This allows architecture specific ptrace_report_syscall_permit_entry()
49 * implementations. If not defined by the architecture this falls back to
50 * to ptrace_report_syscall_permit_entry().
51 */
52 static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs);
53
54 #ifndef arch_ptrace_report_syscall_permit_entry
arch_ptrace_report_syscall_permit_entry(struct pt_regs * regs)55 static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs)
56 {
57 return ptrace_report_syscall_permit_entry(regs);
58 }
59 #endif
60
61 void trace_syscall_enter(struct pt_regs *regs);
62 void trace_syscall_exit(struct pt_regs *regs, long ret);
63 void syscall_enter_audit(struct pt_regs *regs);
64
syscall_trace_enter(struct pt_regs * regs,unsigned long work,long syscall)65 static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work,
66 long syscall)
67 {
68 /*
69 * Handle Syscall User Dispatch. This must comes first, since
70 * the ABI here can be something that doesn't make sense for
71 * other syscall_work features.
72 */
73 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
74 if (syscall_user_dispatch(regs))
75 return false;
76 }
77
78 /*
79 * User space got a time slice extension granted and relinquishes
80 * the CPU. The work stops the slice timer to avoid an extra round
81 * through hrtimer_interrupt().
82 */
83 if (work & SYSCALL_WORK_SYSCALL_RSEQ_SLICE)
84 rseq_syscall_enter_work(syscall);
85
86 /* Handle ptrace */
87 if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) {
88 if (!arch_ptrace_report_syscall_permit_entry(regs) ||
89 (work & SYSCALL_WORK_SYSCALL_EMU))
90 return false;
91
92 /* ptrace might have changed work flags */
93 work = READ_ONCE(current_thread_info()->syscall_work);
94 }
95
96 /* Do seccomp after ptrace, to catch any tracer changes. */
97 if (work & SYSCALL_WORK_SECCOMP) {
98 if (!__seccomp_permit_syscall())
99 return false;
100 }
101
102 if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT))
103 trace_syscall_enter(regs);
104
105 /*
106 * The config check works around broken compilers which fail to
107 * eliminate the dead code in case of CONFIG_AUDITSYSCALL=n as they
108 * insist on creating a always false runtime condition based on
109 * audit_context() which returns NULL in that case. The explicit
110 * IS_ENABLED() check makes that madness go away.
111 */
112 if (IS_ENABLED(CONFIG_AUDITSYSCALL) && unlikely(audit_context()))
113 syscall_enter_audit(regs);
114
115 return true;
116 }
117
118 /**
119 * syscall_enter_from_user_mode_work - Check and handle work before invoking
120 * a syscall
121 * @regs: Pointer to currents pt_regs
122 * @syscall: The syscall number
123 *
124 * Invoked from architecture specific syscall entry code with interrupts enabled
125 * after invoking enter_from_user_mode(), enabling interrupts and extra
126 * architecture specific work with the syscall return value preset to -ENOSYS.
127 *
128 * Returns: True if the syscall should be invoked, False otherwise.
129 *
130 * If the return value is false, the caller must skip the syscall and leave the
131 * syscall return value unmodified as it might have been set by one of the entry
132 * work functions.
133 *
134 * It handles the following work items:
135 *
136 * 1) syscall_work flag dependent invocations of
137 * ptrace_report_syscall_permit_entry(), __seccomp_permit_syscall(), trace_sys_enter()
138 * 2) Invocation of audit_syscall_entry()
139 */
syscall_enter_from_user_mode_work(struct pt_regs * regs,long * syscall)140 static __always_inline bool syscall_enter_from_user_mode_work(struct pt_regs *regs, long *syscall)
141 {
142 unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
143
144 if (!(work & SYSCALL_WORK_ENTER))
145 return true;
146
147 if (unlikely(!syscall_trace_enter(regs, work, *syscall)))
148 return false;
149
150 /* Reread the syscall number as it might have been modified */
151 *syscall = syscall_get_nr(current, regs);
152
153 return true;
154 }
155
156 /**
157 * enter_from_user_mode_randomize_stack - Establish state and add stack randomization
158 * before invoking syscall_enter_from_user_mode_work()
159 * @regs: Pointer to currents pt_regs
160 *
161 * Invoked from architecture specific syscall entry code with interrupts
162 * disabled. The calling code has to be non-instrumentable. When the function
163 * returns all state is correct, interrupts are still disabled and the
164 * subsequent functions can be instrumented.
165 *
166 * Implemented as a macro so that the stack randomization is effective
167 * throughout the function in which it is invoked. An inline would only make it
168 * effective in the scope of the inline function.
169 */
170 #define enter_from_user_mode_randomize_stack(regs) \
171 do { \
172 enter_from_user_mode(regs); \
173 instrumentation_begin(); \
174 add_random_kstack_offset_irqsoff(); \
175 instrumentation_end(); \
176 } while (0)
177
178 /**
179 * syscall_enter_from_user_mode_randomize_stack - Establish state and check and handle work
180 * before invoking a syscall
181 * @regs: Pointer to currents pt_regs
182 * @syscall: The syscall number
183 *
184 * Invoked from architecture specific syscall entry code with interrupts
185 * disabled. The calling code has to be non-instrumentable. When the
186 * function returns all state is correct, interrupts are enabled and the
187 * subsequent functions can be instrumented.
188 *
189 * This is the combination of enter_from_user_mode_randomize_stack() and
190 * syscall_enter_from_user_mode_work() to be used when there is no
191 * architecture specific work to be done between the two.
192 *
193 * Returns: The original or a modified syscall number. See
194 * syscall_enter_from_user_mode_work() for further explanation.
195 *
196 * Implemented as a macro to make stack randomization effective in the calling
197 * scope.
198 */
199 #define syscall_enter_from_user_mode_randomize_stack(regs, syscall) \
200 ({ \
201 enter_from_user_mode_randomize_stack(regs); \
202 \
203 instrumentation_begin(); \
204 local_irq_enable(); \
205 long _ret = syscall_enter_from_user_mode_work(regs, syscall); \
206 instrumentation_end(); \
207 \
208 _ret; \
209 })
210
211 /*
212 * If SYSCALL_EMU is set, then the only reason to report is when SINGLESTEP is
213 * set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall instruction has been
214 * already reported in syscall_enter_from_user_mode_work().
215 */
report_single_step(unsigned long work)216 static __always_inline bool report_single_step(unsigned long work)
217 {
218 if (work & SYSCALL_WORK_SYSCALL_EMU)
219 return false;
220
221 return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP;
222 }
223
224 /**
225 * arch_ptrace_report_syscall_exit - Architecture specific ptrace_report_syscall_exit()
226 * @regs: Pointer to the register state at syscall exit
227 * @step: Indicates a single-step exit rather than a normal syscall exit
228 *
229 * This allows architecture specific ptrace_report_syscall_exit()
230 * implementations. If not defined by the architecture this falls back to
231 * to ptrace_report_syscall_exit().
232 */
233 static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs,
234 int step);
235
236 #ifndef arch_ptrace_report_syscall_exit
arch_ptrace_report_syscall_exit(struct pt_regs * regs,int step)237 static __always_inline void arch_ptrace_report_syscall_exit(struct pt_regs *regs,
238 int step)
239 {
240 ptrace_report_syscall_exit(regs, step);
241 }
242 #endif
243
244 /**
245 * syscall_exit_work - Handle work before returning to user mode
246 * @regs: Pointer to current pt_regs
247 * @work: Current thread syscall work
248 *
249 * Do one-time syscall specific work.
250 */
syscall_exit_work(struct pt_regs * regs,unsigned long work)251 static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned long work)
252 {
253 bool step;
254
255 /*
256 * If the syscall was rolled back due to syscall user dispatching,
257 * then the tracers below are not invoked for the same reason as
258 * the entry side was not invoked in syscall_trace_enter(): The ABI
259 * of these syscalls is unknown.
260 */
261 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
262 if (syscall_user_dispatch_clear_on_dispatch())
263 return;
264 }
265
266 audit_syscall_exit(regs);
267
268 if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT)
269 trace_syscall_exit(regs, syscall_get_return_value(current, regs));
270
271 step = report_single_step(work);
272 if (step || work & SYSCALL_WORK_SYSCALL_TRACE)
273 arch_ptrace_report_syscall_exit(regs, step);
274 }
275
276 /**
277 * syscall_exit_to_user_mode_work - Handle one time work before returning to user mode
278 * @regs: Pointer to currents pt_regs
279 *
280 * Step 1 of syscall_exit_to_user_mode() with the same calling convention.
281 *
282 * The caller must invoke steps 2-3 of syscall_exit_to_user_mode() afterwards.
283 */
syscall_exit_to_user_mode_work(struct pt_regs * regs)284 static __always_inline void syscall_exit_to_user_mode_work(struct pt_regs *regs)
285 {
286 unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
287 unsigned long nr = syscall_get_nr(current, regs);
288
289 CT_WARN_ON(ct_state() != CT_STATE_KERNEL);
290
291 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
292 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
293 local_irq_enable();
294 }
295
296 rseq_debug_syscall_return(regs);
297
298 /*
299 * Do one-time syscall specific work. If these work items are
300 * enabled, we want to run them exactly once per syscall exit with
301 * interrupts enabled.
302 */
303 if (unlikely(work & SYSCALL_WORK_EXIT))
304 syscall_exit_work(regs, work);
305 }
306
307 /**
308 * syscall_exit_to_user_mode - Handle work before returning to user mode
309 * @regs: Pointer to currents pt_regs
310 *
311 * Invoked with interrupts enabled and fully valid @regs. Returns with all
312 * work handled, interrupts disabled such that the caller can immediately
313 * switch to user mode. Called from architecture specific syscall and ret
314 * from fork code.
315 *
316 * The call order is:
317 * 1) One-time syscall exit work:
318 * - rseq syscall exit
319 * - audit
320 * - syscall tracing
321 * - ptrace (single stepping)
322 *
323 * 2) Preparatory work
324 * - Disable interrupts
325 * - Exit to user mode loop (common TIF handling). Invokes
326 * arch_exit_to_user_mode_work() for architecture specific TIF work
327 * - Architecture specific one time work arch_exit_to_user_mode_prepare()
328 * - Address limit and lockdep checks
329 *
330 * 3) Final transition (lockdep, tracing, context tracking, RCU), i.e. the
331 * functionality in exit_to_user_mode().
332 *
333 * This is a combination of syscall_exit_to_user_mode_work() (1), disabling
334 * interrupts followed by syscall_exit_to_user_mode_prepare() (2) and
335 * exit_to_user_mode() (3). This function is preferred unless there is a
336 * compelling architectural reason to invoke the functions separately.
337 */
syscall_exit_to_user_mode(struct pt_regs * regs)338 static __always_inline void syscall_exit_to_user_mode(struct pt_regs *regs)
339 {
340 instrumentation_begin();
341 syscall_exit_to_user_mode_work(regs);
342 local_irq_disable();
343 syscall_exit_to_user_mode_prepare(regs);
344 instrumentation_end();
345 exit_to_user_mode();
346 }
347
348 #endif
349