1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 64-bit system call dispatch */ 3 4 #include <linux/linkage.h> 5 #include <linux/sys.h> 6 #include <linux/cache.h> 7 #include <linux/syscalls.h> 8 #include <linux/entry-common.h> 9 #include <linux/nospec.h> 10 #include <asm/syscall.h> 11 12 #define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *); 13 #define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *); 14 #include <asm/syscalls_64.h> 15 #ifdef CONFIG_X86_X32_ABI 16 #include <asm/syscalls_x32.h> 17 #endif 18 #undef __SYSCALL 19 20 #undef __SYSCALL_NORETURN 21 #define __SYSCALL_NORETURN __SYSCALL 22 23 /* 24 * The sys_call_table[] is no longer used for system calls, but 25 * kernel/trace/trace_syscalls.c still wants to know the system 26 * call address. 27 */ 28 #define __SYSCALL(nr, sym) __x64_##sym, 29 const sys_call_ptr_t sys_call_table[] = { 30 #include <asm/syscalls_64.h> 31 }; 32 #undef __SYSCALL 33 34 #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs); 35 36 /* The unsigned int @nr argument is intentional as it creates denser code */ 37 static noinline long x64_sys_call(const struct pt_regs *regs, unsigned int nr) 38 { 39 switch (nr) { 40 #include <asm/syscalls_64.h> 41 default: return __x64_sys_ni_syscall(regs); 42 } 43 } 44 45 static noinline long x32_sys_call(const struct pt_regs *regs, unsigned int nr) 46 { 47 #ifdef CONFIG_X86_X32_ABI 48 switch (nr) { 49 #include <asm/syscalls_x32.h> 50 default: return __x64_sys_ni_syscall(regs); 51 } 52 #else 53 return -ENOSYS; 54 #endif 55 } 56 57 static __always_inline bool do_syscall_x64(struct pt_regs *regs, unsigned long nr) 58 { 59 if (likely(nr < NR_syscalls)) { 60 nr = array_index_nospec(nr, NR_syscalls); 61 regs->ax = x64_sys_call(regs, (unsigned int)nr); 62 return true; 63 } 64 return false; 65 } 66 67 static __always_inline void do_syscall_x32(struct pt_regs *regs, unsigned long nr) 68 { 69 /* Adjust the starting offset of the table */ 70 nr -= __X32_SYSCALL_BIT; 71 72 if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(nr < X32_NR_syscalls)) { 73 nr = array_index_nospec(nr, X32_NR_syscalls); 74 regs->ax = x32_sys_call(regs, (unsigned int)nr); 75 } 76 } 77 78 /* Returns true to return using SYSRET, or false to use IRET */ 79 __visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr) 80 { 81 if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) { 82 instrumentation_begin(); 83 84 if (!do_syscall_x64(regs, nr)) 85 do_syscall_x32(regs, nr); 86 87 instrumentation_end(); 88 } 89 syscall_exit_to_user_mode(regs); 90 91 /* 92 * Check that the register state is valid for using SYSRET to exit 93 * to userspace. Otherwise use the slower but fully capable IRET 94 * exit path. 95 */ 96 97 /* XEN PV guests always use the IRET path */ 98 if (cpu_feature_enabled(X86_FEATURE_XENPV)) 99 return false; 100 101 /* SYSRET requires RCX == RIP and R11 == EFLAGS */ 102 if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags)) 103 return false; 104 105 /* CS and SS must match the values set in MSR_STAR */ 106 if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS)) 107 return false; 108 109 /* 110 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP 111 * in kernel space. This essentially lets the user take over 112 * the kernel, since userspace controls RSP. 113 * 114 * TASK_SIZE_MAX covers all user-accessible addresses other than 115 * the deprecated vsyscall page. 116 */ 117 if (unlikely(regs->ip >= TASK_SIZE_MAX)) 118 return false; 119 120 /* 121 * SYSRET cannot restore RF. It can restore TF, but unlike IRET, 122 * restoring TF results in a trap from userspace immediately after 123 * SYSRET. 124 */ 125 if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF))) 126 return false; 127 128 /* Use SYSRET to exit to userspace */ 129 return true; 130 } 131