1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 32-bit system call dispatch */ 3 4 #include <linux/linkage.h> 5 #include <linux/sys.h> 6 #include <linux/cache.h> 7 #include <linux/syscalls.h> 8 #include <linux/entry-common.h> 9 #include <linux/nospec.h> 10 #include <linux/uaccess.h> 11 #include <asm/apic.h> 12 #include <asm/traps.h> 13 #include <asm/cpufeature.h> 14 #include <asm/syscall.h> 15 16 #ifdef CONFIG_IA32_EMULATION 17 #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) 18 #else 19 #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) 20 #endif 21 22 #define __SYSCALL(nr, sym) extern long __ia32_##sym(const struct pt_regs *); 23 #define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __ia32_##sym(const struct pt_regs *); 24 #include <asm/syscalls_32.h> 25 #undef __SYSCALL 26 27 #undef __SYSCALL_NORETURN 28 #define __SYSCALL_NORETURN __SYSCALL 29 30 /* 31 * The sys_call_table[] is no longer used for system calls, but 32 * kernel/trace/trace_syscalls.c still wants to know the system 33 * call address. 34 */ 35 #ifdef CONFIG_X86_32 36 #define __SYSCALL(nr, sym) __ia32_##sym, 37 const sys_call_ptr_t sys_call_table[] = { 38 #include <asm/syscalls_32.h> 39 }; 40 #undef __SYSCALL 41 #endif 42 43 #define __SYSCALL(nr, sym) case nr: return __ia32_##sym(regs); 44 45 /* The unsigned int @nr argument is intentional as it creates denser code in a 64-bit build */ 46 static noinline long ia32_sys_call(const struct pt_regs *regs, unsigned int nr) 47 { 48 switch (nr) { 49 #include <asm/syscalls_32.h> 50 default: return __ia32_sys_ni_syscall(regs); 51 } 52 } 53 54 static __always_inline long syscall_32_enter(struct pt_regs *regs) 55 { 56 if (IS_ENABLED(CONFIG_IA32_EMULATION)) 57 current_thread_info()->status |= TS_COMPAT; 58 59 return (int)regs->orig_ax; 60 } 61 62 #ifdef CONFIG_IA32_EMULATION 63 bool __ia32_enabled __ro_after_init = !IS_ENABLED(CONFIG_IA32_EMULATION_DEFAULT_DISABLED); 64 65 static int __init ia32_emulation_override_cmdline(char *arg) 66 { 67 return kstrtobool(arg, &__ia32_enabled); 68 } 69 early_param("ia32_emulation", ia32_emulation_override_cmdline); 70 #endif 71 72 /* 73 * Invoke a 32-bit syscall. Called with IRQs on in CT_STATE_KERNEL. 74 */ 75 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, unsigned long nr) 76 { 77 if (likely(nr < IA32_NR_syscalls)) { 78 nr = array_index_nospec(nr, IA32_NR_syscalls); 79 regs->ax = ia32_sys_call(regs, (unsigned int)nr); 80 } 81 } 82 83 #ifdef CONFIG_IA32_EMULATION 84 static __always_inline bool int80_is_external(void) 85 { 86 const unsigned int offs = (0x80 / 32) * 0x10; 87 const u32 bit = BIT(0x80 % 32); 88 89 /* The local APIC on XENPV guests is fake */ 90 if (cpu_feature_enabled(X86_FEATURE_XENPV)) 91 return false; 92 93 /* 94 * If vector 0x80 is set in the APIC ISR then this is an external 95 * interrupt. Either from broken hardware or injected by a VMM. 96 * 97 * Note: In guest mode this is only valid for secure guests where 98 * the secure module fully controls the vAPIC exposed to the guest. 99 */ 100 return apic_read(APIC_ISR + offs) & bit; 101 } 102 103 /** 104 * do_int80_emulation - 32-bit legacy syscall C entry from asm 105 * @regs: syscall arguments in struct pt_args on the stack. 106 * 107 * This entry point can be used by 32-bit and 64-bit programs to perform 108 * 32-bit system calls. Instances of INT $0x80 can be found inline in 109 * various programs and libraries. It is also used by the vDSO's 110 * __kernel_vsyscall fallback for hardware that doesn't support a faster 111 * entry method. Restarted 32-bit system calls also fall back to INT 112 * $0x80 regardless of what instruction was originally used to do the 113 * system call. 114 * 115 * This is considered a slow path. It is not used by most libc 116 * implementations on modern hardware except during process startup. 117 * 118 * The arguments for the INT $0x80 based syscall are on stack in the 119 * pt_regs structure: 120 * eax: system call number 121 * ebx, ecx, edx, esi, edi, ebp: arg1 - arg 6 122 */ 123 __visible noinstr void do_int80_emulation(struct pt_regs *regs) 124 { 125 long nr; 126 127 /* Kernel does not use INT $0x80! */ 128 if (unlikely(!user_mode(regs))) { 129 irqentry_enter(regs); 130 instrumentation_begin(); 131 panic("Unexpected external interrupt 0x80\n"); 132 } 133 134 /* 135 * Establish kernel context for instrumentation, including for 136 * int80_is_external() below which calls into the APIC driver. 137 * Identical for soft and external interrupts. 138 */ 139 enter_from_user_mode_randomize_stack(regs); 140 141 instrumentation_begin(); 142 143 /* Validate that this is a soft interrupt to the extent possible */ 144 if (unlikely(int80_is_external())) 145 panic("Unexpected external interrupt 0x80\n"); 146 147 /* 148 * The low level idtentry code pushed -1 into regs::orig_ax 149 * and regs::ax contains the syscall number. 150 * 151 * User tracing code (ptrace or signal handlers) might assume 152 * that the regs::orig_ax contains a 32-bit number on invoking 153 * a 32-bit syscall. 154 * 155 * Establish the syscall convention by saving the 32bit truncated 156 * syscall number in regs::orig_ax and by invalidating regs::ax. 157 */ 158 regs->orig_ax = regs->ax & GENMASK(31, 0); 159 regs->ax = -ENOSYS; 160 161 nr = syscall_32_enter(regs); 162 163 local_irq_enable(); 164 165 if (likely(syscall_enter_from_user_mode_work(regs, &nr))) 166 do_syscall_32_irqs_on(regs, nr); 167 168 instrumentation_end(); 169 syscall_exit_to_user_mode(regs); 170 } 171 172 #ifdef CONFIG_X86_FRED 173 /* 174 * A FRED-specific INT80 handler is warranted for the follwing reasons: 175 * 176 * 1) As INT instructions and hardware interrupts are separate event 177 * types, FRED does not preclude the use of vector 0x80 for external 178 * interrupts. As a result, the FRED setup code does not reserve 179 * vector 0x80 and calling int80_is_external() is not merely 180 * suboptimal but actively incorrect: it could cause a system call 181 * to be incorrectly ignored. 182 * 183 * 2) It is called only for handling vector 0x80 of event type 184 * EVENT_TYPE_SWINT and will never be called to handle any external 185 * interrupt (event type EVENT_TYPE_EXTINT). 186 * 187 * 3) FRED has separate entry flows depending on if the event came from 188 * user space or kernel space, and because the kernel does not use 189 * INT insns, the FRED kernel entry handler fred_entry_from_kernel() 190 * falls through to fred_bad_type() if the event type is 191 * EVENT_TYPE_SWINT, i.e., INT insns. So if the kernel is handling 192 * an INT insn, it can only be from a user level. 193 * 194 * 4) int80_emulation() does a CLEAR_BRANCH_HISTORY. While FRED will 195 * likely take a different approach if it is ever needed: it 196 * probably belongs in either fred_intx()/ fred_other() or 197 * asm_fred_entrypoint_user(), depending on if this ought to be done 198 * for all entries from userspace or only system 199 * calls. 200 * 201 * 5) INT $0x80 is the fast path for 32-bit system calls under FRED. 202 */ 203 DEFINE_FREDENTRY_RAW(int80_emulation) 204 { 205 long nr; 206 207 enter_from_user_mode_randomize_stack(regs); 208 209 instrumentation_begin(); 210 /* 211 * FRED pushed 0 into regs::orig_ax and regs::ax contains the 212 * syscall number. 213 * 214 * User tracing code (ptrace or signal handlers) might assume 215 * that the regs::orig_ax contains a 32-bit number on invoking 216 * a 32-bit syscall. 217 * 218 * Establish the syscall convention by saving the 32bit truncated 219 * syscall number in regs::orig_ax and by invalidating regs::ax. 220 */ 221 regs->orig_ax = regs->ax & GENMASK(31, 0); 222 regs->ax = -ENOSYS; 223 224 nr = syscall_32_enter(regs); 225 226 local_irq_enable(); 227 if (likely(syscall_enter_from_user_mode_work(regs, &nr))) 228 do_syscall_32_irqs_on(regs, nr); 229 230 instrumentation_end(); 231 syscall_exit_to_user_mode(regs); 232 } 233 #endif /* CONFIG_X86_FRED */ 234 235 #else /* CONFIG_IA32_EMULATION */ 236 237 /* Handles int $0x80 on a 32bit kernel */ 238 __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) 239 { 240 long nr = syscall_32_enter(regs); 241 242 /* 243 * Subtlety here: if ptrace pokes something larger than 2^31-1 into 244 * orig_ax, the int return value truncates it. This matches 245 * the semantics of syscall_get_nr(). 246 */ 247 if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) { 248 instrumentation_begin(); 249 250 do_syscall_32_irqs_on(regs, nr); 251 252 instrumentation_end(); 253 } 254 syscall_exit_to_user_mode(regs); 255 } 256 #endif /* !CONFIG_IA32_EMULATION */ 257 258 static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) 259 { 260 long nr = syscall_32_enter(regs); 261 int res; 262 263 enter_from_user_mode_randomize_stack(regs); 264 265 instrumentation_begin(); 266 local_irq_enable(); 267 /* Fetch EBP from where the vDSO stashed it. */ 268 if (IS_ENABLED(CONFIG_X86_64)) { 269 /* 270 * Micro-optimization: the pointer we're following is 271 * explicitly 32 bits, so it can't be out of range. 272 */ 273 res = __get_user(*(u32 *)®s->bp, 274 (u32 __user __force *)(unsigned long)(u32)regs->sp); 275 } else { 276 res = get_user(*(u32 *)®s->bp, 277 (u32 __user __force *)(unsigned long)(u32)regs->sp); 278 } 279 280 if (res) { 281 /* User code screwed up. */ 282 regs->ax = -EFAULT; 283 284 local_irq_disable(); 285 instrumentation_end(); 286 irqentry_exit_to_user_mode(regs); 287 return false; 288 } 289 290 if (likely(syscall_enter_from_user_mode_work(regs, &nr))) 291 do_syscall_32_irqs_on(regs, nr); 292 293 instrumentation_end(); 294 syscall_exit_to_user_mode(regs); 295 return true; 296 } 297 298 /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */ 299 __visible noinstr bool do_fast_syscall_32(struct pt_regs *regs) 300 { 301 /* 302 * Called using the internal vDSO SYSENTER/SYSCALL32 calling 303 * convention. Adjust regs so it looks like we entered using int80. 304 */ 305 unsigned long landing_pad = (unsigned long)current->mm->context.vdso + 306 vdso32_image.sym_int80_landing_pad; 307 308 /* 309 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward 310 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction. 311 * Fix it up. 312 */ 313 regs->ip = landing_pad; 314 315 /* Invoke the syscall. If it failed, keep it simple: use IRET. */ 316 if (!__do_fast_syscall_32(regs)) 317 return false; 318 319 /* 320 * Check that the register state is valid for using SYSRETL/SYSEXIT 321 * to exit to userspace. Otherwise use the slower but fully capable 322 * IRET exit path. 323 */ 324 325 /* XEN PV guests always use the IRET path */ 326 if (cpu_feature_enabled(X86_FEATURE_XENPV)) 327 return false; 328 329 /* EIP must point to the VDSO landing pad */ 330 if (unlikely(regs->ip != landing_pad)) 331 return false; 332 333 /* CS and SS must match the values set in MSR_STAR */ 334 if (unlikely(regs->cs != __USER32_CS || regs->ss != __USER_DS)) 335 return false; 336 337 /* If the TF, RF, or VM flags are set, use IRET */ 338 if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM))) 339 return false; 340 341 /* Use SYSRETL/SYSEXIT to exit to userspace */ 342 return true; 343 } 344 345 /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */ 346 __visible noinstr bool do_SYSENTER_32(struct pt_regs *regs) 347 { 348 /* SYSENTER loses RSP, but the vDSO saved it in RBP. */ 349 regs->sp = regs->bp; 350 351 /* SYSENTER clobbers EFLAGS.IF. Assume it was set in usermode. */ 352 regs->flags |= X86_EFLAGS_IF; 353 354 return do_fast_syscall_32(regs); 355 } 356