1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2009 Sunplus Core Technology Co., Ltd. 4 * Chen Liqin <liqin.chen@sunplusct.com> 5 * Lennox Wu <lennox.wu@sunplusct.com> 6 * Copyright (C) 2012 Regents of the University of California 7 * Copyright (C) 2017 SiFive 8 */ 9 10 #include <linux/bitfield.h> 11 #include <linux/cpu.h> 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/sched/debug.h> 15 #include <linux/sched/task_stack.h> 16 #include <linux/tick.h> 17 #include <linux/ptrace.h> 18 #include <linux/uaccess.h> 19 #include <linux/personality.h> 20 #include <linux/entry-common.h> 21 22 #include <asm/asm-prototypes.h> 23 #include <asm/unistd.h> 24 #include <asm/processor.h> 25 #include <asm/csr.h> 26 #include <asm/stacktrace.h> 27 #include <asm/string.h> 28 #include <asm/switch_to.h> 29 #include <asm/thread_info.h> 30 #include <asm/cpuidle.h> 31 #include <asm/vector.h> 32 #include <asm/cpufeature.h> 33 #include <asm/exec.h> 34 #include <asm/usercfi.h> 35 36 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) 37 #include <linux/stackprotector.h> 38 unsigned long __stack_chk_guard __read_mostly; 39 EXPORT_SYMBOL(__stack_chk_guard); 40 #endif 41 42 extern asmlinkage void ret_from_fork_kernel_asm(void); 43 extern asmlinkage void ret_from_fork_user_asm(void); 44 45 void noinstr arch_cpu_idle(void) 46 { 47 cpu_do_idle(); 48 } 49 50 int set_unalign_ctl(struct task_struct *tsk, unsigned int val) 51 { 52 if (!unaligned_ctl_available()) 53 return -EINVAL; 54 55 tsk->thread.align_ctl = val; 56 return 0; 57 } 58 59 int get_unalign_ctl(struct task_struct *tsk, unsigned long adr) 60 { 61 if (!unaligned_ctl_available()) 62 return -EINVAL; 63 64 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr); 65 } 66 67 void __show_regs(struct pt_regs *regs) 68 { 69 show_regs_print_info(KERN_DEFAULT); 70 71 if (!user_mode(regs)) { 72 pr_cont("epc : %pS\n", (void *)regs->epc); 73 pr_cont(" ra : %pS\n", (void *)regs->ra); 74 } 75 76 pr_cont("epc : " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", 77 regs->epc, regs->ra, regs->sp); 78 pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n", 79 regs->gp, regs->tp, regs->t0); 80 pr_cont(" t1 : " REG_FMT " t2 : " REG_FMT " s0 : " REG_FMT "\n", 81 regs->t1, regs->t2, regs->s0); 82 pr_cont(" s1 : " REG_FMT " a0 : " REG_FMT " a1 : " REG_FMT "\n", 83 regs->s1, regs->a0, regs->a1); 84 pr_cont(" a2 : " REG_FMT " a3 : " REG_FMT " a4 : " REG_FMT "\n", 85 regs->a2, regs->a3, regs->a4); 86 pr_cont(" a5 : " REG_FMT " a6 : " REG_FMT " a7 : " REG_FMT "\n", 87 regs->a5, regs->a6, regs->a7); 88 pr_cont(" s2 : " REG_FMT " s3 : " REG_FMT " s4 : " REG_FMT "\n", 89 regs->s2, regs->s3, regs->s4); 90 pr_cont(" s5 : " REG_FMT " s6 : " REG_FMT " s7 : " REG_FMT "\n", 91 regs->s5, regs->s6, regs->s7); 92 pr_cont(" s8 : " REG_FMT " s9 : " REG_FMT " s10: " REG_FMT "\n", 93 regs->s8, regs->s9, regs->s10); 94 pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n", 95 regs->s11, regs->t3, regs->t4); 96 pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT " ssp : " REG_FMT "\n", 97 regs->t5, regs->t6, get_active_shstk(current)); 98 99 pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", 100 regs->status, regs->badaddr, regs->cause); 101 } 102 void show_regs(struct pt_regs *regs) 103 { 104 __show_regs(regs); 105 if (!user_mode(regs)) 106 dump_backtrace(regs, NULL, KERN_DEFAULT); 107 } 108 109 unsigned long arch_align_stack(unsigned long sp) 110 { 111 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) 112 sp -= get_random_u32_below(PAGE_SIZE); 113 return sp & ~0xf; 114 } 115 116 #ifdef CONFIG_COMPAT 117 static bool compat_mode_supported __read_mostly; 118 119 bool compat_elf_check_arch(Elf32_Ehdr *hdr) 120 { 121 return compat_mode_supported && 122 hdr->e_machine == EM_RISCV && 123 hdr->e_ident[EI_CLASS] == ELFCLASS32; 124 } 125 126 static int __init compat_mode_detect(void) 127 { 128 unsigned long tmp = csr_read(CSR_STATUS); 129 130 csr_write(CSR_STATUS, (tmp & ~SR_UXL) | SR_UXL_32); 131 compat_mode_supported = 132 (csr_read(CSR_STATUS) & SR_UXL) == SR_UXL_32; 133 134 csr_write(CSR_STATUS, tmp); 135 136 pr_info("riscv: ELF compat mode %s", 137 compat_mode_supported ? "supported" : "unsupported"); 138 139 return 0; 140 } 141 early_initcall(compat_mode_detect); 142 #endif 143 144 void start_thread(struct pt_regs *regs, unsigned long pc, 145 unsigned long sp) 146 { 147 regs->status = SR_PIE; 148 if (has_fpu()) { 149 regs->status |= SR_FS_INITIAL; 150 /* 151 * Restore the initial value to the FP register 152 * before starting the user program. 153 */ 154 fstate_restore(current, regs); 155 } 156 regs->epc = pc; 157 regs->sp = sp; 158 159 /* 160 * clear shadow stack state on exec. 161 * libc will set it later via prctl. 162 */ 163 set_shstk_status(current, false); 164 set_shstk_base(current, 0, 0); 165 set_active_shstk(current, 0); 166 /* 167 * disable indirect branch tracking on exec. 168 * libc will enable it later via prctl. 169 */ 170 set_indir_lp_status(current, false); 171 172 #ifdef CONFIG_64BIT 173 regs->status &= ~SR_UXL; 174 175 if (is_compat_task()) 176 regs->status |= SR_UXL_32; 177 else 178 regs->status |= SR_UXL_64; 179 #endif 180 } 181 182 void flush_thread(void) 183 { 184 #ifdef CONFIG_FPU 185 /* 186 * Reset FPU state and context 187 * frm: round to nearest, ties to even (IEEE default) 188 * fflags: accrued exceptions cleared 189 */ 190 fstate_off(current, task_pt_regs(current)); 191 memset(¤t->thread.fstate, 0, sizeof(current->thread.fstate)); 192 #endif 193 #ifdef CONFIG_RISCV_ISA_V 194 /* Reset vector state */ 195 riscv_v_vstate_ctrl_init(current); 196 riscv_v_vstate_off(task_pt_regs(current)); 197 kfree(current->thread.vstate.datap); 198 memset(¤t->thread.vstate, 0, sizeof(struct __riscv_v_ext_state)); 199 clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE); 200 #endif 201 #ifdef CONFIG_RISCV_ISA_SUPM 202 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) 203 envcfg_update_bits(current, ENVCFG_PMM, ENVCFG_PMM_PMLEN_0); 204 #endif 205 } 206 207 void arch_release_task_struct(struct task_struct *tsk) 208 { 209 /* Free the vector context of datap. */ 210 if (has_vector() || has_xtheadvector()) 211 riscv_v_thread_free(tsk); 212 } 213 214 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) 215 { 216 fstate_save(src, task_pt_regs(src)); 217 *dst = *src; 218 /* clear entire V context, including datap for a new task */ 219 memset(&dst->thread.vstate, 0, sizeof(struct __riscv_v_ext_state)); 220 memset(&dst->thread.kernel_vstate, 0, sizeof(struct __riscv_v_ext_state)); 221 clear_tsk_thread_flag(dst, TIF_RISCV_V_DEFER_RESTORE); 222 223 return 0; 224 } 225 226 asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs) 227 { 228 fn(fn_arg); 229 230 syscall_exit_to_user_mode(regs); 231 } 232 233 asmlinkage void ret_from_fork_user(struct pt_regs *regs) 234 { 235 syscall_exit_to_user_mode(regs); 236 } 237 238 int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) 239 { 240 u64 clone_flags = args->flags; 241 unsigned long usp = args->stack; 242 unsigned long tls = args->tls; 243 unsigned long ssp = 0; 244 struct pt_regs *childregs = task_pt_regs(p); 245 246 /* Ensure all threads in this mm have the same pointer masking mode. */ 247 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM) && p->mm && (clone_flags & CLONE_VM)) 248 set_bit(MM_CONTEXT_LOCK_PMLEN, &p->mm->context.flags); 249 250 memset(&p->thread.s, 0, sizeof(p->thread.s)); 251 252 /* p->thread holds context to be restored by __switch_to() */ 253 if (unlikely(args->fn)) { 254 /* Kernel thread */ 255 memset(childregs, 0, sizeof(struct pt_regs)); 256 /* Supervisor/Machine, irqs on: */ 257 childregs->status = SR_PP | SR_PIE; 258 259 p->thread.s[0] = (unsigned long)args->fn; 260 p->thread.s[1] = (unsigned long)args->fn_arg; 261 p->thread.ra = (unsigned long)ret_from_fork_kernel_asm; 262 } else { 263 /* allocate new shadow stack if needed. In case of CLONE_VM we have to */ 264 ssp = shstk_alloc_thread_stack(p, args); 265 if (IS_ERR_VALUE(ssp)) 266 return PTR_ERR((void *)ssp); 267 268 *childregs = *(current_pt_regs()); 269 /* Turn off status.VS */ 270 riscv_v_vstate_off(childregs); 271 if (usp) /* User fork */ 272 childregs->sp = usp; 273 /* if needed, set new ssp */ 274 if (ssp) 275 set_active_shstk(p, ssp); 276 if (clone_flags & CLONE_SETTLS) 277 childregs->tp = tls; 278 childregs->a0 = 0; /* Return value of fork() */ 279 p->thread.ra = (unsigned long)ret_from_fork_user_asm; 280 } 281 p->thread.riscv_v_flags = 0; 282 if (has_vector() || has_xtheadvector()) 283 riscv_v_thread_alloc(p); 284 p->thread.sp = (unsigned long)childregs; /* kernel sp */ 285 return 0; 286 } 287 288 void __init arch_task_cache_init(void) 289 { 290 riscv_v_setup_ctx_cache(); 291 } 292 293 #ifdef CONFIG_RISCV_ISA_SUPM 294 enum { 295 PMLEN_0 = 0, 296 PMLEN_7 = 7, 297 PMLEN_16 = 16, 298 }; 299 300 static bool have_user_pmlen_7; 301 static bool have_user_pmlen_16; 302 303 /* 304 * Control the relaxed ABI allowing tagged user addresses into the kernel. 305 */ 306 static unsigned int tagged_addr_disabled; 307 308 long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) 309 { 310 unsigned long valid_mask = PR_PMLEN_MASK | PR_TAGGED_ADDR_ENABLE; 311 struct thread_info *ti = task_thread_info(task); 312 struct mm_struct *mm = task->mm; 313 unsigned long pmm; 314 u8 pmlen; 315 316 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) 317 return -EINVAL; 318 319 if (is_compat_thread(ti)) 320 return -EINVAL; 321 322 if (arg & ~valid_mask) 323 return -EINVAL; 324 325 /* 326 * Prefer the smallest PMLEN that satisfies the user's request, 327 * in case choosing a larger PMLEN has a performance impact. 328 */ 329 pmlen = FIELD_GET(PR_PMLEN_MASK, arg); 330 if (pmlen == PMLEN_0) { 331 pmm = ENVCFG_PMM_PMLEN_0; 332 } else if (pmlen <= PMLEN_7 && have_user_pmlen_7) { 333 pmlen = PMLEN_7; 334 pmm = ENVCFG_PMM_PMLEN_7; 335 } else if (pmlen <= PMLEN_16 && have_user_pmlen_16) { 336 pmlen = PMLEN_16; 337 pmm = ENVCFG_PMM_PMLEN_16; 338 } else { 339 return -EINVAL; 340 } 341 342 /* 343 * Do not allow the enabling of the tagged address ABI if globally 344 * disabled via sysctl abi.tagged_addr_disabled, if pointer masking 345 * is disabled for userspace. 346 */ 347 if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) 348 return -EINVAL; 349 350 if (!(arg & PR_TAGGED_ADDR_ENABLE)) 351 pmlen = PMLEN_0; 352 353 if (mmap_write_lock_killable(mm)) 354 return -EINTR; 355 356 if (test_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags) && mm->context.pmlen != pmlen) { 357 mmap_write_unlock(mm); 358 return -EBUSY; 359 } 360 361 envcfg_update_bits(task, ENVCFG_PMM, pmm); 362 mm->context.pmlen = pmlen; 363 364 mmap_write_unlock(mm); 365 366 return 0; 367 } 368 369 long get_tagged_addr_ctrl(struct task_struct *task) 370 { 371 struct thread_info *ti = task_thread_info(task); 372 long ret = 0; 373 374 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) 375 return -EINVAL; 376 377 if (is_compat_thread(ti)) 378 return -EINVAL; 379 380 /* 381 * The mm context's pmlen is set only when the tagged address ABI is 382 * enabled, so the effective PMLEN must be extracted from envcfg.PMM. 383 */ 384 switch (task->thread.envcfg & ENVCFG_PMM) { 385 case ENVCFG_PMM_PMLEN_7: 386 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_7); 387 break; 388 case ENVCFG_PMM_PMLEN_16: 389 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_16); 390 break; 391 } 392 393 if (task->mm->context.pmlen) 394 ret |= PR_TAGGED_ADDR_ENABLE; 395 396 return ret; 397 } 398 399 static bool try_to_set_pmm(unsigned long value) 400 { 401 csr_set(CSR_ENVCFG, value); 402 return (csr_read_clear(CSR_ENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value; 403 } 404 405 /* 406 * Global sysctl to disable the tagged user addresses support. This control 407 * only prevents the tagged address ABI enabling via prctl() and does not 408 * disable it for tasks that already opted in to the relaxed ABI. 409 */ 410 411 static const struct ctl_table tagged_addr_sysctl_table[] = { 412 { 413 .procname = "tagged_addr_disabled", 414 .mode = 0644, 415 .data = &tagged_addr_disabled, 416 .maxlen = sizeof(int), 417 .proc_handler = proc_dointvec_minmax, 418 .extra1 = SYSCTL_ZERO, 419 .extra2 = SYSCTL_ONE, 420 }, 421 }; 422 423 static int __init tagged_addr_init(void) 424 { 425 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) 426 return 0; 427 428 /* 429 * envcfg.PMM is a WARL field. Detect which values are supported. 430 * Assume the supported PMLEN values are the same on all harts. 431 */ 432 csr_clear(CSR_ENVCFG, ENVCFG_PMM); 433 have_user_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7); 434 have_user_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16); 435 436 if (!register_sysctl("abi", tagged_addr_sysctl_table)) 437 return -EINVAL; 438 439 return 0; 440 } 441 core_initcall(tagged_addr_init); 442 #endif /* CONFIG_RISCV_ISA_SUPM */ 443