1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024 Rivos, Inc. 4 * Deepak Gupta <debug@rivosinc.com> 5 */ 6 7 #include <linux/sched.h> 8 #include <linux/bitops.h> 9 #include <linux/types.h> 10 #include <linux/mm.h> 11 #include <linux/mman.h> 12 #include <linux/uaccess.h> 13 #include <linux/sizes.h> 14 #include <linux/user.h> 15 #include <linux/syscalls.h> 16 #include <linux/prctl.h> 17 #include <asm/csr.h> 18 #include <asm/usercfi.h> 19 20 #define SHSTK_ENTRY_SIZE sizeof(void *) 21 22 /* 23 * Writes on shadow stack can either be `sspush` or `ssamoswap`. `sspush` can happen 24 * implicitly on current shadow stack pointed to by CSR_SSP. `ssamoswap` takes pointer to 25 * shadow stack. To keep it simple, we plan to use `ssamoswap` to perform writes on shadow 26 * stack. 27 */ 28 static noinline unsigned long amo_user_shstk(unsigned long __user *addr, unsigned long val) 29 { 30 /* 31 * Never expect -1 on shadow stack. Expect return addresses and zero 32 */ 33 unsigned long swap = -1; 34 35 __enable_user_access(); 36 asm goto(".option push\n" 37 ".option arch, +zicfiss\n" 38 "1: ssamoswap.d %[swap], %[val], %[addr]\n" 39 _ASM_EXTABLE(1b, %l[fault]) 40 ".option pop\n" 41 : [swap] "=r" (swap), [addr] "+A" (*(__force unsigned long *)addr) 42 : [val] "r" (val) 43 : "memory" 44 : fault 45 ); 46 __disable_user_access(); 47 return swap; 48 fault: 49 __disable_user_access(); 50 return -1; 51 } 52 53 /* 54 * Create a restore token on the shadow stack. A token is always XLEN wide 55 * and aligned to XLEN. 56 */ 57 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr) 58 { 59 unsigned long addr; 60 61 /* Token must be aligned */ 62 if (!IS_ALIGNED(ssp, SHSTK_ENTRY_SIZE)) 63 return -EINVAL; 64 65 /* On RISC-V we're constructing token to be function of address itself */ 66 addr = ssp - SHSTK_ENTRY_SIZE; 67 68 if (amo_user_shstk((unsigned long __user *)addr, (unsigned long)ssp) == -1) 69 return -EFAULT; 70 71 if (token_addr) 72 *token_addr = addr; 73 74 return 0; 75 } 76 77 static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size, 78 unsigned long token_offset, bool set_tok) 79 { 80 int flags = MAP_ANONYMOUS | MAP_PRIVATE; 81 struct mm_struct *mm = current->mm; 82 unsigned long populate; 83 84 if (addr) 85 flags |= MAP_FIXED_NOREPLACE; 86 87 mmap_write_lock(mm); 88 addr = do_mmap(NULL, addr, size, PROT_READ, flags, 89 VM_SHADOW_STACK | VM_WRITE, 0, &populate, NULL); 90 mmap_write_unlock(mm); 91 92 if (!set_tok || IS_ERR_VALUE(addr)) 93 goto out; 94 95 if (create_rstor_token(addr + token_offset, NULL)) { 96 vm_munmap(addr, size); 97 return -EINVAL; 98 } 99 100 out: 101 return addr; 102 } 103 104 SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags) 105 { 106 bool set_tok = flags & SHADOW_STACK_SET_TOKEN; 107 unsigned long aligned_size = 0; 108 109 if (!cpu_supports_shadow_stack()) 110 return -EOPNOTSUPP; 111 112 /* Anything other than set token should result in invalid param */ 113 if (flags & ~SHADOW_STACK_SET_TOKEN) 114 return -EINVAL; 115 116 /* 117 * Unlike other architectures, on RISC-V, SSP pointer is held in CSR_SSP and is an available 118 * CSR in all modes. CSR accesses are performed using 12bit index programmed in instruction 119 * itself. This provides static property on register programming and writes to CSR can't 120 * be unintentional from programmer's perspective. As long as programmer has guarded areas 121 * which perform writes to CSR_SSP properly, shadow stack pivoting is not possible. Since 122 * CSR_SSP is writable by user mode, it itself can setup a shadow stack token subsequent 123 * to allocation. Although in order to provide portablity with other architectures (because 124 * `map_shadow_stack` is arch agnostic syscall), RISC-V will follow expectation of a token 125 * flag in flags and if provided in flags, will setup a token at the base. 126 */ 127 128 /* If there isn't space for a token */ 129 if (set_tok && size < SHSTK_ENTRY_SIZE) 130 return -ENOSPC; 131 132 if (addr && (addr & (PAGE_SIZE - 1))) 133 return -EINVAL; 134 135 aligned_size = PAGE_ALIGN(size); 136 if (aligned_size < size) 137 return -EOVERFLOW; 138 139 return allocate_shadow_stack(addr, aligned_size, size, set_tok); 140 } 141