1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024 Rivos, Inc. 4 * Deepak Gupta <debug@rivosinc.com> 5 */ 6 7 #include <linux/sched.h> 8 #include <linux/bitops.h> 9 #include <linux/types.h> 10 #include <linux/mm.h> 11 #include <linux/mman.h> 12 #include <linux/uaccess.h> 13 #include <linux/sizes.h> 14 #include <linux/user.h> 15 #include <linux/syscalls.h> 16 #include <linux/prctl.h> 17 #include <asm/csr.h> 18 #include <asm/usercfi.h> 19 20 #define SHSTK_ENTRY_SIZE sizeof(void *) 21 22 bool is_shstk_enabled(struct task_struct *task) 23 { 24 return task->thread_info.user_cfi_state.ubcfi_en; 25 } 26 27 void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size) 28 { 29 task->thread_info.user_cfi_state.shdw_stk_base = shstk_addr; 30 task->thread_info.user_cfi_state.shdw_stk_size = size; 31 } 32 33 unsigned long get_shstk_base(struct task_struct *task, unsigned long *size) 34 { 35 if (size) 36 *size = task->thread_info.user_cfi_state.shdw_stk_size; 37 return task->thread_info.user_cfi_state.shdw_stk_base; 38 } 39 40 void set_active_shstk(struct task_struct *task, unsigned long shstk_addr) 41 { 42 task->thread_info.user_cfi_state.user_shdw_stk = shstk_addr; 43 } 44 45 /* 46 * If size is 0, then to be compatible with regular stack we want it to be as big as 47 * regular stack. Else PAGE_ALIGN it and return back 48 */ 49 static unsigned long calc_shstk_size(unsigned long size) 50 { 51 if (size) 52 return PAGE_ALIGN(size); 53 54 return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G)); 55 } 56 57 /* 58 * Writes on shadow stack can either be `sspush` or `ssamoswap`. `sspush` can happen 59 * implicitly on current shadow stack pointed to by CSR_SSP. `ssamoswap` takes pointer to 60 * shadow stack. To keep it simple, we plan to use `ssamoswap` to perform writes on shadow 61 * stack. 62 */ 63 static noinline unsigned long amo_user_shstk(unsigned long __user *addr, unsigned long val) 64 { 65 /* 66 * Never expect -1 on shadow stack. Expect return addresses and zero 67 */ 68 unsigned long swap = -1; 69 70 __enable_user_access(); 71 asm goto(".option push\n" 72 ".option arch, +zicfiss\n" 73 "1: ssamoswap.d %[swap], %[val], %[addr]\n" 74 _ASM_EXTABLE(1b, %l[fault]) 75 ".option pop\n" 76 : [swap] "=r" (swap), [addr] "+A" (*(__force unsigned long *)addr) 77 : [val] "r" (val) 78 : "memory" 79 : fault 80 ); 81 __disable_user_access(); 82 return swap; 83 fault: 84 __disable_user_access(); 85 return -1; 86 } 87 88 /* 89 * Create a restore token on the shadow stack. A token is always XLEN wide 90 * and aligned to XLEN. 91 */ 92 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr) 93 { 94 unsigned long addr; 95 96 /* Token must be aligned */ 97 if (!IS_ALIGNED(ssp, SHSTK_ENTRY_SIZE)) 98 return -EINVAL; 99 100 /* On RISC-V we're constructing token to be function of address itself */ 101 addr = ssp - SHSTK_ENTRY_SIZE; 102 103 if (amo_user_shstk((unsigned long __user *)addr, (unsigned long)ssp) == -1) 104 return -EFAULT; 105 106 if (token_addr) 107 *token_addr = addr; 108 109 return 0; 110 } 111 112 static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size, 113 unsigned long token_offset, bool set_tok) 114 { 115 int flags = MAP_ANONYMOUS | MAP_PRIVATE; 116 struct mm_struct *mm = current->mm; 117 unsigned long populate; 118 119 if (addr) 120 flags |= MAP_FIXED_NOREPLACE; 121 122 mmap_write_lock(mm); 123 addr = do_mmap(NULL, addr, size, PROT_READ, flags, 124 VM_SHADOW_STACK | VM_WRITE, 0, &populate, NULL); 125 mmap_write_unlock(mm); 126 127 if (!set_tok || IS_ERR_VALUE(addr)) 128 goto out; 129 130 if (create_rstor_token(addr + token_offset, NULL)) { 131 vm_munmap(addr, size); 132 return -EINVAL; 133 } 134 135 out: 136 return addr; 137 } 138 139 SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags) 140 { 141 bool set_tok = flags & SHADOW_STACK_SET_TOKEN; 142 unsigned long aligned_size = 0; 143 144 if (!cpu_supports_shadow_stack()) 145 return -EOPNOTSUPP; 146 147 /* Anything other than set token should result in invalid param */ 148 if (flags & ~SHADOW_STACK_SET_TOKEN) 149 return -EINVAL; 150 151 /* 152 * Unlike other architectures, on RISC-V, SSP pointer is held in CSR_SSP and is an available 153 * CSR in all modes. CSR accesses are performed using 12bit index programmed in instruction 154 * itself. This provides static property on register programming and writes to CSR can't 155 * be unintentional from programmer's perspective. As long as programmer has guarded areas 156 * which perform writes to CSR_SSP properly, shadow stack pivoting is not possible. Since 157 * CSR_SSP is writable by user mode, it itself can setup a shadow stack token subsequent 158 * to allocation. Although in order to provide portablity with other architectures (because 159 * `map_shadow_stack` is arch agnostic syscall), RISC-V will follow expectation of a token 160 * flag in flags and if provided in flags, will setup a token at the base. 161 */ 162 163 /* If there isn't space for a token */ 164 if (set_tok && size < SHSTK_ENTRY_SIZE) 165 return -ENOSPC; 166 167 if (addr && (addr & (PAGE_SIZE - 1))) 168 return -EINVAL; 169 170 aligned_size = PAGE_ALIGN(size); 171 if (aligned_size < size) 172 return -EOVERFLOW; 173 174 return allocate_shadow_stack(addr, aligned_size, size, set_tok); 175 } 176 177 /* 178 * This gets called during clone/clone3/fork. And is needed to allocate a shadow stack for 179 * cases where CLONE_VM is specified and thus a different stack is specified by user. We 180 * thus need a separate shadow stack too. How a separate shadow stack is specified by 181 * user is still being debated. Once that's settled, remove this part of the comment. 182 * This function simply returns 0 if shadow stacks are not supported or if separate shadow 183 * stack allocation is not needed (like in case of !CLONE_VM) 184 */ 185 unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, 186 const struct kernel_clone_args *args) 187 { 188 unsigned long addr, size; 189 190 /* If shadow stack is not supported, return 0 */ 191 if (!cpu_supports_shadow_stack()) 192 return 0; 193 194 /* 195 * If shadow stack is not enabled on the new thread, skip any 196 * switch to a new shadow stack. 197 */ 198 if (!is_shstk_enabled(tsk)) 199 return 0; 200 201 /* 202 * For CLONE_VFORK the child will share the parents shadow stack. 203 * Set base = 0 and size = 0, this is special means to track this state 204 * so the freeing logic run for child knows to leave it alone. 205 */ 206 if (args->flags & CLONE_VFORK) { 207 set_shstk_base(tsk, 0, 0); 208 return 0; 209 } 210 211 /* 212 * For !CLONE_VM the child will use a copy of the parents shadow 213 * stack. 214 */ 215 if (!(args->flags & CLONE_VM)) 216 return 0; 217 218 /* 219 * reaching here means, CLONE_VM was specified and thus a separate shadow 220 * stack is needed for new cloned thread. Note: below allocation is happening 221 * using current mm. 222 */ 223 size = calc_shstk_size(args->stack_size); 224 addr = allocate_shadow_stack(0, size, 0, false); 225 if (IS_ERR_VALUE(addr)) 226 return addr; 227 228 set_shstk_base(tsk, addr, size); 229 230 return addr + size; 231 } 232 233 void shstk_release(struct task_struct *tsk) 234 { 235 unsigned long base = 0, size = 0; 236 /* If shadow stack is not supported or not enabled, nothing to release */ 237 if (!cpu_supports_shadow_stack() || !is_shstk_enabled(tsk)) 238 return; 239 240 /* 241 * When fork() with CLONE_VM fails, the child (tsk) already has a 242 * shadow stack allocated, and exit_thread() calls this function to 243 * free it. In this case the parent (current) and the child share 244 * the same mm struct. Move forward only when they're same. 245 */ 246 if (!tsk->mm || tsk->mm != current->mm) 247 return; 248 249 /* 250 * We know shadow stack is enabled but if base is NULL, then 251 * this task is not managing its own shadow stack (CLONE_VFORK). So 252 * skip freeing it. 253 */ 254 base = get_shstk_base(tsk, &size); 255 if (!base) 256 return; 257 258 vm_munmap(base, size); 259 set_shstk_base(tsk, 0, 0); 260 } 261