1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * shstk.c - Intel shadow stack support 4 * 5 * Copyright (c) 2021, Intel Corporation. 6 * Yu-cheng Yu <yu-cheng.yu@intel.com> 7 */ 8 9 #include <linux/sched.h> 10 #include <linux/bitops.h> 11 #include <linux/types.h> 12 #include <linux/mm.h> 13 #include <linux/mman.h> 14 #include <linux/slab.h> 15 #include <linux/uaccess.h> 16 #include <linux/sched/signal.h> 17 #include <linux/compat.h> 18 #include <linux/sizes.h> 19 #include <linux/user.h> 20 #include <linux/syscalls.h> 21 #include <asm/msr.h> 22 #include <asm/fpu/xstate.h> 23 #include <asm/fpu/types.h> 24 #include <asm/shstk.h> 25 #include <asm/special_insns.h> 26 #include <asm/fpu/api.h> 27 #include <asm/prctl.h> 28 29 #define SS_FRAME_SIZE 8 30 31 static bool features_enabled(unsigned long features) 32 { 33 return current->thread.features & features; 34 } 35 36 static void features_set(unsigned long features) 37 { 38 current->thread.features |= features; 39 } 40 41 static void features_clr(unsigned long features) 42 { 43 current->thread.features &= ~features; 44 } 45 46 /* 47 * Create a restore token on the shadow stack. A token is always 8-byte 48 * and aligned to 8. 49 */ 50 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr) 51 { 52 unsigned long addr; 53 54 /* Token must be aligned */ 55 if (!IS_ALIGNED(ssp, 8)) 56 return -EINVAL; 57 58 addr = ssp - SS_FRAME_SIZE; 59 60 /* 61 * SSP is aligned, so reserved bits and mode bit are a zero, just mark 62 * the token 64-bit. 63 */ 64 ssp |= BIT(0); 65 66 if (write_user_shstk_64((u64 __user *)addr, (u64)ssp)) 67 return -EFAULT; 68 69 if (token_addr) 70 *token_addr = addr; 71 72 return 0; 73 } 74 75 /* 76 * VM_SHADOW_STACK will have a guard page. This helps userspace protect 77 * itself from attacks. The reasoning is as follows: 78 * 79 * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The 80 * INCSSP instruction can increment the shadow stack pointer. It is the 81 * shadow stack analog of an instruction like: 82 * 83 * addq $0x80, %rsp 84 * 85 * However, there is one important difference between an ADD on %rsp 86 * and INCSSP. In addition to modifying SSP, INCSSP also reads from the 87 * memory of the first and last elements that were "popped". It can be 88 * thought of as acting like this: 89 * 90 * READ_ONCE(ssp); // read+discard top element on stack 91 * ssp += nr_to_pop * 8; // move the shadow stack 92 * READ_ONCE(ssp-8); // read+discard last popped stack element 93 * 94 * The maximum distance INCSSP can move the SSP is 2040 bytes, before 95 * it would read the memory. Therefore a single page gap will be enough 96 * to prevent any operation from shifting the SSP to an adjacent stack, 97 * since it would have to land in the gap at least once, causing a 98 * fault. 99 */ 100 static unsigned long alloc_shstk(unsigned long addr, unsigned long size, 101 unsigned long token_offset, bool set_res_tok) 102 { 103 int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_ABOVE4G; 104 struct mm_struct *mm = current->mm; 105 unsigned long mapped_addr, unused; 106 107 if (addr) 108 flags |= MAP_FIXED_NOREPLACE; 109 110 mmap_write_lock(mm); 111 mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags, 112 VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL); 113 mmap_write_unlock(mm); 114 115 if (!set_res_tok || IS_ERR_VALUE(mapped_addr)) 116 goto out; 117 118 if (create_rstor_token(mapped_addr + token_offset, NULL)) { 119 vm_munmap(mapped_addr, size); 120 return -EINVAL; 121 } 122 123 out: 124 return mapped_addr; 125 } 126 127 static unsigned long adjust_shstk_size(unsigned long size) 128 { 129 if (size) 130 return PAGE_ALIGN(size); 131 132 return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G)); 133 } 134 135 static void unmap_shadow_stack(u64 base, u64 size) 136 { 137 int r; 138 139 r = vm_munmap(base, size); 140 141 /* 142 * mmap_write_lock_killable() failed with -EINTR. This means 143 * the process is about to die and have it's MM cleaned up. 144 * This task shouldn't ever make it back to userspace. In this 145 * case it is ok to leak a shadow stack, so just exit out. 146 */ 147 if (r == -EINTR) 148 return; 149 150 /* 151 * For all other types of vm_munmap() failure, either the 152 * system is out of memory or there is bug. 153 */ 154 WARN_ON_ONCE(r); 155 } 156 157 static int shstk_setup(void) 158 { 159 struct thread_shstk *shstk = ¤t->thread.shstk; 160 unsigned long addr, size; 161 162 /* Already enabled */ 163 if (features_enabled(ARCH_SHSTK_SHSTK)) 164 return 0; 165 166 /* Also not supported for 32 bit and x32 */ 167 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || in_32bit_syscall()) 168 return -EOPNOTSUPP; 169 170 size = adjust_shstk_size(0); 171 addr = alloc_shstk(0, size, 0, false); 172 if (IS_ERR_VALUE(addr)) 173 return PTR_ERR((void *)addr); 174 175 fpregs_lock_and_load(); 176 wrmsrl(MSR_IA32_PL3_SSP, addr + size); 177 wrmsrl(MSR_IA32_U_CET, CET_SHSTK_EN); 178 fpregs_unlock(); 179 180 shstk->base = addr; 181 shstk->size = size; 182 features_set(ARCH_SHSTK_SHSTK); 183 184 return 0; 185 } 186 187 void reset_thread_features(void) 188 { 189 memset(¤t->thread.shstk, 0, sizeof(struct thread_shstk)); 190 current->thread.features = 0; 191 current->thread.features_locked = 0; 192 } 193 194 unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, unsigned long clone_flags, 195 unsigned long stack_size) 196 { 197 struct thread_shstk *shstk = &tsk->thread.shstk; 198 unsigned long addr, size; 199 200 /* 201 * If shadow stack is not enabled on the new thread, skip any 202 * switch to a new shadow stack. 203 */ 204 if (!features_enabled(ARCH_SHSTK_SHSTK)) 205 return 0; 206 207 /* 208 * For CLONE_VM, except vfork, the child needs a separate shadow 209 * stack. 210 */ 211 if ((clone_flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM) 212 return 0; 213 214 size = adjust_shstk_size(stack_size); 215 addr = alloc_shstk(0, size, 0, false); 216 if (IS_ERR_VALUE(addr)) 217 return addr; 218 219 shstk->base = addr; 220 shstk->size = size; 221 222 return addr + size; 223 } 224 225 static unsigned long get_user_shstk_addr(void) 226 { 227 unsigned long long ssp; 228 229 fpregs_lock_and_load(); 230 231 rdmsrl(MSR_IA32_PL3_SSP, ssp); 232 233 fpregs_unlock(); 234 235 return ssp; 236 } 237 238 #define SHSTK_DATA_BIT BIT(63) 239 240 static int put_shstk_data(u64 __user *addr, u64 data) 241 { 242 if (WARN_ON_ONCE(data & SHSTK_DATA_BIT)) 243 return -EINVAL; 244 245 /* 246 * Mark the high bit so that the sigframe can't be processed as a 247 * return address. 248 */ 249 if (write_user_shstk_64(addr, data | SHSTK_DATA_BIT)) 250 return -EFAULT; 251 return 0; 252 } 253 254 static int get_shstk_data(unsigned long *data, unsigned long __user *addr) 255 { 256 unsigned long ldata; 257 258 if (unlikely(get_user(ldata, addr))) 259 return -EFAULT; 260 261 if (!(ldata & SHSTK_DATA_BIT)) 262 return -EINVAL; 263 264 *data = ldata & ~SHSTK_DATA_BIT; 265 266 return 0; 267 } 268 269 static int shstk_push_sigframe(unsigned long *ssp) 270 { 271 unsigned long target_ssp = *ssp; 272 273 /* Token must be aligned */ 274 if (!IS_ALIGNED(target_ssp, 8)) 275 return -EINVAL; 276 277 *ssp -= SS_FRAME_SIZE; 278 if (put_shstk_data((void __user *)*ssp, target_ssp)) 279 return -EFAULT; 280 281 return 0; 282 } 283 284 static int shstk_pop_sigframe(unsigned long *ssp) 285 { 286 struct vm_area_struct *vma; 287 unsigned long token_addr; 288 bool need_to_check_vma; 289 int err = 1; 290 291 /* 292 * It is possible for the SSP to be off the end of a shadow stack by 4 293 * or 8 bytes. If the shadow stack is at the start of a page or 4 bytes 294 * before it, it might be this case, so check that the address being 295 * read is actually shadow stack. 296 */ 297 if (!IS_ALIGNED(*ssp, 8)) 298 return -EINVAL; 299 300 need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp; 301 302 if (need_to_check_vma) 303 mmap_read_lock_killable(current->mm); 304 305 err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp); 306 if (unlikely(err)) 307 goto out_err; 308 309 if (need_to_check_vma) { 310 vma = find_vma(current->mm, *ssp); 311 if (!vma || !(vma->vm_flags & VM_SHADOW_STACK)) { 312 err = -EFAULT; 313 goto out_err; 314 } 315 316 mmap_read_unlock(current->mm); 317 } 318 319 /* Restore SSP aligned? */ 320 if (unlikely(!IS_ALIGNED(token_addr, 8))) 321 return -EINVAL; 322 323 /* SSP in userspace? */ 324 if (unlikely(token_addr >= TASK_SIZE_MAX)) 325 return -EINVAL; 326 327 *ssp = token_addr; 328 329 return 0; 330 out_err: 331 if (need_to_check_vma) 332 mmap_read_unlock(current->mm); 333 return err; 334 } 335 336 int setup_signal_shadow_stack(struct ksignal *ksig) 337 { 338 void __user *restorer = ksig->ka.sa.sa_restorer; 339 unsigned long ssp; 340 int err; 341 342 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || 343 !features_enabled(ARCH_SHSTK_SHSTK)) 344 return 0; 345 346 if (!restorer) 347 return -EINVAL; 348 349 ssp = get_user_shstk_addr(); 350 if (unlikely(!ssp)) 351 return -EINVAL; 352 353 err = shstk_push_sigframe(&ssp); 354 if (unlikely(err)) 355 return err; 356 357 /* Push restorer address */ 358 ssp -= SS_FRAME_SIZE; 359 err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer); 360 if (unlikely(err)) 361 return -EFAULT; 362 363 fpregs_lock_and_load(); 364 wrmsrl(MSR_IA32_PL3_SSP, ssp); 365 fpregs_unlock(); 366 367 return 0; 368 } 369 370 int restore_signal_shadow_stack(void) 371 { 372 unsigned long ssp; 373 int err; 374 375 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || 376 !features_enabled(ARCH_SHSTK_SHSTK)) 377 return 0; 378 379 ssp = get_user_shstk_addr(); 380 if (unlikely(!ssp)) 381 return -EINVAL; 382 383 err = shstk_pop_sigframe(&ssp); 384 if (unlikely(err)) 385 return err; 386 387 fpregs_lock_and_load(); 388 wrmsrl(MSR_IA32_PL3_SSP, ssp); 389 fpregs_unlock(); 390 391 return 0; 392 } 393 394 void shstk_free(struct task_struct *tsk) 395 { 396 struct thread_shstk *shstk = &tsk->thread.shstk; 397 398 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || 399 !features_enabled(ARCH_SHSTK_SHSTK)) 400 return; 401 402 /* 403 * When fork() with CLONE_VM fails, the child (tsk) already has a 404 * shadow stack allocated, and exit_thread() calls this function to 405 * free it. In this case the parent (current) and the child share 406 * the same mm struct. 407 */ 408 if (!tsk->mm || tsk->mm != current->mm) 409 return; 410 411 unmap_shadow_stack(shstk->base, shstk->size); 412 } 413 414 static int wrss_control(bool enable) 415 { 416 u64 msrval; 417 418 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK)) 419 return -EOPNOTSUPP; 420 421 /* 422 * Only enable WRSS if shadow stack is enabled. If shadow stack is not 423 * enabled, WRSS will already be disabled, so don't bother clearing it 424 * when disabling. 425 */ 426 if (!features_enabled(ARCH_SHSTK_SHSTK)) 427 return -EPERM; 428 429 /* Already enabled/disabled? */ 430 if (features_enabled(ARCH_SHSTK_WRSS) == enable) 431 return 0; 432 433 fpregs_lock_and_load(); 434 rdmsrl(MSR_IA32_U_CET, msrval); 435 436 if (enable) { 437 features_set(ARCH_SHSTK_WRSS); 438 msrval |= CET_WRSS_EN; 439 } else { 440 features_clr(ARCH_SHSTK_WRSS); 441 if (!(msrval & CET_WRSS_EN)) 442 goto unlock; 443 444 msrval &= ~CET_WRSS_EN; 445 } 446 447 wrmsrl(MSR_IA32_U_CET, msrval); 448 449 unlock: 450 fpregs_unlock(); 451 452 return 0; 453 } 454 455 static int shstk_disable(void) 456 { 457 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK)) 458 return -EOPNOTSUPP; 459 460 /* Already disabled? */ 461 if (!features_enabled(ARCH_SHSTK_SHSTK)) 462 return 0; 463 464 fpregs_lock_and_load(); 465 /* Disable WRSS too when disabling shadow stack */ 466 wrmsrl(MSR_IA32_U_CET, 0); 467 wrmsrl(MSR_IA32_PL3_SSP, 0); 468 fpregs_unlock(); 469 470 shstk_free(current); 471 features_clr(ARCH_SHSTK_SHSTK | ARCH_SHSTK_WRSS); 472 473 return 0; 474 } 475 476 SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags) 477 { 478 bool set_tok = flags & SHADOW_STACK_SET_TOKEN; 479 unsigned long aligned_size; 480 481 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK)) 482 return -EOPNOTSUPP; 483 484 if (flags & ~SHADOW_STACK_SET_TOKEN) 485 return -EINVAL; 486 487 /* If there isn't space for a token */ 488 if (set_tok && size < 8) 489 return -ENOSPC; 490 491 if (addr && addr < SZ_4G) 492 return -ERANGE; 493 494 /* 495 * An overflow would result in attempting to write the restore token 496 * to the wrong location. Not catastrophic, but just return the right 497 * error code and block it. 498 */ 499 aligned_size = PAGE_ALIGN(size); 500 if (aligned_size < size) 501 return -EOVERFLOW; 502 503 return alloc_shstk(addr, aligned_size, size, set_tok); 504 } 505 506 long shstk_prctl(struct task_struct *task, int option, unsigned long arg2) 507 { 508 unsigned long features = arg2; 509 510 if (option == ARCH_SHSTK_STATUS) { 511 return put_user(task->thread.features, (unsigned long __user *)arg2); 512 } 513 514 if (option == ARCH_SHSTK_LOCK) { 515 task->thread.features_locked |= features; 516 return 0; 517 } 518 519 /* Only allow via ptrace */ 520 if (task != current) { 521 if (option == ARCH_SHSTK_UNLOCK && IS_ENABLED(CONFIG_CHECKPOINT_RESTORE)) { 522 task->thread.features_locked &= ~features; 523 return 0; 524 } 525 return -EINVAL; 526 } 527 528 /* Do not allow to change locked features */ 529 if (features & task->thread.features_locked) 530 return -EPERM; 531 532 /* Only support enabling/disabling one feature at a time. */ 533 if (hweight_long(features) > 1) 534 return -EINVAL; 535 536 if (option == ARCH_SHSTK_DISABLE) { 537 if (features & ARCH_SHSTK_WRSS) 538 return wrss_control(false); 539 if (features & ARCH_SHSTK_SHSTK) 540 return shstk_disable(); 541 return -EINVAL; 542 } 543 544 /* Handle ARCH_SHSTK_ENABLE */ 545 if (features & ARCH_SHSTK_SHSTK) 546 return shstk_setup(); 547 if (features & ARCH_SHSTK_WRSS) 548 return wrss_control(true); 549 return -EINVAL; 550 } 551