1fc84bc53SMark Brown // SPDX-License-Identifier: GPL-2.0-only 2fc84bc53SMark Brown 3fc84bc53SMark Brown #include <linux/mm.h> 4fc84bc53SMark Brown #include <linux/mman.h> 5fc84bc53SMark Brown #include <linux/syscalls.h> 6fc84bc53SMark Brown #include <linux/types.h> 7fc84bc53SMark Brown 8506496bcSMark Brown #include <asm/cmpxchg.h> 9fc84bc53SMark Brown #include <asm/cpufeature.h> 10506496bcSMark Brown #include <asm/gcs.h> 11fc84bc53SMark Brown #include <asm/page.h> 12fc84bc53SMark Brown 13506496bcSMark Brown static unsigned long alloc_gcs(unsigned long addr, unsigned long size) 14506496bcSMark Brown { 15506496bcSMark Brown int flags = MAP_ANONYMOUS | MAP_PRIVATE; 16506496bcSMark Brown struct mm_struct *mm = current->mm; 17506496bcSMark Brown unsigned long mapped_addr, unused; 18506496bcSMark Brown 19506496bcSMark Brown if (addr) 20506496bcSMark Brown flags |= MAP_FIXED_NOREPLACE; 21506496bcSMark Brown 22506496bcSMark Brown mmap_write_lock(mm); 23506496bcSMark Brown mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags, 24506496bcSMark Brown VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL); 25506496bcSMark Brown mmap_write_unlock(mm); 26506496bcSMark Brown 27506496bcSMark Brown return mapped_addr; 28506496bcSMark Brown } 29506496bcSMark Brown 30506496bcSMark Brown static unsigned long gcs_size(unsigned long size) 31506496bcSMark Brown { 32506496bcSMark Brown if (size) 33506496bcSMark Brown return PAGE_ALIGN(size); 34506496bcSMark Brown 35506496bcSMark Brown /* Allocate RLIMIT_STACK/2 with limits of PAGE_SIZE..2G */ 36506496bcSMark Brown size = PAGE_ALIGN(min_t(unsigned long long, 37506496bcSMark Brown rlimit(RLIMIT_STACK) / 2, SZ_2G)); 38506496bcSMark Brown return max(PAGE_SIZE, size); 39506496bcSMark Brown } 40506496bcSMark Brown 41506496bcSMark Brown unsigned long gcs_alloc_thread_stack(struct task_struct *tsk, 42506496bcSMark Brown const struct kernel_clone_args *args) 43506496bcSMark Brown { 44506496bcSMark Brown unsigned long addr, size; 45506496bcSMark Brown 46506496bcSMark Brown if (!system_supports_gcs()) 47506496bcSMark Brown return 0; 48506496bcSMark Brown 49506496bcSMark Brown if (!task_gcs_el0_enabled(tsk)) 50506496bcSMark Brown return 0; 51506496bcSMark Brown 52506496bcSMark Brown if ((args->flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM) { 53506496bcSMark Brown tsk->thread.gcspr_el0 = read_sysreg_s(SYS_GCSPR_EL0); 54506496bcSMark Brown return 0; 55506496bcSMark Brown } 56506496bcSMark Brown 57506496bcSMark Brown size = args->stack_size / 2; 58506496bcSMark Brown 59506496bcSMark Brown size = gcs_size(size); 60506496bcSMark Brown addr = alloc_gcs(0, size); 61506496bcSMark Brown if (IS_ERR_VALUE(addr)) 62506496bcSMark Brown return addr; 63506496bcSMark Brown 64506496bcSMark Brown tsk->thread.gcs_base = addr; 65506496bcSMark Brown tsk->thread.gcs_size = size; 66506496bcSMark Brown tsk->thread.gcspr_el0 = addr + size - sizeof(u64); 67506496bcSMark Brown 68506496bcSMark Brown return addr; 69506496bcSMark Brown } 70506496bcSMark Brown 71fc84bc53SMark Brown /* 72fc84bc53SMark Brown * Apply the GCS mode configured for the specified task to the 73fc84bc53SMark Brown * hardware. 74fc84bc53SMark Brown */ 75fc84bc53SMark Brown void gcs_set_el0_mode(struct task_struct *task) 76fc84bc53SMark Brown { 77fc84bc53SMark Brown u64 gcscre0_el1 = GCSCRE0_EL1_nTR; 78fc84bc53SMark Brown 79fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE) 80fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_RVCHKEN | GCSCRE0_EL1_PCRSEL; 81fc84bc53SMark Brown 82fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_WRITE) 83fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_STREn; 84fc84bc53SMark Brown 85fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_PUSH) 86fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_PUSHMEn; 87fc84bc53SMark Brown 88fc84bc53SMark Brown write_sysreg_s(gcscre0_el1, SYS_GCSCRE0_EL1); 89fc84bc53SMark Brown } 90fc84bc53SMark Brown 91fc84bc53SMark Brown void gcs_free(struct task_struct *task) 92fc84bc53SMark Brown { 93fc84bc53SMark Brown if (!system_supports_gcs()) 94fc84bc53SMark Brown return; 95fc84bc53SMark Brown 96506496bcSMark Brown /* 97506496bcSMark Brown * When fork() with CLONE_VM fails, the child (tsk) already 98506496bcSMark Brown * has a GCS allocated, and exit_thread() calls this function 99506496bcSMark Brown * to free it. In this case the parent (current) and the 100506496bcSMark Brown * child share the same mm struct. 101506496bcSMark Brown */ 102506496bcSMark Brown if (!task->mm || task->mm != current->mm) 103506496bcSMark Brown return; 104506496bcSMark Brown 105fc84bc53SMark Brown if (task->thread.gcs_base) 106fc84bc53SMark Brown vm_munmap(task->thread.gcs_base, task->thread.gcs_size); 107fc84bc53SMark Brown 108fc84bc53SMark Brown task->thread.gcspr_el0 = 0; 109fc84bc53SMark Brown task->thread.gcs_base = 0; 110fc84bc53SMark Brown task->thread.gcs_size = 0; 111fc84bc53SMark Brown } 112*b57180c7SMark Brown 113*b57180c7SMark Brown int arch_set_shadow_stack_status(struct task_struct *task, unsigned long arg) 114*b57180c7SMark Brown { 115*b57180c7SMark Brown unsigned long gcs, size; 116*b57180c7SMark Brown int ret; 117*b57180c7SMark Brown 118*b57180c7SMark Brown if (!system_supports_gcs()) 119*b57180c7SMark Brown return -EINVAL; 120*b57180c7SMark Brown 121*b57180c7SMark Brown if (is_compat_thread(task_thread_info(task))) 122*b57180c7SMark Brown return -EINVAL; 123*b57180c7SMark Brown 124*b57180c7SMark Brown /* Reject unknown flags */ 125*b57180c7SMark Brown if (arg & ~PR_SHADOW_STACK_SUPPORTED_STATUS_MASK) 126*b57180c7SMark Brown return -EINVAL; 127*b57180c7SMark Brown 128*b57180c7SMark Brown ret = gcs_check_locked(task, arg); 129*b57180c7SMark Brown if (ret != 0) 130*b57180c7SMark Brown return ret; 131*b57180c7SMark Brown 132*b57180c7SMark Brown /* If we are enabling GCS then make sure we have a stack */ 133*b57180c7SMark Brown if (arg & PR_SHADOW_STACK_ENABLE && 134*b57180c7SMark Brown !task_gcs_el0_enabled(task)) { 135*b57180c7SMark Brown /* Do not allow GCS to be reenabled */ 136*b57180c7SMark Brown if (task->thread.gcs_base || task->thread.gcspr_el0) 137*b57180c7SMark Brown return -EINVAL; 138*b57180c7SMark Brown 139*b57180c7SMark Brown if (task != current) 140*b57180c7SMark Brown return -EBUSY; 141*b57180c7SMark Brown 142*b57180c7SMark Brown size = gcs_size(0); 143*b57180c7SMark Brown gcs = alloc_gcs(0, size); 144*b57180c7SMark Brown if (!gcs) 145*b57180c7SMark Brown return -ENOMEM; 146*b57180c7SMark Brown 147*b57180c7SMark Brown task->thread.gcspr_el0 = gcs + size - sizeof(u64); 148*b57180c7SMark Brown task->thread.gcs_base = gcs; 149*b57180c7SMark Brown task->thread.gcs_size = size; 150*b57180c7SMark Brown if (task == current) 151*b57180c7SMark Brown write_sysreg_s(task->thread.gcspr_el0, 152*b57180c7SMark Brown SYS_GCSPR_EL0); 153*b57180c7SMark Brown } 154*b57180c7SMark Brown 155*b57180c7SMark Brown task->thread.gcs_el0_mode = arg; 156*b57180c7SMark Brown if (task == current) 157*b57180c7SMark Brown gcs_set_el0_mode(task); 158*b57180c7SMark Brown 159*b57180c7SMark Brown return 0; 160*b57180c7SMark Brown } 161*b57180c7SMark Brown 162*b57180c7SMark Brown int arch_get_shadow_stack_status(struct task_struct *task, 163*b57180c7SMark Brown unsigned long __user *arg) 164*b57180c7SMark Brown { 165*b57180c7SMark Brown if (!system_supports_gcs()) 166*b57180c7SMark Brown return -EINVAL; 167*b57180c7SMark Brown 168*b57180c7SMark Brown if (is_compat_thread(task_thread_info(task))) 169*b57180c7SMark Brown return -EINVAL; 170*b57180c7SMark Brown 171*b57180c7SMark Brown return put_user(task->thread.gcs_el0_mode, arg); 172*b57180c7SMark Brown } 173*b57180c7SMark Brown 174*b57180c7SMark Brown int arch_lock_shadow_stack_status(struct task_struct *task, 175*b57180c7SMark Brown unsigned long arg) 176*b57180c7SMark Brown { 177*b57180c7SMark Brown if (!system_supports_gcs()) 178*b57180c7SMark Brown return -EINVAL; 179*b57180c7SMark Brown 180*b57180c7SMark Brown if (is_compat_thread(task_thread_info(task))) 181*b57180c7SMark Brown return -EINVAL; 182*b57180c7SMark Brown 183*b57180c7SMark Brown /* 184*b57180c7SMark Brown * We support locking unknown bits so applications can prevent 185*b57180c7SMark Brown * any changes in a future proof manner. 186*b57180c7SMark Brown */ 187*b57180c7SMark Brown task->thread.gcs_el0_locked |= arg; 188*b57180c7SMark Brown 189*b57180c7SMark Brown return 0; 190*b57180c7SMark Brown } 191