1fc84bc53SMark Brown // SPDX-License-Identifier: GPL-2.0-only 2fc84bc53SMark Brown 3fc84bc53SMark Brown #include <linux/mm.h> 4fc84bc53SMark Brown #include <linux/mman.h> 5fc84bc53SMark Brown #include <linux/syscalls.h> 6fc84bc53SMark Brown #include <linux/types.h> 7fc84bc53SMark Brown 8*506496bcSMark Brown #include <asm/cmpxchg.h> 9fc84bc53SMark Brown #include <asm/cpufeature.h> 10*506496bcSMark Brown #include <asm/gcs.h> 11fc84bc53SMark Brown #include <asm/page.h> 12fc84bc53SMark Brown 13*506496bcSMark Brown static unsigned long alloc_gcs(unsigned long addr, unsigned long size) 14*506496bcSMark Brown { 15*506496bcSMark Brown int flags = MAP_ANONYMOUS | MAP_PRIVATE; 16*506496bcSMark Brown struct mm_struct *mm = current->mm; 17*506496bcSMark Brown unsigned long mapped_addr, unused; 18*506496bcSMark Brown 19*506496bcSMark Brown if (addr) 20*506496bcSMark Brown flags |= MAP_FIXED_NOREPLACE; 21*506496bcSMark Brown 22*506496bcSMark Brown mmap_write_lock(mm); 23*506496bcSMark Brown mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags, 24*506496bcSMark Brown VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL); 25*506496bcSMark Brown mmap_write_unlock(mm); 26*506496bcSMark Brown 27*506496bcSMark Brown return mapped_addr; 28*506496bcSMark Brown } 29*506496bcSMark Brown 30*506496bcSMark Brown static unsigned long gcs_size(unsigned long size) 31*506496bcSMark Brown { 32*506496bcSMark Brown if (size) 33*506496bcSMark Brown return PAGE_ALIGN(size); 34*506496bcSMark Brown 35*506496bcSMark Brown /* Allocate RLIMIT_STACK/2 with limits of PAGE_SIZE..2G */ 36*506496bcSMark Brown size = PAGE_ALIGN(min_t(unsigned long long, 37*506496bcSMark Brown rlimit(RLIMIT_STACK) / 2, SZ_2G)); 38*506496bcSMark Brown return max(PAGE_SIZE, size); 39*506496bcSMark Brown } 40*506496bcSMark Brown 41*506496bcSMark Brown unsigned long gcs_alloc_thread_stack(struct task_struct *tsk, 42*506496bcSMark Brown const struct kernel_clone_args *args) 43*506496bcSMark Brown { 44*506496bcSMark Brown unsigned long addr, size; 45*506496bcSMark Brown 46*506496bcSMark Brown if (!system_supports_gcs()) 47*506496bcSMark Brown return 0; 48*506496bcSMark Brown 49*506496bcSMark Brown if (!task_gcs_el0_enabled(tsk)) 50*506496bcSMark Brown return 0; 51*506496bcSMark Brown 52*506496bcSMark Brown if ((args->flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM) { 53*506496bcSMark Brown tsk->thread.gcspr_el0 = read_sysreg_s(SYS_GCSPR_EL0); 54*506496bcSMark Brown return 0; 55*506496bcSMark Brown } 56*506496bcSMark Brown 57*506496bcSMark Brown size = args->stack_size / 2; 58*506496bcSMark Brown 59*506496bcSMark Brown size = gcs_size(size); 60*506496bcSMark Brown addr = alloc_gcs(0, size); 61*506496bcSMark Brown if (IS_ERR_VALUE(addr)) 62*506496bcSMark Brown return addr; 63*506496bcSMark Brown 64*506496bcSMark Brown tsk->thread.gcs_base = addr; 65*506496bcSMark Brown tsk->thread.gcs_size = size; 66*506496bcSMark Brown tsk->thread.gcspr_el0 = addr + size - sizeof(u64); 67*506496bcSMark Brown 68*506496bcSMark Brown return addr; 69*506496bcSMark Brown } 70*506496bcSMark Brown 71fc84bc53SMark Brown /* 72fc84bc53SMark Brown * Apply the GCS mode configured for the specified task to the 73fc84bc53SMark Brown * hardware. 74fc84bc53SMark Brown */ 75fc84bc53SMark Brown void gcs_set_el0_mode(struct task_struct *task) 76fc84bc53SMark Brown { 77fc84bc53SMark Brown u64 gcscre0_el1 = GCSCRE0_EL1_nTR; 78fc84bc53SMark Brown 79fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE) 80fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_RVCHKEN | GCSCRE0_EL1_PCRSEL; 81fc84bc53SMark Brown 82fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_WRITE) 83fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_STREn; 84fc84bc53SMark Brown 85fc84bc53SMark Brown if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_PUSH) 86fc84bc53SMark Brown gcscre0_el1 |= GCSCRE0_EL1_PUSHMEn; 87fc84bc53SMark Brown 88fc84bc53SMark Brown write_sysreg_s(gcscre0_el1, SYS_GCSCRE0_EL1); 89fc84bc53SMark Brown } 90fc84bc53SMark Brown 91fc84bc53SMark Brown void gcs_free(struct task_struct *task) 92fc84bc53SMark Brown { 93fc84bc53SMark Brown if (!system_supports_gcs()) 94fc84bc53SMark Brown return; 95fc84bc53SMark Brown 96*506496bcSMark Brown /* 97*506496bcSMark Brown * When fork() with CLONE_VM fails, the child (tsk) already 98*506496bcSMark Brown * has a GCS allocated, and exit_thread() calls this function 99*506496bcSMark Brown * to free it. In this case the parent (current) and the 100*506496bcSMark Brown * child share the same mm struct. 101*506496bcSMark Brown */ 102*506496bcSMark Brown if (!task->mm || task->mm != current->mm) 103*506496bcSMark Brown return; 104*506496bcSMark Brown 105fc84bc53SMark Brown if (task->thread.gcs_base) 106fc84bc53SMark Brown vm_munmap(task->thread.gcs_base, task->thread.gcs_size); 107fc84bc53SMark Brown 108fc84bc53SMark Brown task->thread.gcspr_el0 = 0; 109fc84bc53SMark Brown task->thread.gcs_base = 0; 110fc84bc53SMark Brown task->thread.gcs_size = 0; 111fc84bc53SMark Brown } 112