1 /* 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2026 Devin Teske <dteske@FreeBSD.org> 5 */ 6 7 /* 8 * x86 memory protection keys (PKU) for the Linuxulator, serving both 9 * the 64-bit and 32-bit Linux ABIs. 10 * 11 * The PKRU register is directly user-visible: Linux programs read and 12 * write it with RDPKRU/WRPKRU, which execute natively. The kernel's 13 * part is key allocation bookkeeping (per address space: inherited on 14 * fork, reset on exec, as with Linux mm->context.pkey_allocation_map), 15 * tagging pages (pkey_mprotect), and applying the initial access 16 * rights of pkey_alloc() to the calling thread's PKRU. 17 */ 18 19 #include <sys/systm.h> 20 #include <sys/imgact.h> 21 #include <sys/lock.h> 22 #include <sys/pcpu.h> 23 #include <sys/proc.h> 24 #include <sys/sx.h> 25 26 #include <machine/cpufunc.h> 27 #include <machine/fpu.h> 28 #include <machine/md_var.h> 29 #include <machine/pcb.h> 30 #include <machine/specialreg.h> 31 #include <machine/sysarch.h> 32 #include <x86/x86_var.h> 33 34 #include <compat/linux/linux_emul.h> 35 #include <compat/linux/linux_mmap.h> 36 37 static bool 38 linux_pkey_supported(void) 39 { 40 41 return ((cpu_stdext_feature2 & CPUID_STDEXT2_OSPKE) != 0); 42 } 43 44 /* 45 * Update the calling thread's PKRU: new value is (PKRU & keep) | set. 46 */ 47 static void 48 linux_pkru_write(struct thread *td, uint32_t keep, uint32_t set) 49 { 50 struct pcb *pcb; 51 struct xstate_hdr *hdr; 52 char *sa; 53 uint32_t *pkru; 54 55 MPASS(td == curthread); 56 pcb = td->td_pcb; 57 58 /* 59 * The critical section is held across the save area update to 60 * exclude preemption: a context switch could otherwise load the 61 * xsave area back into the CPU after fpugetregs(), and the 62 * stores below would then be lost to the next save. 63 */ 64 critical_enter(); 65 if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0 && 66 td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 67 wrpkru((rdpkru() & keep) | set); 68 critical_exit(); 69 return; 70 } 71 72 /* 73 * The user FPU state is in the PCB save area, or is not yet 74 * initialized, in which case fpugetregs() installs the initial 75 * state there. 76 */ 77 (void)fpugetregs(td); 78 sa = (char *)get_pcb_user_save_td(td); 79 hdr = (struct xstate_hdr *)(sa + xsave_area_hdr_offset()); 80 pkru = (uint32_t *)(sa + xsave_area_offset(xsave_mask, 81 XFEATURE_ENABLED_PKRU, false, false)); 82 if ((hdr->xstate_bv & XFEATURE_ENABLED_PKRU) == 0) { 83 hdr->xstate_bv |= XFEATURE_ENABLED_PKRU; 84 *pkru = 0; 85 } 86 *pkru = (*pkru & keep) | set; 87 critical_exit(); 88 } 89 90 /* 91 * Set the calling thread's PKRU access rights for the given key. 92 */ 93 static void 94 linux_pkru_set_perm(struct thread *td, u_int keyidx, uint32_t rights) 95 { 96 97 linux_pkru_write(td, ~(LINUX_PKEY_ACCESS_MASK << (keyidx * 2)), 98 rights << (keyidx * 2)); 99 } 100 101 /* 102 * Called from the Linux sysvecs' exec_setregs. Linux initializes 103 * PKRU at exec to deny access to all keys but key 0 104 * (arch/x86/mm/pkeys.c init_pkru_value), so memory tagged with a not 105 * yet allocated key is inaccessible; FreeBSD's initial PKRU is 0. 106 * This initializes the user FPU state slightly earlier than the lazy 107 * first-use path; the state would be initialized moments later in 108 * rtld/libc startup regardless. 109 */ 110 void 111 linux_pkru_exec_init(struct thread *td) 112 { 113 114 if (!linux_pkey_supported()) 115 return; 116 linux_pkru_write(td, 0, LINUX_PKRU_INIT); 117 } 118 119 /* 120 * Protection keys are a property of the address space: inherit the 121 * allocation map on fork, as Linux does. When a FreeBSD process is 122 * switching to the Linux ABI there is no parent emuldata; start from 123 * the initial map. The unlocked read is atomic on the aligned word; 124 * a pkey_alloc() racing the fork in another thread yields a valid 125 * serialization either way. 126 */ 127 void 128 linux_pemuldata_init_md(struct thread *td, struct linux_pemuldata *pem) 129 { 130 struct linux_pemuldata *ppem; 131 132 ppem = pem_find(td->td_proc); 133 if (ppem != NULL) 134 pem->pem_md.md_pkey_allocation_map = 135 ppem->pem_md.md_pkey_allocation_map; 136 else 137 pem->pem_md.md_pkey_allocation_map = LINUX_PKEY_INITIAL_MAP; 138 } 139 140 void 141 linux_pemuldata_exec_md(struct linux_pemuldata *pem) 142 { 143 144 pem->pem_md.md_pkey_allocation_map = LINUX_PKEY_INITIAL_MAP; 145 } 146 147 int 148 linux_pkey_alloc_machdep(struct thread *td, uint64_t init_val) 149 { 150 struct linux_pemuldata *pem; 151 uint32_t free_keys; 152 int key; 153 154 if (!linux_pkey_supported()) 155 return (ENOSPC); 156 157 pem = pem_find(td->td_proc); 158 LINUX_PEM_XLOCK(pem); 159 free_keys = ~pem->pem_md.md_pkey_allocation_map & 160 ((1u << LINUX_PKEY_MAX) - 1) & ~LINUX_PKEY_INITIAL_MAP; 161 if (free_keys == 0) { 162 LINUX_PEM_XUNLOCK(pem); 163 return (ENOSPC); 164 } 165 key = ffs(free_keys) - 1; 166 pem->pem_md.md_pkey_allocation_map |= 1u << key; 167 LINUX_PEM_XUNLOCK(pem); 168 169 linux_pkru_set_perm(td, key, init_val); 170 td->td_retval[0] = key; 171 return (0); 172 } 173 174 int 175 linux_pkey_free_machdep(struct thread *td, int pkey) 176 { 177 struct linux_pemuldata *pem; 178 179 if (!linux_pkey_supported()) 180 return (EINVAL); 181 182 pem = pem_find(td->td_proc); 183 LINUX_PEM_XLOCK(pem); 184 if ((pem->pem_md.md_pkey_allocation_map & (1u << pkey)) == 0) { 185 LINUX_PEM_XUNLOCK(pem); 186 return (EINVAL); 187 } 188 pem->pem_md.md_pkey_allocation_map &= ~(1u << pkey); 189 LINUX_PEM_XUNLOCK(pem); 190 191 /* 192 * As on Linux, freeing a key neither untags pages nor updates 193 * PKRU; that is the application's responsibility. 194 */ 195 return (0); 196 } 197 198 int 199 linux_pkey_mprotect_machdep(struct thread *td, uintptr_t addr, size_t len, 200 int prot, int pkey) 201 { 202 struct linux_pemuldata *pem; 203 int error; 204 205 if (!linux_pkey_supported()) 206 return (EINVAL); 207 208 pem = pem_find(td->td_proc); 209 LINUX_PEM_SLOCK(pem); 210 if ((pem->pem_md.md_pkey_allocation_map & (1u << pkey)) == 0) { 211 LINUX_PEM_SUNLOCK(pem); 212 return (EINVAL); 213 } 214 LINUX_PEM_SUNLOCK(pem); 215 216 error = linux_mprotect_common(td, addr, len, prot); 217 if (error != 0 || len == 0) 218 return (error); 219 220 /* 221 * Tag the range; a pkey of 0 untags it. The tag is not 222 * persistent: it dies with the mapping, matching Linux VMA 223 * semantics. 224 */ 225 return (amd64_pkru_update(td, addr, len, pkey, 0, pkey == 0)); 226 } 227