1 /*
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2026 Devin Teske <dteske@FreeBSD.org>
5 */
6
7 /*
8 * x86 memory protection keys (PKU) for the Linuxulator, serving both
9 * the 64-bit and 32-bit Linux ABIs.
10 *
11 * The PKRU register is directly user-visible: Linux programs read and
12 * write it with RDPKRU/WRPKRU, which execute natively. The kernel's
13 * part is key allocation bookkeeping (per address space: inherited on
14 * fork, reset on exec, as with Linux mm->context.pkey_allocation_map),
15 * tagging pages (pkey_mprotect), and applying the initial access
16 * rights of pkey_alloc() to the calling thread's PKRU.
17 */
18
19 #include <sys/systm.h>
20 #include <sys/imgact.h>
21 #include <sys/lock.h>
22 #include <sys/pcpu.h>
23 #include <sys/proc.h>
24 #include <sys/sx.h>
25
26 #include <machine/cpufunc.h>
27 #include <machine/fpu.h>
28 #include <machine/md_var.h>
29 #include <machine/pcb.h>
30 #include <machine/specialreg.h>
31 #include <machine/sysarch.h>
32 #include <x86/x86_var.h>
33
34 #include <compat/linux/linux_emul.h>
35 #include <compat/linux/linux_mmap.h>
36
37 static bool
linux_pkey_supported(void)38 linux_pkey_supported(void)
39 {
40
41 return ((cpu_stdext_feature2 & CPUID_STDEXT2_OSPKE) != 0);
42 }
43
44 /*
45 * Update the calling thread's PKRU: new value is (PKRU & keep) | set.
46 */
47 static void
linux_pkru_write(struct thread * td,uint32_t keep,uint32_t set)48 linux_pkru_write(struct thread *td, uint32_t keep, uint32_t set)
49 {
50 struct pcb *pcb;
51 struct xstate_hdr *hdr;
52 char *sa;
53 uint32_t *pkru;
54
55 MPASS(td == curthread);
56 pcb = td->td_pcb;
57
58 /*
59 * The critical section is held across the save area update to
60 * exclude preemption: a context switch could otherwise load the
61 * xsave area back into the CPU after fpugetregs(), and the
62 * stores below would then be lost to the next save.
63 */
64 critical_enter();
65 if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0 &&
66 td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) {
67 wrpkru((rdpkru() & keep) | set);
68 critical_exit();
69 return;
70 }
71
72 /*
73 * The user FPU state is in the PCB save area, or is not yet
74 * initialized, in which case fpugetregs() installs the initial
75 * state there.
76 */
77 (void)fpugetregs(td);
78 sa = (char *)get_pcb_user_save_td(td);
79 hdr = (struct xstate_hdr *)(sa + xsave_area_hdr_offset());
80 pkru = (uint32_t *)(sa + xsave_area_offset(xsave_mask,
81 XFEATURE_ENABLED_PKRU, false, false));
82 if ((hdr->xstate_bv & XFEATURE_ENABLED_PKRU) == 0) {
83 hdr->xstate_bv |= XFEATURE_ENABLED_PKRU;
84 *pkru = 0;
85 }
86 *pkru = (*pkru & keep) | set;
87 critical_exit();
88 }
89
90 /*
91 * Set the calling thread's PKRU access rights for the given key.
92 */
93 static void
linux_pkru_set_perm(struct thread * td,u_int keyidx,uint32_t rights)94 linux_pkru_set_perm(struct thread *td, u_int keyidx, uint32_t rights)
95 {
96
97 linux_pkru_write(td, ~(LINUX_PKEY_ACCESS_MASK << (keyidx * 2)),
98 rights << (keyidx * 2));
99 }
100
101 /*
102 * Called from the Linux sysvecs' exec_setregs. Linux initializes
103 * PKRU at exec to deny access to all keys but key 0
104 * (arch/x86/mm/pkeys.c init_pkru_value), so memory tagged with a not
105 * yet allocated key is inaccessible; FreeBSD's initial PKRU is 0.
106 * This initializes the user FPU state slightly earlier than the lazy
107 * first-use path; the state would be initialized moments later in
108 * rtld/libc startup regardless.
109 */
110 void
linux_pkru_exec_init(struct thread * td)111 linux_pkru_exec_init(struct thread *td)
112 {
113
114 if (!linux_pkey_supported())
115 return;
116 linux_pkru_write(td, 0, LINUX_PKRU_INIT);
117 }
118
119 /*
120 * Protection keys are a property of the address space: inherit the
121 * allocation map on fork, as Linux does. When a FreeBSD process is
122 * switching to the Linux ABI there is no parent emuldata; start from
123 * the initial map. The unlocked read is atomic on the aligned word;
124 * a pkey_alloc() racing the fork in another thread yields a valid
125 * serialization either way.
126 */
127 void
linux_pemuldata_init_md(struct thread * td,struct linux_pemuldata * pem)128 linux_pemuldata_init_md(struct thread *td, struct linux_pemuldata *pem)
129 {
130 struct linux_pemuldata *ppem;
131
132 ppem = pem_find(td->td_proc);
133 if (ppem != NULL)
134 pem->pem_md.md_pkey_allocation_map =
135 ppem->pem_md.md_pkey_allocation_map;
136 else
137 pem->pem_md.md_pkey_allocation_map = LINUX_PKEY_INITIAL_MAP;
138 }
139
140 void
linux_pemuldata_exec_md(struct linux_pemuldata * pem)141 linux_pemuldata_exec_md(struct linux_pemuldata *pem)
142 {
143
144 pem->pem_md.md_pkey_allocation_map = LINUX_PKEY_INITIAL_MAP;
145 }
146
147 int
linux_pkey_alloc_machdep(struct thread * td,uint64_t init_val)148 linux_pkey_alloc_machdep(struct thread *td, uint64_t init_val)
149 {
150 struct linux_pemuldata *pem;
151 uint32_t free_keys;
152 int key;
153
154 if (!linux_pkey_supported())
155 return (ENOSPC);
156
157 pem = pem_find(td->td_proc);
158 LINUX_PEM_XLOCK(pem);
159 free_keys = ~pem->pem_md.md_pkey_allocation_map &
160 ((1u << LINUX_PKEY_MAX) - 1) & ~LINUX_PKEY_INITIAL_MAP;
161 if (free_keys == 0) {
162 LINUX_PEM_XUNLOCK(pem);
163 return (ENOSPC);
164 }
165 key = ffs(free_keys) - 1;
166 pem->pem_md.md_pkey_allocation_map |= 1u << key;
167 LINUX_PEM_XUNLOCK(pem);
168
169 linux_pkru_set_perm(td, key, init_val);
170 td->td_retval[0] = key;
171 return (0);
172 }
173
174 int
linux_pkey_free_machdep(struct thread * td,int pkey)175 linux_pkey_free_machdep(struct thread *td, int pkey)
176 {
177 struct linux_pemuldata *pem;
178
179 if (!linux_pkey_supported())
180 return (EINVAL);
181
182 pem = pem_find(td->td_proc);
183 LINUX_PEM_XLOCK(pem);
184 if ((pem->pem_md.md_pkey_allocation_map & (1u << pkey)) == 0) {
185 LINUX_PEM_XUNLOCK(pem);
186 return (EINVAL);
187 }
188 pem->pem_md.md_pkey_allocation_map &= ~(1u << pkey);
189 LINUX_PEM_XUNLOCK(pem);
190
191 /*
192 * As on Linux, freeing a key neither untags pages nor updates
193 * PKRU; that is the application's responsibility.
194 */
195 return (0);
196 }
197
198 int
linux_pkey_mprotect_machdep(struct thread * td,uintptr_t addr,size_t len,int prot,int pkey)199 linux_pkey_mprotect_machdep(struct thread *td, uintptr_t addr, size_t len,
200 int prot, int pkey)
201 {
202 struct linux_pemuldata *pem;
203 int error;
204
205 if (!linux_pkey_supported())
206 return (EINVAL);
207
208 pem = pem_find(td->td_proc);
209 LINUX_PEM_SLOCK(pem);
210 if ((pem->pem_md.md_pkey_allocation_map & (1u << pkey)) == 0) {
211 LINUX_PEM_SUNLOCK(pem);
212 return (EINVAL);
213 }
214 LINUX_PEM_SUNLOCK(pem);
215
216 error = linux_mprotect_common(td, addr, len, prot);
217 if (error != 0 || len == 0)
218 return (error);
219
220 /*
221 * Tag the range; a pkey of 0 untags it. The tag is not
222 * persistent: it dies with the mapping, matching Linux VMA
223 * semantics.
224 */
225 return (amd64_pkru_update(td, addr, len, pkey, 0, pkey == 0));
226 }
227