1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28 #include "opt_platform.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/limits.h>
33 #include <sys/proc.h>
34 #include <sys/sf_buf.h>
35 #include <sys/signal.h>
36 #include <sys/sysent.h>
37 #include <sys/unistd.h>
38
39 #include <vm/vm.h>
40 #include <vm/vm_page.h>
41 #include <vm/vm_map.h>
42 #include <vm/uma.h>
43 #include <vm/uma_int.h>
44
45 #include <machine/armreg.h>
46 #include <machine/cpu.h>
47 #include <machine/md_var.h>
48 #include <machine/pcb.h>
49 #include <machine/frame.h>
50
51 #ifdef VFP
52 #include <machine/vfp.h>
53 #endif
54
55 #include <dev/psci/psci.h>
56
57 /*
58 * psci.c is "default" in ARM64 kernel config files
59 * psci_reset will do nothing until/unless the psci device probes/attaches.
60 * Therefore, it is safe to default the cpu_reset_hook to psci_reset.
61 */
62 cpu_reset_hook_t cpu_reset_hook = psci_reset;
63
64 /*
65 * Finish a fork operation, with process p2 nearly set up.
66 * Copy and update the pcb, set up the stack so that the child
67 * ready to run and return to user mode.
68 */
69 void
cpu_fork(struct thread * td1,struct proc * p2,struct thread * td2,int flags)70 cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
71 {
72 struct pcb *pcb2;
73 struct trapframe *tf;
74
75 if ((flags & RFPROC) == 0)
76 return;
77
78 if (td1 == curthread) {
79 /*
80 * Save the tpidr_el0 and the vfp state, these normally happen
81 * in cpu_switch, but if userland changes these then forks
82 * this may not have happened.
83 */
84 td1->td_pcb->pcb_tpidr_el0 = READ_SPECIALREG(tpidr_el0);
85 td1->td_pcb->pcb_tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
86 #ifdef VFP
87 if ((td1->td_pcb->pcb_fpflags & PCB_FP_STARTED) != 0)
88 vfp_save_state(td1, td1->td_pcb);
89 #endif
90 }
91
92 pcb2 = (struct pcb *)(td2->td_kstack +
93 td2->td_kstack_pages * PAGE_SIZE) - 1;
94
95 td2->td_pcb = pcb2;
96 bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
97
98 /* Clear the debug register state. */
99 bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs));
100
101 ptrauth_fork(td2, td1);
102
103 tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1);
104 bcopy(td1->td_frame, tf, sizeof(*tf));
105 tf->tf_x[0] = 0;
106 tf->tf_x[1] = 0;
107 tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF);
108
109 td2->td_frame = tf;
110
111 /* Set the return value registers for fork() */
112 td2->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return;
113 td2->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td2;
114 td2->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
115 td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame;
116
117 vfp_new_thread(td2, td1, true);
118
119 /* Setup to release spin count in fork_exit(). */
120 td2->td_md.md_spinlock_count = 1;
121 td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
122
123 /* Copy the TCR_EL1 value */
124 td2->td_proc->p_md.md_tcr = td1->td_proc->p_md.md_tcr;
125
126 #if defined(PERTHREAD_SSP)
127 /* Set the new canary */
128 arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary));
129 #endif
130 }
131
132 void
cpu_reset(void)133 cpu_reset(void)
134 {
135
136 cpu_reset_hook();
137
138 printf("cpu_reset failed");
139 while(1)
140 __asm volatile("wfi" ::: "memory");
141 }
142
143 void
cpu_set_syscall_retval(struct thread * td,int error)144 cpu_set_syscall_retval(struct thread *td, int error)
145 {
146 struct trapframe *frame;
147
148 frame = td->td_frame;
149
150 if (__predict_true(error == 0)) {
151 frame->tf_x[0] = td->td_retval[0];
152 frame->tf_x[1] = td->td_retval[1];
153 frame->tf_spsr &= ~PSR_C; /* carry bit */
154 return;
155 }
156
157 switch (error) {
158 case ERESTART:
159 frame->tf_elr -= 4;
160 break;
161 case EJUSTRETURN:
162 break;
163 default:
164 frame->tf_spsr |= PSR_C; /* carry bit */
165 frame->tf_x[0] = error;
166 break;
167 }
168 }
169
170 /*
171 * Initialize machine state, mostly pcb and trap frame for a new
172 * thread, about to return to userspace. Put enough state in the new
173 * thread's PCB to get it to go back to the fork_return(), which
174 * finalizes the thread state and handles peculiarities of the first
175 * return to userspace for the new thread.
176 */
177 void
cpu_copy_thread(struct thread * td,struct thread * td0)178 cpu_copy_thread(struct thread *td, struct thread *td0)
179 {
180 bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
181 bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
182
183 td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return;
184 td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td;
185 td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
186 td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
187
188 /* Update VFP state for the new thread */
189 vfp_new_thread(td, td0, false);
190
191 /* Setup to release spin count in fork_exit(). */
192 td->td_md.md_spinlock_count = 1;
193 td->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
194
195 #if defined(PERTHREAD_SSP)
196 /* Set the new canary */
197 arc4random_buf(&td->td_md.md_canary, sizeof(td->td_md.md_canary));
198 #endif
199
200 /* Generate new pointer authentication keys. */
201 ptrauth_copy_thread(td, td0);
202 }
203
204 /*
205 * Set that machine state for performing an upcall that starts
206 * the entry function with the given argument.
207 */
208 int
cpu_set_upcall(struct thread * td,void (* entry)(void *),void * arg,stack_t * stack)209 cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg,
210 stack_t *stack)
211 {
212 struct trapframe *tf = td->td_frame;
213
214 /* 32bits processes use r13 for sp */
215 if (td->td_frame->tf_spsr & PSR_M_32) {
216 tf->tf_x[13] = STACKALIGN((uintptr_t)stack->ss_sp +
217 stack->ss_size);
218 if ((register_t)entry & 1)
219 tf->tf_spsr |= PSR_T;
220 } else
221 tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp +
222 stack->ss_size);
223 tf->tf_elr = (register_t)entry;
224 tf->tf_x[0] = (register_t)arg;
225 tf->tf_x[29] = 0;
226 tf->tf_lr = 0;
227 return (0);
228 }
229
230 int
cpu_set_user_tls(struct thread * td,void * tls_base,int thr_flags __unused)231 cpu_set_user_tls(struct thread *td, void *tls_base, int thr_flags __unused)
232 {
233 struct pcb *pcb;
234
235 if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS)
236 return (EINVAL);
237
238 pcb = td->td_pcb;
239 if (td->td_frame->tf_spsr & PSR_M_32) {
240 /* 32bits arm stores the user TLS into tpidrro */
241 pcb->pcb_tpidrro_el0 = (register_t)tls_base;
242 pcb->pcb_tpidr_el0 = (register_t)tls_base;
243 if (td == curthread) {
244 WRITE_SPECIALREG(tpidrro_el0, tls_base);
245 WRITE_SPECIALREG(tpidr_el0, tls_base);
246 }
247 } else {
248 pcb->pcb_tpidr_el0 = (register_t)tls_base;
249 if (td == curthread)
250 WRITE_SPECIALREG(tpidr_el0, tls_base);
251 }
252
253 return (0);
254 }
255
256 void
cpu_thread_exit(struct thread * td)257 cpu_thread_exit(struct thread *td)
258 {
259 }
260
261 void
cpu_thread_alloc(struct thread * td)262 cpu_thread_alloc(struct thread *td)
263 {
264
265 td->td_pcb = (struct pcb *)(td->td_kstack +
266 td->td_kstack_pages * PAGE_SIZE) - 1;
267 td->td_frame = (struct trapframe *)STACKALIGN(
268 (struct trapframe *)td->td_pcb - 1);
269 ptrauth_thread_alloc(td);
270 }
271
272 void
cpu_thread_free(struct thread * td)273 cpu_thread_free(struct thread *td)
274 {
275 }
276
277 void
cpu_thread_clean(struct thread * td)278 cpu_thread_clean(struct thread *td)
279 {
280 }
281
282 /*
283 * Intercept the return address from a freshly forked process that has NOT
284 * been scheduled yet.
285 *
286 * This is needed to make kernel threads stay in kernel mode.
287 */
288 void
cpu_fork_kthread_handler(struct thread * td,void (* func)(void *),void * arg)289 cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg)
290 {
291
292 td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)func;
293 td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)arg;
294 }
295
296 void
cpu_update_pcb(struct thread * td)297 cpu_update_pcb(struct thread *td)
298 {
299 MPASS(td == curthread);
300 td->td_pcb->pcb_tpidr_el0 = READ_SPECIALREG(tpidr_el0);
301 td->td_pcb->pcb_tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
302 }
303
304 void
cpu_exit(struct thread * td)305 cpu_exit(struct thread *td)
306 {
307 }
308
309 bool
cpu_exec_vmspace_reuse(struct proc * p __unused,vm_map_t map __unused)310 cpu_exec_vmspace_reuse(struct proc *p __unused, vm_map_t map __unused)
311 {
312
313 return (true);
314 }
315
316 int
cpu_procctl(struct thread * td __unused,int idtype __unused,id_t id __unused,int com __unused,void * data __unused)317 cpu_procctl(struct thread *td __unused, int idtype __unused, id_t id __unused,
318 int com __unused, void *data __unused)
319 {
320
321 return (EINVAL);
322 }
323
324 void
cpu_sync_core(void)325 cpu_sync_core(void)
326 {
327 /*
328 * Do nothing. According to ARM ARMv8 D1.11 Exception return
329 * If FEAT_ExS is not implemented, or if FEAT_ExS is
330 * implemented and the SCTLR_ELx.EOS field is set, exception
331 * return from ELx is a context synchronization event.
332 */
333 }
334