1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/smp.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/mman.h>
36 #include <sys/pcpu.h>
37 #include <sys/proc.h>
38 #include <sys/sysctl.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/vmem.h>
42
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_map.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_param.h>
49
50 #include <machine/vm.h>
51 #include <machine/cpufunc.h>
52 #include <machine/cpu.h>
53 #include <machine/machdep.h>
54 #include <machine/vmm.h>
55 #include <machine/atomic.h>
56 #include <machine/hypervisor.h>
57 #include <machine/pmap.h>
58
59 #include <dev/vmm/vmm_mem.h>
60 #include <dev/vmm/vmm_vm.h>
61
62 #include "mmu.h"
63 #include "arm64.h"
64 #include "hyp.h"
65 #include "reset.h"
66 #include "io/vgic.h"
67 #include "io/vgic_v3.h"
68 #include "io/vtimer.h"
69 #include "vmm_handlers.h"
70 #include "vmm_stat.h"
71
72 #define HANDLED 1
73 #define UNHANDLED 0
74
75 /* Number of bits in an EL2 virtual address */
76 #define EL2_VIRT_BITS 48
77 CTASSERT((1ul << EL2_VIRT_BITS) >= HYP_VM_MAX_ADDRESS);
78
79 /* TODO: Move the host hypctx off the stack */
80 #define VMM_STACK_PAGES 4
81 #define VMM_STACK_SIZE (VMM_STACK_PAGES * PAGE_SIZE)
82
83 static int vmm_pmap_levels, vmm_virt_bits, vmm_max_ipa_bits;
84
85 /* Register values passed to arm_setup_vectors to set in the hypervisor */
86 struct vmm_init_regs {
87 uint64_t tcr_el2;
88 uint64_t vtcr_el2;
89 };
90
91 MALLOC_DEFINE(M_HYP, "ARM VMM HYP", "ARM VMM HYP");
92
93 extern char hyp_init_vectors[];
94 extern char hyp_vectors[];
95 extern char hyp_stub_vectors[];
96
97 static vm_paddr_t hyp_code_base;
98 static size_t hyp_code_len;
99
100 static char *stack[MAXCPU];
101 static vm_offset_t stack_hyp_va[MAXCPU];
102
103 static vmem_t *el2_mem_alloc;
104
105 static void arm_setup_vectors(void *arg);
106
107 DPCPU_DEFINE_STATIC(struct hypctx *, vcpu);
108
109 static inline void
arm64_set_active_vcpu(struct hypctx * hypctx)110 arm64_set_active_vcpu(struct hypctx *hypctx)
111 {
112 DPCPU_SET(vcpu, hypctx);
113 }
114
115 struct hypctx *
arm64_get_active_vcpu(void)116 arm64_get_active_vcpu(void)
117 {
118 return (DPCPU_GET(vcpu));
119 }
120
121 static void
arm_setup_vectors(void * arg)122 arm_setup_vectors(void *arg)
123 {
124 struct vmm_init_regs *el2_regs;
125 uintptr_t stack_top;
126 uint32_t sctlr_el2;
127 register_t daif;
128
129 el2_regs = arg;
130 arm64_set_active_vcpu(NULL);
131
132 /*
133 * Configure the system control register for EL2:
134 *
135 * SCTLR_EL2_M: MMU on
136 * SCTLR_EL2_C: Data cacheability not affected
137 * SCTLR_EL2_I: Instruction cacheability not affected
138 * SCTLR_EL2_A: Instruction alignment check
139 * SCTLR_EL2_SA: Stack pointer alignment check
140 * SCTLR_EL2_WXN: Treat writable memory as execute never
141 * ~SCTLR_EL2_EE: Data accesses are little-endian
142 */
143 sctlr_el2 = SCTLR_EL2_RES1;
144 sctlr_el2 |= SCTLR_EL2_M | SCTLR_EL2_C | SCTLR_EL2_I;
145 sctlr_el2 |= SCTLR_EL2_A | SCTLR_EL2_SA;
146 sctlr_el2 |= SCTLR_EL2_WXN;
147 sctlr_el2 &= ~SCTLR_EL2_EE;
148
149 daif = intr_disable();
150
151 if (in_vhe()) {
152 WRITE_SPECIALREG(vtcr_el2, el2_regs->vtcr_el2);
153 } else {
154 /*
155 * Install the temporary vectors which will be responsible for
156 * initializing the VMM when we next trap into EL2.
157 *
158 * x0: the exception vector table responsible for hypervisor
159 * initialization on the next call.
160 */
161 vmm_call_hyp(vtophys(&vmm_hyp_code));
162
163 /* Create and map the hypervisor stack */
164 stack_top = stack_hyp_va[PCPU_GET(cpuid)] + VMM_STACK_SIZE;
165
166 /* Special call to initialize EL2 */
167 vmm_call_hyp(vmmpmap_to_ttbr0(), stack_top, el2_regs->tcr_el2,
168 sctlr_el2, el2_regs->vtcr_el2);
169 }
170
171 intr_restore(daif);
172 }
173
174 static void
arm_teardown_vectors(void * arg)175 arm_teardown_vectors(void *arg)
176 {
177 register_t daif;
178
179 /*
180 * vmm_cleanup() will disable the MMU. For the next few instructions,
181 * before the hardware disables the MMU, one of the following is
182 * possible:
183 *
184 * a. The instruction addresses are fetched with the MMU disabled,
185 * and they must represent the actual physical addresses. This will work
186 * because we call the vmm_cleanup() function by its physical address.
187 *
188 * b. The instruction addresses are fetched using the old translation
189 * tables. This will work because we have an identity mapping in place
190 * in the translation tables and vmm_cleanup() is called by its physical
191 * address.
192 */
193 daif = intr_disable();
194 /* TODO: Invalidate the cache */
195 vmm_call_hyp(HYP_CLEANUP, vtophys(hyp_stub_vectors));
196 intr_restore(daif);
197
198 arm64_set_active_vcpu(NULL);
199 }
200
201 static uint64_t
vmm_vtcr_el2_sl(u_int levels)202 vmm_vtcr_el2_sl(u_int levels)
203 {
204 #if PAGE_SIZE == PAGE_SIZE_4K
205 switch (levels) {
206 case 2:
207 return (VTCR_EL2_SL0_4K_LVL2);
208 case 3:
209 return (VTCR_EL2_SL0_4K_LVL1);
210 case 4:
211 return (VTCR_EL2_SL0_4K_LVL0);
212 default:
213 panic("%s: Invalid number of page table levels %u", __func__,
214 levels);
215 }
216 #elif PAGE_SIZE == PAGE_SIZE_16K
217 switch (levels) {
218 case 2:
219 return (VTCR_EL2_SL0_16K_LVL2);
220 case 3:
221 return (VTCR_EL2_SL0_16K_LVL1);
222 case 4:
223 return (VTCR_EL2_SL0_16K_LVL0);
224 default:
225 panic("%s: Invalid number of page table levels %u", __func__,
226 levels);
227 }
228 #else
229 #error Unsupported page size
230 #endif
231 }
232
233 int
vmmops_modinit(int ipinum)234 vmmops_modinit(int ipinum)
235 {
236 struct vmm_init_regs el2_regs;
237 vm_offset_t next_hyp_va;
238 vm_paddr_t vmm_base;
239 uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field;
240 int cpu, i;
241 bool rv __diagused;
242
243 if (!has_hyp()) {
244 printf(
245 "vmm: Processor doesn't have support for virtualization\n");
246 return (ENXIO);
247 }
248
249 if (!vgic_present()) {
250 printf("vmm: No vgic found\n");
251 return (ENODEV);
252 }
253
254 get_kernel_reg(ID_AA64MMFR0_EL1, &id_aa64mmfr0_el1);
255 pa_range_field = ID_AA64MMFR0_PARange_VAL(id_aa64mmfr0_el1);
256 /*
257 * Use 3 levels to give us up to 39 bits with 4k pages, or
258 * 47 bits with 16k pages.
259 */
260 /* TODO: Check the number of levels for 64k pages */
261 vmm_pmap_levels = 3;
262 switch (pa_range_field) {
263 case ID_AA64MMFR0_PARange_4G:
264 printf("vmm: Not enough physical address bits\n");
265 return (ENXIO);
266 case ID_AA64MMFR0_PARange_64G:
267 vmm_virt_bits = 36;
268 #if PAGE_SIZE == PAGE_SIZE_16K
269 vmm_pmap_levels = 2;
270 #endif
271 break;
272 default:
273 vmm_virt_bits = 39;
274 break;
275 }
276 pa_range_bits = pa_range_field >> ID_AA64MMFR0_PARange_SHIFT;
277
278 if (!in_vhe()) {
279 /* Initialise the EL2 MMU */
280 if (!vmmpmap_init()) {
281 printf("vmm: Failed to init the EL2 MMU\n");
282 return (ENOMEM);
283 }
284 }
285
286 /* Set up the stage 2 pmap callbacks */
287 MPASS(pmap_clean_stage2_tlbi == NULL);
288 pmap_clean_stage2_tlbi = vmm_clean_s2_tlbi;
289 pmap_stage2_invalidate_range = vmm_s2_tlbi_range;
290 pmap_stage2_invalidate_all = vmm_s2_tlbi_all;
291
292 if (!in_vhe()) {
293 /*
294 * Create an allocator for the virtual address space used by
295 * EL2. EL2 code is identity-mapped; the allocator is used to
296 * find space for VM structures.
297 */
298 el2_mem_alloc = vmem_create("VMM EL2", 0, 0, PAGE_SIZE, 0,
299 M_WAITOK);
300
301 /* Create the mappings for the hypervisor translation table. */
302 hyp_code_len = round_page(&vmm_hyp_code_end - &vmm_hyp_code);
303
304 /* We need an physical identity mapping for when we activate the MMU */
305 hyp_code_base = vmm_base = vtophys(&vmm_hyp_code);
306 rv = vmmpmap_enter(vmm_base, hyp_code_len, vmm_base,
307 VM_PROT_READ | VM_PROT_EXECUTE);
308 MPASS(rv);
309
310 next_hyp_va = roundup2(vmm_base + hyp_code_len, L2_SIZE);
311
312 /* Create a per-CPU hypervisor stack */
313 CPU_FOREACH(cpu) {
314 stack[cpu] = malloc(VMM_STACK_SIZE, M_HYP, M_WAITOK | M_ZERO);
315 stack_hyp_va[cpu] = next_hyp_va;
316
317 for (i = 0; i < VMM_STACK_PAGES; i++) {
318 rv = vmmpmap_enter(stack_hyp_va[cpu] + ptoa(i),
319 PAGE_SIZE, vtophys(stack[cpu] + ptoa(i)),
320 VM_PROT_READ | VM_PROT_WRITE);
321 MPASS(rv);
322 }
323 next_hyp_va += L2_SIZE;
324 }
325
326 el2_regs.tcr_el2 = TCR_EL2_RES1;
327 el2_regs.tcr_el2 |= min(pa_range_bits << TCR_EL2_PS_SHIFT,
328 TCR_EL2_PS_52BITS);
329 el2_regs.tcr_el2 |= TCR_EL2_T0SZ(64 - EL2_VIRT_BITS);
330 el2_regs.tcr_el2 |= TCR_EL2_IRGN0_WBWA | TCR_EL2_ORGN0_WBWA;
331 #if PAGE_SIZE == PAGE_SIZE_4K
332 el2_regs.tcr_el2 |= TCR_EL2_TG0_4K;
333 #elif PAGE_SIZE == PAGE_SIZE_16K
334 el2_regs.tcr_el2 |= TCR_EL2_TG0_16K;
335 #else
336 #error Unsupported page size
337 #endif
338 #ifdef SMP
339 el2_regs.tcr_el2 |= TCR_EL2_SH0_IS;
340 #endif
341 }
342
343 switch (pa_range_bits << TCR_EL2_PS_SHIFT) {
344 case TCR_EL2_PS_32BITS:
345 vmm_max_ipa_bits = 32;
346 break;
347 case TCR_EL2_PS_36BITS:
348 vmm_max_ipa_bits = 36;
349 break;
350 case TCR_EL2_PS_40BITS:
351 vmm_max_ipa_bits = 40;
352 break;
353 case TCR_EL2_PS_42BITS:
354 vmm_max_ipa_bits = 42;
355 break;
356 case TCR_EL2_PS_44BITS:
357 vmm_max_ipa_bits = 44;
358 break;
359 case TCR_EL2_PS_48BITS:
360 vmm_max_ipa_bits = 48;
361 break;
362 case TCR_EL2_PS_52BITS:
363 default:
364 vmm_max_ipa_bits = 52;
365 break;
366 }
367
368 /*
369 * Configure the Stage 2 translation control register:
370 *
371 * VTCR_IRGN0_WBWA: Translation table walks access inner cacheable
372 * normal memory
373 * VTCR_ORGN0_WBWA: Translation table walks access outer cacheable
374 * normal memory
375 * VTCR_EL2_TG0_4K/16K: Stage 2 uses the same page size as the kernel
376 * VTCR_EL2_SL0_4K_LVL1: Stage 2 uses concatenated level 1 tables
377 * VTCR_EL2_SH0_IS: Memory associated with Stage 2 walks is inner
378 * shareable
379 */
380 el2_regs.vtcr_el2 = VTCR_EL2_RES1;
381 el2_regs.vtcr_el2 |= VTCR_EL2_IRGN0_WBWA | VTCR_EL2_ORGN0_WBWA;
382 el2_regs.vtcr_el2 |= VTCR_EL2_T0SZ(64 - vmm_virt_bits);
383 el2_regs.vtcr_el2 |= vmm_vtcr_el2_sl(vmm_pmap_levels);
384 #if PAGE_SIZE == PAGE_SIZE_4K
385 el2_regs.vtcr_el2 |= VTCR_EL2_TG0_4K;
386 #elif PAGE_SIZE == PAGE_SIZE_16K
387 el2_regs.vtcr_el2 |= VTCR_EL2_TG0_16K;
388 #else
389 #error Unsupported page size
390 #endif
391 #ifdef SMP
392 el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS;
393 #endif
394 if (pmap_vs_enabled())
395 el2_regs.vtcr_el2 |= VTCR_EL2_VS;
396 /*
397 * If FEAT_LPA2 is enabled in the host then we need to enable it here
398 * so the page tables created by pmap.c are correct. The meaning of
399 * the shareability field changes to become address bits when this
400 * is set.
401 */
402 if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0) {
403 el2_regs.vtcr_el2 |= VTCR_EL2_DS;
404 el2_regs.vtcr_el2 |=
405 min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_52BIT);
406 } else {
407 el2_regs.vtcr_el2 |=
408 min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT);
409 }
410
411 smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);
412
413 if (!in_vhe()) {
414 /* Add memory to the vmem allocator (checking there is space) */
415 if (vmm_base > (L2_SIZE + PAGE_SIZE)) {
416 /*
417 * Ensure there is an L2 block before the vmm code to check
418 * for buffer overflows on earlier data. Include the PAGE_SIZE
419 * of the minimum we can allocate.
420 */
421 vmm_base -= L2_SIZE + PAGE_SIZE;
422 vmm_base = rounddown2(vmm_base, L2_SIZE);
423
424 /*
425 * Check there is memory before the vmm code to add.
426 *
427 * Reserve the L2 block at address 0 so NULL dereference will
428 * raise an exception.
429 */
430 if (vmm_base > L2_SIZE)
431 vmem_add(el2_mem_alloc, L2_SIZE, vmm_base - L2_SIZE,
432 M_WAITOK);
433 }
434
435 /*
436 * Add the memory after the stacks. There is most of an L2 block
437 * between the last stack and the first allocation so this should
438 * be safe without adding more padding.
439 */
440 if (next_hyp_va < HYP_VM_MAX_ADDRESS - PAGE_SIZE)
441 vmem_add(el2_mem_alloc, next_hyp_va,
442 HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK);
443 }
444
445 vgic_init();
446 vtimer_init();
447
448 return (0);
449 }
450
451 int
vmmops_modcleanup(void)452 vmmops_modcleanup(void)
453 {
454 int cpu;
455
456 if (!in_vhe()) {
457 smp_rendezvous(NULL, arm_teardown_vectors, NULL, NULL);
458
459 CPU_FOREACH(cpu) {
460 vmmpmap_remove(stack_hyp_va[cpu],
461 VMM_STACK_PAGES * PAGE_SIZE, false);
462 }
463
464 vmmpmap_remove(hyp_code_base, hyp_code_len, false);
465 }
466
467 vtimer_cleanup();
468
469 if (!in_vhe()) {
470 vmmpmap_fini();
471
472 CPU_FOREACH(cpu)
473 free(stack[cpu], M_HYP);
474 }
475
476 pmap_clean_stage2_tlbi = NULL;
477 pmap_stage2_invalidate_range = NULL;
478 pmap_stage2_invalidate_all = NULL;
479
480 return (0);
481 }
482
483 static vm_size_t
el2_hyp_size(struct vm * vm)484 el2_hyp_size(struct vm *vm)
485 {
486 return (round_page(sizeof(struct hyp) +
487 sizeof(struct hypctx *) * vm_get_maxcpus(vm)));
488 }
489
490 static vm_size_t
el2_hypctx_size(void)491 el2_hypctx_size(void)
492 {
493 /* Allocation for hypctx, one vncr page & one host state page */
494 return (round_page(sizeof(struct hypctx) + 2 * VNCR_PAGE_SIZE));
495 }
496
497 static vm_offset_t
el2_map_enter(vm_offset_t data,vm_size_t size,vm_prot_t prot)498 el2_map_enter(vm_offset_t data, vm_size_t size, vm_prot_t prot)
499 {
500 vmem_addr_t addr;
501 int err __diagused;
502 bool rv __diagused;
503
504 err = vmem_alloc(el2_mem_alloc, size, M_NEXTFIT | M_WAITOK, &addr);
505 MPASS(err == 0);
506 rv = vmmpmap_enter(addr, size, vtophys(data), prot);
507 MPASS(rv);
508
509 return (addr);
510 }
511
512 void *
vmmops_init(struct vm * vm,pmap_t pmap)513 vmmops_init(struct vm *vm, pmap_t pmap)
514 {
515 struct hyp *hyp;
516 vm_size_t size;
517 uint64_t idreg;
518
519 size = el2_hyp_size(vm);
520 hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
521
522 hyp->vm = vm;
523 hyp->vgic_attached = false;
524
525 get_kernel_reg(ID_AA64MMFR0_EL1, &idreg);
526 if (ID_AA64MMFR0_ECV_VAL(idreg) >= ID_AA64MMFR0_ECV_POFF)
527 hyp->feats |= HYP_FEAT_ECV_POFF;
528
529 switch (ID_AA64MMFR0_FGT_VAL(idreg)) {
530 case ID_AA64MMFR0_FGT_NONE:
531 break;
532 default:
533 case ID_AA64MMFR0_FGT_8_9:
534 hyp->feats |= HYP_FEAT_FGT2;
535 /* FALLTHROUGH */
536 case ID_AA64MMFR0_FGT_8_6:
537 hyp->feats |= HYP_FEAT_FGT;
538 break;
539 }
540
541 get_kernel_reg(ID_AA64MMFR1_EL1, &idreg);
542 if (ID_AA64MMFR1_HCX_VAL(idreg) >= ID_AA64MMFR1_HCX_IMPL)
543 hyp->feats |= HYP_FEAT_HCX;
544
545 hyp->cntvoff_el2 = READ_SPECIALREG(cntpct_el0);
546 vgic_vminit(hyp);
547
548 if (!in_vhe())
549 hyp->el2_addr = el2_map_enter((vm_offset_t)hyp, size,
550 VM_PROT_READ | VM_PROT_WRITE);
551
552 return (hyp);
553 }
554
555 void *
vmmops_vcpu_init(void * vmi,struct vcpu * vcpu1,int vcpuid)556 vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
557 {
558 struct hyp *hyp = vmi;
559 struct hypctx *hypctx;
560 vm_size_t size;
561
562 size = el2_hypctx_size();
563 /*
564 * Allocation memory layout:
565 0x0000 - 0x0fff <- struct hypctx
566 0x1000 - 0x1fff <- VNCR memory page
567 0x2000 - 0x2fff <- host_ctx memory page
568 */
569 hypctx = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
570 _Static_assert(sizeof(struct hypctx) < VNCR_PAGE_SIZE,
571 "struct hypctx exceeds VNCR_PAGE_SIZE");
572 hypctx->vncr_regs = (void *)((char *)hypctx + VNCR_PAGE_SIZE);
573 hypctx->host_vncr_regs = (void *)((char *)hypctx + 2 * VNCR_PAGE_SIZE);
574
575 KASSERT(vcpuid >= 0 && vcpuid < vm_get_maxcpus(hyp->vm),
576 ("%s: Invalid vcpuid %d", __func__, vcpuid));
577 hyp->ctx[vcpuid] = hypctx;
578
579 hypctx->hyp = hyp;
580 hypctx->vcpu = vcpu1;
581
582 reset_vm_el01_regs(hypctx);
583 reset_vm_el2_regs(hypctx);
584
585 vtimer_cpuinit(hypctx);
586 vgic_cpuinit(hypctx);
587
588 if (!in_vhe()) {
589 hypctx->el2_addr = el2_map_enter((vm_offset_t)hypctx, size,
590 VM_PROT_READ | VM_PROT_WRITE);
591 hypctx->el2_vncr_addr = hypctx->el2_addr + VNCR_PAGE_SIZE;
592 hypctx->el2_host_vncr_addr = hypctx->el2_addr + 2 * VNCR_PAGE_SIZE;
593 }
594
595 return (hypctx);
596 }
597
598 static int
arm_vmm_pinit(pmap_t pmap)599 arm_vmm_pinit(pmap_t pmap)
600 {
601
602 pmap_pinit_stage(pmap, PM_STAGE2, vmm_pmap_levels);
603 return (1);
604 }
605
606 struct vmspace *
vmmops_vmspace_alloc(vm_offset_t min,vm_offset_t max)607 vmmops_vmspace_alloc(vm_offset_t min, vm_offset_t max)
608 {
609 return (vmspace_alloc(min, max, arm_vmm_pinit));
610 }
611
612 void
vmmops_vmspace_free(struct vmspace * vmspace)613 vmmops_vmspace_free(struct vmspace *vmspace)
614 {
615
616 pmap_remove_pages(vmspace_pmap(vmspace));
617 vmspace_free(vmspace);
618 }
619
620 static inline void
arm64_print_hyp_regs(struct vm_exit * vme)621 arm64_print_hyp_regs(struct vm_exit *vme)
622 {
623 printf("esr_el2: 0x%016lx\n", vme->u.hyp.esr_el2);
624 printf("far_el2: 0x%016lx\n", vme->u.hyp.far_el2);
625 printf("hpfar_el2: 0x%016lx\n", vme->u.hyp.hpfar_el2);
626 printf("elr_el2: 0x%016lx\n", vme->pc);
627 }
628
629 static void
arm64_gen_inst_emul_data(struct hypctx * hypctx,uint32_t esr_iss,struct vm_exit * vme_ret)630 arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss,
631 struct vm_exit *vme_ret)
632 {
633 struct vm_guest_paging *paging;
634 struct vie *vie;
635 uint32_t esr_sas, reg_num;
636
637 /*
638 * Get the page address from HPFAR_EL2.
639 */
640 vme_ret->u.inst_emul.gpa =
641 HPFAR_EL2_FIPA_ADDR(hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2));
642 /* Bits [11:0] are the same as bits [11:0] from the virtual address. */
643 vme_ret->u.inst_emul.gpa += hypctx_read_sys_reg(hypctx, HOST_FAR_EL2) &
644 FAR_EL2_HPFAR_PAGE_MASK;
645
646 esr_sas = (esr_iss & ISS_DATA_SAS_MASK) >> ISS_DATA_SAS_SHIFT;
647 reg_num = (esr_iss & ISS_DATA_SRT_MASK) >> ISS_DATA_SRT_SHIFT;
648
649 vie = &vme_ret->u.inst_emul.vie;
650 vie->access_size = 1 << esr_sas;
651 vie->sign_extend = (esr_iss & ISS_DATA_SSE) ? 1 : 0;
652 vie->dir = (esr_iss & ISS_DATA_WnR) ? VM_DIR_WRITE : VM_DIR_READ;
653 vie->reg = reg_num;
654
655 paging = &vme_ret->u.inst_emul.paging;
656 paging->ttbr0_addr = hypctx_read_sys_reg(hypctx, TTBR0_EL1) &
657 ~(TTBR_ASID_MASK | TTBR_CnP);
658 paging->ttbr1_addr = hypctx_read_sys_reg(hypctx, TTBR1_EL1) &
659 ~(TTBR_ASID_MASK | TTBR_CnP);
660 paging->tcr_el1 = hypctx_read_sys_reg(hypctx, TCR_EL1);
661 paging->tcr2_el1 = hypctx_read_sys_reg(hypctx, TCR2_EL1);
662 paging->flags = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & (PSR_M_MASK | PSR_M_32);
663 if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_M) != 0)
664 paging->flags |= VM_GP_MMU_ENABLED;
665 }
666
667 static void
arm64_gen_reg_emul_data(uint32_t esr_iss,struct vm_exit * vme_ret)668 arm64_gen_reg_emul_data(uint32_t esr_iss, struct vm_exit *vme_ret)
669 {
670 uint32_t reg_num;
671 struct vre *vre;
672
673 /* u.hyp member will be replaced by u.reg_emul */
674 vre = &vme_ret->u.reg_emul.vre;
675
676 vre->inst_syndrome = esr_iss;
677 /* ARMv8 Architecture Manual, p. D7-2273: 1 means read */
678 vre->dir = (esr_iss & ISS_MSR_DIR) ? VM_DIR_READ : VM_DIR_WRITE;
679 reg_num = ISS_MSR_Rt(esr_iss);
680 vre->reg = reg_num;
681 }
682
683 void
raise_data_insn_abort(struct hypctx * hypctx,uint64_t far,bool dabort,int fsc)684 raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc)
685 {
686 uint64_t esr;
687
688 if ((hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & PSR_M_MASK) == PSR_M_EL0t)
689 esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT;
690 else
691 esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT;
692 /* Set the bit that changes from insn -> data abort */
693 if (dabort)
694 esr |= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT;
695 /* Set the IL bit if set by hardware */
696 esr |= hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_IL;
697
698 vmmops_exception(hypctx, esr | fsc, far);
699 }
700
701 static int
handle_el1_sync_excp(struct hypctx * hypctx,struct vm_exit * vme_ret,pmap_t pmap)702 handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
703 pmap_t pmap)
704 {
705 uint64_t gpa;
706 uint32_t esr_ec, esr_iss;
707
708 esr_ec = ESR_ELx_EXCEPTION(hypctx_read_sys_reg(hypctx, HOST_ESR_EL2));
709 esr_iss = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK;
710
711 switch (esr_ec) {
712 case EXCP_UNKNOWN:
713 vmm_stat_incr(hypctx->vcpu, VMEXIT_UNKNOWN, 1);
714 arm64_print_hyp_regs(vme_ret);
715 vme_ret->exitcode = VM_EXITCODE_HYP;
716 break;
717 case EXCP_TRAP_WFI_WFE:
718 if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & 0x3) == 0) { /* WFI */
719 vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1);
720 vme_ret->exitcode = VM_EXITCODE_WFI;
721 } else {
722 vmm_stat_incr(hypctx->vcpu, VMEXIT_WFE, 1);
723 vme_ret->exitcode = VM_EXITCODE_HYP;
724 }
725 break;
726 case EXCP_HVC:
727 vmm_stat_incr(hypctx->vcpu, VMEXIT_HVC, 1);
728 vme_ret->exitcode = VM_EXITCODE_HVC;
729 break;
730 case EXCP_MSR:
731 vmm_stat_incr(hypctx->vcpu, VMEXIT_MSR, 1);
732 arm64_gen_reg_emul_data(esr_iss, vme_ret);
733 vme_ret->exitcode = VM_EXITCODE_REG_EMUL;
734 break;
735 case EXCP_BRK:
736 vmm_stat_incr(hypctx->vcpu, VMEXIT_BRK, 1);
737 vme_ret->exitcode = VM_EXITCODE_BRK;
738 break;
739 case EXCP_SOFTSTP_EL0:
740 vmm_stat_incr(hypctx->vcpu, VMEXIT_SS, 1);
741 vme_ret->exitcode = VM_EXITCODE_SS;
742 break;
743 case EXCP_INSN_ABORT_L:
744 case EXCP_DATA_ABORT_L:
745 vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ?
746 VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1);
747 switch (hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ISS_DATA_DFSC_MASK) {
748 case ISS_DATA_DFSC_TF_L0:
749 case ISS_DATA_DFSC_TF_L1:
750 case ISS_DATA_DFSC_TF_L2:
751 case ISS_DATA_DFSC_TF_L3:
752 case ISS_DATA_DFSC_AFF_L1:
753 case ISS_DATA_DFSC_AFF_L2:
754 case ISS_DATA_DFSC_AFF_L3:
755 case ISS_DATA_DFSC_PF_L1:
756 case ISS_DATA_DFSC_PF_L2:
757 case ISS_DATA_DFSC_PF_L3:
758 gpa = HPFAR_EL2_FIPA_ADDR(hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2));
759 /* Check the IPA is valid */
760 if (gpa >= (1ul << vmm_max_ipa_bits)) {
761 raise_data_insn_abort(hypctx,
762 hypctx_read_sys_reg(hypctx, HOST_FAR_EL2),
763 esr_ec == EXCP_DATA_ABORT_L,
764 ISS_DATA_DFSC_ASF_L0);
765 vme_ret->inst_length = 0;
766 return (HANDLED);
767 }
768
769 if (vm_mem_allocated(hypctx->vcpu, gpa)) {
770 vme_ret->exitcode = VM_EXITCODE_PAGING;
771 vme_ret->inst_length = 0;
772 vme_ret->u.paging.esr = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
773 vme_ret->u.paging.gpa = gpa;
774 } else if (esr_ec == EXCP_INSN_ABORT_L) {
775 /*
776 * Raise an external abort. Device memory is
777 * not executable
778 */
779 raise_data_insn_abort(hypctx,
780 hypctx_read_sys_reg(hypctx, HOST_FAR_EL2), false,
781 ISS_DATA_DFSC_EXT);
782 vme_ret->inst_length = 0;
783 return (HANDLED);
784 } else {
785 arm64_gen_inst_emul_data(hypctx, esr_iss,
786 vme_ret);
787 vme_ret->exitcode = VM_EXITCODE_INST_EMUL;
788 }
789 break;
790 default:
791 arm64_print_hyp_regs(vme_ret);
792 vme_ret->exitcode = VM_EXITCODE_HYP;
793 break;
794 }
795
796 break;
797
798 default:
799 vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_SYNC, 1);
800 arm64_print_hyp_regs(vme_ret);
801 vme_ret->exitcode = VM_EXITCODE_HYP;
802 break;
803 }
804
805 /* We don't don't do any instruction emulation here */
806 return (UNHANDLED);
807 }
808
809 static int
arm64_handle_world_switch(struct hypctx * hypctx,int excp_type,struct vm_exit * vme,pmap_t pmap)810 arm64_handle_world_switch(struct hypctx *hypctx, int excp_type,
811 struct vm_exit *vme, pmap_t pmap)
812 {
813 int handled;
814
815 switch (excp_type) {
816 case EXCP_TYPE_EL1_SYNC:
817 /* The exit code will be set by handle_el1_sync_excp(). */
818 handled = handle_el1_sync_excp(hypctx, vme, pmap);
819 break;
820
821 case EXCP_TYPE_EL1_IRQ:
822 case EXCP_TYPE_EL1_FIQ:
823 /* The host kernel will handle IRQs and FIQs. */
824 vmm_stat_incr(hypctx->vcpu,
825 excp_type == EXCP_TYPE_EL1_IRQ ? VMEXIT_IRQ : VMEXIT_FIQ,1);
826 vme->exitcode = VM_EXITCODE_BOGUS;
827 handled = UNHANDLED;
828 break;
829
830 case EXCP_TYPE_EL1_ERROR:
831 case EXCP_TYPE_EL2_SYNC:
832 case EXCP_TYPE_EL2_IRQ:
833 case EXCP_TYPE_EL2_FIQ:
834 case EXCP_TYPE_EL2_ERROR:
835 vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_EL2, 1);
836 vme->exitcode = VM_EXITCODE_BOGUS;
837 handled = UNHANDLED;
838 break;
839
840 default:
841 vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED, 1);
842 vme->exitcode = VM_EXITCODE_BOGUS;
843 handled = UNHANDLED;
844 break;
845 }
846
847 return (handled);
848 }
849
850 static void
ptp_release(void ** cookie)851 ptp_release(void **cookie)
852 {
853 if (*cookie != NULL) {
854 vm_gpa_release(*cookie);
855 *cookie = NULL;
856 }
857 }
858
859 static void *
ptp_hold(struct vcpu * vcpu,vm_paddr_t ptpphys,size_t len,void ** cookie)860 ptp_hold(struct vcpu *vcpu, vm_paddr_t ptpphys, size_t len, void **cookie)
861 {
862 void *ptr;
863
864 ptp_release(cookie);
865 ptr = vm_gpa_hold(vcpu, ptpphys, len, VM_PROT_RW, cookie);
866 return (ptr);
867 }
868
869 /* log2 of the number of bytes in a page table entry */
870 #define PTE_SHIFT 3
871 int
vmmops_gla2gpa(void * vcpui,struct vm_guest_paging * paging,uint64_t gla,int prot,uint64_t * gpa,int * is_fault)872 vmmops_gla2gpa(void *vcpui, struct vm_guest_paging *paging, uint64_t gla,
873 int prot, uint64_t *gpa, int *is_fault)
874 {
875 struct hypctx *hypctx;
876 void *cookie;
877 uint64_t mask, *ptep, pte, pte_addr;
878 int address_bits, granule_shift, ia_bits, levels, pte_shift, tsz;
879 bool is_el0;
880
881 /* Check if the MMU is off */
882 if ((paging->flags & VM_GP_MMU_ENABLED) == 0) {
883 *is_fault = 0;
884 *gpa = gla;
885 return (0);
886 }
887
888 is_el0 = (paging->flags & PSR_M_MASK) == PSR_M_EL0t;
889
890 if (ADDR_IS_KERNEL(gla)) {
891 /* If address translation is disabled raise an exception */
892 if ((paging->tcr_el1 & TCR_EPD1) != 0) {
893 *is_fault = 1;
894 return (0);
895 }
896 if (is_el0 && (paging->tcr_el1 & TCR_E0PD1) != 0) {
897 *is_fault = 1;
898 return (0);
899 }
900 pte_addr = paging->ttbr1_addr;
901 tsz = (paging->tcr_el1 & TCR_T1SZ_MASK) >> TCR_T1SZ_SHIFT;
902 /* Clear the top byte if TBI is on */
903 if ((paging->tcr_el1 & TCR_TBI1) != 0)
904 gla |= (0xfful << 56);
905 switch (paging->tcr_el1 & TCR_TG1_MASK) {
906 case TCR_TG1_4K:
907 granule_shift = PAGE_SHIFT_4K;
908 break;
909 case TCR_TG1_16K:
910 granule_shift = PAGE_SHIFT_16K;
911 break;
912 case TCR_TG1_64K:
913 granule_shift = PAGE_SHIFT_64K;
914 break;
915 default:
916 *is_fault = 1;
917 return (EINVAL);
918 }
919 } else {
920 /* If address translation is disabled raise an exception */
921 if ((paging->tcr_el1 & TCR_EPD0) != 0) {
922 *is_fault = 1;
923 return (0);
924 }
925 if (is_el0 && (paging->tcr_el1 & TCR_E0PD0) != 0) {
926 *is_fault = 1;
927 return (0);
928 }
929 pte_addr = paging->ttbr0_addr;
930 tsz = (paging->tcr_el1 & TCR_T0SZ_MASK) >> TCR_T0SZ_SHIFT;
931 /* Clear the top byte if TBI is on */
932 if ((paging->tcr_el1 & TCR_TBI0) != 0)
933 gla &= ~(0xfful << 56);
934 switch (paging->tcr_el1 & TCR_TG0_MASK) {
935 case TCR_TG0_4K:
936 granule_shift = PAGE_SHIFT_4K;
937 break;
938 case TCR_TG0_16K:
939 granule_shift = PAGE_SHIFT_16K;
940 break;
941 case TCR_TG0_64K:
942 granule_shift = PAGE_SHIFT_64K;
943 break;
944 default:
945 *is_fault = 1;
946 return (EINVAL);
947 }
948 }
949
950 /*
951 * TODO: Support FEAT_TTST for smaller tsz values and FEAT_LPA2
952 * for larger values.
953 */
954 switch (granule_shift) {
955 case PAGE_SHIFT_4K:
956 case PAGE_SHIFT_16K:
957 /*
958 * See "Table D8-11 4KB granule, determining stage 1 initial
959 * lookup level" and "Table D8-21 16KB granule, determining
960 * stage 1 initial lookup level" from the "Arm Architecture
961 * Reference Manual for A-Profile architecture" revision I.a
962 * for the minimum and maximum values.
963 *
964 * TODO: Support less than 16 when FEAT_LPA2 is implemented
965 * and TCR_EL1.DS == 1
966 * TODO: Support more than 39 when FEAT_TTST is implemented
967 */
968 if (tsz < 16 || tsz > 39) {
969 *is_fault = 1;
970 return (EINVAL);
971 }
972 break;
973 case PAGE_SHIFT_64K:
974 /* TODO: Support 64k granule. It will probably work, but is untested */
975 default:
976 *is_fault = 1;
977 return (EINVAL);
978 }
979
980 /*
981 * Calculate the input address bits. These are 64 bit in an address
982 * with the top tsz bits being all 0 or all 1.
983 */
984 ia_bits = 64 - tsz;
985
986 /*
987 * Calculate the number of address bits used in the page table
988 * calculation. This is ia_bits minus the bottom granule_shift
989 * bits that are passed to the output address.
990 */
991 address_bits = ia_bits - granule_shift;
992
993 /*
994 * Calculate the number of levels. Each level uses
995 * granule_shift - PTE_SHIFT bits of the input address.
996 * This is because the table is 1 << granule_shift and each
997 * entry is 1 << PTE_SHIFT bytes.
998 */
999 levels = howmany(address_bits, granule_shift - PTE_SHIFT);
1000
1001 /* Mask of the upper unused bits in the virtual address */
1002 gla &= (1ul << ia_bits) - 1;
1003 hypctx = (struct hypctx *)vcpui;
1004 cookie = NULL;
1005 /* TODO: Check if the level supports block descriptors */
1006 for (;levels > 0; levels--) {
1007 int idx;
1008
1009 pte_shift = (levels - 1) * (granule_shift - PTE_SHIFT) +
1010 granule_shift;
1011 idx = (gla >> pte_shift) &
1012 ((1ul << (granule_shift - PTE_SHIFT)) - 1);
1013 while (idx > PAGE_SIZE / sizeof(pte)) {
1014 idx -= PAGE_SIZE / sizeof(pte);
1015 pte_addr += PAGE_SIZE;
1016 }
1017
1018 ptep = ptp_hold(hypctx->vcpu, pte_addr, PAGE_SIZE, &cookie);
1019 if (ptep == NULL)
1020 goto error;
1021 pte = ptep[idx];
1022
1023 /* Calculate the level we are looking at */
1024 switch (levels) {
1025 default:
1026 goto fault;
1027 /* TODO: Level -1 when FEAT_LPA2 is implemented */
1028 case 4: /* Level 0 */
1029 if ((pte & ATTR_DESCR_MASK) != L0_TABLE)
1030 goto fault;
1031 /* FALLTHROUGH */
1032 case 3: /* Level 1 */
1033 case 2: /* Level 2 */
1034 switch (pte & ATTR_DESCR_MASK) {
1035 /* Use L1 macro as all levels are the same */
1036 case L1_TABLE:
1037 /* Check if EL0 can access this address space */
1038 if (is_el0 &&
1039 (pte & TATTR_AP_TABLE_NO_EL0) != 0)
1040 goto fault;
1041 /* Check if the address space is writable */
1042 if ((prot & PROT_WRITE) != 0 &&
1043 (pte & TATTR_AP_TABLE_RO) != 0)
1044 goto fault;
1045 if ((prot & PROT_EXEC) != 0) {
1046 /* Check the table exec attribute */
1047 if ((is_el0 &&
1048 (pte & TATTR_UXN_TABLE) != 0) ||
1049 (!is_el0 &&
1050 (pte & TATTR_PXN_TABLE) != 0))
1051 goto fault;
1052 }
1053 pte_addr = pte & ~ATTR_MASK;
1054 break;
1055 case L1_BLOCK:
1056 goto done;
1057 default:
1058 goto fault;
1059 }
1060 break;
1061 case 1: /* Level 3 */
1062 if ((pte & ATTR_DESCR_MASK) == L3_PAGE)
1063 goto done;
1064 goto fault;
1065 }
1066 }
1067
1068 done:
1069 /* Check if EL0 has access to the block/page */
1070 if (is_el0 && (pte & ATTR_S1_AP(ATTR_S1_AP_USER)) == 0)
1071 goto fault;
1072 if ((prot & PROT_WRITE) != 0 && (pte & ATTR_S1_AP_RW_BIT) != 0)
1073 goto fault;
1074 if ((prot & PROT_EXEC) != 0) {
1075 if ((is_el0 && (pte & ATTR_S1_UXN) != 0) ||
1076 (!is_el0 && (pte & ATTR_S1_PXN) != 0))
1077 goto fault;
1078 }
1079 mask = (1ul << pte_shift) - 1;
1080 *gpa = (pte & ~ATTR_MASK) | (gla & mask);
1081 *is_fault = 0;
1082 ptp_release(&cookie);
1083 return (0);
1084
1085 error:
1086 ptp_release(&cookie);
1087 return (EFAULT);
1088 fault:
1089 *is_fault = 1;
1090 ptp_release(&cookie);
1091 return (0);
1092 }
1093
1094 int
vmmops_run(void * vcpui,register_t pc,pmap_t pmap,struct vm_eventinfo * evinfo)1095 vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
1096 {
1097 uint64_t excp_type, vbar_el1, new_spsr;
1098 int handled;
1099 register_t daif;
1100 struct hyp *hyp;
1101 struct hypctx *hypctx;
1102 struct vcpu *vcpu;
1103 struct vm_exit *vme;
1104 int mode;
1105
1106 hypctx = (struct hypctx *)vcpui;
1107 hyp = hypctx->hyp;
1108 vcpu = hypctx->vcpu;
1109 vme = vm_exitinfo(vcpu);
1110
1111 hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, (uint64_t)pc);
1112
1113 for (;;) {
1114 if (hypctx->has_exception) {
1115 hypctx->has_exception = false;
1116 hypctx_write_sys_reg(hypctx, ELR_EL1,
1117 hypctx_read_sys_reg(hypctx, HOST_ELR_EL2));
1118
1119 mode = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) &
1120 (PSR_M_MASK | PSR_M_32);
1121
1122 vbar_el1 = hypctx_read_sys_reg(hypctx, VBAR_EL1);
1123 if (mode == PSR_M_EL1t) {
1124 hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
1125 vbar_el1 + 0x0);
1126 } else if (mode == PSR_M_EL1h) {
1127 hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
1128 vbar_el1 + 0x200);
1129 } else if ((mode & PSR_M_32) == PSR_M_64) {
1130 /* 64-bit EL0 */
1131 hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
1132 vbar_el1 + 0x400);
1133 } else {
1134 /* 32-bit EL0 */
1135 hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
1136 vbar_el1 + 0x600);
1137 }
1138
1139 /* Set the new spsr */
1140 new_spsr = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
1141 hypctx_write_sys_reg(hypctx, SPSR_EL1, new_spsr);
1142
1143 /* Set the new cpsr */
1144 hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2,
1145 new_spsr & PSR_FLAGS);
1146 *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_DAIF |
1147 PSR_M_EL1h;
1148
1149 /*
1150 * Update fields that may change on exeption entry
1151 * based on how sctlr_el1 is configured.
1152 */
1153 if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
1154 SCTLR_SPAN) == 0)
1155 *hypctx_sys_reg(hypctx,
1156 HOST_SPSR_EL2) |= PSR_PAN;
1157 if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
1158 SCTLR_DSSBS) == 0)
1159 *hypctx_sys_reg(hypctx,
1160 HOST_SPSR_EL2) &= ~PSR_SSBS;
1161 else
1162 *hypctx_sys_reg(hypctx,
1163 HOST_SPSR_EL2) |= PSR_SSBS;
1164 }
1165
1166 daif = intr_disable();
1167
1168 /* Check if the vcpu is suspended */
1169 if (vcpu_suspended(evinfo)) {
1170 intr_restore(daif);
1171 vm_exit_suspended(vcpu, pc);
1172 break;
1173 }
1174
1175 if (vcpu_debugged(vcpu)) {
1176 intr_restore(daif);
1177 vm_exit_debug(vcpu, pc);
1178 break;
1179 }
1180
1181 /* Activate the stage2 pmap so the vmid is valid */
1182 pmap_activate_vm(pmap);
1183 hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2,
1184 pmap_to_ttbr0(pmap));
1185
1186 /*
1187 * TODO: What happens if a timer interrupt is asserted exactly
1188 * here, but for the previous VM?
1189 */
1190 arm64_set_active_vcpu(hypctx);
1191 vgic_flush_hwstate(hypctx);
1192
1193 /* Call into EL2 to switch to the guest */
1194 excp_type = vmm_enter_guest(hyp, hypctx);
1195
1196 vgic_sync_hwstate(hypctx);
1197 vtimer_sync_hwstate(hypctx);
1198
1199 /*
1200 * Deactivate the stage2 pmap.
1201 */
1202 PCPU_SET(curvmpmap, NULL);
1203 intr_restore(daif);
1204
1205 vmm_stat_incr(vcpu, VMEXIT_COUNT, 1);
1206 if (excp_type == EXCP_TYPE_MAINT_IRQ)
1207 continue;
1208
1209 vme->pc = hypctx_read_sys_reg(hypctx, HOST_ELR_EL2);
1210 vme->inst_length = INSN_SIZE;
1211 vme->u.hyp.exception_nr = excp_type;
1212 vme->u.hyp.esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
1213 vme->u.hyp.far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2);
1214 vme->u.hyp.hpfar_el2 = hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2);
1215
1216 handled = arm64_handle_world_switch(hypctx, excp_type, vme,
1217 pmap);
1218 if (handled == UNHANDLED)
1219 /* Exit loop to emulate instruction. */
1220 break;
1221 else
1222 /* Resume guest execution from the next instruction. */
1223 *hypctx_sys_reg(hypctx, HOST_ELR_EL2) += vme->inst_length;
1224 }
1225
1226 return (0);
1227 }
1228
1229 static void
arm_pcpu_vmcleanup(void * arg)1230 arm_pcpu_vmcleanup(void *arg)
1231 {
1232 struct hyp *hyp;
1233 int i, maxcpus;
1234
1235 hyp = arg;
1236 maxcpus = vm_get_maxcpus(hyp->vm);
1237 for (i = 0; i < maxcpus; i++) {
1238 if (arm64_get_active_vcpu() == hyp->ctx[i]) {
1239 arm64_set_active_vcpu(NULL);
1240 break;
1241 }
1242 }
1243 }
1244
1245 void
vmmops_vcpu_cleanup(void * vcpui)1246 vmmops_vcpu_cleanup(void *vcpui)
1247 {
1248 struct hypctx *hypctx = vcpui;
1249
1250 vtimer_cpucleanup(hypctx);
1251 vgic_cpucleanup(hypctx);
1252
1253 if (!in_vhe())
1254 vmmpmap_remove(hypctx->el2_addr, el2_hypctx_size(), true);
1255
1256 free(hypctx, M_HYP);
1257 }
1258
1259 void
vmmops_cleanup(void * vmi)1260 vmmops_cleanup(void *vmi)
1261 {
1262 struct hyp *hyp = vmi;
1263
1264 vtimer_vmcleanup(hyp);
1265 vgic_vmcleanup(hyp);
1266
1267 smp_rendezvous(NULL, arm_pcpu_vmcleanup, NULL, hyp);
1268
1269 if (!in_vhe())
1270 vmmpmap_remove(hyp->el2_addr, el2_hyp_size(hyp->vm), true);
1271
1272 free(hyp, M_HYP);
1273 }
1274
1275 /*
1276 * Return register value. Registers have different sizes and an explicit cast
1277 * must be made to ensure proper conversion.
1278 */
1279 static uint64_t *
hypctx_regptr(struct hypctx * hypctx,int reg)1280 hypctx_regptr(struct hypctx *hypctx, int reg)
1281 {
1282 switch (reg) {
1283 case VM_REG_GUEST_X0 ... VM_REG_GUEST_X29:
1284 return hypctx_sys_reg(hypctx, GPR_X(reg));
1285 case VM_REG_GUEST_LR:
1286 return hypctx_sys_reg(hypctx, GPR_LR);
1287 case VM_REG_GUEST_SP:
1288 return hypctx_sys_reg(hypctx, HOST_SP_EL1);
1289 case VM_REG_GUEST_CPSR:
1290 return hypctx_sys_reg(hypctx, HOST_SPSR_EL2);
1291 case VM_REG_GUEST_PC:
1292 return hypctx_sys_reg(hypctx, HOST_ELR_EL2);
1293 case VM_REG_GUEST_SCTLR_EL1:
1294 return hypctx_sys_reg(hypctx, SCTLR_EL1);
1295 case VM_REG_GUEST_TTBR0_EL1:
1296 return hypctx_sys_reg(hypctx, TTBR0_EL1);
1297 case VM_REG_GUEST_TTBR1_EL1:
1298 return hypctx_sys_reg(hypctx, TTBR1_EL1);
1299 case VM_REG_GUEST_TCR_EL1:
1300 return hypctx_sys_reg(hypctx, TCR_EL1);
1301 case VM_REG_GUEST_TCR2_EL1:
1302 return hypctx_sys_reg(hypctx, TCR2_EL1);
1303 case VM_REG_GUEST_MPIDR_EL1:
1304 return hypctx_sys_reg(hypctx, HOST_VMPIDR_EL2);
1305 default:
1306 break;
1307 }
1308 return (NULL);
1309 }
1310
1311 int
vmmops_getreg(void * vcpui,int reg,uint64_t * retval)1312 vmmops_getreg(void *vcpui, int reg, uint64_t *retval)
1313 {
1314 uint64_t *regp;
1315 int running, hostcpu;
1316 struct hypctx *hypctx = vcpui;
1317
1318 running = vcpu_is_running(hypctx->vcpu, &hostcpu);
1319 if (running && hostcpu != curcpu)
1320 panic("arm_getreg: %s%d is running", vm_name(hypctx->hyp->vm),
1321 vcpu_vcpuid(hypctx->vcpu));
1322
1323 regp = hypctx_regptr(hypctx, reg);
1324 if (regp == NULL)
1325 return (EINVAL);
1326
1327 *retval = *regp;
1328 return (0);
1329 }
1330
1331 int
vmmops_setreg(void * vcpui,int reg,uint64_t val)1332 vmmops_setreg(void *vcpui, int reg, uint64_t val)
1333 {
1334 uint64_t *regp;
1335 struct hypctx *hypctx = vcpui;
1336 int running, hostcpu;
1337
1338 running = vcpu_is_running(hypctx->vcpu, &hostcpu);
1339 if (running && hostcpu != curcpu)
1340 panic("arm_setreg: %s%d is running", vm_name(hypctx->hyp->vm),
1341 vcpu_vcpuid(hypctx->vcpu));
1342
1343 regp = hypctx_regptr(hypctx, reg);
1344 if (regp == NULL)
1345 return (EINVAL);
1346
1347 *regp = val;
1348 return (0);
1349 }
1350
1351 int
vmmops_exception(void * vcpui,uint64_t esr,uint64_t far)1352 vmmops_exception(void *vcpui, uint64_t esr, uint64_t far)
1353 {
1354 struct hypctx *hypctx = vcpui;
1355 int running, hostcpu;
1356
1357 running = vcpu_is_running(hypctx->vcpu, &hostcpu);
1358 if (running && hostcpu != curcpu)
1359 panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm),
1360 vcpu_vcpuid(hypctx->vcpu));
1361
1362 hypctx_write_sys_reg(hypctx, FAR_EL1, far);
1363 hypctx_write_sys_reg(hypctx, ESR_EL1, esr);
1364 hypctx->has_exception = true;
1365
1366 return (0);
1367 }
1368
1369 int
vmmops_getcap(void * vcpui,int num,int * retval)1370 vmmops_getcap(void *vcpui, int num, int *retval)
1371 {
1372 struct hypctx *hypctx = vcpui;
1373 int ret;
1374
1375 ret = ENOENT;
1376
1377 switch (num) {
1378 case VM_CAP_UNRESTRICTED_GUEST:
1379 *retval = 1;
1380 ret = 0;
1381 break;
1382 case VM_CAP_BRK_EXIT:
1383 case VM_CAP_SS_EXIT:
1384 case VM_CAP_MASK_HWINTR:
1385 *retval = (hypctx->setcaps & (1ul << num)) != 0;
1386 break;
1387 default:
1388 break;
1389 }
1390
1391 return (ret);
1392 }
1393
1394 int
vmmops_setcap(void * vcpui,int num,int val)1395 vmmops_setcap(void *vcpui, int num, int val)
1396 {
1397 struct hypctx *hypctx = vcpui;
1398 int ret;
1399 uint64_t host_spsr_el2;
1400
1401 ret = 0;
1402
1403 switch (num) {
1404 case VM_CAP_BRK_EXIT:
1405 if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0))
1406 break;
1407 if (val != 0)
1408 *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) |= MDCR_EL2_TDE;
1409 else if ((hypctx->setcaps & (1ul << VM_CAP_SS_EXIT)) == 0)
1410 *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
1411 break;
1412 case VM_CAP_SS_EXIT:
1413 if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0))
1414 break;
1415
1416 if (val != 0) {
1417 hypctx->debug_spsr |= (hypctx_read_sys_reg(hypctx,
1418 HOST_SPSR_EL2) & PSR_SS);
1419 hypctx->debug_mdscr |= (hypctx_read_sys_reg(hypctx,
1420 MDSCR_EL1) & MDSCR_SS);
1421
1422 *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_SS;
1423 *hypctx_sys_reg(hypctx, MDSCR_EL1) |= MDSCR_SS;
1424 *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) |= MDCR_EL2_TDE;
1425 } else {
1426 *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) &= ~PSR_SS;
1427 *hypctx_sys_reg(hypctx,
1428 HOST_SPSR_EL2) |= hypctx->debug_spsr;
1429 hypctx->debug_spsr &= ~PSR_SS;
1430 *hypctx_sys_reg(hypctx, MDSCR_EL1) &= ~MDSCR_SS;
1431 *hypctx_sys_reg(hypctx,
1432 MDSCR_EL1) |= hypctx->debug_mdscr;
1433 hypctx->debug_mdscr &= ~MDSCR_SS;
1434 if ((hypctx->setcaps & (1ul << VM_CAP_BRK_EXIT)) == 0)
1435 *hypctx_sys_reg(hypctx,
1436 HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
1437 }
1438 break;
1439 case VM_CAP_MASK_HWINTR:
1440 if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0))
1441 break;
1442
1443 host_spsr_el2 = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
1444 if (val != 0) {
1445 hypctx->debug_spsr |= (host_spsr_el2 & (PSR_I | PSR_F));
1446 host_spsr_el2 |= PSR_I | PSR_F;
1447 } else {
1448 host_spsr_el2 &= ~(PSR_I | PSR_F);
1449 host_spsr_el2 |= (hypctx->debug_spsr & (PSR_I | PSR_F));
1450 hypctx->debug_spsr &= ~(PSR_I | PSR_F);
1451 }
1452 hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, host_spsr_el2);
1453 break;
1454 default:
1455 ret = ENOENT;
1456 break;
1457 }
1458
1459 if (ret == 0) {
1460 if (val == 0)
1461 hypctx->setcaps &= ~(1ul << num);
1462 else
1463 hypctx->setcaps |= (1ul << num);
1464 }
1465
1466 return (ret);
1467 }
1468