1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 #ifndef _VMM_ARM64_H_
29 #define _VMM_ARM64_H_
30
31 #include <machine/reg.h>
32 #include <machine/hypervisor.h>
33 #include <machine/pcpu.h>
34
35 #include "mmu.h"
36 #include "io/vgic_v3.h"
37 #include "io/vtimer.h"
38
39 struct vgic_v3;
40 struct vgic_v3_cpu;
41
42 /*
43 Encode the index of a given register within VNCR into the enum value.
44 We move the indices up by VNCR_START to not interfere with values for
45 non-VNCR registers.
46 */
47 #define VNCR_REG(reg) reg = (VNCR_START + VNCR_##reg / 8)
48 /* Retrieve the VNCR offset from the enum value */
49 #define REG_VNCR_OFFSET(val) ((val - VNCR_START) * 8)
50
51 /* Accessors for indices of PME and DBG registers */
52 #define PMEVCNTR_EL0(n) (PMEVCNTR0_EL0 + MIN(n, 30))
53 #define PMEVTYPER_EL0(n) (PMEVTYPER0_EL0 + MIN(n, 30))
54 #define DBGBCR_EL1(n) (DBGBCR0_EL1 + MIN(n, 15))
55 #define DBGBVR_EL1(n) (DBGBVR0_EL1 + MIN(n, 15))
56 #define DBGWCR_EL1(n) (DBGWCR0_EL1 + MIN(n, 15))
57 #define DBGWVR_EL1(n) (DBGWVR0_EL1 + MIN(n, 15))
58
59 #define HOST_ICH_LR_EL2(n) (HOST_ICH_LR0_EL2 + MIN(n, VGIC_ICH_LR_NUM_MAX-1))
60 #define HOST_ICH_AP0R_EL2(n) (HOST_ICH_AP0R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
61 #define HOST_ICH_AP1R_EL2(n) (HOST_ICH_AP1R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
62 #define GPR_X(n) (GPR_X0 + MIN(n, 30))
63
64 enum hypctx_sysreg {
65 HOST_SPSR_EL2,
66 HOST_ESR_EL2,
67 CSSELR_EL1, /* Cache Size Selection Register */
68 MDCCINT_EL1, /* Monitor DCC Interrupt Enable Register */
69 PAR_EL1, /* Physical Address Register */
70
71 /* PMU Registers */
72 PMCR_EL0, /* Performance Monitors Control Register */
73 PMCCNTR_EL0,
74 PMCCFILTR_EL0,
75 PMUSERENR_EL0,
76 PMSELR_EL0,
77 PMXEVCNTR_EL0,
78 PMCNTENSET_EL0,
79 PMINTENSET_EL1,
80 PMOVSSET_EL0,
81 /* Access these through macros defined above, e.g. PMEVCNTR_EL0(5) */
82 PMEVCNTR0_EL0,
83 PMEVCNTR30_EL0 = PMEVCNTR0_EL0 + 30,
84 PMEVTYPER0_EL0,
85 PMEVTYPER30_EL0 = PMEVTYPER0_EL0 + 30,
86
87 /* DBG Registers */
88 DBGCLAIMSET_EL1,
89 /* Access these through macros defined above, e.g. DBGBCR_EL1(5) */
90 DBGBCR0_EL1, /* Debug Breakpoint Control Registers */
91 DBGBCR15_EL1 = DBGBCR0_EL1 + 15,
92 DBGBVR0_EL1, /* Debug Breakpoint Value Registers */
93 DBGBVR15_EL1 = DBGBVR0_EL1 + 15,
94 DBGWCR0_EL1, /* Debug Watchpoint Control Registers */
95 DBGWCR15_EL1 = DBGWCR0_EL1 + 15,
96 DBGWVR0_EL1, /* Debug Watchpoint Value Registers */
97 DBGWVR15_EL1 = DBGWVR0_EL1 + 15,
98
99 /* EL2 registers used to control the guest, but not exposed to it */
100 HOST_CPTR_EL2, /* Architectural Feature Trap Register */
101 HOST_HCR_EL2, /* Hypervisor Configuration Register */
102 HOST_HCRX_EL2, /* Extended Hypervisor Configuration Register */
103 HOST_MDCR_EL2, /* Monitor Debug Configuration Register */
104 HOST_VPIDR_EL2, /* Virtualization Processor ID Register */
105 HOST_VMPIDR_EL2, /* Virtualization Multiprocessor ID Register */
106 /* On systems without NV2 this still points to the register storage
107 memory page */
108 HOST_VNCR_EL2, /* Virtual Nested Control Register */
109 HOST_VTTBR_EL2,
110
111 /* FEAT_FGT registers */
112 /*HOST_HAFGRTR_EL2; */ /* For FEAT_AMUv1 (not supported) */
113 HOST_HDFGRTR_EL2,
114 HOST_HDFGWTR_EL2,
115 HOST_HFGITR_EL2,
116 HOST_HFGRTR_EL2,
117 HOST_HFGWTR_EL2,
118
119 /* FEAT_FGT2 registers */
120 HOST_HDFGRTR2_EL2,
121 HOST_HDFGWTR2_EL2,
122 HOST_HFGITR2_EL2,
123 HOST_HFGRTR2_EL2,
124 HOST_HFGWTR2_EL2,
125
126 /* Exit info registers */
127 HOST_FAR_EL2, /* Fault Address Register */
128 HOST_HPFAR_EL2, /* Hypervisor IPA Fault Address Register */
129
130 HOST_ICH_EISR_EL2, /* End of Interrupt Status Register */
131 HOST_ICH_ELRSR_EL2, /* Empty List Register Status Register */
132 HOST_ICH_HCR_EL2, /* Hyp Control Register */
133 HOST_ICH_MISR_EL2, /* Maintenance Interrupt State Register */
134 HOST_ICH_VMCR_EL2, /* Virtual Machine Control Register */
135
136 /*
137 * The List Registers are part of the VM context and are modified on a
138 * world switch. They need to be allocated statically so they are
139 * mapped in the EL2 translation tables when struct hypctx is mapped.
140 */
141 HOST_ICH_LR0_EL2,
142 HOST_ICH_LR_MAX_EL2 = HOST_ICH_LR0_EL2 + (VGIC_ICH_LR_NUM_MAX - 1),
143 /* Active Priorities Registers for Group 0 and 1 interrupts */
144 HOST_ICH_AP0R0_EL2,
145 HOST_ICH_AP0R_MAX_EL2 = HOST_ICH_AP0R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
146 HOST_ICH_AP1R0_EL2,
147 HOST_ICH_AP1R_MAX_EL2 = HOST_ICH_AP1R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
148
149 HOST_CNTHCTL_EL2,
150 HOST_CNTVOFF_EL2,
151
152 NR_NON_VNCR_REGS,
153
154 /* VNCR Registers */
155 VNCR_START,
156
157 VNCR_REG(VTTBR_EL2),
158 VNCR_REG(VSTTBR_EL2),
159 VNCR_REG(VTCR_EL2),
160 VNCR_REG(VSTCR_EL2),
161 VNCR_REG(VMPIDR_EL2), /* Virtualization Multiprocessor ID Register */
162 VNCR_REG(CNTVOFF_EL2),
163 VNCR_REG(HCR_EL2), /* Hypervisor Configuration Register */
164 VNCR_REG(HSTR_EL2),
165 VNCR_REG(VPIDR_EL2), /* Virtualization Processor ID Register */
166 VNCR_REG(TPIDR_EL2),
167 VNCR_REG(HCRX_EL2), /* Extended Hypervisor Configuration Register */
168 VNCR_REG(VNCR_EL2),
169 VNCR_REG(CPACR_EL1), /* Architectural Feature Access Control Register */
170 VNCR_REG(CONTEXTIDR_EL1), /* Current Process Identifier */
171 VNCR_REG(SCTLR_EL1), /* System Control Register */
172 VNCR_REG(ACTLR_EL1), /* Auxiliary Control Register */
173 VNCR_REG(TCR_EL1), /* Translation Control Register */
174 VNCR_REG(AFSR0_EL1), /* Auxiliary Fault Status Register 0 */
175 VNCR_REG(AFSR1_EL1), /* Auxiliary Fault Status Register 1 */
176 VNCR_REG(ESR_EL1), /* Exception Syndrome Register */
177 VNCR_REG(MAIR_EL1), /* Memory Attribute Indirection Register */
178 VNCR_REG(AMAIR_EL1), /* Auxiliary Memory Attribute Indirection Register */
179 VNCR_REG(MDSCR_EL1), /* Monitor Debug System Control Register */
180 VNCR_REG(SPSR_EL1), /* Saved Program Status Register */
181 VNCR_REG(CNTV_CVAL_EL0),
182 VNCR_REG(CNTV_CTL_EL0),
183 VNCR_REG(CNTP_CVAL_EL0),
184 VNCR_REG(CNTP_CTL_EL0),
185 VNCR_REG(SCXTNUM_EL1),
186 VNCR_REG(TFSR_EL1),
187 VNCR_REG(HDFGRTR2_EL2),
188 VNCR_REG(CNTPOFF_EL2),
189 VNCR_REG(HDFGWTR2_EL2),
190 VNCR_REG(HFGRTR_EL2),
191 VNCR_REG(HFGWTR_EL2),
192 VNCR_REG(HFGITR_EL2),
193 VNCR_REG(HDFGRTR_EL2),
194 VNCR_REG(HDFGWTR_EL2),
195 VNCR_REG(ZCR_EL1),
196 VNCR_REG(HAFGRTR_EL2),
197 VNCR_REG(SMCR_EL1),
198 VNCR_REG(SMPRIMAP_EL2),
199 VNCR_REG(TTBR0_EL1), /* Translation Table Base Register 0 */
200 VNCR_REG(TTBR1_EL1), /* Translation Table Base Register 1 */
201 VNCR_REG(FAR_EL1), /* Fault Address Register */
202 VNCR_REG(ELR_EL1), /* Exception Link Register */
203 VNCR_REG(SP_EL1),
204 VNCR_REG(VBAR_EL1), /* Vector Base Address Register */
205 VNCR_REG(TCR2_EL1), /* Translation Control Register 2 */
206 VNCR_REG(SCTLR2_EL1),
207 VNCR_REG(MAIR2_EL1),
208 VNCR_REG(AMAIR2_EL1),
209 VNCR_REG(PIRE0_EL1),
210 VNCR_REG(PIRE0_EL2),
211 VNCR_REG(PIR_EL1),
212 VNCR_REG(POR_EL1),
213 VNCR_REG(S2PIR_EL2),
214 VNCR_REG(S2POR_EL1),
215 VNCR_REG(HFGRTR2_EL2),
216 VNCR_REG(HFGWTR2_EL2),
217 VNCR_REG(PFAR_EL1),
218 VNCR_REG(HFGITR2_EL2),
219 VNCR_REG(SCTLRMASK_EL1),
220 VNCR_REG(CPACRMASK_EL1),
221 VNCR_REG(SCTLR2MASK_EL1),
222 VNCR_REG(TCRMASK_EL1),
223 VNCR_REG(TCR2MASK_EL1),
224 VNCR_REG(ACTLRMASK_EL1),
225 VNCR_REG(ICH_HCR_EL2),
226 VNCR_REG(ICH_VMCR_EL2),
227 VNCR_REG(VDISR_EL2),
228 VNCR_REG(VSESR_EL2),
229 VNCR_REG(PMBLIMITR_EL1),
230 VNCR_REG(PMBPTR_EL1),
231 VNCR_REG(PMBSR_EL1),
232 VNCR_REG(PMSCR_EL1),
233 VNCR_REG(PMSEVFR_EL1),
234 VNCR_REG(PMSICR_EL1),
235 VNCR_REG(PMSIRR_EL1),
236 VNCR_REG(PMSLATFR_EL1),
237 VNCR_REG(PMSNEVFR_EL1),
238 VNCR_REG(PMSDSFR_EL1),
239 VNCR_REG(TRFCR_EL1),
240 VNCR_REG(TRCITECR_EL1),
241 VNCR_REG(GCSPR_EL1),
242 VNCR_REG(GCSCR_EL1),
243 VNCR_REG(BRBCR_EL1),
244 VNCR_REG(SPMACCESSR_EL1),
245
246 /* Virtual-address-sized registers */
247 VA_REGS_START,
248
249 /* Do not reorder these without matching asm changes */
250 GPR_LR,
251 GPR_X0,
252 GPR_X30 = GPR_X0 + 30,
253 /* These can be reordered freely */
254 SP_EL0, /* Stack pointer */
255 TPIDR_EL0, /* EL0 Software ID Register */
256 TPIDRRO_EL0, /* Read-only Thread ID Register */
257 TPIDR_EL1, /* EL1 Software ID Register */
258 HOST_SP_EL1,
259 HOST_ELR_EL2,
260
261 VA_REGS_END,
262 };
263
264 /*
265 * Per-vCPU hypervisor state.
266 */
267 struct hypctx {
268 /* Virtual-address-sized registers */
269 uint64_t va_regs[VA_REGS_END - VA_REGS_START - 1];
270 /* Non-VNCR register state */
271 uint64_t sys_regs[NR_NON_VNCR_REGS];
272
273 struct hyp *hyp;
274 struct vcpu *vcpu;
275
276 struct vtimer_cpu vtimer_cpu;
277
278 uint64_t setcaps; /* Currently enabled capabilities. */
279
280 /* vCPU state used to handle guest debugging. */
281 uint64_t debug_spsr; /* Saved guest SPSR */
282 uint64_t debug_mdscr; /* Saved guest MDSCR */
283
284 struct {
285 uint16_t ich_lr_num;
286 uint16_t ich_apr_num;
287 } vgic_v3;
288 struct vgic_v3_cpu *vgic_cpu;
289 bool has_exception;
290 bool dbg_oslock;
291
292 /*
293 * Memory page pointed at by VNCR_EL2. Contains storage for registers
294 * the accesses to which are redirected to memory by FEAT_NV2.
295 * NOTE: The storage is still used even if FEAT_NV2 is not present.
296 */
297 void *vncr_regs;
298 /* Memory page used to store the host's values of VNCR registers. */
299 void *host_vncr_regs;
300
301 uint64_t el2_addr; /* The address of this in el2 space */
302 uint64_t el2_vncr_addr; /* The address of vncr_regs in el2 space */
303 uint64_t el2_host_vncr_addr;
304 };
305
306 /* For non-VHE vmm_hyp.c, this will already be defined in vmm_nvhe.c */
307 #ifndef __hypctx_vncr_sysreg
308 #define __hypctx_vncr_sysreg(hypctx, reg) \
309 ((uint64_t *)((char *)hypctx->vncr_regs + REG_VNCR_OFFSET(reg)))
310 #endif
311
312 #ifndef __hypctx_va_sysreg
313 #define __hypctx_va_sysreg(hypctx, reg) \
314 (&hypctx->va_regs[reg - VA_REGS_START - 1])
315 #endif
316
317 /* Calling this with markers like reg=VNCR_START or reg=NR_NON_VNCR_REGS
318 is a bad idea */
319 static inline uint64_t *
hypctx_sys_reg(struct hypctx * hypctx,int reg)320 hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */)
321 {
322 /* Extract this into a separate helper when we actually support 128-bit regs */
323 if (reg > VA_REGS_START)
324 return (__hypctx_va_sysreg(hypctx, reg));
325 if (reg > VNCR_START)
326 return (__hypctx_vncr_sysreg(hypctx, reg));
327 return (&hypctx->sys_regs[reg]);
328 }
329
330 static inline void
hypctx_write_sys_reg(struct hypctx * hypctx,enum hypctx_sysreg reg,uint64_t val)331 hypctx_write_sys_reg(struct hypctx *hypctx, enum hypctx_sysreg reg,
332 uint64_t val)
333 {
334 *hypctx_sys_reg(hypctx, reg) = val;
335 }
336
337 static inline uint64_t
hypctx_read_sys_reg(struct hypctx * hypctx,enum hypctx_sysreg reg)338 hypctx_read_sys_reg(struct hypctx *hypctx, enum hypctx_sysreg reg)
339 {
340 return (*hypctx_sys_reg(hypctx, reg));
341 }
342
343 struct hyp {
344 struct vm *vm;
345 uint64_t vmid_generation;
346 uint64_t cntvoff_el2; /* VM-wide virtual timer offset */
347 uint64_t el2_addr; /* The address of this in el2 space */
348 uint64_t feats; /* Which features are enabled */
349 #define HYP_FEAT_HCX (0x1ul << 0)
350 #define HYP_FEAT_ECV_POFF (0x1ul << 1)
351 #define HYP_FEAT_FGT (0x1ul << 2)
352 #define HYP_FEAT_FGT2 (0x1ul << 3)
353 bool vgic_attached;
354 struct vgic_v3 *vgic;
355 struct hypctx *ctx[];
356 };
357
358 uint64_t vmm_call_hyp(uint64_t, ...);
359
360 #if 0
361 #define eprintf(fmt, ...) printf("%s:%d " fmt, __func__, __LINE__, ##__VA_ARGS__)
362 #else
363 #define eprintf(fmt, ...) do {} while(0)
364 #endif
365
366 struct hypctx *arm64_get_active_vcpu(void);
367 void raise_data_insn_abort(struct hypctx *, uint64_t, bool, int);
368
369 #endif /* !_VMM_ARM64_H_ */
370