1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Andrew Turner
5 *
6 * This work was supported by Innovate UK project 105694, "Digital Security
7 * by Design (DSbD) Technology Platform Prototype".
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/types.h>
33 #include <sys/proc.h>
34
35 #include <machine/armreg.h>
36
37 #include "arm64.h"
38 #include "hyp.h"
39
40 struct hypctx;
41
42 uint64_t VMM_HYP_FUNC(do_call_guest)(struct hypctx *);
43
44 static void
vmm_hyp_reg_store(struct hypctx * hypctx,struct hyp * hyp,bool guest)45 vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
46 {
47 uint64_t dfr0;
48
49 /* Store the guest VFP registers */
50 if (guest) {
51 /* Store the timer registers */
52 hypctx->vtimer_cpu.cntkctl_el1 =
53 READ_SPECIALREG(EL1_REG(CNTKCTL));
54 hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
55 READ_SPECIALREG(EL0_REG(CNTV_CVAL));
56 hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
57 READ_SPECIALREG(EL0_REG(CNTV_CTL));
58
59 /* Store the GICv3 registers */
60 hypctx->vgic_v3_regs.ich_eisr_el2 =
61 READ_SPECIALREG(ich_eisr_el2);
62 hypctx->vgic_v3_regs.ich_elrsr_el2 =
63 READ_SPECIALREG(ich_elrsr_el2);
64 hypctx->vgic_v3_regs.ich_hcr_el2 =
65 READ_SPECIALREG(ich_hcr_el2);
66 hypctx->vgic_v3_regs.ich_misr_el2 =
67 READ_SPECIALREG(ich_misr_el2);
68 hypctx->vgic_v3_regs.ich_vmcr_el2 =
69 READ_SPECIALREG(ich_vmcr_el2);
70 switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
71 #define STORE_LR(x) \
72 case x: \
73 hypctx->vgic_v3_regs.ich_lr_el2[x] = \
74 READ_SPECIALREG(ich_lr ## x ##_el2)
75 STORE_LR(15);
76 STORE_LR(14);
77 STORE_LR(13);
78 STORE_LR(12);
79 STORE_LR(11);
80 STORE_LR(10);
81 STORE_LR(9);
82 STORE_LR(8);
83 STORE_LR(7);
84 STORE_LR(6);
85 STORE_LR(5);
86 STORE_LR(4);
87 STORE_LR(3);
88 STORE_LR(2);
89 STORE_LR(1);
90 default:
91 STORE_LR(0);
92 #undef STORE_LR
93 }
94
95 switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
96 #define STORE_APR(x) \
97 case x: \
98 hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
99 READ_SPECIALREG(ich_ap0r ## x ##_el2); \
100 hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
101 READ_SPECIALREG(ich_ap1r ## x ##_el2)
102 STORE_APR(3);
103 STORE_APR(2);
104 STORE_APR(1);
105 default:
106 STORE_APR(0);
107 #undef STORE_APR
108 }
109 }
110
111 dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
112 switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
113 #define STORE_DBG_BRP(x) \
114 case x: \
115 hypctx->dbgbcr_el1[x] = \
116 READ_SPECIALREG(dbgbcr ## x ## _el1); \
117 hypctx->dbgbvr_el1[x] = \
118 READ_SPECIALREG(dbgbvr ## x ## _el1)
119 STORE_DBG_BRP(15);
120 STORE_DBG_BRP(14);
121 STORE_DBG_BRP(13);
122 STORE_DBG_BRP(12);
123 STORE_DBG_BRP(11);
124 STORE_DBG_BRP(10);
125 STORE_DBG_BRP(9);
126 STORE_DBG_BRP(8);
127 STORE_DBG_BRP(7);
128 STORE_DBG_BRP(6);
129 STORE_DBG_BRP(5);
130 STORE_DBG_BRP(4);
131 STORE_DBG_BRP(3);
132 STORE_DBG_BRP(2);
133 STORE_DBG_BRP(1);
134 default:
135 STORE_DBG_BRP(0);
136 #undef STORE_DBG_BRP
137 }
138
139 switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
140 #define STORE_DBG_WRP(x) \
141 case x: \
142 hypctx->dbgwcr_el1[x] = \
143 READ_SPECIALREG(dbgwcr ## x ## _el1); \
144 hypctx->dbgwvr_el1[x] = \
145 READ_SPECIALREG(dbgwvr ## x ## _el1)
146 STORE_DBG_WRP(15);
147 STORE_DBG_WRP(14);
148 STORE_DBG_WRP(13);
149 STORE_DBG_WRP(12);
150 STORE_DBG_WRP(11);
151 STORE_DBG_WRP(10);
152 STORE_DBG_WRP(9);
153 STORE_DBG_WRP(8);
154 STORE_DBG_WRP(7);
155 STORE_DBG_WRP(6);
156 STORE_DBG_WRP(5);
157 STORE_DBG_WRP(4);
158 STORE_DBG_WRP(3);
159 STORE_DBG_WRP(2);
160 STORE_DBG_WRP(1);
161 default:
162 STORE_DBG_WRP(0);
163 #undef STORE_DBG_WRP
164 }
165
166 /* Store the PMU registers */
167 hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0);
168 hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0);
169 hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0);
170 hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0);
171 hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1);
172 hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0);
173 hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
174 switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
175 #define STORE_PMU(x) \
176 case (x + 1): \
177 hypctx->pmevcntr_el0[x] = \
178 READ_SPECIALREG(pmevcntr ## x ## _el0); \
179 hypctx->pmevtyper_el0[x] = \
180 READ_SPECIALREG(pmevtyper ## x ## _el0)
181 STORE_PMU(30);
182 STORE_PMU(29);
183 STORE_PMU(28);
184 STORE_PMU(27);
185 STORE_PMU(26);
186 STORE_PMU(25);
187 STORE_PMU(24);
188 STORE_PMU(23);
189 STORE_PMU(22);
190 STORE_PMU(21);
191 STORE_PMU(20);
192 STORE_PMU(19);
193 STORE_PMU(18);
194 STORE_PMU(17);
195 STORE_PMU(16);
196 STORE_PMU(15);
197 STORE_PMU(14);
198 STORE_PMU(13);
199 STORE_PMU(12);
200 STORE_PMU(11);
201 STORE_PMU(10);
202 STORE_PMU(9);
203 STORE_PMU(8);
204 STORE_PMU(7);
205 STORE_PMU(6);
206 STORE_PMU(5);
207 STORE_PMU(4);
208 STORE_PMU(3);
209 STORE_PMU(2);
210 STORE_PMU(1);
211 STORE_PMU(0);
212 default: /* N == 0 when only PMCCNTR_EL0 is available */
213 break;
214 #undef STORE_PMU
215 }
216
217 /* Store the special to from the trapframe */
218 hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
219 hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
220 hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
221 if (guest) {
222 hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
223 hypctx->par_el1 = READ_SPECIALREG(par_el1);
224 }
225
226 /* Store the guest special registers */
227 hypctx->sp_el0 = READ_SPECIALREG(sp_el0);
228 hypctx->tpidr_el0 = READ_SPECIALREG(tpidr_el0);
229 hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
230 hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1);
231
232 hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1);
233 hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1);
234 hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1);
235 hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1);
236
237 if (guest_or_nonvhe(guest)) {
238 hypctx->elr_el1 = READ_SPECIALREG(EL1_REG(ELR));
239 hypctx->vbar_el1 = READ_SPECIALREG(EL1_REG(VBAR));
240
241 hypctx->afsr0_el1 = READ_SPECIALREG(EL1_REG(AFSR0));
242 hypctx->afsr1_el1 = READ_SPECIALREG(EL1_REG(AFSR1));
243 hypctx->amair_el1 = READ_SPECIALREG(EL1_REG(AMAIR));
244 hypctx->contextidr_el1 = READ_SPECIALREG(EL1_REG(CONTEXTIDR));
245 hypctx->cpacr_el1 = READ_SPECIALREG(EL1_REG(CPACR));
246 hypctx->esr_el1 = READ_SPECIALREG(EL1_REG(ESR));
247 hypctx->far_el1 = READ_SPECIALREG(EL1_REG(FAR));
248 hypctx->mair_el1 = READ_SPECIALREG(EL1_REG(MAIR));
249 hypctx->sctlr_el1 = READ_SPECIALREG(EL1_REG(SCTLR));
250 hypctx->spsr_el1 = READ_SPECIALREG(EL1_REG(SPSR));
251 hypctx->tcr_el1 = READ_SPECIALREG(EL1_REG(TCR));
252 /* TODO: Support when this is not res0 */
253 hypctx->tcr2_el1 = 0;
254 hypctx->ttbr0_el1 = READ_SPECIALREG(EL1_REG(TTBR0));
255 hypctx->ttbr1_el1 = READ_SPECIALREG(EL1_REG(TTBR1));
256 }
257
258 hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2);
259 hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
260 hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
261 hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
262 }
263
264 static void
vmm_hyp_reg_restore(struct hypctx * hypctx,struct hyp * hyp,bool guest)265 vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
266 {
267 uint64_t dfr0;
268
269 /* Restore the special registers */
270 WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
271 isb();
272
273 WRITE_SPECIALREG(sp_el0, hypctx->sp_el0);
274 WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0);
275 WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0);
276 WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1);
277
278 WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1);
279 WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1);
280 WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1);
281 WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1);
282
283 if (guest_or_nonvhe(guest)) {
284 WRITE_SPECIALREG(EL1_REG(ELR), hypctx->elr_el1);
285 WRITE_SPECIALREG(EL1_REG(VBAR), hypctx->vbar_el1);
286
287 WRITE_SPECIALREG(EL1_REG(AFSR0), hypctx->afsr0_el1);
288 WRITE_SPECIALREG(EL1_REG(AFSR1), hypctx->afsr1_el1);
289 WRITE_SPECIALREG(EL1_REG(AMAIR), hypctx->amair_el1);
290 WRITE_SPECIALREG(EL1_REG(CONTEXTIDR), hypctx->contextidr_el1);
291 WRITE_SPECIALREG(EL1_REG(CPACR), hypctx->cpacr_el1);
292 WRITE_SPECIALREG(EL1_REG(ESR), hypctx->esr_el1);
293 WRITE_SPECIALREG(EL1_REG(FAR), hypctx->far_el1);
294 WRITE_SPECIALREG(EL1_REG(MAIR), hypctx->mair_el1); //
295
296 WRITE_SPECIALREG(EL1_REG(SCTLR), hypctx->sctlr_el1);
297 WRITE_SPECIALREG(EL1_REG(SPSR), hypctx->spsr_el1);
298 WRITE_SPECIALREG(EL1_REG(TCR), hypctx->tcr_el1);
299 /* TODO: tcr2_el1 */
300 WRITE_SPECIALREG(EL1_REG(TTBR0), hypctx->ttbr0_el1);
301 WRITE_SPECIALREG(EL1_REG(TTBR1), hypctx->ttbr1_el1);
302 }
303
304 if (guest) {
305 WRITE_SPECIALREG(par_el1, hypctx->par_el1);
306 }
307
308 WRITE_SPECIALREG(cptr_el2, hypctx->cptr_el2);
309 WRITE_SPECIALREG(vpidr_el2, hypctx->vpidr_el2);
310 WRITE_SPECIALREG(vmpidr_el2, hypctx->vmpidr_el2);
311
312 /* Load the special regs from the trapframe */
313 WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
314 WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
315 WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
316
317 /* Restore the PMU registers */
318 WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0);
319 WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0);
320 WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0);
321 /* Clear all events/interrupts then enable them */
322 WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful);
323 WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0);
324 WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful);
325 WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1);
326 WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful);
327 WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0);
328
329 switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
330 #define LOAD_PMU(x) \
331 case (x + 1): \
332 WRITE_SPECIALREG(pmevcntr ## x ## _el0, \
333 hypctx->pmevcntr_el0[x]); \
334 WRITE_SPECIALREG(pmevtyper ## x ## _el0, \
335 hypctx->pmevtyper_el0[x])
336 LOAD_PMU(30);
337 LOAD_PMU(29);
338 LOAD_PMU(28);
339 LOAD_PMU(27);
340 LOAD_PMU(26);
341 LOAD_PMU(25);
342 LOAD_PMU(24);
343 LOAD_PMU(23);
344 LOAD_PMU(22);
345 LOAD_PMU(21);
346 LOAD_PMU(20);
347 LOAD_PMU(19);
348 LOAD_PMU(18);
349 LOAD_PMU(17);
350 LOAD_PMU(16);
351 LOAD_PMU(15);
352 LOAD_PMU(14);
353 LOAD_PMU(13);
354 LOAD_PMU(12);
355 LOAD_PMU(11);
356 LOAD_PMU(10);
357 LOAD_PMU(9);
358 LOAD_PMU(8);
359 LOAD_PMU(7);
360 LOAD_PMU(6);
361 LOAD_PMU(5);
362 LOAD_PMU(4);
363 LOAD_PMU(3);
364 LOAD_PMU(2);
365 LOAD_PMU(1);
366 LOAD_PMU(0);
367 default: /* N == 0 when only PMCCNTR_EL0 is available */
368 break;
369 #undef LOAD_PMU
370 }
371
372 dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
373 switch (ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
374 #define LOAD_DBG_BRP(x) \
375 case x: \
376 WRITE_SPECIALREG(dbgbcr ## x ## _el1, \
377 hypctx->dbgbcr_el1[x]); \
378 WRITE_SPECIALREG(dbgbvr ## x ## _el1, \
379 hypctx->dbgbvr_el1[x])
380 LOAD_DBG_BRP(15);
381 LOAD_DBG_BRP(14);
382 LOAD_DBG_BRP(13);
383 LOAD_DBG_BRP(12);
384 LOAD_DBG_BRP(11);
385 LOAD_DBG_BRP(10);
386 LOAD_DBG_BRP(9);
387 LOAD_DBG_BRP(8);
388 LOAD_DBG_BRP(7);
389 LOAD_DBG_BRP(6);
390 LOAD_DBG_BRP(5);
391 LOAD_DBG_BRP(4);
392 LOAD_DBG_BRP(3);
393 LOAD_DBG_BRP(2);
394 LOAD_DBG_BRP(1);
395 default:
396 LOAD_DBG_BRP(0);
397 #undef LOAD_DBG_BRP
398 }
399
400 switch (ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
401 #define LOAD_DBG_WRP(x) \
402 case x: \
403 WRITE_SPECIALREG(dbgwcr ## x ## _el1, \
404 hypctx->dbgwcr_el1[x]); \
405 WRITE_SPECIALREG(dbgwvr ## x ## _el1, \
406 hypctx->dbgwvr_el1[x])
407 LOAD_DBG_WRP(15);
408 LOAD_DBG_WRP(14);
409 LOAD_DBG_WRP(13);
410 LOAD_DBG_WRP(12);
411 LOAD_DBG_WRP(11);
412 LOAD_DBG_WRP(10);
413 LOAD_DBG_WRP(9);
414 LOAD_DBG_WRP(8);
415 LOAD_DBG_WRP(7);
416 LOAD_DBG_WRP(6);
417 LOAD_DBG_WRP(5);
418 LOAD_DBG_WRP(4);
419 LOAD_DBG_WRP(3);
420 LOAD_DBG_WRP(2);
421 LOAD_DBG_WRP(1);
422 default:
423 LOAD_DBG_WRP(0);
424 #undef LOAD_DBG_WRP
425 }
426
427 if (guest) {
428 /* Load the timer registers */
429 WRITE_SPECIALREG(EL1_REG(CNTKCTL),
430 hypctx->vtimer_cpu.cntkctl_el1);
431 WRITE_SPECIALREG(EL0_REG(CNTV_CVAL),
432 hypctx->vtimer_cpu.virt_timer.cntx_cval_el0);
433 WRITE_SPECIALREG(EL0_REG(CNTV_CTL),
434 hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0);
435 WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
436 WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
437
438 /* Load the GICv3 registers */
439 WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
440 WRITE_SPECIALREG(ich_vmcr_el2,
441 hypctx->vgic_v3_regs.ich_vmcr_el2);
442 switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
443 #define LOAD_LR(x) \
444 case x: \
445 WRITE_SPECIALREG(ich_lr ## x ##_el2, \
446 hypctx->vgic_v3_regs.ich_lr_el2[x])
447 LOAD_LR(15);
448 LOAD_LR(14);
449 LOAD_LR(13);
450 LOAD_LR(12);
451 LOAD_LR(11);
452 LOAD_LR(10);
453 LOAD_LR(9);
454 LOAD_LR(8);
455 LOAD_LR(7);
456 LOAD_LR(6);
457 LOAD_LR(5);
458 LOAD_LR(4);
459 LOAD_LR(3);
460 LOAD_LR(2);
461 LOAD_LR(1);
462 default:
463 LOAD_LR(0);
464 #undef LOAD_LR
465 }
466
467 switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
468 #define LOAD_APR(x) \
469 case x: \
470 WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
471 hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
472 WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
473 hypctx->vgic_v3_regs.ich_ap1r_el2[x])
474 LOAD_APR(3);
475 LOAD_APR(2);
476 LOAD_APR(1);
477 default:
478 LOAD_APR(0);
479 #undef LOAD_APR
480 }
481 }
482 }
483
484 static uint64_t
vmm_hyp_call_guest(struct hyp * hyp,struct hypctx * hypctx)485 vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
486 {
487 struct hypctx host_hypctx;
488 uint64_t cntvoff_el2;
489 uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
490 uint64_t ret;
491 uint64_t s1e1r, hpfar_el2;
492 bool hpfar_valid;
493
494 vmm_hyp_reg_store(&host_hypctx, NULL, false);
495
496 /* Save the host special registers */
497 cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
498 cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
499 cntvoff_el2 = READ_SPECIALREG(cntvoff_el2);
500
501 ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
502 ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
503
504 vmm_hyp_reg_restore(hypctx, hyp, true);
505
506 /* Load the common hypervisor registers */
507 WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
508
509 host_hypctx.mdcr_el2 = READ_SPECIALREG(mdcr_el2);
510 WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
511
512 /* Call into the guest */
513 ret = VMM_HYP_FUNC(do_call_guest)(hypctx);
514
515 WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2);
516 isb();
517
518 /* Store the exit info */
519 hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
520 vmm_hyp_reg_store(hypctx, hyp, true);
521
522 hpfar_valid = true;
523 if (ret == EXCP_TYPE_EL1_SYNC) {
524 switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
525 case EXCP_INSN_ABORT_L:
526 case EXCP_DATA_ABORT_L:
527 /*
528 * The hpfar_el2 register is valid for:
529 * - Translation and Access faults.
530 * - Translation, Access, and permission faults on
531 * the translation table walk on the stage 1 tables.
532 * - A stage 2 Address size fault.
533 *
534 * As we only need it in the first 2 cases we can just
535 * exclude it on permission faults that are not from
536 * the stage 1 table walk.
537 *
538 * TODO: Add a case for Arm erratum 834220.
539 */
540 if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
541 break;
542 switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
543 case ISS_DATA_DFSC_PF_L1:
544 case ISS_DATA_DFSC_PF_L2:
545 case ISS_DATA_DFSC_PF_L3:
546 hpfar_valid = false;
547 break;
548 }
549 break;
550 }
551 }
552 if (hpfar_valid) {
553 hypctx->exit_info.hpfar_el2 = READ_SPECIALREG(hpfar_el2);
554 } else {
555 /*
556 * TODO: There is a risk the at instruction could cause an
557 * exception here. We should handle it & return a failure.
558 */
559 s1e1r =
560 arm64_address_translate_s1e1r(hypctx->exit_info.far_el2);
561 if (PAR_SUCCESS(s1e1r)) {
562 hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
563 hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
564 hypctx->exit_info.hpfar_el2 = hpfar_el2;
565 } else {
566 ret = EXCP_TYPE_REENTER;
567 }
568 }
569
570 vmm_hyp_reg_restore(&host_hypctx, NULL, false);
571
572 /* Restore the host special registers */
573 WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
574 WRITE_SPECIALREG(ich_vmcr_el2, ich_vmcr_el2);
575
576 WRITE_SPECIALREG(cnthctl_el2, cnthctl_el2);
577 WRITE_SPECIALREG(cntkctl_el1, cntkctl_el1);
578 WRITE_SPECIALREG(cntvoff_el2, cntvoff_el2);
579
580 return (ret);
581 }
582
583 VMM_STATIC uint64_t
VMM_HYP_FUNC(enter_guest)584 VMM_HYP_FUNC(enter_guest)(struct hyp *hyp, struct hypctx *hypctx)
585 {
586 uint64_t ret;
587
588 do {
589 ret = vmm_hyp_call_guest(hyp, hypctx);
590 } while (ret == EXCP_TYPE_REENTER);
591
592 return (ret);
593 }
594
595 VMM_STATIC uint64_t
VMM_HYP_FUNC(read_reg)596 VMM_HYP_FUNC(read_reg)(uint64_t reg)
597 {
598 switch (reg) {
599 case HYP_REG_ICH_VTR:
600 return (READ_SPECIALREG(ich_vtr_el2));
601 case HYP_REG_CNTHCTL:
602 return (READ_SPECIALREG(cnthctl_el2));
603 }
604
605 return (0);
606 }
607
608 VMM_STATIC void
VMM_HYP_FUNC(clean_s2_tlbi)609 VMM_HYP_FUNC(clean_s2_tlbi)(void)
610 {
611 dsb(ishst);
612 __asm __volatile("tlbi alle1is");
613 dsb(ish);
614 }
615
616 VMM_STATIC void
VMM_HYP_FUNC(s2_tlbi_range)617 VMM_HYP_FUNC(s2_tlbi_range)(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva,
618 bool final_only)
619 {
620 uint64_t end, r, start;
621 uint64_t host_vttbr;
622 #ifdef VMM_VHE
623 uint64_t host_tcr;
624 #endif
625
626 #ifdef VMM_VHE
627 dsb(ishst);
628 #endif
629
630 #define TLBI_VA_SHIFT 12
631 #define TLBI_VA_MASK ((1ul << 44) - 1)
632 #define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK)
633 #define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
634
635 /* Switch to the guest vttbr */
636 /* TODO: Handle Cortex-A57/A72 erratum 131936 */
637 host_vttbr = READ_SPECIALREG(vttbr_el2);
638 WRITE_SPECIALREG(vttbr_el2, vttbr);
639 isb();
640
641 #ifdef VMM_VHE
642 host_tcr = READ_SPECIALREG(tcr_el2);
643 WRITE_SPECIALREG(tcr_el2, host_tcr & ~HCR_TGE);
644 isb();
645 #endif
646
647 /*
648 * The CPU can cache the stage 1 + 2 combination so we need to ensure
649 * the stage 2 is invalidated first, then when this has completed we
650 * invalidate the stage 1 TLB. As we don't know which stage 1 virtual
651 * addresses point at the stage 2 IPA we need to invalidate the entire
652 * stage 1 TLB.
653 */
654
655 start = TLBI_VA(sva);
656 end = TLBI_VA(eva);
657 for (r = start; r < end; r += TLBI_VA_L3_INCR) {
658 /* Invalidate the stage 2 TLB entry */
659 if (final_only)
660 __asm __volatile("tlbi ipas2le1is, %0" : : "r"(r));
661 else
662 __asm __volatile("tlbi ipas2e1is, %0" : : "r"(r));
663 }
664 /* Ensure the entry has been invalidated */
665 dsb(ish);
666 /* Invalidate the stage 1 TLB. */
667 __asm __volatile("tlbi vmalle1is");
668 dsb(ish);
669 isb();
670
671 #ifdef VMM_VHE
672 WRITE_SPECIALREG(tcr_el2, host_tcr);
673 isb();
674 #endif
675
676 /* Switch back to the host vttbr */
677 WRITE_SPECIALREG(vttbr_el2, host_vttbr);
678 isb();
679 }
680
681 VMM_STATIC void
VMM_HYP_FUNC(s2_tlbi_all)682 VMM_HYP_FUNC(s2_tlbi_all)(uint64_t vttbr)
683 {
684 uint64_t host_vttbr;
685
686 #ifdef VMM_VHE
687 dsb(ishst);
688 #endif
689
690 /* Switch to the guest vttbr */
691 /* TODO: Handle Cortex-A57/A72 erratum 131936 */
692 host_vttbr = READ_SPECIALREG(vttbr_el2);
693 WRITE_SPECIALREG(vttbr_el2, vttbr);
694 isb();
695
696 __asm __volatile("tlbi vmalls12e1is");
697 dsb(ish);
698 isb();
699
700 /* Switch back t othe host vttbr */
701 WRITE_SPECIALREG(vttbr_el2, host_vttbr);
702 isb();
703 }
704