1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/arm-smccc.h> 8#include <linux/cfi_types.h> 9#include <linux/linkage.h> 10 11#include <asm/alternative.h> 12#include <asm/assembler.h> 13#include <asm/el2_setup.h> 14#include <asm/kvm_arm.h> 15#include <asm/kvm_asm.h> 16#include <asm/kvm_mmu.h> 17#include <asm/pgtable-hwdef.h> 18#include <asm/sysreg.h> 19#include <asm/virt.h> 20 21 .text 22 .pushsection .idmap.text, "ax" 23 24 .align 11 25 26SYM_CODE_START(__kvm_hyp_init) 27 ventry . // Synchronous EL2t 28 ventry . // IRQ EL2t 29 ventry . // FIQ EL2t 30 ventry . // Error EL2t 31 32 ventry . // Synchronous EL2h 33 ventry . // IRQ EL2h 34 ventry . // FIQ EL2h 35 ventry . // Error EL2h 36 37 ventry __do_hyp_init // Synchronous 64-bit EL1 38 ventry . // IRQ 64-bit EL1 39 ventry . // FIQ 64-bit EL1 40 ventry . // Error 64-bit EL1 41 42 ventry . // Synchronous 32-bit EL1 43 ventry . // IRQ 32-bit EL1 44 ventry . // FIQ 32-bit EL1 45 ventry . // Error 32-bit EL1 46 47 /* 48 * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers. 49 * 50 * x0: SMCCC function ID 51 * x1: struct kvm_nvhe_init_params PA 52 */ 53__do_hyp_init: 54 /* Check for a stub HVC call */ 55 cmp x0, #HVC_STUB_HCALL_NR 56 b.lo __kvm_handle_stub_hvc 57 58 bic x0, x0, #ARM_SMCCC_CALL_HINTS 59 mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) 60 cmp x0, x3 61 b.eq 1f 62 63 mov x0, #SMCCC_RET_NOT_SUPPORTED 64 eret 65 661: mov x0, x1 67 mov x3, lr 68 bl ___kvm_hyp_init // Clobbers x0..x2 69 mov lr, x3 70 71 /* Hello, World! */ 72 mov x0, #SMCCC_RET_SUCCESS 73 eret 74SYM_CODE_END(__kvm_hyp_init) 75 76/* 77 * Initialize EL2 CPU state to sane values. 78 * 79 * HCR_EL2.E2H must have been initialized already. 80 */ 81SYM_CODE_START_LOCAL(__kvm_init_el2_state) 82 init_el2_state // Clobbers x0..x2 83 finalise_el2_state 84 ret 85SYM_CODE_END(__kvm_init_el2_state) 86 87/* 88 * Initialize the hypervisor in EL2. 89 * 90 * Only uses x0..x2 so as to not clobber callee-saved SMCCC registers 91 * and leave x3 for the caller. 92 * 93 * x0: struct kvm_nvhe_init_params PA 94 */ 95SYM_CODE_START_LOCAL(___kvm_hyp_init) 96 ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] 97 mov sp, x1 98 99 ldr x1, [x0, #NVHE_INIT_MAIR_EL2] 100 msr mair_el2, x1 101 102 ldr x1, [x0, #NVHE_INIT_HCR_EL2] 103 msr_hcr_el2 x1 104 105 mov x2, #HCR_E2H 106 and x2, x1, x2 107 cbz x2, 1f 108 109 // hVHE: Replay the EL2 setup to account for the E2H bit 110 // TPIDR_EL2 is used to preserve x0 across the macro maze... 111 isb 112 msr tpidr_el2, x0 113 str lr, [x0, #NVHE_INIT_TMP] 114 115 bl __kvm_init_el2_state 116 117 mrs x0, tpidr_el2 118 ldr lr, [x0, #NVHE_INIT_TMP] 119 1201: 121 ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] 122 msr tpidr_el2, x1 123 124 ldr x1, [x0, #NVHE_INIT_VTTBR] 125 msr vttbr_el2, x1 126 127 ldr x1, [x0, #NVHE_INIT_VTCR] 128 msr vtcr_el2, x1 129 130 ldr x1, [x0, #NVHE_INIT_PGD_PA] 131 phys_to_ttbr x2, x1 132alternative_if ARM64_HAS_CNP 133 orr x2, x2, #TTBRx_EL1_CnP 134alternative_else_nop_endif 135 msr ttbr0_el2, x2 136 137 ldr x0, [x0, #NVHE_INIT_TCR_EL2] 138 msr tcr_el2, x0 139 140 isb 141 142 /* Invalidate the stale TLBs from Bootloader */ 143 tlbi alle2 144 tlbi alle1 145 dsb sy 146 147 mov_q x0, INIT_SCTLR_EL2_MMU_ON 148alternative_if ARM64_HAS_ADDRESS_AUTH 149 mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ 150 SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) 151 orr x0, x0, x1 152alternative_else_nop_endif 153 154#ifdef CONFIG_ARM64_BTI_KERNEL 155alternative_if ARM64_BTI 156 orr x0, x0, #SCTLR_EL2_BT 157alternative_else_nop_endif 158#endif /* CONFIG_ARM64_BTI_KERNEL */ 159 160 msr sctlr_el2, x0 161 isb 162 163 /* Set the host vector */ 164 ldr x0, =__kvm_hyp_host_vector 165 msr vbar_el2, x0 166 167 ret 168SYM_CODE_END(___kvm_hyp_init) 169 170/* 171 * PSCI CPU_ON entry point 172 * 173 * x0: struct kvm_nvhe_init_params PA 174 */ 175SYM_CODE_START(kvm_hyp_cpu_entry) 176 ldr x29, =__kvm_host_psci_cpu_on_entry 177 b __kvm_hyp_init_cpu 178 179/* 180 * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point 181 * 182 * x0: struct kvm_nvhe_init_params PA 183 */ 184SYM_CODE_START(kvm_hyp_cpu_resume) 185 ldr x29, =__kvm_host_psci_cpu_resume_entry 186 187SYM_INNER_LABEL(__kvm_hyp_init_cpu, SYM_L_LOCAL) 188 mov x28, x0 // Stash arguments 189 190 /* Check that the core was booted in EL2. */ 191 mrs x0, CurrentEL 192 cmp x0, #CurrentEL_EL2 193 b.ne 1f 194 195 msr SPsel, #1 // We want to use SP_EL2 196 197 init_el2_hcr 0 198 199 bl __kvm_init_el2_state 200 201 /* Enable MMU, set vectors and stack. */ 202 mov x0, x28 203 bl ___kvm_hyp_init // Clobbers x0..x2 204 205 /* Leave idmap -- using BLR is OK, LR is restored from host context */ 206 blr x29 207 208 // The core booted in EL1, or the C code unexpectedly returned. 209 // Either way, KVM cannot be initialized on it. 2101: wfe 211 wfi 212 b 1b 213SYM_CODE_END(kvm_hyp_cpu_resume) 214SYM_CODE_END(kvm_hyp_cpu_entry) 215 216SYM_CODE_START(__kvm_handle_stub_hvc) 217 /* 218 * __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so 219 * we need bti j at beginning. 220 */ 221 bti j 222 cmp x0, #HVC_SOFT_RESTART 223 b.ne 1f 224 225 /* This is where we're about to jump, staying at EL2 */ 226 msr elr_el2, x1 227 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h) 228 msr spsr_el2, x0 229 230 /* Shuffle the arguments, and don't come back */ 231 mov x0, x2 232 mov x1, x3 233 mov x2, x4 234 b reset 235 2361: cmp x0, #HVC_RESET_VECTORS 237 b.ne 1f 238 239 /* 240 * Set the HVC_RESET_VECTORS return code before entering the common 241 * path so that we do not clobber x0-x2 in case we are coming via 242 * HVC_SOFT_RESTART. 243 */ 244 mov x0, xzr 245reset: 246 /* Reset kvm back to the hyp stub. */ 247 mov_q x5, INIT_SCTLR_EL2_MMU_OFF 248 pre_disable_mmu_workaround 249 msr sctlr_el2, x5 250 isb 251 252 /* Install stub vectors */ 253 adr_l x5, __hyp_stub_vectors 254 msr vbar_el2, x5 255 eret 256 2571: /* Bad stub call */ 258 mov_q x0, HVC_STUB_ERR 259 eret 260 261SYM_CODE_END(__kvm_handle_stub_hvc) 262 263/* 264 * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp, 265 * void (*fn)(void)); 266 * 267 * SYM_TYPED_FUNC_START() allows C to call this ID-mapped function indirectly 268 * using a physical pointer without triggering a kCFI failure. 269 */ 270SYM_TYPED_FUNC_START(__pkvm_init_switch_pgd) 271 /* Turn the MMU off */ 272 pre_disable_mmu_workaround 273 mrs x3, sctlr_el2 274 bic x4, x3, #SCTLR_ELx_M 275 msr sctlr_el2, x4 276 isb 277 278 tlbi alle2 279 280 /* Install the new pgtables */ 281 phys_to_ttbr x5, x0 282alternative_if ARM64_HAS_CNP 283 orr x5, x5, #TTBRx_EL1_CnP 284alternative_else_nop_endif 285 msr ttbr0_el2, x5 286 287 /* Set the new stack pointer */ 288 mov sp, x1 289 290 /* And turn the MMU back on! */ 291 dsb nsh 292 isb 293 set_sctlr_el2 x3 294 ret x2 295SYM_FUNC_END(__pkvm_init_switch_pgd) 296 297 .popsection 298