1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/arm-smccc.h> 8#include <linux/linkage.h> 9 10#include <asm/alternative.h> 11#include <asm/assembler.h> 12#include <asm/el2_setup.h> 13#include <asm/kvm_arm.h> 14#include <asm/kvm_asm.h> 15#include <asm/kvm_mmu.h> 16#include <asm/pgtable-hwdef.h> 17#include <asm/sysreg.h> 18#include <asm/virt.h> 19 20 .text 21 .pushsection .idmap.text, "ax" 22 23 .align 11 24 25SYM_CODE_START(__kvm_hyp_init) 26 ventry __invalid // Synchronous EL2t 27 ventry __invalid // IRQ EL2t 28 ventry __invalid // FIQ EL2t 29 ventry __invalid // Error EL2t 30 31 ventry __invalid // Synchronous EL2h 32 ventry __invalid // IRQ EL2h 33 ventry __invalid // FIQ EL2h 34 ventry __invalid // Error EL2h 35 36 ventry __do_hyp_init // Synchronous 64-bit EL1 37 ventry __invalid // IRQ 64-bit EL1 38 ventry __invalid // FIQ 64-bit EL1 39 ventry __invalid // Error 64-bit EL1 40 41 ventry __invalid // Synchronous 32-bit EL1 42 ventry __invalid // IRQ 32-bit EL1 43 ventry __invalid // FIQ 32-bit EL1 44 ventry __invalid // Error 32-bit EL1 45 46__invalid: 47 b . 48 49 /* 50 * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers. 51 * 52 * x0: SMCCC function ID 53 * x1: struct kvm_nvhe_init_params PA 54 */ 55__do_hyp_init: 56 /* Check for a stub HVC call */ 57 cmp x0, #HVC_STUB_HCALL_NR 58 b.lo __kvm_handle_stub_hvc 59 60 bic x0, x0, #ARM_SMCCC_CALL_HINTS 61 mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) 62 cmp x0, x3 63 b.eq 1f 64 65 mov x0, #SMCCC_RET_NOT_SUPPORTED 66 eret 67 681: mov x0, x1 69 mov x3, lr 70 bl ___kvm_hyp_init // Clobbers x0..x2 71 mov lr, x3 72 73 /* Hello, World! */ 74 mov x0, #SMCCC_RET_SUCCESS 75 eret 76SYM_CODE_END(__kvm_hyp_init) 77 78/* 79 * Initialize the hypervisor in EL2. 80 * 81 * Only uses x0..x2 so as to not clobber callee-saved SMCCC registers 82 * and leave x3 for the caller. 83 * 84 * x0: struct kvm_nvhe_init_params PA 85 */ 86SYM_CODE_START_LOCAL(___kvm_hyp_init) 87 ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] 88 mov sp, x1 89 90 ldr x1, [x0, #NVHE_INIT_MAIR_EL2] 91 msr mair_el2, x1 92 93 ldr x1, [x0, #NVHE_INIT_HCR_EL2] 94 msr hcr_el2, x1 95 96 mov x2, #HCR_E2H 97 and x2, x1, x2 98 cbz x2, 1f 99 100 // hVHE: Replay the EL2 setup to account for the E2H bit 101 // TPIDR_EL2 is used to preserve x0 across the macro maze... 102 isb 103 msr tpidr_el2, x0 104 init_el2_state 105 finalise_el2_state 106 mrs x0, tpidr_el2 107 1081: 109 ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] 110 msr tpidr_el2, x1 111 112 ldr x1, [x0, #NVHE_INIT_VTTBR] 113 msr vttbr_el2, x1 114 115 ldr x1, [x0, #NVHE_INIT_VTCR] 116 msr vtcr_el2, x1 117 118 ldr x1, [x0, #NVHE_INIT_PGD_PA] 119 phys_to_ttbr x2, x1 120alternative_if ARM64_HAS_CNP 121 orr x2, x2, #TTBR_CNP_BIT 122alternative_else_nop_endif 123 msr ttbr0_el2, x2 124 125 ldr x0, [x0, #NVHE_INIT_TCR_EL2] 126 msr tcr_el2, x0 127 128 isb 129 130 /* Invalidate the stale TLBs from Bootloader */ 131 tlbi alle2 132 tlbi vmalls12e1 133 dsb sy 134 135 mov_q x0, INIT_SCTLR_EL2_MMU_ON 136alternative_if ARM64_HAS_ADDRESS_AUTH 137 mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ 138 SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) 139 orr x0, x0, x1 140alternative_else_nop_endif 141 142#ifdef CONFIG_ARM64_BTI_KERNEL 143alternative_if ARM64_BTI 144 orr x0, x0, #SCTLR_EL2_BT 145alternative_else_nop_endif 146#endif /* CONFIG_ARM64_BTI_KERNEL */ 147 148 msr sctlr_el2, x0 149 isb 150 151 /* Set the host vector */ 152 ldr x0, =__kvm_hyp_host_vector 153 msr vbar_el2, x0 154 155 ret 156SYM_CODE_END(___kvm_hyp_init) 157 158/* 159 * PSCI CPU_ON entry point 160 * 161 * x0: struct kvm_nvhe_init_params PA 162 */ 163SYM_CODE_START(kvm_hyp_cpu_entry) 164 mov x1, #1 // is_cpu_on = true 165 b __kvm_hyp_init_cpu 166SYM_CODE_END(kvm_hyp_cpu_entry) 167 168/* 169 * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point 170 * 171 * x0: struct kvm_nvhe_init_params PA 172 */ 173SYM_CODE_START(kvm_hyp_cpu_resume) 174 mov x1, #0 // is_cpu_on = false 175 b __kvm_hyp_init_cpu 176SYM_CODE_END(kvm_hyp_cpu_resume) 177 178/* 179 * Common code for CPU entry points. Initializes EL2 state and 180 * installs the hypervisor before handing over to a C handler. 181 * 182 * x0: struct kvm_nvhe_init_params PA 183 * x1: bool is_cpu_on 184 */ 185SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) 186 mov x28, x0 // Stash arguments 187 mov x29, x1 188 189 /* Check that the core was booted in EL2. */ 190 mrs x0, CurrentEL 191 cmp x0, #CurrentEL_EL2 192 b.eq 2f 193 194 /* The core booted in EL1. KVM cannot be initialized on it. */ 1951: wfe 196 wfi 197 b 1b 198 1992: msr SPsel, #1 // We want to use SP_EL{1,2} 200 201 /* Initialize EL2 CPU state to sane values. */ 202 init_el2_state // Clobbers x0..x2 203 finalise_el2_state 204 __init_el2_nvhe_prepare_eret 205 206 /* Enable MMU, set vectors and stack. */ 207 mov x0, x28 208 bl ___kvm_hyp_init // Clobbers x0..x2 209 210 /* Leave idmap. */ 211 mov x0, x29 212 ldr x1, =kvm_host_psci_cpu_entry 213 br x1 214SYM_CODE_END(__kvm_hyp_init_cpu) 215 216SYM_CODE_START(__kvm_handle_stub_hvc) 217 /* 218 * __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so 219 * we need bti j at beginning. 220 */ 221 bti j 222 cmp x0, #HVC_SOFT_RESTART 223 b.ne 1f 224 225 /* This is where we're about to jump, staying at EL2 */ 226 msr elr_el2, x1 227 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT | PSR_MODE_EL2h) 228 msr spsr_el2, x0 229 230 /* Shuffle the arguments, and don't come back */ 231 mov x0, x2 232 mov x1, x3 233 mov x2, x4 234 b reset 235 2361: cmp x0, #HVC_RESET_VECTORS 237 b.ne 1f 238 239 /* 240 * Set the HVC_RESET_VECTORS return code before entering the common 241 * path so that we do not clobber x0-x2 in case we are coming via 242 * HVC_SOFT_RESTART. 243 */ 244 mov x0, xzr 245reset: 246 /* Reset kvm back to the hyp stub. */ 247 mov_q x5, INIT_SCTLR_EL2_MMU_OFF 248 pre_disable_mmu_workaround 249 msr sctlr_el2, x5 250 isb 251 252alternative_if ARM64_KVM_PROTECTED_MODE 253 mov_q x5, HCR_HOST_NVHE_FLAGS 254 msr hcr_el2, x5 255alternative_else_nop_endif 256 257 /* Install stub vectors */ 258 adr_l x5, __hyp_stub_vectors 259 msr vbar_el2, x5 260 eret 261 2621: /* Bad stub call */ 263 mov_q x0, HVC_STUB_ERR 264 eret 265 266SYM_CODE_END(__kvm_handle_stub_hvc) 267 268SYM_FUNC_START(__pkvm_init_switch_pgd) 269 /* Turn the MMU off */ 270 pre_disable_mmu_workaround 271 mrs x2, sctlr_el2 272 bic x3, x2, #SCTLR_ELx_M 273 msr sctlr_el2, x3 274 isb 275 276 tlbi alle2 277 278 /* Install the new pgtables */ 279 ldr x3, [x0, #NVHE_INIT_PGD_PA] 280 phys_to_ttbr x4, x3 281alternative_if ARM64_HAS_CNP 282 orr x4, x4, #TTBR_CNP_BIT 283alternative_else_nop_endif 284 msr ttbr0_el2, x4 285 286 /* Set the new stack pointer */ 287 ldr x0, [x0, #NVHE_INIT_STACK_HYP_VA] 288 mov sp, x0 289 290 /* And turn the MMU back on! */ 291 dsb nsh 292 isb 293 set_sctlr_el2 x2 294 ret x1 295SYM_FUNC_END(__pkvm_init_switch_pgd) 296 297 .popsection 298