1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Hypervisor stub 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 */ 8 9#include <linux/init.h> 10#include <linux/linkage.h> 11 12#include <asm/assembler.h> 13#include <asm/el2_setup.h> 14#include <asm/kvm_arm.h> 15#include <asm/kvm_asm.h> 16#include <asm/ptrace.h> 17#include <asm/virt.h> 18 19// Warning, hardcoded register allocation 20// This will clobber x1 and x2, and expect x1 to contain 21// the id register value as read from the HW 22.macro __check_override idreg, fld, width, pass, fail 23 ubfx x1, x1, #\fld, #\width 24 cbz x1, \fail 25 26 adr_l x1, \idreg\()_override 27 ldr x2, [x1, FTR_OVR_VAL_OFFSET] 28 ldr x1, [x1, FTR_OVR_MASK_OFFSET] 29 ubfx x2, x2, #\fld, #\width 30 ubfx x1, x1, #\fld, #\width 31 cmp x1, xzr 32 and x2, x2, x1 33 csinv x2, x2, xzr, ne 34 cbnz x2, \pass 35 b \fail 36.endm 37 38.macro check_override idreg, fld, pass, fail 39 mrs x1, \idreg\()_el1 40 __check_override \idreg \fld 4 \pass \fail 41.endm 42 43 .text 44 .pushsection .hyp.text, "ax" 45 46 .align 11 47 48SYM_CODE_START(__hyp_stub_vectors) 49 ventry el2_sync_invalid // Synchronous EL2t 50 ventry el2_irq_invalid // IRQ EL2t 51 ventry el2_fiq_invalid // FIQ EL2t 52 ventry el2_error_invalid // Error EL2t 53 54 ventry elx_sync // Synchronous EL2h 55 ventry el2_irq_invalid // IRQ EL2h 56 ventry el2_fiq_invalid // FIQ EL2h 57 ventry el2_error_invalid // Error EL2h 58 59 ventry elx_sync // Synchronous 64-bit EL1 60 ventry el1_irq_invalid // IRQ 64-bit EL1 61 ventry el1_fiq_invalid // FIQ 64-bit EL1 62 ventry el1_error_invalid // Error 64-bit EL1 63 64 ventry el1_sync_invalid // Synchronous 32-bit EL1 65 ventry el1_irq_invalid // IRQ 32-bit EL1 66 ventry el1_fiq_invalid // FIQ 32-bit EL1 67 ventry el1_error_invalid // Error 32-bit EL1 68SYM_CODE_END(__hyp_stub_vectors) 69 70 .align 11 71 72SYM_CODE_START_LOCAL(elx_sync) 73 cmp x0, #HVC_SET_VECTORS 74 b.ne 1f 75 msr vbar_el2, x1 76 b 9f 77 781: cmp x0, #HVC_FINALISE_EL2 79 b.eq __finalise_el2 80 812: cmp x0, #HVC_SOFT_RESTART 82 b.ne 3f 83 mov x0, x2 84 mov x2, x4 85 mov x4, x1 86 mov x1, x3 87 br x4 // no return 88 893: cmp x0, #HVC_RESET_VECTORS 90 beq 9f // Nothing to reset! 91 92 /* Someone called kvm_call_hyp() against the hyp-stub... */ 93 mov_q x0, HVC_STUB_ERR 94 eret 95 969: mov x0, xzr 97 eret 98SYM_CODE_END(elx_sync) 99 100SYM_CODE_START_LOCAL(__finalise_el2) 101 check_override id_aa64pfr0 ID_AA64PFR0_EL1_SVE_SHIFT .Linit_sve .Lskip_sve 102 103.Linit_sve: /* SVE register access */ 104 mrs x0, cptr_el2 // Disable SVE traps 105 bic x0, x0, #CPTR_EL2_TZ 106 msr cptr_el2, x0 107 isb 108 mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector 109 msr_s SYS_ZCR_EL2, x1 // length for EL1. 110 111.Lskip_sve: 112 check_override id_aa64pfr1 ID_AA64PFR1_EL1_SME_SHIFT .Linit_sme .Lskip_sme 113 114.Linit_sme: /* SME register access and priority mapping */ 115 mrs x0, cptr_el2 // Disable SME traps 116 bic x0, x0, #CPTR_EL2_TSM 117 msr cptr_el2, x0 118 isb 119 120 mrs x1, sctlr_el2 121 orr x1, x1, #SCTLR_ELx_ENTP2 // Disable TPIDR2 traps 122 msr sctlr_el2, x1 123 isb 124 125 mov x0, #0 // SMCR controls 126 127 // Full FP in SM? 128 mrs_s x1, SYS_ID_AA64SMFR0_EL1 129 __check_override id_aa64smfr0 ID_AA64SMFR0_EL1_FA64_SHIFT 1 .Linit_sme_fa64 .Lskip_sme_fa64 130 131.Linit_sme_fa64: 132 orr x0, x0, SMCR_ELx_FA64_MASK 133.Lskip_sme_fa64: 134 135 // ZT0 available? 136 mrs_s x1, SYS_ID_AA64SMFR0_EL1 137 __check_override id_aa64smfr0 ID_AA64SMFR0_EL1_SMEver_SHIFT 4 .Linit_sme_zt0 .Lskip_sme_zt0 138.Linit_sme_zt0: 139 orr x0, x0, SMCR_ELx_EZT0_MASK 140.Lskip_sme_zt0: 141 142 orr x0, x0, #SMCR_ELx_LEN_MASK // Enable full SME vector 143 msr_s SYS_SMCR_EL2, x0 // length for EL1. 144 145 mrs_s x1, SYS_SMIDR_EL1 // Priority mapping supported? 146 ubfx x1, x1, #SMIDR_EL1_SMPS_SHIFT, #1 147 cbz x1, .Lskip_sme 148 149 msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal 150 151 mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present? 152 ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4 153 cbz x1, .Lskip_sme 154 155 mrs_s x1, SYS_HCRX_EL2 156 orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping 157 msr_s SYS_HCRX_EL2, x1 158 159.Lskip_sme: 160 161 // nVHE? No way! Give me the real thing! 162 // Sanity check: MMU *must* be off 163 mrs x1, sctlr_el2 164 tbnz x1, #0, 1f 165 166 // Needs to be VHE capable, obviously 167 check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 2f 1f 168 1691: mov_q x0, HVC_STUB_ERR 170 eret 1712: 172 // Engage the VHE magic! 173 mov_q x0, HCR_HOST_VHE_FLAGS 174 msr hcr_el2, x0 175 isb 176 177 // Use the EL1 allocated stack, per-cpu offset 178 mrs x0, sp_el1 179 mov sp, x0 180 mrs x0, tpidr_el1 181 msr tpidr_el2, x0 182 183 // FP configuration, vectors 184 mrs_s x0, SYS_CPACR_EL12 185 msr cpacr_el1, x0 186 mrs_s x0, SYS_VBAR_EL12 187 msr vbar_el1, x0 188 189 // Use EL2 translations for SPE & TRBE and disable access from EL1 190 mrs x0, mdcr_el2 191 bic x0, x0, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT) 192 bic x0, x0, #(MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT) 193 msr mdcr_el2, x0 194 195 // Transfer the MM state from EL1 to EL2 196 mrs_s x0, SYS_TCR_EL12 197 msr tcr_el1, x0 198 mrs_s x0, SYS_TTBR0_EL12 199 msr ttbr0_el1, x0 200 mrs_s x0, SYS_TTBR1_EL12 201 msr ttbr1_el1, x0 202 mrs_s x0, SYS_MAIR_EL12 203 msr mair_el1, x0 204 isb 205 206 // Hack the exception return to stay at EL2 207 mrs x0, spsr_el1 208 and x0, x0, #~PSR_MODE_MASK 209 mov x1, #PSR_MODE_EL2h 210 orr x0, x0, x1 211 msr spsr_el1, x0 212 213 b enter_vhe 214SYM_CODE_END(__finalise_el2) 215 216 // At the point where we reach enter_vhe(), we run with 217 // the MMU off (which is enforced by __finalise_el2()). 218 // We thus need to be in the idmap, or everything will 219 // explode when enabling the MMU. 220 221 .pushsection .idmap.text, "ax" 222 223SYM_CODE_START_LOCAL(enter_vhe) 224 // Invalidate TLBs before enabling the MMU 225 tlbi vmalle1 226 dsb nsh 227 isb 228 229 // Enable the EL2 S1 MMU, as set up from EL1 230 mrs_s x0, SYS_SCTLR_EL12 231 set_sctlr_el1 x0 232 233 // Disable the EL1 S1 MMU for a good measure 234 mov_q x0, INIT_SCTLR_EL1_MMU_OFF 235 msr_s SYS_SCTLR_EL12, x0 236 237 mov x0, xzr 238 239 eret 240SYM_CODE_END(enter_vhe) 241 242 .popsection 243 244.macro invalid_vector label 245SYM_CODE_START_LOCAL(\label) 246 b \label 247SYM_CODE_END(\label) 248.endm 249 250 invalid_vector el2_sync_invalid 251 invalid_vector el2_irq_invalid 252 invalid_vector el2_fiq_invalid 253 invalid_vector el2_error_invalid 254 invalid_vector el1_sync_invalid 255 invalid_vector el1_irq_invalid 256 invalid_vector el1_fiq_invalid 257 invalid_vector el1_error_invalid 258 259 .popsection 260 261/* 262 * __hyp_set_vectors: Call this after boot to set the initial hypervisor 263 * vectors as part of hypervisor installation. On an SMP system, this should 264 * be called on each CPU. 265 * 266 * x0 must be the physical address of the new vector table, and must be 267 * 2KB aligned. 268 * 269 * Before calling this, you must check that the stub hypervisor is installed 270 * everywhere, by waiting for any secondary CPUs to be brought up and then 271 * checking that is_hyp_mode_available() is true. 272 * 273 * If not, there is a pre-existing hypervisor, some CPUs failed to boot, or 274 * something else went wrong... in such cases, trying to install a new 275 * hypervisor is unlikely to work as desired. 276 * 277 * When you call into your shiny new hypervisor, sp_el2 will contain junk, 278 * so you will need to set that to something sensible at the new hypervisor's 279 * initialisation entry point. 280 */ 281 282SYM_FUNC_START(__hyp_set_vectors) 283 mov x1, x0 284 mov x0, #HVC_SET_VECTORS 285 hvc #0 286 ret 287SYM_FUNC_END(__hyp_set_vectors) 288 289SYM_FUNC_START(__hyp_reset_vectors) 290 mov x0, #HVC_RESET_VECTORS 291 hvc #0 292 ret 293SYM_FUNC_END(__hyp_reset_vectors) 294 295/* 296 * Entry point to finalise EL2 and switch to VHE if deemed capable 297 * 298 * w0: boot mode, as returned by init_kernel_el() 299 */ 300SYM_FUNC_START(finalise_el2) 301 // Need to have booted at EL2 302 cmp w0, #BOOT_CPU_MODE_EL2 303 b.ne 1f 304 305 // and still be at EL1 306 mrs x0, CurrentEL 307 cmp x0, #CurrentEL_EL1 308 b.ne 1f 309 310 mov x0, #HVC_FINALISE_EL2 311 hvc #0 3121: 313 ret 314SYM_FUNC_END(finalise_el2) 315