1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * AArch64 processor specific defines 4 * 5 * Copyright (C) 2018, Red Hat, Inc. 6 */ 7 #ifndef SELFTEST_KVM_PROCESSOR_H 8 #define SELFTEST_KVM_PROCESSOR_H 9 10 #include "kvm_util.h" 11 #include "ucall_common.h" 12 13 #include <linux/stringify.h> 14 #include <linux/types.h> 15 #include <asm/barrier.h> 16 #include <asm/brk-imm.h> 17 #include <asm/esr.h> 18 #include <asm/sysreg.h> 19 20 21 #define ARM64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ 22 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 23 24 /* 25 * KVM_ARM64_SYS_REG(sys_reg_id): Helper macro to convert 26 * SYS_* register definitions in asm/sysreg.h to use in KVM 27 * calls such as vcpu_get_reg() and vcpu_set_reg(). 28 */ 29 #define KVM_ARM64_SYS_REG(sys_reg_id) \ 30 ARM64_SYS_REG(sys_reg_Op0(sys_reg_id), \ 31 sys_reg_Op1(sys_reg_id), \ 32 sys_reg_CRn(sys_reg_id), \ 33 sys_reg_CRm(sys_reg_id), \ 34 sys_reg_Op2(sys_reg_id)) 35 36 /* 37 * Default MAIR 38 * index attribute 39 * DEVICE_nGnRnE 0 0000:0000 40 * DEVICE_nGnRE 1 0000:0100 41 * DEVICE_GRE 2 0000:1100 42 * NORMAL_NC 3 0100:0100 43 * NORMAL 4 1111:1111 44 * NORMAL_WT 5 1011:1011 45 */ 46 47 /* Linux doesn't use these memory types, so let's define them. */ 48 #define MAIR_ATTR_DEVICE_GRE UL(0x0c) 49 #define MAIR_ATTR_NORMAL_WT UL(0xbb) 50 51 #define MT_DEVICE_nGnRnE 0 52 #define MT_DEVICE_nGnRE 1 53 #define MT_DEVICE_GRE 2 54 #define MT_NORMAL_NC 3 55 #define MT_NORMAL 4 56 #define MT_NORMAL_WT 5 57 58 #define DEFAULT_MAIR_EL1 \ 59 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 60 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 61 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \ 62 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 63 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 64 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT)) 65 66 /* TCR_EL1 specific flags */ 67 #define TCR_T0SZ_OFFSET 0 68 #define TCR_T0SZ(x) ((UL(64) - (x)) << TCR_T0SZ_OFFSET) 69 70 #define TCR_IRGN0_SHIFT 8 71 #define TCR_IRGN0_MASK (UL(3) << TCR_IRGN0_SHIFT) 72 #define TCR_IRGN0_NC (UL(0) << TCR_IRGN0_SHIFT) 73 #define TCR_IRGN0_WBWA (UL(1) << TCR_IRGN0_SHIFT) 74 #define TCR_IRGN0_WT (UL(2) << TCR_IRGN0_SHIFT) 75 #define TCR_IRGN0_WBnWA (UL(3) << TCR_IRGN0_SHIFT) 76 77 #define TCR_ORGN0_SHIFT 10 78 #define TCR_ORGN0_MASK (UL(3) << TCR_ORGN0_SHIFT) 79 #define TCR_ORGN0_NC (UL(0) << TCR_ORGN0_SHIFT) 80 #define TCR_ORGN0_WBWA (UL(1) << TCR_ORGN0_SHIFT) 81 #define TCR_ORGN0_WT (UL(2) << TCR_ORGN0_SHIFT) 82 #define TCR_ORGN0_WBnWA (UL(3) << TCR_ORGN0_SHIFT) 83 84 #define TCR_SH0_SHIFT 12 85 #define TCR_SH0_MASK (UL(3) << TCR_SH0_SHIFT) 86 #define TCR_SH0_INNER (UL(3) << TCR_SH0_SHIFT) 87 88 #define TCR_TG0_SHIFT 14 89 #define TCR_TG0_MASK (UL(3) << TCR_TG0_SHIFT) 90 #define TCR_TG0_4K (UL(0) << TCR_TG0_SHIFT) 91 #define TCR_TG0_64K (UL(1) << TCR_TG0_SHIFT) 92 #define TCR_TG0_16K (UL(2) << TCR_TG0_SHIFT) 93 94 #define TCR_EPD1_SHIFT 23 95 #define TCR_EPD1_MASK (UL(1) << TCR_EPD1_SHIFT) 96 97 #define TCR_IPS_SHIFT 32 98 #define TCR_IPS_MASK (UL(7) << TCR_IPS_SHIFT) 99 #define TCR_IPS_52_BITS (UL(6) << TCR_IPS_SHIFT) 100 #define TCR_IPS_48_BITS (UL(5) << TCR_IPS_SHIFT) 101 #define TCR_IPS_40_BITS (UL(2) << TCR_IPS_SHIFT) 102 #define TCR_IPS_36_BITS (UL(1) << TCR_IPS_SHIFT) 103 104 #define TCR_TBI1 (UL(1) << 38) 105 #define TCR_HA (UL(1) << 39) 106 #define TCR_DS (UL(1) << 59) 107 108 /* 109 * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers). 110 */ 111 #define PTE_ATTRINDX(t) ((t) << 2) 112 #define PTE_ATTRINDX_MASK GENMASK(4, 2) 113 #define PTE_ATTRINDX_SHIFT 2 114 115 #define PTE_VALID BIT(0) 116 #define PGD_TYPE_TABLE BIT(1) 117 #define PUD_TYPE_TABLE BIT(1) 118 #define PMD_TYPE_TABLE BIT(1) 119 #define PTE_TYPE_PAGE BIT(1) 120 121 #define PTE_SHARED (UL(3) << 8) /* SH[1:0], inner shareable */ 122 #define PTE_AF BIT(10) 123 124 #define PTE_ADDR_MASK(page_shift) GENMASK(47, (page_shift)) 125 #define PTE_ADDR_51_48 GENMASK(15, 12) 126 #define PTE_ADDR_51_48_SHIFT 12 127 #define PTE_ADDR_MASK_LPA2(page_shift) GENMASK(49, (page_shift)) 128 #define PTE_ADDR_51_50_LPA2 GENMASK(9, 8) 129 #define PTE_ADDR_51_50_LPA2_SHIFT 8 130 131 void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init); 132 struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, u32 vcpu_id, 133 struct kvm_vcpu_init *init, void *guest_code); 134 135 struct ex_regs { 136 u64 regs[31]; 137 u64 sp; 138 u64 pc; 139 u64 pstate; 140 }; 141 142 #define VECTOR_NUM 16 143 144 enum { 145 VECTOR_SYNC_CURRENT_SP0, 146 VECTOR_IRQ_CURRENT_SP0, 147 VECTOR_FIQ_CURRENT_SP0, 148 VECTOR_ERROR_CURRENT_SP0, 149 150 VECTOR_SYNC_CURRENT, 151 VECTOR_IRQ_CURRENT, 152 VECTOR_FIQ_CURRENT, 153 VECTOR_ERROR_CURRENT, 154 155 VECTOR_SYNC_LOWER_64, 156 VECTOR_IRQ_LOWER_64, 157 VECTOR_FIQ_LOWER_64, 158 VECTOR_ERROR_LOWER_64, 159 160 VECTOR_SYNC_LOWER_32, 161 VECTOR_IRQ_LOWER_32, 162 VECTOR_FIQ_LOWER_32, 163 VECTOR_ERROR_LOWER_32, 164 }; 165 166 #define VECTOR_IS_SYNC(v) ((v) == VECTOR_SYNC_CURRENT_SP0 || \ 167 (v) == VECTOR_SYNC_CURRENT || \ 168 (v) == VECTOR_SYNC_LOWER_64 || \ 169 (v) == VECTOR_SYNC_LOWER_32) 170 171 void aarch64_get_supported_page_sizes(u32 ipa, u32 *ipa4k, 172 u32 *ipa16k, u32 *ipa64k); 173 174 void vm_init_descriptor_tables(struct kvm_vm *vm); 175 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu); 176 177 typedef void(*handler_fn)(struct ex_regs *); 178 void vm_install_exception_handler(struct kvm_vm *vm, 179 int vector, handler_fn handler); 180 void vm_install_sync_handler(struct kvm_vm *vm, 181 int vector, int ec, handler_fn handler); 182 183 u64 *virt_get_pte_hva_at_level(struct kvm_vm *vm, gva_t gva, int level); 184 u64 *virt_get_pte_hva(struct kvm_vm *vm, gva_t gva); 185 186 static inline void cpu_relax(void) 187 { 188 asm volatile("yield" ::: "memory"); 189 } 190 191 #define isb() asm volatile("isb" : : : "memory") 192 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 193 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 194 195 #define __iowmb() dma_wmb() 196 197 #define __iormb(v) \ 198 ({ \ 199 unsigned long tmp; \ 200 \ 201 dma_rmb(); \ 202 \ 203 /* \ 204 * Courtesy of arch/arm64/include/asm/io.h: \ 205 * Create a dummy control dependency from the IO read to any \ 206 * later instructions. This ensures that a subsequent call \ 207 * to udelay() will be ordered due to the ISB in __delay(). \ 208 */ \ 209 asm volatile("eor %0, %1, %1\n" \ 210 "cbnz %0, ." \ 211 : "=r" (tmp) : "r" ((unsigned long)(v)) \ 212 : "memory"); \ 213 }) 214 215 static __always_inline void __raw_writel(u32 val, volatile void *addr) 216 { 217 asm volatile("str %w0, [%1]" : : "rZ" (val), "r" (addr)); 218 } 219 220 static __always_inline u32 __raw_readl(const volatile void *addr) 221 { 222 u32 val; 223 asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr)); 224 return val; 225 } 226 227 static __always_inline void __raw_writeq(u64 val, volatile void *addr) 228 { 229 asm volatile("str %0, [%1]" : : "rZ" (val), "r" (addr)); 230 } 231 232 static __always_inline u64 __raw_readq(const volatile void *addr) 233 { 234 u64 val; 235 asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr)); 236 return val; 237 } 238 239 #define writel_relaxed(v,c) ((void)__raw_writel((__force u32)cpu_to_le32(v),(c))) 240 #define readl_relaxed(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; }) 241 #define writeq_relaxed(v,c) ((void)__raw_writeq((__force u64)cpu_to_le64(v),(c))) 242 #define readq_relaxed(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; }) 243 244 #define writel(v,c) ({ __iowmb(); writel_relaxed((v),(c));}) 245 #define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(__v); __v; }) 246 #define writeq(v,c) ({ __iowmb(); writeq_relaxed((v),(c));}) 247 #define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(__v); __v; }) 248 249 250 static inline void local_irq_enable(void) 251 { 252 asm volatile("msr daifclr, #3" : : : "memory"); 253 } 254 255 static inline void local_irq_disable(void) 256 { 257 asm volatile("msr daifset, #3" : : : "memory"); 258 } 259 260 static inline void local_serror_enable(void) 261 { 262 asm volatile("msr daifclr, #4" : : : "memory"); 263 } 264 265 static inline void local_serror_disable(void) 266 { 267 asm volatile("msr daifset, #4" : : : "memory"); 268 } 269 270 /** 271 * struct arm_smccc_res - Result from SMC/HVC call 272 * @a0-a3 result values from registers 0 to 3 273 */ 274 struct arm_smccc_res { 275 unsigned long a0; 276 unsigned long a1; 277 unsigned long a2; 278 unsigned long a3; 279 }; 280 281 /** 282 * smccc_hvc - Invoke a SMCCC function using the hvc conduit 283 * @function_id: the SMCCC function to be called 284 * @arg0-arg6: SMCCC function arguments, corresponding to registers x1-x7 285 * @res: pointer to write the return values from registers x0-x3 286 * 287 */ 288 void smccc_hvc(u32 function_id, u64 arg0, u64 arg1, 289 u64 arg2, u64 arg3, u64 arg4, u64 arg5, 290 u64 arg6, struct arm_smccc_res *res); 291 292 /** 293 * smccc_smc - Invoke a SMCCC function using the smc conduit 294 * @function_id: the SMCCC function to be called 295 * @arg0-arg6: SMCCC function arguments, corresponding to registers x1-x7 296 * @res: pointer to write the return values from registers x0-x3 297 * 298 */ 299 void smccc_smc(u32 function_id, u64 arg0, u64 arg1, 300 u64 arg2, u64 arg3, u64 arg4, u64 arg5, 301 u64 arg6, struct arm_smccc_res *res); 302 303 /* Execute a Wait For Interrupt instruction. */ 304 void wfi(void); 305 306 void test_wants_mte(void); 307 void test_disable_default_vgic(void); 308 309 bool vm_supports_el2(struct kvm_vm *vm); 310 311 static inline bool test_supports_el2(void) 312 { 313 struct kvm_vm *vm = vm_create(1); 314 bool supported = vm_supports_el2(vm); 315 316 kvm_vm_free(vm); 317 return supported; 318 } 319 320 static inline bool vcpu_has_el2(struct kvm_vcpu *vcpu) 321 { 322 return vcpu->init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2); 323 } 324 325 #define MAPPED_EL2_SYSREG(el2, el1) \ 326 case SYS_##el1: \ 327 if (vcpu_has_el2(vcpu)) \ 328 alias = SYS_##el2; \ 329 break 330 331 332 static __always_inline u64 ctxt_reg_alias(struct kvm_vcpu *vcpu, u32 encoding) 333 { 334 u32 alias = encoding; 335 336 BUILD_BUG_ON(!__builtin_constant_p(encoding)); 337 338 switch (encoding) { 339 MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1); 340 MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1); 341 MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1); 342 MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1); 343 MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1); 344 MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1); 345 MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1); 346 MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1); 347 MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1); 348 MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1); 349 MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1); 350 MAPPED_EL2_SYSREG(TCR2_EL2, TCR2_EL1); 351 MAPPED_EL2_SYSREG(PIR_EL2, PIR_EL1); 352 MAPPED_EL2_SYSREG(PIRE0_EL2, PIRE0_EL1); 353 MAPPED_EL2_SYSREG(POR_EL2, POR_EL1); 354 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1); 355 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1); 356 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1); 357 MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1); 358 MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1); 359 MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1); 360 MAPPED_EL2_SYSREG(CNTHCTL_EL2, CNTKCTL_EL1); 361 case SYS_SP_EL1: 362 if (!vcpu_has_el2(vcpu)) 363 return ARM64_CORE_REG(sp_el1); 364 365 alias = SYS_SP_EL2; 366 break; 367 default: 368 BUILD_BUG(); 369 } 370 371 return KVM_ARM64_SYS_REG(alias); 372 } 373 374 void kvm_get_default_vcpu_target(struct kvm_vm *vm, struct kvm_vcpu_init *init); 375 376 static inline unsigned int get_current_el(void) 377 { 378 return (read_sysreg(CurrentEL) >> 2) & 0x3; 379 } 380 381 #define do_smccc(...) \ 382 do { \ 383 if (get_current_el() == 2) \ 384 smccc_smc(__VA_ARGS__); \ 385 else \ 386 smccc_hvc(__VA_ARGS__); \ 387 } while (0) 388 389 #endif /* SELFTEST_KVM_PROCESSOR_H */ 390