1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 ARM Ltd. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <linux/random.h> 9 #include <linux/memblock.h> 10 #include <asm/alternative.h> 11 #include <asm/debug-monitors.h> 12 #include <asm/insn.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/memory.h> 15 16 /* 17 * The LSB of the HYP VA tag 18 */ 19 static u8 tag_lsb; 20 /* 21 * The HYP VA tag value with the region bit 22 */ 23 static u64 tag_val; 24 static u64 va_mask; 25 26 /* 27 * Compute HYP VA by using the same computation as kern_hyp_va(). 28 */ 29 static u64 __early_kern_hyp_va(u64 addr) 30 { 31 addr &= va_mask; 32 addr |= tag_val << tag_lsb; 33 return addr; 34 } 35 36 /* 37 * Store a hyp VA <-> PA offset into a EL2-owned variable. 38 */ 39 static void init_hyp_physvirt_offset(void) 40 { 41 u64 kern_va, hyp_va; 42 43 /* Compute the offset from the hyp VA and PA of a random symbol. */ 44 kern_va = (u64)lm_alias(__hyp_text_start); 45 hyp_va = __early_kern_hyp_va(kern_va); 46 hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va; 47 } 48 49 /* 50 * Calculate the actual VA size used by the hypervisor 51 */ 52 __init u32 kvm_hyp_va_bits(void) 53 { 54 /* 55 * The ID map is always configured for 48 bits of translation, which may 56 * be different from the number of VA bits used by the regular kernel 57 * stage 1. 58 * 59 * At EL2, there is only one TTBR register, and we can't switch between 60 * translation tables *and* update TCR_EL2.T0SZ at the same time. Bottom 61 * line: we need to use the extended range with *both* our translation 62 * tables. 63 * 64 * So use the maximum of the idmap VA bits and the regular kernel stage 65 * 1 VA bits as the hypervisor VA size to assure that the hypervisor can 66 * both ID map its code page and map any kernel memory. 67 */ 68 return max(IDMAP_VA_BITS, vabits_actual); 69 } 70 71 /* 72 * We want to generate a hyp VA with the following format (with V == 73 * hypervisor VA bits): 74 * 75 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0 76 * --------------------------------------------------------- 77 * | 0000000 | hyp_va_msb | random tag | kern linear VA | 78 * |--------- tag_val -----------|----- va_mask ---| 79 * 80 * which does not conflict with the idmap regions. 81 */ 82 __init void kvm_compute_layout(void) 83 { 84 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start); 85 u64 hyp_va_msb; 86 u32 hyp_va_bits = kvm_hyp_va_bits(); 87 88 /* Where is my RAM region? */ 89 hyp_va_msb = idmap_addr & BIT(hyp_va_bits - 1); 90 hyp_va_msb ^= BIT(hyp_va_bits - 1); 91 92 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^ 93 (u64)(high_memory - 1)); 94 95 va_mask = GENMASK_ULL(tag_lsb - 1, 0); 96 tag_val = hyp_va_msb; 97 98 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (hyp_va_bits - 1)) { 99 /* We have some free bits to insert a random tag. */ 100 tag_val |= get_random_long() & GENMASK_ULL(hyp_va_bits - 2, tag_lsb); 101 } 102 tag_val >>= tag_lsb; 103 104 init_hyp_physvirt_offset(); 105 } 106 107 /* 108 * The .hyp.reloc ELF section contains a list of kimg positions that 109 * contains kimg VAs but will be accessed only in hyp execution context. 110 * Convert them to hyp VAs. See gen-hyprel.c for more details. 111 */ 112 __init void kvm_apply_hyp_relocations(void) 113 { 114 int32_t *rel; 115 int32_t *begin = (int32_t *)__hyp_reloc_begin; 116 int32_t *end = (int32_t *)__hyp_reloc_end; 117 118 for (rel = begin; rel < end; ++rel) { 119 uintptr_t *ptr, kimg_va; 120 121 /* 122 * Each entry contains a 32-bit relative offset from itself 123 * to a kimg VA position. 124 */ 125 ptr = (uintptr_t *)lm_alias((char *)rel + *rel); 126 127 /* Read the kimg VA value at the relocation address. */ 128 kimg_va = *ptr; 129 130 /* Convert to hyp VA and store back to the relocation address. */ 131 *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va)); 132 } 133 } 134 135 static u32 compute_instruction(int n, u32 rd, u32 rn) 136 { 137 u32 insn = AARCH64_BREAK_FAULT; 138 139 switch (n) { 140 case 0: 141 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND, 142 AARCH64_INSN_VARIANT_64BIT, 143 rn, rd, va_mask); 144 break; 145 146 case 1: 147 /* ROR is a variant of EXTR with Rm = Rn */ 148 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 149 rn, rn, rd, 150 tag_lsb); 151 break; 152 153 case 2: 154 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 155 tag_val & GENMASK(11, 0), 156 AARCH64_INSN_VARIANT_64BIT, 157 AARCH64_INSN_ADSB_ADD); 158 break; 159 160 case 3: 161 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 162 tag_val & GENMASK(23, 12), 163 AARCH64_INSN_VARIANT_64BIT, 164 AARCH64_INSN_ADSB_ADD); 165 break; 166 167 case 4: 168 /* ROR is a variant of EXTR with Rm = Rn */ 169 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 170 rn, rn, rd, 64 - tag_lsb); 171 break; 172 } 173 174 return insn; 175 } 176 177 void __init kvm_update_va_mask(struct alt_instr *alt, 178 __le32 *origptr, __le32 *updptr, int nr_inst) 179 { 180 int i; 181 182 BUG_ON(nr_inst != 5); 183 184 for (i = 0; i < nr_inst; i++) { 185 u32 rd, rn, insn, oinsn; 186 187 /* 188 * VHE doesn't need any address translation, let's NOP 189 * everything. 190 * 191 * Alternatively, if the tag is zero (because the layout 192 * dictates it and we don't have any spare bits in the 193 * address), NOP everything after masking the kernel VA. 194 */ 195 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) { 196 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); 197 continue; 198 } 199 200 oinsn = le32_to_cpu(origptr[i]); 201 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); 202 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn); 203 204 insn = compute_instruction(i, rd, rn); 205 BUG_ON(insn == AARCH64_BREAK_FAULT); 206 207 updptr[i] = cpu_to_le32(insn); 208 } 209 } 210 211 void kvm_patch_vector_branch(struct alt_instr *alt, 212 __le32 *origptr, __le32 *updptr, int nr_inst) 213 { 214 u64 addr; 215 u32 insn; 216 217 BUG_ON(nr_inst != 4); 218 219 if (!cpus_have_cap(ARM64_SPECTRE_V3A) || 220 WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))) 221 return; 222 223 /* 224 * Compute HYP VA by using the same computation as kern_hyp_va() 225 */ 226 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector)); 227 228 /* Use PC[10:7] to branch to the same vector in KVM */ 229 addr |= ((u64)origptr & GENMASK_ULL(10, 7)); 230 231 /* 232 * Branch over the preamble in order to avoid the initial store on 233 * the stack (which we already perform in the hardening vectors). 234 */ 235 addr += KVM_VECTOR_PREAMBLE; 236 237 /* movz x0, #(addr & 0xffff) */ 238 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 239 (u16)addr, 240 0, 241 AARCH64_INSN_VARIANT_64BIT, 242 AARCH64_INSN_MOVEWIDE_ZERO); 243 *updptr++ = cpu_to_le32(insn); 244 245 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */ 246 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 247 (u16)(addr >> 16), 248 16, 249 AARCH64_INSN_VARIANT_64BIT, 250 AARCH64_INSN_MOVEWIDE_KEEP); 251 *updptr++ = cpu_to_le32(insn); 252 253 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */ 254 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 255 (u16)(addr >> 32), 256 32, 257 AARCH64_INSN_VARIANT_64BIT, 258 AARCH64_INSN_MOVEWIDE_KEEP); 259 *updptr++ = cpu_to_le32(insn); 260 261 /* br x0 */ 262 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0, 263 AARCH64_INSN_BRANCH_NOLINK); 264 *updptr++ = cpu_to_le32(insn); 265 } 266 267 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst) 268 { 269 u32 insn, oinsn, rd; 270 271 BUG_ON(nr_inst != 4); 272 273 /* Compute target register */ 274 oinsn = le32_to_cpu(*origptr); 275 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); 276 277 /* movz rd, #(val & 0xffff) */ 278 insn = aarch64_insn_gen_movewide(rd, 279 (u16)val, 280 0, 281 AARCH64_INSN_VARIANT_64BIT, 282 AARCH64_INSN_MOVEWIDE_ZERO); 283 *updptr++ = cpu_to_le32(insn); 284 285 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */ 286 insn = aarch64_insn_gen_movewide(rd, 287 (u16)(val >> 16), 288 16, 289 AARCH64_INSN_VARIANT_64BIT, 290 AARCH64_INSN_MOVEWIDE_KEEP); 291 *updptr++ = cpu_to_le32(insn); 292 293 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */ 294 insn = aarch64_insn_gen_movewide(rd, 295 (u16)(val >> 32), 296 32, 297 AARCH64_INSN_VARIANT_64BIT, 298 AARCH64_INSN_MOVEWIDE_KEEP); 299 *updptr++ = cpu_to_le32(insn); 300 301 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */ 302 insn = aarch64_insn_gen_movewide(rd, 303 (u16)(val >> 48), 304 48, 305 AARCH64_INSN_VARIANT_64BIT, 306 AARCH64_INSN_MOVEWIDE_KEEP); 307 *updptr++ = cpu_to_le32(insn); 308 } 309 310 void kvm_get_kimage_voffset(struct alt_instr *alt, 311 __le32 *origptr, __le32 *updptr, int nr_inst) 312 { 313 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst); 314 } 315 316 void kvm_compute_final_ctr_el0(struct alt_instr *alt, 317 __le32 *origptr, __le32 *updptr, int nr_inst) 318 { 319 generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0), 320 origptr, updptr, nr_inst); 321 } 322