1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/kvm_host.h>
8 #include <linux/random.h>
9 #include <linux/memblock.h>
10 #include <asm/alternative.h>
11 #include <asm/debug-monitors.h>
12 #include <asm/insn.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/memory.h>
15
16 /*
17 * The LSB of the HYP VA tag
18 */
19 static u8 tag_lsb;
20 /*
21 * The HYP VA tag value with the region bit
22 */
23 static u64 tag_val;
24 static u64 va_mask;
25
26 /*
27 * Compute HYP VA by using the same computation as kern_hyp_va().
28 */
__early_kern_hyp_va(u64 addr)29 static u64 __early_kern_hyp_va(u64 addr)
30 {
31 addr &= va_mask;
32 addr |= tag_val << tag_lsb;
33 return addr;
34 }
35
36 /*
37 * Store a hyp VA <-> PA offset into a EL2-owned variable.
38 */
init_hyp_physvirt_offset(void)39 static void init_hyp_physvirt_offset(void)
40 {
41 u64 kern_va, hyp_va;
42
43 /* Compute the offset from the hyp VA and PA of a random symbol. */
44 kern_va = (u64)lm_alias(__hyp_text_start);
45 hyp_va = __early_kern_hyp_va(kern_va);
46 hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
47 }
48
49 /*
50 * Calculate the actual VA size used by the hypervisor
51 */
kvm_hyp_va_bits(void)52 __init u32 kvm_hyp_va_bits(void)
53 {
54 /*
55 * The ID map is always configured for 48 bits of translation, which may
56 * be different from the number of VA bits used by the regular kernel
57 * stage 1.
58 *
59 * At EL2, there is only one TTBR register, and we can't switch between
60 * translation tables *and* update TCR_EL2.T0SZ at the same time. Bottom
61 * line: we need to use the extended range with *both* our translation
62 * tables.
63 *
64 * So use the maximum of the idmap VA bits and the regular kernel stage
65 * 1 VA bits as the hypervisor VA size to assure that the hypervisor can
66 * both ID map its code page and map any kernel memory.
67 */
68 return max(IDMAP_VA_BITS, vabits_actual);
69 }
70
71 /*
72 * We want to generate a hyp VA with the following format (with V ==
73 * hypervisor VA bits):
74 *
75 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0
76 * ---------------------------------------------------------
77 * | 0000000 | hyp_va_msb | random tag | kern linear VA |
78 * |--------- tag_val -----------|----- va_mask ---|
79 *
80 * which does not conflict with the idmap regions.
81 */
kvm_compute_layout(void)82 __init void kvm_compute_layout(void)
83 {
84 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
85 u64 hyp_va_msb;
86 u32 hyp_va_bits = kvm_hyp_va_bits();
87
88 /* Where is my RAM region? */
89 hyp_va_msb = idmap_addr & BIT(hyp_va_bits - 1);
90 hyp_va_msb ^= BIT(hyp_va_bits - 1);
91
92 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
93 (u64)(high_memory - 1));
94
95 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
96 tag_val = hyp_va_msb;
97
98 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (hyp_va_bits - 1)) {
99 /* We have some free bits to insert a random tag. */
100 tag_val |= get_random_long() & GENMASK_ULL(hyp_va_bits - 2, tag_lsb);
101 }
102 tag_val >>= tag_lsb;
103
104 init_hyp_physvirt_offset();
105 }
106
107 /*
108 * The .hyp.reloc ELF section contains a list of kimg positions that
109 * contains kimg VAs but will be accessed only in hyp execution context.
110 * Convert them to hyp VAs. See gen-hyprel.c for more details.
111 */
kvm_apply_hyp_relocations(void)112 __init void kvm_apply_hyp_relocations(void)
113 {
114 int32_t *rel;
115 int32_t *begin = (int32_t *)__hyp_reloc_begin;
116 int32_t *end = (int32_t *)__hyp_reloc_end;
117
118 for (rel = begin; rel < end; ++rel) {
119 uintptr_t *ptr, kimg_va;
120
121 /*
122 * Each entry contains a 32-bit relative offset from itself
123 * to a kimg VA position.
124 */
125 ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
126
127 /* Read the kimg VA value at the relocation address. */
128 kimg_va = *ptr;
129
130 /* Convert to hyp VA and store back to the relocation address. */
131 *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
132 }
133 }
134
compute_instruction(int n,u32 rd,u32 rn)135 static u32 compute_instruction(int n, u32 rd, u32 rn)
136 {
137 u32 insn = AARCH64_BREAK_FAULT;
138
139 switch (n) {
140 case 0:
141 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
142 AARCH64_INSN_VARIANT_64BIT,
143 rn, rd, va_mask);
144 break;
145
146 case 1:
147 /* ROR is a variant of EXTR with Rm = Rn */
148 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
149 rn, rn, rd,
150 tag_lsb);
151 break;
152
153 case 2:
154 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
155 tag_val & GENMASK(11, 0),
156 AARCH64_INSN_VARIANT_64BIT,
157 AARCH64_INSN_ADSB_ADD);
158 break;
159
160 case 3:
161 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
162 tag_val & GENMASK(23, 12),
163 AARCH64_INSN_VARIANT_64BIT,
164 AARCH64_INSN_ADSB_ADD);
165 break;
166
167 case 4:
168 /* ROR is a variant of EXTR with Rm = Rn */
169 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
170 rn, rn, rd, 64 - tag_lsb);
171 break;
172 }
173
174 return insn;
175 }
176
kvm_update_va_mask(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)177 void __init kvm_update_va_mask(struct alt_instr *alt,
178 __le32 *origptr, __le32 *updptr, int nr_inst)
179 {
180 int i;
181
182 BUG_ON(nr_inst != 5);
183
184 for (i = 0; i < nr_inst; i++) {
185 u32 rd, rn, insn, oinsn;
186
187 /*
188 * VHE doesn't need any address translation, let's NOP
189 * everything.
190 *
191 * Alternatively, if the tag is zero (because the layout
192 * dictates it and we don't have any spare bits in the
193 * address), NOP everything after masking the kernel VA.
194 */
195 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
196 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
197 continue;
198 }
199
200 oinsn = le32_to_cpu(origptr[i]);
201 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
202 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
203
204 insn = compute_instruction(i, rd, rn);
205 BUG_ON(insn == AARCH64_BREAK_FAULT);
206
207 updptr[i] = cpu_to_le32(insn);
208 }
209 }
210
kvm_patch_vector_branch(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)211 void kvm_patch_vector_branch(struct alt_instr *alt,
212 __le32 *origptr, __le32 *updptr, int nr_inst)
213 {
214 u64 addr;
215 u32 insn;
216
217 BUG_ON(nr_inst != 4);
218
219 if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
220 WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
221 return;
222
223 /*
224 * Compute HYP VA by using the same computation as kern_hyp_va()
225 */
226 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
227
228 /* Use PC[10:7] to branch to the same vector in KVM */
229 addr |= ((u64)origptr & GENMASK_ULL(10, 7));
230
231 /*
232 * Branch over the preamble in order to avoid the initial store on
233 * the stack (which we already perform in the hardening vectors).
234 */
235 addr += KVM_VECTOR_PREAMBLE;
236
237 /* movz x0, #(addr & 0xffff) */
238 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
239 (u16)addr,
240 0,
241 AARCH64_INSN_VARIANT_64BIT,
242 AARCH64_INSN_MOVEWIDE_ZERO);
243 *updptr++ = cpu_to_le32(insn);
244
245 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
246 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
247 (u16)(addr >> 16),
248 16,
249 AARCH64_INSN_VARIANT_64BIT,
250 AARCH64_INSN_MOVEWIDE_KEEP);
251 *updptr++ = cpu_to_le32(insn);
252
253 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
254 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
255 (u16)(addr >> 32),
256 32,
257 AARCH64_INSN_VARIANT_64BIT,
258 AARCH64_INSN_MOVEWIDE_KEEP);
259 *updptr++ = cpu_to_le32(insn);
260
261 /* br x0 */
262 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
263 AARCH64_INSN_BRANCH_NOLINK);
264 *updptr++ = cpu_to_le32(insn);
265 }
266
generate_mov_q(u64 val,__le32 * origptr,__le32 * updptr,int nr_inst)267 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
268 {
269 u32 insn, oinsn, rd;
270
271 BUG_ON(nr_inst != 4);
272
273 /* Compute target register */
274 oinsn = le32_to_cpu(*origptr);
275 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
276
277 /* movz rd, #(val & 0xffff) */
278 insn = aarch64_insn_gen_movewide(rd,
279 (u16)val,
280 0,
281 AARCH64_INSN_VARIANT_64BIT,
282 AARCH64_INSN_MOVEWIDE_ZERO);
283 *updptr++ = cpu_to_le32(insn);
284
285 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
286 insn = aarch64_insn_gen_movewide(rd,
287 (u16)(val >> 16),
288 16,
289 AARCH64_INSN_VARIANT_64BIT,
290 AARCH64_INSN_MOVEWIDE_KEEP);
291 *updptr++ = cpu_to_le32(insn);
292
293 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
294 insn = aarch64_insn_gen_movewide(rd,
295 (u16)(val >> 32),
296 32,
297 AARCH64_INSN_VARIANT_64BIT,
298 AARCH64_INSN_MOVEWIDE_KEEP);
299 *updptr++ = cpu_to_le32(insn);
300
301 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
302 insn = aarch64_insn_gen_movewide(rd,
303 (u16)(val >> 48),
304 48,
305 AARCH64_INSN_VARIANT_64BIT,
306 AARCH64_INSN_MOVEWIDE_KEEP);
307 *updptr++ = cpu_to_le32(insn);
308 }
309
kvm_get_kimage_voffset(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)310 void kvm_get_kimage_voffset(struct alt_instr *alt,
311 __le32 *origptr, __le32 *updptr, int nr_inst)
312 {
313 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
314 }
315
kvm_compute_final_ctr_el0(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)316 void kvm_compute_final_ctr_el0(struct alt_instr *alt,
317 __le32 *origptr, __le32 *updptr, int nr_inst)
318 {
319 generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0),
320 origptr, updptr, nr_inst);
321 }
322