xref: /linux/arch/arm64/kvm/hyp/nvhe/host.S (revision 26fbb4c8c7c3ee9a4c3b4de555a8587b5a19154e)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020 - Google Inc
4 * Author: Andrew Scull <ascull@google.com>
5 */
6
7#include <linux/linkage.h>
8
9#include <asm/assembler.h>
10#include <asm/kvm_asm.h>
11#include <asm/kvm_mmu.h>
12
13	.text
14
15SYM_FUNC_START(__host_exit)
16	get_host_ctxt	x0, x1
17
18	/* Store the host regs x2 and x3 */
19	stp	x2, x3,   [x0, #CPU_XREG_OFFSET(2)]
20
21	/* Retrieve the host regs x0-x1 from the stack */
22	ldp	x2, x3, [sp], #16	// x0, x1
23
24	/* Store the host regs x0-x1 and x4-x17 */
25	stp	x2, x3,   [x0, #CPU_XREG_OFFSET(0)]
26	stp	x4, x5,   [x0, #CPU_XREG_OFFSET(4)]
27	stp	x6, x7,   [x0, #CPU_XREG_OFFSET(6)]
28	stp	x8, x9,   [x0, #CPU_XREG_OFFSET(8)]
29	stp	x10, x11, [x0, #CPU_XREG_OFFSET(10)]
30	stp	x12, x13, [x0, #CPU_XREG_OFFSET(12)]
31	stp	x14, x15, [x0, #CPU_XREG_OFFSET(14)]
32	stp	x16, x17, [x0, #CPU_XREG_OFFSET(16)]
33
34	/* Store the host regs x18-x29, lr */
35	save_callee_saved_regs x0
36
37	/* Save the host context pointer in x29 across the function call */
38	mov	x29, x0
39	bl	handle_trap
40
41	/* Restore host regs x0-x17 */
42__host_enter_restore_full:
43	ldp	x0, x1,   [x29, #CPU_XREG_OFFSET(0)]
44	ldp	x2, x3,   [x29, #CPU_XREG_OFFSET(2)]
45	ldp	x4, x5,   [x29, #CPU_XREG_OFFSET(4)]
46	ldp	x6, x7,   [x29, #CPU_XREG_OFFSET(6)]
47
48	/* x0-7 are use for panic arguments */
49__host_enter_for_panic:
50	ldp	x8, x9,   [x29, #CPU_XREG_OFFSET(8)]
51	ldp	x10, x11, [x29, #CPU_XREG_OFFSET(10)]
52	ldp	x12, x13, [x29, #CPU_XREG_OFFSET(12)]
53	ldp	x14, x15, [x29, #CPU_XREG_OFFSET(14)]
54	ldp	x16, x17, [x29, #CPU_XREG_OFFSET(16)]
55
56	/* Restore host regs x18-x29, lr */
57	restore_callee_saved_regs x29
58
59	/* Do not touch any register after this! */
60__host_enter_without_restoring:
61	eret
62	sb
63SYM_FUNC_END(__host_exit)
64
65/*
66 * void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
67 */
68SYM_FUNC_START(__host_enter)
69	mov	x29, x0
70	b	__host_enter_restore_full
71SYM_FUNC_END(__host_enter)
72
73/*
74 * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
75 */
76SYM_FUNC_START(__hyp_do_panic)
77	/* Load the format arguments into x1-7 */
78	mov	x6, x3
79	get_vcpu_ptr x7, x3
80
81	mrs	x3, esr_el2
82	mrs	x4, far_el2
83	mrs	x5, hpfar_el2
84
85	/* Prepare and exit to the host's panic funciton. */
86	mov	lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
87		      PSR_MODE_EL1h)
88	msr	spsr_el2, lr
89	ldr	lr, =panic
90	msr	elr_el2, lr
91
92	/*
93	 * Set the panic format string and enter the host, conditionally
94	 * restoring the host context.
95	 */
96	cmp	x0, xzr
97	ldr	x0, =__hyp_panic_string
98	b.eq	__host_enter_without_restoring
99	b	__host_enter_for_panic
100SYM_FUNC_END(__hyp_do_panic)
101
102.macro host_el1_sync_vect
103	.align 7
104.L__vect_start\@:
105	stp	x0, x1, [sp, #-16]!
106	mrs	x0, esr_el2
107	lsr	x0, x0, #ESR_ELx_EC_SHIFT
108	cmp	x0, #ESR_ELx_EC_HVC64
109	b.ne	__host_exit
110
111	ldp	x0, x1, [sp]		// Don't fixup the stack yet
112
113	/* Check for a stub HVC call */
114	cmp	x0, #HVC_STUB_HCALL_NR
115	b.hs	__host_exit
116
117	add	sp, sp, #16
118	/*
119	 * Compute the idmap address of __kvm_handle_stub_hvc and
120	 * jump there. Since we use kimage_voffset, do not use the
121	 * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
122	 * (by loading it from the constant pool).
123	 *
124	 * Preserve x0-x4, which may contain stub parameters.
125	 */
126	ldr	x5, =__kvm_handle_stub_hvc
127	kimg_pa x5, x6
128	br	x5
129.L__vect_end\@:
130.if ((.L__vect_end\@ - .L__vect_start\@) > 0x80)
131	.error "host_el1_sync_vect larger than vector entry"
132.endif
133.endm
134
135.macro invalid_host_el2_vect
136	.align 7
137	/* If a guest is loaded, panic out of it. */
138	stp	x0, x1, [sp, #-16]!
139	get_loaded_vcpu x0, x1
140	cbnz	x0, __guest_exit_panic
141	add	sp, sp, #16
142
143	/*
144	 * The panic may not be clean if the exception is taken before the host
145	 * context has been saved by __host_exit or after the hyp context has
146	 * been partially clobbered by __host_enter.
147	 */
148	b	hyp_panic
149.endm
150
151.macro invalid_host_el1_vect
152	.align 7
153	mov	x0, xzr		/* restore_host = false */
154	mrs	x1, spsr_el2
155	mrs	x2, elr_el2
156	mrs	x3, par_el1
157	b	__hyp_do_panic
158.endm
159
160/*
161 * The host vector does not use an ESB instruction in order to avoid consuming
162 * SErrors that should only be consumed by the host. Guest entry is deferred by
163 * __guest_enter if there are any pending asynchronous exceptions so hyp will
164 * always return to the host without having consumerd host SErrors.
165 *
166 * CONFIG_KVM_INDIRECT_VECTORS is not applied to the host vectors because the
167 * host knows about the EL2 vectors already, and there is no point in hiding
168 * them.
169 */
170	.align 11
171SYM_CODE_START(__kvm_hyp_host_vector)
172	invalid_host_el2_vect			// Synchronous EL2t
173	invalid_host_el2_vect			// IRQ EL2t
174	invalid_host_el2_vect			// FIQ EL2t
175	invalid_host_el2_vect			// Error EL2t
176
177	invalid_host_el2_vect			// Synchronous EL2h
178	invalid_host_el2_vect			// IRQ EL2h
179	invalid_host_el2_vect			// FIQ EL2h
180	invalid_host_el2_vect			// Error EL2h
181
182	host_el1_sync_vect			// Synchronous 64-bit EL1
183	invalid_host_el1_vect			// IRQ 64-bit EL1
184	invalid_host_el1_vect			// FIQ 64-bit EL1
185	invalid_host_el1_vect			// Error 64-bit EL1
186
187	invalid_host_el1_vect			// Synchronous 32-bit EL1
188	invalid_host_el1_vect			// IRQ 32-bit EL1
189	invalid_host_el1_vect			// FIQ 32-bit EL1
190	invalid_host_el1_vect			// Error 32-bit EL1
191SYM_CODE_END(__kvm_hyp_host_vector)
192
193/*
194 * Forward SMC with arguments in struct kvm_cpu_context, and
195 * store the result into the same struct. Assumes SMCCC 1.2 or older.
196 *
197 * x0: struct kvm_cpu_context*
198 */
199SYM_CODE_START(__kvm_hyp_host_forward_smc)
200	/*
201	 * Use x18 to keep the pointer to the host context because
202	 * x18 is callee-saved in SMCCC but not in AAPCS64.
203	 */
204	mov	x18, x0
205
206	ldp	x0, x1,   [x18, #CPU_XREG_OFFSET(0)]
207	ldp	x2, x3,   [x18, #CPU_XREG_OFFSET(2)]
208	ldp	x4, x5,   [x18, #CPU_XREG_OFFSET(4)]
209	ldp	x6, x7,   [x18, #CPU_XREG_OFFSET(6)]
210	ldp	x8, x9,   [x18, #CPU_XREG_OFFSET(8)]
211	ldp	x10, x11, [x18, #CPU_XREG_OFFSET(10)]
212	ldp	x12, x13, [x18, #CPU_XREG_OFFSET(12)]
213	ldp	x14, x15, [x18, #CPU_XREG_OFFSET(14)]
214	ldp	x16, x17, [x18, #CPU_XREG_OFFSET(16)]
215
216	smc	#0
217
218	stp	x0, x1,   [x18, #CPU_XREG_OFFSET(0)]
219	stp	x2, x3,   [x18, #CPU_XREG_OFFSET(2)]
220	stp	x4, x5,   [x18, #CPU_XREG_OFFSET(4)]
221	stp	x6, x7,   [x18, #CPU_XREG_OFFSET(6)]
222	stp	x8, x9,   [x18, #CPU_XREG_OFFSET(8)]
223	stp	x10, x11, [x18, #CPU_XREG_OFFSET(10)]
224	stp	x12, x13, [x18, #CPU_XREG_OFFSET(12)]
225	stp	x14, x15, [x18, #CPU_XREG_OFFSET(14)]
226	stp	x16, x17, [x18, #CPU_XREG_OFFSET(16)]
227
228	ret
229SYM_CODE_END(__kvm_hyp_host_forward_smc)
230