xref: /linux/arch/powerpc/kvm/tm.S (revision 3d0fe49454652117522f60bfbefb978ba0e5300b) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *
4 * Derived from book3s_hv_rmhandlers.S, which is:
5 *
6 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7 */
8
9#include <linux/export.h>
10#include <asm/reg.h>
11#include <asm/ppc_asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/tm.h>
14#include <asm/cputable.h>
15
16#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
17#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
18
19/*
20 * Save transactional state and TM-related registers.
21 * Called with:
22 * - r3 pointing to the vcpu struct
23 * - r4 containing the MSR with current TS bits:
24 * 	(For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR).
25 * - r5 containing a flag indicating that non-volatile registers
26 *	must be preserved.
27 * If r5 == 0, this can modify all checkpointed registers, but
28 * restores r1, r2 before exit.  If r5 != 0, this restores the
29 * MSR TM/FP/VEC/VSX bits to their state on entry.
30 */
31_GLOBAL(__kvmppc_save_tm)
32	mflr	r0
33	std	r0, PPC_LR_STKOFF(r1)
34	stdu    r1, -SWITCH_FRAME_SIZE(r1)
35
36	mr	r9, r3
37	cmpdi	cr7, r5, 0
38
39	/* Turn on TM. */
40	mfmsr	r8
41	mr	r10, r8
42	li	r0, 1
43	rldimi	r8, r0, MSR_TM_LG, 63-MSR_TM_LG
44	ori     r8, r8, MSR_FP
45	oris    r8, r8, (MSR_VEC | MSR_VSX)@h
46	mtmsrd	r8
47
48	rldicl. r4, r4, 64 - MSR_TS_S_LG, 62
49	beq	1f	/* TM not active in guest. */
50
51	std	r1, HSTATE_SCRATCH2(r13)
52	std	r3, HSTATE_SCRATCH1(r13)
53
54	/* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */
55	mfcr	r6
56	SAVE_GPR(6, r1)
57
58	/* Save DSCR so we can restore it to avoid running with user value */
59	mfspr	r7, SPRN_DSCR
60	SAVE_GPR(7, r1)
61
62	/*
63	 * We are going to do treclaim., which will modify all checkpointed
64	 * registers.  Save the non-volatile registers on the stack if
65	 * preservation of non-volatile state has been requested.
66	 */
67	beq	cr7, 3f
68	SAVE_NVGPRS(r1)
69
70	/* MSR[TS] will be 0 (non-transactional) once we do treclaim. */
71	li	r0, 0
72	rldimi	r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
73	SAVE_GPR(10, r1)	/* final MSR value */
743:
75#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
76BEGIN_FTR_SECTION
77	/* Emulation of the treclaim instruction needs TEXASR before treclaim */
78	mfspr	r6, SPRN_TEXASR
79	std	r6, VCPU_ORIG_TEXASR(r3)
80END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
81#endif
82
83	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
84	li	r5, 0
85	mtmsrd	r5, 1
86
87	li	r3, TM_CAUSE_KVM_RESCHED
88
89	/* All GPRs are volatile at this point. */
90	TRECLAIM(R3)
91
92	/* Temporarily store r13 and r9 so we have some regs to play with */
93	SET_SCRATCH0(r13)
94	GET_PACA(r13)
95	std	r9, PACATMSCRATCH(r13)
96	ld	r9, HSTATE_SCRATCH1(r13)
97
98	/* Save away PPR soon so we don't run with user value. */
99	std	r0, VCPU_GPRS_TM(0)(r9)
100	mfspr	r0, SPRN_PPR
101	HMT_MEDIUM
102
103	/* Reload stack pointer. */
104	std	r1, VCPU_GPRS_TM(1)(r9)
105	ld	r1, HSTATE_SCRATCH2(r13)
106
107	/* Set MSR RI now we have r1 and r13 back. */
108	std	r2, VCPU_GPRS_TM(2)(r9)
109	li	r2, MSR_RI
110	mtmsrd	r2, 1
111
112	/* Reload TOC pointer. */
113	LOAD_PACA_TOC()
114
115	/* Save all but r0-r2, r9 & r13 */
116	reg = 3
117	.rept	29
118	.if (reg != 9) && (reg != 13)
119	std	reg, VCPU_GPRS_TM(reg)(r9)
120	.endif
121	reg = reg + 1
122	.endr
123	/* ... now save r13 */
124	GET_SCRATCH0(r4)
125	std	r4, VCPU_GPRS_TM(13)(r9)
126	/* ... and save r9 */
127	ld	r4, PACATMSCRATCH(r13)
128	std	r4, VCPU_GPRS_TM(9)(r9)
129
130	/* Restore host DSCR and CR values, after saving guest values */
131	mfcr	r6
132	mfspr	r7, SPRN_DSCR
133	stw	r6, VCPU_CR_TM(r9)
134	std	r7, VCPU_DSCR_TM(r9)
135	REST_GPR(6, r1)
136	REST_GPR(7, r1)
137	mtcr	r6
138	mtspr	SPRN_DSCR, r7
139
140	/* Save away checkpointed SPRs. */
141	std	r0, VCPU_PPR_TM(r9)
142	mflr	r5
143	mfctr	r7
144	mfspr	r8, SPRN_AMR
145	mfspr	r10, SPRN_TAR
146	mfxer	r11
147	std	r5, VCPU_LR_TM(r9)
148	std	r7, VCPU_CTR_TM(r9)
149	std	r8, VCPU_AMR_TM(r9)
150	std	r10, VCPU_TAR_TM(r9)
151	std	r11, VCPU_XER_TM(r9)
152
153	/* Save FP/VSX. */
154	addi	r3, r9, VCPU_FPRS_TM
155	bl	store_fp_state
156	addi	r3, r9, VCPU_VRS_TM
157	bl	store_vr_state
158	mfspr	r6, SPRN_VRSAVE
159	stw	r6, VCPU_VRSAVE_TM(r9)
160
161	/* Restore non-volatile registers if requested to */
162	beq	cr7, 1f
163	REST_NVGPRS(r1)
164	REST_GPR(10, r1)
1651:
166	/*
167	 * We need to save these SPRs after the treclaim so that the software
168	 * error code is recorded correctly in the TEXASR.  Also the user may
169	 * change these outside of a transaction, so they must always be
170	 * context switched.
171	 */
172	mfspr	r7, SPRN_TEXASR
173	std	r7, VCPU_TEXASR(r9)
174	mfspr	r5, SPRN_TFHAR
175	mfspr	r6, SPRN_TFIAR
176	std	r5, VCPU_TFHAR(r9)
177	std	r6, VCPU_TFIAR(r9)
178
179	/* Restore MSR state if requested */
180	beq	cr7, 2f
181	mtmsrd	r10, 0
1822:
183	addi	r1, r1, SWITCH_FRAME_SIZE
184	ld	r0, PPC_LR_STKOFF(r1)
185	mtlr	r0
186	blr
187
188/*
189 * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can
190 * be invoked from C function by PR KVM only.
191 */
192_GLOBAL(_kvmppc_save_tm_pr)
193	mflr	r0
194	std	r0, PPC_LR_STKOFF(r1)
195	stdu    r1, -PPC_MIN_STKFRM(r1)
196
197	mfspr   r8, SPRN_TAR
198	std	r8, PPC_MIN_STKFRM-8(r1)
199
200	li	r5, 1		/* preserve non-volatile registers */
201	bl	__kvmppc_save_tm
202
203	ld	r8, PPC_MIN_STKFRM-8(r1)
204	mtspr   SPRN_TAR, r8
205
206	addi    r1, r1, PPC_MIN_STKFRM
207	ld	r0, PPC_LR_STKOFF(r1)
208	mtlr	r0
209	blr
210
211EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr);
212
213/*
214 * Restore transactional state and TM-related registers.
215 * Called with:
216 *  - r3 pointing to the vcpu struct.
217 *  - r4 is the guest MSR with desired TS bits:
218 * 	For HV KVM, it is VCPU_MSR
219 * 	For PR KVM, it is provided by caller
220 * - r5 containing a flag indicating that non-volatile registers
221 *	must be preserved.
222 * If r5 == 0, this potentially modifies all checkpointed registers, but
223 * restores r1, r2 from the PACA before exit.
224 * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry.
225 */
226_GLOBAL(__kvmppc_restore_tm)
227	mflr	r0
228	std	r0, PPC_LR_STKOFF(r1)
229
230	cmpdi	cr7, r5, 0
231
232	/* Turn on TM/FP/VSX/VMX so we can restore them. */
233	mfmsr	r5
234	mr	r10, r5
235	li	r6, MSR_TM >> 32
236	sldi	r6, r6, 32
237	or	r5, r5, r6
238	ori	r5, r5, MSR_FP
239	oris	r5, r5, (MSR_VEC | MSR_VSX)@h
240	mtmsrd	r5
241
242	/*
243	 * The user may change these outside of a transaction, so they must
244	 * always be context switched.
245	 */
246	ld	r5, VCPU_TFHAR(r3)
247	ld	r6, VCPU_TFIAR(r3)
248	ld	r7, VCPU_TEXASR(r3)
249	mtspr	SPRN_TFHAR, r5
250	mtspr	SPRN_TFIAR, r6
251	mtspr	SPRN_TEXASR, r7
252
253	mr	r5, r4
254	rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
255	beq	9f		/* TM not active in guest */
256
257	/* Make sure the failure summary is set, otherwise we'll program check
258	 * when we trechkpt.  It's possible that this might have been not set
259	 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
260	 * host.
261	 */
262	oris	r7, r7, (TEXASR_FS)@h
263	mtspr	SPRN_TEXASR, r7
264
265	/*
266	 * Make a stack frame and save non-volatile registers if requested.
267	 */
268	stdu	r1, -SWITCH_FRAME_SIZE(r1)
269	std	r1, HSTATE_SCRATCH2(r13)
270
271	mfcr	r6
272	mfspr	r7, SPRN_DSCR
273	SAVE_GPR(2, r1)
274	SAVE_GPR(6, r1)
275	SAVE_GPR(7, r1)
276
277	beq	cr7, 4f
278	SAVE_NVGPRS(r1)
279
280	/* MSR[TS] will be 1 (suspended) once we do trechkpt */
281	li	r0, 1
282	rldimi	r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
283	SAVE_GPR(10, r1)	/* final MSR value */
2844:
285	/*
286	 * We need to load up the checkpointed state for the guest.
287	 * We need to do this early as it will blow away any GPRs, VSRs and
288	 * some SPRs.
289	 */
290
291	mr	r31, r3
292	addi	r3, r31, VCPU_FPRS_TM
293	bl	load_fp_state
294	addi	r3, r31, VCPU_VRS_TM
295	bl	load_vr_state
296	mr	r3, r31
297	lwz	r7, VCPU_VRSAVE_TM(r3)
298	mtspr	SPRN_VRSAVE, r7
299
300	ld	r5, VCPU_LR_TM(r3)
301	lwz	r6, VCPU_CR_TM(r3)
302	ld	r7, VCPU_CTR_TM(r3)
303	ld	r8, VCPU_AMR_TM(r3)
304	ld	r9, VCPU_TAR_TM(r3)
305	ld	r10, VCPU_XER_TM(r3)
306	mtlr	r5
307	mtcr	r6
308	mtctr	r7
309	mtspr	SPRN_AMR, r8
310	mtspr	SPRN_TAR, r9
311	mtxer	r10
312
313	/*
314	 * Load up PPR and DSCR values but don't put them in the actual SPRs
315	 * till the last moment to avoid running with userspace PPR and DSCR for
316	 * too long.
317	 */
318	ld	r29, VCPU_DSCR_TM(r3)
319	ld	r30, VCPU_PPR_TM(r3)
320
321	/* Clear the MSR RI since r1, r13 are all going to be foobar. */
322	li	r5, 0
323	mtmsrd	r5, 1
324
325	/* Load GPRs r0-r28 */
326	reg = 0
327	.rept	29
328	ld	reg, VCPU_GPRS_TM(reg)(r31)
329	reg = reg + 1
330	.endr
331
332	mtspr	SPRN_DSCR, r29
333	mtspr	SPRN_PPR, r30
334
335	/* Load final GPRs */
336	ld	29, VCPU_GPRS_TM(29)(r31)
337	ld	30, VCPU_GPRS_TM(30)(r31)
338	ld	31, VCPU_GPRS_TM(31)(r31)
339
340	/* TM checkpointed state is now setup.  All GPRs are now volatile. */
341	TRECHKPT
342
343	/* Now let's get back the state we need. */
344	HMT_MEDIUM
345	GET_PACA(r13)
346	ld	r1, HSTATE_SCRATCH2(r13)
347	REST_GPR(7, r1)
348	mtspr	SPRN_DSCR, r7
349
350	/* Set the MSR RI since we have our registers back. */
351	li	r5, MSR_RI
352	mtmsrd	r5, 1
353
354	/* Restore TOC pointer and CR */
355	REST_GPR(2, r1)
356	REST_GPR(6, r1)
357	mtcr	r6
358
359	/* Restore non-volatile registers if requested to. */
360	beq	cr7, 5f
361	REST_GPR(10, r1)
362	REST_NVGPRS(r1)
363
3645:	addi	r1, r1, SWITCH_FRAME_SIZE
365	ld	r0, PPC_LR_STKOFF(r1)
366	mtlr	r0
367
3689:	/* Restore MSR bits if requested */
369	beqlr	cr7
370	mtmsrd	r10, 0
371	blr
372
373/*
374 * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it
375 * can be invoked from C function by PR KVM only.
376 */
377_GLOBAL(_kvmppc_restore_tm_pr)
378	mflr	r0
379	std	r0, PPC_LR_STKOFF(r1)
380	stdu    r1, -PPC_MIN_STKFRM(r1)
381
382	/* save TAR so that it can be recovered later */
383	mfspr   r8, SPRN_TAR
384	std	r8, PPC_MIN_STKFRM-8(r1)
385
386	li	r5, 1
387	bl	__kvmppc_restore_tm
388
389	ld	r8, PPC_MIN_STKFRM-8(r1)
390	mtspr   SPRN_TAR, r8
391
392	addi    r1, r1, PPC_MIN_STKFRM
393	ld	r0, PPC_LR_STKOFF(r1)
394	mtlr	r0
395	blr
396
397EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr);
398#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
399