1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * 4 * Derived from book3s_hv_rmhandlers.S, which is: 5 * 6 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 7 */ 8 9#include <asm/reg.h> 10#include <asm/ppc_asm.h> 11#include <asm/asm-offsets.h> 12#include <asm/export.h> 13#include <asm/tm.h> 14#include <asm/cputable.h> 15 16#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 17#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) 18 19/* 20 * Save transactional state and TM-related registers. 21 * Called with: 22 * - r3 pointing to the vcpu struct 23 * - r4 containing the MSR with current TS bits: 24 * (For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR). 25 * - r5 containing a flag indicating that non-volatile registers 26 * must be preserved. 27 * If r5 == 0, this can modify all checkpointed registers, but 28 * restores r1, r2 before exit. If r5 != 0, this restores the 29 * MSR TM/FP/VEC/VSX bits to their state on entry. 30 */ 31_GLOBAL(__kvmppc_save_tm) 32 mflr r0 33 std r0, PPC_LR_STKOFF(r1) 34 stdu r1, -SWITCH_FRAME_SIZE(r1) 35 36 mr r9, r3 37 cmpdi cr7, r5, 0 38 39 /* Turn on TM. */ 40 mfmsr r8 41 mr r10, r8 42 li r0, 1 43 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 44 ori r8, r8, MSR_FP 45 oris r8, r8, (MSR_VEC | MSR_VSX)@h 46 mtmsrd r8 47 48 rldicl. r4, r4, 64 - MSR_TS_S_LG, 62 49 beq 1f /* TM not active in guest. */ 50 51 std r1, HSTATE_SCRATCH2(r13) 52 std r3, HSTATE_SCRATCH1(r13) 53 54 /* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */ 55 mfcr r6 56 SAVE_GPR(6, r1) 57 58 /* Save DSCR so we can restore it to avoid running with user value */ 59 mfspr r7, SPRN_DSCR 60 SAVE_GPR(7, r1) 61 62 /* 63 * We are going to do treclaim., which will modify all checkpointed 64 * registers. Save the non-volatile registers on the stack if 65 * preservation of non-volatile state has been requested. 66 */ 67 beq cr7, 3f 68 SAVE_NVGPRS(r1) 69 70 /* MSR[TS] will be 0 (non-transactional) once we do treclaim. */ 71 li r0, 0 72 rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 73 SAVE_GPR(10, r1) /* final MSR value */ 743: 75#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 76BEGIN_FTR_SECTION 77 /* Emulation of the treclaim instruction needs TEXASR before treclaim */ 78 mfspr r6, SPRN_TEXASR 79 std r6, VCPU_ORIG_TEXASR(r3) 80END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST) 81#endif 82 83 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 84 li r5, 0 85 mtmsrd r5, 1 86 87 li r3, TM_CAUSE_KVM_RESCHED 88 89 /* All GPRs are volatile at this point. */ 90 TRECLAIM(R3) 91 92 /* Temporarily store r13 and r9 so we have some regs to play with */ 93 SET_SCRATCH0(r13) 94 GET_PACA(r13) 95 std r9, PACATMSCRATCH(r13) 96 ld r9, HSTATE_SCRATCH1(r13) 97 98 /* Save away PPR soon so we don't run with user value. */ 99 std r0, VCPU_GPRS_TM(0)(r9) 100 mfspr r0, SPRN_PPR 101 HMT_MEDIUM 102 103 /* Reload stack pointer. */ 104 std r1, VCPU_GPRS_TM(1)(r9) 105 ld r1, HSTATE_SCRATCH2(r13) 106 107 /* Set MSR RI now we have r1 and r13 back. */ 108 std r2, VCPU_GPRS_TM(2)(r9) 109 li r2, MSR_RI 110 mtmsrd r2, 1 111 112 /* Reload TOC pointer. */ 113 LOAD_PACA_TOC() 114 115 /* Save all but r0-r2, r9 & r13 */ 116 reg = 3 117 .rept 29 118 .if (reg != 9) && (reg != 13) 119 std reg, VCPU_GPRS_TM(reg)(r9) 120 .endif 121 reg = reg + 1 122 .endr 123 /* ... now save r13 */ 124 GET_SCRATCH0(r4) 125 std r4, VCPU_GPRS_TM(13)(r9) 126 /* ... and save r9 */ 127 ld r4, PACATMSCRATCH(r13) 128 std r4, VCPU_GPRS_TM(9)(r9) 129 130 /* Restore host DSCR and CR values, after saving guest values */ 131 mfcr r6 132 mfspr r7, SPRN_DSCR 133 stw r6, VCPU_CR_TM(r9) 134 std r7, VCPU_DSCR_TM(r9) 135 REST_GPR(6, r1) 136 REST_GPR(7, r1) 137 mtcr r6 138 mtspr SPRN_DSCR, r7 139 140 /* Save away checkpointed SPRs. */ 141 std r0, VCPU_PPR_TM(r9) 142 mflr r5 143 mfctr r7 144 mfspr r8, SPRN_AMR 145 mfspr r10, SPRN_TAR 146 mfxer r11 147 std r5, VCPU_LR_TM(r9) 148 std r7, VCPU_CTR_TM(r9) 149 std r8, VCPU_AMR_TM(r9) 150 std r10, VCPU_TAR_TM(r9) 151 std r11, VCPU_XER_TM(r9) 152 153 /* Save FP/VSX. */ 154 addi r3, r9, VCPU_FPRS_TM 155 bl store_fp_state 156 addi r3, r9, VCPU_VRS_TM 157 bl store_vr_state 158 mfspr r6, SPRN_VRSAVE 159 stw r6, VCPU_VRSAVE_TM(r9) 160 161 /* Restore non-volatile registers if requested to */ 162 beq cr7, 1f 163 REST_NVGPRS(r1) 164 REST_GPR(10, r1) 1651: 166 /* 167 * We need to save these SPRs after the treclaim so that the software 168 * error code is recorded correctly in the TEXASR. Also the user may 169 * change these outside of a transaction, so they must always be 170 * context switched. 171 */ 172 mfspr r7, SPRN_TEXASR 173 std r7, VCPU_TEXASR(r9) 174 mfspr r5, SPRN_TFHAR 175 mfspr r6, SPRN_TFIAR 176 std r5, VCPU_TFHAR(r9) 177 std r6, VCPU_TFIAR(r9) 178 179 /* Restore MSR state if requested */ 180 beq cr7, 2f 181 mtmsrd r10, 0 1822: 183 addi r1, r1, SWITCH_FRAME_SIZE 184 ld r0, PPC_LR_STKOFF(r1) 185 mtlr r0 186 blr 187 188/* 189 * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can 190 * be invoked from C function by PR KVM only. 191 */ 192_GLOBAL(_kvmppc_save_tm_pr) 193 mflr r0 194 std r0, PPC_LR_STKOFF(r1) 195 stdu r1, -PPC_MIN_STKFRM(r1) 196 197 mfspr r8, SPRN_TAR 198 std r8, PPC_MIN_STKFRM-8(r1) 199 200 li r5, 1 /* preserve non-volatile registers */ 201 bl __kvmppc_save_tm 202 203 ld r8, PPC_MIN_STKFRM-8(r1) 204 mtspr SPRN_TAR, r8 205 206 addi r1, r1, PPC_MIN_STKFRM 207 ld r0, PPC_LR_STKOFF(r1) 208 mtlr r0 209 blr 210 211EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr); 212 213/* 214 * Restore transactional state and TM-related registers. 215 * Called with: 216 * - r3 pointing to the vcpu struct. 217 * - r4 is the guest MSR with desired TS bits: 218 * For HV KVM, it is VCPU_MSR 219 * For PR KVM, it is provided by caller 220 * - r5 containing a flag indicating that non-volatile registers 221 * must be preserved. 222 * If r5 == 0, this potentially modifies all checkpointed registers, but 223 * restores r1, r2 from the PACA before exit. 224 * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry. 225 */ 226_GLOBAL(__kvmppc_restore_tm) 227 mflr r0 228 std r0, PPC_LR_STKOFF(r1) 229 230 cmpdi cr7, r5, 0 231 232 /* Turn on TM/FP/VSX/VMX so we can restore them. */ 233 mfmsr r5 234 mr r10, r5 235 li r6, MSR_TM >> 32 236 sldi r6, r6, 32 237 or r5, r5, r6 238 ori r5, r5, MSR_FP 239 oris r5, r5, (MSR_VEC | MSR_VSX)@h 240 mtmsrd r5 241 242 /* 243 * The user may change these outside of a transaction, so they must 244 * always be context switched. 245 */ 246 ld r5, VCPU_TFHAR(r3) 247 ld r6, VCPU_TFIAR(r3) 248 ld r7, VCPU_TEXASR(r3) 249 mtspr SPRN_TFHAR, r5 250 mtspr SPRN_TFIAR, r6 251 mtspr SPRN_TEXASR, r7 252 253 mr r5, r4 254 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 255 beq 9f /* TM not active in guest */ 256 257 /* Make sure the failure summary is set, otherwise we'll program check 258 * when we trechkpt. It's possible that this might have been not set 259 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the 260 * host. 261 */ 262 oris r7, r7, (TEXASR_FS)@h 263 mtspr SPRN_TEXASR, r7 264 265 /* 266 * Make a stack frame and save non-volatile registers if requested. 267 */ 268 stdu r1, -SWITCH_FRAME_SIZE(r1) 269 std r1, HSTATE_SCRATCH2(r13) 270 271 mfcr r6 272 mfspr r7, SPRN_DSCR 273 SAVE_GPR(2, r1) 274 SAVE_GPR(6, r1) 275 SAVE_GPR(7, r1) 276 277 beq cr7, 4f 278 SAVE_NVGPRS(r1) 279 280 /* MSR[TS] will be 1 (suspended) once we do trechkpt */ 281 li r0, 1 282 rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 283 SAVE_GPR(10, r1) /* final MSR value */ 2844: 285 /* 286 * We need to load up the checkpointed state for the guest. 287 * We need to do this early as it will blow away any GPRs, VSRs and 288 * some SPRs. 289 */ 290 291 mr r31, r3 292 addi r3, r31, VCPU_FPRS_TM 293 bl load_fp_state 294 addi r3, r31, VCPU_VRS_TM 295 bl load_vr_state 296 mr r3, r31 297 lwz r7, VCPU_VRSAVE_TM(r3) 298 mtspr SPRN_VRSAVE, r7 299 300 ld r5, VCPU_LR_TM(r3) 301 lwz r6, VCPU_CR_TM(r3) 302 ld r7, VCPU_CTR_TM(r3) 303 ld r8, VCPU_AMR_TM(r3) 304 ld r9, VCPU_TAR_TM(r3) 305 ld r10, VCPU_XER_TM(r3) 306 mtlr r5 307 mtcr r6 308 mtctr r7 309 mtspr SPRN_AMR, r8 310 mtspr SPRN_TAR, r9 311 mtxer r10 312 313 /* 314 * Load up PPR and DSCR values but don't put them in the actual SPRs 315 * till the last moment to avoid running with userspace PPR and DSCR for 316 * too long. 317 */ 318 ld r29, VCPU_DSCR_TM(r3) 319 ld r30, VCPU_PPR_TM(r3) 320 321 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 322 li r5, 0 323 mtmsrd r5, 1 324 325 /* Load GPRs r0-r28 */ 326 reg = 0 327 .rept 29 328 ld reg, VCPU_GPRS_TM(reg)(r31) 329 reg = reg + 1 330 .endr 331 332 mtspr SPRN_DSCR, r29 333 mtspr SPRN_PPR, r30 334 335 /* Load final GPRs */ 336 ld 29, VCPU_GPRS_TM(29)(r31) 337 ld 30, VCPU_GPRS_TM(30)(r31) 338 ld 31, VCPU_GPRS_TM(31)(r31) 339 340 /* TM checkpointed state is now setup. All GPRs are now volatile. */ 341 TRECHKPT 342 343 /* Now let's get back the state we need. */ 344 HMT_MEDIUM 345 GET_PACA(r13) 346 ld r1, HSTATE_SCRATCH2(r13) 347 REST_GPR(7, r1) 348 mtspr SPRN_DSCR, r7 349 350 /* Set the MSR RI since we have our registers back. */ 351 li r5, MSR_RI 352 mtmsrd r5, 1 353 354 /* Restore TOC pointer and CR */ 355 REST_GPR(2, r1) 356 REST_GPR(6, r1) 357 mtcr r6 358 359 /* Restore non-volatile registers if requested to. */ 360 beq cr7, 5f 361 REST_GPR(10, r1) 362 REST_NVGPRS(r1) 363 3645: addi r1, r1, SWITCH_FRAME_SIZE 365 ld r0, PPC_LR_STKOFF(r1) 366 mtlr r0 367 3689: /* Restore MSR bits if requested */ 369 beqlr cr7 370 mtmsrd r10, 0 371 blr 372 373/* 374 * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it 375 * can be invoked from C function by PR KVM only. 376 */ 377_GLOBAL(_kvmppc_restore_tm_pr) 378 mflr r0 379 std r0, PPC_LR_STKOFF(r1) 380 stdu r1, -PPC_MIN_STKFRM(r1) 381 382 /* save TAR so that it can be recovered later */ 383 mfspr r8, SPRN_TAR 384 std r8, PPC_MIN_STKFRM-8(r1) 385 386 li r5, 1 387 bl __kvmppc_restore_tm 388 389 ld r8, PPC_MIN_STKFRM-8(r1) 390 mtspr SPRN_TAR, r8 391 392 addi r1, r1, PPC_MIN_STKFRM 393 ld r0, PPC_LR_STKOFF(r1) 394 mtlr r0 395 blr 396 397EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr); 398#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 399