1/* SPDX-License-Identifier: GPL-2.0-only */ 2#include <linux/export.h> 3#include <asm/asm-offsets.h> 4#include <asm/cache.h> 5#include <asm/code-patching-asm.h> 6#include <asm/exception-64s.h> 7#include <asm/kvm_asm.h> 8#include <asm/kvm_book3s_asm.h> 9#include <asm/mmu.h> 10#include <asm/ppc_asm.h> 11#include <asm/ptrace.h> 12#include <asm/reg.h> 13#include <asm/ultravisor-api.h> 14 15/* 16 * These are branched to from interrupt handlers in exception-64s.S which set 17 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero. 18 */ 19 20/* 21 * This is a hcall, so register convention is as 22 * Documentation/powerpc/papr_hcalls.rst. 23 * 24 * This may also be a syscall from PR-KVM userspace that is to be 25 * reflected to the PR guest kernel, so registers may be set up for 26 * a system call rather than hcall. We don't currently clobber 27 * anything here, but the 0xc00 handler has already clobbered CTR 28 * and CR0, so PR-KVM can not support a guest kernel that preserves 29 * those registers across its system calls. 30 * 31 * The state of registers is as kvmppc_interrupt, except CFAR is not 32 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap. 33 */ 34.global kvmppc_hcall 35.balign IFETCH_ALIGN_BYTES 36kvmppc_hcall: 37#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 38 lbz r10,HSTATE_IN_GUEST(r13) 39 cmpwi r10,KVM_GUEST_MODE_HV_P9 40 beq kvmppc_p9_exit_hcall 41#endif 42 ld r10,PACA_EXGEN+EX_R13(r13) 43 SET_SCRATCH0(r10) 44 li r10,0xc00 45 /* Now we look like kvmppc_interrupt */ 46 li r11,PACA_EXGEN 47 b .Lgot_save_area 48 49/* 50 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that 51 * call convention: 52 * 53 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area 54 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts 55 * guest R13 also saved in SCRATCH0 56 * R13 = PACA 57 * R11 = (H)SRR0 58 * R12 = (H)SRR1 59 * R9 = guest CR 60 * PPR is set to medium 61 * 62 * With the addition for KVM: 63 * R10 = trap vector 64 */ 65.global kvmppc_interrupt 66.balign IFETCH_ALIGN_BYTES 67kvmppc_interrupt: 68#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 69 std r10,HSTATE_SCRATCH0(r13) 70 lbz r10,HSTATE_IN_GUEST(r13) 71 cmpwi r10,KVM_GUEST_MODE_HV_P9 72 beq kvmppc_p9_exit_interrupt 73 ld r10,HSTATE_SCRATCH0(r13) 74#endif 75 li r11,PACA_EXGEN 76 cmpdi r10,0x200 77 bgt+ .Lgot_save_area 78 li r11,PACA_EXMC 79 beq .Lgot_save_area 80 li r11,PACA_EXNMI 81.Lgot_save_area: 82 add r11,r11,r13 83BEGIN_FTR_SECTION 84 ld r12,EX_CFAR(r11) 85 std r12,HSTATE_CFAR(r13) 86END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 87 ld r12,EX_CTR(r11) 88 mtctr r12 89BEGIN_FTR_SECTION 90 ld r12,EX_PPR(r11) 91 std r12,HSTATE_PPR(r13) 92END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 93 ld r12,EX_R12(r11) 94 std r12,HSTATE_SCRATCH0(r13) 95 sldi r12,r9,32 96 or r12,r12,r10 97 ld r9,EX_R9(r11) 98 ld r10,EX_R10(r11) 99 ld r11,EX_R11(r11) 100 101 /* 102 * Hcalls and other interrupts come here after normalising register 103 * contents and save locations: 104 * 105 * R12 = (guest CR << 32) | interrupt vector 106 * R13 = PACA 107 * guest R12 saved in shadow HSTATE_SCRATCH0 108 * guest R13 saved in SPRN_SCRATCH0 109 */ 110 std r9,HSTATE_SCRATCH2(r13) 111 lbz r9,HSTATE_IN_GUEST(r13) 112 cmpwi r9,KVM_GUEST_MODE_SKIP 113 beq- .Lmaybe_skip 114.Lno_skip: 115#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 116#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 117 cmpwi r9,KVM_GUEST_MODE_GUEST 118 beq kvmppc_interrupt_pr 119#endif 120 b kvmppc_interrupt_hv 121#else 122 b kvmppc_interrupt_pr 123#endif 124 125/* 126 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load 127 * the faulting instruction in guest memory from the hypervisor without 128 * walking page tables. 129 * 130 * When the guest takes a fault that requires the hypervisor to load the 131 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1 132 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets 133 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions 134 * but loads and stores will access the guest context. This is used to load 135 * the faulting instruction using the faulting guest effective address. 136 * 137 * However the guest context may not be able to translate, or it may cause a 138 * machine check or other issue, which results in a fault in the host 139 * (even with KVM-HV). 140 * 141 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they 142 * are (or are likely) caused by that load, the instruction is skipped by 143 * just returning with the PC advanced +4, where it is noticed the load did 144 * not execute and it goes to the slow path which walks the page tables to 145 * read guest memory. 146 */ 147.Lmaybe_skip: 148 cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK 149 beq 1f 150 cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE 151 beq 1f 152 cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT 153 beq 1f 154#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 155 /* HSRR interrupts get 2 added to interrupt number */ 156 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2 157 beq 2f 158#endif 159 b .Lno_skip 1601: mfspr r9,SPRN_SRR0 161 addi r9,r9,4 162 mtspr SPRN_SRR0,r9 163 ld r12,HSTATE_SCRATCH0(r13) 164 ld r9,HSTATE_SCRATCH2(r13) 165 GET_SCRATCH0(r13) 166 RFI_TO_KERNEL 167#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 1682: mfspr r9,SPRN_HSRR0 169 addi r9,r9,4 170 mtspr SPRN_HSRR0,r9 171 ld r12,HSTATE_SCRATCH0(r13) 172 ld r9,HSTATE_SCRATCH2(r13) 173 GET_SCRATCH0(r13) 174 HRFI_TO_KERNEL 175#endif 176 177#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 178 179/* Stack frame offsets for kvmppc_p9_enter_guest */ 180#define SFS (144 + STACK_FRAME_MIN_SIZE) 181#define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */ 182 183/* 184 * void kvmppc_p9_enter_guest(struct vcpu *vcpu); 185 * 186 * Enter the guest on a ISAv3.0 or later system. 187 */ 188.balign IFETCH_ALIGN_BYTES 189_GLOBAL(kvmppc_p9_enter_guest) 190EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest) 191 mflr r0 192 std r0,PPC_LR_STKOFF(r1) 193 stdu r1,-SFS(r1) 194 195 std r1,HSTATE_HOST_R1(r13) 196 197 mfcr r4 198 stw r4,SFS+8(r1) 199 200 reg = 14 201 .rept 18 202 std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 203 reg = reg + 1 204 .endr 205 206 ld r4,VCPU_LR(r3) 207 mtlr r4 208 ld r4,VCPU_CTR(r3) 209 mtctr r4 210 ld r4,VCPU_XER(r3) 211 mtspr SPRN_XER,r4 212 213 ld r1,VCPU_CR(r3) 214 215BEGIN_FTR_SECTION 216 ld r4,VCPU_CFAR(r3) 217 mtspr SPRN_CFAR,r4 218END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 219BEGIN_FTR_SECTION 220 ld r4,VCPU_PPR(r3) 221 mtspr SPRN_PPR,r4 222END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 223 224 reg = 4 225 .rept 28 226 ld reg,__VCPU_GPR(reg)(r3) 227 reg = reg + 1 228 .endr 229 230 ld r4,VCPU_KVM(r3) 231 lbz r4,KVM_SECURE_GUEST(r4) 232 cmpdi r4,0 233 ld r4,VCPU_GPR(R4)(r3) 234 bne .Lret_to_ultra 235 236 mtcr r1 237 238 ld r0,VCPU_GPR(R0)(r3) 239 ld r1,VCPU_GPR(R1)(r3) 240 ld r2,VCPU_GPR(R2)(r3) 241 ld r3,VCPU_GPR(R3)(r3) 242 243 HRFI_TO_GUEST 244 b . 245 246 /* 247 * Use UV_RETURN ultracall to return control back to the Ultravisor 248 * after processing an hypercall or interrupt that was forwarded 249 * (a.k.a. reflected) to the Hypervisor. 250 * 251 * All registers have already been reloaded except the ucall requires: 252 * R0 = hcall result 253 * R2 = SRR1, so UV can detect a synthesized interrupt (if any) 254 * R3 = UV_RETURN 255 */ 256.Lret_to_ultra: 257 mtcr r1 258 ld r1,VCPU_GPR(R1)(r3) 259 260 ld r0,VCPU_GPR(R3)(r3) 261 mfspr r2,SPRN_SRR1 262 LOAD_REG_IMMEDIATE(r3, UV_RETURN) 263 sc 2 264 265/* 266 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from 267 * above if the interrupt was taken for a guest that was entered via 268 * kvmppc_p9_enter_guest(). 269 * 270 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs 271 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re- 272 * establishes the host stack and registers to return from the 273 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers 274 * (SPRs and FP, VEC, etc). 275 */ 276.balign IFETCH_ALIGN_BYTES 277kvmppc_p9_exit_hcall: 278 mfspr r11,SPRN_SRR0 279 mfspr r12,SPRN_SRR1 280 li r10,0xc00 281 std r10,HSTATE_SCRATCH0(r13) 282 283.balign IFETCH_ALIGN_BYTES 284kvmppc_p9_exit_interrupt: 285 /* 286 * If set to KVM_GUEST_MODE_HV_P9 but we're still in the 287 * hypervisor, that means we can't return from the entry stack. 288 */ 289 rldicl. r10,r12,64-MSR_HV_LG,63 290 bne- kvmppc_p9_bad_interrupt 291 292 std r1,HSTATE_SCRATCH1(r13) 293 std r3,HSTATE_SCRATCH2(r13) 294 ld r1,HSTATE_HOST_R1(r13) 295 ld r3,HSTATE_KVM_VCPU(r13) 296 297 std r9,VCPU_CR(r3) 298 2991: 300 std r11,VCPU_PC(r3) 301 std r12,VCPU_MSR(r3) 302 303 reg = 14 304 .rept 18 305 std reg,__VCPU_GPR(reg)(r3) 306 reg = reg + 1 307 .endr 308 309 /* r1, r3, r9-r13 are saved to vcpu by C code */ 310 std r0,VCPU_GPR(R0)(r3) 311 std r2,VCPU_GPR(R2)(r3) 312 reg = 4 313 .rept 5 314 std reg,__VCPU_GPR(reg)(r3) 315 reg = reg + 1 316 .endr 317 318 LOAD_PACA_TOC() 319 320 mflr r4 321 std r4,VCPU_LR(r3) 322 mfspr r4,SPRN_XER 323 std r4,VCPU_XER(r3) 324 325 reg = 14 326 .rept 18 327 ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 328 reg = reg + 1 329 .endr 330 331 lwz r4,SFS+8(r1) 332 mtcr r4 333 334 /* 335 * Flush the link stack here, before executing the first blr on the 336 * way out of the guest. 337 * 338 * The link stack won't match coming out of the guest anyway so the 339 * only cost is the flush itself. The call clobbers r0. 340 */ 3411: nop 342 patch_site 1b patch__call_kvm_flush_link_stack_p9 343 344 addi r1,r1,SFS 345 ld r0,PPC_LR_STKOFF(r1) 346 mtlr r0 347 blr 348 349/* 350 * Took an interrupt somewhere right before HRFID to guest, so registers are 351 * in a bad way. Return things hopefully enough to run host virtual code and 352 * run the Linux interrupt handler (SRESET or MCE) to print something useful. 353 * 354 * We could be really clever and save all host registers in known locations 355 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting 356 * return address to a fixup that sets them up again. But that's a lot of 357 * effort for a small bit of code. Lots of other things to do first. 358 */ 359kvmppc_p9_bad_interrupt: 360BEGIN_MMU_FTR_SECTION 361 /* 362 * Hash host doesn't try to recover MMU (requires host SLB reload) 363 */ 364 b . 365END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) 366 /* 367 * Clean up guest registers to give host a chance to run. 368 */ 369 li r10,0 370 mtspr SPRN_AMR,r10 371 mtspr SPRN_IAMR,r10 372 mtspr SPRN_CIABR,r10 373 mtspr SPRN_DAWRX0,r10 374BEGIN_FTR_SECTION 375 mtspr SPRN_DAWRX1,r10 376END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) 377 378 /* 379 * Switch to host MMU mode (don't have the real host PID but we aren't 380 * going back to userspace). 381 */ 382 hwsync 383 isync 384 385 mtspr SPRN_PID,r10 386 387 ld r10, HSTATE_KVM_VCPU(r13) 388 ld r10, VCPU_KVM(r10) 389 lwz r10, KVM_HOST_LPID(r10) 390 mtspr SPRN_LPID,r10 391 392 ld r10, HSTATE_KVM_VCPU(r13) 393 ld r10, VCPU_KVM(r10) 394 ld r10, KVM_HOST_LPCR(r10) 395 mtspr SPRN_LPCR,r10 396 397 isync 398 399 /* 400 * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear 401 * MSR_RI in r12 ([H]SRR1) so the handler won't try to return. 402 */ 403 li r10,KVM_GUEST_MODE_NONE 404 stb r10,HSTATE_IN_GUEST(r13) 405 li r10,MSR_RI 406 andc r12,r12,r10 407 408 /* 409 * Go back to interrupt handler. MCE and SRESET have their specific 410 * PACA save area so they should be used directly. They set up their 411 * own stack. The other handlers all use EXGEN. They will use the 412 * guest r1 if it looks like a kernel stack, so just load the 413 * emergency stack and go to program check for all other interrupts. 414 */ 415 ld r10,HSTATE_SCRATCH0(r13) 416 cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK 417 beq .Lcall_machine_check_common 418 419 cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET 420 beq .Lcall_system_reset_common 421 422 b . 423 424.Lcall_machine_check_common: 425 b machine_check_common 426 427.Lcall_system_reset_common: 428 b system_reset_common 429#endif 430