1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/tm.h> 32#include <asm/opal.h> 33#include <asm/xive-regs.h> 34#include <asm/thread_info.h> 35 36/* Sign-extend HDEC if not on POWER9 */ 37#define EXTEND_HDEC(reg) \ 38BEGIN_FTR_SECTION; \ 39 extsw reg, reg; \ 40END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 41 42#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) 43 44/* Values in HSTATE_NAPPING(r13) */ 45#define NAPPING_CEDE 1 46#define NAPPING_NOVCPU 2 47 48/* Stack frame offsets for kvmppc_hv_entry */ 49#define SFS 160 50#define STACK_SLOT_TRAP (SFS-4) 51#define STACK_SLOT_TID (SFS-16) 52#define STACK_SLOT_PSSCR (SFS-24) 53#define STACK_SLOT_PID (SFS-32) 54#define STACK_SLOT_IAMR (SFS-40) 55#define STACK_SLOT_CIABR (SFS-48) 56#define STACK_SLOT_DAWR (SFS-56) 57#define STACK_SLOT_DAWRX (SFS-64) 58#define STACK_SLOT_HFSCR (SFS-72) 59 60/* 61 * Call kvmppc_hv_entry in real mode. 62 * Must be called with interrupts hard-disabled. 63 * 64 * Input Registers: 65 * 66 * LR = return address to continue at after eventually re-enabling MMU 67 */ 68_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 69 mflr r0 70 std r0, PPC_LR_STKOFF(r1) 71 stdu r1, -112(r1) 72 mfmsr r10 73 std r10, HSTATE_HOST_MSR(r13) 74 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 75 li r0,MSR_RI 76 andc r0,r10,r0 77 li r6,MSR_IR | MSR_DR 78 andc r6,r10,r6 79 mtmsrd r0,1 /* clear RI in MSR */ 80 mtsrr0 r5 81 mtsrr1 r6 82 RFI_TO_KERNEL 83 84kvmppc_call_hv_entry: 85BEGIN_FTR_SECTION 86 /* On P9, do LPCR setting, if necessary */ 87 ld r3, HSTATE_SPLIT_MODE(r13) 88 cmpdi r3, 0 89 beq 46f 90 lwz r4, KVM_SPLIT_DO_SET(r3) 91 cmpwi r4, 0 92 beq 46f 93 bl kvmhv_p9_set_lpcr 94 nop 9546: 96END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 97 98 ld r4, HSTATE_KVM_VCPU(r13) 99 bl kvmppc_hv_entry 100 101 /* Back from guest - restore host state and return to caller */ 102 103BEGIN_FTR_SECTION 104 /* Restore host DABR and DABRX */ 105 ld r5,HSTATE_DABR(r13) 106 li r6,7 107 mtspr SPRN_DABR,r5 108 mtspr SPRN_DABRX,r6 109END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 110 111 /* Restore SPRG3 */ 112 ld r3,PACA_SPRG_VDSO(r13) 113 mtspr SPRN_SPRG_VDSO_WRITE,r3 114 115 /* Reload the host's PMU registers */ 116 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */ 117 lbz r4, LPPACA_PMCINUSE(r3) 118 cmpwi r4, 0 119 beq 23f /* skip if not */ 120BEGIN_FTR_SECTION 121 ld r3, HSTATE_MMCR0(r13) 122 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 123 cmpwi r4, MMCR0_PMAO 124 beql kvmppc_fix_pmao 125END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 126 lwz r3, HSTATE_PMC1(r13) 127 lwz r4, HSTATE_PMC2(r13) 128 lwz r5, HSTATE_PMC3(r13) 129 lwz r6, HSTATE_PMC4(r13) 130 lwz r8, HSTATE_PMC5(r13) 131 lwz r9, HSTATE_PMC6(r13) 132 mtspr SPRN_PMC1, r3 133 mtspr SPRN_PMC2, r4 134 mtspr SPRN_PMC3, r5 135 mtspr SPRN_PMC4, r6 136 mtspr SPRN_PMC5, r8 137 mtspr SPRN_PMC6, r9 138 ld r3, HSTATE_MMCR0(r13) 139 ld r4, HSTATE_MMCR1(r13) 140 ld r5, HSTATE_MMCRA(r13) 141 ld r6, HSTATE_SIAR(r13) 142 ld r7, HSTATE_SDAR(r13) 143 mtspr SPRN_MMCR1, r4 144 mtspr SPRN_MMCRA, r5 145 mtspr SPRN_SIAR, r6 146 mtspr SPRN_SDAR, r7 147BEGIN_FTR_SECTION 148 ld r8, HSTATE_MMCR2(r13) 149 ld r9, HSTATE_SIER(r13) 150 mtspr SPRN_MMCR2, r8 151 mtspr SPRN_SIER, r9 152END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 153 mtspr SPRN_MMCR0, r3 154 isync 15523: 156 157 /* 158 * Reload DEC. HDEC interrupts were disabled when 159 * we reloaded the host's LPCR value. 160 */ 161 ld r3, HSTATE_DECEXP(r13) 162 mftb r4 163 subf r4, r4, r3 164 mtspr SPRN_DEC, r4 165 166 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 167 li r0, 0 168 stb r0, HSTATE_HWTHREAD_REQ(r13) 169 170 /* 171 * For external interrupts we need to call the Linux 172 * handler to process the interrupt. We do that by jumping 173 * to absolute address 0x500 for external interrupts. 174 * The [h]rfid at the end of the handler will return to 175 * the book3s_hv_interrupts.S code. For other interrupts 176 * we do the rfid to get back to the book3s_hv_interrupts.S 177 * code here. 178 */ 179 ld r8, 112+PPC_LR_STKOFF(r1) 180 addi r1, r1, 112 181 ld r7, HSTATE_HOST_MSR(r13) 182 183 /* Return the trap number on this thread as the return value */ 184 mr r3, r12 185 186 /* 187 * If we came back from the guest via a relocation-on interrupt, 188 * we will be in virtual mode at this point, which makes it a 189 * little easier to get back to the caller. 190 */ 191 mfmsr r0 192 andi. r0, r0, MSR_IR /* in real mode? */ 193 bne .Lvirt_return 194 195 /* RFI into the highmem handler */ 196 mfmsr r6 197 li r0, MSR_RI 198 andc r6, r6, r0 199 mtmsrd r6, 1 /* Clear RI in MSR */ 200 mtsrr0 r8 201 mtsrr1 r7 202 RFI_TO_KERNEL 203 204 /* Virtual-mode return */ 205.Lvirt_return: 206 mtlr r8 207 blr 208 209kvmppc_primary_no_guest: 210 /* We handle this much like a ceded vcpu */ 211 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 212 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 213 /* HDEC value came from DEC in the first place, it will fit */ 214 mfspr r3, SPRN_HDEC 215 mtspr SPRN_DEC, r3 216 /* 217 * Make sure the primary has finished the MMU switch. 218 * We should never get here on a secondary thread, but 219 * check it for robustness' sake. 220 */ 221 ld r5, HSTATE_KVM_VCORE(r13) 22265: lbz r0, VCORE_IN_GUEST(r5) 223 cmpwi r0, 0 224 beq 65b 225 /* Set LPCR. */ 226 ld r8,VCORE_LPCR(r5) 227 mtspr SPRN_LPCR,r8 228 isync 229 /* set our bit in napping_threads */ 230 ld r5, HSTATE_KVM_VCORE(r13) 231 lbz r7, HSTATE_PTID(r13) 232 li r0, 1 233 sld r0, r0, r7 234 addi r6, r5, VCORE_NAPPING_THREADS 2351: lwarx r3, 0, r6 236 or r3, r3, r0 237 stwcx. r3, 0, r6 238 bne 1b 239 /* order napping_threads update vs testing entry_exit_map */ 240 isync 241 li r12, 0 242 lwz r7, VCORE_ENTRY_EXIT(r5) 243 cmpwi r7, 0x100 244 bge kvm_novcpu_exit /* another thread already exiting */ 245 li r3, NAPPING_NOVCPU 246 stb r3, HSTATE_NAPPING(r13) 247 248 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 249 b kvm_do_nap 250 251/* 252 * kvm_novcpu_wakeup 253 * Entered from kvm_start_guest if kvm_hstate.napping is set 254 * to NAPPING_NOVCPU 255 * r2 = kernel TOC 256 * r13 = paca 257 */ 258kvm_novcpu_wakeup: 259 ld r1, HSTATE_HOST_R1(r13) 260 ld r5, HSTATE_KVM_VCORE(r13) 261 li r0, 0 262 stb r0, HSTATE_NAPPING(r13) 263 264 /* check the wake reason */ 265 bl kvmppc_check_wake_reason 266 267 /* 268 * Restore volatile registers since we could have called 269 * a C routine in kvmppc_check_wake_reason. 270 * r5 = VCORE 271 */ 272 ld r5, HSTATE_KVM_VCORE(r13) 273 274 /* see if any other thread is already exiting */ 275 lwz r0, VCORE_ENTRY_EXIT(r5) 276 cmpwi r0, 0x100 277 bge kvm_novcpu_exit 278 279 /* clear our bit in napping_threads */ 280 lbz r7, HSTATE_PTID(r13) 281 li r0, 1 282 sld r0, r0, r7 283 addi r6, r5, VCORE_NAPPING_THREADS 2844: lwarx r7, 0, r6 285 andc r7, r7, r0 286 stwcx. r7, 0, r6 287 bne 4b 288 289 /* See if the wake reason means we need to exit */ 290 cmpdi r3, 0 291 bge kvm_novcpu_exit 292 293 /* See if our timeslice has expired (HDEC is negative) */ 294 mfspr r0, SPRN_HDEC 295 EXTEND_HDEC(r0) 296 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 297 cmpdi r0, 0 298 blt kvm_novcpu_exit 299 300 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 301 ld r4, HSTATE_KVM_VCPU(r13) 302 cmpdi r4, 0 303 beq kvmppc_primary_no_guest 304 305#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 306 addi r3, r4, VCPU_TB_RMENTRY 307 bl kvmhv_start_timing 308#endif 309 b kvmppc_got_guest 310 311kvm_novcpu_exit: 312#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 313 ld r4, HSTATE_KVM_VCPU(r13) 314 cmpdi r4, 0 315 beq 13f 316 addi r3, r4, VCPU_TB_RMEXIT 317 bl kvmhv_accumulate_time 318#endif 31913: mr r3, r12 320 stw r12, STACK_SLOT_TRAP(r1) 321 bl kvmhv_commence_exit 322 nop 323 lwz r12, STACK_SLOT_TRAP(r1) 324 b kvmhv_switch_to_host 325 326/* 327 * We come in here when wakened from nap mode. 328 * Relocation is off and most register values are lost. 329 * r13 points to the PACA. 330 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 331 */ 332 .globl kvm_start_guest 333kvm_start_guest: 334 /* Set runlatch bit the minute you wake up from nap */ 335 mfspr r0, SPRN_CTRLF 336 ori r0, r0, 1 337 mtspr SPRN_CTRLT, r0 338 339 /* 340 * Could avoid this and pass it through in r3. For now, 341 * code expects it to be in SRR1. 342 */ 343 mtspr SPRN_SRR1,r3 344 345 ld r2,PACATOC(r13) 346 347 li r0,KVM_HWTHREAD_IN_KVM 348 stb r0,HSTATE_HWTHREAD_STATE(r13) 349 350 /* NV GPR values from power7_idle() will no longer be valid */ 351 li r0,1 352 stb r0,PACA_NAPSTATELOST(r13) 353 354 /* were we napping due to cede? */ 355 lbz r0,HSTATE_NAPPING(r13) 356 cmpwi r0,NAPPING_CEDE 357 beq kvm_end_cede 358 cmpwi r0,NAPPING_NOVCPU 359 beq kvm_novcpu_wakeup 360 361 ld r1,PACAEMERGSP(r13) 362 subi r1,r1,STACK_FRAME_OVERHEAD 363 364 /* 365 * We weren't napping due to cede, so this must be a secondary 366 * thread being woken up to run a guest, or being woken up due 367 * to a stray IPI. (Or due to some machine check or hypervisor 368 * maintenance interrupt while the core is in KVM.) 369 */ 370 371 /* Check the wake reason in SRR1 to see why we got here */ 372 bl kvmppc_check_wake_reason 373 /* 374 * kvmppc_check_wake_reason could invoke a C routine, but we 375 * have no volatile registers to restore when we return. 376 */ 377 378 cmpdi r3, 0 379 bge kvm_no_guest 380 381 /* get vcore pointer, NULL if we have nothing to run */ 382 ld r5,HSTATE_KVM_VCORE(r13) 383 cmpdi r5,0 384 /* if we have no vcore to run, go back to sleep */ 385 beq kvm_no_guest 386 387kvm_secondary_got_guest: 388 389 /* Set HSTATE_DSCR(r13) to something sensible */ 390 ld r6, PACA_DSCR_DEFAULT(r13) 391 std r6, HSTATE_DSCR(r13) 392 393 /* On thread 0 of a subcore, set HDEC to max */ 394 lbz r4, HSTATE_PTID(r13) 395 cmpwi r4, 0 396 bne 63f 397 LOAD_REG_ADDR(r6, decrementer_max) 398 ld r6, 0(r6) 399 mtspr SPRN_HDEC, r6 400 /* and set per-LPAR registers, if doing dynamic micro-threading */ 401 ld r6, HSTATE_SPLIT_MODE(r13) 402 cmpdi r6, 0 403 beq 63f 404BEGIN_FTR_SECTION 405 ld r0, KVM_SPLIT_RPR(r6) 406 mtspr SPRN_RPR, r0 407 ld r0, KVM_SPLIT_PMMAR(r6) 408 mtspr SPRN_PMMAR, r0 409 ld r0, KVM_SPLIT_LDBAR(r6) 410 mtspr SPRN_LDBAR, r0 411 isync 412FTR_SECTION_ELSE 413 /* On P9 we use the split_info for coordinating LPCR changes */ 414 lwz r4, KVM_SPLIT_DO_SET(r6) 415 cmpwi r4, 0 416 beq 1f 417 mr r3, r6 418 bl kvmhv_p9_set_lpcr 419 nop 4201: 421ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 42263: 423 /* Order load of vcpu after load of vcore */ 424 lwsync 425 ld r4, HSTATE_KVM_VCPU(r13) 426 bl kvmppc_hv_entry 427 428 /* Back from the guest, go back to nap */ 429 /* Clear our vcpu and vcore pointers so we don't come back in early */ 430 li r0, 0 431 std r0, HSTATE_KVM_VCPU(r13) 432 /* 433 * Once we clear HSTATE_KVM_VCORE(r13), the code in 434 * kvmppc_run_core() is going to assume that all our vcpu 435 * state is visible in memory. This lwsync makes sure 436 * that that is true. 437 */ 438 lwsync 439 std r0, HSTATE_KVM_VCORE(r13) 440 441 /* 442 * All secondaries exiting guest will fall through this path. 443 * Before proceeding, just check for HMI interrupt and 444 * invoke opal hmi handler. By now we are sure that the 445 * primary thread on this core/subcore has already made partition 446 * switch/TB resync and we are good to call opal hmi handler. 447 */ 448 cmpwi r12, BOOK3S_INTERRUPT_HMI 449 bne kvm_no_guest 450 451 li r3,0 /* NULL argument */ 452 bl hmi_exception_realmode 453/* 454 * At this point we have finished executing in the guest. 455 * We need to wait for hwthread_req to become zero, since 456 * we may not turn on the MMU while hwthread_req is non-zero. 457 * While waiting we also need to check if we get given a vcpu to run. 458 */ 459kvm_no_guest: 460 lbz r3, HSTATE_HWTHREAD_REQ(r13) 461 cmpwi r3, 0 462 bne 53f 463 HMT_MEDIUM 464 li r0, KVM_HWTHREAD_IN_KERNEL 465 stb r0, HSTATE_HWTHREAD_STATE(r13) 466 /* need to recheck hwthread_req after a barrier, to avoid race */ 467 sync 468 lbz r3, HSTATE_HWTHREAD_REQ(r13) 469 cmpwi r3, 0 470 bne 54f 471/* 472 * We jump to pnv_wakeup_loss, which will return to the caller 473 * of power7_nap in the powernv cpu offline loop. The value we 474 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 475 * requires SRR1 in r12. 476 */ 477 li r3, LPCR_PECE0 478 mfspr r4, SPRN_LPCR 479 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 480 mtspr SPRN_LPCR, r4 481 li r3, 0 482 mfspr r12,SPRN_SRR1 483 b pnv_wakeup_loss 484 48553: HMT_LOW 486 ld r5, HSTATE_KVM_VCORE(r13) 487 cmpdi r5, 0 488 bne 60f 489 ld r3, HSTATE_SPLIT_MODE(r13) 490 cmpdi r3, 0 491 beq kvm_no_guest 492 lwz r0, KVM_SPLIT_DO_SET(r3) 493 cmpwi r0, 0 494 bne kvmhv_do_set 495 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 496 cmpwi r0, 0 497 bne kvmhv_do_restore 498 lbz r0, KVM_SPLIT_DO_NAP(r3) 499 cmpwi r0, 0 500 beq kvm_no_guest 501 HMT_MEDIUM 502 b kvm_unsplit_nap 50360: HMT_MEDIUM 504 b kvm_secondary_got_guest 505 50654: li r0, KVM_HWTHREAD_IN_KVM 507 stb r0, HSTATE_HWTHREAD_STATE(r13) 508 b kvm_no_guest 509 510kvmhv_do_set: 511 /* Set LPCR, LPIDR etc. on P9 */ 512 HMT_MEDIUM 513 bl kvmhv_p9_set_lpcr 514 nop 515 b kvm_no_guest 516 517kvmhv_do_restore: 518 HMT_MEDIUM 519 bl kvmhv_p9_restore_lpcr 520 nop 521 b kvm_no_guest 522 523/* 524 * Here the primary thread is trying to return the core to 525 * whole-core mode, so we need to nap. 526 */ 527kvm_unsplit_nap: 528 /* 529 * When secondaries are napping in kvm_unsplit_nap() with 530 * hwthread_req = 1, HMI goes ignored even though subcores are 531 * already exited the guest. Hence HMI keeps waking up secondaries 532 * from nap in a loop and secondaries always go back to nap since 533 * no vcore is assigned to them. This makes impossible for primary 534 * thread to get hold of secondary threads resulting into a soft 535 * lockup in KVM path. 536 * 537 * Let us check if HMI is pending and handle it before we go to nap. 538 */ 539 cmpwi r12, BOOK3S_INTERRUPT_HMI 540 bne 55f 541 li r3, 0 /* NULL argument */ 542 bl hmi_exception_realmode 54355: 544 /* 545 * Ensure that secondary doesn't nap when it has 546 * its vcore pointer set. 547 */ 548 sync /* matches smp_mb() before setting split_info.do_nap */ 549 ld r0, HSTATE_KVM_VCORE(r13) 550 cmpdi r0, 0 551 bne kvm_no_guest 552 /* clear any pending message */ 553BEGIN_FTR_SECTION 554 lis r6, (PPC_DBELL_SERVER << (63-36))@h 555 PPC_MSGCLR(6) 556END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 557 /* Set kvm_split_mode.napped[tid] = 1 */ 558 ld r3, HSTATE_SPLIT_MODE(r13) 559 li r0, 1 560 lbz r4, HSTATE_TID(r13) 561 addi r4, r4, KVM_SPLIT_NAPPED 562 stbx r0, r3, r4 563 /* Check the do_nap flag again after setting napped[] */ 564 sync 565 lbz r0, KVM_SPLIT_DO_NAP(r3) 566 cmpwi r0, 0 567 beq 57f 568 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 569 mfspr r5, SPRN_LPCR 570 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 571 b kvm_nap_sequence 572 57357: li r0, 0 574 stbx r0, r3, r4 575 b kvm_no_guest 576 577/****************************************************************************** 578 * * 579 * Entry code * 580 * * 581 *****************************************************************************/ 582 583.global kvmppc_hv_entry 584kvmppc_hv_entry: 585 586 /* Required state: 587 * 588 * R4 = vcpu pointer (or NULL) 589 * MSR = ~IR|DR 590 * R13 = PACA 591 * R1 = host R1 592 * R2 = TOC 593 * all other volatile GPRS = free 594 * Does not preserve non-volatile GPRs or CR fields 595 */ 596 mflr r0 597 std r0, PPC_LR_STKOFF(r1) 598 stdu r1, -SFS(r1) 599 600 /* Save R1 in the PACA */ 601 std r1, HSTATE_HOST_R1(r13) 602 603 li r6, KVM_GUEST_MODE_HOST_HV 604 stb r6, HSTATE_IN_GUEST(r13) 605 606#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 607 /* Store initial timestamp */ 608 cmpdi r4, 0 609 beq 1f 610 addi r3, r4, VCPU_TB_RMENTRY 611 bl kvmhv_start_timing 6121: 613#endif 614 615 /* Use cr7 as an indication of radix mode */ 616 ld r5, HSTATE_KVM_VCORE(r13) 617 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 618 lbz r0, KVM_RADIX(r9) 619 cmpwi cr7, r0, 0 620 621 /* 622 * POWER7/POWER8 host -> guest partition switch code. 623 * We don't have to lock against concurrent tlbies, 624 * but we do have to coordinate across hardware threads. 625 */ 626 /* Set bit in entry map iff exit map is zero. */ 627 li r7, 1 628 lbz r6, HSTATE_PTID(r13) 629 sld r7, r7, r6 630 addi r8, r5, VCORE_ENTRY_EXIT 63121: lwarx r3, 0, r8 632 cmpwi r3, 0x100 /* any threads starting to exit? */ 633 bge secondary_too_late /* if so we're too late to the party */ 634 or r3, r3, r7 635 stwcx. r3, 0, r8 636 bne 21b 637 638 /* Primary thread switches to guest partition. */ 639 cmpwi r6,0 640 bne 10f 641 lwz r7,KVM_LPID(r9) 642BEGIN_FTR_SECTION 643 ld r6,KVM_SDR1(r9) 644 li r0,LPID_RSVD /* switch to reserved LPID */ 645 mtspr SPRN_LPID,r0 646 ptesync 647 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 648END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 649 mtspr SPRN_LPID,r7 650 isync 651 652 /* See if we need to flush the TLB */ 653 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 654BEGIN_FTR_SECTION 655 /* 656 * On POWER9, individual threads can come in here, but the 657 * TLB is shared between the 4 threads in a core, hence 658 * invalidating on one thread invalidates for all. 659 * Thus we make all 4 threads use the same bit here. 660 */ 661 clrrdi r6,r6,2 662END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 663 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 664 srdi r6,r6,6 /* doubleword number */ 665 sldi r6,r6,3 /* address offset */ 666 add r6,r6,r9 667 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 668 li r8,1 669 sld r8,r8,r7 670 ld r7,0(r6) 671 and. r7,r7,r8 672 beq 22f 673 /* Flush the TLB of any entries for this LPID */ 674 lwz r0,KVM_TLB_SETS(r9) 675 mtctr r0 676 li r7,0x800 /* IS field = 0b10 */ 677 ptesync 678 li r0,0 /* RS for P9 version of tlbiel */ 679 bne cr7, 29f 68028: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 681 addi r7,r7,0x1000 682 bdnz 28b 683 b 30f 68429: PPC_TLBIEL(7,0,2,1,1) /* for radix, RIC=2, PRS=1, R=1 */ 685 addi r7,r7,0x1000 686 bdnz 29b 68730: ptesync 68823: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 689 andc r7,r7,r8 690 stdcx. r7,0,r6 691 bne 23b 692 693 /* Add timebase offset onto timebase */ 69422: ld r8,VCORE_TB_OFFSET(r5) 695 cmpdi r8,0 696 beq 37f 697 mftb r6 /* current host timebase */ 698 add r8,r8,r6 699 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 700 mftb r7 /* check if lower 24 bits overflowed */ 701 clrldi r6,r6,40 702 clrldi r7,r7,40 703 cmpld r7,r6 704 bge 37f 705 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 706 mtspr SPRN_TBU40,r8 707 708 /* Load guest PCR value to select appropriate compat mode */ 70937: ld r7, VCORE_PCR(r5) 710 cmpdi r7, 0 711 beq 38f 712 mtspr SPRN_PCR, r7 71338: 714 715BEGIN_FTR_SECTION 716 /* DPDES and VTB are shared between threads */ 717 ld r8, VCORE_DPDES(r5) 718 ld r7, VCORE_VTB(r5) 719 mtspr SPRN_DPDES, r8 720 mtspr SPRN_VTB, r7 721END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 722 723 /* Mark the subcore state as inside guest */ 724 bl kvmppc_subcore_enter_guest 725 nop 726 ld r5, HSTATE_KVM_VCORE(r13) 727 ld r4, HSTATE_KVM_VCPU(r13) 728 li r0,1 729 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 730 731 /* Do we have a guest vcpu to run? */ 73210: cmpdi r4, 0 733 beq kvmppc_primary_no_guest 734kvmppc_got_guest: 735 /* Increment yield count if they have a VPA */ 736 ld r3, VCPU_VPA(r4) 737 cmpdi r3, 0 738 beq 25f 739 li r6, LPPACA_YIELDCOUNT 740 LWZX_BE r5, r3, r6 741 addi r5, r5, 1 742 STWX_BE r5, r3, r6 743 li r6, 1 744 stb r6, VCPU_VPA_DIRTY(r4) 74525: 746 747 /* Save purr/spurr */ 748 mfspr r5,SPRN_PURR 749 mfspr r6,SPRN_SPURR 750 std r5,HSTATE_PURR(r13) 751 std r6,HSTATE_SPURR(r13) 752 ld r7,VCPU_PURR(r4) 753 ld r8,VCPU_SPURR(r4) 754 mtspr SPRN_PURR,r7 755 mtspr SPRN_SPURR,r8 756 757 /* Save host values of some registers */ 758BEGIN_FTR_SECTION 759 mfspr r5, SPRN_TIDR 760 mfspr r6, SPRN_PSSCR 761 mfspr r7, SPRN_PID 762 mfspr r8, SPRN_IAMR 763 std r5, STACK_SLOT_TID(r1) 764 std r6, STACK_SLOT_PSSCR(r1) 765 std r7, STACK_SLOT_PID(r1) 766 std r8, STACK_SLOT_IAMR(r1) 767 mfspr r5, SPRN_HFSCR 768 std r5, STACK_SLOT_HFSCR(r1) 769END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 770BEGIN_FTR_SECTION 771 mfspr r5, SPRN_CIABR 772 mfspr r6, SPRN_DAWR 773 mfspr r7, SPRN_DAWRX 774 std r5, STACK_SLOT_CIABR(r1) 775 std r6, STACK_SLOT_DAWR(r1) 776 std r7, STACK_SLOT_DAWRX(r1) 777END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 778 779BEGIN_FTR_SECTION 780 /* Set partition DABR */ 781 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 782 lwz r5,VCPU_DABRX(r4) 783 ld r6,VCPU_DABR(r4) 784 mtspr SPRN_DABRX,r5 785 mtspr SPRN_DABR,r6 786 isync 787END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 788 789#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 790BEGIN_FTR_SECTION 791 /* 792 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 793 */ 794 bl kvmppc_restore_tm 795END_FTR_SECTION_IFSET(CPU_FTR_TM) 796#endif 797 798 /* Load guest PMU registers */ 799 /* R4 is live here (vcpu pointer) */ 800 li r3, 1 801 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 802 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 803 isync 804BEGIN_FTR_SECTION 805 ld r3, VCPU_MMCR(r4) 806 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 807 cmpwi r5, MMCR0_PMAO 808 beql kvmppc_fix_pmao 809END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 810 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 811 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 812 lwz r6, VCPU_PMC + 8(r4) 813 lwz r7, VCPU_PMC + 12(r4) 814 lwz r8, VCPU_PMC + 16(r4) 815 lwz r9, VCPU_PMC + 20(r4) 816 mtspr SPRN_PMC1, r3 817 mtspr SPRN_PMC2, r5 818 mtspr SPRN_PMC3, r6 819 mtspr SPRN_PMC4, r7 820 mtspr SPRN_PMC5, r8 821 mtspr SPRN_PMC6, r9 822 ld r3, VCPU_MMCR(r4) 823 ld r5, VCPU_MMCR + 8(r4) 824 ld r6, VCPU_MMCR + 16(r4) 825 ld r7, VCPU_SIAR(r4) 826 ld r8, VCPU_SDAR(r4) 827 mtspr SPRN_MMCR1, r5 828 mtspr SPRN_MMCRA, r6 829 mtspr SPRN_SIAR, r7 830 mtspr SPRN_SDAR, r8 831BEGIN_FTR_SECTION 832 ld r5, VCPU_MMCR + 24(r4) 833 ld r6, VCPU_SIER(r4) 834 mtspr SPRN_MMCR2, r5 835 mtspr SPRN_SIER, r6 836BEGIN_FTR_SECTION_NESTED(96) 837 lwz r7, VCPU_PMC + 24(r4) 838 lwz r8, VCPU_PMC + 28(r4) 839 ld r9, VCPU_MMCR + 32(r4) 840 mtspr SPRN_SPMC1, r7 841 mtspr SPRN_SPMC2, r8 842 mtspr SPRN_MMCRS, r9 843END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 844END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 845 mtspr SPRN_MMCR0, r3 846 isync 847 848 /* Load up FP, VMX and VSX registers */ 849 bl kvmppc_load_fp 850 851 ld r14, VCPU_GPR(R14)(r4) 852 ld r15, VCPU_GPR(R15)(r4) 853 ld r16, VCPU_GPR(R16)(r4) 854 ld r17, VCPU_GPR(R17)(r4) 855 ld r18, VCPU_GPR(R18)(r4) 856 ld r19, VCPU_GPR(R19)(r4) 857 ld r20, VCPU_GPR(R20)(r4) 858 ld r21, VCPU_GPR(R21)(r4) 859 ld r22, VCPU_GPR(R22)(r4) 860 ld r23, VCPU_GPR(R23)(r4) 861 ld r24, VCPU_GPR(R24)(r4) 862 ld r25, VCPU_GPR(R25)(r4) 863 ld r26, VCPU_GPR(R26)(r4) 864 ld r27, VCPU_GPR(R27)(r4) 865 ld r28, VCPU_GPR(R28)(r4) 866 ld r29, VCPU_GPR(R29)(r4) 867 ld r30, VCPU_GPR(R30)(r4) 868 ld r31, VCPU_GPR(R31)(r4) 869 870 /* Switch DSCR to guest value */ 871 ld r5, VCPU_DSCR(r4) 872 mtspr SPRN_DSCR, r5 873 874BEGIN_FTR_SECTION 875 /* Skip next section on POWER7 */ 876 b 8f 877END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 878 /* Load up POWER8-specific registers */ 879 ld r5, VCPU_IAMR(r4) 880 lwz r6, VCPU_PSPB(r4) 881 ld r7, VCPU_FSCR(r4) 882 mtspr SPRN_IAMR, r5 883 mtspr SPRN_PSPB, r6 884 mtspr SPRN_FSCR, r7 885 ld r5, VCPU_DAWR(r4) 886 ld r6, VCPU_DAWRX(r4) 887 ld r7, VCPU_CIABR(r4) 888 ld r8, VCPU_TAR(r4) 889 mtspr SPRN_DAWR, r5 890 mtspr SPRN_DAWRX, r6 891 mtspr SPRN_CIABR, r7 892 mtspr SPRN_TAR, r8 893 ld r5, VCPU_IC(r4) 894 ld r8, VCPU_EBBHR(r4) 895 mtspr SPRN_IC, r5 896 mtspr SPRN_EBBHR, r8 897 ld r5, VCPU_EBBRR(r4) 898 ld r6, VCPU_BESCR(r4) 899 lwz r7, VCPU_GUEST_PID(r4) 900 ld r8, VCPU_WORT(r4) 901 mtspr SPRN_EBBRR, r5 902 mtspr SPRN_BESCR, r6 903 mtspr SPRN_PID, r7 904 mtspr SPRN_WORT, r8 905BEGIN_FTR_SECTION 906 PPC_INVALIDATE_ERAT 907END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 908BEGIN_FTR_SECTION 909 /* POWER8-only registers */ 910 ld r5, VCPU_TCSCR(r4) 911 ld r6, VCPU_ACOP(r4) 912 ld r7, VCPU_CSIGR(r4) 913 ld r8, VCPU_TACR(r4) 914 mtspr SPRN_TCSCR, r5 915 mtspr SPRN_ACOP, r6 916 mtspr SPRN_CSIGR, r7 917 mtspr SPRN_TACR, r8 918FTR_SECTION_ELSE 919 /* POWER9-only registers */ 920 ld r5, VCPU_TID(r4) 921 ld r6, VCPU_PSSCR(r4) 922 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 923 ld r7, VCPU_HFSCR(r4) 924 mtspr SPRN_TIDR, r5 925 mtspr SPRN_PSSCR, r6 926 mtspr SPRN_HFSCR, r7 927ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 9288: 929 930 /* 931 * Set the decrementer to the guest decrementer. 932 */ 933 ld r8,VCPU_DEC_EXPIRES(r4) 934 /* r8 is a host timebase value here, convert to guest TB */ 935 ld r5,HSTATE_KVM_VCORE(r13) 936 ld r6,VCORE_TB_OFFSET(r5) 937 add r8,r8,r6 938 mftb r7 939 subf r3,r7,r8 940 mtspr SPRN_DEC,r3 941 942 ld r5, VCPU_SPRG0(r4) 943 ld r6, VCPU_SPRG1(r4) 944 ld r7, VCPU_SPRG2(r4) 945 ld r8, VCPU_SPRG3(r4) 946 mtspr SPRN_SPRG0, r5 947 mtspr SPRN_SPRG1, r6 948 mtspr SPRN_SPRG2, r7 949 mtspr SPRN_SPRG3, r8 950 951 /* Load up DAR and DSISR */ 952 ld r5, VCPU_DAR(r4) 953 lwz r6, VCPU_DSISR(r4) 954 mtspr SPRN_DAR, r5 955 mtspr SPRN_DSISR, r6 956 957 /* Restore AMR and UAMOR, set AMOR to all 1s */ 958 ld r5,VCPU_AMR(r4) 959 ld r6,VCPU_UAMOR(r4) 960 li r7,-1 961 mtspr SPRN_AMR,r5 962 mtspr SPRN_UAMOR,r6 963 mtspr SPRN_AMOR,r7 964 965 /* Restore state of CTRL run bit; assume 1 on entry */ 966 lwz r5,VCPU_CTRL(r4) 967 andi. r5,r5,1 968 bne 4f 969 mfspr r6,SPRN_CTRLF 970 clrrdi r6,r6,1 971 mtspr SPRN_CTRLT,r6 9724: 973 /* Secondary threads wait for primary to have done partition switch */ 974 ld r5, HSTATE_KVM_VCORE(r13) 975 lbz r6, HSTATE_PTID(r13) 976 cmpwi r6, 0 977 beq 21f 978 lbz r0, VCORE_IN_GUEST(r5) 979 cmpwi r0, 0 980 bne 21f 981 HMT_LOW 98220: lwz r3, VCORE_ENTRY_EXIT(r5) 983 cmpwi r3, 0x100 984 bge no_switch_exit 985 lbz r0, VCORE_IN_GUEST(r5) 986 cmpwi r0, 0 987 beq 20b 988 HMT_MEDIUM 98921: 990 /* Set LPCR. */ 991 ld r8,VCORE_LPCR(r5) 992 mtspr SPRN_LPCR,r8 993 isync 994 995 /* Check if HDEC expires soon */ 996 mfspr r3, SPRN_HDEC 997 EXTEND_HDEC(r3) 998 cmpdi r3, 512 /* 1 microsecond */ 999 blt hdec_soon 1000 1001 /* For hash guest, clear out and reload the SLB */ 1002 ld r6, VCPU_KVM(r4) 1003 lbz r0, KVM_RADIX(r6) 1004 cmpwi r0, 0 1005 bne 9f 1006 li r6, 0 1007 slbmte r6, r6 1008 slbia 1009 ptesync 1010 1011 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 1012 lwz r5,VCPU_SLB_MAX(r4) 1013 cmpwi r5,0 1014 beq 9f 1015 mtctr r5 1016 addi r6,r4,VCPU_SLB 10171: ld r8,VCPU_SLB_E(r6) 1018 ld r9,VCPU_SLB_V(r6) 1019 slbmte r9,r8 1020 addi r6,r6,VCPU_SLB_SIZE 1021 bdnz 1b 10229: 1023 1024#ifdef CONFIG_KVM_XICS 1025 /* We are entering the guest on that thread, push VCPU to XIVE */ 1026 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1027 cmpldi cr0, r10, 0 1028 beq no_xive 1029 ld r11, VCPU_XIVE_SAVED_STATE(r4) 1030 li r9, TM_QW1_OS 1031 eieio 1032 stdcix r11,r9,r10 1033 lwz r11, VCPU_XIVE_CAM_WORD(r4) 1034 li r9, TM_QW1_OS + TM_WORD2 1035 stwcix r11,r9,r10 1036 li r9, 1 1037 stb r9, VCPU_XIVE_PUSHED(r4) 1038 eieio 1039 1040 /* 1041 * We clear the irq_pending flag. There is a small chance of a 1042 * race vs. the escalation interrupt happening on another 1043 * processor setting it again, but the only consequence is to 1044 * cause a spurrious wakeup on the next H_CEDE which is not an 1045 * issue. 1046 */ 1047 li r0,0 1048 stb r0, VCPU_IRQ_PENDING(r4) 1049 1050 /* 1051 * In single escalation mode, if the escalation interrupt is 1052 * on, we mask it. 1053 */ 1054 lbz r0, VCPU_XIVE_ESC_ON(r4) 1055 cmpwi r0,0 1056 beq 1f 1057 ld r10, VCPU_XIVE_ESC_RADDR(r4) 1058 li r9, XIVE_ESB_SET_PQ_01 1059 ldcix r0, r10, r9 1060 sync 1061 1062 /* We have a possible subtle race here: The escalation interrupt might 1063 * have fired and be on its way to the host queue while we mask it, 1064 * and if we unmask it early enough (re-cede right away), there is 1065 * a theorical possibility that it fires again, thus landing in the 1066 * target queue more than once which is a big no-no. 1067 * 1068 * Fortunately, solving this is rather easy. If the above load setting 1069 * PQ to 01 returns a previous value where P is set, then we know the 1070 * escalation interrupt is somewhere on its way to the host. In that 1071 * case we simply don't clear the xive_esc_on flag below. It will be 1072 * eventually cleared by the handler for the escalation interrupt. 1073 * 1074 * Then, when doing a cede, we check that flag again before re-enabling 1075 * the escalation interrupt, and if set, we abort the cede. 1076 */ 1077 andi. r0, r0, XIVE_ESB_VAL_P 1078 bne- 1f 1079 1080 /* Now P is 0, we can clear the flag */ 1081 li r0, 0 1082 stb r0, VCPU_XIVE_ESC_ON(r4) 10831: 1084no_xive: 1085#endif /* CONFIG_KVM_XICS */ 1086 1087deliver_guest_interrupt: 1088 ld r6, VCPU_CTR(r4) 1089 ld r7, VCPU_XER(r4) 1090 1091 mtctr r6 1092 mtxer r7 1093 1094kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ 1095 ld r10, VCPU_PC(r4) 1096 ld r11, VCPU_MSR(r4) 1097 ld r6, VCPU_SRR0(r4) 1098 ld r7, VCPU_SRR1(r4) 1099 mtspr SPRN_SRR0, r6 1100 mtspr SPRN_SRR1, r7 1101 1102 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1103 rldicl r11, r11, 63 - MSR_HV_LG, 1 1104 rotldi r11, r11, 1 + MSR_HV_LG 1105 ori r11, r11, MSR_ME 1106 1107 /* Check if we can deliver an external or decrementer interrupt now */ 1108 ld r0, VCPU_PENDING_EXC(r4) 1109 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 1110 cmpdi cr1, r0, 0 1111 andi. r8, r11, MSR_EE 1112 mfspr r8, SPRN_LPCR 1113 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ 1114 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH 1115 mtspr SPRN_LPCR, r8 1116 isync 1117 beq 5f 1118 li r0, BOOK3S_INTERRUPT_EXTERNAL 1119 bne cr1, 12f 1120 mfspr r0, SPRN_DEC 1121BEGIN_FTR_SECTION 1122 /* On POWER9 check whether the guest has large decrementer enabled */ 1123 andis. r8, r8, LPCR_LD@h 1124 bne 15f 1125END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1126 extsw r0, r0 112715: cmpdi r0, 0 1128 li r0, BOOK3S_INTERRUPT_DECREMENTER 1129 bge 5f 1130 113112: mtspr SPRN_SRR0, r10 1132 mr r10,r0 1133 mtspr SPRN_SRR1, r11 1134 mr r9, r4 1135 bl kvmppc_msr_interrupt 11365: 1137BEGIN_FTR_SECTION 1138 b fast_guest_return 1139END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1140 /* On POWER9, check for pending doorbell requests */ 1141 lbz r0, VCPU_DBELL_REQ(r4) 1142 cmpwi r0, 0 1143 beq fast_guest_return 1144 ld r5, HSTATE_KVM_VCORE(r13) 1145 /* Set DPDES register so the CPU will take a doorbell interrupt */ 1146 li r0, 1 1147 mtspr SPRN_DPDES, r0 1148 std r0, VCORE_DPDES(r5) 1149 /* Make sure other cpus see vcore->dpdes set before dbell req clear */ 1150 lwsync 1151 /* Clear the pending doorbell request */ 1152 li r0, 0 1153 stb r0, VCPU_DBELL_REQ(r4) 1154 1155/* 1156 * Required state: 1157 * R4 = vcpu 1158 * R10: value for HSRR0 1159 * R11: value for HSRR1 1160 * R13 = PACA 1161 */ 1162fast_guest_return: 1163 li r0,0 1164 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1165 mtspr SPRN_HSRR0,r10 1166 mtspr SPRN_HSRR1,r11 1167 1168 /* Activate guest mode, so faults get handled by KVM */ 1169 li r9, KVM_GUEST_MODE_GUEST_HV 1170 stb r9, HSTATE_IN_GUEST(r13) 1171 1172#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1173 /* Accumulate timing */ 1174 addi r3, r4, VCPU_TB_GUEST 1175 bl kvmhv_accumulate_time 1176#endif 1177 1178 /* Enter guest */ 1179 1180BEGIN_FTR_SECTION 1181 ld r5, VCPU_CFAR(r4) 1182 mtspr SPRN_CFAR, r5 1183END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1184BEGIN_FTR_SECTION 1185 ld r0, VCPU_PPR(r4) 1186END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1187 1188 ld r5, VCPU_LR(r4) 1189 lwz r6, VCPU_CR(r4) 1190 mtlr r5 1191 mtcr r6 1192 1193 ld r1, VCPU_GPR(R1)(r4) 1194 ld r2, VCPU_GPR(R2)(r4) 1195 ld r3, VCPU_GPR(R3)(r4) 1196 ld r5, VCPU_GPR(R5)(r4) 1197 ld r6, VCPU_GPR(R6)(r4) 1198 ld r7, VCPU_GPR(R7)(r4) 1199 ld r8, VCPU_GPR(R8)(r4) 1200 ld r9, VCPU_GPR(R9)(r4) 1201 ld r10, VCPU_GPR(R10)(r4) 1202 ld r11, VCPU_GPR(R11)(r4) 1203 ld r12, VCPU_GPR(R12)(r4) 1204 ld r13, VCPU_GPR(R13)(r4) 1205 1206BEGIN_FTR_SECTION 1207 mtspr SPRN_PPR, r0 1208END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1209 1210/* Move canary into DSISR to check for later */ 1211BEGIN_FTR_SECTION 1212 li r0, 0x7fff 1213 mtspr SPRN_HDSISR, r0 1214END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1215 1216 ld r0, VCPU_GPR(R0)(r4) 1217 ld r4, VCPU_GPR(R4)(r4) 1218 HRFI_TO_GUEST 1219 b . 1220 1221secondary_too_late: 1222 li r12, 0 1223 cmpdi r4, 0 1224 beq 11f 1225 stw r12, VCPU_TRAP(r4) 1226#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1227 addi r3, r4, VCPU_TB_RMEXIT 1228 bl kvmhv_accumulate_time 1229#endif 123011: b kvmhv_switch_to_host 1231 1232no_switch_exit: 1233 HMT_MEDIUM 1234 li r12, 0 1235 b 12f 1236hdec_soon: 1237 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 123812: stw r12, VCPU_TRAP(r4) 1239 mr r9, r4 1240#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1241 addi r3, r4, VCPU_TB_RMEXIT 1242 bl kvmhv_accumulate_time 1243#endif 1244 b guest_bypass 1245 1246/****************************************************************************** 1247 * * 1248 * Exit code * 1249 * * 1250 *****************************************************************************/ 1251 1252/* 1253 * We come here from the first-level interrupt handlers. 1254 */ 1255 .globl kvmppc_interrupt_hv 1256kvmppc_interrupt_hv: 1257 /* 1258 * Register contents: 1259 * R12 = (guest CR << 32) | interrupt vector 1260 * R13 = PACA 1261 * guest R12 saved in shadow VCPU SCRATCH0 1262 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1263 * guest R13 saved in SPRN_SCRATCH0 1264 */ 1265 std r9, HSTATE_SCRATCH2(r13) 1266 lbz r9, HSTATE_IN_GUEST(r13) 1267 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1268 beq kvmppc_bad_host_intr 1269#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1270 cmpwi r9, KVM_GUEST_MODE_GUEST 1271 ld r9, HSTATE_SCRATCH2(r13) 1272 beq kvmppc_interrupt_pr 1273#endif 1274 /* We're now back in the host but in guest MMU context */ 1275 li r9, KVM_GUEST_MODE_HOST_HV 1276 stb r9, HSTATE_IN_GUEST(r13) 1277 1278 ld r9, HSTATE_KVM_VCPU(r13) 1279 1280 /* Save registers */ 1281 1282 std r0, VCPU_GPR(R0)(r9) 1283 std r1, VCPU_GPR(R1)(r9) 1284 std r2, VCPU_GPR(R2)(r9) 1285 std r3, VCPU_GPR(R3)(r9) 1286 std r4, VCPU_GPR(R4)(r9) 1287 std r5, VCPU_GPR(R5)(r9) 1288 std r6, VCPU_GPR(R6)(r9) 1289 std r7, VCPU_GPR(R7)(r9) 1290 std r8, VCPU_GPR(R8)(r9) 1291 ld r0, HSTATE_SCRATCH2(r13) 1292 std r0, VCPU_GPR(R9)(r9) 1293 std r10, VCPU_GPR(R10)(r9) 1294 std r11, VCPU_GPR(R11)(r9) 1295 ld r3, HSTATE_SCRATCH0(r13) 1296 std r3, VCPU_GPR(R12)(r9) 1297 /* CR is in the high half of r12 */ 1298 srdi r4, r12, 32 1299 stw r4, VCPU_CR(r9) 1300BEGIN_FTR_SECTION 1301 ld r3, HSTATE_CFAR(r13) 1302 std r3, VCPU_CFAR(r9) 1303END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1304BEGIN_FTR_SECTION 1305 ld r4, HSTATE_PPR(r13) 1306 std r4, VCPU_PPR(r9) 1307END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1308 1309 /* Restore R1/R2 so we can handle faults */ 1310 ld r1, HSTATE_HOST_R1(r13) 1311 ld r2, PACATOC(r13) 1312 1313 mfspr r10, SPRN_SRR0 1314 mfspr r11, SPRN_SRR1 1315 std r10, VCPU_SRR0(r9) 1316 std r11, VCPU_SRR1(r9) 1317 /* trap is in the low half of r12, clear CR from the high half */ 1318 clrldi r12, r12, 32 1319 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1320 beq 1f 1321 mfspr r10, SPRN_HSRR0 1322 mfspr r11, SPRN_HSRR1 1323 clrrdi r12, r12, 2 13241: std r10, VCPU_PC(r9) 1325 std r11, VCPU_MSR(r9) 1326 1327 GET_SCRATCH0(r3) 1328 mflr r4 1329 std r3, VCPU_GPR(R13)(r9) 1330 std r4, VCPU_LR(r9) 1331 1332 stw r12,VCPU_TRAP(r9) 1333 1334 /* 1335 * Now that we have saved away SRR0/1 and HSRR0/1, 1336 * interrupts are recoverable in principle, so set MSR_RI. 1337 * This becomes important for relocation-on interrupts from 1338 * the guest, which we can get in radix mode on POWER9. 1339 */ 1340 li r0, MSR_RI 1341 mtmsrd r0, 1 1342 1343#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1344 addi r3, r9, VCPU_TB_RMINTR 1345 mr r4, r9 1346 bl kvmhv_accumulate_time 1347 ld r5, VCPU_GPR(R5)(r9) 1348 ld r6, VCPU_GPR(R6)(r9) 1349 ld r7, VCPU_GPR(R7)(r9) 1350 ld r8, VCPU_GPR(R8)(r9) 1351#endif 1352 1353 /* Save HEIR (HV emulation assist reg) in emul_inst 1354 if this is an HEI (HV emulation interrupt, e40) */ 1355 li r3,KVM_INST_FETCH_FAILED 1356 stw r3,VCPU_LAST_INST(r9) 1357 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1358 bne 11f 1359 mfspr r3,SPRN_HEIR 136011: stw r3,VCPU_HEIR(r9) 1361 1362 /* these are volatile across C function calls */ 1363#ifdef CONFIG_RELOCATABLE 1364 ld r3, HSTATE_SCRATCH1(r13) 1365 mtctr r3 1366#else 1367 mfctr r3 1368#endif 1369 mfxer r4 1370 std r3, VCPU_CTR(r9) 1371 std r4, VCPU_XER(r9) 1372 1373 /* If this is a page table miss then see if it's theirs or ours */ 1374 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1375 beq kvmppc_hdsi 1376 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1377 beq kvmppc_hisi 1378 1379 /* See if this is a leftover HDEC interrupt */ 1380 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1381 bne 2f 1382 mfspr r3,SPRN_HDEC 1383 EXTEND_HDEC(r3) 1384 cmpdi r3,0 1385 mr r4,r9 1386 bge fast_guest_return 13872: 1388 /* See if this is an hcall we can handle in real mode */ 1389 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1390 beq hcall_try_real_mode 1391 1392 /* Hypervisor doorbell - exit only if host IPI flag set */ 1393 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1394 bne 3f 1395BEGIN_FTR_SECTION 1396 PPC_MSGSYNC 1397 lwsync 1398END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1399 lbz r0, HSTATE_HOST_IPI(r13) 1400 cmpwi r0, 0 1401 beq 4f 1402 b guest_exit_cont 14033: 1404 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1405 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1406 bne 14f 1407 mfspr r3, SPRN_HFSCR 1408 std r3, VCPU_HFSCR(r9) 1409 b guest_exit_cont 141014: 1411 /* External interrupt ? */ 1412 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1413 bne+ guest_exit_cont 1414 1415 /* External interrupt, first check for host_ipi. If this is 1416 * set, we know the host wants us out so let's do it now 1417 */ 1418 bl kvmppc_read_intr 1419 1420 /* 1421 * Restore the active volatile registers after returning from 1422 * a C function. 1423 */ 1424 ld r9, HSTATE_KVM_VCPU(r13) 1425 li r12, BOOK3S_INTERRUPT_EXTERNAL 1426 1427 /* 1428 * kvmppc_read_intr return codes: 1429 * 1430 * Exit to host (r3 > 0) 1431 * 1 An interrupt is pending that needs to be handled by the host 1432 * Exit guest and return to host by branching to guest_exit_cont 1433 * 1434 * 2 Passthrough that needs completion in the host 1435 * Exit guest and return to host by branching to guest_exit_cont 1436 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1437 * to indicate to the host to complete handling the interrupt 1438 * 1439 * Before returning to guest, we check if any CPU is heading out 1440 * to the host and if so, we head out also. If no CPUs are heading 1441 * check return values <= 0. 1442 * 1443 * Return to guest (r3 <= 0) 1444 * 0 No external interrupt is pending 1445 * -1 A guest wakeup IPI (which has now been cleared) 1446 * In either case, we return to guest to deliver any pending 1447 * guest interrupts. 1448 * 1449 * -2 A PCI passthrough external interrupt was handled 1450 * (interrupt was delivered directly to guest) 1451 * Return to guest to deliver any pending guest interrupts. 1452 */ 1453 1454 cmpdi r3, 1 1455 ble 1f 1456 1457 /* Return code = 2 */ 1458 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1459 stw r12, VCPU_TRAP(r9) 1460 b guest_exit_cont 1461 14621: /* Return code <= 1 */ 1463 cmpdi r3, 0 1464 bgt guest_exit_cont 1465 1466 /* Return code <= 0 */ 14674: ld r5, HSTATE_KVM_VCORE(r13) 1468 lwz r0, VCORE_ENTRY_EXIT(r5) 1469 cmpwi r0, 0x100 1470 mr r4, r9 1471 blt deliver_guest_interrupt 1472 1473guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1474 /* Save more register state */ 1475 mfdar r6 1476 mfdsisr r7 1477 std r6, VCPU_DAR(r9) 1478 stw r7, VCPU_DSISR(r9) 1479 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1480 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1481 beq mc_cont 1482 std r6, VCPU_FAULT_DAR(r9) 1483 stw r7, VCPU_FAULT_DSISR(r9) 1484 1485 /* See if it is a machine check */ 1486 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1487 beq machine_check_realmode 1488mc_cont: 1489#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1490 addi r3, r9, VCPU_TB_RMEXIT 1491 mr r4, r9 1492 bl kvmhv_accumulate_time 1493#endif 1494#ifdef CONFIG_KVM_XICS 1495 /* We are exiting, pull the VP from the XIVE */ 1496 lbz r0, VCPU_XIVE_PUSHED(r9) 1497 cmpwi cr0, r0, 0 1498 beq 1f 1499 li r7, TM_SPC_PULL_OS_CTX 1500 li r6, TM_QW1_OS 1501 mfmsr r0 1502 andi. r0, r0, MSR_DR /* in real mode? */ 1503 beq 2f 1504 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1505 cmpldi cr0, r10, 0 1506 beq 1f 1507 /* First load to pull the context, we ignore the value */ 1508 eieio 1509 lwzx r11, r7, r10 1510 /* Second load to recover the context state (Words 0 and 1) */ 1511 ldx r11, r6, r10 1512 b 3f 15132: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1514 cmpldi cr0, r10, 0 1515 beq 1f 1516 /* First load to pull the context, we ignore the value */ 1517 eieio 1518 lwzcix r11, r7, r10 1519 /* Second load to recover the context state (Words 0 and 1) */ 1520 ldcix r11, r6, r10 15213: std r11, VCPU_XIVE_SAVED_STATE(r9) 1522 /* Fixup some of the state for the next load */ 1523 li r10, 0 1524 li r0, 0xff 1525 stb r10, VCPU_XIVE_PUSHED(r9) 1526 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1527 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1528 eieio 15291: 1530#endif /* CONFIG_KVM_XICS */ 1531 1532 /* For hash guest, read the guest SLB and save it away */ 1533 ld r5, VCPU_KVM(r9) 1534 lbz r0, KVM_RADIX(r5) 1535 li r5, 0 1536 cmpwi r0, 0 1537 bne 3f /* for radix, save 0 entries */ 1538 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1539 mtctr r0 1540 li r6,0 1541 addi r7,r9,VCPU_SLB 15421: slbmfee r8,r6 1543 andis. r0,r8,SLB_ESID_V@h 1544 beq 2f 1545 add r8,r8,r6 /* put index in */ 1546 slbmfev r3,r6 1547 std r8,VCPU_SLB_E(r7) 1548 std r3,VCPU_SLB_V(r7) 1549 addi r7,r7,VCPU_SLB_SIZE 1550 addi r5,r5,1 15512: addi r6,r6,1 1552 bdnz 1b 1553 /* Finally clear out the SLB */ 1554 li r0,0 1555 slbmte r0,r0 1556 slbia 1557 ptesync 15583: stw r5,VCPU_SLB_MAX(r9) 1559 1560guest_bypass: 1561 mr r3, r12 1562 /* Increment exit count, poke other threads to exit */ 1563 bl kvmhv_commence_exit 1564 nop 1565 ld r9, HSTATE_KVM_VCPU(r13) 1566 lwz r12, VCPU_TRAP(r9) 1567 1568 /* Stop others sending VCPU interrupts to this physical CPU */ 1569 li r0, -1 1570 stw r0, VCPU_CPU(r9) 1571 stw r0, VCPU_THREAD_CPU(r9) 1572 1573 /* Save guest CTRL register, set runlatch to 1 */ 1574 mfspr r6,SPRN_CTRLF 1575 stw r6,VCPU_CTRL(r9) 1576 andi. r0,r6,1 1577 bne 4f 1578 ori r6,r6,1 1579 mtspr SPRN_CTRLT,r6 15804: 1581 /* 1582 * Save the guest PURR/SPURR 1583 */ 1584 mfspr r5,SPRN_PURR 1585 mfspr r6,SPRN_SPURR 1586 ld r7,VCPU_PURR(r9) 1587 ld r8,VCPU_SPURR(r9) 1588 std r5,VCPU_PURR(r9) 1589 std r6,VCPU_SPURR(r9) 1590 subf r5,r7,r5 1591 subf r6,r8,r6 1592 1593 /* 1594 * Restore host PURR/SPURR and add guest times 1595 * so that the time in the guest gets accounted. 1596 */ 1597 ld r3,HSTATE_PURR(r13) 1598 ld r4,HSTATE_SPURR(r13) 1599 add r3,r3,r5 1600 add r4,r4,r6 1601 mtspr SPRN_PURR,r3 1602 mtspr SPRN_SPURR,r4 1603 1604 /* Save DEC */ 1605 ld r3, HSTATE_KVM_VCORE(r13) 1606 mfspr r5,SPRN_DEC 1607 mftb r6 1608 /* On P9, if the guest has large decr enabled, don't sign extend */ 1609BEGIN_FTR_SECTION 1610 ld r4, VCORE_LPCR(r3) 1611 andis. r4, r4, LPCR_LD@h 1612 bne 16f 1613END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1614 extsw r5,r5 161516: add r5,r5,r6 1616 /* r5 is a guest timebase value here, convert to host TB */ 1617 ld r4,VCORE_TB_OFFSET(r3) 1618 subf r5,r4,r5 1619 std r5,VCPU_DEC_EXPIRES(r9) 1620 1621BEGIN_FTR_SECTION 1622 b 8f 1623END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1624 /* Save POWER8-specific registers */ 1625 mfspr r5, SPRN_IAMR 1626 mfspr r6, SPRN_PSPB 1627 mfspr r7, SPRN_FSCR 1628 std r5, VCPU_IAMR(r9) 1629 stw r6, VCPU_PSPB(r9) 1630 std r7, VCPU_FSCR(r9) 1631 mfspr r5, SPRN_IC 1632 mfspr r7, SPRN_TAR 1633 std r5, VCPU_IC(r9) 1634 std r7, VCPU_TAR(r9) 1635 mfspr r8, SPRN_EBBHR 1636 std r8, VCPU_EBBHR(r9) 1637 mfspr r5, SPRN_EBBRR 1638 mfspr r6, SPRN_BESCR 1639 mfspr r7, SPRN_PID 1640 mfspr r8, SPRN_WORT 1641 std r5, VCPU_EBBRR(r9) 1642 std r6, VCPU_BESCR(r9) 1643 stw r7, VCPU_GUEST_PID(r9) 1644 std r8, VCPU_WORT(r9) 1645BEGIN_FTR_SECTION 1646 mfspr r5, SPRN_TCSCR 1647 mfspr r6, SPRN_ACOP 1648 mfspr r7, SPRN_CSIGR 1649 mfspr r8, SPRN_TACR 1650 std r5, VCPU_TCSCR(r9) 1651 std r6, VCPU_ACOP(r9) 1652 std r7, VCPU_CSIGR(r9) 1653 std r8, VCPU_TACR(r9) 1654FTR_SECTION_ELSE 1655 mfspr r5, SPRN_TIDR 1656 mfspr r6, SPRN_PSSCR 1657 std r5, VCPU_TID(r9) 1658 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1659 rotldi r6, r6, 60 1660 std r6, VCPU_PSSCR(r9) 1661 /* Restore host HFSCR value */ 1662 ld r7, STACK_SLOT_HFSCR(r1) 1663 mtspr SPRN_HFSCR, r7 1664ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1665 /* 1666 * Restore various registers to 0, where non-zero values 1667 * set by the guest could disrupt the host. 1668 */ 1669 li r0, 0 1670 mtspr SPRN_PSPB, r0 1671 mtspr SPRN_WORT, r0 1672BEGIN_FTR_SECTION 1673 mtspr SPRN_IAMR, r0 1674 mtspr SPRN_TCSCR, r0 1675 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1676 li r0, 1 1677 sldi r0, r0, 31 1678 mtspr SPRN_MMCRS, r0 1679END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 16808: 1681 1682 /* Save and reset AMR and UAMOR before turning on the MMU */ 1683 mfspr r5,SPRN_AMR 1684 mfspr r6,SPRN_UAMOR 1685 std r5,VCPU_AMR(r9) 1686 std r6,VCPU_UAMOR(r9) 1687 li r6,0 1688 mtspr SPRN_AMR,r6 1689 mtspr SPRN_UAMOR, r6 1690 1691 /* Switch DSCR back to host value */ 1692 mfspr r8, SPRN_DSCR 1693 ld r7, HSTATE_DSCR(r13) 1694 std r8, VCPU_DSCR(r9) 1695 mtspr SPRN_DSCR, r7 1696 1697 /* Save non-volatile GPRs */ 1698 std r14, VCPU_GPR(R14)(r9) 1699 std r15, VCPU_GPR(R15)(r9) 1700 std r16, VCPU_GPR(R16)(r9) 1701 std r17, VCPU_GPR(R17)(r9) 1702 std r18, VCPU_GPR(R18)(r9) 1703 std r19, VCPU_GPR(R19)(r9) 1704 std r20, VCPU_GPR(R20)(r9) 1705 std r21, VCPU_GPR(R21)(r9) 1706 std r22, VCPU_GPR(R22)(r9) 1707 std r23, VCPU_GPR(R23)(r9) 1708 std r24, VCPU_GPR(R24)(r9) 1709 std r25, VCPU_GPR(R25)(r9) 1710 std r26, VCPU_GPR(R26)(r9) 1711 std r27, VCPU_GPR(R27)(r9) 1712 std r28, VCPU_GPR(R28)(r9) 1713 std r29, VCPU_GPR(R29)(r9) 1714 std r30, VCPU_GPR(R30)(r9) 1715 std r31, VCPU_GPR(R31)(r9) 1716 1717 /* Save SPRGs */ 1718 mfspr r3, SPRN_SPRG0 1719 mfspr r4, SPRN_SPRG1 1720 mfspr r5, SPRN_SPRG2 1721 mfspr r6, SPRN_SPRG3 1722 std r3, VCPU_SPRG0(r9) 1723 std r4, VCPU_SPRG1(r9) 1724 std r5, VCPU_SPRG2(r9) 1725 std r6, VCPU_SPRG3(r9) 1726 1727 /* save FP state */ 1728 mr r3, r9 1729 bl kvmppc_save_fp 1730 1731#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1732BEGIN_FTR_SECTION 1733 /* 1734 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 1735 */ 1736 bl kvmppc_save_tm 1737END_FTR_SECTION_IFSET(CPU_FTR_TM) 1738#endif 1739 1740 /* Increment yield count if they have a VPA */ 1741 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1742 cmpdi r8, 0 1743 beq 25f 1744 li r4, LPPACA_YIELDCOUNT 1745 LWZX_BE r3, r8, r4 1746 addi r3, r3, 1 1747 STWX_BE r3, r8, r4 1748 li r3, 1 1749 stb r3, VCPU_VPA_DIRTY(r9) 175025: 1751 /* Save PMU registers if requested */ 1752 /* r8 and cr0.eq are live here */ 1753BEGIN_FTR_SECTION 1754 /* 1755 * POWER8 seems to have a hardware bug where setting 1756 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 1757 * when some counters are already negative doesn't seem 1758 * to cause a performance monitor alert (and hence interrupt). 1759 * The effect of this is that when saving the PMU state, 1760 * if there is no PMU alert pending when we read MMCR0 1761 * before freezing the counters, but one becomes pending 1762 * before we read the counters, we lose it. 1763 * To work around this, we need a way to freeze the counters 1764 * before reading MMCR0. Normally, freezing the counters 1765 * is done by writing MMCR0 (to set MMCR0[FC]) which 1766 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 1767 * we can also freeze the counters using MMCR2, by writing 1768 * 1s to all the counter freeze condition bits (there are 1769 * 9 bits each for 6 counters). 1770 */ 1771 li r3, -1 /* set all freeze bits */ 1772 clrrdi r3, r3, 10 1773 mfspr r10, SPRN_MMCR2 1774 mtspr SPRN_MMCR2, r3 1775 isync 1776END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1777 li r3, 1 1778 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 1779 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 1780 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 1781 mfspr r6, SPRN_MMCRA 1782 /* Clear MMCRA in order to disable SDAR updates */ 1783 li r7, 0 1784 mtspr SPRN_MMCRA, r7 1785 isync 1786 beq 21f /* if no VPA, save PMU stuff anyway */ 1787 lbz r7, LPPACA_PMCINUSE(r8) 1788 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */ 1789 bne 21f 1790 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 1791 b 22f 179221: mfspr r5, SPRN_MMCR1 1793 mfspr r7, SPRN_SIAR 1794 mfspr r8, SPRN_SDAR 1795 std r4, VCPU_MMCR(r9) 1796 std r5, VCPU_MMCR + 8(r9) 1797 std r6, VCPU_MMCR + 16(r9) 1798BEGIN_FTR_SECTION 1799 std r10, VCPU_MMCR + 24(r9) 1800END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1801 std r7, VCPU_SIAR(r9) 1802 std r8, VCPU_SDAR(r9) 1803 mfspr r3, SPRN_PMC1 1804 mfspr r4, SPRN_PMC2 1805 mfspr r5, SPRN_PMC3 1806 mfspr r6, SPRN_PMC4 1807 mfspr r7, SPRN_PMC5 1808 mfspr r8, SPRN_PMC6 1809 stw r3, VCPU_PMC(r9) 1810 stw r4, VCPU_PMC + 4(r9) 1811 stw r5, VCPU_PMC + 8(r9) 1812 stw r6, VCPU_PMC + 12(r9) 1813 stw r7, VCPU_PMC + 16(r9) 1814 stw r8, VCPU_PMC + 20(r9) 1815BEGIN_FTR_SECTION 1816 mfspr r5, SPRN_SIER 1817 std r5, VCPU_SIER(r9) 1818BEGIN_FTR_SECTION_NESTED(96) 1819 mfspr r6, SPRN_SPMC1 1820 mfspr r7, SPRN_SPMC2 1821 mfspr r8, SPRN_MMCRS 1822 stw r6, VCPU_PMC + 24(r9) 1823 stw r7, VCPU_PMC + 28(r9) 1824 std r8, VCPU_MMCR + 32(r9) 1825 lis r4, 0x8000 1826 mtspr SPRN_MMCRS, r4 1827END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 1828END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 182922: 1830 1831 /* Restore host values of some registers */ 1832BEGIN_FTR_SECTION 1833 ld r5, STACK_SLOT_CIABR(r1) 1834 ld r6, STACK_SLOT_DAWR(r1) 1835 ld r7, STACK_SLOT_DAWRX(r1) 1836 mtspr SPRN_CIABR, r5 1837 mtspr SPRN_DAWR, r6 1838 mtspr SPRN_DAWRX, r7 1839END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1840BEGIN_FTR_SECTION 1841 ld r5, STACK_SLOT_TID(r1) 1842 ld r6, STACK_SLOT_PSSCR(r1) 1843 ld r7, STACK_SLOT_PID(r1) 1844 ld r8, STACK_SLOT_IAMR(r1) 1845 mtspr SPRN_TIDR, r5 1846 mtspr SPRN_PSSCR, r6 1847 mtspr SPRN_PID, r7 1848 mtspr SPRN_IAMR, r8 1849END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1850 1851#ifdef CONFIG_PPC_RADIX_MMU 1852 /* 1853 * Are we running hash or radix ? 1854 */ 1855 ld r5, VCPU_KVM(r9) 1856 lbz r0, KVM_RADIX(r5) 1857 cmpwi cr2, r0, 0 1858 beq cr2, 4f 1859 1860 /* Radix: Handle the case where the guest used an illegal PID */ 1861 LOAD_REG_ADDR(r4, mmu_base_pid) 1862 lwz r3, VCPU_GUEST_PID(r9) 1863 lwz r5, 0(r4) 1864 cmpw cr0,r3,r5 1865 blt 2f 1866 1867 /* 1868 * Illegal PID, the HW might have prefetched and cached in the TLB 1869 * some translations for the LPID 0 / guest PID combination which 1870 * Linux doesn't know about, so we need to flush that PID out of 1871 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1872 * the right context. 1873 */ 1874 li r0,0 1875 mtspr SPRN_LPID,r0 1876 isync 1877 1878 /* Then do a congruence class local flush */ 1879 ld r6,VCPU_KVM(r9) 1880 lwz r0,KVM_TLB_SETS(r6) 1881 mtctr r0 1882 li r7,0x400 /* IS field = 0b01 */ 1883 ptesync 1884 sldi r0,r3,32 /* RS has PID */ 18851: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1886 addi r7,r7,0x1000 1887 bdnz 1b 1888 ptesync 1889 18902: /* Flush the ERAT on radix P9 DD1 guest exit */ 1891BEGIN_FTR_SECTION 1892 PPC_INVALIDATE_ERAT 1893END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 18944: 1895#endif /* CONFIG_PPC_RADIX_MMU */ 1896 1897 /* 1898 * POWER7/POWER8 guest -> host partition switch code. 1899 * We don't have to lock against tlbies but we do 1900 * have to coordinate the hardware threads. 1901 */ 1902kvmhv_switch_to_host: 1903 /* Secondary threads wait for primary to do partition switch */ 1904 ld r5,HSTATE_KVM_VCORE(r13) 1905 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1906 lbz r3,HSTATE_PTID(r13) 1907 cmpwi r3,0 1908 beq 15f 1909 HMT_LOW 191013: lbz r3,VCORE_IN_GUEST(r5) 1911 cmpwi r3,0 1912 bne 13b 1913 HMT_MEDIUM 1914 b 16f 1915 1916 /* Primary thread waits for all the secondaries to exit guest */ 191715: lwz r3,VCORE_ENTRY_EXIT(r5) 1918 rlwinm r0,r3,32-8,0xff 1919 clrldi r3,r3,56 1920 cmpw r3,r0 1921 bne 15b 1922 isync 1923 1924 /* Did we actually switch to the guest at all? */ 1925 lbz r6, VCORE_IN_GUEST(r5) 1926 cmpwi r6, 0 1927 beq 19f 1928 1929 /* Primary thread switches back to host partition */ 1930 lwz r7,KVM_HOST_LPID(r4) 1931BEGIN_FTR_SECTION 1932 ld r6,KVM_HOST_SDR1(r4) 1933 li r8,LPID_RSVD /* switch to reserved LPID */ 1934 mtspr SPRN_LPID,r8 1935 ptesync 1936 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1937END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1938 mtspr SPRN_LPID,r7 1939 isync 1940 1941BEGIN_FTR_SECTION 1942 /* DPDES and VTB are shared between threads */ 1943 mfspr r7, SPRN_DPDES 1944 mfspr r8, SPRN_VTB 1945 std r7, VCORE_DPDES(r5) 1946 std r8, VCORE_VTB(r5) 1947 /* clear DPDES so we don't get guest doorbells in the host */ 1948 li r8, 0 1949 mtspr SPRN_DPDES, r8 1950END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1951 1952 /* If HMI, call kvmppc_realmode_hmi_handler() */ 1953 cmpwi r12, BOOK3S_INTERRUPT_HMI 1954 bne 27f 1955 bl kvmppc_realmode_hmi_handler 1956 nop 1957 cmpdi r3, 0 1958 li r12, BOOK3S_INTERRUPT_HMI 1959 /* 1960 * At this point kvmppc_realmode_hmi_handler may have resync-ed 1961 * the TB, and if it has, we must not subtract the guest timebase 1962 * offset from the timebase. So, skip it. 1963 * 1964 * Also, do not call kvmppc_subcore_exit_guest() because it has 1965 * been invoked as part of kvmppc_realmode_hmi_handler(). 1966 */ 1967 beq 30f 1968 196927: 1970 /* Subtract timebase offset from timebase */ 1971 ld r8,VCORE_TB_OFFSET(r5) 1972 cmpdi r8,0 1973 beq 17f 1974 mftb r6 /* current guest timebase */ 1975 subf r8,r8,r6 1976 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1977 mftb r7 /* check if lower 24 bits overflowed */ 1978 clrldi r6,r6,40 1979 clrldi r7,r7,40 1980 cmpld r7,r6 1981 bge 17f 1982 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1983 mtspr SPRN_TBU40,r8 1984 198517: bl kvmppc_subcore_exit_guest 1986 nop 198730: ld r5,HSTATE_KVM_VCORE(r13) 1988 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1989 1990 /* Reset PCR */ 1991 ld r0, VCORE_PCR(r5) 1992 cmpdi r0, 0 1993 beq 18f 1994 li r0, 0 1995 mtspr SPRN_PCR, r0 199618: 1997 /* Signal secondary CPUs to continue */ 1998 stb r0,VCORE_IN_GUEST(r5) 199919: lis r8,0x7fff /* MAX_INT@h */ 2000 mtspr SPRN_HDEC,r8 2001 200216: 2003BEGIN_FTR_SECTION 2004 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 2005 ld r3, HSTATE_SPLIT_MODE(r13) 2006 cmpdi r3, 0 2007 beq 47f 2008 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 2009 cmpwi r8, 0 2010 beq 47f 2011 stw r12, STACK_SLOT_TRAP(r1) 2012 bl kvmhv_p9_restore_lpcr 2013 nop 2014 lwz r12, STACK_SLOT_TRAP(r1) 2015 b 48f 201647: 2017END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2018 ld r8,KVM_HOST_LPCR(r4) 2019 mtspr SPRN_LPCR,r8 2020 isync 202148: 2022 /* load host SLB entries */ 2023BEGIN_MMU_FTR_SECTION 2024 b 0f 2025END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 2026 ld r8,PACA_SLBSHADOWPTR(r13) 2027 2028 .rept SLB_NUM_BOLTED 2029 li r3, SLBSHADOW_SAVEAREA 2030 LDX_BE r5, r8, r3 2031 addi r3, r3, 8 2032 LDX_BE r6, r8, r3 2033 andis. r7,r5,SLB_ESID_V@h 2034 beq 1f 2035 slbmte r6,r5 20361: addi r8,r8,16 2037 .endr 20380: 2039#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2040 /* Finish timing, if we have a vcpu */ 2041 ld r4, HSTATE_KVM_VCPU(r13) 2042 cmpdi r4, 0 2043 li r3, 0 2044 beq 2f 2045 bl kvmhv_accumulate_time 20462: 2047#endif 2048 /* Unset guest mode */ 2049 li r0, KVM_GUEST_MODE_NONE 2050 stb r0, HSTATE_IN_GUEST(r13) 2051 2052 ld r0, SFS+PPC_LR_STKOFF(r1) 2053 addi r1, r1, SFS 2054 mtlr r0 2055 blr 2056 2057/* 2058 * Check whether an HDSI is an HPTE not found fault or something else. 2059 * If it is an HPTE not found fault that is due to the guest accessing 2060 * a page that they have mapped but which we have paged out, then 2061 * we continue on with the guest exit path. In all other cases, 2062 * reflect the HDSI to the guest as a DSI. 2063 */ 2064kvmppc_hdsi: 2065 ld r3, VCPU_KVM(r9) 2066 lbz r0, KVM_RADIX(r3) 2067 mfspr r4, SPRN_HDAR 2068 mfspr r6, SPRN_HDSISR 2069BEGIN_FTR_SECTION 2070 /* Look for DSISR canary. If we find it, retry instruction */ 2071 cmpdi r6, 0x7fff 2072 beq 6f 2073END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2074 cmpwi r0, 0 2075 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2076 /* HPTE not found fault or protection fault? */ 2077 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2078 beq 1f /* if not, send it to the guest */ 2079 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2080 beq 3f 2081BEGIN_FTR_SECTION 2082 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2083 b 4f 2084END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2085 clrrdi r0, r4, 28 2086 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2087 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2088 bne 7f /* if no SLB entry found */ 20894: std r4, VCPU_FAULT_DAR(r9) 2090 stw r6, VCPU_FAULT_DSISR(r9) 2091 2092 /* Search the hash table. */ 2093 mr r3, r9 /* vcpu pointer */ 2094 li r7, 1 /* data fault */ 2095 bl kvmppc_hpte_hv_fault 2096 ld r9, HSTATE_KVM_VCPU(r13) 2097 ld r10, VCPU_PC(r9) 2098 ld r11, VCPU_MSR(r9) 2099 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2100 cmpdi r3, 0 /* retry the instruction */ 2101 beq 6f 2102 cmpdi r3, -1 /* handle in kernel mode */ 2103 beq guest_exit_cont 2104 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2105 beq 2f 2106 2107 /* Synthesize a DSI (or DSegI) for the guest */ 2108 ld r4, VCPU_FAULT_DAR(r9) 2109 mr r6, r3 21101: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2111 mtspr SPRN_DSISR, r6 21127: mtspr SPRN_DAR, r4 2113 mtspr SPRN_SRR0, r10 2114 mtspr SPRN_SRR1, r11 2115 mr r10, r0 2116 bl kvmppc_msr_interrupt 2117fast_interrupt_c_return: 21186: ld r7, VCPU_CTR(r9) 2119 ld r8, VCPU_XER(r9) 2120 mtctr r7 2121 mtxer r8 2122 mr r4, r9 2123 b fast_guest_return 2124 21253: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2126 ld r5, KVM_VRMA_SLB_V(r5) 2127 b 4b 2128 2129 /* If this is for emulated MMIO, load the instruction word */ 21302: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2131 2132 /* Set guest mode to 'jump over instruction' so if lwz faults 2133 * we'll just continue at the next IP. */ 2134 li r0, KVM_GUEST_MODE_SKIP 2135 stb r0, HSTATE_IN_GUEST(r13) 2136 2137 /* Do the access with MSR:DR enabled */ 2138 mfmsr r3 2139 ori r4, r3, MSR_DR /* Enable paging for data */ 2140 mtmsrd r4 2141 lwz r8, 0(r10) 2142 mtmsrd r3 2143 2144 /* Store the result */ 2145 stw r8, VCPU_LAST_INST(r9) 2146 2147 /* Unset guest mode. */ 2148 li r0, KVM_GUEST_MODE_HOST_HV 2149 stb r0, HSTATE_IN_GUEST(r13) 2150 b guest_exit_cont 2151 2152.Lradix_hdsi: 2153 std r4, VCPU_FAULT_DAR(r9) 2154 stw r6, VCPU_FAULT_DSISR(r9) 2155.Lradix_hisi: 2156 mfspr r5, SPRN_ASDR 2157 std r5, VCPU_FAULT_GPA(r9) 2158 b guest_exit_cont 2159 2160/* 2161 * Similarly for an HISI, reflect it to the guest as an ISI unless 2162 * it is an HPTE not found fault for a page that we have paged out. 2163 */ 2164kvmppc_hisi: 2165 ld r3, VCPU_KVM(r9) 2166 lbz r0, KVM_RADIX(r3) 2167 cmpwi r0, 0 2168 bne .Lradix_hisi /* for radix, just save ASDR */ 2169 andis. r0, r11, SRR1_ISI_NOPT@h 2170 beq 1f 2171 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2172 beq 3f 2173BEGIN_FTR_SECTION 2174 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2175 b 4f 2176END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2177 clrrdi r0, r10, 28 2178 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2179 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2180 bne 7f /* if no SLB entry found */ 21814: 2182 /* Search the hash table. */ 2183 mr r3, r9 /* vcpu pointer */ 2184 mr r4, r10 2185 mr r6, r11 2186 li r7, 0 /* instruction fault */ 2187 bl kvmppc_hpte_hv_fault 2188 ld r9, HSTATE_KVM_VCPU(r13) 2189 ld r10, VCPU_PC(r9) 2190 ld r11, VCPU_MSR(r9) 2191 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2192 cmpdi r3, 0 /* retry the instruction */ 2193 beq fast_interrupt_c_return 2194 cmpdi r3, -1 /* handle in kernel mode */ 2195 beq guest_exit_cont 2196 2197 /* Synthesize an ISI (or ISegI) for the guest */ 2198 mr r11, r3 21991: li r0, BOOK3S_INTERRUPT_INST_STORAGE 22007: mtspr SPRN_SRR0, r10 2201 mtspr SPRN_SRR1, r11 2202 mr r10, r0 2203 bl kvmppc_msr_interrupt 2204 b fast_interrupt_c_return 2205 22063: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2207 ld r5, KVM_VRMA_SLB_V(r6) 2208 b 4b 2209 2210/* 2211 * Try to handle an hcall in real mode. 2212 * Returns to the guest if we handle it, or continues on up to 2213 * the kernel if we can't (i.e. if we don't have a handler for 2214 * it, or if the handler returns H_TOO_HARD). 2215 * 2216 * r5 - r8 contain hcall args, 2217 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2218 */ 2219hcall_try_real_mode: 2220 ld r3,VCPU_GPR(R3)(r9) 2221 andi. r0,r11,MSR_PR 2222 /* sc 1 from userspace - reflect to guest syscall */ 2223 bne sc_1_fast_return 2224 clrrdi r3,r3,2 2225 cmpldi r3,hcall_real_table_end - hcall_real_table 2226 bge guest_exit_cont 2227 /* See if this hcall is enabled for in-kernel handling */ 2228 ld r4, VCPU_KVM(r9) 2229 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2230 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2231 add r4, r4, r0 2232 ld r0, KVM_ENABLED_HCALLS(r4) 2233 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2234 srd r0, r0, r4 2235 andi. r0, r0, 1 2236 beq guest_exit_cont 2237 /* Get pointer to handler, if any, and call it */ 2238 LOAD_REG_ADDR(r4, hcall_real_table) 2239 lwax r3,r3,r4 2240 cmpwi r3,0 2241 beq guest_exit_cont 2242 add r12,r3,r4 2243 mtctr r12 2244 mr r3,r9 /* get vcpu pointer */ 2245 ld r4,VCPU_GPR(R4)(r9) 2246 bctrl 2247 cmpdi r3,H_TOO_HARD 2248 beq hcall_real_fallback 2249 ld r4,HSTATE_KVM_VCPU(r13) 2250 std r3,VCPU_GPR(R3)(r4) 2251 ld r10,VCPU_PC(r4) 2252 ld r11,VCPU_MSR(r4) 2253 b fast_guest_return 2254 2255sc_1_fast_return: 2256 mtspr SPRN_SRR0,r10 2257 mtspr SPRN_SRR1,r11 2258 li r10, BOOK3S_INTERRUPT_SYSCALL 2259 bl kvmppc_msr_interrupt 2260 mr r4,r9 2261 b fast_guest_return 2262 2263 /* We've attempted a real mode hcall, but it's punted it back 2264 * to userspace. We need to restore some clobbered volatiles 2265 * before resuming the pass-it-to-qemu path */ 2266hcall_real_fallback: 2267 li r12,BOOK3S_INTERRUPT_SYSCALL 2268 ld r9, HSTATE_KVM_VCPU(r13) 2269 2270 b guest_exit_cont 2271 2272 .globl hcall_real_table 2273hcall_real_table: 2274 .long 0 /* 0 - unused */ 2275 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2276 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2277 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2278 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2279 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2280 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2281 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2282 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2283 .long 0 /* 0x24 - H_SET_SPRG0 */ 2284 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2285 .long 0 /* 0x2c */ 2286 .long 0 /* 0x30 */ 2287 .long 0 /* 0x34 */ 2288 .long 0 /* 0x38 */ 2289 .long 0 /* 0x3c */ 2290 .long 0 /* 0x40 */ 2291 .long 0 /* 0x44 */ 2292 .long 0 /* 0x48 */ 2293 .long 0 /* 0x4c */ 2294 .long 0 /* 0x50 */ 2295 .long 0 /* 0x54 */ 2296 .long 0 /* 0x58 */ 2297 .long 0 /* 0x5c */ 2298 .long 0 /* 0x60 */ 2299#ifdef CONFIG_KVM_XICS 2300 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2301 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2302 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2303 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2304 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2305#else 2306 .long 0 /* 0x64 - H_EOI */ 2307 .long 0 /* 0x68 - H_CPPR */ 2308 .long 0 /* 0x6c - H_IPI */ 2309 .long 0 /* 0x70 - H_IPOLL */ 2310 .long 0 /* 0x74 - H_XIRR */ 2311#endif 2312 .long 0 /* 0x78 */ 2313 .long 0 /* 0x7c */ 2314 .long 0 /* 0x80 */ 2315 .long 0 /* 0x84 */ 2316 .long 0 /* 0x88 */ 2317 .long 0 /* 0x8c */ 2318 .long 0 /* 0x90 */ 2319 .long 0 /* 0x94 */ 2320 .long 0 /* 0x98 */ 2321 .long 0 /* 0x9c */ 2322 .long 0 /* 0xa0 */ 2323 .long 0 /* 0xa4 */ 2324 .long 0 /* 0xa8 */ 2325 .long 0 /* 0xac */ 2326 .long 0 /* 0xb0 */ 2327 .long 0 /* 0xb4 */ 2328 .long 0 /* 0xb8 */ 2329 .long 0 /* 0xbc */ 2330 .long 0 /* 0xc0 */ 2331 .long 0 /* 0xc4 */ 2332 .long 0 /* 0xc8 */ 2333 .long 0 /* 0xcc */ 2334 .long 0 /* 0xd0 */ 2335 .long 0 /* 0xd4 */ 2336 .long 0 /* 0xd8 */ 2337 .long 0 /* 0xdc */ 2338 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2339 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2340 .long 0 /* 0xe8 */ 2341 .long 0 /* 0xec */ 2342 .long 0 /* 0xf0 */ 2343 .long 0 /* 0xf4 */ 2344 .long 0 /* 0xf8 */ 2345 .long 0 /* 0xfc */ 2346 .long 0 /* 0x100 */ 2347 .long 0 /* 0x104 */ 2348 .long 0 /* 0x108 */ 2349 .long 0 /* 0x10c */ 2350 .long 0 /* 0x110 */ 2351 .long 0 /* 0x114 */ 2352 .long 0 /* 0x118 */ 2353 .long 0 /* 0x11c */ 2354 .long 0 /* 0x120 */ 2355 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2356 .long 0 /* 0x128 */ 2357 .long 0 /* 0x12c */ 2358 .long 0 /* 0x130 */ 2359 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2360 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2361 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2362 .long 0 /* 0x140 */ 2363 .long 0 /* 0x144 */ 2364 .long 0 /* 0x148 */ 2365 .long 0 /* 0x14c */ 2366 .long 0 /* 0x150 */ 2367 .long 0 /* 0x154 */ 2368 .long 0 /* 0x158 */ 2369 .long 0 /* 0x15c */ 2370 .long 0 /* 0x160 */ 2371 .long 0 /* 0x164 */ 2372 .long 0 /* 0x168 */ 2373 .long 0 /* 0x16c */ 2374 .long 0 /* 0x170 */ 2375 .long 0 /* 0x174 */ 2376 .long 0 /* 0x178 */ 2377 .long 0 /* 0x17c */ 2378 .long 0 /* 0x180 */ 2379 .long 0 /* 0x184 */ 2380 .long 0 /* 0x188 */ 2381 .long 0 /* 0x18c */ 2382 .long 0 /* 0x190 */ 2383 .long 0 /* 0x194 */ 2384 .long 0 /* 0x198 */ 2385 .long 0 /* 0x19c */ 2386 .long 0 /* 0x1a0 */ 2387 .long 0 /* 0x1a4 */ 2388 .long 0 /* 0x1a8 */ 2389 .long 0 /* 0x1ac */ 2390 .long 0 /* 0x1b0 */ 2391 .long 0 /* 0x1b4 */ 2392 .long 0 /* 0x1b8 */ 2393 .long 0 /* 0x1bc */ 2394 .long 0 /* 0x1c0 */ 2395 .long 0 /* 0x1c4 */ 2396 .long 0 /* 0x1c8 */ 2397 .long 0 /* 0x1cc */ 2398 .long 0 /* 0x1d0 */ 2399 .long 0 /* 0x1d4 */ 2400 .long 0 /* 0x1d8 */ 2401 .long 0 /* 0x1dc */ 2402 .long 0 /* 0x1e0 */ 2403 .long 0 /* 0x1e4 */ 2404 .long 0 /* 0x1e8 */ 2405 .long 0 /* 0x1ec */ 2406 .long 0 /* 0x1f0 */ 2407 .long 0 /* 0x1f4 */ 2408 .long 0 /* 0x1f8 */ 2409 .long 0 /* 0x1fc */ 2410 .long 0 /* 0x200 */ 2411 .long 0 /* 0x204 */ 2412 .long 0 /* 0x208 */ 2413 .long 0 /* 0x20c */ 2414 .long 0 /* 0x210 */ 2415 .long 0 /* 0x214 */ 2416 .long 0 /* 0x218 */ 2417 .long 0 /* 0x21c */ 2418 .long 0 /* 0x220 */ 2419 .long 0 /* 0x224 */ 2420 .long 0 /* 0x228 */ 2421 .long 0 /* 0x22c */ 2422 .long 0 /* 0x230 */ 2423 .long 0 /* 0x234 */ 2424 .long 0 /* 0x238 */ 2425 .long 0 /* 0x23c */ 2426 .long 0 /* 0x240 */ 2427 .long 0 /* 0x244 */ 2428 .long 0 /* 0x248 */ 2429 .long 0 /* 0x24c */ 2430 .long 0 /* 0x250 */ 2431 .long 0 /* 0x254 */ 2432 .long 0 /* 0x258 */ 2433 .long 0 /* 0x25c */ 2434 .long 0 /* 0x260 */ 2435 .long 0 /* 0x264 */ 2436 .long 0 /* 0x268 */ 2437 .long 0 /* 0x26c */ 2438 .long 0 /* 0x270 */ 2439 .long 0 /* 0x274 */ 2440 .long 0 /* 0x278 */ 2441 .long 0 /* 0x27c */ 2442 .long 0 /* 0x280 */ 2443 .long 0 /* 0x284 */ 2444 .long 0 /* 0x288 */ 2445 .long 0 /* 0x28c */ 2446 .long 0 /* 0x290 */ 2447 .long 0 /* 0x294 */ 2448 .long 0 /* 0x298 */ 2449 .long 0 /* 0x29c */ 2450 .long 0 /* 0x2a0 */ 2451 .long 0 /* 0x2a4 */ 2452 .long 0 /* 0x2a8 */ 2453 .long 0 /* 0x2ac */ 2454 .long 0 /* 0x2b0 */ 2455 .long 0 /* 0x2b4 */ 2456 .long 0 /* 0x2b8 */ 2457 .long 0 /* 0x2bc */ 2458 .long 0 /* 0x2c0 */ 2459 .long 0 /* 0x2c4 */ 2460 .long 0 /* 0x2c8 */ 2461 .long 0 /* 0x2cc */ 2462 .long 0 /* 0x2d0 */ 2463 .long 0 /* 0x2d4 */ 2464 .long 0 /* 0x2d8 */ 2465 .long 0 /* 0x2dc */ 2466 .long 0 /* 0x2e0 */ 2467 .long 0 /* 0x2e4 */ 2468 .long 0 /* 0x2e8 */ 2469 .long 0 /* 0x2ec */ 2470 .long 0 /* 0x2f0 */ 2471 .long 0 /* 0x2f4 */ 2472 .long 0 /* 0x2f8 */ 2473#ifdef CONFIG_KVM_XICS 2474 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2475#else 2476 .long 0 /* 0x2fc - H_XIRR_X*/ 2477#endif 2478 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2479 .globl hcall_real_table_end 2480hcall_real_table_end: 2481 2482_GLOBAL(kvmppc_h_set_xdabr) 2483 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2484 beq 6f 2485 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2486 andc. r0, r5, r0 2487 beq 3f 24886: li r3, H_PARAMETER 2489 blr 2490 2491_GLOBAL(kvmppc_h_set_dabr) 2492 li r5, DABRX_USER | DABRX_KERNEL 24933: 2494BEGIN_FTR_SECTION 2495 b 2f 2496END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2497 std r4,VCPU_DABR(r3) 2498 stw r5, VCPU_DABRX(r3) 2499 mtspr SPRN_DABRX, r5 2500 /* Work around P7 bug where DABR can get corrupted on mtspr */ 25011: mtspr SPRN_DABR,r4 2502 mfspr r5, SPRN_DABR 2503 cmpd r4, r5 2504 bne 1b 2505 isync 2506 li r3,0 2507 blr 2508 2509 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 25102: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2511 rlwimi r5, r4, 2, DAWRX_WT 2512 clrrdi r4, r4, 3 2513 std r4, VCPU_DAWR(r3) 2514 std r5, VCPU_DAWRX(r3) 2515 mtspr SPRN_DAWR, r4 2516 mtspr SPRN_DAWRX, r5 2517 li r3, 0 2518 blr 2519 2520_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2521 ori r11,r11,MSR_EE 2522 std r11,VCPU_MSR(r3) 2523 li r0,1 2524 stb r0,VCPU_CEDED(r3) 2525 sync /* order setting ceded vs. testing prodded */ 2526 lbz r5,VCPU_PRODDED(r3) 2527 cmpwi r5,0 2528 bne kvm_cede_prodded 2529 li r12,0 /* set trap to 0 to say hcall is handled */ 2530 stw r12,VCPU_TRAP(r3) 2531 li r0,H_SUCCESS 2532 std r0,VCPU_GPR(R3)(r3) 2533 2534 /* 2535 * Set our bit in the bitmask of napping threads unless all the 2536 * other threads are already napping, in which case we send this 2537 * up to the host. 2538 */ 2539 ld r5,HSTATE_KVM_VCORE(r13) 2540 lbz r6,HSTATE_PTID(r13) 2541 lwz r8,VCORE_ENTRY_EXIT(r5) 2542 clrldi r8,r8,56 2543 li r0,1 2544 sld r0,r0,r6 2545 addi r6,r5,VCORE_NAPPING_THREADS 254631: lwarx r4,0,r6 2547 or r4,r4,r0 2548 cmpw r4,r8 2549 beq kvm_cede_exit 2550 stwcx. r4,0,r6 2551 bne 31b 2552 /* order napping_threads update vs testing entry_exit_map */ 2553 isync 2554 li r0,NAPPING_CEDE 2555 stb r0,HSTATE_NAPPING(r13) 2556 lwz r7,VCORE_ENTRY_EXIT(r5) 2557 cmpwi r7,0x100 2558 bge 33f /* another thread already exiting */ 2559 2560/* 2561 * Although not specifically required by the architecture, POWER7 2562 * preserves the following registers in nap mode, even if an SMT mode 2563 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2564 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2565 */ 2566 /* Save non-volatile GPRs */ 2567 std r14, VCPU_GPR(R14)(r3) 2568 std r15, VCPU_GPR(R15)(r3) 2569 std r16, VCPU_GPR(R16)(r3) 2570 std r17, VCPU_GPR(R17)(r3) 2571 std r18, VCPU_GPR(R18)(r3) 2572 std r19, VCPU_GPR(R19)(r3) 2573 std r20, VCPU_GPR(R20)(r3) 2574 std r21, VCPU_GPR(R21)(r3) 2575 std r22, VCPU_GPR(R22)(r3) 2576 std r23, VCPU_GPR(R23)(r3) 2577 std r24, VCPU_GPR(R24)(r3) 2578 std r25, VCPU_GPR(R25)(r3) 2579 std r26, VCPU_GPR(R26)(r3) 2580 std r27, VCPU_GPR(R27)(r3) 2581 std r28, VCPU_GPR(R28)(r3) 2582 std r29, VCPU_GPR(R29)(r3) 2583 std r30, VCPU_GPR(R30)(r3) 2584 std r31, VCPU_GPR(R31)(r3) 2585 2586 /* save FP state */ 2587 bl kvmppc_save_fp 2588 2589#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2590BEGIN_FTR_SECTION 2591 /* 2592 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2593 */ 2594 ld r9, HSTATE_KVM_VCPU(r13) 2595 bl kvmppc_save_tm 2596END_FTR_SECTION_IFSET(CPU_FTR_TM) 2597#endif 2598 2599 /* 2600 * Set DEC to the smaller of DEC and HDEC, so that we wake 2601 * no later than the end of our timeslice (HDEC interrupts 2602 * don't wake us from nap). 2603 */ 2604 mfspr r3, SPRN_DEC 2605 mfspr r4, SPRN_HDEC 2606 mftb r5 2607BEGIN_FTR_SECTION 2608 /* On P9 check whether the guest has large decrementer mode enabled */ 2609 ld r6, HSTATE_KVM_VCORE(r13) 2610 ld r6, VCORE_LPCR(r6) 2611 andis. r6, r6, LPCR_LD@h 2612 bne 68f 2613END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2614 extsw r3, r3 261568: EXTEND_HDEC(r4) 2616 cmpd r3, r4 2617 ble 67f 2618 mtspr SPRN_DEC, r4 261967: 2620 /* save expiry time of guest decrementer */ 2621 add r3, r3, r5 2622 ld r4, HSTATE_KVM_VCPU(r13) 2623 ld r5, HSTATE_KVM_VCORE(r13) 2624 ld r6, VCORE_TB_OFFSET(r5) 2625 subf r3, r6, r3 /* convert to host TB value */ 2626 std r3, VCPU_DEC_EXPIRES(r4) 2627 2628#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2629 ld r4, HSTATE_KVM_VCPU(r13) 2630 addi r3, r4, VCPU_TB_CEDE 2631 bl kvmhv_accumulate_time 2632#endif 2633 2634 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2635 2636 /* 2637 * Take a nap until a decrementer or external or doobell interrupt 2638 * occurs, with PECE1 and PECE0 set in LPCR. 2639 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2640 * Also clear the runlatch bit before napping. 2641 */ 2642kvm_do_nap: 2643 mfspr r0, SPRN_CTRLF 2644 clrrdi r0, r0, 1 2645 mtspr SPRN_CTRLT, r0 2646 2647 li r0,1 2648 stb r0,HSTATE_HWTHREAD_REQ(r13) 2649 mfspr r5,SPRN_LPCR 2650 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2651BEGIN_FTR_SECTION 2652 ori r5, r5, LPCR_PECEDH 2653 rlwimi r5, r3, 0, LPCR_PECEDP 2654END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2655 2656kvm_nap_sequence: /* desired LPCR value in r5 */ 2657BEGIN_FTR_SECTION 2658 /* 2659 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2660 * enable state loss = 1 (allow SMT mode switch) 2661 * requested level = 0 (just stop dispatching) 2662 */ 2663 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2664 mtspr SPRN_PSSCR, r3 2665 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2666 li r4, LPCR_PECE_HVEE@higher 2667 sldi r4, r4, 32 2668 or r5, r5, r4 2669END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2670 mtspr SPRN_LPCR,r5 2671 isync 2672 li r0, 0 2673 std r0, HSTATE_SCRATCH0(r13) 2674 ptesync 2675 ld r0, HSTATE_SCRATCH0(r13) 26761: cmpd r0, r0 2677 bne 1b 2678BEGIN_FTR_SECTION 2679 nap 2680FTR_SECTION_ELSE 2681 PPC_STOP 2682ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2683 b . 2684 268533: mr r4, r3 2686 li r3, 0 2687 li r12, 0 2688 b 34f 2689 2690kvm_end_cede: 2691 /* get vcpu pointer */ 2692 ld r4, HSTATE_KVM_VCPU(r13) 2693 2694 /* Woken by external or decrementer interrupt */ 2695 ld r1, HSTATE_HOST_R1(r13) 2696 2697#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2698 addi r3, r4, VCPU_TB_RMINTR 2699 bl kvmhv_accumulate_time 2700#endif 2701 2702#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2703BEGIN_FTR_SECTION 2704 /* 2705 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2706 */ 2707 bl kvmppc_restore_tm 2708END_FTR_SECTION_IFSET(CPU_FTR_TM) 2709#endif 2710 2711 /* load up FP state */ 2712 bl kvmppc_load_fp 2713 2714 /* Restore guest decrementer */ 2715 ld r3, VCPU_DEC_EXPIRES(r4) 2716 ld r5, HSTATE_KVM_VCORE(r13) 2717 ld r6, VCORE_TB_OFFSET(r5) 2718 add r3, r3, r6 /* convert host TB to guest TB value */ 2719 mftb r7 2720 subf r3, r7, r3 2721 mtspr SPRN_DEC, r3 2722 2723 /* Load NV GPRS */ 2724 ld r14, VCPU_GPR(R14)(r4) 2725 ld r15, VCPU_GPR(R15)(r4) 2726 ld r16, VCPU_GPR(R16)(r4) 2727 ld r17, VCPU_GPR(R17)(r4) 2728 ld r18, VCPU_GPR(R18)(r4) 2729 ld r19, VCPU_GPR(R19)(r4) 2730 ld r20, VCPU_GPR(R20)(r4) 2731 ld r21, VCPU_GPR(R21)(r4) 2732 ld r22, VCPU_GPR(R22)(r4) 2733 ld r23, VCPU_GPR(R23)(r4) 2734 ld r24, VCPU_GPR(R24)(r4) 2735 ld r25, VCPU_GPR(R25)(r4) 2736 ld r26, VCPU_GPR(R26)(r4) 2737 ld r27, VCPU_GPR(R27)(r4) 2738 ld r28, VCPU_GPR(R28)(r4) 2739 ld r29, VCPU_GPR(R29)(r4) 2740 ld r30, VCPU_GPR(R30)(r4) 2741 ld r31, VCPU_GPR(R31)(r4) 2742 2743 /* Check the wake reason in SRR1 to see why we got here */ 2744 bl kvmppc_check_wake_reason 2745 2746 /* 2747 * Restore volatile registers since we could have called a 2748 * C routine in kvmppc_check_wake_reason 2749 * r4 = VCPU 2750 * r3 tells us whether we need to return to host or not 2751 * WARNING: it gets checked further down: 2752 * should not modify r3 until this check is done. 2753 */ 2754 ld r4, HSTATE_KVM_VCPU(r13) 2755 2756 /* clear our bit in vcore->napping_threads */ 275734: ld r5,HSTATE_KVM_VCORE(r13) 2758 lbz r7,HSTATE_PTID(r13) 2759 li r0,1 2760 sld r0,r0,r7 2761 addi r6,r5,VCORE_NAPPING_THREADS 276232: lwarx r7,0,r6 2763 andc r7,r7,r0 2764 stwcx. r7,0,r6 2765 bne 32b 2766 li r0,0 2767 stb r0,HSTATE_NAPPING(r13) 2768 2769 /* See if the wake reason saved in r3 means we need to exit */ 2770 stw r12, VCPU_TRAP(r4) 2771 mr r9, r4 2772 cmpdi r3, 0 2773 bgt guest_exit_cont 2774 2775 /* see if any other thread is already exiting */ 2776 lwz r0,VCORE_ENTRY_EXIT(r5) 2777 cmpwi r0,0x100 2778 bge guest_exit_cont 2779 2780 b kvmppc_cede_reentry /* if not go back to guest */ 2781 2782 /* cede when already previously prodded case */ 2783kvm_cede_prodded: 2784 li r0,0 2785 stb r0,VCPU_PRODDED(r3) 2786 sync /* order testing prodded vs. clearing ceded */ 2787 stb r0,VCPU_CEDED(r3) 2788 li r3,H_SUCCESS 2789 blr 2790 2791 /* we've ceded but we want to give control to the host */ 2792kvm_cede_exit: 2793 ld r9, HSTATE_KVM_VCPU(r13) 2794#ifdef CONFIG_KVM_XICS 2795 /* Abort if we still have a pending escalation */ 2796 lbz r5, VCPU_XIVE_ESC_ON(r9) 2797 cmpwi r5, 0 2798 beq 1f 2799 li r0, 0 2800 stb r0, VCPU_CEDED(r9) 28011: /* Enable XIVE escalation */ 2802 li r5, XIVE_ESB_SET_PQ_00 2803 mfmsr r0 2804 andi. r0, r0, MSR_DR /* in real mode? */ 2805 beq 1f 2806 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2807 cmpdi r10, 0 2808 beq 3f 2809 ldx r0, r10, r5 2810 b 2f 28111: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2812 cmpdi r10, 0 2813 beq 3f 2814 ldcix r0, r10, r5 28152: sync 2816 li r0, 1 2817 stb r0, VCPU_XIVE_ESC_ON(r9) 2818#endif /* CONFIG_KVM_XICS */ 28193: b guest_exit_cont 2820 2821 /* Try to handle a machine check in real mode */ 2822machine_check_realmode: 2823 mr r3, r9 /* get vcpu pointer */ 2824 bl kvmppc_realmode_machine_check 2825 nop 2826 ld r9, HSTATE_KVM_VCPU(r13) 2827 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2828 /* 2829 * For the guest that is FWNMI capable, deliver all the MCE errors 2830 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2831 * reason. This new approach injects machine check errors in guest 2832 * address space to guest with additional information in the form 2833 * of RTAS event, thus enabling guest kernel to suitably handle 2834 * such errors. 2835 * 2836 * For the guest that is not FWNMI capable (old QEMU) fallback 2837 * to old behaviour for backward compatibility: 2838 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2839 * through machine check interrupt (set HSRR0 to 0x200). 2840 * For handled errors (no-fatal), just go back to guest execution 2841 * with current HSRR0. 2842 * if we receive machine check with MSR(RI=0) then deliver it to 2843 * guest as machine check causing guest to crash. 2844 */ 2845 ld r11, VCPU_MSR(r9) 2846 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2847 bne mc_cont /* if so, exit to host */ 2848 /* Check if guest is capable of handling NMI exit */ 2849 ld r10, VCPU_KVM(r9) 2850 lbz r10, KVM_FWNMI(r10) 2851 cmpdi r10, 1 /* FWNMI capable? */ 2852 beq mc_cont /* if so, exit with KVM_EXIT_NMI. */ 2853 2854 /* if not, fall through for backward compatibility. */ 2855 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2856 beq 1f /* Deliver a machine check to guest */ 2857 ld r10, VCPU_PC(r9) 2858 cmpdi r3, 0 /* Did we handle MCE ? */ 2859 bne 2f /* Continue guest execution. */ 2860 /* If not, deliver a machine check. SRR0/1 are already set */ 28611: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2862 bl kvmppc_msr_interrupt 28632: b fast_interrupt_c_return 2864 2865/* 2866 * Check the reason we woke from nap, and take appropriate action. 2867 * Returns (in r3): 2868 * 0 if nothing needs to be done 2869 * 1 if something happened that needs to be handled by the host 2870 * -1 if there was a guest wakeup (IPI or msgsnd) 2871 * -2 if we handled a PCI passthrough interrupt (returned by 2872 * kvmppc_read_intr only) 2873 * 2874 * Also sets r12 to the interrupt vector for any interrupt that needs 2875 * to be handled now by the host (0x500 for external interrupt), or zero. 2876 * Modifies all volatile registers (since it may call a C function). 2877 * This routine calls kvmppc_read_intr, a C function, if an external 2878 * interrupt is pending. 2879 */ 2880kvmppc_check_wake_reason: 2881 mfspr r6, SPRN_SRR1 2882BEGIN_FTR_SECTION 2883 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2884FTR_SECTION_ELSE 2885 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2886ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2887 cmpwi r6, 8 /* was it an external interrupt? */ 2888 beq 7f /* if so, see what it was */ 2889 li r3, 0 2890 li r12, 0 2891 cmpwi r6, 6 /* was it the decrementer? */ 2892 beq 0f 2893BEGIN_FTR_SECTION 2894 cmpwi r6, 5 /* privileged doorbell? */ 2895 beq 0f 2896 cmpwi r6, 3 /* hypervisor doorbell? */ 2897 beq 3f 2898END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2899 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2900 beq 4f 2901 li r3, 1 /* anything else, return 1 */ 29020: blr 2903 2904 /* hypervisor doorbell */ 29053: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2906 2907 /* 2908 * Clear the doorbell as we will invoke the handler 2909 * explicitly in the guest exit path. 2910 */ 2911 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2912 PPC_MSGCLR(6) 2913 /* see if it's a host IPI */ 2914 li r3, 1 2915BEGIN_FTR_SECTION 2916 PPC_MSGSYNC 2917 lwsync 2918END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2919 lbz r0, HSTATE_HOST_IPI(r13) 2920 cmpwi r0, 0 2921 bnelr 2922 /* if not, return -1 */ 2923 li r3, -1 2924 blr 2925 2926 /* Woken up due to Hypervisor maintenance interrupt */ 29274: li r12, BOOK3S_INTERRUPT_HMI 2928 li r3, 1 2929 blr 2930 2931 /* external interrupt - create a stack frame so we can call C */ 29327: mflr r0 2933 std r0, PPC_LR_STKOFF(r1) 2934 stdu r1, -PPC_MIN_STKFRM(r1) 2935 bl kvmppc_read_intr 2936 nop 2937 li r12, BOOK3S_INTERRUPT_EXTERNAL 2938 cmpdi r3, 1 2939 ble 1f 2940 2941 /* 2942 * Return code of 2 means PCI passthrough interrupt, but 2943 * we need to return back to host to complete handling the 2944 * interrupt. Trap reason is expected in r12 by guest 2945 * exit code. 2946 */ 2947 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29481: 2949 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2950 addi r1, r1, PPC_MIN_STKFRM 2951 mtlr r0 2952 blr 2953 2954/* 2955 * Save away FP, VMX and VSX registers. 2956 * r3 = vcpu pointer 2957 * N.B. r30 and r31 are volatile across this function, 2958 * thus it is not callable from C. 2959 */ 2960kvmppc_save_fp: 2961 mflr r30 2962 mr r31,r3 2963 mfmsr r5 2964 ori r8,r5,MSR_FP 2965#ifdef CONFIG_ALTIVEC 2966BEGIN_FTR_SECTION 2967 oris r8,r8,MSR_VEC@h 2968END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2969#endif 2970#ifdef CONFIG_VSX 2971BEGIN_FTR_SECTION 2972 oris r8,r8,MSR_VSX@h 2973END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2974#endif 2975 mtmsrd r8 2976 addi r3,r3,VCPU_FPRS 2977 bl store_fp_state 2978#ifdef CONFIG_ALTIVEC 2979BEGIN_FTR_SECTION 2980 addi r3,r31,VCPU_VRS 2981 bl store_vr_state 2982END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2983#endif 2984 mfspr r6,SPRN_VRSAVE 2985 stw r6,VCPU_VRSAVE(r31) 2986 mtlr r30 2987 blr 2988 2989/* 2990 * Load up FP, VMX and VSX registers 2991 * r4 = vcpu pointer 2992 * N.B. r30 and r31 are volatile across this function, 2993 * thus it is not callable from C. 2994 */ 2995kvmppc_load_fp: 2996 mflr r30 2997 mr r31,r4 2998 mfmsr r9 2999 ori r8,r9,MSR_FP 3000#ifdef CONFIG_ALTIVEC 3001BEGIN_FTR_SECTION 3002 oris r8,r8,MSR_VEC@h 3003END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3004#endif 3005#ifdef CONFIG_VSX 3006BEGIN_FTR_SECTION 3007 oris r8,r8,MSR_VSX@h 3008END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3009#endif 3010 mtmsrd r8 3011 addi r3,r4,VCPU_FPRS 3012 bl load_fp_state 3013#ifdef CONFIG_ALTIVEC 3014BEGIN_FTR_SECTION 3015 addi r3,r31,VCPU_VRS 3016 bl load_vr_state 3017END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3018#endif 3019 lwz r7,VCPU_VRSAVE(r31) 3020 mtspr SPRN_VRSAVE,r7 3021 mtlr r30 3022 mr r4,r31 3023 blr 3024 3025#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3026/* 3027 * Save transactional state and TM-related registers. 3028 * Called with r9 pointing to the vcpu struct. 3029 * This can modify all checkpointed registers, but 3030 * restores r1, r2 and r9 (vcpu pointer) before exit. 3031 */ 3032kvmppc_save_tm: 3033 mflr r0 3034 std r0, PPC_LR_STKOFF(r1) 3035 3036 /* Turn on TM. */ 3037 mfmsr r8 3038 li r0, 1 3039 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3040 mtmsrd r8 3041 3042 ld r5, VCPU_MSR(r9) 3043 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3044 beq 1f /* TM not active in guest. */ 3045 3046 std r1, HSTATE_HOST_R1(r13) 3047 li r3, TM_CAUSE_KVM_RESCHED 3048 3049 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3050 li r5, 0 3051 mtmsrd r5, 1 3052 3053 /* All GPRs are volatile at this point. */ 3054 TRECLAIM(R3) 3055 3056 /* Temporarily store r13 and r9 so we have some regs to play with */ 3057 SET_SCRATCH0(r13) 3058 GET_PACA(r13) 3059 std r9, PACATMSCRATCH(r13) 3060 ld r9, HSTATE_KVM_VCPU(r13) 3061 3062 /* Get a few more GPRs free. */ 3063 std r29, VCPU_GPRS_TM(29)(r9) 3064 std r30, VCPU_GPRS_TM(30)(r9) 3065 std r31, VCPU_GPRS_TM(31)(r9) 3066 3067 /* Save away PPR and DSCR soon so don't run with user values. */ 3068 mfspr r31, SPRN_PPR 3069 HMT_MEDIUM 3070 mfspr r30, SPRN_DSCR 3071 ld r29, HSTATE_DSCR(r13) 3072 mtspr SPRN_DSCR, r29 3073 3074 /* Save all but r9, r13 & r29-r31 */ 3075 reg = 0 3076 .rept 29 3077 .if (reg != 9) && (reg != 13) 3078 std reg, VCPU_GPRS_TM(reg)(r9) 3079 .endif 3080 reg = reg + 1 3081 .endr 3082 /* ... now save r13 */ 3083 GET_SCRATCH0(r4) 3084 std r4, VCPU_GPRS_TM(13)(r9) 3085 /* ... and save r9 */ 3086 ld r4, PACATMSCRATCH(r13) 3087 std r4, VCPU_GPRS_TM(9)(r9) 3088 3089 /* Reload stack pointer and TOC. */ 3090 ld r1, HSTATE_HOST_R1(r13) 3091 ld r2, PACATOC(r13) 3092 3093 /* Set MSR RI now we have r1 and r13 back. */ 3094 li r5, MSR_RI 3095 mtmsrd r5, 1 3096 3097 /* Save away checkpinted SPRs. */ 3098 std r31, VCPU_PPR_TM(r9) 3099 std r30, VCPU_DSCR_TM(r9) 3100 mflr r5 3101 mfcr r6 3102 mfctr r7 3103 mfspr r8, SPRN_AMR 3104 mfspr r10, SPRN_TAR 3105 mfxer r11 3106 std r5, VCPU_LR_TM(r9) 3107 stw r6, VCPU_CR_TM(r9) 3108 std r7, VCPU_CTR_TM(r9) 3109 std r8, VCPU_AMR_TM(r9) 3110 std r10, VCPU_TAR_TM(r9) 3111 std r11, VCPU_XER_TM(r9) 3112 3113 /* Restore r12 as trap number. */ 3114 lwz r12, VCPU_TRAP(r9) 3115 3116 /* Save FP/VSX. */ 3117 addi r3, r9, VCPU_FPRS_TM 3118 bl store_fp_state 3119 addi r3, r9, VCPU_VRS_TM 3120 bl store_vr_state 3121 mfspr r6, SPRN_VRSAVE 3122 stw r6, VCPU_VRSAVE_TM(r9) 31231: 3124 /* 3125 * We need to save these SPRs after the treclaim so that the software 3126 * error code is recorded correctly in the TEXASR. Also the user may 3127 * change these outside of a transaction, so they must always be 3128 * context switched. 3129 */ 3130 mfspr r5, SPRN_TFHAR 3131 mfspr r6, SPRN_TFIAR 3132 mfspr r7, SPRN_TEXASR 3133 std r5, VCPU_TFHAR(r9) 3134 std r6, VCPU_TFIAR(r9) 3135 std r7, VCPU_TEXASR(r9) 3136 3137 ld r0, PPC_LR_STKOFF(r1) 3138 mtlr r0 3139 blr 3140 3141/* 3142 * Restore transactional state and TM-related registers. 3143 * Called with r4 pointing to the vcpu struct. 3144 * This potentially modifies all checkpointed registers. 3145 * It restores r1, r2, r4 from the PACA. 3146 */ 3147kvmppc_restore_tm: 3148 mflr r0 3149 std r0, PPC_LR_STKOFF(r1) 3150 3151 /* Turn on TM/FP/VSX/VMX so we can restore them. */ 3152 mfmsr r5 3153 li r6, MSR_TM >> 32 3154 sldi r6, r6, 32 3155 or r5, r5, r6 3156 ori r5, r5, MSR_FP 3157 oris r5, r5, (MSR_VEC | MSR_VSX)@h 3158 mtmsrd r5 3159 3160 /* 3161 * The user may change these outside of a transaction, so they must 3162 * always be context switched. 3163 */ 3164 ld r5, VCPU_TFHAR(r4) 3165 ld r6, VCPU_TFIAR(r4) 3166 ld r7, VCPU_TEXASR(r4) 3167 mtspr SPRN_TFHAR, r5 3168 mtspr SPRN_TFIAR, r6 3169 mtspr SPRN_TEXASR, r7 3170 3171 ld r5, VCPU_MSR(r4) 3172 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 3173 beqlr /* TM not active in guest */ 3174 std r1, HSTATE_HOST_R1(r13) 3175 3176 /* Make sure the failure summary is set, otherwise we'll program check 3177 * when we trechkpt. It's possible that this might have been not set 3178 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the 3179 * host. 3180 */ 3181 oris r7, r7, (TEXASR_FS)@h 3182 mtspr SPRN_TEXASR, r7 3183 3184 /* 3185 * We need to load up the checkpointed state for the guest. 3186 * We need to do this early as it will blow away any GPRs, VSRs and 3187 * some SPRs. 3188 */ 3189 3190 mr r31, r4 3191 addi r3, r31, VCPU_FPRS_TM 3192 bl load_fp_state 3193 addi r3, r31, VCPU_VRS_TM 3194 bl load_vr_state 3195 mr r4, r31 3196 lwz r7, VCPU_VRSAVE_TM(r4) 3197 mtspr SPRN_VRSAVE, r7 3198 3199 ld r5, VCPU_LR_TM(r4) 3200 lwz r6, VCPU_CR_TM(r4) 3201 ld r7, VCPU_CTR_TM(r4) 3202 ld r8, VCPU_AMR_TM(r4) 3203 ld r9, VCPU_TAR_TM(r4) 3204 ld r10, VCPU_XER_TM(r4) 3205 mtlr r5 3206 mtcr r6 3207 mtctr r7 3208 mtspr SPRN_AMR, r8 3209 mtspr SPRN_TAR, r9 3210 mtxer r10 3211 3212 /* 3213 * Load up PPR and DSCR values but don't put them in the actual SPRs 3214 * till the last moment to avoid running with userspace PPR and DSCR for 3215 * too long. 3216 */ 3217 ld r29, VCPU_DSCR_TM(r4) 3218 ld r30, VCPU_PPR_TM(r4) 3219 3220 std r2, PACATMSCRATCH(r13) /* Save TOC */ 3221 3222 /* Clear the MSR RI since r1, r13 are all going to be foobar. */ 3223 li r5, 0 3224 mtmsrd r5, 1 3225 3226 /* Load GPRs r0-r28 */ 3227 reg = 0 3228 .rept 29 3229 ld reg, VCPU_GPRS_TM(reg)(r31) 3230 reg = reg + 1 3231 .endr 3232 3233 mtspr SPRN_DSCR, r29 3234 mtspr SPRN_PPR, r30 3235 3236 /* Load final GPRs */ 3237 ld 29, VCPU_GPRS_TM(29)(r31) 3238 ld 30, VCPU_GPRS_TM(30)(r31) 3239 ld 31, VCPU_GPRS_TM(31)(r31) 3240 3241 /* TM checkpointed state is now setup. All GPRs are now volatile. */ 3242 TRECHKPT 3243 3244 /* Now let's get back the state we need. */ 3245 HMT_MEDIUM 3246 GET_PACA(r13) 3247 ld r29, HSTATE_DSCR(r13) 3248 mtspr SPRN_DSCR, r29 3249 ld r4, HSTATE_KVM_VCPU(r13) 3250 ld r1, HSTATE_HOST_R1(r13) 3251 ld r2, PACATMSCRATCH(r13) 3252 3253 /* Set the MSR RI since we have our registers back. */ 3254 li r5, MSR_RI 3255 mtmsrd r5, 1 3256 3257 ld r0, PPC_LR_STKOFF(r1) 3258 mtlr r0 3259 blr 3260#endif 3261 3262/* 3263 * We come here if we get any exception or interrupt while we are 3264 * executing host real mode code while in guest MMU context. 3265 * r12 is (CR << 32) | vector 3266 * r13 points to our PACA 3267 * r12 is saved in HSTATE_SCRATCH0(r13) 3268 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3269 * r9 is saved in HSTATE_SCRATCH2(r13) 3270 * r13 is saved in HSPRG1 3271 * cfar is saved in HSTATE_CFAR(r13) 3272 * ppr is saved in HSTATE_PPR(r13) 3273 */ 3274kvmppc_bad_host_intr: 3275 /* 3276 * Switch to the emergency stack, but start half-way down in 3277 * case we were already on it. 3278 */ 3279 mr r9, r1 3280 std r1, PACAR1(r13) 3281 ld r1, PACAEMERGSP(r13) 3282 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3283 std r9, 0(r1) 3284 std r0, GPR0(r1) 3285 std r9, GPR1(r1) 3286 std r2, GPR2(r1) 3287 SAVE_4GPRS(3, r1) 3288 SAVE_2GPRS(7, r1) 3289 srdi r0, r12, 32 3290 clrldi r12, r12, 32 3291 std r0, _CCR(r1) 3292 std r12, _TRAP(r1) 3293 andi. r0, r12, 2 3294 beq 1f 3295 mfspr r3, SPRN_HSRR0 3296 mfspr r4, SPRN_HSRR1 3297 mfspr r5, SPRN_HDAR 3298 mfspr r6, SPRN_HDSISR 3299 b 2f 33001: mfspr r3, SPRN_SRR0 3301 mfspr r4, SPRN_SRR1 3302 mfspr r5, SPRN_DAR 3303 mfspr r6, SPRN_DSISR 33042: std r3, _NIP(r1) 3305 std r4, _MSR(r1) 3306 std r5, _DAR(r1) 3307 std r6, _DSISR(r1) 3308 ld r9, HSTATE_SCRATCH2(r13) 3309 ld r12, HSTATE_SCRATCH0(r13) 3310 GET_SCRATCH0(r0) 3311 SAVE_4GPRS(9, r1) 3312 std r0, GPR13(r1) 3313 SAVE_NVGPRS(r1) 3314 ld r5, HSTATE_CFAR(r13) 3315 std r5, ORIG_GPR3(r1) 3316 mflr r3 3317#ifdef CONFIG_RELOCATABLE 3318 ld r4, HSTATE_SCRATCH1(r13) 3319#else 3320 mfctr r4 3321#endif 3322 mfxer r5 3323 lbz r6, PACAIRQSOFTMASK(r13) 3324 std r3, _LINK(r1) 3325 std r4, _CTR(r1) 3326 std r5, _XER(r1) 3327 std r6, SOFTE(r1) 3328 ld r2, PACATOC(r13) 3329 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3330 std r3, STACK_FRAME_OVERHEAD-16(r1) 3331 3332 /* 3333 * On POWER9 do a minimal restore of the MMU and call C code, 3334 * which will print a message and panic. 3335 * XXX On POWER7 and POWER8, we just spin here since we don't 3336 * know what the other threads are doing (and we don't want to 3337 * coordinate with them) - but at least we now have register state 3338 * in memory that we might be able to look at from another CPU. 3339 */ 3340BEGIN_FTR_SECTION 3341 b . 3342END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3343 ld r9, HSTATE_KVM_VCPU(r13) 3344 ld r10, VCPU_KVM(r9) 3345 3346 li r0, 0 3347 mtspr SPRN_AMR, r0 3348 mtspr SPRN_IAMR, r0 3349 mtspr SPRN_CIABR, r0 3350 mtspr SPRN_DAWRX, r0 3351 3352 /* Flush the ERAT on radix P9 DD1 guest exit */ 3353BEGIN_FTR_SECTION 3354 PPC_INVALIDATE_ERAT 3355END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) 3356 3357BEGIN_MMU_FTR_SECTION 3358 b 4f 3359END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3360 3361 slbmte r0, r0 3362 slbia 3363 ptesync 3364 ld r8, PACA_SLBSHADOWPTR(r13) 3365 .rept SLB_NUM_BOLTED 3366 li r3, SLBSHADOW_SAVEAREA 3367 LDX_BE r5, r8, r3 3368 addi r3, r3, 8 3369 LDX_BE r6, r8, r3 3370 andis. r7, r5, SLB_ESID_V@h 3371 beq 3f 3372 slbmte r6, r5 33733: addi r8, r8, 16 3374 .endr 3375 33764: lwz r7, KVM_HOST_LPID(r10) 3377 mtspr SPRN_LPID, r7 3378 mtspr SPRN_PID, r0 3379 ld r8, KVM_HOST_LPCR(r10) 3380 mtspr SPRN_LPCR, r8 3381 isync 3382 li r0, KVM_GUEST_MODE_NONE 3383 stb r0, HSTATE_IN_GUEST(r13) 3384 3385 /* 3386 * Turn on the MMU and jump to C code 3387 */ 3388 bcl 20, 31, .+4 33895: mflr r3 3390 addi r3, r3, 9f - 5b 3391 ld r4, PACAKMSR(r13) 3392 mtspr SPRN_SRR0, r3 3393 mtspr SPRN_SRR1, r4 3394 RFI_TO_KERNEL 33959: addi r3, r1, STACK_FRAME_OVERHEAD 3396 bl kvmppc_bad_interrupt 3397 b 9b 3398 3399/* 3400 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3401 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3402 * r11 has the guest MSR value (in/out) 3403 * r9 has a vcpu pointer (in) 3404 * r0 is used as a scratch register 3405 */ 3406kvmppc_msr_interrupt: 3407 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3408 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3409 ld r11, VCPU_INTR_MSR(r9) 3410 bne 1f 3411 /* ... if transactional, change to suspended */ 3412 li r0, 1 34131: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3414 blr 3415 3416/* 3417 * This works around a hardware bug on POWER8E processors, where 3418 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3419 * performance monitor interrupt. Instead, when we need to have 3420 * an interrupt pending, we have to arrange for a counter to overflow. 3421 */ 3422kvmppc_fix_pmao: 3423 li r3, 0 3424 mtspr SPRN_MMCR2, r3 3425 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3426 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3427 mtspr SPRN_MMCR0, r3 3428 lis r3, 0x7fff 3429 ori r3, r3, 0xffff 3430 mtspr SPRN_PMC6, r3 3431 isync 3432 blr 3433 3434#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3435/* 3436 * Start timing an activity 3437 * r3 = pointer to time accumulation struct, r4 = vcpu 3438 */ 3439kvmhv_start_timing: 3440 ld r5, HSTATE_KVM_VCORE(r13) 3441 lbz r6, VCORE_IN_GUEST(r5) 3442 cmpwi r6, 0 3443 beq 5f /* if in guest, need to */ 3444 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 34455: mftb r5 3446 subf r5, r6, r5 3447 std r3, VCPU_CUR_ACTIVITY(r4) 3448 std r5, VCPU_ACTIVITY_START(r4) 3449 blr 3450 3451/* 3452 * Accumulate time to one activity and start another. 3453 * r3 = pointer to new time accumulation struct, r4 = vcpu 3454 */ 3455kvmhv_accumulate_time: 3456 ld r5, HSTATE_KVM_VCORE(r13) 3457 lbz r8, VCORE_IN_GUEST(r5) 3458 cmpwi r8, 0 3459 beq 4f /* if in guest, need to */ 3460 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */ 34614: ld r5, VCPU_CUR_ACTIVITY(r4) 3462 ld r6, VCPU_ACTIVITY_START(r4) 3463 std r3, VCPU_CUR_ACTIVITY(r4) 3464 mftb r7 3465 subf r7, r8, r7 3466 std r7, VCPU_ACTIVITY_START(r4) 3467 cmpdi r5, 0 3468 beqlr 3469 subf r3, r6, r7 3470 ld r8, TAS_SEQCOUNT(r5) 3471 cmpdi r8, 0 3472 addi r8, r8, 1 3473 std r8, TAS_SEQCOUNT(r5) 3474 lwsync 3475 ld r7, TAS_TOTAL(r5) 3476 add r7, r7, r3 3477 std r7, TAS_TOTAL(r5) 3478 ld r6, TAS_MIN(r5) 3479 ld r7, TAS_MAX(r5) 3480 beq 3f 3481 cmpd r3, r6 3482 bge 1f 34833: std r3, TAS_MIN(r5) 34841: cmpd r3, r7 3485 ble 2f 3486 std r3, TAS_MAX(r5) 34872: lwsync 3488 addi r8, r8, 1 3489 std r8, TAS_SEQCOUNT(r5) 3490 blr 3491#endif 3492