1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/tm.h> 32#include <asm/opal.h> 33#include <asm/xive-regs.h> 34#include <asm/thread_info.h> 35#include <asm/asm-compat.h> 36#include <asm/feature-fixups.h> 37 38/* Sign-extend HDEC if not on POWER9 */ 39#define EXTEND_HDEC(reg) \ 40BEGIN_FTR_SECTION; \ 41 extsw reg, reg; \ 42END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 43 44/* Values in HSTATE_NAPPING(r13) */ 45#define NAPPING_CEDE 1 46#define NAPPING_NOVCPU 2 47 48/* Stack frame offsets for kvmppc_hv_entry */ 49#define SFS 160 50#define STACK_SLOT_TRAP (SFS-4) 51#define STACK_SLOT_TID (SFS-16) 52#define STACK_SLOT_PSSCR (SFS-24) 53#define STACK_SLOT_PID (SFS-32) 54#define STACK_SLOT_IAMR (SFS-40) 55#define STACK_SLOT_CIABR (SFS-48) 56#define STACK_SLOT_DAWR (SFS-56) 57#define STACK_SLOT_DAWRX (SFS-64) 58#define STACK_SLOT_HFSCR (SFS-72) 59 60/* 61 * Call kvmppc_hv_entry in real mode. 62 * Must be called with interrupts hard-disabled. 63 * 64 * Input Registers: 65 * 66 * LR = return address to continue at after eventually re-enabling MMU 67 */ 68_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 69 mflr r0 70 std r0, PPC_LR_STKOFF(r1) 71 stdu r1, -112(r1) 72 mfmsr r10 73 std r10, HSTATE_HOST_MSR(r13) 74 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 75 li r0,MSR_RI 76 andc r0,r10,r0 77 li r6,MSR_IR | MSR_DR 78 andc r6,r10,r6 79 mtmsrd r0,1 /* clear RI in MSR */ 80 mtsrr0 r5 81 mtsrr1 r6 82 RFI_TO_KERNEL 83 84kvmppc_call_hv_entry: 85BEGIN_FTR_SECTION 86 /* On P9, do LPCR setting, if necessary */ 87 ld r3, HSTATE_SPLIT_MODE(r13) 88 cmpdi r3, 0 89 beq 46f 90 lwz r4, KVM_SPLIT_DO_SET(r3) 91 cmpwi r4, 0 92 beq 46f 93 bl kvmhv_p9_set_lpcr 94 nop 9546: 96END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 97 98 ld r4, HSTATE_KVM_VCPU(r13) 99 bl kvmppc_hv_entry 100 101 /* Back from guest - restore host state and return to caller */ 102 103BEGIN_FTR_SECTION 104 /* Restore host DABR and DABRX */ 105 ld r5,HSTATE_DABR(r13) 106 li r6,7 107 mtspr SPRN_DABR,r5 108 mtspr SPRN_DABRX,r6 109END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 110 111 /* Restore SPRG3 */ 112 ld r3,PACA_SPRG_VDSO(r13) 113 mtspr SPRN_SPRG_VDSO_WRITE,r3 114 115 /* Reload the host's PMU registers */ 116 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 117 cmpwi r4, 0 118 beq 23f /* skip if not */ 119BEGIN_FTR_SECTION 120 ld r3, HSTATE_MMCR0(r13) 121 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 122 cmpwi r4, MMCR0_PMAO 123 beql kvmppc_fix_pmao 124END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 125 lwz r3, HSTATE_PMC1(r13) 126 lwz r4, HSTATE_PMC2(r13) 127 lwz r5, HSTATE_PMC3(r13) 128 lwz r6, HSTATE_PMC4(r13) 129 lwz r8, HSTATE_PMC5(r13) 130 lwz r9, HSTATE_PMC6(r13) 131 mtspr SPRN_PMC1, r3 132 mtspr SPRN_PMC2, r4 133 mtspr SPRN_PMC3, r5 134 mtspr SPRN_PMC4, r6 135 mtspr SPRN_PMC5, r8 136 mtspr SPRN_PMC6, r9 137 ld r3, HSTATE_MMCR0(r13) 138 ld r4, HSTATE_MMCR1(r13) 139 ld r5, HSTATE_MMCRA(r13) 140 ld r6, HSTATE_SIAR(r13) 141 ld r7, HSTATE_SDAR(r13) 142 mtspr SPRN_MMCR1, r4 143 mtspr SPRN_MMCRA, r5 144 mtspr SPRN_SIAR, r6 145 mtspr SPRN_SDAR, r7 146BEGIN_FTR_SECTION 147 ld r8, HSTATE_MMCR2(r13) 148 ld r9, HSTATE_SIER(r13) 149 mtspr SPRN_MMCR2, r8 150 mtspr SPRN_SIER, r9 151END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 152 mtspr SPRN_MMCR0, r3 153 isync 15423: 155 156 /* 157 * Reload DEC. HDEC interrupts were disabled when 158 * we reloaded the host's LPCR value. 159 */ 160 ld r3, HSTATE_DECEXP(r13) 161 mftb r4 162 subf r4, r4, r3 163 mtspr SPRN_DEC, r4 164 165 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 166 li r0, 0 167 stb r0, HSTATE_HWTHREAD_REQ(r13) 168 169 /* 170 * For external interrupts we need to call the Linux 171 * handler to process the interrupt. We do that by jumping 172 * to absolute address 0x500 for external interrupts. 173 * The [h]rfid at the end of the handler will return to 174 * the book3s_hv_interrupts.S code. For other interrupts 175 * we do the rfid to get back to the book3s_hv_interrupts.S 176 * code here. 177 */ 178 ld r8, 112+PPC_LR_STKOFF(r1) 179 addi r1, r1, 112 180 ld r7, HSTATE_HOST_MSR(r13) 181 182 /* Return the trap number on this thread as the return value */ 183 mr r3, r12 184 185 /* 186 * If we came back from the guest via a relocation-on interrupt, 187 * we will be in virtual mode at this point, which makes it a 188 * little easier to get back to the caller. 189 */ 190 mfmsr r0 191 andi. r0, r0, MSR_IR /* in real mode? */ 192 bne .Lvirt_return 193 194 /* RFI into the highmem handler */ 195 mfmsr r6 196 li r0, MSR_RI 197 andc r6, r6, r0 198 mtmsrd r6, 1 /* Clear RI in MSR */ 199 mtsrr0 r8 200 mtsrr1 r7 201 RFI_TO_KERNEL 202 203 /* Virtual-mode return */ 204.Lvirt_return: 205 mtlr r8 206 blr 207 208kvmppc_primary_no_guest: 209 /* We handle this much like a ceded vcpu */ 210 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 211 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 212 /* HDEC value came from DEC in the first place, it will fit */ 213 mfspr r3, SPRN_HDEC 214 mtspr SPRN_DEC, r3 215 /* 216 * Make sure the primary has finished the MMU switch. 217 * We should never get here on a secondary thread, but 218 * check it for robustness' sake. 219 */ 220 ld r5, HSTATE_KVM_VCORE(r13) 22165: lbz r0, VCORE_IN_GUEST(r5) 222 cmpwi r0, 0 223 beq 65b 224 /* Set LPCR. */ 225 ld r8,VCORE_LPCR(r5) 226 mtspr SPRN_LPCR,r8 227 isync 228 /* set our bit in napping_threads */ 229 ld r5, HSTATE_KVM_VCORE(r13) 230 lbz r7, HSTATE_PTID(r13) 231 li r0, 1 232 sld r0, r0, r7 233 addi r6, r5, VCORE_NAPPING_THREADS 2341: lwarx r3, 0, r6 235 or r3, r3, r0 236 stwcx. r3, 0, r6 237 bne 1b 238 /* order napping_threads update vs testing entry_exit_map */ 239 isync 240 li r12, 0 241 lwz r7, VCORE_ENTRY_EXIT(r5) 242 cmpwi r7, 0x100 243 bge kvm_novcpu_exit /* another thread already exiting */ 244 li r3, NAPPING_NOVCPU 245 stb r3, HSTATE_NAPPING(r13) 246 247 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 248 b kvm_do_nap 249 250/* 251 * kvm_novcpu_wakeup 252 * Entered from kvm_start_guest if kvm_hstate.napping is set 253 * to NAPPING_NOVCPU 254 * r2 = kernel TOC 255 * r13 = paca 256 */ 257kvm_novcpu_wakeup: 258 ld r1, HSTATE_HOST_R1(r13) 259 ld r5, HSTATE_KVM_VCORE(r13) 260 li r0, 0 261 stb r0, HSTATE_NAPPING(r13) 262 263 /* check the wake reason */ 264 bl kvmppc_check_wake_reason 265 266 /* 267 * Restore volatile registers since we could have called 268 * a C routine in kvmppc_check_wake_reason. 269 * r5 = VCORE 270 */ 271 ld r5, HSTATE_KVM_VCORE(r13) 272 273 /* see if any other thread is already exiting */ 274 lwz r0, VCORE_ENTRY_EXIT(r5) 275 cmpwi r0, 0x100 276 bge kvm_novcpu_exit 277 278 /* clear our bit in napping_threads */ 279 lbz r7, HSTATE_PTID(r13) 280 li r0, 1 281 sld r0, r0, r7 282 addi r6, r5, VCORE_NAPPING_THREADS 2834: lwarx r7, 0, r6 284 andc r7, r7, r0 285 stwcx. r7, 0, r6 286 bne 4b 287 288 /* See if the wake reason means we need to exit */ 289 cmpdi r3, 0 290 bge kvm_novcpu_exit 291 292 /* See if our timeslice has expired (HDEC is negative) */ 293 mfspr r0, SPRN_HDEC 294 EXTEND_HDEC(r0) 295 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 296 cmpdi r0, 0 297 blt kvm_novcpu_exit 298 299 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 300 ld r4, HSTATE_KVM_VCPU(r13) 301 cmpdi r4, 0 302 beq kvmppc_primary_no_guest 303 304#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 305 addi r3, r4, VCPU_TB_RMENTRY 306 bl kvmhv_start_timing 307#endif 308 b kvmppc_got_guest 309 310kvm_novcpu_exit: 311#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 312 ld r4, HSTATE_KVM_VCPU(r13) 313 cmpdi r4, 0 314 beq 13f 315 addi r3, r4, VCPU_TB_RMEXIT 316 bl kvmhv_accumulate_time 317#endif 31813: mr r3, r12 319 stw r12, STACK_SLOT_TRAP(r1) 320 bl kvmhv_commence_exit 321 nop 322 b kvmhv_switch_to_host 323 324/* 325 * We come in here when wakened from nap mode. 326 * Relocation is off and most register values are lost. 327 * r13 points to the PACA. 328 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 329 */ 330 .globl kvm_start_guest 331kvm_start_guest: 332 /* Set runlatch bit the minute you wake up from nap */ 333 mfspr r0, SPRN_CTRLF 334 ori r0, r0, 1 335 mtspr SPRN_CTRLT, r0 336 337 /* 338 * Could avoid this and pass it through in r3. For now, 339 * code expects it to be in SRR1. 340 */ 341 mtspr SPRN_SRR1,r3 342 343 ld r2,PACATOC(r13) 344 345 li r0,0 346 stb r0,PACA_FTRACE_ENABLED(r13) 347 348 li r0,KVM_HWTHREAD_IN_KVM 349 stb r0,HSTATE_HWTHREAD_STATE(r13) 350 351 /* NV GPR values from power7_idle() will no longer be valid */ 352 li r0,1 353 stb r0,PACA_NAPSTATELOST(r13) 354 355 /* were we napping due to cede? */ 356 lbz r0,HSTATE_NAPPING(r13) 357 cmpwi r0,NAPPING_CEDE 358 beq kvm_end_cede 359 cmpwi r0,NAPPING_NOVCPU 360 beq kvm_novcpu_wakeup 361 362 ld r1,PACAEMERGSP(r13) 363 subi r1,r1,STACK_FRAME_OVERHEAD 364 365 /* 366 * We weren't napping due to cede, so this must be a secondary 367 * thread being woken up to run a guest, or being woken up due 368 * to a stray IPI. (Or due to some machine check or hypervisor 369 * maintenance interrupt while the core is in KVM.) 370 */ 371 372 /* Check the wake reason in SRR1 to see why we got here */ 373 bl kvmppc_check_wake_reason 374 /* 375 * kvmppc_check_wake_reason could invoke a C routine, but we 376 * have no volatile registers to restore when we return. 377 */ 378 379 cmpdi r3, 0 380 bge kvm_no_guest 381 382 /* get vcore pointer, NULL if we have nothing to run */ 383 ld r5,HSTATE_KVM_VCORE(r13) 384 cmpdi r5,0 385 /* if we have no vcore to run, go back to sleep */ 386 beq kvm_no_guest 387 388kvm_secondary_got_guest: 389 390 /* Set HSTATE_DSCR(r13) to something sensible */ 391 ld r6, PACA_DSCR_DEFAULT(r13) 392 std r6, HSTATE_DSCR(r13) 393 394 /* On thread 0 of a subcore, set HDEC to max */ 395 lbz r4, HSTATE_PTID(r13) 396 cmpwi r4, 0 397 bne 63f 398 LOAD_REG_ADDR(r6, decrementer_max) 399 ld r6, 0(r6) 400 mtspr SPRN_HDEC, r6 401 /* and set per-LPAR registers, if doing dynamic micro-threading */ 402 ld r6, HSTATE_SPLIT_MODE(r13) 403 cmpdi r6, 0 404 beq 63f 405BEGIN_FTR_SECTION 406 ld r0, KVM_SPLIT_RPR(r6) 407 mtspr SPRN_RPR, r0 408 ld r0, KVM_SPLIT_PMMAR(r6) 409 mtspr SPRN_PMMAR, r0 410 ld r0, KVM_SPLIT_LDBAR(r6) 411 mtspr SPRN_LDBAR, r0 412 isync 413FTR_SECTION_ELSE 414 /* On P9 we use the split_info for coordinating LPCR changes */ 415 lwz r4, KVM_SPLIT_DO_SET(r6) 416 cmpwi r4, 0 417 beq 1f 418 mr r3, r6 419 bl kvmhv_p9_set_lpcr 420 nop 4211: 422ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 42363: 424 /* Order load of vcpu after load of vcore */ 425 lwsync 426 ld r4, HSTATE_KVM_VCPU(r13) 427 bl kvmppc_hv_entry 428 429 /* Back from the guest, go back to nap */ 430 /* Clear our vcpu and vcore pointers so we don't come back in early */ 431 li r0, 0 432 std r0, HSTATE_KVM_VCPU(r13) 433 /* 434 * Once we clear HSTATE_KVM_VCORE(r13), the code in 435 * kvmppc_run_core() is going to assume that all our vcpu 436 * state is visible in memory. This lwsync makes sure 437 * that that is true. 438 */ 439 lwsync 440 std r0, HSTATE_KVM_VCORE(r13) 441 442 /* 443 * All secondaries exiting guest will fall through this path. 444 * Before proceeding, just check for HMI interrupt and 445 * invoke opal hmi handler. By now we are sure that the 446 * primary thread on this core/subcore has already made partition 447 * switch/TB resync and we are good to call opal hmi handler. 448 */ 449 cmpwi r12, BOOK3S_INTERRUPT_HMI 450 bne kvm_no_guest 451 452 li r3,0 /* NULL argument */ 453 bl hmi_exception_realmode 454/* 455 * At this point we have finished executing in the guest. 456 * We need to wait for hwthread_req to become zero, since 457 * we may not turn on the MMU while hwthread_req is non-zero. 458 * While waiting we also need to check if we get given a vcpu to run. 459 */ 460kvm_no_guest: 461 lbz r3, HSTATE_HWTHREAD_REQ(r13) 462 cmpwi r3, 0 463 bne 53f 464 HMT_MEDIUM 465 li r0, KVM_HWTHREAD_IN_KERNEL 466 stb r0, HSTATE_HWTHREAD_STATE(r13) 467 /* need to recheck hwthread_req after a barrier, to avoid race */ 468 sync 469 lbz r3, HSTATE_HWTHREAD_REQ(r13) 470 cmpwi r3, 0 471 bne 54f 472/* 473 * We jump to pnv_wakeup_loss, which will return to the caller 474 * of power7_nap in the powernv cpu offline loop. The value we 475 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 476 * requires SRR1 in r12. 477 */ 478 li r3, LPCR_PECE0 479 mfspr r4, SPRN_LPCR 480 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 481 mtspr SPRN_LPCR, r4 482 li r3, 0 483 mfspr r12,SPRN_SRR1 484 b pnv_wakeup_loss 485 48653: HMT_LOW 487 ld r5, HSTATE_KVM_VCORE(r13) 488 cmpdi r5, 0 489 bne 60f 490 ld r3, HSTATE_SPLIT_MODE(r13) 491 cmpdi r3, 0 492 beq kvm_no_guest 493 lwz r0, KVM_SPLIT_DO_SET(r3) 494 cmpwi r0, 0 495 bne kvmhv_do_set 496 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 497 cmpwi r0, 0 498 bne kvmhv_do_restore 499 lbz r0, KVM_SPLIT_DO_NAP(r3) 500 cmpwi r0, 0 501 beq kvm_no_guest 502 HMT_MEDIUM 503 b kvm_unsplit_nap 50460: HMT_MEDIUM 505 b kvm_secondary_got_guest 506 50754: li r0, KVM_HWTHREAD_IN_KVM 508 stb r0, HSTATE_HWTHREAD_STATE(r13) 509 b kvm_no_guest 510 511kvmhv_do_set: 512 /* Set LPCR, LPIDR etc. on P9 */ 513 HMT_MEDIUM 514 bl kvmhv_p9_set_lpcr 515 nop 516 b kvm_no_guest 517 518kvmhv_do_restore: 519 HMT_MEDIUM 520 bl kvmhv_p9_restore_lpcr 521 nop 522 b kvm_no_guest 523 524/* 525 * Here the primary thread is trying to return the core to 526 * whole-core mode, so we need to nap. 527 */ 528kvm_unsplit_nap: 529 /* 530 * When secondaries are napping in kvm_unsplit_nap() with 531 * hwthread_req = 1, HMI goes ignored even though subcores are 532 * already exited the guest. Hence HMI keeps waking up secondaries 533 * from nap in a loop and secondaries always go back to nap since 534 * no vcore is assigned to them. This makes impossible for primary 535 * thread to get hold of secondary threads resulting into a soft 536 * lockup in KVM path. 537 * 538 * Let us check if HMI is pending and handle it before we go to nap. 539 */ 540 cmpwi r12, BOOK3S_INTERRUPT_HMI 541 bne 55f 542 li r3, 0 /* NULL argument */ 543 bl hmi_exception_realmode 54455: 545 /* 546 * Ensure that secondary doesn't nap when it has 547 * its vcore pointer set. 548 */ 549 sync /* matches smp_mb() before setting split_info.do_nap */ 550 ld r0, HSTATE_KVM_VCORE(r13) 551 cmpdi r0, 0 552 bne kvm_no_guest 553 /* clear any pending message */ 554BEGIN_FTR_SECTION 555 lis r6, (PPC_DBELL_SERVER << (63-36))@h 556 PPC_MSGCLR(6) 557END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 558 /* Set kvm_split_mode.napped[tid] = 1 */ 559 ld r3, HSTATE_SPLIT_MODE(r13) 560 li r0, 1 561 lbz r4, HSTATE_TID(r13) 562 addi r4, r4, KVM_SPLIT_NAPPED 563 stbx r0, r3, r4 564 /* Check the do_nap flag again after setting napped[] */ 565 sync 566 lbz r0, KVM_SPLIT_DO_NAP(r3) 567 cmpwi r0, 0 568 beq 57f 569 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 570 mfspr r5, SPRN_LPCR 571 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 572 b kvm_nap_sequence 573 57457: li r0, 0 575 stbx r0, r3, r4 576 b kvm_no_guest 577 578/****************************************************************************** 579 * * 580 * Entry code * 581 * * 582 *****************************************************************************/ 583 584.global kvmppc_hv_entry 585kvmppc_hv_entry: 586 587 /* Required state: 588 * 589 * R4 = vcpu pointer (or NULL) 590 * MSR = ~IR|DR 591 * R13 = PACA 592 * R1 = host R1 593 * R2 = TOC 594 * all other volatile GPRS = free 595 * Does not preserve non-volatile GPRs or CR fields 596 */ 597 mflr r0 598 std r0, PPC_LR_STKOFF(r1) 599 stdu r1, -SFS(r1) 600 601 /* Save R1 in the PACA */ 602 std r1, HSTATE_HOST_R1(r13) 603 604 li r6, KVM_GUEST_MODE_HOST_HV 605 stb r6, HSTATE_IN_GUEST(r13) 606 607#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 608 /* Store initial timestamp */ 609 cmpdi r4, 0 610 beq 1f 611 addi r3, r4, VCPU_TB_RMENTRY 612 bl kvmhv_start_timing 6131: 614#endif 615 616 /* Use cr7 as an indication of radix mode */ 617 ld r5, HSTATE_KVM_VCORE(r13) 618 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 619 lbz r0, KVM_RADIX(r9) 620 cmpwi cr7, r0, 0 621 622 /* 623 * POWER7/POWER8 host -> guest partition switch code. 624 * We don't have to lock against concurrent tlbies, 625 * but we do have to coordinate across hardware threads. 626 */ 627 /* Set bit in entry map iff exit map is zero. */ 628 li r7, 1 629 lbz r6, HSTATE_PTID(r13) 630 sld r7, r7, r6 631 addi r8, r5, VCORE_ENTRY_EXIT 63221: lwarx r3, 0, r8 633 cmpwi r3, 0x100 /* any threads starting to exit? */ 634 bge secondary_too_late /* if so we're too late to the party */ 635 or r3, r3, r7 636 stwcx. r3, 0, r8 637 bne 21b 638 639 /* Primary thread switches to guest partition. */ 640 cmpwi r6,0 641 bne 10f 642 643 /* Radix has already switched LPID and flushed core TLB */ 644 bne cr7, 22f 645 646 lwz r7,KVM_LPID(r9) 647BEGIN_FTR_SECTION 648 ld r6,KVM_SDR1(r9) 649 li r0,LPID_RSVD /* switch to reserved LPID */ 650 mtspr SPRN_LPID,r0 651 ptesync 652 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 653END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 654 mtspr SPRN_LPID,r7 655 isync 656 657 /* See if we need to flush the TLB. Hash has to be done in RM */ 658 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 659BEGIN_FTR_SECTION 660 /* 661 * On POWER9, individual threads can come in here, but the 662 * TLB is shared between the 4 threads in a core, hence 663 * invalidating on one thread invalidates for all. 664 * Thus we make all 4 threads use the same bit here. 665 */ 666 clrrdi r6,r6,2 667END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 668 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 669 srdi r6,r6,6 /* doubleword number */ 670 sldi r6,r6,3 /* address offset */ 671 add r6,r6,r9 672 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 673 li r8,1 674 sld r8,r8,r7 675 ld r7,0(r6) 676 and. r7,r7,r8 677 beq 22f 678 /* Flush the TLB of any entries for this LPID */ 679 lwz r0,KVM_TLB_SETS(r9) 680 mtctr r0 681 li r7,0x800 /* IS field = 0b10 */ 682 ptesync 683 li r0,0 /* RS for P9 version of tlbiel */ 68428: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 685 addi r7,r7,0x1000 686 bdnz 28b 687 ptesync 68823: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 689 andc r7,r7,r8 690 stdcx. r7,0,r6 691 bne 23b 692 693 /* Add timebase offset onto timebase */ 69422: ld r8,VCORE_TB_OFFSET(r5) 695 cmpdi r8,0 696 beq 37f 697 std r8, VCORE_TB_OFFSET_APPL(r5) 698 mftb r6 /* current host timebase */ 699 add r8,r8,r6 700 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 701 mftb r7 /* check if lower 24 bits overflowed */ 702 clrldi r6,r6,40 703 clrldi r7,r7,40 704 cmpld r7,r6 705 bge 37f 706 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 707 mtspr SPRN_TBU40,r8 708 709 /* Load guest PCR value to select appropriate compat mode */ 71037: ld r7, VCORE_PCR(r5) 711 cmpdi r7, 0 712 beq 38f 713 mtspr SPRN_PCR, r7 71438: 715 716BEGIN_FTR_SECTION 717 /* DPDES and VTB are shared between threads */ 718 ld r8, VCORE_DPDES(r5) 719 ld r7, VCORE_VTB(r5) 720 mtspr SPRN_DPDES, r8 721 mtspr SPRN_VTB, r7 722END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 723 724 /* Mark the subcore state as inside guest */ 725 bl kvmppc_subcore_enter_guest 726 nop 727 ld r5, HSTATE_KVM_VCORE(r13) 728 ld r4, HSTATE_KVM_VCPU(r13) 729 li r0,1 730 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 731 732 /* Do we have a guest vcpu to run? */ 73310: cmpdi r4, 0 734 beq kvmppc_primary_no_guest 735kvmppc_got_guest: 736 /* Increment yield count if they have a VPA */ 737 ld r3, VCPU_VPA(r4) 738 cmpdi r3, 0 739 beq 25f 740 li r6, LPPACA_YIELDCOUNT 741 LWZX_BE r5, r3, r6 742 addi r5, r5, 1 743 STWX_BE r5, r3, r6 744 li r6, 1 745 stb r6, VCPU_VPA_DIRTY(r4) 74625: 747 748 /* Save purr/spurr */ 749 mfspr r5,SPRN_PURR 750 mfspr r6,SPRN_SPURR 751 std r5,HSTATE_PURR(r13) 752 std r6,HSTATE_SPURR(r13) 753 ld r7,VCPU_PURR(r4) 754 ld r8,VCPU_SPURR(r4) 755 mtspr SPRN_PURR,r7 756 mtspr SPRN_SPURR,r8 757 758 /* Save host values of some registers */ 759BEGIN_FTR_SECTION 760 mfspr r5, SPRN_TIDR 761 mfspr r6, SPRN_PSSCR 762 mfspr r7, SPRN_PID 763 mfspr r8, SPRN_IAMR 764 std r5, STACK_SLOT_TID(r1) 765 std r6, STACK_SLOT_PSSCR(r1) 766 std r7, STACK_SLOT_PID(r1) 767 std r8, STACK_SLOT_IAMR(r1) 768 mfspr r5, SPRN_HFSCR 769 std r5, STACK_SLOT_HFSCR(r1) 770END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 771BEGIN_FTR_SECTION 772 mfspr r5, SPRN_CIABR 773 mfspr r6, SPRN_DAWR 774 mfspr r7, SPRN_DAWRX 775 std r5, STACK_SLOT_CIABR(r1) 776 std r6, STACK_SLOT_DAWR(r1) 777 std r7, STACK_SLOT_DAWRX(r1) 778END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 779 780BEGIN_FTR_SECTION 781 /* Set partition DABR */ 782 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 783 lwz r5,VCPU_DABRX(r4) 784 ld r6,VCPU_DABR(r4) 785 mtspr SPRN_DABRX,r5 786 mtspr SPRN_DABR,r6 787 isync 788END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 789 790#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 791/* 792 * Branch around the call if both CPU_FTR_TM and 793 * CPU_FTR_P9_TM_HV_ASSIST are off. 794 */ 795BEGIN_FTR_SECTION 796 b 91f 797END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 798 /* 799 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 800 */ 801 mr r3, r4 802 ld r4, VCPU_MSR(r3) 803 bl kvmppc_restore_tm_hv 804 ld r4, HSTATE_KVM_VCPU(r13) 80591: 806#endif 807 808 /* Load guest PMU registers */ 809 /* R4 is live here (vcpu pointer) */ 810 li r3, 1 811 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 812 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 813 isync 814BEGIN_FTR_SECTION 815 ld r3, VCPU_MMCR(r4) 816 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 817 cmpwi r5, MMCR0_PMAO 818 beql kvmppc_fix_pmao 819END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 820 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 821 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 822 lwz r6, VCPU_PMC + 8(r4) 823 lwz r7, VCPU_PMC + 12(r4) 824 lwz r8, VCPU_PMC + 16(r4) 825 lwz r9, VCPU_PMC + 20(r4) 826 mtspr SPRN_PMC1, r3 827 mtspr SPRN_PMC2, r5 828 mtspr SPRN_PMC3, r6 829 mtspr SPRN_PMC4, r7 830 mtspr SPRN_PMC5, r8 831 mtspr SPRN_PMC6, r9 832 ld r3, VCPU_MMCR(r4) 833 ld r5, VCPU_MMCR + 8(r4) 834 ld r6, VCPU_MMCR + 16(r4) 835 ld r7, VCPU_SIAR(r4) 836 ld r8, VCPU_SDAR(r4) 837 mtspr SPRN_MMCR1, r5 838 mtspr SPRN_MMCRA, r6 839 mtspr SPRN_SIAR, r7 840 mtspr SPRN_SDAR, r8 841BEGIN_FTR_SECTION 842 ld r5, VCPU_MMCR + 24(r4) 843 ld r6, VCPU_SIER(r4) 844 mtspr SPRN_MMCR2, r5 845 mtspr SPRN_SIER, r6 846BEGIN_FTR_SECTION_NESTED(96) 847 lwz r7, VCPU_PMC + 24(r4) 848 lwz r8, VCPU_PMC + 28(r4) 849 ld r9, VCPU_MMCR + 32(r4) 850 mtspr SPRN_SPMC1, r7 851 mtspr SPRN_SPMC2, r8 852 mtspr SPRN_MMCRS, r9 853END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 854END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 855 mtspr SPRN_MMCR0, r3 856 isync 857 858 /* Load up FP, VMX and VSX registers */ 859 bl kvmppc_load_fp 860 861 ld r14, VCPU_GPR(R14)(r4) 862 ld r15, VCPU_GPR(R15)(r4) 863 ld r16, VCPU_GPR(R16)(r4) 864 ld r17, VCPU_GPR(R17)(r4) 865 ld r18, VCPU_GPR(R18)(r4) 866 ld r19, VCPU_GPR(R19)(r4) 867 ld r20, VCPU_GPR(R20)(r4) 868 ld r21, VCPU_GPR(R21)(r4) 869 ld r22, VCPU_GPR(R22)(r4) 870 ld r23, VCPU_GPR(R23)(r4) 871 ld r24, VCPU_GPR(R24)(r4) 872 ld r25, VCPU_GPR(R25)(r4) 873 ld r26, VCPU_GPR(R26)(r4) 874 ld r27, VCPU_GPR(R27)(r4) 875 ld r28, VCPU_GPR(R28)(r4) 876 ld r29, VCPU_GPR(R29)(r4) 877 ld r30, VCPU_GPR(R30)(r4) 878 ld r31, VCPU_GPR(R31)(r4) 879 880 /* Switch DSCR to guest value */ 881 ld r5, VCPU_DSCR(r4) 882 mtspr SPRN_DSCR, r5 883 884BEGIN_FTR_SECTION 885 /* Skip next section on POWER7 */ 886 b 8f 887END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 888 /* Load up POWER8-specific registers */ 889 ld r5, VCPU_IAMR(r4) 890 lwz r6, VCPU_PSPB(r4) 891 ld r7, VCPU_FSCR(r4) 892 mtspr SPRN_IAMR, r5 893 mtspr SPRN_PSPB, r6 894 mtspr SPRN_FSCR, r7 895 ld r5, VCPU_DAWR(r4) 896 ld r6, VCPU_DAWRX(r4) 897 ld r7, VCPU_CIABR(r4) 898 ld r8, VCPU_TAR(r4) 899 /* 900 * Handle broken DAWR case by not writing it. This means we 901 * can still store the DAWR register for migration. 902 */ 903BEGIN_FTR_SECTION 904 mtspr SPRN_DAWR, r5 905 mtspr SPRN_DAWRX, r6 906END_FTR_SECTION_IFSET(CPU_FTR_DAWR) 907 mtspr SPRN_CIABR, r7 908 mtspr SPRN_TAR, r8 909 ld r5, VCPU_IC(r4) 910 ld r8, VCPU_EBBHR(r4) 911 mtspr SPRN_IC, r5 912 mtspr SPRN_EBBHR, r8 913 ld r5, VCPU_EBBRR(r4) 914 ld r6, VCPU_BESCR(r4) 915 lwz r7, VCPU_GUEST_PID(r4) 916 ld r8, VCPU_WORT(r4) 917 mtspr SPRN_EBBRR, r5 918 mtspr SPRN_BESCR, r6 919 mtspr SPRN_PID, r7 920 mtspr SPRN_WORT, r8 921BEGIN_FTR_SECTION 922 /* POWER8-only registers */ 923 ld r5, VCPU_TCSCR(r4) 924 ld r6, VCPU_ACOP(r4) 925 ld r7, VCPU_CSIGR(r4) 926 ld r8, VCPU_TACR(r4) 927 mtspr SPRN_TCSCR, r5 928 mtspr SPRN_ACOP, r6 929 mtspr SPRN_CSIGR, r7 930 mtspr SPRN_TACR, r8 931 nop 932FTR_SECTION_ELSE 933 /* POWER9-only registers */ 934 ld r5, VCPU_TID(r4) 935 ld r6, VCPU_PSSCR(r4) 936 lbz r8, HSTATE_FAKE_SUSPEND(r13) 937 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 938 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG 939 ld r7, VCPU_HFSCR(r4) 940 mtspr SPRN_TIDR, r5 941 mtspr SPRN_PSSCR, r6 942 mtspr SPRN_HFSCR, r7 943ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 9448: 945 946 ld r5, VCPU_SPRG0(r4) 947 ld r6, VCPU_SPRG1(r4) 948 ld r7, VCPU_SPRG2(r4) 949 ld r8, VCPU_SPRG3(r4) 950 mtspr SPRN_SPRG0, r5 951 mtspr SPRN_SPRG1, r6 952 mtspr SPRN_SPRG2, r7 953 mtspr SPRN_SPRG3, r8 954 955 /* Load up DAR and DSISR */ 956 ld r5, VCPU_DAR(r4) 957 lwz r6, VCPU_DSISR(r4) 958 mtspr SPRN_DAR, r5 959 mtspr SPRN_DSISR, r6 960 961 /* Restore AMR and UAMOR, set AMOR to all 1s */ 962 ld r5,VCPU_AMR(r4) 963 ld r6,VCPU_UAMOR(r4) 964 li r7,-1 965 mtspr SPRN_AMR,r5 966 mtspr SPRN_UAMOR,r6 967 mtspr SPRN_AMOR,r7 968 969 /* Restore state of CTRL run bit; assume 1 on entry */ 970 lwz r5,VCPU_CTRL(r4) 971 andi. r5,r5,1 972 bne 4f 973 mfspr r6,SPRN_CTRLF 974 clrrdi r6,r6,1 975 mtspr SPRN_CTRLT,r6 9764: 977 /* Secondary threads wait for primary to have done partition switch */ 978 ld r5, HSTATE_KVM_VCORE(r13) 979 lbz r6, HSTATE_PTID(r13) 980 cmpwi r6, 0 981 beq 21f 982 lbz r0, VCORE_IN_GUEST(r5) 983 cmpwi r0, 0 984 bne 21f 985 HMT_LOW 98620: lwz r3, VCORE_ENTRY_EXIT(r5) 987 cmpwi r3, 0x100 988 bge no_switch_exit 989 lbz r0, VCORE_IN_GUEST(r5) 990 cmpwi r0, 0 991 beq 20b 992 HMT_MEDIUM 99321: 994 /* Set LPCR. */ 995 ld r8,VCORE_LPCR(r5) 996 mtspr SPRN_LPCR,r8 997 isync 998 999 /* 1000 * Set the decrementer to the guest decrementer. 1001 */ 1002 ld r8,VCPU_DEC_EXPIRES(r4) 1003 /* r8 is a host timebase value here, convert to guest TB */ 1004 ld r5,HSTATE_KVM_VCORE(r13) 1005 ld r6,VCORE_TB_OFFSET_APPL(r5) 1006 add r8,r8,r6 1007 mftb r7 1008 subf r3,r7,r8 1009 mtspr SPRN_DEC,r3 1010 1011 /* Check if HDEC expires soon */ 1012 mfspr r3, SPRN_HDEC 1013 EXTEND_HDEC(r3) 1014 cmpdi r3, 512 /* 1 microsecond */ 1015 blt hdec_soon 1016 1017 /* For hash guest, clear out and reload the SLB */ 1018 ld r6, VCPU_KVM(r4) 1019 lbz r0, KVM_RADIX(r6) 1020 cmpwi r0, 0 1021 bne 9f 1022 li r6, 0 1023 slbmte r6, r6 1024 slbia 1025 ptesync 1026 1027 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 1028 lwz r5,VCPU_SLB_MAX(r4) 1029 cmpwi r5,0 1030 beq 9f 1031 mtctr r5 1032 addi r6,r4,VCPU_SLB 10331: ld r8,VCPU_SLB_E(r6) 1034 ld r9,VCPU_SLB_V(r6) 1035 slbmte r9,r8 1036 addi r6,r6,VCPU_SLB_SIZE 1037 bdnz 1b 10389: 1039 1040#ifdef CONFIG_KVM_XICS 1041 /* We are entering the guest on that thread, push VCPU to XIVE */ 1042 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1043 cmpldi cr0, r10, 0 1044 beq no_xive 1045 ld r11, VCPU_XIVE_SAVED_STATE(r4) 1046 li r9, TM_QW1_OS 1047 eieio 1048 stdcix r11,r9,r10 1049 lwz r11, VCPU_XIVE_CAM_WORD(r4) 1050 li r9, TM_QW1_OS + TM_WORD2 1051 stwcix r11,r9,r10 1052 li r9, 1 1053 stb r9, VCPU_XIVE_PUSHED(r4) 1054 eieio 1055 1056 /* 1057 * We clear the irq_pending flag. There is a small chance of a 1058 * race vs. the escalation interrupt happening on another 1059 * processor setting it again, but the only consequence is to 1060 * cause a spurrious wakeup on the next H_CEDE which is not an 1061 * issue. 1062 */ 1063 li r0,0 1064 stb r0, VCPU_IRQ_PENDING(r4) 1065 1066 /* 1067 * In single escalation mode, if the escalation interrupt is 1068 * on, we mask it. 1069 */ 1070 lbz r0, VCPU_XIVE_ESC_ON(r4) 1071 cmpwi r0,0 1072 beq 1f 1073 ld r10, VCPU_XIVE_ESC_RADDR(r4) 1074 li r9, XIVE_ESB_SET_PQ_01 1075 ldcix r0, r10, r9 1076 sync 1077 1078 /* We have a possible subtle race here: The escalation interrupt might 1079 * have fired and be on its way to the host queue while we mask it, 1080 * and if we unmask it early enough (re-cede right away), there is 1081 * a theorical possibility that it fires again, thus landing in the 1082 * target queue more than once which is a big no-no. 1083 * 1084 * Fortunately, solving this is rather easy. If the above load setting 1085 * PQ to 01 returns a previous value where P is set, then we know the 1086 * escalation interrupt is somewhere on its way to the host. In that 1087 * case we simply don't clear the xive_esc_on flag below. It will be 1088 * eventually cleared by the handler for the escalation interrupt. 1089 * 1090 * Then, when doing a cede, we check that flag again before re-enabling 1091 * the escalation interrupt, and if set, we abort the cede. 1092 */ 1093 andi. r0, r0, XIVE_ESB_VAL_P 1094 bne- 1f 1095 1096 /* Now P is 0, we can clear the flag */ 1097 li r0, 0 1098 stb r0, VCPU_XIVE_ESC_ON(r4) 10991: 1100no_xive: 1101#endif /* CONFIG_KVM_XICS */ 1102 1103deliver_guest_interrupt: 1104 ld r6, VCPU_CTR(r4) 1105 ld r7, VCPU_XER(r4) 1106 1107 mtctr r6 1108 mtxer r7 1109 1110kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ 1111 ld r10, VCPU_PC(r4) 1112 ld r11, VCPU_MSR(r4) 1113 ld r6, VCPU_SRR0(r4) 1114 ld r7, VCPU_SRR1(r4) 1115 mtspr SPRN_SRR0, r6 1116 mtspr SPRN_SRR1, r7 1117 1118 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1119 rldicl r11, r11, 63 - MSR_HV_LG, 1 1120 rotldi r11, r11, 1 + MSR_HV_LG 1121 ori r11, r11, MSR_ME 1122 1123 /* Check if we can deliver an external or decrementer interrupt now */ 1124 ld r0, VCPU_PENDING_EXC(r4) 1125 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 1126 cmpdi cr1, r0, 0 1127 andi. r8, r11, MSR_EE 1128 mfspr r8, SPRN_LPCR 1129 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ 1130 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH 1131 mtspr SPRN_LPCR, r8 1132 isync 1133 beq 5f 1134 li r0, BOOK3S_INTERRUPT_EXTERNAL 1135 bne cr1, 12f 1136 mfspr r0, SPRN_DEC 1137BEGIN_FTR_SECTION 1138 /* On POWER9 check whether the guest has large decrementer enabled */ 1139 andis. r8, r8, LPCR_LD@h 1140 bne 15f 1141END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1142 extsw r0, r0 114315: cmpdi r0, 0 1144 li r0, BOOK3S_INTERRUPT_DECREMENTER 1145 bge 5f 1146 114712: mtspr SPRN_SRR0, r10 1148 mr r10,r0 1149 mtspr SPRN_SRR1, r11 1150 mr r9, r4 1151 bl kvmppc_msr_interrupt 11525: 1153BEGIN_FTR_SECTION 1154 b fast_guest_return 1155END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1156 /* On POWER9, check for pending doorbell requests */ 1157 lbz r0, VCPU_DBELL_REQ(r4) 1158 cmpwi r0, 0 1159 beq fast_guest_return 1160 ld r5, HSTATE_KVM_VCORE(r13) 1161 /* Set DPDES register so the CPU will take a doorbell interrupt */ 1162 li r0, 1 1163 mtspr SPRN_DPDES, r0 1164 std r0, VCORE_DPDES(r5) 1165 /* Make sure other cpus see vcore->dpdes set before dbell req clear */ 1166 lwsync 1167 /* Clear the pending doorbell request */ 1168 li r0, 0 1169 stb r0, VCPU_DBELL_REQ(r4) 1170 1171/* 1172 * Required state: 1173 * R4 = vcpu 1174 * R10: value for HSRR0 1175 * R11: value for HSRR1 1176 * R13 = PACA 1177 */ 1178fast_guest_return: 1179 li r0,0 1180 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1181 mtspr SPRN_HSRR0,r10 1182 mtspr SPRN_HSRR1,r11 1183 1184 /* Activate guest mode, so faults get handled by KVM */ 1185 li r9, KVM_GUEST_MODE_GUEST_HV 1186 stb r9, HSTATE_IN_GUEST(r13) 1187 1188#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1189 /* Accumulate timing */ 1190 addi r3, r4, VCPU_TB_GUEST 1191 bl kvmhv_accumulate_time 1192#endif 1193 1194 /* Enter guest */ 1195 1196BEGIN_FTR_SECTION 1197 ld r5, VCPU_CFAR(r4) 1198 mtspr SPRN_CFAR, r5 1199END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1200BEGIN_FTR_SECTION 1201 ld r0, VCPU_PPR(r4) 1202END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1203 1204 ld r5, VCPU_LR(r4) 1205 lwz r6, VCPU_CR(r4) 1206 mtlr r5 1207 mtcr r6 1208 1209 ld r1, VCPU_GPR(R1)(r4) 1210 ld r2, VCPU_GPR(R2)(r4) 1211 ld r3, VCPU_GPR(R3)(r4) 1212 ld r5, VCPU_GPR(R5)(r4) 1213 ld r6, VCPU_GPR(R6)(r4) 1214 ld r7, VCPU_GPR(R7)(r4) 1215 ld r8, VCPU_GPR(R8)(r4) 1216 ld r9, VCPU_GPR(R9)(r4) 1217 ld r10, VCPU_GPR(R10)(r4) 1218 ld r11, VCPU_GPR(R11)(r4) 1219 ld r12, VCPU_GPR(R12)(r4) 1220 ld r13, VCPU_GPR(R13)(r4) 1221 1222BEGIN_FTR_SECTION 1223 mtspr SPRN_PPR, r0 1224END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1225 1226/* Move canary into DSISR to check for later */ 1227BEGIN_FTR_SECTION 1228 li r0, 0x7fff 1229 mtspr SPRN_HDSISR, r0 1230END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1231 1232 ld r0, VCPU_GPR(R0)(r4) 1233 ld r4, VCPU_GPR(R4)(r4) 1234 HRFI_TO_GUEST 1235 b . 1236 1237secondary_too_late: 1238 li r12, 0 1239 stw r12, STACK_SLOT_TRAP(r1) 1240 cmpdi r4, 0 1241 beq 11f 1242 stw r12, VCPU_TRAP(r4) 1243#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1244 addi r3, r4, VCPU_TB_RMEXIT 1245 bl kvmhv_accumulate_time 1246#endif 124711: b kvmhv_switch_to_host 1248 1249no_switch_exit: 1250 HMT_MEDIUM 1251 li r12, 0 1252 b 12f 1253hdec_soon: 1254 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 125512: stw r12, VCPU_TRAP(r4) 1256 mr r9, r4 1257#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1258 addi r3, r4, VCPU_TB_RMEXIT 1259 bl kvmhv_accumulate_time 1260#endif 1261 b guest_bypass 1262 1263/****************************************************************************** 1264 * * 1265 * Exit code * 1266 * * 1267 *****************************************************************************/ 1268 1269/* 1270 * We come here from the first-level interrupt handlers. 1271 */ 1272 .globl kvmppc_interrupt_hv 1273kvmppc_interrupt_hv: 1274 /* 1275 * Register contents: 1276 * R12 = (guest CR << 32) | interrupt vector 1277 * R13 = PACA 1278 * guest R12 saved in shadow VCPU SCRATCH0 1279 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1280 * guest R13 saved in SPRN_SCRATCH0 1281 */ 1282 std r9, HSTATE_SCRATCH2(r13) 1283 lbz r9, HSTATE_IN_GUEST(r13) 1284 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1285 beq kvmppc_bad_host_intr 1286#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1287 cmpwi r9, KVM_GUEST_MODE_GUEST 1288 ld r9, HSTATE_SCRATCH2(r13) 1289 beq kvmppc_interrupt_pr 1290#endif 1291 /* We're now back in the host but in guest MMU context */ 1292 li r9, KVM_GUEST_MODE_HOST_HV 1293 stb r9, HSTATE_IN_GUEST(r13) 1294 1295 ld r9, HSTATE_KVM_VCPU(r13) 1296 1297 /* Save registers */ 1298 1299 std r0, VCPU_GPR(R0)(r9) 1300 std r1, VCPU_GPR(R1)(r9) 1301 std r2, VCPU_GPR(R2)(r9) 1302 std r3, VCPU_GPR(R3)(r9) 1303 std r4, VCPU_GPR(R4)(r9) 1304 std r5, VCPU_GPR(R5)(r9) 1305 std r6, VCPU_GPR(R6)(r9) 1306 std r7, VCPU_GPR(R7)(r9) 1307 std r8, VCPU_GPR(R8)(r9) 1308 ld r0, HSTATE_SCRATCH2(r13) 1309 std r0, VCPU_GPR(R9)(r9) 1310 std r10, VCPU_GPR(R10)(r9) 1311 std r11, VCPU_GPR(R11)(r9) 1312 ld r3, HSTATE_SCRATCH0(r13) 1313 std r3, VCPU_GPR(R12)(r9) 1314 /* CR is in the high half of r12 */ 1315 srdi r4, r12, 32 1316 stw r4, VCPU_CR(r9) 1317BEGIN_FTR_SECTION 1318 ld r3, HSTATE_CFAR(r13) 1319 std r3, VCPU_CFAR(r9) 1320END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1321BEGIN_FTR_SECTION 1322 ld r4, HSTATE_PPR(r13) 1323 std r4, VCPU_PPR(r9) 1324END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1325 1326 /* Restore R1/R2 so we can handle faults */ 1327 ld r1, HSTATE_HOST_R1(r13) 1328 ld r2, PACATOC(r13) 1329 1330 mfspr r10, SPRN_SRR0 1331 mfspr r11, SPRN_SRR1 1332 std r10, VCPU_SRR0(r9) 1333 std r11, VCPU_SRR1(r9) 1334 /* trap is in the low half of r12, clear CR from the high half */ 1335 clrldi r12, r12, 32 1336 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1337 beq 1f 1338 mfspr r10, SPRN_HSRR0 1339 mfspr r11, SPRN_HSRR1 1340 clrrdi r12, r12, 2 13411: std r10, VCPU_PC(r9) 1342 std r11, VCPU_MSR(r9) 1343 1344 GET_SCRATCH0(r3) 1345 mflr r4 1346 std r3, VCPU_GPR(R13)(r9) 1347 std r4, VCPU_LR(r9) 1348 1349 stw r12,VCPU_TRAP(r9) 1350 1351 /* 1352 * Now that we have saved away SRR0/1 and HSRR0/1, 1353 * interrupts are recoverable in principle, so set MSR_RI. 1354 * This becomes important for relocation-on interrupts from 1355 * the guest, which we can get in radix mode on POWER9. 1356 */ 1357 li r0, MSR_RI 1358 mtmsrd r0, 1 1359 1360#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1361 addi r3, r9, VCPU_TB_RMINTR 1362 mr r4, r9 1363 bl kvmhv_accumulate_time 1364 ld r5, VCPU_GPR(R5)(r9) 1365 ld r6, VCPU_GPR(R6)(r9) 1366 ld r7, VCPU_GPR(R7)(r9) 1367 ld r8, VCPU_GPR(R8)(r9) 1368#endif 1369 1370 /* Save HEIR (HV emulation assist reg) in emul_inst 1371 if this is an HEI (HV emulation interrupt, e40) */ 1372 li r3,KVM_INST_FETCH_FAILED 1373 stw r3,VCPU_LAST_INST(r9) 1374 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1375 bne 11f 1376 mfspr r3,SPRN_HEIR 137711: stw r3,VCPU_HEIR(r9) 1378 1379 /* these are volatile across C function calls */ 1380#ifdef CONFIG_RELOCATABLE 1381 ld r3, HSTATE_SCRATCH1(r13) 1382 mtctr r3 1383#else 1384 mfctr r3 1385#endif 1386 mfxer r4 1387 std r3, VCPU_CTR(r9) 1388 std r4, VCPU_XER(r9) 1389 1390#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1391 /* For softpatch interrupt, go off and do TM instruction emulation */ 1392 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1393 beq kvmppc_tm_emul 1394#endif 1395 1396 /* If this is a page table miss then see if it's theirs or ours */ 1397 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1398 beq kvmppc_hdsi 1399 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1400 beq kvmppc_hisi 1401 1402 /* See if this is a leftover HDEC interrupt */ 1403 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1404 bne 2f 1405 mfspr r3,SPRN_HDEC 1406 EXTEND_HDEC(r3) 1407 cmpdi r3,0 1408 mr r4,r9 1409 bge fast_guest_return 14102: 1411 /* See if this is an hcall we can handle in real mode */ 1412 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1413 beq hcall_try_real_mode 1414 1415 /* Hypervisor doorbell - exit only if host IPI flag set */ 1416 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1417 bne 3f 1418BEGIN_FTR_SECTION 1419 PPC_MSGSYNC 1420 lwsync 1421END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1422 lbz r0, HSTATE_HOST_IPI(r13) 1423 cmpwi r0, 0 1424 beq 4f 1425 b guest_exit_cont 14263: 1427 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1428 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1429 bne 14f 1430 mfspr r3, SPRN_HFSCR 1431 std r3, VCPU_HFSCR(r9) 1432 b guest_exit_cont 143314: 1434 /* External interrupt ? */ 1435 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1436 bne+ guest_exit_cont 1437 1438 /* External interrupt, first check for host_ipi. If this is 1439 * set, we know the host wants us out so let's do it now 1440 */ 1441 bl kvmppc_read_intr 1442 1443 /* 1444 * Restore the active volatile registers after returning from 1445 * a C function. 1446 */ 1447 ld r9, HSTATE_KVM_VCPU(r13) 1448 li r12, BOOK3S_INTERRUPT_EXTERNAL 1449 1450 /* 1451 * kvmppc_read_intr return codes: 1452 * 1453 * Exit to host (r3 > 0) 1454 * 1 An interrupt is pending that needs to be handled by the host 1455 * Exit guest and return to host by branching to guest_exit_cont 1456 * 1457 * 2 Passthrough that needs completion in the host 1458 * Exit guest and return to host by branching to guest_exit_cont 1459 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1460 * to indicate to the host to complete handling the interrupt 1461 * 1462 * Before returning to guest, we check if any CPU is heading out 1463 * to the host and if so, we head out also. If no CPUs are heading 1464 * check return values <= 0. 1465 * 1466 * Return to guest (r3 <= 0) 1467 * 0 No external interrupt is pending 1468 * -1 A guest wakeup IPI (which has now been cleared) 1469 * In either case, we return to guest to deliver any pending 1470 * guest interrupts. 1471 * 1472 * -2 A PCI passthrough external interrupt was handled 1473 * (interrupt was delivered directly to guest) 1474 * Return to guest to deliver any pending guest interrupts. 1475 */ 1476 1477 cmpdi r3, 1 1478 ble 1f 1479 1480 /* Return code = 2 */ 1481 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1482 stw r12, VCPU_TRAP(r9) 1483 b guest_exit_cont 1484 14851: /* Return code <= 1 */ 1486 cmpdi r3, 0 1487 bgt guest_exit_cont 1488 1489 /* Return code <= 0 */ 14904: ld r5, HSTATE_KVM_VCORE(r13) 1491 lwz r0, VCORE_ENTRY_EXIT(r5) 1492 cmpwi r0, 0x100 1493 mr r4, r9 1494 blt deliver_guest_interrupt 1495 1496guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1497 /* Save more register state */ 1498 mfdar r6 1499 mfdsisr r7 1500 std r6, VCPU_DAR(r9) 1501 stw r7, VCPU_DSISR(r9) 1502 /* don't overwrite fault_dar/fault_dsisr if HDSI */ 1503 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE 1504 beq mc_cont 1505 std r6, VCPU_FAULT_DAR(r9) 1506 stw r7, VCPU_FAULT_DSISR(r9) 1507 1508 /* See if it is a machine check */ 1509 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1510 beq machine_check_realmode 1511mc_cont: 1512#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1513 addi r3, r9, VCPU_TB_RMEXIT 1514 mr r4, r9 1515 bl kvmhv_accumulate_time 1516#endif 1517#ifdef CONFIG_KVM_XICS 1518 /* We are exiting, pull the VP from the XIVE */ 1519 lbz r0, VCPU_XIVE_PUSHED(r9) 1520 cmpwi cr0, r0, 0 1521 beq 1f 1522 li r7, TM_SPC_PULL_OS_CTX 1523 li r6, TM_QW1_OS 1524 mfmsr r0 1525 andi. r0, r0, MSR_DR /* in real mode? */ 1526 beq 2f 1527 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1528 cmpldi cr0, r10, 0 1529 beq 1f 1530 /* First load to pull the context, we ignore the value */ 1531 eieio 1532 lwzx r11, r7, r10 1533 /* Second load to recover the context state (Words 0 and 1) */ 1534 ldx r11, r6, r10 1535 b 3f 15362: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1537 cmpldi cr0, r10, 0 1538 beq 1f 1539 /* First load to pull the context, we ignore the value */ 1540 eieio 1541 lwzcix r11, r7, r10 1542 /* Second load to recover the context state (Words 0 and 1) */ 1543 ldcix r11, r6, r10 15443: std r11, VCPU_XIVE_SAVED_STATE(r9) 1545 /* Fixup some of the state for the next load */ 1546 li r10, 0 1547 li r0, 0xff 1548 stb r10, VCPU_XIVE_PUSHED(r9) 1549 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1550 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1551 eieio 15521: 1553#endif /* CONFIG_KVM_XICS */ 1554 1555 /* For hash guest, read the guest SLB and save it away */ 1556 ld r5, VCPU_KVM(r9) 1557 lbz r0, KVM_RADIX(r5) 1558 li r5, 0 1559 cmpwi r0, 0 1560 bne 3f /* for radix, save 0 entries */ 1561 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1562 mtctr r0 1563 li r6,0 1564 addi r7,r9,VCPU_SLB 15651: slbmfee r8,r6 1566 andis. r0,r8,SLB_ESID_V@h 1567 beq 2f 1568 add r8,r8,r6 /* put index in */ 1569 slbmfev r3,r6 1570 std r8,VCPU_SLB_E(r7) 1571 std r3,VCPU_SLB_V(r7) 1572 addi r7,r7,VCPU_SLB_SIZE 1573 addi r5,r5,1 15742: addi r6,r6,1 1575 bdnz 1b 1576 /* Finally clear out the SLB */ 1577 li r0,0 1578 slbmte r0,r0 1579 slbia 1580 ptesync 15813: stw r5,VCPU_SLB_MAX(r9) 1582 1583 /* load host SLB entries */ 1584BEGIN_MMU_FTR_SECTION 1585 b 0f 1586END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1587 ld r8,PACA_SLBSHADOWPTR(r13) 1588 1589 .rept SLB_NUM_BOLTED 1590 li r3, SLBSHADOW_SAVEAREA 1591 LDX_BE r5, r8, r3 1592 addi r3, r3, 8 1593 LDX_BE r6, r8, r3 1594 andis. r7,r5,SLB_ESID_V@h 1595 beq 1f 1596 slbmte r6,r5 15971: addi r8,r8,16 1598 .endr 15990: 1600 1601guest_bypass: 1602 stw r12, STACK_SLOT_TRAP(r1) 1603 1604 /* Save DEC */ 1605 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1606 ld r3, HSTATE_KVM_VCORE(r13) 1607 mfspr r5,SPRN_DEC 1608 mftb r6 1609 /* On P9, if the guest has large decr enabled, don't sign extend */ 1610BEGIN_FTR_SECTION 1611 ld r4, VCORE_LPCR(r3) 1612 andis. r4, r4, LPCR_LD@h 1613 bne 16f 1614END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1615 extsw r5,r5 161616: add r5,r5,r6 1617 /* r5 is a guest timebase value here, convert to host TB */ 1618 ld r4,VCORE_TB_OFFSET_APPL(r3) 1619 subf r5,r4,r5 1620 std r5,VCPU_DEC_EXPIRES(r9) 1621 1622 /* Increment exit count, poke other threads to exit */ 1623 mr r3, r12 1624 bl kvmhv_commence_exit 1625 nop 1626 ld r9, HSTATE_KVM_VCPU(r13) 1627 1628 /* Stop others sending VCPU interrupts to this physical CPU */ 1629 li r0, -1 1630 stw r0, VCPU_CPU(r9) 1631 stw r0, VCPU_THREAD_CPU(r9) 1632 1633 /* Save guest CTRL register, set runlatch to 1 */ 1634 mfspr r6,SPRN_CTRLF 1635 stw r6,VCPU_CTRL(r9) 1636 andi. r0,r6,1 1637 bne 4f 1638 ori r6,r6,1 1639 mtspr SPRN_CTRLT,r6 16404: 1641 /* 1642 * Save the guest PURR/SPURR 1643 */ 1644 mfspr r5,SPRN_PURR 1645 mfspr r6,SPRN_SPURR 1646 ld r7,VCPU_PURR(r9) 1647 ld r8,VCPU_SPURR(r9) 1648 std r5,VCPU_PURR(r9) 1649 std r6,VCPU_SPURR(r9) 1650 subf r5,r7,r5 1651 subf r6,r8,r6 1652 1653 /* 1654 * Restore host PURR/SPURR and add guest times 1655 * so that the time in the guest gets accounted. 1656 */ 1657 ld r3,HSTATE_PURR(r13) 1658 ld r4,HSTATE_SPURR(r13) 1659 add r3,r3,r5 1660 add r4,r4,r6 1661 mtspr SPRN_PURR,r3 1662 mtspr SPRN_SPURR,r4 1663 1664BEGIN_FTR_SECTION 1665 b 8f 1666END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1667 /* Save POWER8-specific registers */ 1668 mfspr r5, SPRN_IAMR 1669 mfspr r6, SPRN_PSPB 1670 mfspr r7, SPRN_FSCR 1671 std r5, VCPU_IAMR(r9) 1672 stw r6, VCPU_PSPB(r9) 1673 std r7, VCPU_FSCR(r9) 1674 mfspr r5, SPRN_IC 1675 mfspr r7, SPRN_TAR 1676 std r5, VCPU_IC(r9) 1677 std r7, VCPU_TAR(r9) 1678 mfspr r8, SPRN_EBBHR 1679 std r8, VCPU_EBBHR(r9) 1680 mfspr r5, SPRN_EBBRR 1681 mfspr r6, SPRN_BESCR 1682 mfspr r7, SPRN_PID 1683 mfspr r8, SPRN_WORT 1684 std r5, VCPU_EBBRR(r9) 1685 std r6, VCPU_BESCR(r9) 1686 stw r7, VCPU_GUEST_PID(r9) 1687 std r8, VCPU_WORT(r9) 1688BEGIN_FTR_SECTION 1689 mfspr r5, SPRN_TCSCR 1690 mfspr r6, SPRN_ACOP 1691 mfspr r7, SPRN_CSIGR 1692 mfspr r8, SPRN_TACR 1693 std r5, VCPU_TCSCR(r9) 1694 std r6, VCPU_ACOP(r9) 1695 std r7, VCPU_CSIGR(r9) 1696 std r8, VCPU_TACR(r9) 1697FTR_SECTION_ELSE 1698 mfspr r5, SPRN_TIDR 1699 mfspr r6, SPRN_PSSCR 1700 std r5, VCPU_TID(r9) 1701 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1702 rotldi r6, r6, 60 1703 std r6, VCPU_PSSCR(r9) 1704 /* Restore host HFSCR value */ 1705 ld r7, STACK_SLOT_HFSCR(r1) 1706 mtspr SPRN_HFSCR, r7 1707ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1708 /* 1709 * Restore various registers to 0, where non-zero values 1710 * set by the guest could disrupt the host. 1711 */ 1712 li r0, 0 1713 mtspr SPRN_PSPB, r0 1714 mtspr SPRN_WORT, r0 1715BEGIN_FTR_SECTION 1716 mtspr SPRN_IAMR, r0 1717 mtspr SPRN_TCSCR, r0 1718 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1719 li r0, 1 1720 sldi r0, r0, 31 1721 mtspr SPRN_MMCRS, r0 1722END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 17238: 1724 1725 /* Save and reset AMR and UAMOR before turning on the MMU */ 1726 mfspr r5,SPRN_AMR 1727 mfspr r6,SPRN_UAMOR 1728 std r5,VCPU_AMR(r9) 1729 std r6,VCPU_UAMOR(r9) 1730 li r6,0 1731 mtspr SPRN_AMR,r6 1732 mtspr SPRN_UAMOR, r6 1733 1734 /* Switch DSCR back to host value */ 1735 mfspr r8, SPRN_DSCR 1736 ld r7, HSTATE_DSCR(r13) 1737 std r8, VCPU_DSCR(r9) 1738 mtspr SPRN_DSCR, r7 1739 1740 /* Save non-volatile GPRs */ 1741 std r14, VCPU_GPR(R14)(r9) 1742 std r15, VCPU_GPR(R15)(r9) 1743 std r16, VCPU_GPR(R16)(r9) 1744 std r17, VCPU_GPR(R17)(r9) 1745 std r18, VCPU_GPR(R18)(r9) 1746 std r19, VCPU_GPR(R19)(r9) 1747 std r20, VCPU_GPR(R20)(r9) 1748 std r21, VCPU_GPR(R21)(r9) 1749 std r22, VCPU_GPR(R22)(r9) 1750 std r23, VCPU_GPR(R23)(r9) 1751 std r24, VCPU_GPR(R24)(r9) 1752 std r25, VCPU_GPR(R25)(r9) 1753 std r26, VCPU_GPR(R26)(r9) 1754 std r27, VCPU_GPR(R27)(r9) 1755 std r28, VCPU_GPR(R28)(r9) 1756 std r29, VCPU_GPR(R29)(r9) 1757 std r30, VCPU_GPR(R30)(r9) 1758 std r31, VCPU_GPR(R31)(r9) 1759 1760 /* Save SPRGs */ 1761 mfspr r3, SPRN_SPRG0 1762 mfspr r4, SPRN_SPRG1 1763 mfspr r5, SPRN_SPRG2 1764 mfspr r6, SPRN_SPRG3 1765 std r3, VCPU_SPRG0(r9) 1766 std r4, VCPU_SPRG1(r9) 1767 std r5, VCPU_SPRG2(r9) 1768 std r6, VCPU_SPRG3(r9) 1769 1770 /* save FP state */ 1771 mr r3, r9 1772 bl kvmppc_save_fp 1773 1774#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1775/* 1776 * Branch around the call if both CPU_FTR_TM and 1777 * CPU_FTR_P9_TM_HV_ASSIST are off. 1778 */ 1779BEGIN_FTR_SECTION 1780 b 91f 1781END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 1782 /* 1783 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 1784 */ 1785 mr r3, r9 1786 ld r4, VCPU_MSR(r3) 1787 bl kvmppc_save_tm_hv 1788 ld r9, HSTATE_KVM_VCPU(r13) 178991: 1790#endif 1791 1792 /* Increment yield count if they have a VPA */ 1793 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1794 cmpdi r8, 0 1795 beq 25f 1796 li r4, LPPACA_YIELDCOUNT 1797 LWZX_BE r3, r8, r4 1798 addi r3, r3, 1 1799 STWX_BE r3, r8, r4 1800 li r3, 1 1801 stb r3, VCPU_VPA_DIRTY(r9) 180225: 1803 /* Save PMU registers if requested */ 1804 /* r8 and cr0.eq are live here */ 1805BEGIN_FTR_SECTION 1806 /* 1807 * POWER8 seems to have a hardware bug where setting 1808 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 1809 * when some counters are already negative doesn't seem 1810 * to cause a performance monitor alert (and hence interrupt). 1811 * The effect of this is that when saving the PMU state, 1812 * if there is no PMU alert pending when we read MMCR0 1813 * before freezing the counters, but one becomes pending 1814 * before we read the counters, we lose it. 1815 * To work around this, we need a way to freeze the counters 1816 * before reading MMCR0. Normally, freezing the counters 1817 * is done by writing MMCR0 (to set MMCR0[FC]) which 1818 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 1819 * we can also freeze the counters using MMCR2, by writing 1820 * 1s to all the counter freeze condition bits (there are 1821 * 9 bits each for 6 counters). 1822 */ 1823 li r3, -1 /* set all freeze bits */ 1824 clrrdi r3, r3, 10 1825 mfspr r10, SPRN_MMCR2 1826 mtspr SPRN_MMCR2, r3 1827 isync 1828END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1829 li r3, 1 1830 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 1831 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 1832 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 1833 mfspr r6, SPRN_MMCRA 1834 /* Clear MMCRA in order to disable SDAR updates */ 1835 li r7, 0 1836 mtspr SPRN_MMCRA, r7 1837 isync 1838 beq 21f /* if no VPA, save PMU stuff anyway */ 1839 lbz r7, LPPACA_PMCINUSE(r8) 1840 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */ 1841 bne 21f 1842 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 1843 b 22f 184421: mfspr r5, SPRN_MMCR1 1845 mfspr r7, SPRN_SIAR 1846 mfspr r8, SPRN_SDAR 1847 std r4, VCPU_MMCR(r9) 1848 std r5, VCPU_MMCR + 8(r9) 1849 std r6, VCPU_MMCR + 16(r9) 1850BEGIN_FTR_SECTION 1851 std r10, VCPU_MMCR + 24(r9) 1852END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1853 std r7, VCPU_SIAR(r9) 1854 std r8, VCPU_SDAR(r9) 1855 mfspr r3, SPRN_PMC1 1856 mfspr r4, SPRN_PMC2 1857 mfspr r5, SPRN_PMC3 1858 mfspr r6, SPRN_PMC4 1859 mfspr r7, SPRN_PMC5 1860 mfspr r8, SPRN_PMC6 1861 stw r3, VCPU_PMC(r9) 1862 stw r4, VCPU_PMC + 4(r9) 1863 stw r5, VCPU_PMC + 8(r9) 1864 stw r6, VCPU_PMC + 12(r9) 1865 stw r7, VCPU_PMC + 16(r9) 1866 stw r8, VCPU_PMC + 20(r9) 1867BEGIN_FTR_SECTION 1868 mfspr r5, SPRN_SIER 1869 std r5, VCPU_SIER(r9) 1870BEGIN_FTR_SECTION_NESTED(96) 1871 mfspr r6, SPRN_SPMC1 1872 mfspr r7, SPRN_SPMC2 1873 mfspr r8, SPRN_MMCRS 1874 stw r6, VCPU_PMC + 24(r9) 1875 stw r7, VCPU_PMC + 28(r9) 1876 std r8, VCPU_MMCR + 32(r9) 1877 lis r4, 0x8000 1878 mtspr SPRN_MMCRS, r4 1879END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 1880END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 188122: 1882 1883 /* Restore host values of some registers */ 1884BEGIN_FTR_SECTION 1885 ld r5, STACK_SLOT_CIABR(r1) 1886 ld r6, STACK_SLOT_DAWR(r1) 1887 ld r7, STACK_SLOT_DAWRX(r1) 1888 mtspr SPRN_CIABR, r5 1889 /* 1890 * If the DAWR doesn't work, it's ok to write these here as 1891 * this value should always be zero 1892 */ 1893 mtspr SPRN_DAWR, r6 1894 mtspr SPRN_DAWRX, r7 1895END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1896BEGIN_FTR_SECTION 1897 ld r5, STACK_SLOT_TID(r1) 1898 ld r6, STACK_SLOT_PSSCR(r1) 1899 ld r7, STACK_SLOT_PID(r1) 1900 ld r8, STACK_SLOT_IAMR(r1) 1901 mtspr SPRN_TIDR, r5 1902 mtspr SPRN_PSSCR, r6 1903 mtspr SPRN_PID, r7 1904 mtspr SPRN_IAMR, r8 1905END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1906 1907#ifdef CONFIG_PPC_RADIX_MMU 1908 /* 1909 * Are we running hash or radix ? 1910 */ 1911 ld r5, VCPU_KVM(r9) 1912 lbz r0, KVM_RADIX(r5) 1913 cmpwi cr2, r0, 0 1914 beq cr2, 2f 1915 1916 /* 1917 * Radix: do eieio; tlbsync; ptesync sequence in case we 1918 * interrupted the guest between a tlbie and a ptesync. 1919 */ 1920 eieio 1921 tlbsync 1922 ptesync 1923 1924 /* Radix: Handle the case where the guest used an illegal PID */ 1925 LOAD_REG_ADDR(r4, mmu_base_pid) 1926 lwz r3, VCPU_GUEST_PID(r9) 1927 lwz r5, 0(r4) 1928 cmpw cr0,r3,r5 1929 blt 2f 1930 1931 /* 1932 * Illegal PID, the HW might have prefetched and cached in the TLB 1933 * some translations for the LPID 0 / guest PID combination which 1934 * Linux doesn't know about, so we need to flush that PID out of 1935 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1936 * the right context. 1937 */ 1938 li r0,0 1939 mtspr SPRN_LPID,r0 1940 isync 1941 1942 /* Then do a congruence class local flush */ 1943 ld r6,VCPU_KVM(r9) 1944 lwz r0,KVM_TLB_SETS(r6) 1945 mtctr r0 1946 li r7,0x400 /* IS field = 0b01 */ 1947 ptesync 1948 sldi r0,r3,32 /* RS has PID */ 19491: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1950 addi r7,r7,0x1000 1951 bdnz 1b 1952 ptesync 1953 19542: 1955#endif /* CONFIG_PPC_RADIX_MMU */ 1956 1957 /* 1958 * POWER7/POWER8 guest -> host partition switch code. 1959 * We don't have to lock against tlbies but we do 1960 * have to coordinate the hardware threads. 1961 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1962 */ 1963kvmhv_switch_to_host: 1964 /* Secondary threads wait for primary to do partition switch */ 1965 ld r5,HSTATE_KVM_VCORE(r13) 1966 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1967 lbz r3,HSTATE_PTID(r13) 1968 cmpwi r3,0 1969 beq 15f 1970 HMT_LOW 197113: lbz r3,VCORE_IN_GUEST(r5) 1972 cmpwi r3,0 1973 bne 13b 1974 HMT_MEDIUM 1975 b 16f 1976 1977 /* Primary thread waits for all the secondaries to exit guest */ 197815: lwz r3,VCORE_ENTRY_EXIT(r5) 1979 rlwinm r0,r3,32-8,0xff 1980 clrldi r3,r3,56 1981 cmpw r3,r0 1982 bne 15b 1983 isync 1984 1985 /* Did we actually switch to the guest at all? */ 1986 lbz r6, VCORE_IN_GUEST(r5) 1987 cmpwi r6, 0 1988 beq 19f 1989 1990 /* Primary thread switches back to host partition */ 1991 lwz r7,KVM_HOST_LPID(r4) 1992BEGIN_FTR_SECTION 1993 ld r6,KVM_HOST_SDR1(r4) 1994 li r8,LPID_RSVD /* switch to reserved LPID */ 1995 mtspr SPRN_LPID,r8 1996 ptesync 1997 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1998END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1999 mtspr SPRN_LPID,r7 2000 isync 2001 2002BEGIN_FTR_SECTION 2003 /* DPDES and VTB are shared between threads */ 2004 mfspr r7, SPRN_DPDES 2005 mfspr r8, SPRN_VTB 2006 std r7, VCORE_DPDES(r5) 2007 std r8, VCORE_VTB(r5) 2008 /* clear DPDES so we don't get guest doorbells in the host */ 2009 li r8, 0 2010 mtspr SPRN_DPDES, r8 2011END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2012 2013 /* If HMI, call kvmppc_realmode_hmi_handler() */ 2014 lwz r12, STACK_SLOT_TRAP(r1) 2015 cmpwi r12, BOOK3S_INTERRUPT_HMI 2016 bne 27f 2017 bl kvmppc_realmode_hmi_handler 2018 nop 2019 cmpdi r3, 0 2020 /* 2021 * At this point kvmppc_realmode_hmi_handler may have resync-ed 2022 * the TB, and if it has, we must not subtract the guest timebase 2023 * offset from the timebase. So, skip it. 2024 * 2025 * Also, do not call kvmppc_subcore_exit_guest() because it has 2026 * been invoked as part of kvmppc_realmode_hmi_handler(). 2027 */ 2028 beq 30f 2029 203027: 2031 /* Subtract timebase offset from timebase */ 2032 ld r8, VCORE_TB_OFFSET_APPL(r5) 2033 cmpdi r8,0 2034 beq 17f 2035 li r0, 0 2036 std r0, VCORE_TB_OFFSET_APPL(r5) 2037 mftb r6 /* current guest timebase */ 2038 subf r8,r8,r6 2039 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 2040 mftb r7 /* check if lower 24 bits overflowed */ 2041 clrldi r6,r6,40 2042 clrldi r7,r7,40 2043 cmpld r7,r6 2044 bge 17f 2045 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 2046 mtspr SPRN_TBU40,r8 2047 204817: bl kvmppc_subcore_exit_guest 2049 nop 205030: ld r5,HSTATE_KVM_VCORE(r13) 2051 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 2052 2053 /* Reset PCR */ 2054 ld r0, VCORE_PCR(r5) 2055 cmpdi r0, 0 2056 beq 18f 2057 li r0, 0 2058 mtspr SPRN_PCR, r0 205918: 2060 /* Signal secondary CPUs to continue */ 2061 stb r0,VCORE_IN_GUEST(r5) 206219: lis r8,0x7fff /* MAX_INT@h */ 2063 mtspr SPRN_HDEC,r8 2064 206516: 2066BEGIN_FTR_SECTION 2067 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 2068 ld r3, HSTATE_SPLIT_MODE(r13) 2069 cmpdi r3, 0 2070 beq 47f 2071 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 2072 cmpwi r8, 0 2073 beq 47f 2074 bl kvmhv_p9_restore_lpcr 2075 nop 2076 b 48f 207747: 2078END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2079 ld r8,KVM_HOST_LPCR(r4) 2080 mtspr SPRN_LPCR,r8 2081 isync 208248: 2083#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2084 /* Finish timing, if we have a vcpu */ 2085 ld r4, HSTATE_KVM_VCPU(r13) 2086 cmpdi r4, 0 2087 li r3, 0 2088 beq 2f 2089 bl kvmhv_accumulate_time 20902: 2091#endif 2092 /* Unset guest mode */ 2093 li r0, KVM_GUEST_MODE_NONE 2094 stb r0, HSTATE_IN_GUEST(r13) 2095 2096 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 2097 ld r0, SFS+PPC_LR_STKOFF(r1) 2098 addi r1, r1, SFS 2099 mtlr r0 2100 blr 2101 2102#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2103/* 2104 * Softpatch interrupt for transactional memory emulation cases 2105 * on POWER9 DD2.2. This is early in the guest exit path - we 2106 * haven't saved registers or done a treclaim yet. 2107 */ 2108kvmppc_tm_emul: 2109 /* Save instruction image in HEIR */ 2110 mfspr r3, SPRN_HEIR 2111 stw r3, VCPU_HEIR(r9) 2112 2113 /* 2114 * The cases we want to handle here are those where the guest 2115 * is in real suspend mode and is trying to transition to 2116 * transactional mode. 2117 */ 2118 lbz r0, HSTATE_FAKE_SUSPEND(r13) 2119 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 2120 bne guest_exit_cont 2121 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 2122 cmpwi r3, 1 /* or if not in suspend state */ 2123 bne guest_exit_cont 2124 2125 /* Call C code to do the emulation */ 2126 mr r3, r9 2127 bl kvmhv_p9_tm_emulation_early 2128 nop 2129 ld r9, HSTATE_KVM_VCPU(r13) 2130 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 2131 cmpwi r3, 0 2132 beq guest_exit_cont /* continue exiting if not handled */ 2133 ld r10, VCPU_PC(r9) 2134 ld r11, VCPU_MSR(r9) 2135 b fast_interrupt_c_return /* go back to guest if handled */ 2136#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2137 2138/* 2139 * Check whether an HDSI is an HPTE not found fault or something else. 2140 * If it is an HPTE not found fault that is due to the guest accessing 2141 * a page that they have mapped but which we have paged out, then 2142 * we continue on with the guest exit path. In all other cases, 2143 * reflect the HDSI to the guest as a DSI. 2144 */ 2145kvmppc_hdsi: 2146 ld r3, VCPU_KVM(r9) 2147 lbz r0, KVM_RADIX(r3) 2148 mfspr r4, SPRN_HDAR 2149 mfspr r6, SPRN_HDSISR 2150BEGIN_FTR_SECTION 2151 /* Look for DSISR canary. If we find it, retry instruction */ 2152 cmpdi r6, 0x7fff 2153 beq 6f 2154END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2155 cmpwi r0, 0 2156 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2157 /* HPTE not found fault or protection fault? */ 2158 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2159 beq 1f /* if not, send it to the guest */ 2160 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2161 beq 3f 2162BEGIN_FTR_SECTION 2163 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2164 b 4f 2165END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2166 clrrdi r0, r4, 28 2167 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2168 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2169 bne 7f /* if no SLB entry found */ 21704: std r4, VCPU_FAULT_DAR(r9) 2171 stw r6, VCPU_FAULT_DSISR(r9) 2172 2173 /* Search the hash table. */ 2174 mr r3, r9 /* vcpu pointer */ 2175 li r7, 1 /* data fault */ 2176 bl kvmppc_hpte_hv_fault 2177 ld r9, HSTATE_KVM_VCPU(r13) 2178 ld r10, VCPU_PC(r9) 2179 ld r11, VCPU_MSR(r9) 2180 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2181 cmpdi r3, 0 /* retry the instruction */ 2182 beq 6f 2183 cmpdi r3, -1 /* handle in kernel mode */ 2184 beq guest_exit_cont 2185 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2186 beq 2f 2187 2188 /* Synthesize a DSI (or DSegI) for the guest */ 2189 ld r4, VCPU_FAULT_DAR(r9) 2190 mr r6, r3 21911: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2192 mtspr SPRN_DSISR, r6 21937: mtspr SPRN_DAR, r4 2194 mtspr SPRN_SRR0, r10 2195 mtspr SPRN_SRR1, r11 2196 mr r10, r0 2197 bl kvmppc_msr_interrupt 2198fast_interrupt_c_return: 21996: ld r7, VCPU_CTR(r9) 2200 ld r8, VCPU_XER(r9) 2201 mtctr r7 2202 mtxer r8 2203 mr r4, r9 2204 b fast_guest_return 2205 22063: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2207 ld r5, KVM_VRMA_SLB_V(r5) 2208 b 4b 2209 2210 /* If this is for emulated MMIO, load the instruction word */ 22112: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2212 2213 /* Set guest mode to 'jump over instruction' so if lwz faults 2214 * we'll just continue at the next IP. */ 2215 li r0, KVM_GUEST_MODE_SKIP 2216 stb r0, HSTATE_IN_GUEST(r13) 2217 2218 /* Do the access with MSR:DR enabled */ 2219 mfmsr r3 2220 ori r4, r3, MSR_DR /* Enable paging for data */ 2221 mtmsrd r4 2222 lwz r8, 0(r10) 2223 mtmsrd r3 2224 2225 /* Store the result */ 2226 stw r8, VCPU_LAST_INST(r9) 2227 2228 /* Unset guest mode. */ 2229 li r0, KVM_GUEST_MODE_HOST_HV 2230 stb r0, HSTATE_IN_GUEST(r13) 2231 b guest_exit_cont 2232 2233.Lradix_hdsi: 2234 std r4, VCPU_FAULT_DAR(r9) 2235 stw r6, VCPU_FAULT_DSISR(r9) 2236.Lradix_hisi: 2237 mfspr r5, SPRN_ASDR 2238 std r5, VCPU_FAULT_GPA(r9) 2239 b guest_exit_cont 2240 2241/* 2242 * Similarly for an HISI, reflect it to the guest as an ISI unless 2243 * it is an HPTE not found fault for a page that we have paged out. 2244 */ 2245kvmppc_hisi: 2246 ld r3, VCPU_KVM(r9) 2247 lbz r0, KVM_RADIX(r3) 2248 cmpwi r0, 0 2249 bne .Lradix_hisi /* for radix, just save ASDR */ 2250 andis. r0, r11, SRR1_ISI_NOPT@h 2251 beq 1f 2252 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2253 beq 3f 2254BEGIN_FTR_SECTION 2255 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2256 b 4f 2257END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2258 clrrdi r0, r10, 28 2259 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2260 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2261 bne 7f /* if no SLB entry found */ 22624: 2263 /* Search the hash table. */ 2264 mr r3, r9 /* vcpu pointer */ 2265 mr r4, r10 2266 mr r6, r11 2267 li r7, 0 /* instruction fault */ 2268 bl kvmppc_hpte_hv_fault 2269 ld r9, HSTATE_KVM_VCPU(r13) 2270 ld r10, VCPU_PC(r9) 2271 ld r11, VCPU_MSR(r9) 2272 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2273 cmpdi r3, 0 /* retry the instruction */ 2274 beq fast_interrupt_c_return 2275 cmpdi r3, -1 /* handle in kernel mode */ 2276 beq guest_exit_cont 2277 2278 /* Synthesize an ISI (or ISegI) for the guest */ 2279 mr r11, r3 22801: li r0, BOOK3S_INTERRUPT_INST_STORAGE 22817: mtspr SPRN_SRR0, r10 2282 mtspr SPRN_SRR1, r11 2283 mr r10, r0 2284 bl kvmppc_msr_interrupt 2285 b fast_interrupt_c_return 2286 22873: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2288 ld r5, KVM_VRMA_SLB_V(r6) 2289 b 4b 2290 2291/* 2292 * Try to handle an hcall in real mode. 2293 * Returns to the guest if we handle it, or continues on up to 2294 * the kernel if we can't (i.e. if we don't have a handler for 2295 * it, or if the handler returns H_TOO_HARD). 2296 * 2297 * r5 - r8 contain hcall args, 2298 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2299 */ 2300hcall_try_real_mode: 2301 ld r3,VCPU_GPR(R3)(r9) 2302 andi. r0,r11,MSR_PR 2303 /* sc 1 from userspace - reflect to guest syscall */ 2304 bne sc_1_fast_return 2305 clrrdi r3,r3,2 2306 cmpldi r3,hcall_real_table_end - hcall_real_table 2307 bge guest_exit_cont 2308 /* See if this hcall is enabled for in-kernel handling */ 2309 ld r4, VCPU_KVM(r9) 2310 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2311 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2312 add r4, r4, r0 2313 ld r0, KVM_ENABLED_HCALLS(r4) 2314 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2315 srd r0, r0, r4 2316 andi. r0, r0, 1 2317 beq guest_exit_cont 2318 /* Get pointer to handler, if any, and call it */ 2319 LOAD_REG_ADDR(r4, hcall_real_table) 2320 lwax r3,r3,r4 2321 cmpwi r3,0 2322 beq guest_exit_cont 2323 add r12,r3,r4 2324 mtctr r12 2325 mr r3,r9 /* get vcpu pointer */ 2326 ld r4,VCPU_GPR(R4)(r9) 2327 bctrl 2328 cmpdi r3,H_TOO_HARD 2329 beq hcall_real_fallback 2330 ld r4,HSTATE_KVM_VCPU(r13) 2331 std r3,VCPU_GPR(R3)(r4) 2332 ld r10,VCPU_PC(r4) 2333 ld r11,VCPU_MSR(r4) 2334 b fast_guest_return 2335 2336sc_1_fast_return: 2337 mtspr SPRN_SRR0,r10 2338 mtspr SPRN_SRR1,r11 2339 li r10, BOOK3S_INTERRUPT_SYSCALL 2340 bl kvmppc_msr_interrupt 2341 mr r4,r9 2342 b fast_guest_return 2343 2344 /* We've attempted a real mode hcall, but it's punted it back 2345 * to userspace. We need to restore some clobbered volatiles 2346 * before resuming the pass-it-to-qemu path */ 2347hcall_real_fallback: 2348 li r12,BOOK3S_INTERRUPT_SYSCALL 2349 ld r9, HSTATE_KVM_VCPU(r13) 2350 2351 b guest_exit_cont 2352 2353 .globl hcall_real_table 2354hcall_real_table: 2355 .long 0 /* 0 - unused */ 2356 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2357 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2358 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2359 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2360 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2361 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2362 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2363 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2364 .long 0 /* 0x24 - H_SET_SPRG0 */ 2365 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2366 .long 0 /* 0x2c */ 2367 .long 0 /* 0x30 */ 2368 .long 0 /* 0x34 */ 2369 .long 0 /* 0x38 */ 2370 .long 0 /* 0x3c */ 2371 .long 0 /* 0x40 */ 2372 .long 0 /* 0x44 */ 2373 .long 0 /* 0x48 */ 2374 .long 0 /* 0x4c */ 2375 .long 0 /* 0x50 */ 2376 .long 0 /* 0x54 */ 2377 .long 0 /* 0x58 */ 2378 .long 0 /* 0x5c */ 2379 .long 0 /* 0x60 */ 2380#ifdef CONFIG_KVM_XICS 2381 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2382 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2383 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2384 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2385 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2386#else 2387 .long 0 /* 0x64 - H_EOI */ 2388 .long 0 /* 0x68 - H_CPPR */ 2389 .long 0 /* 0x6c - H_IPI */ 2390 .long 0 /* 0x70 - H_IPOLL */ 2391 .long 0 /* 0x74 - H_XIRR */ 2392#endif 2393 .long 0 /* 0x78 */ 2394 .long 0 /* 0x7c */ 2395 .long 0 /* 0x80 */ 2396 .long 0 /* 0x84 */ 2397 .long 0 /* 0x88 */ 2398 .long 0 /* 0x8c */ 2399 .long 0 /* 0x90 */ 2400 .long 0 /* 0x94 */ 2401 .long 0 /* 0x98 */ 2402 .long 0 /* 0x9c */ 2403 .long 0 /* 0xa0 */ 2404 .long 0 /* 0xa4 */ 2405 .long 0 /* 0xa8 */ 2406 .long 0 /* 0xac */ 2407 .long 0 /* 0xb0 */ 2408 .long 0 /* 0xb4 */ 2409 .long 0 /* 0xb8 */ 2410 .long 0 /* 0xbc */ 2411 .long 0 /* 0xc0 */ 2412 .long 0 /* 0xc4 */ 2413 .long 0 /* 0xc8 */ 2414 .long 0 /* 0xcc */ 2415 .long 0 /* 0xd0 */ 2416 .long 0 /* 0xd4 */ 2417 .long 0 /* 0xd8 */ 2418 .long 0 /* 0xdc */ 2419 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2420 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2421 .long 0 /* 0xe8 */ 2422 .long 0 /* 0xec */ 2423 .long 0 /* 0xf0 */ 2424 .long 0 /* 0xf4 */ 2425 .long 0 /* 0xf8 */ 2426 .long 0 /* 0xfc */ 2427 .long 0 /* 0x100 */ 2428 .long 0 /* 0x104 */ 2429 .long 0 /* 0x108 */ 2430 .long 0 /* 0x10c */ 2431 .long 0 /* 0x110 */ 2432 .long 0 /* 0x114 */ 2433 .long 0 /* 0x118 */ 2434 .long 0 /* 0x11c */ 2435 .long 0 /* 0x120 */ 2436 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2437 .long 0 /* 0x128 */ 2438 .long 0 /* 0x12c */ 2439 .long 0 /* 0x130 */ 2440 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2441 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2442 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2443 .long 0 /* 0x140 */ 2444 .long 0 /* 0x144 */ 2445 .long 0 /* 0x148 */ 2446 .long 0 /* 0x14c */ 2447 .long 0 /* 0x150 */ 2448 .long 0 /* 0x154 */ 2449 .long 0 /* 0x158 */ 2450 .long 0 /* 0x15c */ 2451 .long 0 /* 0x160 */ 2452 .long 0 /* 0x164 */ 2453 .long 0 /* 0x168 */ 2454 .long 0 /* 0x16c */ 2455 .long 0 /* 0x170 */ 2456 .long 0 /* 0x174 */ 2457 .long 0 /* 0x178 */ 2458 .long 0 /* 0x17c */ 2459 .long 0 /* 0x180 */ 2460 .long 0 /* 0x184 */ 2461 .long 0 /* 0x188 */ 2462 .long 0 /* 0x18c */ 2463 .long 0 /* 0x190 */ 2464 .long 0 /* 0x194 */ 2465 .long 0 /* 0x198 */ 2466 .long 0 /* 0x19c */ 2467 .long 0 /* 0x1a0 */ 2468 .long 0 /* 0x1a4 */ 2469 .long 0 /* 0x1a8 */ 2470 .long 0 /* 0x1ac */ 2471 .long 0 /* 0x1b0 */ 2472 .long 0 /* 0x1b4 */ 2473 .long 0 /* 0x1b8 */ 2474 .long 0 /* 0x1bc */ 2475 .long 0 /* 0x1c0 */ 2476 .long 0 /* 0x1c4 */ 2477 .long 0 /* 0x1c8 */ 2478 .long 0 /* 0x1cc */ 2479 .long 0 /* 0x1d0 */ 2480 .long 0 /* 0x1d4 */ 2481 .long 0 /* 0x1d8 */ 2482 .long 0 /* 0x1dc */ 2483 .long 0 /* 0x1e0 */ 2484 .long 0 /* 0x1e4 */ 2485 .long 0 /* 0x1e8 */ 2486 .long 0 /* 0x1ec */ 2487 .long 0 /* 0x1f0 */ 2488 .long 0 /* 0x1f4 */ 2489 .long 0 /* 0x1f8 */ 2490 .long 0 /* 0x1fc */ 2491 .long 0 /* 0x200 */ 2492 .long 0 /* 0x204 */ 2493 .long 0 /* 0x208 */ 2494 .long 0 /* 0x20c */ 2495 .long 0 /* 0x210 */ 2496 .long 0 /* 0x214 */ 2497 .long 0 /* 0x218 */ 2498 .long 0 /* 0x21c */ 2499 .long 0 /* 0x220 */ 2500 .long 0 /* 0x224 */ 2501 .long 0 /* 0x228 */ 2502 .long 0 /* 0x22c */ 2503 .long 0 /* 0x230 */ 2504 .long 0 /* 0x234 */ 2505 .long 0 /* 0x238 */ 2506 .long 0 /* 0x23c */ 2507 .long 0 /* 0x240 */ 2508 .long 0 /* 0x244 */ 2509 .long 0 /* 0x248 */ 2510 .long 0 /* 0x24c */ 2511 .long 0 /* 0x250 */ 2512 .long 0 /* 0x254 */ 2513 .long 0 /* 0x258 */ 2514 .long 0 /* 0x25c */ 2515 .long 0 /* 0x260 */ 2516 .long 0 /* 0x264 */ 2517 .long 0 /* 0x268 */ 2518 .long 0 /* 0x26c */ 2519 .long 0 /* 0x270 */ 2520 .long 0 /* 0x274 */ 2521 .long 0 /* 0x278 */ 2522 .long 0 /* 0x27c */ 2523 .long 0 /* 0x280 */ 2524 .long 0 /* 0x284 */ 2525 .long 0 /* 0x288 */ 2526 .long 0 /* 0x28c */ 2527 .long 0 /* 0x290 */ 2528 .long 0 /* 0x294 */ 2529 .long 0 /* 0x298 */ 2530 .long 0 /* 0x29c */ 2531 .long 0 /* 0x2a0 */ 2532 .long 0 /* 0x2a4 */ 2533 .long 0 /* 0x2a8 */ 2534 .long 0 /* 0x2ac */ 2535 .long 0 /* 0x2b0 */ 2536 .long 0 /* 0x2b4 */ 2537 .long 0 /* 0x2b8 */ 2538 .long 0 /* 0x2bc */ 2539 .long 0 /* 0x2c0 */ 2540 .long 0 /* 0x2c4 */ 2541 .long 0 /* 0x2c8 */ 2542 .long 0 /* 0x2cc */ 2543 .long 0 /* 0x2d0 */ 2544 .long 0 /* 0x2d4 */ 2545 .long 0 /* 0x2d8 */ 2546 .long 0 /* 0x2dc */ 2547 .long 0 /* 0x2e0 */ 2548 .long 0 /* 0x2e4 */ 2549 .long 0 /* 0x2e8 */ 2550 .long 0 /* 0x2ec */ 2551 .long 0 /* 0x2f0 */ 2552 .long 0 /* 0x2f4 */ 2553 .long 0 /* 0x2f8 */ 2554#ifdef CONFIG_KVM_XICS 2555 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2556#else 2557 .long 0 /* 0x2fc - H_XIRR_X*/ 2558#endif 2559 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2560 .globl hcall_real_table_end 2561hcall_real_table_end: 2562 2563_GLOBAL(kvmppc_h_set_xdabr) 2564 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2565 beq 6f 2566 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2567 andc. r0, r5, r0 2568 beq 3f 25696: li r3, H_PARAMETER 2570 blr 2571 2572_GLOBAL(kvmppc_h_set_dabr) 2573 li r5, DABRX_USER | DABRX_KERNEL 25743: 2575BEGIN_FTR_SECTION 2576 b 2f 2577END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2578 std r4,VCPU_DABR(r3) 2579 stw r5, VCPU_DABRX(r3) 2580 mtspr SPRN_DABRX, r5 2581 /* Work around P7 bug where DABR can get corrupted on mtspr */ 25821: mtspr SPRN_DABR,r4 2583 mfspr r5, SPRN_DABR 2584 cmpd r4, r5 2585 bne 1b 2586 isync 2587 li r3,0 2588 blr 2589 25902: 2591BEGIN_FTR_SECTION 2592 /* POWER9 with disabled DAWR */ 2593 li r3, H_HARDWARE 2594 blr 2595END_FTR_SECTION_IFCLR(CPU_FTR_DAWR) 2596 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2597 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2598 rlwimi r5, r4, 2, DAWRX_WT 2599 clrrdi r4, r4, 3 2600 std r4, VCPU_DAWR(r3) 2601 std r5, VCPU_DAWRX(r3) 2602 mtspr SPRN_DAWR, r4 2603 mtspr SPRN_DAWRX, r5 2604 li r3, 0 2605 blr 2606 2607_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2608 ori r11,r11,MSR_EE 2609 std r11,VCPU_MSR(r3) 2610 li r0,1 2611 stb r0,VCPU_CEDED(r3) 2612 sync /* order setting ceded vs. testing prodded */ 2613 lbz r5,VCPU_PRODDED(r3) 2614 cmpwi r5,0 2615 bne kvm_cede_prodded 2616 li r12,0 /* set trap to 0 to say hcall is handled */ 2617 stw r12,VCPU_TRAP(r3) 2618 li r0,H_SUCCESS 2619 std r0,VCPU_GPR(R3)(r3) 2620 2621 /* 2622 * Set our bit in the bitmask of napping threads unless all the 2623 * other threads are already napping, in which case we send this 2624 * up to the host. 2625 */ 2626 ld r5,HSTATE_KVM_VCORE(r13) 2627 lbz r6,HSTATE_PTID(r13) 2628 lwz r8,VCORE_ENTRY_EXIT(r5) 2629 clrldi r8,r8,56 2630 li r0,1 2631 sld r0,r0,r6 2632 addi r6,r5,VCORE_NAPPING_THREADS 263331: lwarx r4,0,r6 2634 or r4,r4,r0 2635 cmpw r4,r8 2636 beq kvm_cede_exit 2637 stwcx. r4,0,r6 2638 bne 31b 2639 /* order napping_threads update vs testing entry_exit_map */ 2640 isync 2641 li r0,NAPPING_CEDE 2642 stb r0,HSTATE_NAPPING(r13) 2643 lwz r7,VCORE_ENTRY_EXIT(r5) 2644 cmpwi r7,0x100 2645 bge 33f /* another thread already exiting */ 2646 2647/* 2648 * Although not specifically required by the architecture, POWER7 2649 * preserves the following registers in nap mode, even if an SMT mode 2650 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2651 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2652 */ 2653 /* Save non-volatile GPRs */ 2654 std r14, VCPU_GPR(R14)(r3) 2655 std r15, VCPU_GPR(R15)(r3) 2656 std r16, VCPU_GPR(R16)(r3) 2657 std r17, VCPU_GPR(R17)(r3) 2658 std r18, VCPU_GPR(R18)(r3) 2659 std r19, VCPU_GPR(R19)(r3) 2660 std r20, VCPU_GPR(R20)(r3) 2661 std r21, VCPU_GPR(R21)(r3) 2662 std r22, VCPU_GPR(R22)(r3) 2663 std r23, VCPU_GPR(R23)(r3) 2664 std r24, VCPU_GPR(R24)(r3) 2665 std r25, VCPU_GPR(R25)(r3) 2666 std r26, VCPU_GPR(R26)(r3) 2667 std r27, VCPU_GPR(R27)(r3) 2668 std r28, VCPU_GPR(R28)(r3) 2669 std r29, VCPU_GPR(R29)(r3) 2670 std r30, VCPU_GPR(R30)(r3) 2671 std r31, VCPU_GPR(R31)(r3) 2672 2673 /* save FP state */ 2674 bl kvmppc_save_fp 2675 2676#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2677/* 2678 * Branch around the call if both CPU_FTR_TM and 2679 * CPU_FTR_P9_TM_HV_ASSIST are off. 2680 */ 2681BEGIN_FTR_SECTION 2682 b 91f 2683END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2684 /* 2685 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2686 */ 2687 ld r3, HSTATE_KVM_VCPU(r13) 2688 ld r4, VCPU_MSR(r3) 2689 bl kvmppc_save_tm_hv 269091: 2691#endif 2692 2693 /* 2694 * Set DEC to the smaller of DEC and HDEC, so that we wake 2695 * no later than the end of our timeslice (HDEC interrupts 2696 * don't wake us from nap). 2697 */ 2698 mfspr r3, SPRN_DEC 2699 mfspr r4, SPRN_HDEC 2700 mftb r5 2701BEGIN_FTR_SECTION 2702 /* On P9 check whether the guest has large decrementer mode enabled */ 2703 ld r6, HSTATE_KVM_VCORE(r13) 2704 ld r6, VCORE_LPCR(r6) 2705 andis. r6, r6, LPCR_LD@h 2706 bne 68f 2707END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2708 extsw r3, r3 270968: EXTEND_HDEC(r4) 2710 cmpd r3, r4 2711 ble 67f 2712 mtspr SPRN_DEC, r4 271367: 2714 /* save expiry time of guest decrementer */ 2715 add r3, r3, r5 2716 ld r4, HSTATE_KVM_VCPU(r13) 2717 ld r5, HSTATE_KVM_VCORE(r13) 2718 ld r6, VCORE_TB_OFFSET_APPL(r5) 2719 subf r3, r6, r3 /* convert to host TB value */ 2720 std r3, VCPU_DEC_EXPIRES(r4) 2721 2722#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2723 ld r4, HSTATE_KVM_VCPU(r13) 2724 addi r3, r4, VCPU_TB_CEDE 2725 bl kvmhv_accumulate_time 2726#endif 2727 2728 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2729 2730 /* 2731 * Take a nap until a decrementer or external or doobell interrupt 2732 * occurs, with PECE1 and PECE0 set in LPCR. 2733 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2734 * Also clear the runlatch bit before napping. 2735 */ 2736kvm_do_nap: 2737 mfspr r0, SPRN_CTRLF 2738 clrrdi r0, r0, 1 2739 mtspr SPRN_CTRLT, r0 2740 2741 li r0,1 2742 stb r0,HSTATE_HWTHREAD_REQ(r13) 2743 mfspr r5,SPRN_LPCR 2744 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2745BEGIN_FTR_SECTION 2746 ori r5, r5, LPCR_PECEDH 2747 rlwimi r5, r3, 0, LPCR_PECEDP 2748END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2749 2750kvm_nap_sequence: /* desired LPCR value in r5 */ 2751BEGIN_FTR_SECTION 2752 /* 2753 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2754 * enable state loss = 1 (allow SMT mode switch) 2755 * requested level = 0 (just stop dispatching) 2756 */ 2757 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2758 mtspr SPRN_PSSCR, r3 2759 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2760 li r4, LPCR_PECE_HVEE@higher 2761 sldi r4, r4, 32 2762 or r5, r5, r4 2763END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2764 mtspr SPRN_LPCR,r5 2765 isync 2766 li r0, 0 2767 std r0, HSTATE_SCRATCH0(r13) 2768 ptesync 2769 ld r0, HSTATE_SCRATCH0(r13) 27701: cmpd r0, r0 2771 bne 1b 2772BEGIN_FTR_SECTION 2773 nap 2774FTR_SECTION_ELSE 2775 PPC_STOP 2776ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2777 b . 2778 277933: mr r4, r3 2780 li r3, 0 2781 li r12, 0 2782 b 34f 2783 2784kvm_end_cede: 2785 /* get vcpu pointer */ 2786 ld r4, HSTATE_KVM_VCPU(r13) 2787 2788 /* Woken by external or decrementer interrupt */ 2789 ld r1, HSTATE_HOST_R1(r13) 2790 2791#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2792 addi r3, r4, VCPU_TB_RMINTR 2793 bl kvmhv_accumulate_time 2794#endif 2795 2796#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2797/* 2798 * Branch around the call if both CPU_FTR_TM and 2799 * CPU_FTR_P9_TM_HV_ASSIST are off. 2800 */ 2801BEGIN_FTR_SECTION 2802 b 91f 2803END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2804 /* 2805 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR 2806 */ 2807 mr r3, r4 2808 ld r4, VCPU_MSR(r3) 2809 bl kvmppc_restore_tm_hv 2810 ld r4, HSTATE_KVM_VCPU(r13) 281191: 2812#endif 2813 2814 /* load up FP state */ 2815 bl kvmppc_load_fp 2816 2817 /* Restore guest decrementer */ 2818 ld r3, VCPU_DEC_EXPIRES(r4) 2819 ld r5, HSTATE_KVM_VCORE(r13) 2820 ld r6, VCORE_TB_OFFSET_APPL(r5) 2821 add r3, r3, r6 /* convert host TB to guest TB value */ 2822 mftb r7 2823 subf r3, r7, r3 2824 mtspr SPRN_DEC, r3 2825 2826 /* Load NV GPRS */ 2827 ld r14, VCPU_GPR(R14)(r4) 2828 ld r15, VCPU_GPR(R15)(r4) 2829 ld r16, VCPU_GPR(R16)(r4) 2830 ld r17, VCPU_GPR(R17)(r4) 2831 ld r18, VCPU_GPR(R18)(r4) 2832 ld r19, VCPU_GPR(R19)(r4) 2833 ld r20, VCPU_GPR(R20)(r4) 2834 ld r21, VCPU_GPR(R21)(r4) 2835 ld r22, VCPU_GPR(R22)(r4) 2836 ld r23, VCPU_GPR(R23)(r4) 2837 ld r24, VCPU_GPR(R24)(r4) 2838 ld r25, VCPU_GPR(R25)(r4) 2839 ld r26, VCPU_GPR(R26)(r4) 2840 ld r27, VCPU_GPR(R27)(r4) 2841 ld r28, VCPU_GPR(R28)(r4) 2842 ld r29, VCPU_GPR(R29)(r4) 2843 ld r30, VCPU_GPR(R30)(r4) 2844 ld r31, VCPU_GPR(R31)(r4) 2845 2846 /* Check the wake reason in SRR1 to see why we got here */ 2847 bl kvmppc_check_wake_reason 2848 2849 /* 2850 * Restore volatile registers since we could have called a 2851 * C routine in kvmppc_check_wake_reason 2852 * r4 = VCPU 2853 * r3 tells us whether we need to return to host or not 2854 * WARNING: it gets checked further down: 2855 * should not modify r3 until this check is done. 2856 */ 2857 ld r4, HSTATE_KVM_VCPU(r13) 2858 2859 /* clear our bit in vcore->napping_threads */ 286034: ld r5,HSTATE_KVM_VCORE(r13) 2861 lbz r7,HSTATE_PTID(r13) 2862 li r0,1 2863 sld r0,r0,r7 2864 addi r6,r5,VCORE_NAPPING_THREADS 286532: lwarx r7,0,r6 2866 andc r7,r7,r0 2867 stwcx. r7,0,r6 2868 bne 32b 2869 li r0,0 2870 stb r0,HSTATE_NAPPING(r13) 2871 2872 /* See if the wake reason saved in r3 means we need to exit */ 2873 stw r12, VCPU_TRAP(r4) 2874 mr r9, r4 2875 cmpdi r3, 0 2876 bgt guest_exit_cont 2877 2878 /* see if any other thread is already exiting */ 2879 lwz r0,VCORE_ENTRY_EXIT(r5) 2880 cmpwi r0,0x100 2881 bge guest_exit_cont 2882 2883 b kvmppc_cede_reentry /* if not go back to guest */ 2884 2885 /* cede when already previously prodded case */ 2886kvm_cede_prodded: 2887 li r0,0 2888 stb r0,VCPU_PRODDED(r3) 2889 sync /* order testing prodded vs. clearing ceded */ 2890 stb r0,VCPU_CEDED(r3) 2891 li r3,H_SUCCESS 2892 blr 2893 2894 /* we've ceded but we want to give control to the host */ 2895kvm_cede_exit: 2896 ld r9, HSTATE_KVM_VCPU(r13) 2897#ifdef CONFIG_KVM_XICS 2898 /* Abort if we still have a pending escalation */ 2899 lbz r5, VCPU_XIVE_ESC_ON(r9) 2900 cmpwi r5, 0 2901 beq 1f 2902 li r0, 0 2903 stb r0, VCPU_CEDED(r9) 29041: /* Enable XIVE escalation */ 2905 li r5, XIVE_ESB_SET_PQ_00 2906 mfmsr r0 2907 andi. r0, r0, MSR_DR /* in real mode? */ 2908 beq 1f 2909 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2910 cmpdi r10, 0 2911 beq 3f 2912 ldx r0, r10, r5 2913 b 2f 29141: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2915 cmpdi r10, 0 2916 beq 3f 2917 ldcix r0, r10, r5 29182: sync 2919 li r0, 1 2920 stb r0, VCPU_XIVE_ESC_ON(r9) 2921#endif /* CONFIG_KVM_XICS */ 29223: b guest_exit_cont 2923 2924 /* Try to handle a machine check in real mode */ 2925machine_check_realmode: 2926 mr r3, r9 /* get vcpu pointer */ 2927 bl kvmppc_realmode_machine_check 2928 nop 2929 ld r9, HSTATE_KVM_VCPU(r13) 2930 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2931 /* 2932 * For the guest that is FWNMI capable, deliver all the MCE errors 2933 * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit 2934 * reason. This new approach injects machine check errors in guest 2935 * address space to guest with additional information in the form 2936 * of RTAS event, thus enabling guest kernel to suitably handle 2937 * such errors. 2938 * 2939 * For the guest that is not FWNMI capable (old QEMU) fallback 2940 * to old behaviour for backward compatibility: 2941 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either 2942 * through machine check interrupt (set HSRR0 to 0x200). 2943 * For handled errors (no-fatal), just go back to guest execution 2944 * with current HSRR0. 2945 * if we receive machine check with MSR(RI=0) then deliver it to 2946 * guest as machine check causing guest to crash. 2947 */ 2948 ld r11, VCPU_MSR(r9) 2949 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */ 2950 bne mc_cont /* if so, exit to host */ 2951 /* Check if guest is capable of handling NMI exit */ 2952 ld r10, VCPU_KVM(r9) 2953 lbz r10, KVM_FWNMI(r10) 2954 cmpdi r10, 1 /* FWNMI capable? */ 2955 beq mc_cont /* if so, exit with KVM_EXIT_NMI. */ 2956 2957 /* if not, fall through for backward compatibility. */ 2958 andi. r10, r11, MSR_RI /* check for unrecoverable exception */ 2959 beq 1f /* Deliver a machine check to guest */ 2960 ld r10, VCPU_PC(r9) 2961 cmpdi r3, 0 /* Did we handle MCE ? */ 2962 bne 2f /* Continue guest execution. */ 2963 /* If not, deliver a machine check. SRR0/1 are already set */ 29641: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK 2965 bl kvmppc_msr_interrupt 29662: b fast_interrupt_c_return 2967 2968/* 2969 * Check the reason we woke from nap, and take appropriate action. 2970 * Returns (in r3): 2971 * 0 if nothing needs to be done 2972 * 1 if something happened that needs to be handled by the host 2973 * -1 if there was a guest wakeup (IPI or msgsnd) 2974 * -2 if we handled a PCI passthrough interrupt (returned by 2975 * kvmppc_read_intr only) 2976 * 2977 * Also sets r12 to the interrupt vector for any interrupt that needs 2978 * to be handled now by the host (0x500 for external interrupt), or zero. 2979 * Modifies all volatile registers (since it may call a C function). 2980 * This routine calls kvmppc_read_intr, a C function, if an external 2981 * interrupt is pending. 2982 */ 2983kvmppc_check_wake_reason: 2984 mfspr r6, SPRN_SRR1 2985BEGIN_FTR_SECTION 2986 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2987FTR_SECTION_ELSE 2988 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2989ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2990 cmpwi r6, 8 /* was it an external interrupt? */ 2991 beq 7f /* if so, see what it was */ 2992 li r3, 0 2993 li r12, 0 2994 cmpwi r6, 6 /* was it the decrementer? */ 2995 beq 0f 2996BEGIN_FTR_SECTION 2997 cmpwi r6, 5 /* privileged doorbell? */ 2998 beq 0f 2999 cmpwi r6, 3 /* hypervisor doorbell? */ 3000 beq 3f 3001END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3002 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 3003 beq 4f 3004 li r3, 1 /* anything else, return 1 */ 30050: blr 3006 3007 /* hypervisor doorbell */ 30083: li r12, BOOK3S_INTERRUPT_H_DOORBELL 3009 3010 /* 3011 * Clear the doorbell as we will invoke the handler 3012 * explicitly in the guest exit path. 3013 */ 3014 lis r6, (PPC_DBELL_SERVER << (63-36))@h 3015 PPC_MSGCLR(6) 3016 /* see if it's a host IPI */ 3017 li r3, 1 3018BEGIN_FTR_SECTION 3019 PPC_MSGSYNC 3020 lwsync 3021END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 3022 lbz r0, HSTATE_HOST_IPI(r13) 3023 cmpwi r0, 0 3024 bnelr 3025 /* if not, return -1 */ 3026 li r3, -1 3027 blr 3028 3029 /* Woken up due to Hypervisor maintenance interrupt */ 30304: li r12, BOOK3S_INTERRUPT_HMI 3031 li r3, 1 3032 blr 3033 3034 /* external interrupt - create a stack frame so we can call C */ 30357: mflr r0 3036 std r0, PPC_LR_STKOFF(r1) 3037 stdu r1, -PPC_MIN_STKFRM(r1) 3038 bl kvmppc_read_intr 3039 nop 3040 li r12, BOOK3S_INTERRUPT_EXTERNAL 3041 cmpdi r3, 1 3042 ble 1f 3043 3044 /* 3045 * Return code of 2 means PCI passthrough interrupt, but 3046 * we need to return back to host to complete handling the 3047 * interrupt. Trap reason is expected in r12 by guest 3048 * exit code. 3049 */ 3050 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 30511: 3052 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 3053 addi r1, r1, PPC_MIN_STKFRM 3054 mtlr r0 3055 blr 3056 3057/* 3058 * Save away FP, VMX and VSX registers. 3059 * r3 = vcpu pointer 3060 * N.B. r30 and r31 are volatile across this function, 3061 * thus it is not callable from C. 3062 */ 3063kvmppc_save_fp: 3064 mflr r30 3065 mr r31,r3 3066 mfmsr r5 3067 ori r8,r5,MSR_FP 3068#ifdef CONFIG_ALTIVEC 3069BEGIN_FTR_SECTION 3070 oris r8,r8,MSR_VEC@h 3071END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3072#endif 3073#ifdef CONFIG_VSX 3074BEGIN_FTR_SECTION 3075 oris r8,r8,MSR_VSX@h 3076END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3077#endif 3078 mtmsrd r8 3079 addi r3,r3,VCPU_FPRS 3080 bl store_fp_state 3081#ifdef CONFIG_ALTIVEC 3082BEGIN_FTR_SECTION 3083 addi r3,r31,VCPU_VRS 3084 bl store_vr_state 3085END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3086#endif 3087 mfspr r6,SPRN_VRSAVE 3088 stw r6,VCPU_VRSAVE(r31) 3089 mtlr r30 3090 blr 3091 3092/* 3093 * Load up FP, VMX and VSX registers 3094 * r4 = vcpu pointer 3095 * N.B. r30 and r31 are volatile across this function, 3096 * thus it is not callable from C. 3097 */ 3098kvmppc_load_fp: 3099 mflr r30 3100 mr r31,r4 3101 mfmsr r9 3102 ori r8,r9,MSR_FP 3103#ifdef CONFIG_ALTIVEC 3104BEGIN_FTR_SECTION 3105 oris r8,r8,MSR_VEC@h 3106END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3107#endif 3108#ifdef CONFIG_VSX 3109BEGIN_FTR_SECTION 3110 oris r8,r8,MSR_VSX@h 3111END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3112#endif 3113 mtmsrd r8 3114 addi r3,r4,VCPU_FPRS 3115 bl load_fp_state 3116#ifdef CONFIG_ALTIVEC 3117BEGIN_FTR_SECTION 3118 addi r3,r31,VCPU_VRS 3119 bl load_vr_state 3120END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3121#endif 3122 lwz r7,VCPU_VRSAVE(r31) 3123 mtspr SPRN_VRSAVE,r7 3124 mtlr r30 3125 mr r4,r31 3126 blr 3127 3128#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3129/* 3130 * Save transactional state and TM-related registers. 3131 * Called with r3 pointing to the vcpu struct and r4 containing 3132 * the guest MSR value. 3133 * This can modify all checkpointed registers, but 3134 * restores r1 and r2 before exit. 3135 */ 3136kvmppc_save_tm_hv: 3137 /* See if we need to handle fake suspend mode */ 3138BEGIN_FTR_SECTION 3139 b __kvmppc_save_tm 3140END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3141 3142 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 3143 cmpwi r0, 0 3144 beq __kvmppc_save_tm 3145 3146 /* The following code handles the fake_suspend = 1 case */ 3147 mflr r0 3148 std r0, PPC_LR_STKOFF(r1) 3149 stdu r1, -PPC_MIN_STKFRM(r1) 3150 3151 /* Turn on TM. */ 3152 mfmsr r8 3153 li r0, 1 3154 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3155 mtmsrd r8 3156 3157 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 3158 beq 4f 3159BEGIN_FTR_SECTION 3160 bl pnv_power9_force_smt4_catch 3161END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3162 nop 3163 3164 std r1, HSTATE_HOST_R1(r13) 3165 3166 /* Clear the MSR RI since r1, r13 may be foobar. */ 3167 li r5, 0 3168 mtmsrd r5, 1 3169 3170 /* We have to treclaim here because that's the only way to do S->N */ 3171 li r3, TM_CAUSE_KVM_RESCHED 3172 TRECLAIM(R3) 3173 3174 /* 3175 * We were in fake suspend, so we are not going to save the 3176 * register state as the guest checkpointed state (since 3177 * we already have it), therefore we can now use any volatile GPR. 3178 */ 3179 /* Reload PACA pointer, stack pointer and TOC. */ 3180 GET_PACA(r13) 3181 ld r1, HSTATE_HOST_R1(r13) 3182 ld r2, PACATOC(r13) 3183 3184 /* Set MSR RI now we have r1 and r13 back. */ 3185 li r5, MSR_RI 3186 mtmsrd r5, 1 3187 3188 HMT_MEDIUM 3189 ld r6, HSTATE_DSCR(r13) 3190 mtspr SPRN_DSCR, r6 3191BEGIN_FTR_SECTION_NESTED(96) 3192 bl pnv_power9_force_smt4_release 3193END_FTR_SECTION_NESTED(CPU_FTR_P9_TM_XER_SO_BUG, CPU_FTR_P9_TM_XER_SO_BUG, 96) 3194 nop 3195 31964: 3197 mfspr r3, SPRN_PSSCR 3198 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 3199 li r0, PSSCR_FAKE_SUSPEND 3200 andc r3, r3, r0 3201 mtspr SPRN_PSSCR, r3 3202 3203 /* Don't save TEXASR, use value from last exit in real suspend state */ 3204 ld r9, HSTATE_KVM_VCPU(r13) 3205 mfspr r5, SPRN_TFHAR 3206 mfspr r6, SPRN_TFIAR 3207 std r5, VCPU_TFHAR(r9) 3208 std r6, VCPU_TFIAR(r9) 3209 3210 addi r1, r1, PPC_MIN_STKFRM 3211 ld r0, PPC_LR_STKOFF(r1) 3212 mtlr r0 3213 blr 3214 3215/* 3216 * Restore transactional state and TM-related registers. 3217 * Called with r3 pointing to the vcpu struct 3218 * and r4 containing the guest MSR value. 3219 * This potentially modifies all checkpointed registers. 3220 * It restores r1 and r2 from the PACA. 3221 */ 3222kvmppc_restore_tm_hv: 3223 /* 3224 * If we are doing TM emulation for the guest on a POWER9 DD2, 3225 * then we don't actually do a trechkpt -- we either set up 3226 * fake-suspend mode, or emulate a TM rollback. 3227 */ 3228BEGIN_FTR_SECTION 3229 b __kvmppc_restore_tm 3230END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3231 mflr r0 3232 std r0, PPC_LR_STKOFF(r1) 3233 3234 li r0, 0 3235 stb r0, HSTATE_FAKE_SUSPEND(r13) 3236 3237 /* Turn on TM so we can restore TM SPRs */ 3238 mfmsr r5 3239 li r0, 1 3240 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 3241 mtmsrd r5 3242 3243 /* 3244 * The user may change these outside of a transaction, so they must 3245 * always be context switched. 3246 */ 3247 ld r5, VCPU_TFHAR(r3) 3248 ld r6, VCPU_TFIAR(r3) 3249 ld r7, VCPU_TEXASR(r3) 3250 mtspr SPRN_TFHAR, r5 3251 mtspr SPRN_TFIAR, r6 3252 mtspr SPRN_TEXASR, r7 3253 3254 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 3255 beqlr /* TM not active in guest */ 3256 3257 /* Make sure the failure summary is set */ 3258 oris r7, r7, (TEXASR_FS)@h 3259 mtspr SPRN_TEXASR, r7 3260 3261 cmpwi r5, 1 /* check for suspended state */ 3262 bgt 10f 3263 stb r5, HSTATE_FAKE_SUSPEND(r13) 3264 b 9f /* and return */ 326510: stdu r1, -PPC_MIN_STKFRM(r1) 3266 /* guest is in transactional state, so simulate rollback */ 3267 bl kvmhv_emulate_tm_rollback 3268 nop 3269 addi r1, r1, PPC_MIN_STKFRM 32709: ld r0, PPC_LR_STKOFF(r1) 3271 mtlr r0 3272 blr 3273#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 3274 3275/* 3276 * We come here if we get any exception or interrupt while we are 3277 * executing host real mode code while in guest MMU context. 3278 * r12 is (CR << 32) | vector 3279 * r13 points to our PACA 3280 * r12 is saved in HSTATE_SCRATCH0(r13) 3281 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3282 * r9 is saved in HSTATE_SCRATCH2(r13) 3283 * r13 is saved in HSPRG1 3284 * cfar is saved in HSTATE_CFAR(r13) 3285 * ppr is saved in HSTATE_PPR(r13) 3286 */ 3287kvmppc_bad_host_intr: 3288 /* 3289 * Switch to the emergency stack, but start half-way down in 3290 * case we were already on it. 3291 */ 3292 mr r9, r1 3293 std r1, PACAR1(r13) 3294 ld r1, PACAEMERGSP(r13) 3295 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3296 std r9, 0(r1) 3297 std r0, GPR0(r1) 3298 std r9, GPR1(r1) 3299 std r2, GPR2(r1) 3300 SAVE_4GPRS(3, r1) 3301 SAVE_2GPRS(7, r1) 3302 srdi r0, r12, 32 3303 clrldi r12, r12, 32 3304 std r0, _CCR(r1) 3305 std r12, _TRAP(r1) 3306 andi. r0, r12, 2 3307 beq 1f 3308 mfspr r3, SPRN_HSRR0 3309 mfspr r4, SPRN_HSRR1 3310 mfspr r5, SPRN_HDAR 3311 mfspr r6, SPRN_HDSISR 3312 b 2f 33131: mfspr r3, SPRN_SRR0 3314 mfspr r4, SPRN_SRR1 3315 mfspr r5, SPRN_DAR 3316 mfspr r6, SPRN_DSISR 33172: std r3, _NIP(r1) 3318 std r4, _MSR(r1) 3319 std r5, _DAR(r1) 3320 std r6, _DSISR(r1) 3321 ld r9, HSTATE_SCRATCH2(r13) 3322 ld r12, HSTATE_SCRATCH0(r13) 3323 GET_SCRATCH0(r0) 3324 SAVE_4GPRS(9, r1) 3325 std r0, GPR13(r1) 3326 SAVE_NVGPRS(r1) 3327 ld r5, HSTATE_CFAR(r13) 3328 std r5, ORIG_GPR3(r1) 3329 mflr r3 3330#ifdef CONFIG_RELOCATABLE 3331 ld r4, HSTATE_SCRATCH1(r13) 3332#else 3333 mfctr r4 3334#endif 3335 mfxer r5 3336 lbz r6, PACAIRQSOFTMASK(r13) 3337 std r3, _LINK(r1) 3338 std r4, _CTR(r1) 3339 std r5, _XER(r1) 3340 std r6, SOFTE(r1) 3341 ld r2, PACATOC(r13) 3342 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3343 std r3, STACK_FRAME_OVERHEAD-16(r1) 3344 3345 /* 3346 * On POWER9 do a minimal restore of the MMU and call C code, 3347 * which will print a message and panic. 3348 * XXX On POWER7 and POWER8, we just spin here since we don't 3349 * know what the other threads are doing (and we don't want to 3350 * coordinate with them) - but at least we now have register state 3351 * in memory that we might be able to look at from another CPU. 3352 */ 3353BEGIN_FTR_SECTION 3354 b . 3355END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3356 ld r9, HSTATE_KVM_VCPU(r13) 3357 ld r10, VCPU_KVM(r9) 3358 3359 li r0, 0 3360 mtspr SPRN_AMR, r0 3361 mtspr SPRN_IAMR, r0 3362 mtspr SPRN_CIABR, r0 3363 mtspr SPRN_DAWRX, r0 3364 3365BEGIN_MMU_FTR_SECTION 3366 b 4f 3367END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3368 3369 slbmte r0, r0 3370 slbia 3371 ptesync 3372 ld r8, PACA_SLBSHADOWPTR(r13) 3373 .rept SLB_NUM_BOLTED 3374 li r3, SLBSHADOW_SAVEAREA 3375 LDX_BE r5, r8, r3 3376 addi r3, r3, 8 3377 LDX_BE r6, r8, r3 3378 andis. r7, r5, SLB_ESID_V@h 3379 beq 3f 3380 slbmte r6, r5 33813: addi r8, r8, 16 3382 .endr 3383 33844: lwz r7, KVM_HOST_LPID(r10) 3385 mtspr SPRN_LPID, r7 3386 mtspr SPRN_PID, r0 3387 ld r8, KVM_HOST_LPCR(r10) 3388 mtspr SPRN_LPCR, r8 3389 isync 3390 li r0, KVM_GUEST_MODE_NONE 3391 stb r0, HSTATE_IN_GUEST(r13) 3392 3393 /* 3394 * Turn on the MMU and jump to C code 3395 */ 3396 bcl 20, 31, .+4 33975: mflr r3 3398 addi r3, r3, 9f - 5b 3399 li r4, -1 3400 rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ 3401 ld r4, PACAKMSR(r13) 3402 mtspr SPRN_SRR0, r3 3403 mtspr SPRN_SRR1, r4 3404 RFI_TO_KERNEL 34059: addi r3, r1, STACK_FRAME_OVERHEAD 3406 bl kvmppc_bad_interrupt 3407 b 9b 3408 3409/* 3410 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3411 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3412 * r11 has the guest MSR value (in/out) 3413 * r9 has a vcpu pointer (in) 3414 * r0 is used as a scratch register 3415 */ 3416kvmppc_msr_interrupt: 3417 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3418 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3419 ld r11, VCPU_INTR_MSR(r9) 3420 bne 1f 3421 /* ... if transactional, change to suspended */ 3422 li r0, 1 34231: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3424 blr 3425 3426/* 3427 * This works around a hardware bug on POWER8E processors, where 3428 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3429 * performance monitor interrupt. Instead, when we need to have 3430 * an interrupt pending, we have to arrange for a counter to overflow. 3431 */ 3432kvmppc_fix_pmao: 3433 li r3, 0 3434 mtspr SPRN_MMCR2, r3 3435 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3436 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3437 mtspr SPRN_MMCR0, r3 3438 lis r3, 0x7fff 3439 ori r3, r3, 0xffff 3440 mtspr SPRN_PMC6, r3 3441 isync 3442 blr 3443 3444#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3445/* 3446 * Start timing an activity 3447 * r3 = pointer to time accumulation struct, r4 = vcpu 3448 */ 3449kvmhv_start_timing: 3450 ld r5, HSTATE_KVM_VCORE(r13) 3451 ld r6, VCORE_TB_OFFSET_APPL(r5) 3452 mftb r5 3453 subf r5, r6, r5 /* subtract current timebase offset */ 3454 std r3, VCPU_CUR_ACTIVITY(r4) 3455 std r5, VCPU_ACTIVITY_START(r4) 3456 blr 3457 3458/* 3459 * Accumulate time to one activity and start another. 3460 * r3 = pointer to new time accumulation struct, r4 = vcpu 3461 */ 3462kvmhv_accumulate_time: 3463 ld r5, HSTATE_KVM_VCORE(r13) 3464 ld r8, VCORE_TB_OFFSET_APPL(r5) 3465 ld r5, VCPU_CUR_ACTIVITY(r4) 3466 ld r6, VCPU_ACTIVITY_START(r4) 3467 std r3, VCPU_CUR_ACTIVITY(r4) 3468 mftb r7 3469 subf r7, r8, r7 /* subtract current timebase offset */ 3470 std r7, VCPU_ACTIVITY_START(r4) 3471 cmpdi r5, 0 3472 beqlr 3473 subf r3, r6, r7 3474 ld r8, TAS_SEQCOUNT(r5) 3475 cmpdi r8, 0 3476 addi r8, r8, 1 3477 std r8, TAS_SEQCOUNT(r5) 3478 lwsync 3479 ld r7, TAS_TOTAL(r5) 3480 add r7, r7, r3 3481 std r7, TAS_TOTAL(r5) 3482 ld r6, TAS_MIN(r5) 3483 ld r7, TAS_MAX(r5) 3484 beq 3f 3485 cmpd r3, r6 3486 bge 1f 34873: std r3, TAS_MIN(r5) 34881: cmpd r3, r7 3489 ble 2f 3490 std r3, TAS_MAX(r5) 34912: lwsync 3492 addi r8, r8, 1 3493 std r8, TAS_SEQCOUNT(r5) 3494 blr 3495#endif 3496