1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License, version 2, as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 12 * 13 * Derived from book3s_rmhandlers.S and other files, which are: 14 * 15 * Copyright SUSE Linux Products GmbH 2009 16 * 17 * Authors: Alexander Graf <agraf@suse.de> 18 */ 19 20#include <asm/ppc_asm.h> 21#include <asm/kvm_asm.h> 22#include <asm/reg.h> 23#include <asm/mmu.h> 24#include <asm/page.h> 25#include <asm/ptrace.h> 26#include <asm/hvcall.h> 27#include <asm/asm-offsets.h> 28#include <asm/exception-64s.h> 29#include <asm/kvm_book3s_asm.h> 30#include <asm/book3s/64/mmu-hash.h> 31#include <asm/export.h> 32#include <asm/tm.h> 33#include <asm/opal.h> 34#include <asm/xive-regs.h> 35#include <asm/thread_info.h> 36#include <asm/asm-compat.h> 37#include <asm/feature-fixups.h> 38 39/* Sign-extend HDEC if not on POWER9 */ 40#define EXTEND_HDEC(reg) \ 41BEGIN_FTR_SECTION; \ 42 extsw reg, reg; \ 43END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 44 45/* Values in HSTATE_NAPPING(r13) */ 46#define NAPPING_CEDE 1 47#define NAPPING_NOVCPU 2 48 49/* Stack frame offsets for kvmppc_hv_entry */ 50#define SFS 208 51#define STACK_SLOT_TRAP (SFS-4) 52#define STACK_SLOT_SHORT_PATH (SFS-8) 53#define STACK_SLOT_TID (SFS-16) 54#define STACK_SLOT_PSSCR (SFS-24) 55#define STACK_SLOT_PID (SFS-32) 56#define STACK_SLOT_IAMR (SFS-40) 57#define STACK_SLOT_CIABR (SFS-48) 58#define STACK_SLOT_DAWR (SFS-56) 59#define STACK_SLOT_DAWRX (SFS-64) 60#define STACK_SLOT_HFSCR (SFS-72) 61#define STACK_SLOT_AMR (SFS-80) 62#define STACK_SLOT_UAMOR (SFS-88) 63/* the following is used by the P9 short path */ 64#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */ 65 66/* 67 * Call kvmppc_hv_entry in real mode. 68 * Must be called with interrupts hard-disabled. 69 * 70 * Input Registers: 71 * 72 * LR = return address to continue at after eventually re-enabling MMU 73 */ 74_GLOBAL_TOC(kvmppc_hv_entry_trampoline) 75 mflr r0 76 std r0, PPC_LR_STKOFF(r1) 77 stdu r1, -112(r1) 78 mfmsr r10 79 std r10, HSTATE_HOST_MSR(r13) 80 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 81 li r0,MSR_RI 82 andc r0,r10,r0 83 li r6,MSR_IR | MSR_DR 84 andc r6,r10,r6 85 mtmsrd r0,1 /* clear RI in MSR */ 86 mtsrr0 r5 87 mtsrr1 r6 88 RFI_TO_KERNEL 89 90kvmppc_call_hv_entry: 91BEGIN_FTR_SECTION 92 /* On P9, do LPCR setting, if necessary */ 93 ld r3, HSTATE_SPLIT_MODE(r13) 94 cmpdi r3, 0 95 beq 46f 96 lwz r4, KVM_SPLIT_DO_SET(r3) 97 cmpwi r4, 0 98 beq 46f 99 bl kvmhv_p9_set_lpcr 100 nop 10146: 102END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 103 104 ld r4, HSTATE_KVM_VCPU(r13) 105 bl kvmppc_hv_entry 106 107 /* Back from guest - restore host state and return to caller */ 108 109BEGIN_FTR_SECTION 110 /* Restore host DABR and DABRX */ 111 ld r5,HSTATE_DABR(r13) 112 li r6,7 113 mtspr SPRN_DABR,r5 114 mtspr SPRN_DABRX,r6 115END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 116 117 /* Restore SPRG3 */ 118 ld r3,PACA_SPRG_VDSO(r13) 119 mtspr SPRN_SPRG_VDSO_WRITE,r3 120 121 /* Reload the host's PMU registers */ 122 bl kvmhv_load_host_pmu 123 124 /* 125 * Reload DEC. HDEC interrupts were disabled when 126 * we reloaded the host's LPCR value. 127 */ 128 ld r3, HSTATE_DECEXP(r13) 129 mftb r4 130 subf r4, r4, r3 131 mtspr SPRN_DEC, r4 132 133 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 134 li r0, 0 135 stb r0, HSTATE_HWTHREAD_REQ(r13) 136 137 /* 138 * For external interrupts we need to call the Linux 139 * handler to process the interrupt. We do that by jumping 140 * to absolute address 0x500 for external interrupts. 141 * The [h]rfid at the end of the handler will return to 142 * the book3s_hv_interrupts.S code. For other interrupts 143 * we do the rfid to get back to the book3s_hv_interrupts.S 144 * code here. 145 */ 146 ld r8, 112+PPC_LR_STKOFF(r1) 147 addi r1, r1, 112 148 ld r7, HSTATE_HOST_MSR(r13) 149 150 /* Return the trap number on this thread as the return value */ 151 mr r3, r12 152 153 /* 154 * If we came back from the guest via a relocation-on interrupt, 155 * we will be in virtual mode at this point, which makes it a 156 * little easier to get back to the caller. 157 */ 158 mfmsr r0 159 andi. r0, r0, MSR_IR /* in real mode? */ 160 bne .Lvirt_return 161 162 /* RFI into the highmem handler */ 163 mfmsr r6 164 li r0, MSR_RI 165 andc r6, r6, r0 166 mtmsrd r6, 1 /* Clear RI in MSR */ 167 mtsrr0 r8 168 mtsrr1 r7 169 RFI_TO_KERNEL 170 171 /* Virtual-mode return */ 172.Lvirt_return: 173 mtlr r8 174 blr 175 176kvmppc_primary_no_guest: 177 /* We handle this much like a ceded vcpu */ 178 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 179 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 180 /* HDEC value came from DEC in the first place, it will fit */ 181 mfspr r3, SPRN_HDEC 182 mtspr SPRN_DEC, r3 183 /* 184 * Make sure the primary has finished the MMU switch. 185 * We should never get here on a secondary thread, but 186 * check it for robustness' sake. 187 */ 188 ld r5, HSTATE_KVM_VCORE(r13) 18965: lbz r0, VCORE_IN_GUEST(r5) 190 cmpwi r0, 0 191 beq 65b 192 /* Set LPCR. */ 193 ld r8,VCORE_LPCR(r5) 194 mtspr SPRN_LPCR,r8 195 isync 196 /* set our bit in napping_threads */ 197 ld r5, HSTATE_KVM_VCORE(r13) 198 lbz r7, HSTATE_PTID(r13) 199 li r0, 1 200 sld r0, r0, r7 201 addi r6, r5, VCORE_NAPPING_THREADS 2021: lwarx r3, 0, r6 203 or r3, r3, r0 204 stwcx. r3, 0, r6 205 bne 1b 206 /* order napping_threads update vs testing entry_exit_map */ 207 isync 208 li r12, 0 209 lwz r7, VCORE_ENTRY_EXIT(r5) 210 cmpwi r7, 0x100 211 bge kvm_novcpu_exit /* another thread already exiting */ 212 li r3, NAPPING_NOVCPU 213 stb r3, HSTATE_NAPPING(r13) 214 215 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 216 b kvm_do_nap 217 218/* 219 * kvm_novcpu_wakeup 220 * Entered from kvm_start_guest if kvm_hstate.napping is set 221 * to NAPPING_NOVCPU 222 * r2 = kernel TOC 223 * r13 = paca 224 */ 225kvm_novcpu_wakeup: 226 ld r1, HSTATE_HOST_R1(r13) 227 ld r5, HSTATE_KVM_VCORE(r13) 228 li r0, 0 229 stb r0, HSTATE_NAPPING(r13) 230 231 /* check the wake reason */ 232 bl kvmppc_check_wake_reason 233 234 /* 235 * Restore volatile registers since we could have called 236 * a C routine in kvmppc_check_wake_reason. 237 * r5 = VCORE 238 */ 239 ld r5, HSTATE_KVM_VCORE(r13) 240 241 /* see if any other thread is already exiting */ 242 lwz r0, VCORE_ENTRY_EXIT(r5) 243 cmpwi r0, 0x100 244 bge kvm_novcpu_exit 245 246 /* clear our bit in napping_threads */ 247 lbz r7, HSTATE_PTID(r13) 248 li r0, 1 249 sld r0, r0, r7 250 addi r6, r5, VCORE_NAPPING_THREADS 2514: lwarx r7, 0, r6 252 andc r7, r7, r0 253 stwcx. r7, 0, r6 254 bne 4b 255 256 /* See if the wake reason means we need to exit */ 257 cmpdi r3, 0 258 bge kvm_novcpu_exit 259 260 /* See if our timeslice has expired (HDEC is negative) */ 261 mfspr r0, SPRN_HDEC 262 EXTEND_HDEC(r0) 263 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 264 cmpdi r0, 0 265 blt kvm_novcpu_exit 266 267 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 268 ld r4, HSTATE_KVM_VCPU(r13) 269 cmpdi r4, 0 270 beq kvmppc_primary_no_guest 271 272#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 273 addi r3, r4, VCPU_TB_RMENTRY 274 bl kvmhv_start_timing 275#endif 276 b kvmppc_got_guest 277 278kvm_novcpu_exit: 279#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 280 ld r4, HSTATE_KVM_VCPU(r13) 281 cmpdi r4, 0 282 beq 13f 283 addi r3, r4, VCPU_TB_RMEXIT 284 bl kvmhv_accumulate_time 285#endif 28613: mr r3, r12 287 stw r12, STACK_SLOT_TRAP(r1) 288 bl kvmhv_commence_exit 289 nop 290 b kvmhv_switch_to_host 291 292/* 293 * We come in here when wakened from nap mode. 294 * Relocation is off and most register values are lost. 295 * r13 points to the PACA. 296 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 297 */ 298 .globl kvm_start_guest 299kvm_start_guest: 300 /* Set runlatch bit the minute you wake up from nap */ 301 mfspr r0, SPRN_CTRLF 302 ori r0, r0, 1 303 mtspr SPRN_CTRLT, r0 304 305 /* 306 * Could avoid this and pass it through in r3. For now, 307 * code expects it to be in SRR1. 308 */ 309 mtspr SPRN_SRR1,r3 310 311 ld r2,PACATOC(r13) 312 313 li r0,0 314 stb r0,PACA_FTRACE_ENABLED(r13) 315 316 li r0,KVM_HWTHREAD_IN_KVM 317 stb r0,HSTATE_HWTHREAD_STATE(r13) 318 319 /* NV GPR values from power7_idle() will no longer be valid */ 320 li r0,1 321 stb r0,PACA_NAPSTATELOST(r13) 322 323 /* were we napping due to cede? */ 324 lbz r0,HSTATE_NAPPING(r13) 325 cmpwi r0,NAPPING_CEDE 326 beq kvm_end_cede 327 cmpwi r0,NAPPING_NOVCPU 328 beq kvm_novcpu_wakeup 329 330 ld r1,PACAEMERGSP(r13) 331 subi r1,r1,STACK_FRAME_OVERHEAD 332 333 /* 334 * We weren't napping due to cede, so this must be a secondary 335 * thread being woken up to run a guest, or being woken up due 336 * to a stray IPI. (Or due to some machine check or hypervisor 337 * maintenance interrupt while the core is in KVM.) 338 */ 339 340 /* Check the wake reason in SRR1 to see why we got here */ 341 bl kvmppc_check_wake_reason 342 /* 343 * kvmppc_check_wake_reason could invoke a C routine, but we 344 * have no volatile registers to restore when we return. 345 */ 346 347 cmpdi r3, 0 348 bge kvm_no_guest 349 350 /* get vcore pointer, NULL if we have nothing to run */ 351 ld r5,HSTATE_KVM_VCORE(r13) 352 cmpdi r5,0 353 /* if we have no vcore to run, go back to sleep */ 354 beq kvm_no_guest 355 356kvm_secondary_got_guest: 357 358 /* Set HSTATE_DSCR(r13) to something sensible */ 359 ld r6, PACA_DSCR_DEFAULT(r13) 360 std r6, HSTATE_DSCR(r13) 361 362 /* On thread 0 of a subcore, set HDEC to max */ 363 lbz r4, HSTATE_PTID(r13) 364 cmpwi r4, 0 365 bne 63f 366 LOAD_REG_ADDR(r6, decrementer_max) 367 ld r6, 0(r6) 368 mtspr SPRN_HDEC, r6 369 /* and set per-LPAR registers, if doing dynamic micro-threading */ 370 ld r6, HSTATE_SPLIT_MODE(r13) 371 cmpdi r6, 0 372 beq 63f 373BEGIN_FTR_SECTION 374 ld r0, KVM_SPLIT_RPR(r6) 375 mtspr SPRN_RPR, r0 376 ld r0, KVM_SPLIT_PMMAR(r6) 377 mtspr SPRN_PMMAR, r0 378 ld r0, KVM_SPLIT_LDBAR(r6) 379 mtspr SPRN_LDBAR, r0 380 isync 381FTR_SECTION_ELSE 382 /* On P9 we use the split_info for coordinating LPCR changes */ 383 lwz r4, KVM_SPLIT_DO_SET(r6) 384 cmpwi r4, 0 385 beq 1f 386 mr r3, r6 387 bl kvmhv_p9_set_lpcr 388 nop 3891: 390ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 39163: 392 /* Order load of vcpu after load of vcore */ 393 lwsync 394 ld r4, HSTATE_KVM_VCPU(r13) 395 bl kvmppc_hv_entry 396 397 /* Back from the guest, go back to nap */ 398 /* Clear our vcpu and vcore pointers so we don't come back in early */ 399 li r0, 0 400 std r0, HSTATE_KVM_VCPU(r13) 401 /* 402 * Once we clear HSTATE_KVM_VCORE(r13), the code in 403 * kvmppc_run_core() is going to assume that all our vcpu 404 * state is visible in memory. This lwsync makes sure 405 * that that is true. 406 */ 407 lwsync 408 std r0, HSTATE_KVM_VCORE(r13) 409 410 /* 411 * All secondaries exiting guest will fall through this path. 412 * Before proceeding, just check for HMI interrupt and 413 * invoke opal hmi handler. By now we are sure that the 414 * primary thread on this core/subcore has already made partition 415 * switch/TB resync and we are good to call opal hmi handler. 416 */ 417 cmpwi r12, BOOK3S_INTERRUPT_HMI 418 bne kvm_no_guest 419 420 li r3,0 /* NULL argument */ 421 bl hmi_exception_realmode 422/* 423 * At this point we have finished executing in the guest. 424 * We need to wait for hwthread_req to become zero, since 425 * we may not turn on the MMU while hwthread_req is non-zero. 426 * While waiting we also need to check if we get given a vcpu to run. 427 */ 428kvm_no_guest: 429 lbz r3, HSTATE_HWTHREAD_REQ(r13) 430 cmpwi r3, 0 431 bne 53f 432 HMT_MEDIUM 433 li r0, KVM_HWTHREAD_IN_KERNEL 434 stb r0, HSTATE_HWTHREAD_STATE(r13) 435 /* need to recheck hwthread_req after a barrier, to avoid race */ 436 sync 437 lbz r3, HSTATE_HWTHREAD_REQ(r13) 438 cmpwi r3, 0 439 bne 54f 440/* 441 * We jump to pnv_wakeup_loss, which will return to the caller 442 * of power7_nap in the powernv cpu offline loop. The value we 443 * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss 444 * requires SRR1 in r12. 445 */ 446 li r3, LPCR_PECE0 447 mfspr r4, SPRN_LPCR 448 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 449 mtspr SPRN_LPCR, r4 450 li r3, 0 451 mfspr r12,SPRN_SRR1 452 b pnv_wakeup_loss 453 45453: HMT_LOW 455 ld r5, HSTATE_KVM_VCORE(r13) 456 cmpdi r5, 0 457 bne 60f 458 ld r3, HSTATE_SPLIT_MODE(r13) 459 cmpdi r3, 0 460 beq kvm_no_guest 461 lwz r0, KVM_SPLIT_DO_SET(r3) 462 cmpwi r0, 0 463 bne kvmhv_do_set 464 lwz r0, KVM_SPLIT_DO_RESTORE(r3) 465 cmpwi r0, 0 466 bne kvmhv_do_restore 467 lbz r0, KVM_SPLIT_DO_NAP(r3) 468 cmpwi r0, 0 469 beq kvm_no_guest 470 HMT_MEDIUM 471 b kvm_unsplit_nap 47260: HMT_MEDIUM 473 b kvm_secondary_got_guest 474 47554: li r0, KVM_HWTHREAD_IN_KVM 476 stb r0, HSTATE_HWTHREAD_STATE(r13) 477 b kvm_no_guest 478 479kvmhv_do_set: 480 /* Set LPCR, LPIDR etc. on P9 */ 481 HMT_MEDIUM 482 bl kvmhv_p9_set_lpcr 483 nop 484 b kvm_no_guest 485 486kvmhv_do_restore: 487 HMT_MEDIUM 488 bl kvmhv_p9_restore_lpcr 489 nop 490 b kvm_no_guest 491 492/* 493 * Here the primary thread is trying to return the core to 494 * whole-core mode, so we need to nap. 495 */ 496kvm_unsplit_nap: 497 /* 498 * When secondaries are napping in kvm_unsplit_nap() with 499 * hwthread_req = 1, HMI goes ignored even though subcores are 500 * already exited the guest. Hence HMI keeps waking up secondaries 501 * from nap in a loop and secondaries always go back to nap since 502 * no vcore is assigned to them. This makes impossible for primary 503 * thread to get hold of secondary threads resulting into a soft 504 * lockup in KVM path. 505 * 506 * Let us check if HMI is pending and handle it before we go to nap. 507 */ 508 cmpwi r12, BOOK3S_INTERRUPT_HMI 509 bne 55f 510 li r3, 0 /* NULL argument */ 511 bl hmi_exception_realmode 51255: 513 /* 514 * Ensure that secondary doesn't nap when it has 515 * its vcore pointer set. 516 */ 517 sync /* matches smp_mb() before setting split_info.do_nap */ 518 ld r0, HSTATE_KVM_VCORE(r13) 519 cmpdi r0, 0 520 bne kvm_no_guest 521 /* clear any pending message */ 522BEGIN_FTR_SECTION 523 lis r6, (PPC_DBELL_SERVER << (63-36))@h 524 PPC_MSGCLR(6) 525END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 526 /* Set kvm_split_mode.napped[tid] = 1 */ 527 ld r3, HSTATE_SPLIT_MODE(r13) 528 li r0, 1 529 lbz r4, HSTATE_TID(r13) 530 addi r4, r4, KVM_SPLIT_NAPPED 531 stbx r0, r3, r4 532 /* Check the do_nap flag again after setting napped[] */ 533 sync 534 lbz r0, KVM_SPLIT_DO_NAP(r3) 535 cmpwi r0, 0 536 beq 57f 537 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 538 mfspr r5, SPRN_LPCR 539 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 540 b kvm_nap_sequence 541 54257: li r0, 0 543 stbx r0, r3, r4 544 b kvm_no_guest 545 546/****************************************************************************** 547 * * 548 * Entry code * 549 * * 550 *****************************************************************************/ 551 552.global kvmppc_hv_entry 553kvmppc_hv_entry: 554 555 /* Required state: 556 * 557 * R4 = vcpu pointer (or NULL) 558 * MSR = ~IR|DR 559 * R13 = PACA 560 * R1 = host R1 561 * R2 = TOC 562 * all other volatile GPRS = free 563 * Does not preserve non-volatile GPRs or CR fields 564 */ 565 mflr r0 566 std r0, PPC_LR_STKOFF(r1) 567 stdu r1, -SFS(r1) 568 569 /* Save R1 in the PACA */ 570 std r1, HSTATE_HOST_R1(r13) 571 572 li r6, KVM_GUEST_MODE_HOST_HV 573 stb r6, HSTATE_IN_GUEST(r13) 574 575#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 576 /* Store initial timestamp */ 577 cmpdi r4, 0 578 beq 1f 579 addi r3, r4, VCPU_TB_RMENTRY 580 bl kvmhv_start_timing 5811: 582#endif 583 584 /* Use cr7 as an indication of radix mode */ 585 ld r5, HSTATE_KVM_VCORE(r13) 586 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 587 lbz r0, KVM_RADIX(r9) 588 cmpwi cr7, r0, 0 589 590 /* 591 * POWER7/POWER8 host -> guest partition switch code. 592 * We don't have to lock against concurrent tlbies, 593 * but we do have to coordinate across hardware threads. 594 */ 595 /* Set bit in entry map iff exit map is zero. */ 596 li r7, 1 597 lbz r6, HSTATE_PTID(r13) 598 sld r7, r7, r6 599 addi r8, r5, VCORE_ENTRY_EXIT 60021: lwarx r3, 0, r8 601 cmpwi r3, 0x100 /* any threads starting to exit? */ 602 bge secondary_too_late /* if so we're too late to the party */ 603 or r3, r3, r7 604 stwcx. r3, 0, r8 605 bne 21b 606 607 /* Primary thread switches to guest partition. */ 608 cmpwi r6,0 609 bne 10f 610 611 /* Radix has already switched LPID and flushed core TLB */ 612 bne cr7, 22f 613 614 lwz r7,KVM_LPID(r9) 615BEGIN_FTR_SECTION 616 ld r6,KVM_SDR1(r9) 617 li r0,LPID_RSVD /* switch to reserved LPID */ 618 mtspr SPRN_LPID,r0 619 ptesync 620 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 621END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 622 mtspr SPRN_LPID,r7 623 isync 624 625 /* See if we need to flush the TLB. Hash has to be done in RM */ 626 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */ 627BEGIN_FTR_SECTION 628 /* 629 * On POWER9, individual threads can come in here, but the 630 * TLB is shared between the 4 threads in a core, hence 631 * invalidating on one thread invalidates for all. 632 * Thus we make all 4 threads use the same bit here. 633 */ 634 clrrdi r6,r6,2 635END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 636 clrldi r7,r6,64-6 /* extract bit number (6 bits) */ 637 srdi r6,r6,6 /* doubleword number */ 638 sldi r6,r6,3 /* address offset */ 639 add r6,r6,r9 640 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */ 641 li r8,1 642 sld r8,r8,r7 643 ld r7,0(r6) 644 and. r7,r7,r8 645 beq 22f 646 /* Flush the TLB of any entries for this LPID */ 647 lwz r0,KVM_TLB_SETS(r9) 648 mtctr r0 649 li r7,0x800 /* IS field = 0b10 */ 650 ptesync 651 li r0,0 /* RS for P9 version of tlbiel */ 65228: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */ 653 addi r7,r7,0x1000 654 bdnz 28b 655 ptesync 65623: ldarx r7,0,r6 /* clear the bit after TLB flushed */ 657 andc r7,r7,r8 658 stdcx. r7,0,r6 659 bne 23b 660 661 /* Add timebase offset onto timebase */ 66222: ld r8,VCORE_TB_OFFSET(r5) 663 cmpdi r8,0 664 beq 37f 665 std r8, VCORE_TB_OFFSET_APPL(r5) 666 mftb r6 /* current host timebase */ 667 add r8,r8,r6 668 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 669 mftb r7 /* check if lower 24 bits overflowed */ 670 clrldi r6,r6,40 671 clrldi r7,r7,40 672 cmpld r7,r6 673 bge 37f 674 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 675 mtspr SPRN_TBU40,r8 676 677 /* Load guest PCR value to select appropriate compat mode */ 67837: ld r7, VCORE_PCR(r5) 679 cmpdi r7, 0 680 beq 38f 681 mtspr SPRN_PCR, r7 68238: 683 684BEGIN_FTR_SECTION 685 /* DPDES and VTB are shared between threads */ 686 ld r8, VCORE_DPDES(r5) 687 ld r7, VCORE_VTB(r5) 688 mtspr SPRN_DPDES, r8 689 mtspr SPRN_VTB, r7 690END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 691 692 /* Mark the subcore state as inside guest */ 693 bl kvmppc_subcore_enter_guest 694 nop 695 ld r5, HSTATE_KVM_VCORE(r13) 696 ld r4, HSTATE_KVM_VCPU(r13) 697 li r0,1 698 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 699 700 /* Do we have a guest vcpu to run? */ 70110: cmpdi r4, 0 702 beq kvmppc_primary_no_guest 703kvmppc_got_guest: 704 /* Increment yield count if they have a VPA */ 705 ld r3, VCPU_VPA(r4) 706 cmpdi r3, 0 707 beq 25f 708 li r6, LPPACA_YIELDCOUNT 709 LWZX_BE r5, r3, r6 710 addi r5, r5, 1 711 STWX_BE r5, r3, r6 712 li r6, 1 713 stb r6, VCPU_VPA_DIRTY(r4) 71425: 715 716 /* Save purr/spurr */ 717 mfspr r5,SPRN_PURR 718 mfspr r6,SPRN_SPURR 719 std r5,HSTATE_PURR(r13) 720 std r6,HSTATE_SPURR(r13) 721 ld r7,VCPU_PURR(r4) 722 ld r8,VCPU_SPURR(r4) 723 mtspr SPRN_PURR,r7 724 mtspr SPRN_SPURR,r8 725 726 /* Save host values of some registers */ 727BEGIN_FTR_SECTION 728 mfspr r5, SPRN_TIDR 729 mfspr r6, SPRN_PSSCR 730 mfspr r7, SPRN_PID 731 std r5, STACK_SLOT_TID(r1) 732 std r6, STACK_SLOT_PSSCR(r1) 733 std r7, STACK_SLOT_PID(r1) 734 mfspr r5, SPRN_HFSCR 735 std r5, STACK_SLOT_HFSCR(r1) 736END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 737BEGIN_FTR_SECTION 738 mfspr r5, SPRN_CIABR 739 mfspr r6, SPRN_DAWR 740 mfspr r7, SPRN_DAWRX 741 mfspr r8, SPRN_IAMR 742 std r5, STACK_SLOT_CIABR(r1) 743 std r6, STACK_SLOT_DAWR(r1) 744 std r7, STACK_SLOT_DAWRX(r1) 745 std r8, STACK_SLOT_IAMR(r1) 746END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 747 748 mfspr r5, SPRN_AMR 749 std r5, STACK_SLOT_AMR(r1) 750 mfspr r6, SPRN_UAMOR 751 std r6, STACK_SLOT_UAMOR(r1) 752 753BEGIN_FTR_SECTION 754 /* Set partition DABR */ 755 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 756 lwz r5,VCPU_DABRX(r4) 757 ld r6,VCPU_DABR(r4) 758 mtspr SPRN_DABRX,r5 759 mtspr SPRN_DABR,r6 760 isync 761END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 762 763#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 764/* 765 * Branch around the call if both CPU_FTR_TM and 766 * CPU_FTR_P9_TM_HV_ASSIST are off. 767 */ 768BEGIN_FTR_SECTION 769 b 91f 770END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 771 /* 772 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 773 */ 774 mr r3, r4 775 ld r4, VCPU_MSR(r3) 776 li r5, 0 /* don't preserve non-vol regs */ 777 bl kvmppc_restore_tm_hv 778 nop 779 ld r4, HSTATE_KVM_VCPU(r13) 78091: 781#endif 782 783 /* Load guest PMU registers; r4 = vcpu pointer here */ 784 mr r3, r4 785 bl kvmhv_load_guest_pmu 786 787 /* Load up FP, VMX and VSX registers */ 788 ld r4, HSTATE_KVM_VCPU(r13) 789 bl kvmppc_load_fp 790 791 ld r14, VCPU_GPR(R14)(r4) 792 ld r15, VCPU_GPR(R15)(r4) 793 ld r16, VCPU_GPR(R16)(r4) 794 ld r17, VCPU_GPR(R17)(r4) 795 ld r18, VCPU_GPR(R18)(r4) 796 ld r19, VCPU_GPR(R19)(r4) 797 ld r20, VCPU_GPR(R20)(r4) 798 ld r21, VCPU_GPR(R21)(r4) 799 ld r22, VCPU_GPR(R22)(r4) 800 ld r23, VCPU_GPR(R23)(r4) 801 ld r24, VCPU_GPR(R24)(r4) 802 ld r25, VCPU_GPR(R25)(r4) 803 ld r26, VCPU_GPR(R26)(r4) 804 ld r27, VCPU_GPR(R27)(r4) 805 ld r28, VCPU_GPR(R28)(r4) 806 ld r29, VCPU_GPR(R29)(r4) 807 ld r30, VCPU_GPR(R30)(r4) 808 ld r31, VCPU_GPR(R31)(r4) 809 810 /* Switch DSCR to guest value */ 811 ld r5, VCPU_DSCR(r4) 812 mtspr SPRN_DSCR, r5 813 814BEGIN_FTR_SECTION 815 /* Skip next section on POWER7 */ 816 b 8f 817END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 818 /* Load up POWER8-specific registers */ 819 ld r5, VCPU_IAMR(r4) 820 lwz r6, VCPU_PSPB(r4) 821 ld r7, VCPU_FSCR(r4) 822 mtspr SPRN_IAMR, r5 823 mtspr SPRN_PSPB, r6 824 mtspr SPRN_FSCR, r7 825 ld r5, VCPU_DAWR(r4) 826 ld r6, VCPU_DAWRX(r4) 827 ld r7, VCPU_CIABR(r4) 828 ld r8, VCPU_TAR(r4) 829 /* 830 * Handle broken DAWR case by not writing it. This means we 831 * can still store the DAWR register for migration. 832 */ 833BEGIN_FTR_SECTION 834 mtspr SPRN_DAWR, r5 835 mtspr SPRN_DAWRX, r6 836END_FTR_SECTION_IFSET(CPU_FTR_DAWR) 837 mtspr SPRN_CIABR, r7 838 mtspr SPRN_TAR, r8 839 ld r5, VCPU_IC(r4) 840 ld r8, VCPU_EBBHR(r4) 841 mtspr SPRN_IC, r5 842 mtspr SPRN_EBBHR, r8 843 ld r5, VCPU_EBBRR(r4) 844 ld r6, VCPU_BESCR(r4) 845 lwz r7, VCPU_GUEST_PID(r4) 846 ld r8, VCPU_WORT(r4) 847 mtspr SPRN_EBBRR, r5 848 mtspr SPRN_BESCR, r6 849 mtspr SPRN_PID, r7 850 mtspr SPRN_WORT, r8 851BEGIN_FTR_SECTION 852 /* POWER8-only registers */ 853 ld r5, VCPU_TCSCR(r4) 854 ld r6, VCPU_ACOP(r4) 855 ld r7, VCPU_CSIGR(r4) 856 ld r8, VCPU_TACR(r4) 857 mtspr SPRN_TCSCR, r5 858 mtspr SPRN_ACOP, r6 859 mtspr SPRN_CSIGR, r7 860 mtspr SPRN_TACR, r8 861 nop 862FTR_SECTION_ELSE 863 /* POWER9-only registers */ 864 ld r5, VCPU_TID(r4) 865 ld r6, VCPU_PSSCR(r4) 866 lbz r8, HSTATE_FAKE_SUSPEND(r13) 867 oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ 868 rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG 869 ld r7, VCPU_HFSCR(r4) 870 mtspr SPRN_TIDR, r5 871 mtspr SPRN_PSSCR, r6 872 mtspr SPRN_HFSCR, r7 873ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8748: 875 876 ld r5, VCPU_SPRG0(r4) 877 ld r6, VCPU_SPRG1(r4) 878 ld r7, VCPU_SPRG2(r4) 879 ld r8, VCPU_SPRG3(r4) 880 mtspr SPRN_SPRG0, r5 881 mtspr SPRN_SPRG1, r6 882 mtspr SPRN_SPRG2, r7 883 mtspr SPRN_SPRG3, r8 884 885 /* Load up DAR and DSISR */ 886 ld r5, VCPU_DAR(r4) 887 lwz r6, VCPU_DSISR(r4) 888 mtspr SPRN_DAR, r5 889 mtspr SPRN_DSISR, r6 890 891 /* Restore AMR and UAMOR, set AMOR to all 1s */ 892 ld r5,VCPU_AMR(r4) 893 ld r6,VCPU_UAMOR(r4) 894 li r7,-1 895 mtspr SPRN_AMR,r5 896 mtspr SPRN_UAMOR,r6 897 mtspr SPRN_AMOR,r7 898 899 /* Restore state of CTRL run bit; assume 1 on entry */ 900 lwz r5,VCPU_CTRL(r4) 901 andi. r5,r5,1 902 bne 4f 903 mfspr r6,SPRN_CTRLF 904 clrrdi r6,r6,1 905 mtspr SPRN_CTRLT,r6 9064: 907 /* Secondary threads wait for primary to have done partition switch */ 908 ld r5, HSTATE_KVM_VCORE(r13) 909 lbz r6, HSTATE_PTID(r13) 910 cmpwi r6, 0 911 beq 21f 912 lbz r0, VCORE_IN_GUEST(r5) 913 cmpwi r0, 0 914 bne 21f 915 HMT_LOW 91620: lwz r3, VCORE_ENTRY_EXIT(r5) 917 cmpwi r3, 0x100 918 bge no_switch_exit 919 lbz r0, VCORE_IN_GUEST(r5) 920 cmpwi r0, 0 921 beq 20b 922 HMT_MEDIUM 92321: 924 /* Set LPCR. */ 925 ld r8,VCORE_LPCR(r5) 926 mtspr SPRN_LPCR,r8 927 isync 928 929 /* 930 * Set the decrementer to the guest decrementer. 931 */ 932 ld r8,VCPU_DEC_EXPIRES(r4) 933 /* r8 is a host timebase value here, convert to guest TB */ 934 ld r5,HSTATE_KVM_VCORE(r13) 935 ld r6,VCORE_TB_OFFSET_APPL(r5) 936 add r8,r8,r6 937 mftb r7 938 subf r3,r7,r8 939 mtspr SPRN_DEC,r3 940 941 /* Check if HDEC expires soon */ 942 mfspr r3, SPRN_HDEC 943 EXTEND_HDEC(r3) 944 cmpdi r3, 512 /* 1 microsecond */ 945 blt hdec_soon 946 947 /* For hash guest, clear out and reload the SLB */ 948 ld r6, VCPU_KVM(r4) 949 lbz r0, KVM_RADIX(r6) 950 cmpwi r0, 0 951 bne 9f 952 li r6, 0 953 slbmte r6, r6 954 slbia 955 ptesync 956 957 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 958 lwz r5,VCPU_SLB_MAX(r4) 959 cmpwi r5,0 960 beq 9f 961 mtctr r5 962 addi r6,r4,VCPU_SLB 9631: ld r8,VCPU_SLB_E(r6) 964 ld r9,VCPU_SLB_V(r6) 965 slbmte r9,r8 966 addi r6,r6,VCPU_SLB_SIZE 967 bdnz 1b 9689: 969 970#ifdef CONFIG_KVM_XICS 971 /* We are entering the guest on that thread, push VCPU to XIVE */ 972 ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 973 cmpldi cr0, r10, 0 974 beq no_xive 975 ld r11, VCPU_XIVE_SAVED_STATE(r4) 976 li r9, TM_QW1_OS 977 eieio 978 stdcix r11,r9,r10 979 lwz r11, VCPU_XIVE_CAM_WORD(r4) 980 li r9, TM_QW1_OS + TM_WORD2 981 stwcix r11,r9,r10 982 li r9, 1 983 stb r9, VCPU_XIVE_PUSHED(r4) 984 eieio 985 986 /* 987 * We clear the irq_pending flag. There is a small chance of a 988 * race vs. the escalation interrupt happening on another 989 * processor setting it again, but the only consequence is to 990 * cause a spurrious wakeup on the next H_CEDE which is not an 991 * issue. 992 */ 993 li r0,0 994 stb r0, VCPU_IRQ_PENDING(r4) 995 996 /* 997 * In single escalation mode, if the escalation interrupt is 998 * on, we mask it. 999 */ 1000 lbz r0, VCPU_XIVE_ESC_ON(r4) 1001 cmpwi r0,0 1002 beq 1f 1003 ld r10, VCPU_XIVE_ESC_RADDR(r4) 1004 li r9, XIVE_ESB_SET_PQ_01 1005 ldcix r0, r10, r9 1006 sync 1007 1008 /* We have a possible subtle race here: The escalation interrupt might 1009 * have fired and be on its way to the host queue while we mask it, 1010 * and if we unmask it early enough (re-cede right away), there is 1011 * a theorical possibility that it fires again, thus landing in the 1012 * target queue more than once which is a big no-no. 1013 * 1014 * Fortunately, solving this is rather easy. If the above load setting 1015 * PQ to 01 returns a previous value where P is set, then we know the 1016 * escalation interrupt is somewhere on its way to the host. In that 1017 * case we simply don't clear the xive_esc_on flag below. It will be 1018 * eventually cleared by the handler for the escalation interrupt. 1019 * 1020 * Then, when doing a cede, we check that flag again before re-enabling 1021 * the escalation interrupt, and if set, we abort the cede. 1022 */ 1023 andi. r0, r0, XIVE_ESB_VAL_P 1024 bne- 1f 1025 1026 /* Now P is 0, we can clear the flag */ 1027 li r0, 0 1028 stb r0, VCPU_XIVE_ESC_ON(r4) 10291: 1030no_xive: 1031#endif /* CONFIG_KVM_XICS */ 1032 1033 li r0, 0 1034 stw r0, STACK_SLOT_SHORT_PATH(r1) 1035 1036deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 1037 /* Check if we can deliver an external or decrementer interrupt now */ 1038 ld r0, VCPU_PENDING_EXC(r4) 1039BEGIN_FTR_SECTION 1040 /* On POWER9, also check for emulated doorbell interrupt */ 1041 lbz r3, VCPU_DBELL_REQ(r4) 1042 or r0, r0, r3 1043END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1044 cmpdi r0, 0 1045 beq 71f 1046 mr r3, r4 1047 bl kvmppc_guest_entry_inject_int 1048 ld r4, HSTATE_KVM_VCPU(r13) 104971: 1050 ld r6, VCPU_SRR0(r4) 1051 ld r7, VCPU_SRR1(r4) 1052 mtspr SPRN_SRR0, r6 1053 mtspr SPRN_SRR1, r7 1054 1055fast_guest_entry_c: 1056 ld r10, VCPU_PC(r4) 1057 ld r11, VCPU_MSR(r4) 1058 /* r11 = vcpu->arch.msr & ~MSR_HV */ 1059 rldicl r11, r11, 63 - MSR_HV_LG, 1 1060 rotldi r11, r11, 1 + MSR_HV_LG 1061 ori r11, r11, MSR_ME 1062 1063 ld r6, VCPU_CTR(r4) 1064 ld r7, VCPU_XER(r4) 1065 mtctr r6 1066 mtxer r7 1067 1068/* 1069 * Required state: 1070 * R4 = vcpu 1071 * R10: value for HSRR0 1072 * R11: value for HSRR1 1073 * R13 = PACA 1074 */ 1075fast_guest_return: 1076 li r0,0 1077 stb r0,VCPU_CEDED(r4) /* cancel cede */ 1078 mtspr SPRN_HSRR0,r10 1079 mtspr SPRN_HSRR1,r11 1080 1081 /* Activate guest mode, so faults get handled by KVM */ 1082 li r9, KVM_GUEST_MODE_GUEST_HV 1083 stb r9, HSTATE_IN_GUEST(r13) 1084 1085#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1086 /* Accumulate timing */ 1087 addi r3, r4, VCPU_TB_GUEST 1088 bl kvmhv_accumulate_time 1089#endif 1090 1091 /* Enter guest */ 1092 1093BEGIN_FTR_SECTION 1094 ld r5, VCPU_CFAR(r4) 1095 mtspr SPRN_CFAR, r5 1096END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1097BEGIN_FTR_SECTION 1098 ld r0, VCPU_PPR(r4) 1099END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1100 1101 ld r5, VCPU_LR(r4) 1102 ld r6, VCPU_CR(r4) 1103 mtlr r5 1104 mtcr r6 1105 1106 ld r1, VCPU_GPR(R1)(r4) 1107 ld r2, VCPU_GPR(R2)(r4) 1108 ld r3, VCPU_GPR(R3)(r4) 1109 ld r5, VCPU_GPR(R5)(r4) 1110 ld r6, VCPU_GPR(R6)(r4) 1111 ld r7, VCPU_GPR(R7)(r4) 1112 ld r8, VCPU_GPR(R8)(r4) 1113 ld r9, VCPU_GPR(R9)(r4) 1114 ld r10, VCPU_GPR(R10)(r4) 1115 ld r11, VCPU_GPR(R11)(r4) 1116 ld r12, VCPU_GPR(R12)(r4) 1117 ld r13, VCPU_GPR(R13)(r4) 1118 1119BEGIN_FTR_SECTION 1120 mtspr SPRN_PPR, r0 1121END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1122 1123/* Move canary into DSISR to check for later */ 1124BEGIN_FTR_SECTION 1125 li r0, 0x7fff 1126 mtspr SPRN_HDSISR, r0 1127END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1128 1129 ld r0, VCPU_GPR(R0)(r4) 1130 ld r4, VCPU_GPR(R4)(r4) 1131 HRFI_TO_GUEST 1132 b . 1133 1134/* 1135 * Enter the guest on a P9 or later system where we have exactly 1136 * one vcpu per vcore and we don't need to go to real mode 1137 * (which implies that host and guest are both using radix MMU mode). 1138 * r3 = vcpu pointer 1139 * Most SPRs and all the VSRs have been loaded already. 1140 */ 1141_GLOBAL(__kvmhv_vcpu_entry_p9) 1142EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) 1143 mflr r0 1144 std r0, PPC_LR_STKOFF(r1) 1145 stdu r1, -SFS(r1) 1146 1147 li r0, 1 1148 stw r0, STACK_SLOT_SHORT_PATH(r1) 1149 1150 std r3, HSTATE_KVM_VCPU(r13) 1151 mfcr r4 1152 stw r4, SFS+8(r1) 1153 1154 std r1, HSTATE_HOST_R1(r13) 1155 1156 reg = 14 1157 .rept 18 1158 std reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1159 reg = reg + 1 1160 .endr 1161 1162 reg = 14 1163 .rept 18 1164 ld reg, __VCPU_GPR(reg)(r3) 1165 reg = reg + 1 1166 .endr 1167 1168 mfmsr r10 1169 std r10, HSTATE_HOST_MSR(r13) 1170 1171 mr r4, r3 1172 b fast_guest_entry_c 1173guest_exit_short_path: 1174 1175 li r0, KVM_GUEST_MODE_NONE 1176 stb r0, HSTATE_IN_GUEST(r13) 1177 1178 reg = 14 1179 .rept 18 1180 std reg, __VCPU_GPR(reg)(r9) 1181 reg = reg + 1 1182 .endr 1183 1184 reg = 14 1185 .rept 18 1186 ld reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) 1187 reg = reg + 1 1188 .endr 1189 1190 lwz r4, SFS+8(r1) 1191 mtcr r4 1192 1193 mr r3, r12 /* trap number */ 1194 1195 addi r1, r1, SFS 1196 ld r0, PPC_LR_STKOFF(r1) 1197 mtlr r0 1198 1199 /* If we are in real mode, do a rfid to get back to the caller */ 1200 mfmsr r4 1201 andi. r5, r4, MSR_IR 1202 bnelr 1203 rldicl r5, r4, 64 - MSR_TS_S_LG, 62 /* extract TS field */ 1204 mtspr SPRN_SRR0, r0 1205 ld r10, HSTATE_HOST_MSR(r13) 1206 rldimi r10, r5, MSR_TS_S_LG, 63 - MSR_TS_T_LG 1207 mtspr SPRN_SRR1, r10 1208 RFI_TO_KERNEL 1209 b . 1210 1211secondary_too_late: 1212 li r12, 0 1213 stw r12, STACK_SLOT_TRAP(r1) 1214 cmpdi r4, 0 1215 beq 11f 1216 stw r12, VCPU_TRAP(r4) 1217#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1218 addi r3, r4, VCPU_TB_RMEXIT 1219 bl kvmhv_accumulate_time 1220#endif 122111: b kvmhv_switch_to_host 1222 1223no_switch_exit: 1224 HMT_MEDIUM 1225 li r12, 0 1226 b 12f 1227hdec_soon: 1228 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 122912: stw r12, VCPU_TRAP(r4) 1230 mr r9, r4 1231#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1232 addi r3, r4, VCPU_TB_RMEXIT 1233 bl kvmhv_accumulate_time 1234#endif 1235 b guest_bypass 1236 1237/****************************************************************************** 1238 * * 1239 * Exit code * 1240 * * 1241 *****************************************************************************/ 1242 1243/* 1244 * We come here from the first-level interrupt handlers. 1245 */ 1246 .globl kvmppc_interrupt_hv 1247kvmppc_interrupt_hv: 1248 /* 1249 * Register contents: 1250 * R12 = (guest CR << 32) | interrupt vector 1251 * R13 = PACA 1252 * guest R12 saved in shadow VCPU SCRATCH0 1253 * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE 1254 * guest R13 saved in SPRN_SCRATCH0 1255 */ 1256 std r9, HSTATE_SCRATCH2(r13) 1257 lbz r9, HSTATE_IN_GUEST(r13) 1258 cmpwi r9, KVM_GUEST_MODE_HOST_HV 1259 beq kvmppc_bad_host_intr 1260#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1261 cmpwi r9, KVM_GUEST_MODE_GUEST 1262 ld r9, HSTATE_SCRATCH2(r13) 1263 beq kvmppc_interrupt_pr 1264#endif 1265 /* We're now back in the host but in guest MMU context */ 1266 li r9, KVM_GUEST_MODE_HOST_HV 1267 stb r9, HSTATE_IN_GUEST(r13) 1268 1269 ld r9, HSTATE_KVM_VCPU(r13) 1270 1271 /* Save registers */ 1272 1273 std r0, VCPU_GPR(R0)(r9) 1274 std r1, VCPU_GPR(R1)(r9) 1275 std r2, VCPU_GPR(R2)(r9) 1276 std r3, VCPU_GPR(R3)(r9) 1277 std r4, VCPU_GPR(R4)(r9) 1278 std r5, VCPU_GPR(R5)(r9) 1279 std r6, VCPU_GPR(R6)(r9) 1280 std r7, VCPU_GPR(R7)(r9) 1281 std r8, VCPU_GPR(R8)(r9) 1282 ld r0, HSTATE_SCRATCH2(r13) 1283 std r0, VCPU_GPR(R9)(r9) 1284 std r10, VCPU_GPR(R10)(r9) 1285 std r11, VCPU_GPR(R11)(r9) 1286 ld r3, HSTATE_SCRATCH0(r13) 1287 std r3, VCPU_GPR(R12)(r9) 1288 /* CR is in the high half of r12 */ 1289 srdi r4, r12, 32 1290 std r4, VCPU_CR(r9) 1291BEGIN_FTR_SECTION 1292 ld r3, HSTATE_CFAR(r13) 1293 std r3, VCPU_CFAR(r9) 1294END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1295BEGIN_FTR_SECTION 1296 ld r4, HSTATE_PPR(r13) 1297 std r4, VCPU_PPR(r9) 1298END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1299 1300 /* Restore R1/R2 so we can handle faults */ 1301 ld r1, HSTATE_HOST_R1(r13) 1302 ld r2, PACATOC(r13) 1303 1304 mfspr r10, SPRN_SRR0 1305 mfspr r11, SPRN_SRR1 1306 std r10, VCPU_SRR0(r9) 1307 std r11, VCPU_SRR1(r9) 1308 /* trap is in the low half of r12, clear CR from the high half */ 1309 clrldi r12, r12, 32 1310 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1311 beq 1f 1312 mfspr r10, SPRN_HSRR0 1313 mfspr r11, SPRN_HSRR1 1314 clrrdi r12, r12, 2 13151: std r10, VCPU_PC(r9) 1316 std r11, VCPU_MSR(r9) 1317 1318 GET_SCRATCH0(r3) 1319 mflr r4 1320 std r3, VCPU_GPR(R13)(r9) 1321 std r4, VCPU_LR(r9) 1322 1323 stw r12,VCPU_TRAP(r9) 1324 1325 /* 1326 * Now that we have saved away SRR0/1 and HSRR0/1, 1327 * interrupts are recoverable in principle, so set MSR_RI. 1328 * This becomes important for relocation-on interrupts from 1329 * the guest, which we can get in radix mode on POWER9. 1330 */ 1331 li r0, MSR_RI 1332 mtmsrd r0, 1 1333 1334#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1335 addi r3, r9, VCPU_TB_RMINTR 1336 mr r4, r9 1337 bl kvmhv_accumulate_time 1338 ld r5, VCPU_GPR(R5)(r9) 1339 ld r6, VCPU_GPR(R6)(r9) 1340 ld r7, VCPU_GPR(R7)(r9) 1341 ld r8, VCPU_GPR(R8)(r9) 1342#endif 1343 1344 /* Save HEIR (HV emulation assist reg) in emul_inst 1345 if this is an HEI (HV emulation interrupt, e40) */ 1346 li r3,KVM_INST_FETCH_FAILED 1347 stw r3,VCPU_LAST_INST(r9) 1348 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1349 bne 11f 1350 mfspr r3,SPRN_HEIR 135111: stw r3,VCPU_HEIR(r9) 1352 1353 /* these are volatile across C function calls */ 1354#ifdef CONFIG_RELOCATABLE 1355 ld r3, HSTATE_SCRATCH1(r13) 1356 mtctr r3 1357#else 1358 mfctr r3 1359#endif 1360 mfxer r4 1361 std r3, VCPU_CTR(r9) 1362 std r4, VCPU_XER(r9) 1363 1364 /* Save more register state */ 1365 mfdar r3 1366 mfdsisr r4 1367 std r3, VCPU_DAR(r9) 1368 stw r4, VCPU_DSISR(r9) 1369 1370 /* If this is a page table miss then see if it's theirs or ours */ 1371 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1372 beq kvmppc_hdsi 1373 std r3, VCPU_FAULT_DAR(r9) 1374 stw r4, VCPU_FAULT_DSISR(r9) 1375 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1376 beq kvmppc_hisi 1377 1378#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1379 /* For softpatch interrupt, go off and do TM instruction emulation */ 1380 cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 1381 beq kvmppc_tm_emul 1382#endif 1383 1384 /* See if this is a leftover HDEC interrupt */ 1385 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1386 bne 2f 1387 mfspr r3,SPRN_HDEC 1388 EXTEND_HDEC(r3) 1389 cmpdi r3,0 1390 mr r4,r9 1391 bge fast_guest_return 13922: 1393 /* See if this is an hcall we can handle in real mode */ 1394 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1395 beq hcall_try_real_mode 1396 1397 /* Hypervisor doorbell - exit only if host IPI flag set */ 1398 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1399 bne 3f 1400BEGIN_FTR_SECTION 1401 PPC_MSGSYNC 1402 lwsync 1403 /* always exit if we're running a nested guest */ 1404 ld r0, VCPU_NESTED(r9) 1405 cmpdi r0, 0 1406 bne guest_exit_cont 1407END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1408 lbz r0, HSTATE_HOST_IPI(r13) 1409 cmpwi r0, 0 1410 beq maybe_reenter_guest 1411 b guest_exit_cont 14123: 1413 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1414 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1415 bne 14f 1416 mfspr r3, SPRN_HFSCR 1417 std r3, VCPU_HFSCR(r9) 1418 b guest_exit_cont 141914: 1420 /* External interrupt ? */ 1421 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1422 beq kvmppc_guest_external 1423 /* See if it is a machine check */ 1424 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1425 beq machine_check_realmode 1426 /* Or a hypervisor maintenance interrupt */ 1427 cmpwi r12, BOOK3S_INTERRUPT_HMI 1428 beq hmi_realmode 1429 1430guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1431 1432#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1433 addi r3, r9, VCPU_TB_RMEXIT 1434 mr r4, r9 1435 bl kvmhv_accumulate_time 1436#endif 1437#ifdef CONFIG_KVM_XICS 1438 /* We are exiting, pull the VP from the XIVE */ 1439 lbz r0, VCPU_XIVE_PUSHED(r9) 1440 cmpwi cr0, r0, 0 1441 beq 1f 1442 li r7, TM_SPC_PULL_OS_CTX 1443 li r6, TM_QW1_OS 1444 mfmsr r0 1445 andi. r0, r0, MSR_DR /* in real mode? */ 1446 beq 2f 1447 ld r10, HSTATE_XIVE_TIMA_VIRT(r13) 1448 cmpldi cr0, r10, 0 1449 beq 1f 1450 /* First load to pull the context, we ignore the value */ 1451 eieio 1452 lwzx r11, r7, r10 1453 /* Second load to recover the context state (Words 0 and 1) */ 1454 ldx r11, r6, r10 1455 b 3f 14562: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) 1457 cmpldi cr0, r10, 0 1458 beq 1f 1459 /* First load to pull the context, we ignore the value */ 1460 eieio 1461 lwzcix r11, r7, r10 1462 /* Second load to recover the context state (Words 0 and 1) */ 1463 ldcix r11, r6, r10 14643: std r11, VCPU_XIVE_SAVED_STATE(r9) 1465 /* Fixup some of the state for the next load */ 1466 li r10, 0 1467 li r0, 0xff 1468 stb r10, VCPU_XIVE_PUSHED(r9) 1469 stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) 1470 stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) 1471 eieio 14721: 1473#endif /* CONFIG_KVM_XICS */ 1474 1475 /* If we came in through the P9 short path, go back out to C now */ 1476 lwz r0, STACK_SLOT_SHORT_PATH(r1) 1477 cmpwi r0, 0 1478 bne guest_exit_short_path 1479 1480 /* For hash guest, read the guest SLB and save it away */ 1481 ld r5, VCPU_KVM(r9) 1482 lbz r0, KVM_RADIX(r5) 1483 li r5, 0 1484 cmpwi r0, 0 1485 bne 3f /* for radix, save 0 entries */ 1486 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1487 mtctr r0 1488 li r6,0 1489 addi r7,r9,VCPU_SLB 14901: slbmfee r8,r6 1491 andis. r0,r8,SLB_ESID_V@h 1492 beq 2f 1493 add r8,r8,r6 /* put index in */ 1494 slbmfev r3,r6 1495 std r8,VCPU_SLB_E(r7) 1496 std r3,VCPU_SLB_V(r7) 1497 addi r7,r7,VCPU_SLB_SIZE 1498 addi r5,r5,1 14992: addi r6,r6,1 1500 bdnz 1b 1501 /* Finally clear out the SLB */ 1502 li r0,0 1503 slbmte r0,r0 1504 slbia 1505 ptesync 15063: stw r5,VCPU_SLB_MAX(r9) 1507 1508 /* load host SLB entries */ 1509BEGIN_MMU_FTR_SECTION 1510 b 0f 1511END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 1512 ld r8,PACA_SLBSHADOWPTR(r13) 1513 1514 .rept SLB_NUM_BOLTED 1515 li r3, SLBSHADOW_SAVEAREA 1516 LDX_BE r5, r8, r3 1517 addi r3, r3, 8 1518 LDX_BE r6, r8, r3 1519 andis. r7,r5,SLB_ESID_V@h 1520 beq 1f 1521 slbmte r6,r5 15221: addi r8,r8,16 1523 .endr 15240: 1525 1526guest_bypass: 1527 stw r12, STACK_SLOT_TRAP(r1) 1528 1529 /* Save DEC */ 1530 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1531 ld r3, HSTATE_KVM_VCORE(r13) 1532 mfspr r5,SPRN_DEC 1533 mftb r6 1534 /* On P9, if the guest has large decr enabled, don't sign extend */ 1535BEGIN_FTR_SECTION 1536 ld r4, VCORE_LPCR(r3) 1537 andis. r4, r4, LPCR_LD@h 1538 bne 16f 1539END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1540 extsw r5,r5 154116: add r5,r5,r6 1542 /* r5 is a guest timebase value here, convert to host TB */ 1543 ld r4,VCORE_TB_OFFSET_APPL(r3) 1544 subf r5,r4,r5 1545 std r5,VCPU_DEC_EXPIRES(r9) 1546 1547 /* Increment exit count, poke other threads to exit */ 1548 mr r3, r12 1549 bl kvmhv_commence_exit 1550 nop 1551 ld r9, HSTATE_KVM_VCPU(r13) 1552 1553 /* Stop others sending VCPU interrupts to this physical CPU */ 1554 li r0, -1 1555 stw r0, VCPU_CPU(r9) 1556 stw r0, VCPU_THREAD_CPU(r9) 1557 1558 /* Save guest CTRL register, set runlatch to 1 */ 1559 mfspr r6,SPRN_CTRLF 1560 stw r6,VCPU_CTRL(r9) 1561 andi. r0,r6,1 1562 bne 4f 1563 ori r6,r6,1 1564 mtspr SPRN_CTRLT,r6 15654: 1566 /* 1567 * Save the guest PURR/SPURR 1568 */ 1569 mfspr r5,SPRN_PURR 1570 mfspr r6,SPRN_SPURR 1571 ld r7,VCPU_PURR(r9) 1572 ld r8,VCPU_SPURR(r9) 1573 std r5,VCPU_PURR(r9) 1574 std r6,VCPU_SPURR(r9) 1575 subf r5,r7,r5 1576 subf r6,r8,r6 1577 1578 /* 1579 * Restore host PURR/SPURR and add guest times 1580 * so that the time in the guest gets accounted. 1581 */ 1582 ld r3,HSTATE_PURR(r13) 1583 ld r4,HSTATE_SPURR(r13) 1584 add r3,r3,r5 1585 add r4,r4,r6 1586 mtspr SPRN_PURR,r3 1587 mtspr SPRN_SPURR,r4 1588 1589BEGIN_FTR_SECTION 1590 b 8f 1591END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1592 /* Save POWER8-specific registers */ 1593 mfspr r5, SPRN_IAMR 1594 mfspr r6, SPRN_PSPB 1595 mfspr r7, SPRN_FSCR 1596 std r5, VCPU_IAMR(r9) 1597 stw r6, VCPU_PSPB(r9) 1598 std r7, VCPU_FSCR(r9) 1599 mfspr r5, SPRN_IC 1600 mfspr r7, SPRN_TAR 1601 std r5, VCPU_IC(r9) 1602 std r7, VCPU_TAR(r9) 1603 mfspr r8, SPRN_EBBHR 1604 std r8, VCPU_EBBHR(r9) 1605 mfspr r5, SPRN_EBBRR 1606 mfspr r6, SPRN_BESCR 1607 mfspr r7, SPRN_PID 1608 mfspr r8, SPRN_WORT 1609 std r5, VCPU_EBBRR(r9) 1610 std r6, VCPU_BESCR(r9) 1611 stw r7, VCPU_GUEST_PID(r9) 1612 std r8, VCPU_WORT(r9) 1613BEGIN_FTR_SECTION 1614 mfspr r5, SPRN_TCSCR 1615 mfspr r6, SPRN_ACOP 1616 mfspr r7, SPRN_CSIGR 1617 mfspr r8, SPRN_TACR 1618 std r5, VCPU_TCSCR(r9) 1619 std r6, VCPU_ACOP(r9) 1620 std r7, VCPU_CSIGR(r9) 1621 std r8, VCPU_TACR(r9) 1622FTR_SECTION_ELSE 1623 mfspr r5, SPRN_TIDR 1624 mfspr r6, SPRN_PSSCR 1625 std r5, VCPU_TID(r9) 1626 rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ 1627 rotldi r6, r6, 60 1628 std r6, VCPU_PSSCR(r9) 1629 /* Restore host HFSCR value */ 1630 ld r7, STACK_SLOT_HFSCR(r1) 1631 mtspr SPRN_HFSCR, r7 1632ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 1633 /* 1634 * Restore various registers to 0, where non-zero values 1635 * set by the guest could disrupt the host. 1636 */ 1637 li r0, 0 1638 mtspr SPRN_PSPB, r0 1639 mtspr SPRN_WORT, r0 1640BEGIN_FTR_SECTION 1641 mtspr SPRN_TCSCR, r0 1642 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1643 li r0, 1 1644 sldi r0, r0, 31 1645 mtspr SPRN_MMCRS, r0 1646END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1647 1648 /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ 1649 ld r8, STACK_SLOT_IAMR(r1) 1650 mtspr SPRN_IAMR, r8 1651 16528: /* Power7 jumps back in here */ 1653 mfspr r5,SPRN_AMR 1654 mfspr r6,SPRN_UAMOR 1655 std r5,VCPU_AMR(r9) 1656 std r6,VCPU_UAMOR(r9) 1657 ld r5,STACK_SLOT_AMR(r1) 1658 ld r6,STACK_SLOT_UAMOR(r1) 1659 mtspr SPRN_AMR, r5 1660 mtspr SPRN_UAMOR, r6 1661 1662 /* Switch DSCR back to host value */ 1663 mfspr r8, SPRN_DSCR 1664 ld r7, HSTATE_DSCR(r13) 1665 std r8, VCPU_DSCR(r9) 1666 mtspr SPRN_DSCR, r7 1667 1668 /* Save non-volatile GPRs */ 1669 std r14, VCPU_GPR(R14)(r9) 1670 std r15, VCPU_GPR(R15)(r9) 1671 std r16, VCPU_GPR(R16)(r9) 1672 std r17, VCPU_GPR(R17)(r9) 1673 std r18, VCPU_GPR(R18)(r9) 1674 std r19, VCPU_GPR(R19)(r9) 1675 std r20, VCPU_GPR(R20)(r9) 1676 std r21, VCPU_GPR(R21)(r9) 1677 std r22, VCPU_GPR(R22)(r9) 1678 std r23, VCPU_GPR(R23)(r9) 1679 std r24, VCPU_GPR(R24)(r9) 1680 std r25, VCPU_GPR(R25)(r9) 1681 std r26, VCPU_GPR(R26)(r9) 1682 std r27, VCPU_GPR(R27)(r9) 1683 std r28, VCPU_GPR(R28)(r9) 1684 std r29, VCPU_GPR(R29)(r9) 1685 std r30, VCPU_GPR(R30)(r9) 1686 std r31, VCPU_GPR(R31)(r9) 1687 1688 /* Save SPRGs */ 1689 mfspr r3, SPRN_SPRG0 1690 mfspr r4, SPRN_SPRG1 1691 mfspr r5, SPRN_SPRG2 1692 mfspr r6, SPRN_SPRG3 1693 std r3, VCPU_SPRG0(r9) 1694 std r4, VCPU_SPRG1(r9) 1695 std r5, VCPU_SPRG2(r9) 1696 std r6, VCPU_SPRG3(r9) 1697 1698 /* save FP state */ 1699 mr r3, r9 1700 bl kvmppc_save_fp 1701 1702#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1703/* 1704 * Branch around the call if both CPU_FTR_TM and 1705 * CPU_FTR_P9_TM_HV_ASSIST are off. 1706 */ 1707BEGIN_FTR_SECTION 1708 b 91f 1709END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 1710 /* 1711 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1712 */ 1713 mr r3, r9 1714 ld r4, VCPU_MSR(r3) 1715 li r5, 0 /* don't preserve non-vol regs */ 1716 bl kvmppc_save_tm_hv 1717 nop 1718 ld r9, HSTATE_KVM_VCPU(r13) 171991: 1720#endif 1721 1722 /* Increment yield count if they have a VPA */ 1723 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1724 cmpdi r8, 0 1725 beq 25f 1726 li r4, LPPACA_YIELDCOUNT 1727 LWZX_BE r3, r8, r4 1728 addi r3, r3, 1 1729 STWX_BE r3, r8, r4 1730 li r3, 1 1731 stb r3, VCPU_VPA_DIRTY(r9) 173225: 1733 /* Save PMU registers if requested */ 1734 /* r8 and cr0.eq are live here */ 1735 mr r3, r9 1736 li r4, 1 1737 beq 21f /* if no VPA, save PMU stuff anyway */ 1738 lbz r4, LPPACA_PMCINUSE(r8) 173921: bl kvmhv_save_guest_pmu 1740 ld r9, HSTATE_KVM_VCPU(r13) 1741 1742 /* Restore host values of some registers */ 1743BEGIN_FTR_SECTION 1744 ld r5, STACK_SLOT_CIABR(r1) 1745 ld r6, STACK_SLOT_DAWR(r1) 1746 ld r7, STACK_SLOT_DAWRX(r1) 1747 mtspr SPRN_CIABR, r5 1748 /* 1749 * If the DAWR doesn't work, it's ok to write these here as 1750 * this value should always be zero 1751 */ 1752 mtspr SPRN_DAWR, r6 1753 mtspr SPRN_DAWRX, r7 1754END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1755BEGIN_FTR_SECTION 1756 ld r5, STACK_SLOT_TID(r1) 1757 ld r6, STACK_SLOT_PSSCR(r1) 1758 ld r7, STACK_SLOT_PID(r1) 1759 mtspr SPRN_TIDR, r5 1760 mtspr SPRN_PSSCR, r6 1761 mtspr SPRN_PID, r7 1762END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1763 1764#ifdef CONFIG_PPC_RADIX_MMU 1765 /* 1766 * Are we running hash or radix ? 1767 */ 1768 ld r5, VCPU_KVM(r9) 1769 lbz r0, KVM_RADIX(r5) 1770 cmpwi cr2, r0, 0 1771 beq cr2, 2f 1772 1773 /* 1774 * Radix: do eieio; tlbsync; ptesync sequence in case we 1775 * interrupted the guest between a tlbie and a ptesync. 1776 */ 1777 eieio 1778 tlbsync 1779 ptesync 1780 1781 /* Radix: Handle the case where the guest used an illegal PID */ 1782 LOAD_REG_ADDR(r4, mmu_base_pid) 1783 lwz r3, VCPU_GUEST_PID(r9) 1784 lwz r5, 0(r4) 1785 cmpw cr0,r3,r5 1786 blt 2f 1787 1788 /* 1789 * Illegal PID, the HW might have prefetched and cached in the TLB 1790 * some translations for the LPID 0 / guest PID combination which 1791 * Linux doesn't know about, so we need to flush that PID out of 1792 * the TLB. First we need to set LPIDR to 0 so tlbiel applies to 1793 * the right context. 1794 */ 1795 li r0,0 1796 mtspr SPRN_LPID,r0 1797 isync 1798 1799 /* Then do a congruence class local flush */ 1800 ld r6,VCPU_KVM(r9) 1801 lwz r0,KVM_TLB_SETS(r6) 1802 mtctr r0 1803 li r7,0x400 /* IS field = 0b01 */ 1804 ptesync 1805 sldi r0,r3,32 /* RS has PID */ 18061: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ 1807 addi r7,r7,0x1000 1808 bdnz 1b 1809 ptesync 1810 18112: 1812#endif /* CONFIG_PPC_RADIX_MMU */ 1813 1814 /* 1815 * POWER7/POWER8 guest -> host partition switch code. 1816 * We don't have to lock against tlbies but we do 1817 * have to coordinate the hardware threads. 1818 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1819 */ 1820kvmhv_switch_to_host: 1821 /* Secondary threads wait for primary to do partition switch */ 1822 ld r5,HSTATE_KVM_VCORE(r13) 1823 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1824 lbz r3,HSTATE_PTID(r13) 1825 cmpwi r3,0 1826 beq 15f 1827 HMT_LOW 182813: lbz r3,VCORE_IN_GUEST(r5) 1829 cmpwi r3,0 1830 bne 13b 1831 HMT_MEDIUM 1832 b 16f 1833 1834 /* Primary thread waits for all the secondaries to exit guest */ 183515: lwz r3,VCORE_ENTRY_EXIT(r5) 1836 rlwinm r0,r3,32-8,0xff 1837 clrldi r3,r3,56 1838 cmpw r3,r0 1839 bne 15b 1840 isync 1841 1842 /* Did we actually switch to the guest at all? */ 1843 lbz r6, VCORE_IN_GUEST(r5) 1844 cmpwi r6, 0 1845 beq 19f 1846 1847 /* Primary thread switches back to host partition */ 1848 lwz r7,KVM_HOST_LPID(r4) 1849BEGIN_FTR_SECTION 1850 ld r6,KVM_HOST_SDR1(r4) 1851 li r8,LPID_RSVD /* switch to reserved LPID */ 1852 mtspr SPRN_LPID,r8 1853 ptesync 1854 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1855END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 1856 mtspr SPRN_LPID,r7 1857 isync 1858 1859BEGIN_FTR_SECTION 1860 /* DPDES and VTB are shared between threads */ 1861 mfspr r7, SPRN_DPDES 1862 mfspr r8, SPRN_VTB 1863 std r7, VCORE_DPDES(r5) 1864 std r8, VCORE_VTB(r5) 1865 /* clear DPDES so we don't get guest doorbells in the host */ 1866 li r8, 0 1867 mtspr SPRN_DPDES, r8 1868END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1869 1870 /* Subtract timebase offset from timebase */ 1871 ld r8, VCORE_TB_OFFSET_APPL(r5) 1872 cmpdi r8,0 1873 beq 17f 1874 li r0, 0 1875 std r0, VCORE_TB_OFFSET_APPL(r5) 1876 mftb r6 /* current guest timebase */ 1877 subf r8,r8,r6 1878 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1879 mftb r7 /* check if lower 24 bits overflowed */ 1880 clrldi r6,r6,40 1881 clrldi r7,r7,40 1882 cmpld r7,r6 1883 bge 17f 1884 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1885 mtspr SPRN_TBU40,r8 1886 188717: 1888 /* 1889 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1890 * above, which may or may not have already called 1891 * kvmppc_subcore_exit_guest. Fortunately, all that 1892 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1893 * it again here is benign even if kvmppc_realmode_hmi_handler 1894 * has already called it. 1895 */ 1896 bl kvmppc_subcore_exit_guest 1897 nop 189830: ld r5,HSTATE_KVM_VCORE(r13) 1899 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1900 1901 /* Reset PCR */ 1902 ld r0, VCORE_PCR(r5) 1903 cmpdi r0, 0 1904 beq 18f 1905 li r0, 0 1906 mtspr SPRN_PCR, r0 190718: 1908 /* Signal secondary CPUs to continue */ 1909 stb r0,VCORE_IN_GUEST(r5) 191019: lis r8,0x7fff /* MAX_INT@h */ 1911 mtspr SPRN_HDEC,r8 1912 191316: 1914BEGIN_FTR_SECTION 1915 /* On POWER9 with HPT-on-radix we need to wait for all other threads */ 1916 ld r3, HSTATE_SPLIT_MODE(r13) 1917 cmpdi r3, 0 1918 beq 47f 1919 lwz r8, KVM_SPLIT_DO_RESTORE(r3) 1920 cmpwi r8, 0 1921 beq 47f 1922 bl kvmhv_p9_restore_lpcr 1923 nop 1924 b 48f 192547: 1926END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1927 ld r8,KVM_HOST_LPCR(r4) 1928 mtspr SPRN_LPCR,r8 1929 isync 193048: 1931#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1932 /* Finish timing, if we have a vcpu */ 1933 ld r4, HSTATE_KVM_VCPU(r13) 1934 cmpdi r4, 0 1935 li r3, 0 1936 beq 2f 1937 bl kvmhv_accumulate_time 19382: 1939#endif 1940 /* Unset guest mode */ 1941 li r0, KVM_GUEST_MODE_NONE 1942 stb r0, HSTATE_IN_GUEST(r13) 1943 1944 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1945 ld r0, SFS+PPC_LR_STKOFF(r1) 1946 addi r1, r1, SFS 1947 mtlr r0 1948 blr 1949 1950kvmppc_guest_external: 1951 /* External interrupt, first check for host_ipi. If this is 1952 * set, we know the host wants us out so let's do it now 1953 */ 1954 bl kvmppc_read_intr 1955 1956 /* 1957 * Restore the active volatile registers after returning from 1958 * a C function. 1959 */ 1960 ld r9, HSTATE_KVM_VCPU(r13) 1961 li r12, BOOK3S_INTERRUPT_EXTERNAL 1962 1963 /* 1964 * kvmppc_read_intr return codes: 1965 * 1966 * Exit to host (r3 > 0) 1967 * 1 An interrupt is pending that needs to be handled by the host 1968 * Exit guest and return to host by branching to guest_exit_cont 1969 * 1970 * 2 Passthrough that needs completion in the host 1971 * Exit guest and return to host by branching to guest_exit_cont 1972 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1973 * to indicate to the host to complete handling the interrupt 1974 * 1975 * Before returning to guest, we check if any CPU is heading out 1976 * to the host and if so, we head out also. If no CPUs are heading 1977 * check return values <= 0. 1978 * 1979 * Return to guest (r3 <= 0) 1980 * 0 No external interrupt is pending 1981 * -1 A guest wakeup IPI (which has now been cleared) 1982 * In either case, we return to guest to deliver any pending 1983 * guest interrupts. 1984 * 1985 * -2 A PCI passthrough external interrupt was handled 1986 * (interrupt was delivered directly to guest) 1987 * Return to guest to deliver any pending guest interrupts. 1988 */ 1989 1990 cmpdi r3, 1 1991 ble 1f 1992 1993 /* Return code = 2 */ 1994 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1995 stw r12, VCPU_TRAP(r9) 1996 b guest_exit_cont 1997 19981: /* Return code <= 1 */ 1999 cmpdi r3, 0 2000 bgt guest_exit_cont 2001 2002 /* Return code <= 0 */ 2003maybe_reenter_guest: 2004 ld r5, HSTATE_KVM_VCORE(r13) 2005 lwz r0, VCORE_ENTRY_EXIT(r5) 2006 cmpwi r0, 0x100 2007 mr r4, r9 2008 blt deliver_guest_interrupt 2009 b guest_exit_cont 2010 2011#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2012/* 2013 * Softpatch interrupt for transactional memory emulation cases 2014 * on POWER9 DD2.2. This is early in the guest exit path - we 2015 * haven't saved registers or done a treclaim yet. 2016 */ 2017kvmppc_tm_emul: 2018 /* Save instruction image in HEIR */ 2019 mfspr r3, SPRN_HEIR 2020 stw r3, VCPU_HEIR(r9) 2021 2022 /* 2023 * The cases we want to handle here are those where the guest 2024 * is in real suspend mode and is trying to transition to 2025 * transactional mode. 2026 */ 2027 lbz r0, HSTATE_FAKE_SUSPEND(r13) 2028 cmpwi r0, 0 /* keep exiting guest if in fake suspend */ 2029 bne guest_exit_cont 2030 rldicl r3, r11, 64 - MSR_TS_S_LG, 62 2031 cmpwi r3, 1 /* or if not in suspend state */ 2032 bne guest_exit_cont 2033 2034 /* Call C code to do the emulation */ 2035 mr r3, r9 2036 bl kvmhv_p9_tm_emulation_early 2037 nop 2038 ld r9, HSTATE_KVM_VCPU(r13) 2039 li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH 2040 cmpwi r3, 0 2041 beq guest_exit_cont /* continue exiting if not handled */ 2042 ld r10, VCPU_PC(r9) 2043 ld r11, VCPU_MSR(r9) 2044 b fast_interrupt_c_return /* go back to guest if handled */ 2045#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2046 2047/* 2048 * Check whether an HDSI is an HPTE not found fault or something else. 2049 * If it is an HPTE not found fault that is due to the guest accessing 2050 * a page that they have mapped but which we have paged out, then 2051 * we continue on with the guest exit path. In all other cases, 2052 * reflect the HDSI to the guest as a DSI. 2053 */ 2054kvmppc_hdsi: 2055 ld r3, VCPU_KVM(r9) 2056 lbz r0, KVM_RADIX(r3) 2057 mfspr r4, SPRN_HDAR 2058 mfspr r6, SPRN_HDSISR 2059BEGIN_FTR_SECTION 2060 /* Look for DSISR canary. If we find it, retry instruction */ 2061 cmpdi r6, 0x7fff 2062 beq 6f 2063END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2064 cmpwi r0, 0 2065 bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ 2066 /* HPTE not found fault or protection fault? */ 2067 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 2068 beq 1f /* if not, send it to the guest */ 2069 andi. r0, r11, MSR_DR /* data relocation enabled? */ 2070 beq 3f 2071BEGIN_FTR_SECTION 2072 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2073 b 4f 2074END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2075 clrrdi r0, r4, 28 2076 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2077 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 2078 bne 7f /* if no SLB entry found */ 20794: std r4, VCPU_FAULT_DAR(r9) 2080 stw r6, VCPU_FAULT_DSISR(r9) 2081 2082 /* Search the hash table. */ 2083 mr r3, r9 /* vcpu pointer */ 2084 li r7, 1 /* data fault */ 2085 bl kvmppc_hpte_hv_fault 2086 ld r9, HSTATE_KVM_VCPU(r13) 2087 ld r10, VCPU_PC(r9) 2088 ld r11, VCPU_MSR(r9) 2089 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 2090 cmpdi r3, 0 /* retry the instruction */ 2091 beq 6f 2092 cmpdi r3, -1 /* handle in kernel mode */ 2093 beq guest_exit_cont 2094 cmpdi r3, -2 /* MMIO emulation; need instr word */ 2095 beq 2f 2096 2097 /* Synthesize a DSI (or DSegI) for the guest */ 2098 ld r4, VCPU_FAULT_DAR(r9) 2099 mr r6, r3 21001: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 2101 mtspr SPRN_DSISR, r6 21027: mtspr SPRN_DAR, r4 2103 mtspr SPRN_SRR0, r10 2104 mtspr SPRN_SRR1, r11 2105 mr r10, r0 2106 bl kvmppc_msr_interrupt 2107fast_interrupt_c_return: 21086: ld r7, VCPU_CTR(r9) 2109 ld r8, VCPU_XER(r9) 2110 mtctr r7 2111 mtxer r8 2112 mr r4, r9 2113 b fast_guest_return 2114 21153: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 2116 ld r5, KVM_VRMA_SLB_V(r5) 2117 b 4b 2118 2119 /* If this is for emulated MMIO, load the instruction word */ 21202: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 2121 2122 /* Set guest mode to 'jump over instruction' so if lwz faults 2123 * we'll just continue at the next IP. */ 2124 li r0, KVM_GUEST_MODE_SKIP 2125 stb r0, HSTATE_IN_GUEST(r13) 2126 2127 /* Do the access with MSR:DR enabled */ 2128 mfmsr r3 2129 ori r4, r3, MSR_DR /* Enable paging for data */ 2130 mtmsrd r4 2131 lwz r8, 0(r10) 2132 mtmsrd r3 2133 2134 /* Store the result */ 2135 stw r8, VCPU_LAST_INST(r9) 2136 2137 /* Unset guest mode. */ 2138 li r0, KVM_GUEST_MODE_HOST_HV 2139 stb r0, HSTATE_IN_GUEST(r13) 2140 b guest_exit_cont 2141 2142.Lradix_hdsi: 2143 std r4, VCPU_FAULT_DAR(r9) 2144 stw r6, VCPU_FAULT_DSISR(r9) 2145.Lradix_hisi: 2146 mfspr r5, SPRN_ASDR 2147 std r5, VCPU_FAULT_GPA(r9) 2148 b guest_exit_cont 2149 2150/* 2151 * Similarly for an HISI, reflect it to the guest as an ISI unless 2152 * it is an HPTE not found fault for a page that we have paged out. 2153 */ 2154kvmppc_hisi: 2155 ld r3, VCPU_KVM(r9) 2156 lbz r0, KVM_RADIX(r3) 2157 cmpwi r0, 0 2158 bne .Lradix_hisi /* for radix, just save ASDR */ 2159 andis. r0, r11, SRR1_ISI_NOPT@h 2160 beq 1f 2161 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 2162 beq 3f 2163BEGIN_FTR_SECTION 2164 mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ 2165 b 4f 2166END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2167 clrrdi r0, r10, 28 2168 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 2169 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 2170 bne 7f /* if no SLB entry found */ 21714: 2172 /* Search the hash table. */ 2173 mr r3, r9 /* vcpu pointer */ 2174 mr r4, r10 2175 mr r6, r11 2176 li r7, 0 /* instruction fault */ 2177 bl kvmppc_hpte_hv_fault 2178 ld r9, HSTATE_KVM_VCPU(r13) 2179 ld r10, VCPU_PC(r9) 2180 ld r11, VCPU_MSR(r9) 2181 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 2182 cmpdi r3, 0 /* retry the instruction */ 2183 beq fast_interrupt_c_return 2184 cmpdi r3, -1 /* handle in kernel mode */ 2185 beq guest_exit_cont 2186 2187 /* Synthesize an ISI (or ISegI) for the guest */ 2188 mr r11, r3 21891: li r0, BOOK3S_INTERRUPT_INST_STORAGE 21907: mtspr SPRN_SRR0, r10 2191 mtspr SPRN_SRR1, r11 2192 mr r10, r0 2193 bl kvmppc_msr_interrupt 2194 b fast_interrupt_c_return 2195 21963: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 2197 ld r5, KVM_VRMA_SLB_V(r6) 2198 b 4b 2199 2200/* 2201 * Try to handle an hcall in real mode. 2202 * Returns to the guest if we handle it, or continues on up to 2203 * the kernel if we can't (i.e. if we don't have a handler for 2204 * it, or if the handler returns H_TOO_HARD). 2205 * 2206 * r5 - r8 contain hcall args, 2207 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 2208 */ 2209hcall_try_real_mode: 2210 ld r3,VCPU_GPR(R3)(r9) 2211 andi. r0,r11,MSR_PR 2212 /* sc 1 from userspace - reflect to guest syscall */ 2213 bne sc_1_fast_return 2214 /* sc 1 from nested guest - give it to L1 to handle */ 2215 ld r0, VCPU_NESTED(r9) 2216 cmpdi r0, 0 2217 bne guest_exit_cont 2218 clrrdi r3,r3,2 2219 cmpldi r3,hcall_real_table_end - hcall_real_table 2220 bge guest_exit_cont 2221 /* See if this hcall is enabled for in-kernel handling */ 2222 ld r4, VCPU_KVM(r9) 2223 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 2224 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 2225 add r4, r4, r0 2226 ld r0, KVM_ENABLED_HCALLS(r4) 2227 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 2228 srd r0, r0, r4 2229 andi. r0, r0, 1 2230 beq guest_exit_cont 2231 /* Get pointer to handler, if any, and call it */ 2232 LOAD_REG_ADDR(r4, hcall_real_table) 2233 lwax r3,r3,r4 2234 cmpwi r3,0 2235 beq guest_exit_cont 2236 add r12,r3,r4 2237 mtctr r12 2238 mr r3,r9 /* get vcpu pointer */ 2239 ld r4,VCPU_GPR(R4)(r9) 2240 bctrl 2241 cmpdi r3,H_TOO_HARD 2242 beq hcall_real_fallback 2243 ld r4,HSTATE_KVM_VCPU(r13) 2244 std r3,VCPU_GPR(R3)(r4) 2245 ld r10,VCPU_PC(r4) 2246 ld r11,VCPU_MSR(r4) 2247 b fast_guest_return 2248 2249sc_1_fast_return: 2250 mtspr SPRN_SRR0,r10 2251 mtspr SPRN_SRR1,r11 2252 li r10, BOOK3S_INTERRUPT_SYSCALL 2253 bl kvmppc_msr_interrupt 2254 mr r4,r9 2255 b fast_guest_return 2256 2257 /* We've attempted a real mode hcall, but it's punted it back 2258 * to userspace. We need to restore some clobbered volatiles 2259 * before resuming the pass-it-to-qemu path */ 2260hcall_real_fallback: 2261 li r12,BOOK3S_INTERRUPT_SYSCALL 2262 ld r9, HSTATE_KVM_VCPU(r13) 2263 2264 b guest_exit_cont 2265 2266 .globl hcall_real_table 2267hcall_real_table: 2268 .long 0 /* 0 - unused */ 2269 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 2270 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 2271 .long DOTSYM(kvmppc_h_read) - hcall_real_table 2272 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 2273 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 2274 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 2275#ifdef CONFIG_SPAPR_TCE_IOMMU 2276 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table 2277 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table 2278#else 2279 .long 0 /* 0x1c */ 2280 .long 0 /* 0x20 */ 2281#endif 2282 .long 0 /* 0x24 - H_SET_SPRG0 */ 2283 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 2284 .long 0 /* 0x2c */ 2285 .long 0 /* 0x30 */ 2286 .long 0 /* 0x34 */ 2287 .long 0 /* 0x38 */ 2288 .long 0 /* 0x3c */ 2289 .long 0 /* 0x40 */ 2290 .long 0 /* 0x44 */ 2291 .long 0 /* 0x48 */ 2292 .long 0 /* 0x4c */ 2293 .long 0 /* 0x50 */ 2294 .long 0 /* 0x54 */ 2295 .long 0 /* 0x58 */ 2296 .long 0 /* 0x5c */ 2297 .long 0 /* 0x60 */ 2298#ifdef CONFIG_KVM_XICS 2299 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table 2300 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table 2301 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table 2302 .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table 2303 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table 2304#else 2305 .long 0 /* 0x64 - H_EOI */ 2306 .long 0 /* 0x68 - H_CPPR */ 2307 .long 0 /* 0x6c - H_IPI */ 2308 .long 0 /* 0x70 - H_IPOLL */ 2309 .long 0 /* 0x74 - H_XIRR */ 2310#endif 2311 .long 0 /* 0x78 */ 2312 .long 0 /* 0x7c */ 2313 .long 0 /* 0x80 */ 2314 .long 0 /* 0x84 */ 2315 .long 0 /* 0x88 */ 2316 .long 0 /* 0x8c */ 2317 .long 0 /* 0x90 */ 2318 .long 0 /* 0x94 */ 2319 .long 0 /* 0x98 */ 2320 .long 0 /* 0x9c */ 2321 .long 0 /* 0xa0 */ 2322 .long 0 /* 0xa4 */ 2323 .long 0 /* 0xa8 */ 2324 .long 0 /* 0xac */ 2325 .long 0 /* 0xb0 */ 2326 .long 0 /* 0xb4 */ 2327 .long 0 /* 0xb8 */ 2328 .long 0 /* 0xbc */ 2329 .long 0 /* 0xc0 */ 2330 .long 0 /* 0xc4 */ 2331 .long 0 /* 0xc8 */ 2332 .long 0 /* 0xcc */ 2333 .long 0 /* 0xd0 */ 2334 .long 0 /* 0xd4 */ 2335 .long 0 /* 0xd8 */ 2336 .long 0 /* 0xdc */ 2337 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 2338 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 2339 .long 0 /* 0xe8 */ 2340 .long 0 /* 0xec */ 2341 .long 0 /* 0xf0 */ 2342 .long 0 /* 0xf4 */ 2343 .long 0 /* 0xf8 */ 2344 .long 0 /* 0xfc */ 2345 .long 0 /* 0x100 */ 2346 .long 0 /* 0x104 */ 2347 .long 0 /* 0x108 */ 2348 .long 0 /* 0x10c */ 2349 .long 0 /* 0x110 */ 2350 .long 0 /* 0x114 */ 2351 .long 0 /* 0x118 */ 2352 .long 0 /* 0x11c */ 2353 .long 0 /* 0x120 */ 2354 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 2355 .long 0 /* 0x128 */ 2356 .long 0 /* 0x12c */ 2357 .long 0 /* 0x130 */ 2358 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 2359#ifdef CONFIG_SPAPR_TCE_IOMMU 2360 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table 2361 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table 2362#else 2363 .long 0 /* 0x138 */ 2364 .long 0 /* 0x13c */ 2365#endif 2366 .long 0 /* 0x140 */ 2367 .long 0 /* 0x144 */ 2368 .long 0 /* 0x148 */ 2369 .long 0 /* 0x14c */ 2370 .long 0 /* 0x150 */ 2371 .long 0 /* 0x154 */ 2372 .long 0 /* 0x158 */ 2373 .long 0 /* 0x15c */ 2374 .long 0 /* 0x160 */ 2375 .long 0 /* 0x164 */ 2376 .long 0 /* 0x168 */ 2377 .long 0 /* 0x16c */ 2378 .long 0 /* 0x170 */ 2379 .long 0 /* 0x174 */ 2380 .long 0 /* 0x178 */ 2381 .long 0 /* 0x17c */ 2382 .long 0 /* 0x180 */ 2383 .long 0 /* 0x184 */ 2384 .long 0 /* 0x188 */ 2385 .long 0 /* 0x18c */ 2386 .long 0 /* 0x190 */ 2387 .long 0 /* 0x194 */ 2388 .long 0 /* 0x198 */ 2389 .long 0 /* 0x19c */ 2390 .long 0 /* 0x1a0 */ 2391 .long 0 /* 0x1a4 */ 2392 .long 0 /* 0x1a8 */ 2393 .long 0 /* 0x1ac */ 2394 .long 0 /* 0x1b0 */ 2395 .long 0 /* 0x1b4 */ 2396 .long 0 /* 0x1b8 */ 2397 .long 0 /* 0x1bc */ 2398 .long 0 /* 0x1c0 */ 2399 .long 0 /* 0x1c4 */ 2400 .long 0 /* 0x1c8 */ 2401 .long 0 /* 0x1cc */ 2402 .long 0 /* 0x1d0 */ 2403 .long 0 /* 0x1d4 */ 2404 .long 0 /* 0x1d8 */ 2405 .long 0 /* 0x1dc */ 2406 .long 0 /* 0x1e0 */ 2407 .long 0 /* 0x1e4 */ 2408 .long 0 /* 0x1e8 */ 2409 .long 0 /* 0x1ec */ 2410 .long 0 /* 0x1f0 */ 2411 .long 0 /* 0x1f4 */ 2412 .long 0 /* 0x1f8 */ 2413 .long 0 /* 0x1fc */ 2414 .long 0 /* 0x200 */ 2415 .long 0 /* 0x204 */ 2416 .long 0 /* 0x208 */ 2417 .long 0 /* 0x20c */ 2418 .long 0 /* 0x210 */ 2419 .long 0 /* 0x214 */ 2420 .long 0 /* 0x218 */ 2421 .long 0 /* 0x21c */ 2422 .long 0 /* 0x220 */ 2423 .long 0 /* 0x224 */ 2424 .long 0 /* 0x228 */ 2425 .long 0 /* 0x22c */ 2426 .long 0 /* 0x230 */ 2427 .long 0 /* 0x234 */ 2428 .long 0 /* 0x238 */ 2429 .long 0 /* 0x23c */ 2430 .long 0 /* 0x240 */ 2431 .long 0 /* 0x244 */ 2432 .long 0 /* 0x248 */ 2433 .long 0 /* 0x24c */ 2434 .long 0 /* 0x250 */ 2435 .long 0 /* 0x254 */ 2436 .long 0 /* 0x258 */ 2437 .long 0 /* 0x25c */ 2438 .long 0 /* 0x260 */ 2439 .long 0 /* 0x264 */ 2440 .long 0 /* 0x268 */ 2441 .long 0 /* 0x26c */ 2442 .long 0 /* 0x270 */ 2443 .long 0 /* 0x274 */ 2444 .long 0 /* 0x278 */ 2445 .long 0 /* 0x27c */ 2446 .long 0 /* 0x280 */ 2447 .long 0 /* 0x284 */ 2448 .long 0 /* 0x288 */ 2449 .long 0 /* 0x28c */ 2450 .long 0 /* 0x290 */ 2451 .long 0 /* 0x294 */ 2452 .long 0 /* 0x298 */ 2453 .long 0 /* 0x29c */ 2454 .long 0 /* 0x2a0 */ 2455 .long 0 /* 0x2a4 */ 2456 .long 0 /* 0x2a8 */ 2457 .long 0 /* 0x2ac */ 2458 .long 0 /* 0x2b0 */ 2459 .long 0 /* 0x2b4 */ 2460 .long 0 /* 0x2b8 */ 2461 .long 0 /* 0x2bc */ 2462 .long 0 /* 0x2c0 */ 2463 .long 0 /* 0x2c4 */ 2464 .long 0 /* 0x2c8 */ 2465 .long 0 /* 0x2cc */ 2466 .long 0 /* 0x2d0 */ 2467 .long 0 /* 0x2d4 */ 2468 .long 0 /* 0x2d8 */ 2469 .long 0 /* 0x2dc */ 2470 .long 0 /* 0x2e0 */ 2471 .long 0 /* 0x2e4 */ 2472 .long 0 /* 0x2e8 */ 2473 .long 0 /* 0x2ec */ 2474 .long 0 /* 0x2f0 */ 2475 .long 0 /* 0x2f4 */ 2476 .long 0 /* 0x2f8 */ 2477#ifdef CONFIG_KVM_XICS 2478 .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table 2479#else 2480 .long 0 /* 0x2fc - H_XIRR_X*/ 2481#endif 2482 .long DOTSYM(kvmppc_h_random) - hcall_real_table 2483 .globl hcall_real_table_end 2484hcall_real_table_end: 2485 2486_GLOBAL(kvmppc_h_set_xdabr) 2487EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2488 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2489 beq 6f 2490 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2491 andc. r0, r5, r0 2492 beq 3f 24936: li r3, H_PARAMETER 2494 blr 2495 2496_GLOBAL(kvmppc_h_set_dabr) 2497EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2498 li r5, DABRX_USER | DABRX_KERNEL 24993: 2500BEGIN_FTR_SECTION 2501 b 2f 2502END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2503 std r4,VCPU_DABR(r3) 2504 stw r5, VCPU_DABRX(r3) 2505 mtspr SPRN_DABRX, r5 2506 /* Work around P7 bug where DABR can get corrupted on mtspr */ 25071: mtspr SPRN_DABR,r4 2508 mfspr r5, SPRN_DABR 2509 cmpd r4, r5 2510 bne 1b 2511 isync 2512 li r3,0 2513 blr 2514 25152: 2516BEGIN_FTR_SECTION 2517 /* POWER9 with disabled DAWR */ 2518 li r3, H_HARDWARE 2519 blr 2520END_FTR_SECTION_IFCLR(CPU_FTR_DAWR) 2521 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2522 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2523 rlwimi r5, r4, 2, DAWRX_WT 2524 clrrdi r4, r4, 3 2525 std r4, VCPU_DAWR(r3) 2526 std r5, VCPU_DAWRX(r3) 2527 mtspr SPRN_DAWR, r4 2528 mtspr SPRN_DAWRX, r5 2529 li r3, 0 2530 blr 2531 2532_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2533 ori r11,r11,MSR_EE 2534 std r11,VCPU_MSR(r3) 2535 li r0,1 2536 stb r0,VCPU_CEDED(r3) 2537 sync /* order setting ceded vs. testing prodded */ 2538 lbz r5,VCPU_PRODDED(r3) 2539 cmpwi r5,0 2540 bne kvm_cede_prodded 2541 li r12,0 /* set trap to 0 to say hcall is handled */ 2542 stw r12,VCPU_TRAP(r3) 2543 li r0,H_SUCCESS 2544 std r0,VCPU_GPR(R3)(r3) 2545 2546 /* 2547 * Set our bit in the bitmask of napping threads unless all the 2548 * other threads are already napping, in which case we send this 2549 * up to the host. 2550 */ 2551 ld r5,HSTATE_KVM_VCORE(r13) 2552 lbz r6,HSTATE_PTID(r13) 2553 lwz r8,VCORE_ENTRY_EXIT(r5) 2554 clrldi r8,r8,56 2555 li r0,1 2556 sld r0,r0,r6 2557 addi r6,r5,VCORE_NAPPING_THREADS 255831: lwarx r4,0,r6 2559 or r4,r4,r0 2560 cmpw r4,r8 2561 beq kvm_cede_exit 2562 stwcx. r4,0,r6 2563 bne 31b 2564 /* order napping_threads update vs testing entry_exit_map */ 2565 isync 2566 li r0,NAPPING_CEDE 2567 stb r0,HSTATE_NAPPING(r13) 2568 lwz r7,VCORE_ENTRY_EXIT(r5) 2569 cmpwi r7,0x100 2570 bge 33f /* another thread already exiting */ 2571 2572/* 2573 * Although not specifically required by the architecture, POWER7 2574 * preserves the following registers in nap mode, even if an SMT mode 2575 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2576 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2577 */ 2578 /* Save non-volatile GPRs */ 2579 std r14, VCPU_GPR(R14)(r3) 2580 std r15, VCPU_GPR(R15)(r3) 2581 std r16, VCPU_GPR(R16)(r3) 2582 std r17, VCPU_GPR(R17)(r3) 2583 std r18, VCPU_GPR(R18)(r3) 2584 std r19, VCPU_GPR(R19)(r3) 2585 std r20, VCPU_GPR(R20)(r3) 2586 std r21, VCPU_GPR(R21)(r3) 2587 std r22, VCPU_GPR(R22)(r3) 2588 std r23, VCPU_GPR(R23)(r3) 2589 std r24, VCPU_GPR(R24)(r3) 2590 std r25, VCPU_GPR(R25)(r3) 2591 std r26, VCPU_GPR(R26)(r3) 2592 std r27, VCPU_GPR(R27)(r3) 2593 std r28, VCPU_GPR(R28)(r3) 2594 std r29, VCPU_GPR(R29)(r3) 2595 std r30, VCPU_GPR(R30)(r3) 2596 std r31, VCPU_GPR(R31)(r3) 2597 2598 /* save FP state */ 2599 bl kvmppc_save_fp 2600 2601#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2602/* 2603 * Branch around the call if both CPU_FTR_TM and 2604 * CPU_FTR_P9_TM_HV_ASSIST are off. 2605 */ 2606BEGIN_FTR_SECTION 2607 b 91f 2608END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2609 /* 2610 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2611 */ 2612 ld r3, HSTATE_KVM_VCPU(r13) 2613 ld r4, VCPU_MSR(r3) 2614 li r5, 0 /* don't preserve non-vol regs */ 2615 bl kvmppc_save_tm_hv 2616 nop 261791: 2618#endif 2619 2620 /* 2621 * Set DEC to the smaller of DEC and HDEC, so that we wake 2622 * no later than the end of our timeslice (HDEC interrupts 2623 * don't wake us from nap). 2624 */ 2625 mfspr r3, SPRN_DEC 2626 mfspr r4, SPRN_HDEC 2627 mftb r5 2628BEGIN_FTR_SECTION 2629 /* On P9 check whether the guest has large decrementer mode enabled */ 2630 ld r6, HSTATE_KVM_VCORE(r13) 2631 ld r6, VCORE_LPCR(r6) 2632 andis. r6, r6, LPCR_LD@h 2633 bne 68f 2634END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2635 extsw r3, r3 263668: EXTEND_HDEC(r4) 2637 cmpd r3, r4 2638 ble 67f 2639 mtspr SPRN_DEC, r4 264067: 2641 /* save expiry time of guest decrementer */ 2642 add r3, r3, r5 2643 ld r4, HSTATE_KVM_VCPU(r13) 2644 ld r5, HSTATE_KVM_VCORE(r13) 2645 ld r6, VCORE_TB_OFFSET_APPL(r5) 2646 subf r3, r6, r3 /* convert to host TB value */ 2647 std r3, VCPU_DEC_EXPIRES(r4) 2648 2649#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2650 ld r4, HSTATE_KVM_VCPU(r13) 2651 addi r3, r4, VCPU_TB_CEDE 2652 bl kvmhv_accumulate_time 2653#endif 2654 2655 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2656 2657 /* 2658 * Take a nap until a decrementer or external or doobell interrupt 2659 * occurs, with PECE1 and PECE0 set in LPCR. 2660 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2661 * Also clear the runlatch bit before napping. 2662 */ 2663kvm_do_nap: 2664 mfspr r0, SPRN_CTRLF 2665 clrrdi r0, r0, 1 2666 mtspr SPRN_CTRLT, r0 2667 2668 li r0,1 2669 stb r0,HSTATE_HWTHREAD_REQ(r13) 2670 mfspr r5,SPRN_LPCR 2671 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2672BEGIN_FTR_SECTION 2673 ori r5, r5, LPCR_PECEDH 2674 rlwimi r5, r3, 0, LPCR_PECEDP 2675END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2676 2677kvm_nap_sequence: /* desired LPCR value in r5 */ 2678BEGIN_FTR_SECTION 2679 /* 2680 * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) 2681 * enable state loss = 1 (allow SMT mode switch) 2682 * requested level = 0 (just stop dispatching) 2683 */ 2684 lis r3, (PSSCR_EC | PSSCR_ESL)@h 2685 mtspr SPRN_PSSCR, r3 2686 /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ 2687 li r4, LPCR_PECE_HVEE@higher 2688 sldi r4, r4, 32 2689 or r5, r5, r4 2690END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2691 mtspr SPRN_LPCR,r5 2692 isync 2693 li r0, 0 2694 std r0, HSTATE_SCRATCH0(r13) 2695 ptesync 2696 ld r0, HSTATE_SCRATCH0(r13) 26971: cmpd r0, r0 2698 bne 1b 2699BEGIN_FTR_SECTION 2700 nap 2701FTR_SECTION_ELSE 2702 PPC_STOP 2703ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 2704 b . 2705 270633: mr r4, r3 2707 li r3, 0 2708 li r12, 0 2709 b 34f 2710 2711kvm_end_cede: 2712 /* get vcpu pointer */ 2713 ld r4, HSTATE_KVM_VCPU(r13) 2714 2715 /* Woken by external or decrementer interrupt */ 2716 ld r1, HSTATE_HOST_R1(r13) 2717 2718#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 2719 addi r3, r4, VCPU_TB_RMINTR 2720 bl kvmhv_accumulate_time 2721#endif 2722 2723#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2724/* 2725 * Branch around the call if both CPU_FTR_TM and 2726 * CPU_FTR_P9_TM_HV_ASSIST are off. 2727 */ 2728BEGIN_FTR_SECTION 2729 b 91f 2730END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) 2731 /* 2732 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2733 */ 2734 mr r3, r4 2735 ld r4, VCPU_MSR(r3) 2736 li r5, 0 /* don't preserve non-vol regs */ 2737 bl kvmppc_restore_tm_hv 2738 nop 2739 ld r4, HSTATE_KVM_VCPU(r13) 274091: 2741#endif 2742 2743 /* load up FP state */ 2744 bl kvmppc_load_fp 2745 2746 /* Restore guest decrementer */ 2747 ld r3, VCPU_DEC_EXPIRES(r4) 2748 ld r5, HSTATE_KVM_VCORE(r13) 2749 ld r6, VCORE_TB_OFFSET_APPL(r5) 2750 add r3, r3, r6 /* convert host TB to guest TB value */ 2751 mftb r7 2752 subf r3, r7, r3 2753 mtspr SPRN_DEC, r3 2754 2755 /* Load NV GPRS */ 2756 ld r14, VCPU_GPR(R14)(r4) 2757 ld r15, VCPU_GPR(R15)(r4) 2758 ld r16, VCPU_GPR(R16)(r4) 2759 ld r17, VCPU_GPR(R17)(r4) 2760 ld r18, VCPU_GPR(R18)(r4) 2761 ld r19, VCPU_GPR(R19)(r4) 2762 ld r20, VCPU_GPR(R20)(r4) 2763 ld r21, VCPU_GPR(R21)(r4) 2764 ld r22, VCPU_GPR(R22)(r4) 2765 ld r23, VCPU_GPR(R23)(r4) 2766 ld r24, VCPU_GPR(R24)(r4) 2767 ld r25, VCPU_GPR(R25)(r4) 2768 ld r26, VCPU_GPR(R26)(r4) 2769 ld r27, VCPU_GPR(R27)(r4) 2770 ld r28, VCPU_GPR(R28)(r4) 2771 ld r29, VCPU_GPR(R29)(r4) 2772 ld r30, VCPU_GPR(R30)(r4) 2773 ld r31, VCPU_GPR(R31)(r4) 2774 2775 /* Check the wake reason in SRR1 to see why we got here */ 2776 bl kvmppc_check_wake_reason 2777 2778 /* 2779 * Restore volatile registers since we could have called a 2780 * C routine in kvmppc_check_wake_reason 2781 * r4 = VCPU 2782 * r3 tells us whether we need to return to host or not 2783 * WARNING: it gets checked further down: 2784 * should not modify r3 until this check is done. 2785 */ 2786 ld r4, HSTATE_KVM_VCPU(r13) 2787 2788 /* clear our bit in vcore->napping_threads */ 278934: ld r5,HSTATE_KVM_VCORE(r13) 2790 lbz r7,HSTATE_PTID(r13) 2791 li r0,1 2792 sld r0,r0,r7 2793 addi r6,r5,VCORE_NAPPING_THREADS 279432: lwarx r7,0,r6 2795 andc r7,r7,r0 2796 stwcx. r7,0,r6 2797 bne 32b 2798 li r0,0 2799 stb r0,HSTATE_NAPPING(r13) 2800 2801 /* See if the wake reason saved in r3 means we need to exit */ 2802 stw r12, VCPU_TRAP(r4) 2803 mr r9, r4 2804 cmpdi r3, 0 2805 bgt guest_exit_cont 2806 b maybe_reenter_guest 2807 2808 /* cede when already previously prodded case */ 2809kvm_cede_prodded: 2810 li r0,0 2811 stb r0,VCPU_PRODDED(r3) 2812 sync /* order testing prodded vs. clearing ceded */ 2813 stb r0,VCPU_CEDED(r3) 2814 li r3,H_SUCCESS 2815 blr 2816 2817 /* we've ceded but we want to give control to the host */ 2818kvm_cede_exit: 2819 ld r9, HSTATE_KVM_VCPU(r13) 2820#ifdef CONFIG_KVM_XICS 2821 /* Abort if we still have a pending escalation */ 2822 lbz r5, VCPU_XIVE_ESC_ON(r9) 2823 cmpwi r5, 0 2824 beq 1f 2825 li r0, 0 2826 stb r0, VCPU_CEDED(r9) 28271: /* Enable XIVE escalation */ 2828 li r5, XIVE_ESB_SET_PQ_00 2829 mfmsr r0 2830 andi. r0, r0, MSR_DR /* in real mode? */ 2831 beq 1f 2832 ld r10, VCPU_XIVE_ESC_VADDR(r9) 2833 cmpdi r10, 0 2834 beq 3f 2835 ldx r0, r10, r5 2836 b 2f 28371: ld r10, VCPU_XIVE_ESC_RADDR(r9) 2838 cmpdi r10, 0 2839 beq 3f 2840 ldcix r0, r10, r5 28412: sync 2842 li r0, 1 2843 stb r0, VCPU_XIVE_ESC_ON(r9) 2844#endif /* CONFIG_KVM_XICS */ 28453: b guest_exit_cont 2846 2847 /* Try to do machine check recovery in real mode */ 2848machine_check_realmode: 2849 mr r3, r9 /* get vcpu pointer */ 2850 bl kvmppc_realmode_machine_check 2851 nop 2852 /* all machine checks go to virtual mode for further handling */ 2853 ld r9, HSTATE_KVM_VCPU(r13) 2854 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2855 b guest_exit_cont 2856 2857/* 2858 * Call C code to handle a HMI in real mode. 2859 * Only the primary thread does the call, secondary threads are handled 2860 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2861 * r9 points to the vcpu on entry 2862 */ 2863hmi_realmode: 2864 lbz r0, HSTATE_PTID(r13) 2865 cmpwi r0, 0 2866 bne guest_exit_cont 2867 bl kvmppc_realmode_hmi_handler 2868 ld r9, HSTATE_KVM_VCPU(r13) 2869 li r12, BOOK3S_INTERRUPT_HMI 2870 b guest_exit_cont 2871 2872/* 2873 * Check the reason we woke from nap, and take appropriate action. 2874 * Returns (in r3): 2875 * 0 if nothing needs to be done 2876 * 1 if something happened that needs to be handled by the host 2877 * -1 if there was a guest wakeup (IPI or msgsnd) 2878 * -2 if we handled a PCI passthrough interrupt (returned by 2879 * kvmppc_read_intr only) 2880 * 2881 * Also sets r12 to the interrupt vector for any interrupt that needs 2882 * to be handled now by the host (0x500 for external interrupt), or zero. 2883 * Modifies all volatile registers (since it may call a C function). 2884 * This routine calls kvmppc_read_intr, a C function, if an external 2885 * interrupt is pending. 2886 */ 2887kvmppc_check_wake_reason: 2888 mfspr r6, SPRN_SRR1 2889BEGIN_FTR_SECTION 2890 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2891FTR_SECTION_ELSE 2892 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2893ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2894 cmpwi r6, 8 /* was it an external interrupt? */ 2895 beq 7f /* if so, see what it was */ 2896 li r3, 0 2897 li r12, 0 2898 cmpwi r6, 6 /* was it the decrementer? */ 2899 beq 0f 2900BEGIN_FTR_SECTION 2901 cmpwi r6, 5 /* privileged doorbell? */ 2902 beq 0f 2903 cmpwi r6, 3 /* hypervisor doorbell? */ 2904 beq 3f 2905END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2906 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2907 beq 4f 2908 li r3, 1 /* anything else, return 1 */ 29090: blr 2910 2911 /* hypervisor doorbell */ 29123: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2913 2914 /* 2915 * Clear the doorbell as we will invoke the handler 2916 * explicitly in the guest exit path. 2917 */ 2918 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2919 PPC_MSGCLR(6) 2920 /* see if it's a host IPI */ 2921 li r3, 1 2922BEGIN_FTR_SECTION 2923 PPC_MSGSYNC 2924 lwsync 2925END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 2926 lbz r0, HSTATE_HOST_IPI(r13) 2927 cmpwi r0, 0 2928 bnelr 2929 /* if not, return -1 */ 2930 li r3, -1 2931 blr 2932 2933 /* Woken up due to Hypervisor maintenance interrupt */ 29344: li r12, BOOK3S_INTERRUPT_HMI 2935 li r3, 1 2936 blr 2937 2938 /* external interrupt - create a stack frame so we can call C */ 29397: mflr r0 2940 std r0, PPC_LR_STKOFF(r1) 2941 stdu r1, -PPC_MIN_STKFRM(r1) 2942 bl kvmppc_read_intr 2943 nop 2944 li r12, BOOK3S_INTERRUPT_EXTERNAL 2945 cmpdi r3, 1 2946 ble 1f 2947 2948 /* 2949 * Return code of 2 means PCI passthrough interrupt, but 2950 * we need to return back to host to complete handling the 2951 * interrupt. Trap reason is expected in r12 by guest 2952 * exit code. 2953 */ 2954 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 29551: 2956 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2957 addi r1, r1, PPC_MIN_STKFRM 2958 mtlr r0 2959 blr 2960 2961/* 2962 * Save away FP, VMX and VSX registers. 2963 * r3 = vcpu pointer 2964 * N.B. r30 and r31 are volatile across this function, 2965 * thus it is not callable from C. 2966 */ 2967kvmppc_save_fp: 2968 mflr r30 2969 mr r31,r3 2970 mfmsr r5 2971 ori r8,r5,MSR_FP 2972#ifdef CONFIG_ALTIVEC 2973BEGIN_FTR_SECTION 2974 oris r8,r8,MSR_VEC@h 2975END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2976#endif 2977#ifdef CONFIG_VSX 2978BEGIN_FTR_SECTION 2979 oris r8,r8,MSR_VSX@h 2980END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2981#endif 2982 mtmsrd r8 2983 addi r3,r3,VCPU_FPRS 2984 bl store_fp_state 2985#ifdef CONFIG_ALTIVEC 2986BEGIN_FTR_SECTION 2987 addi r3,r31,VCPU_VRS 2988 bl store_vr_state 2989END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2990#endif 2991 mfspr r6,SPRN_VRSAVE 2992 stw r6,VCPU_VRSAVE(r31) 2993 mtlr r30 2994 blr 2995 2996/* 2997 * Load up FP, VMX and VSX registers 2998 * r4 = vcpu pointer 2999 * N.B. r30 and r31 are volatile across this function, 3000 * thus it is not callable from C. 3001 */ 3002kvmppc_load_fp: 3003 mflr r30 3004 mr r31,r4 3005 mfmsr r9 3006 ori r8,r9,MSR_FP 3007#ifdef CONFIG_ALTIVEC 3008BEGIN_FTR_SECTION 3009 oris r8,r8,MSR_VEC@h 3010END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3011#endif 3012#ifdef CONFIG_VSX 3013BEGIN_FTR_SECTION 3014 oris r8,r8,MSR_VSX@h 3015END_FTR_SECTION_IFSET(CPU_FTR_VSX) 3016#endif 3017 mtmsrd r8 3018 addi r3,r4,VCPU_FPRS 3019 bl load_fp_state 3020#ifdef CONFIG_ALTIVEC 3021BEGIN_FTR_SECTION 3022 addi r3,r31,VCPU_VRS 3023 bl load_vr_state 3024END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 3025#endif 3026 lwz r7,VCPU_VRSAVE(r31) 3027 mtspr SPRN_VRSAVE,r7 3028 mtlr r30 3029 mr r4,r31 3030 blr 3031 3032#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3033/* 3034 * Save transactional state and TM-related registers. 3035 * Called with r3 pointing to the vcpu struct and r4 containing 3036 * the guest MSR value. 3037 * r5 is non-zero iff non-volatile register state needs to be maintained. 3038 * If r5 == 0, this can modify all checkpointed registers, but 3039 * restores r1 and r2 before exit. 3040 */ 3041_GLOBAL_TOC(kvmppc_save_tm_hv) 3042EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 3043 /* See if we need to handle fake suspend mode */ 3044BEGIN_FTR_SECTION 3045 b __kvmppc_save_tm 3046END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3047 3048 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 3049 cmpwi r0, 0 3050 beq __kvmppc_save_tm 3051 3052 /* The following code handles the fake_suspend = 1 case */ 3053 mflr r0 3054 std r0, PPC_LR_STKOFF(r1) 3055 stdu r1, -PPC_MIN_STKFRM(r1) 3056 3057 /* Turn on TM. */ 3058 mfmsr r8 3059 li r0, 1 3060 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 3061 mtmsrd r8 3062 3063 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 3064 beq 4f 3065BEGIN_FTR_SECTION 3066 bl pnv_power9_force_smt4_catch 3067END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3068 nop 3069 3070 /* We have to treclaim here because that's the only way to do S->N */ 3071 li r3, TM_CAUSE_KVM_RESCHED 3072 TRECLAIM(R3) 3073 3074 /* 3075 * We were in fake suspend, so we are not going to save the 3076 * register state as the guest checkpointed state (since 3077 * we already have it), therefore we can now use any volatile GPR. 3078 * In fact treclaim in fake suspend state doesn't modify 3079 * any registers. 3080 */ 3081 3082BEGIN_FTR_SECTION 3083 bl pnv_power9_force_smt4_release 3084END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 3085 nop 3086 30874: 3088 mfspr r3, SPRN_PSSCR 3089 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 3090 li r0, PSSCR_FAKE_SUSPEND 3091 andc r3, r3, r0 3092 mtspr SPRN_PSSCR, r3 3093 3094 /* Don't save TEXASR, use value from last exit in real suspend state */ 3095 ld r9, HSTATE_KVM_VCPU(r13) 3096 mfspr r5, SPRN_TFHAR 3097 mfspr r6, SPRN_TFIAR 3098 std r5, VCPU_TFHAR(r9) 3099 std r6, VCPU_TFIAR(r9) 3100 3101 addi r1, r1, PPC_MIN_STKFRM 3102 ld r0, PPC_LR_STKOFF(r1) 3103 mtlr r0 3104 blr 3105 3106/* 3107 * Restore transactional state and TM-related registers. 3108 * Called with r3 pointing to the vcpu struct 3109 * and r4 containing the guest MSR value. 3110 * r5 is non-zero iff non-volatile register state needs to be maintained. 3111 * This potentially modifies all checkpointed registers. 3112 * It restores r1 and r2 from the PACA. 3113 */ 3114_GLOBAL_TOC(kvmppc_restore_tm_hv) 3115EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 3116 /* 3117 * If we are doing TM emulation for the guest on a POWER9 DD2, 3118 * then we don't actually do a trechkpt -- we either set up 3119 * fake-suspend mode, or emulate a TM rollback. 3120 */ 3121BEGIN_FTR_SECTION 3122 b __kvmppc_restore_tm 3123END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 3124 mflr r0 3125 std r0, PPC_LR_STKOFF(r1) 3126 3127 li r0, 0 3128 stb r0, HSTATE_FAKE_SUSPEND(r13) 3129 3130 /* Turn on TM so we can restore TM SPRs */ 3131 mfmsr r5 3132 li r0, 1 3133 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 3134 mtmsrd r5 3135 3136 /* 3137 * The user may change these outside of a transaction, so they must 3138 * always be context switched. 3139 */ 3140 ld r5, VCPU_TFHAR(r3) 3141 ld r6, VCPU_TFIAR(r3) 3142 ld r7, VCPU_TEXASR(r3) 3143 mtspr SPRN_TFHAR, r5 3144 mtspr SPRN_TFIAR, r6 3145 mtspr SPRN_TEXASR, r7 3146 3147 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 3148 beqlr /* TM not active in guest */ 3149 3150 /* Make sure the failure summary is set */ 3151 oris r7, r7, (TEXASR_FS)@h 3152 mtspr SPRN_TEXASR, r7 3153 3154 cmpwi r5, 1 /* check for suspended state */ 3155 bgt 10f 3156 stb r5, HSTATE_FAKE_SUSPEND(r13) 3157 b 9f /* and return */ 315810: stdu r1, -PPC_MIN_STKFRM(r1) 3159 /* guest is in transactional state, so simulate rollback */ 3160 bl kvmhv_emulate_tm_rollback 3161 nop 3162 addi r1, r1, PPC_MIN_STKFRM 31639: ld r0, PPC_LR_STKOFF(r1) 3164 mtlr r0 3165 blr 3166#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 3167 3168/* 3169 * We come here if we get any exception or interrupt while we are 3170 * executing host real mode code while in guest MMU context. 3171 * r12 is (CR << 32) | vector 3172 * r13 points to our PACA 3173 * r12 is saved in HSTATE_SCRATCH0(r13) 3174 * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE 3175 * r9 is saved in HSTATE_SCRATCH2(r13) 3176 * r13 is saved in HSPRG1 3177 * cfar is saved in HSTATE_CFAR(r13) 3178 * ppr is saved in HSTATE_PPR(r13) 3179 */ 3180kvmppc_bad_host_intr: 3181 /* 3182 * Switch to the emergency stack, but start half-way down in 3183 * case we were already on it. 3184 */ 3185 mr r9, r1 3186 std r1, PACAR1(r13) 3187 ld r1, PACAEMERGSP(r13) 3188 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 3189 std r9, 0(r1) 3190 std r0, GPR0(r1) 3191 std r9, GPR1(r1) 3192 std r2, GPR2(r1) 3193 SAVE_4GPRS(3, r1) 3194 SAVE_2GPRS(7, r1) 3195 srdi r0, r12, 32 3196 clrldi r12, r12, 32 3197 std r0, _CCR(r1) 3198 std r12, _TRAP(r1) 3199 andi. r0, r12, 2 3200 beq 1f 3201 mfspr r3, SPRN_HSRR0 3202 mfspr r4, SPRN_HSRR1 3203 mfspr r5, SPRN_HDAR 3204 mfspr r6, SPRN_HDSISR 3205 b 2f 32061: mfspr r3, SPRN_SRR0 3207 mfspr r4, SPRN_SRR1 3208 mfspr r5, SPRN_DAR 3209 mfspr r6, SPRN_DSISR 32102: std r3, _NIP(r1) 3211 std r4, _MSR(r1) 3212 std r5, _DAR(r1) 3213 std r6, _DSISR(r1) 3214 ld r9, HSTATE_SCRATCH2(r13) 3215 ld r12, HSTATE_SCRATCH0(r13) 3216 GET_SCRATCH0(r0) 3217 SAVE_4GPRS(9, r1) 3218 std r0, GPR13(r1) 3219 SAVE_NVGPRS(r1) 3220 ld r5, HSTATE_CFAR(r13) 3221 std r5, ORIG_GPR3(r1) 3222 mflr r3 3223#ifdef CONFIG_RELOCATABLE 3224 ld r4, HSTATE_SCRATCH1(r13) 3225#else 3226 mfctr r4 3227#endif 3228 mfxer r5 3229 lbz r6, PACAIRQSOFTMASK(r13) 3230 std r3, _LINK(r1) 3231 std r4, _CTR(r1) 3232 std r5, _XER(r1) 3233 std r6, SOFTE(r1) 3234 ld r2, PACATOC(r13) 3235 LOAD_REG_IMMEDIATE(3, 0x7265677368657265) 3236 std r3, STACK_FRAME_OVERHEAD-16(r1) 3237 3238 /* 3239 * On POWER9 do a minimal restore of the MMU and call C code, 3240 * which will print a message and panic. 3241 * XXX On POWER7 and POWER8, we just spin here since we don't 3242 * know what the other threads are doing (and we don't want to 3243 * coordinate with them) - but at least we now have register state 3244 * in memory that we might be able to look at from another CPU. 3245 */ 3246BEGIN_FTR_SECTION 3247 b . 3248END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 3249 ld r9, HSTATE_KVM_VCPU(r13) 3250 ld r10, VCPU_KVM(r9) 3251 3252 li r0, 0 3253 mtspr SPRN_AMR, r0 3254 mtspr SPRN_IAMR, r0 3255 mtspr SPRN_CIABR, r0 3256 mtspr SPRN_DAWRX, r0 3257 3258BEGIN_MMU_FTR_SECTION 3259 b 4f 3260END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) 3261 3262 slbmte r0, r0 3263 slbia 3264 ptesync 3265 ld r8, PACA_SLBSHADOWPTR(r13) 3266 .rept SLB_NUM_BOLTED 3267 li r3, SLBSHADOW_SAVEAREA 3268 LDX_BE r5, r8, r3 3269 addi r3, r3, 8 3270 LDX_BE r6, r8, r3 3271 andis. r7, r5, SLB_ESID_V@h 3272 beq 3f 3273 slbmte r6, r5 32743: addi r8, r8, 16 3275 .endr 3276 32774: lwz r7, KVM_HOST_LPID(r10) 3278 mtspr SPRN_LPID, r7 3279 mtspr SPRN_PID, r0 3280 ld r8, KVM_HOST_LPCR(r10) 3281 mtspr SPRN_LPCR, r8 3282 isync 3283 li r0, KVM_GUEST_MODE_NONE 3284 stb r0, HSTATE_IN_GUEST(r13) 3285 3286 /* 3287 * Turn on the MMU and jump to C code 3288 */ 3289 bcl 20, 31, .+4 32905: mflr r3 3291 addi r3, r3, 9f - 5b 3292 li r4, -1 3293 rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ 3294 ld r4, PACAKMSR(r13) 3295 mtspr SPRN_SRR0, r3 3296 mtspr SPRN_SRR1, r4 3297 RFI_TO_KERNEL 32989: addi r3, r1, STACK_FRAME_OVERHEAD 3299 bl kvmppc_bad_interrupt 3300 b 9b 3301 3302/* 3303 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 3304 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 3305 * r11 has the guest MSR value (in/out) 3306 * r9 has a vcpu pointer (in) 3307 * r0 is used as a scratch register 3308 */ 3309kvmppc_msr_interrupt: 3310 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 3311 cmpwi r0, 2 /* Check if we are in transactional state.. */ 3312 ld r11, VCPU_INTR_MSR(r9) 3313 bne 1f 3314 /* ... if transactional, change to suspended */ 3315 li r0, 1 33161: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 3317 blr 3318 3319/* 3320 * Load up guest PMU state. R3 points to the vcpu struct. 3321 */ 3322_GLOBAL(kvmhv_load_guest_pmu) 3323EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) 3324 mr r4, r3 3325 mflr r0 3326 li r3, 1 3327 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3328 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3329 isync 3330BEGIN_FTR_SECTION 3331 ld r3, VCPU_MMCR(r4) 3332 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3333 cmpwi r5, MMCR0_PMAO 3334 beql kvmppc_fix_pmao 3335END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3336 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 3337 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 3338 lwz r6, VCPU_PMC + 8(r4) 3339 lwz r7, VCPU_PMC + 12(r4) 3340 lwz r8, VCPU_PMC + 16(r4) 3341 lwz r9, VCPU_PMC + 20(r4) 3342 mtspr SPRN_PMC1, r3 3343 mtspr SPRN_PMC2, r5 3344 mtspr SPRN_PMC3, r6 3345 mtspr SPRN_PMC4, r7 3346 mtspr SPRN_PMC5, r8 3347 mtspr SPRN_PMC6, r9 3348 ld r3, VCPU_MMCR(r4) 3349 ld r5, VCPU_MMCR + 8(r4) 3350 ld r6, VCPU_MMCR + 16(r4) 3351 ld r7, VCPU_SIAR(r4) 3352 ld r8, VCPU_SDAR(r4) 3353 mtspr SPRN_MMCR1, r5 3354 mtspr SPRN_MMCRA, r6 3355 mtspr SPRN_SIAR, r7 3356 mtspr SPRN_SDAR, r8 3357BEGIN_FTR_SECTION 3358 ld r5, VCPU_MMCR + 24(r4) 3359 ld r6, VCPU_SIER(r4) 3360 mtspr SPRN_MMCR2, r5 3361 mtspr SPRN_SIER, r6 3362BEGIN_FTR_SECTION_NESTED(96) 3363 lwz r7, VCPU_PMC + 24(r4) 3364 lwz r8, VCPU_PMC + 28(r4) 3365 ld r9, VCPU_MMCR + 32(r4) 3366 mtspr SPRN_SPMC1, r7 3367 mtspr SPRN_SPMC2, r8 3368 mtspr SPRN_MMCRS, r9 3369END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3370END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3371 mtspr SPRN_MMCR0, r3 3372 isync 3373 mtlr r0 3374 blr 3375 3376/* 3377 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 3378 */ 3379_GLOBAL(kvmhv_load_host_pmu) 3380EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) 3381 mflr r0 3382 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 3383 cmpwi r4, 0 3384 beq 23f /* skip if not */ 3385BEGIN_FTR_SECTION 3386 ld r3, HSTATE_MMCR0(r13) 3387 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 3388 cmpwi r4, MMCR0_PMAO 3389 beql kvmppc_fix_pmao 3390END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 3391 lwz r3, HSTATE_PMC1(r13) 3392 lwz r4, HSTATE_PMC2(r13) 3393 lwz r5, HSTATE_PMC3(r13) 3394 lwz r6, HSTATE_PMC4(r13) 3395 lwz r8, HSTATE_PMC5(r13) 3396 lwz r9, HSTATE_PMC6(r13) 3397 mtspr SPRN_PMC1, r3 3398 mtspr SPRN_PMC2, r4 3399 mtspr SPRN_PMC3, r5 3400 mtspr SPRN_PMC4, r6 3401 mtspr SPRN_PMC5, r8 3402 mtspr SPRN_PMC6, r9 3403 ld r3, HSTATE_MMCR0(r13) 3404 ld r4, HSTATE_MMCR1(r13) 3405 ld r5, HSTATE_MMCRA(r13) 3406 ld r6, HSTATE_SIAR(r13) 3407 ld r7, HSTATE_SDAR(r13) 3408 mtspr SPRN_MMCR1, r4 3409 mtspr SPRN_MMCRA, r5 3410 mtspr SPRN_SIAR, r6 3411 mtspr SPRN_SDAR, r7 3412BEGIN_FTR_SECTION 3413 ld r8, HSTATE_MMCR2(r13) 3414 ld r9, HSTATE_SIER(r13) 3415 mtspr SPRN_MMCR2, r8 3416 mtspr SPRN_SIER, r9 3417END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3418 mtspr SPRN_MMCR0, r3 3419 isync 3420 mtlr r0 342123: blr 3422 3423/* 3424 * Save guest PMU state into the vcpu struct. 3425 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 3426 */ 3427_GLOBAL(kvmhv_save_guest_pmu) 3428EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) 3429 mr r9, r3 3430 mr r8, r4 3431BEGIN_FTR_SECTION 3432 /* 3433 * POWER8 seems to have a hardware bug where setting 3434 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 3435 * when some counters are already negative doesn't seem 3436 * to cause a performance monitor alert (and hence interrupt). 3437 * The effect of this is that when saving the PMU state, 3438 * if there is no PMU alert pending when we read MMCR0 3439 * before freezing the counters, but one becomes pending 3440 * before we read the counters, we lose it. 3441 * To work around this, we need a way to freeze the counters 3442 * before reading MMCR0. Normally, freezing the counters 3443 * is done by writing MMCR0 (to set MMCR0[FC]) which 3444 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 3445 * we can also freeze the counters using MMCR2, by writing 3446 * 1s to all the counter freeze condition bits (there are 3447 * 9 bits each for 6 counters). 3448 */ 3449 li r3, -1 /* set all freeze bits */ 3450 clrrdi r3, r3, 10 3451 mfspr r10, SPRN_MMCR2 3452 mtspr SPRN_MMCR2, r3 3453 isync 3454END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3455 li r3, 1 3456 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 3457 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 3458 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 3459 mfspr r6, SPRN_MMCRA 3460 /* Clear MMCRA in order to disable SDAR updates */ 3461 li r7, 0 3462 mtspr SPRN_MMCRA, r7 3463 isync 3464 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 3465 bne 21f 3466 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 3467 b 22f 346821: mfspr r5, SPRN_MMCR1 3469 mfspr r7, SPRN_SIAR 3470 mfspr r8, SPRN_SDAR 3471 std r4, VCPU_MMCR(r9) 3472 std r5, VCPU_MMCR + 8(r9) 3473 std r6, VCPU_MMCR + 16(r9) 3474BEGIN_FTR_SECTION 3475 std r10, VCPU_MMCR + 24(r9) 3476END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 3477 std r7, VCPU_SIAR(r9) 3478 std r8, VCPU_SDAR(r9) 3479 mfspr r3, SPRN_PMC1 3480 mfspr r4, SPRN_PMC2 3481 mfspr r5, SPRN_PMC3 3482 mfspr r6, SPRN_PMC4 3483 mfspr r7, SPRN_PMC5 3484 mfspr r8, SPRN_PMC6 3485 stw r3, VCPU_PMC(r9) 3486 stw r4, VCPU_PMC + 4(r9) 3487 stw r5, VCPU_PMC + 8(r9) 3488 stw r6, VCPU_PMC + 12(r9) 3489 stw r7, VCPU_PMC + 16(r9) 3490 stw r8, VCPU_PMC + 20(r9) 3491BEGIN_FTR_SECTION 3492 mfspr r5, SPRN_SIER 3493 std r5, VCPU_SIER(r9) 3494BEGIN_FTR_SECTION_NESTED(96) 3495 mfspr r6, SPRN_SPMC1 3496 mfspr r7, SPRN_SPMC2 3497 mfspr r8, SPRN_MMCRS 3498 stw r6, VCPU_PMC + 24(r9) 3499 stw r7, VCPU_PMC + 28(r9) 3500 std r8, VCPU_MMCR + 32(r9) 3501 lis r4, 0x8000 3502 mtspr SPRN_MMCRS, r4 3503END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) 3504END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 350522: blr 3506 3507/* 3508 * This works around a hardware bug on POWER8E processors, where 3509 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 3510 * performance monitor interrupt. Instead, when we need to have 3511 * an interrupt pending, we have to arrange for a counter to overflow. 3512 */ 3513kvmppc_fix_pmao: 3514 li r3, 0 3515 mtspr SPRN_MMCR2, r3 3516 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 3517 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 3518 mtspr SPRN_MMCR0, r3 3519 lis r3, 0x7fff 3520 ori r3, r3, 0xffff 3521 mtspr SPRN_PMC6, r3 3522 isync 3523 blr 3524 3525#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 3526/* 3527 * Start timing an activity 3528 * r3 = pointer to time accumulation struct, r4 = vcpu 3529 */ 3530kvmhv_start_timing: 3531 ld r5, HSTATE_KVM_VCORE(r13) 3532 ld r6, VCORE_TB_OFFSET_APPL(r5) 3533 mftb r5 3534 subf r5, r6, r5 /* subtract current timebase offset */ 3535 std r3, VCPU_CUR_ACTIVITY(r4) 3536 std r5, VCPU_ACTIVITY_START(r4) 3537 blr 3538 3539/* 3540 * Accumulate time to one activity and start another. 3541 * r3 = pointer to new time accumulation struct, r4 = vcpu 3542 */ 3543kvmhv_accumulate_time: 3544 ld r5, HSTATE_KVM_VCORE(r13) 3545 ld r8, VCORE_TB_OFFSET_APPL(r5) 3546 ld r5, VCPU_CUR_ACTIVITY(r4) 3547 ld r6, VCPU_ACTIVITY_START(r4) 3548 std r3, VCPU_CUR_ACTIVITY(r4) 3549 mftb r7 3550 subf r7, r8, r7 /* subtract current timebase offset */ 3551 std r7, VCPU_ACTIVITY_START(r4) 3552 cmpdi r5, 0 3553 beqlr 3554 subf r3, r6, r7 3555 ld r8, TAS_SEQCOUNT(r5) 3556 cmpdi r8, 0 3557 addi r8, r8, 1 3558 std r8, TAS_SEQCOUNT(r5) 3559 lwsync 3560 ld r7, TAS_TOTAL(r5) 3561 add r7, r7, r3 3562 std r7, TAS_TOTAL(r5) 3563 ld r6, TAS_MIN(r5) 3564 ld r7, TAS_MAX(r5) 3565 beq 3f 3566 cmpd r3, r6 3567 bge 1f 35683: std r3, TAS_MIN(r5) 35691: cmpd r3, r7 3570 ble 2f 3571 std r3, TAS_MAX(r5) 35722: lwsync 3573 addi r8, r8, 1 3574 std r8, TAS_SEQCOUNT(r5) 3575 blr 3576#endif 3577