1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/kvm_host.h> 4 #include <asm/asm-prototypes.h> 5 #include <asm/dbell.h> 6 #include <asm/ppc-opcode.h> 7 8 #include "book3s_hv.h" 9 10 static void load_spr_state(struct kvm_vcpu *vcpu, 11 struct p9_host_os_sprs *host_os_sprs) 12 { 13 /* TAR is very fast */ 14 mtspr(SPRN_TAR, vcpu->arch.tar); 15 16 #ifdef CONFIG_ALTIVEC 17 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 18 current->thread.vrsave != vcpu->arch.vrsave) 19 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 20 #endif 21 22 if (vcpu->arch.hfscr & HFSCR_EBB) { 23 if (current->thread.ebbhr != vcpu->arch.ebbhr) 24 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); 25 if (current->thread.ebbrr != vcpu->arch.ebbrr) 26 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); 27 if (current->thread.bescr != vcpu->arch.bescr) 28 mtspr(SPRN_BESCR, vcpu->arch.bescr); 29 } 30 31 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 32 current->thread.tidr != vcpu->arch.tid) 33 mtspr(SPRN_TIDR, vcpu->arch.tid); 34 if (host_os_sprs->iamr != vcpu->arch.iamr) 35 mtspr(SPRN_IAMR, vcpu->arch.iamr); 36 if (host_os_sprs->amr != vcpu->arch.amr) 37 mtspr(SPRN_AMR, vcpu->arch.amr); 38 if (vcpu->arch.uamor != 0) 39 mtspr(SPRN_UAMOR, vcpu->arch.uamor); 40 if (current->thread.fscr != vcpu->arch.fscr) 41 mtspr(SPRN_FSCR, vcpu->arch.fscr); 42 if (current->thread.dscr != vcpu->arch.dscr) 43 mtspr(SPRN_DSCR, vcpu->arch.dscr); 44 if (vcpu->arch.pspb != 0) 45 mtspr(SPRN_PSPB, vcpu->arch.pspb); 46 47 /* 48 * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] 49 * clear (or hstate set appropriately to catch those registers 50 * being clobbered if we take a MCE or SRESET), so those are done 51 * later. 52 */ 53 54 if (!(vcpu->arch.ctrl & 1)) 55 mtspr(SPRN_CTRLT, 0); 56 } 57 58 static void store_spr_state(struct kvm_vcpu *vcpu) 59 { 60 vcpu->arch.tar = mfspr(SPRN_TAR); 61 62 #ifdef CONFIG_ALTIVEC 63 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 64 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 65 #endif 66 67 if (vcpu->arch.hfscr & HFSCR_EBB) { 68 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); 69 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); 70 vcpu->arch.bescr = mfspr(SPRN_BESCR); 71 } 72 73 if (cpu_has_feature(CPU_FTR_P9_TIDR)) 74 vcpu->arch.tid = mfspr(SPRN_TIDR); 75 vcpu->arch.iamr = mfspr(SPRN_IAMR); 76 vcpu->arch.amr = mfspr(SPRN_AMR); 77 vcpu->arch.uamor = mfspr(SPRN_UAMOR); 78 vcpu->arch.fscr = mfspr(SPRN_FSCR); 79 vcpu->arch.dscr = mfspr(SPRN_DSCR); 80 vcpu->arch.pspb = mfspr(SPRN_PSPB); 81 82 vcpu->arch.ctrl = mfspr(SPRN_CTRLF); 83 } 84 85 /* Returns true if current MSR and/or guest MSR may have changed */ 86 bool load_vcpu_state(struct kvm_vcpu *vcpu, 87 struct p9_host_os_sprs *host_os_sprs) 88 { 89 bool ret = false; 90 91 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 92 if (cpu_has_feature(CPU_FTR_TM) || 93 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 94 unsigned long guest_msr = vcpu->arch.shregs.msr; 95 if (MSR_TM_ACTIVE(guest_msr)) { 96 kvmppc_restore_tm_hv(vcpu, guest_msr, true); 97 ret = true; 98 } else if (vcpu->arch.hfscr & HFSCR_TM) { 99 mtspr(SPRN_TEXASR, vcpu->arch.texasr); 100 mtspr(SPRN_TFHAR, vcpu->arch.tfhar); 101 mtspr(SPRN_TFIAR, vcpu->arch.tfiar); 102 } 103 } 104 #endif 105 106 load_spr_state(vcpu, host_os_sprs); 107 108 load_fp_state(&vcpu->arch.fp); 109 #ifdef CONFIG_ALTIVEC 110 load_vr_state(&vcpu->arch.vr); 111 #endif 112 113 return ret; 114 } 115 EXPORT_SYMBOL_GPL(load_vcpu_state); 116 117 void store_vcpu_state(struct kvm_vcpu *vcpu) 118 { 119 store_spr_state(vcpu); 120 121 store_fp_state(&vcpu->arch.fp); 122 #ifdef CONFIG_ALTIVEC 123 store_vr_state(&vcpu->arch.vr); 124 #endif 125 126 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 127 if (cpu_has_feature(CPU_FTR_TM) || 128 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 129 unsigned long guest_msr = vcpu->arch.shregs.msr; 130 if (MSR_TM_ACTIVE(guest_msr)) { 131 kvmppc_save_tm_hv(vcpu, guest_msr, true); 132 } else if (vcpu->arch.hfscr & HFSCR_TM) { 133 vcpu->arch.texasr = mfspr(SPRN_TEXASR); 134 vcpu->arch.tfhar = mfspr(SPRN_TFHAR); 135 vcpu->arch.tfiar = mfspr(SPRN_TFIAR); 136 137 if (!vcpu->arch.nested) { 138 vcpu->arch.load_tm++; /* see load_ebb comment */ 139 if (!vcpu->arch.load_tm) 140 vcpu->arch.hfscr &= ~HFSCR_TM; 141 } 142 } 143 } 144 #endif 145 } 146 EXPORT_SYMBOL_GPL(store_vcpu_state); 147 148 void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) 149 { 150 host_os_sprs->iamr = mfspr(SPRN_IAMR); 151 host_os_sprs->amr = mfspr(SPRN_AMR); 152 } 153 EXPORT_SYMBOL_GPL(save_p9_host_os_sprs); 154 155 /* vcpu guest regs must already be saved */ 156 void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, 157 struct p9_host_os_sprs *host_os_sprs) 158 { 159 /* 160 * current->thread.xxx registers must all be restored to host 161 * values before a potential context switch, otherwise the context 162 * switch itself will overwrite current->thread.xxx with the values 163 * from the guest SPRs. 164 */ 165 166 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); 167 168 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 169 current->thread.tidr != vcpu->arch.tid) 170 mtspr(SPRN_TIDR, current->thread.tidr); 171 if (host_os_sprs->iamr != vcpu->arch.iamr) 172 mtspr(SPRN_IAMR, host_os_sprs->iamr); 173 if (vcpu->arch.uamor != 0) 174 mtspr(SPRN_UAMOR, 0); 175 if (host_os_sprs->amr != vcpu->arch.amr) 176 mtspr(SPRN_AMR, host_os_sprs->amr); 177 if (current->thread.fscr != vcpu->arch.fscr) 178 mtspr(SPRN_FSCR, current->thread.fscr); 179 if (current->thread.dscr != vcpu->arch.dscr) 180 mtspr(SPRN_DSCR, current->thread.dscr); 181 if (vcpu->arch.pspb != 0) 182 mtspr(SPRN_PSPB, 0); 183 184 /* Save guest CTRL register, set runlatch to 1 */ 185 if (!(vcpu->arch.ctrl & 1)) 186 mtspr(SPRN_CTRLT, 1); 187 188 #ifdef CONFIG_ALTIVEC 189 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 190 vcpu->arch.vrsave != current->thread.vrsave) 191 mtspr(SPRN_VRSAVE, current->thread.vrsave); 192 #endif 193 if (vcpu->arch.hfscr & HFSCR_EBB) { 194 if (vcpu->arch.bescr != current->thread.bescr) 195 mtspr(SPRN_BESCR, current->thread.bescr); 196 if (vcpu->arch.ebbhr != current->thread.ebbhr) 197 mtspr(SPRN_EBBHR, current->thread.ebbhr); 198 if (vcpu->arch.ebbrr != current->thread.ebbrr) 199 mtspr(SPRN_EBBRR, current->thread.ebbrr); 200 201 if (!vcpu->arch.nested) { 202 /* 203 * This is like load_fp in context switching, turn off 204 * the facility after it wraps the u8 to try avoiding 205 * saving and restoring the registers each partition 206 * switch. 207 */ 208 vcpu->arch.load_ebb++; 209 if (!vcpu->arch.load_ebb) 210 vcpu->arch.hfscr &= ~HFSCR_EBB; 211 } 212 } 213 214 if (vcpu->arch.tar != current->thread.tar) 215 mtspr(SPRN_TAR, current->thread.tar); 216 } 217 EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs); 218 219 #ifdef CONFIG_KVM_BOOK3S_HV_P9_TIMING 220 void accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) 221 { 222 struct kvmppc_vcore *vc = vcpu->arch.vcore; 223 struct kvmhv_tb_accumulator *curr; 224 u64 tb = mftb() - vc->tb_offset_applied; 225 u64 prev_tb; 226 u64 delta; 227 u64 seq; 228 229 curr = vcpu->arch.cur_activity; 230 vcpu->arch.cur_activity = next; 231 prev_tb = vcpu->arch.cur_tb_start; 232 vcpu->arch.cur_tb_start = tb; 233 234 if (!curr) 235 return; 236 237 delta = tb - prev_tb; 238 239 seq = curr->seqcount; 240 curr->seqcount = seq + 1; 241 smp_wmb(); 242 curr->tb_total += delta; 243 if (seq == 0 || delta < curr->tb_min) 244 curr->tb_min = delta; 245 if (delta > curr->tb_max) 246 curr->tb_max = delta; 247 smp_wmb(); 248 curr->seqcount = seq + 2; 249 } 250 EXPORT_SYMBOL_GPL(accumulate_time); 251 #endif 252 253 static inline u64 mfslbv(unsigned int idx) 254 { 255 u64 slbev; 256 257 asm volatile("slbmfev %0,%1" : "=r" (slbev) : "r" (idx)); 258 259 return slbev; 260 } 261 262 static inline u64 mfslbe(unsigned int idx) 263 { 264 u64 slbee; 265 266 asm volatile("slbmfee %0,%1" : "=r" (slbee) : "r" (idx)); 267 268 return slbee; 269 } 270 271 static inline void mtslb(u64 slbee, u64 slbev) 272 { 273 asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee)); 274 } 275 276 static inline void clear_slb_entry(unsigned int idx) 277 { 278 mtslb(idx, 0); 279 } 280 281 static inline void slb_clear_invalidate_partition(void) 282 { 283 clear_slb_entry(0); 284 asm volatile(PPC_SLBIA(6)); 285 } 286 287 /* 288 * Malicious or buggy radix guests may have inserted SLB entries 289 * (only 0..3 because radix always runs with UPRT=1), so these must 290 * be cleared here to avoid side-channels. slbmte is used rather 291 * than slbia, as it won't clear cached translations. 292 */ 293 static void radix_clear_slb(void) 294 { 295 int i; 296 297 for (i = 0; i < 4; i++) 298 clear_slb_entry(i); 299 } 300 301 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 302 { 303 struct kvm_nested_guest *nested = vcpu->arch.nested; 304 u32 lpid; 305 u32 pid; 306 307 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; 308 pid = kvmppc_get_pid(vcpu); 309 310 /* 311 * Prior memory accesses to host PID Q3 must be completed before we 312 * start switching, and stores must be drained to avoid not-my-LPAR 313 * logic (see switch_mmu_to_host). 314 */ 315 asm volatile("hwsync" ::: "memory"); 316 isync(); 317 mtspr(SPRN_LPID, lpid); 318 mtspr(SPRN_LPCR, lpcr); 319 mtspr(SPRN_PID, pid); 320 /* 321 * isync not required here because we are HRFID'ing to guest before 322 * any guest context access, which is context synchronising. 323 */ 324 } 325 326 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 327 { 328 u32 lpid; 329 u32 pid; 330 int i; 331 332 lpid = kvm->arch.lpid; 333 pid = kvmppc_get_pid(vcpu); 334 335 /* 336 * See switch_mmu_to_guest_radix. ptesync should not be required here 337 * even if the host is in HPT mode because speculative accesses would 338 * not cause RC updates (we are in real mode). 339 */ 340 asm volatile("hwsync" ::: "memory"); 341 isync(); 342 mtspr(SPRN_LPID, lpid); 343 mtspr(SPRN_LPCR, lpcr); 344 mtspr(SPRN_PID, pid); 345 346 for (i = 0; i < vcpu->arch.slb_max; i++) 347 mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv); 348 /* 349 * isync not required here, see switch_mmu_to_guest_radix. 350 */ 351 } 352 353 static void switch_mmu_to_host(struct kvm *kvm, u32 pid) 354 { 355 u32 lpid = kvm->arch.host_lpid; 356 u64 lpcr = kvm->arch.host_lpcr; 357 358 /* 359 * The guest has exited, so guest MMU context is no longer being 360 * non-speculatively accessed, but a hwsync is needed before the 361 * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained, 362 * so the not-my-LPAR tlbie logic does not overlook them. 363 */ 364 asm volatile("hwsync" ::: "memory"); 365 isync(); 366 mtspr(SPRN_PID, pid); 367 mtspr(SPRN_LPID, lpid); 368 mtspr(SPRN_LPCR, lpcr); 369 /* 370 * isync is not required after the switch, because mtmsrd with L=0 371 * is performed after this switch, which is context synchronising. 372 */ 373 374 if (!radix_enabled()) 375 slb_restore_bolted_realmode(); 376 } 377 378 static void save_clear_host_mmu(struct kvm *kvm) 379 { 380 if (!radix_enabled()) { 381 /* 382 * Hash host could save and restore host SLB entries to 383 * reduce SLB fault overheads of VM exits, but for now the 384 * existing code clears all entries and restores just the 385 * bolted ones when switching back to host. 386 */ 387 slb_clear_invalidate_partition(); 388 } 389 } 390 391 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) 392 { 393 if (kvm_is_radix(kvm)) { 394 radix_clear_slb(); 395 } else { 396 int i; 397 int nr = 0; 398 399 /* 400 * This must run before switching to host (radix host can't 401 * access all SLBs). 402 */ 403 for (i = 0; i < vcpu->arch.slb_nr; i++) { 404 u64 slbee, slbev; 405 406 slbee = mfslbe(i); 407 if (slbee & SLB_ESID_V) { 408 slbev = mfslbv(i); 409 vcpu->arch.slb[nr].orige = slbee | i; 410 vcpu->arch.slb[nr].origv = slbev; 411 nr++; 412 } 413 } 414 vcpu->arch.slb_max = nr; 415 slb_clear_invalidate_partition(); 416 } 417 } 418 419 static void flush_guest_tlb(struct kvm *kvm) 420 { 421 unsigned long rb, set; 422 423 rb = PPC_BIT(52); /* IS = 2 */ 424 if (kvm_is_radix(kvm)) { 425 /* R=1 PRS=1 RIC=2 */ 426 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 427 : : "r" (rb), "i" (1), "i" (1), "i" (2), 428 "r" (0) : "memory"); 429 for (set = 1; set < kvm->arch.tlb_sets; ++set) { 430 rb += PPC_BIT(51); /* increment set number */ 431 /* R=1 PRS=1 RIC=0 */ 432 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 433 : : "r" (rb), "i" (1), "i" (1), "i" (0), 434 "r" (0) : "memory"); 435 } 436 asm volatile("ptesync": : :"memory"); 437 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 438 asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); 439 } else { 440 for (set = 0; set < kvm->arch.tlb_sets; ++set) { 441 /* R=0 PRS=0 RIC=0 */ 442 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 443 : : "r" (rb), "i" (0), "i" (0), "i" (0), 444 "r" (0) : "memory"); 445 rb += PPC_BIT(51); /* increment set number */ 446 } 447 asm volatile("ptesync": : :"memory"); 448 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 449 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); 450 } 451 } 452 453 static void check_need_tlb_flush(struct kvm *kvm, int pcpu, 454 struct kvm_nested_guest *nested) 455 { 456 cpumask_t *need_tlb_flush; 457 bool all_set = true; 458 int i; 459 460 if (nested) 461 need_tlb_flush = &nested->need_tlb_flush; 462 else 463 need_tlb_flush = &kvm->arch.need_tlb_flush; 464 465 if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush))) 466 return; 467 468 /* 469 * Individual threads can come in here, but the TLB is shared between 470 * the 4 threads in a core, hence invalidating on one thread 471 * invalidates for all, so only invalidate the first time (if all bits 472 * were set. The others must still execute a ptesync. 473 * 474 * If a race occurs and two threads do the TLB flush, that is not a 475 * problem, just sub-optimal. 476 */ 477 for (i = cpu_first_tlb_thread_sibling(pcpu); 478 i <= cpu_last_tlb_thread_sibling(pcpu); 479 i += cpu_tlb_thread_sibling_step()) { 480 if (!cpumask_test_cpu(i, need_tlb_flush)) { 481 all_set = false; 482 break; 483 } 484 } 485 if (all_set) 486 flush_guest_tlb(kvm); 487 else 488 asm volatile("ptesync" ::: "memory"); 489 490 /* Clear the bit after the TLB flush */ 491 cpumask_clear_cpu(pcpu, need_tlb_flush); 492 } 493 494 unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr) 495 { 496 unsigned long msr_needed = 0; 497 498 msr &= ~MSR_EE; 499 500 /* MSR bits may have been cleared by context switch so must recheck */ 501 if (IS_ENABLED(CONFIG_PPC_FPU)) 502 msr_needed |= MSR_FP; 503 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 504 msr_needed |= MSR_VEC; 505 if (cpu_has_feature(CPU_FTR_VSX)) 506 msr_needed |= MSR_VSX; 507 if ((cpu_has_feature(CPU_FTR_TM) || 508 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && 509 (vcpu->arch.hfscr & HFSCR_TM)) 510 msr_needed |= MSR_TM; 511 512 /* 513 * This could be combined with MSR[RI] clearing, but that expands 514 * the unrecoverable window. It would be better to cover unrecoverable 515 * with KVM bad interrupt handling rather than use MSR[RI] at all. 516 * 517 * Much more difficult and less worthwhile to combine with IR/DR 518 * disable. 519 */ 520 if ((msr & msr_needed) != msr_needed) { 521 msr |= msr_needed; 522 __mtmsrd(msr, 0); 523 } else { 524 __hard_irq_disable(); 525 } 526 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 527 528 return msr; 529 } 530 EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities); 531 532 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) 533 { 534 struct p9_host_os_sprs host_os_sprs; 535 struct kvm *kvm = vcpu->kvm; 536 struct kvm_nested_guest *nested = vcpu->arch.nested; 537 struct kvmppc_vcore *vc = vcpu->arch.vcore; 538 s64 hdec, dec; 539 u64 purr, spurr; 540 u64 *exsave; 541 int trap; 542 unsigned long msr; 543 unsigned long host_hfscr; 544 unsigned long host_ciabr; 545 unsigned long host_dawr0; 546 unsigned long host_dawrx0; 547 unsigned long host_psscr; 548 unsigned long host_hpsscr; 549 unsigned long host_pidr; 550 unsigned long host_dawr1; 551 unsigned long host_dawrx1; 552 unsigned long dpdes; 553 554 hdec = time_limit - *tb; 555 if (hdec < 0) 556 return BOOK3S_INTERRUPT_HV_DECREMENTER; 557 558 WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV); 559 WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME)); 560 561 vcpu->arch.ceded = 0; 562 563 /* Save MSR for restore, with EE clear. */ 564 msr = mfmsr() & ~MSR_EE; 565 566 host_hfscr = mfspr(SPRN_HFSCR); 567 host_ciabr = mfspr(SPRN_CIABR); 568 host_psscr = mfspr(SPRN_PSSCR_PR); 569 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) 570 host_hpsscr = mfspr(SPRN_PSSCR); 571 host_pidr = mfspr(SPRN_PID); 572 573 if (dawr_enabled()) { 574 host_dawr0 = mfspr(SPRN_DAWR0); 575 host_dawrx0 = mfspr(SPRN_DAWRX0); 576 if (cpu_has_feature(CPU_FTR_DAWR1)) { 577 host_dawr1 = mfspr(SPRN_DAWR1); 578 host_dawrx1 = mfspr(SPRN_DAWRX1); 579 } 580 } 581 582 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); 583 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); 584 585 save_p9_host_os_sprs(&host_os_sprs); 586 587 msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr); 588 if (lazy_irq_pending()) { 589 trap = 0; 590 goto out; 591 } 592 593 if (unlikely(load_vcpu_state(vcpu, &host_os_sprs))) 594 msr = mfmsr(); /* MSR may have been updated */ 595 596 if (vc->tb_offset) { 597 u64 new_tb = *tb + vc->tb_offset; 598 mtspr(SPRN_TBU40, new_tb); 599 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 600 new_tb += 0x1000000; 601 mtspr(SPRN_TBU40, new_tb); 602 } 603 *tb = new_tb; 604 vc->tb_offset_applied = vc->tb_offset; 605 } 606 607 mtspr(SPRN_VTB, vc->vtb); 608 mtspr(SPRN_PURR, vcpu->arch.purr); 609 mtspr(SPRN_SPURR, vcpu->arch.spurr); 610 611 if (vc->pcr) 612 mtspr(SPRN_PCR, vc->pcr | PCR_MASK); 613 if (vcpu->arch.doorbell_request) { 614 vcpu->arch.doorbell_request = 0; 615 mtspr(SPRN_DPDES, 1); 616 } 617 618 if (dawr_enabled()) { 619 if (vcpu->arch.dawr0 != host_dawr0) 620 mtspr(SPRN_DAWR0, vcpu->arch.dawr0); 621 if (vcpu->arch.dawrx0 != host_dawrx0) 622 mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); 623 if (cpu_has_feature(CPU_FTR_DAWR1)) { 624 if (vcpu->arch.dawr1 != host_dawr1) 625 mtspr(SPRN_DAWR1, vcpu->arch.dawr1); 626 if (vcpu->arch.dawrx1 != host_dawrx1) 627 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); 628 } 629 } 630 if (vcpu->arch.ciabr != host_ciabr) 631 mtspr(SPRN_CIABR, vcpu->arch.ciabr); 632 633 634 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 635 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | 636 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 637 } else { 638 if (vcpu->arch.psscr != host_psscr) 639 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); 640 } 641 642 mtspr(SPRN_HFSCR, vcpu->arch.hfscr); 643 644 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 645 mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME); 646 647 /* 648 * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage 649 * Interrupt (HDSI) the HDSISR is not be updated at all. 650 * 651 * To work around this we put a canary value into the HDSISR before 652 * returning to a guest and then check for this canary when we take a 653 * HDSI. If we find the canary on a HDSI, we know the hardware didn't 654 * update the HDSISR. In this case we return to the guest to retake the 655 * HDSI which should correctly update the HDSISR the second time HDSI 656 * entry. 657 * 658 * The "radix prefetch bug" test can be used to test for this bug, as 659 * it also exists fo DD2.1 and below. 660 */ 661 if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 662 mtspr(SPRN_HDSISR, HDSISR_CANARY); 663 664 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); 665 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); 666 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); 667 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); 668 669 /* 670 * It might be preferable to load_vcpu_state here, in order to get the 671 * GPR/FP register loads executing in parallel with the previous mtSPR 672 * instructions, but for now that can't be done because the TM handling 673 * in load_vcpu_state can change some SPRs and vcpu state (nip, msr). 674 * But TM could be split out if this would be a significant benefit. 675 */ 676 677 /* 678 * MSR[RI] does not need to be cleared (and is not, for radix guests 679 * with no prefetch bug), because in_guest is set. If we take a SRESET 680 * or MCE with in_guest set but still in HV mode, then 681 * kvmppc_p9_bad_interrupt handles the interrupt, which effectively 682 * clears MSR[RI] and doesn't return. 683 */ 684 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9); 685 barrier(); /* Open in_guest critical section */ 686 687 /* 688 * Hash host, hash guest, or radix guest with prefetch bug, all have 689 * to disable the MMU before switching to guest MMU state. 690 */ 691 if (!radix_enabled() || !kvm_is_radix(kvm) || 692 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 693 __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0); 694 695 save_clear_host_mmu(kvm); 696 697 if (kvm_is_radix(kvm)) 698 switch_mmu_to_guest_radix(kvm, vcpu, lpcr); 699 else 700 switch_mmu_to_guest_hpt(kvm, vcpu, lpcr); 701 702 /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */ 703 check_need_tlb_flush(kvm, vc->pcpu, nested); 704 705 /* 706 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0, 707 * so set guest LPCR (with HDICE) before writing HDEC. 708 */ 709 mtspr(SPRN_HDEC, hdec); 710 711 mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb); 712 713 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 714 tm_return_to_guest: 715 #endif 716 mtspr(SPRN_DAR, vcpu->arch.shregs.dar); 717 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); 718 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0); 719 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1); 720 721 switch_pmu_to_guest(vcpu, &host_os_sprs); 722 accumulate_time(vcpu, &vcpu->arch.in_guest); 723 724 kvmppc_p9_enter_guest(vcpu); 725 726 accumulate_time(vcpu, &vcpu->arch.guest_exit); 727 switch_pmu_to_host(vcpu, &host_os_sprs); 728 729 /* XXX: Could get these from r11/12 and paca exsave instead */ 730 vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0); 731 vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1); 732 vcpu->arch.shregs.dar = mfspr(SPRN_DAR); 733 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); 734 735 /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */ 736 trap = local_paca->kvm_hstate.scratch0 & ~0x2; 737 738 if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) 739 exsave = local_paca->exgen; 740 else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) 741 exsave = local_paca->exnmi; 742 else /* trap == 0x200 */ 743 exsave = local_paca->exmc; 744 745 vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1; 746 vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2; 747 748 /* 749 * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate 750 * scratch (which we need to move into exsave to make re-entrant vs 751 * SRESET/MCE), register state is protected from reentrancy. However 752 * timebase, MMU, among other state is still set to guest, so don't 753 * enable MSR[RI] here. It gets enabled at the end, after in_guest 754 * is cleared. 755 * 756 * It is possible an NMI could come in here, which is why it is 757 * important to save the above state early so it can be debugged. 758 */ 759 760 vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)]; 761 vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)]; 762 vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)]; 763 vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)]; 764 vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)]; 765 vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)]; 766 vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)]; 767 vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)]; 768 769 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; 770 771 if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) { 772 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 773 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 774 kvmppc_realmode_machine_check(vcpu); 775 776 } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) { 777 kvmppc_p9_realmode_hmi_handler(vcpu); 778 779 } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) { 780 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 781 782 } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) { 783 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 784 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 785 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 786 787 } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) { 788 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 789 790 } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) { 791 vcpu->arch.hfscr = mfspr(SPRN_HFSCR); 792 793 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 794 /* 795 * Softpatch interrupt for transactional memory emulation cases 796 * on POWER9 DD2.2. This is early in the guest exit path - we 797 * haven't saved registers or done a treclaim yet. 798 */ 799 } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) { 800 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 801 802 /* 803 * The cases we want to handle here are those where the guest 804 * is in real suspend mode and is trying to transition to 805 * transactional mode. 806 */ 807 if (!local_paca->kvm_hstate.fake_suspend && 808 (vcpu->arch.shregs.msr & MSR_TS_S)) { 809 if (kvmhv_p9_tm_emulation_early(vcpu)) { 810 /* 811 * Go straight back into the guest with the 812 * new NIP/MSR as set by TM emulation. 813 */ 814 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 815 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr); 816 goto tm_return_to_guest; 817 } 818 } 819 #endif 820 } 821 822 /* Advance host PURR/SPURR by the amount used by guest */ 823 purr = mfspr(SPRN_PURR); 824 spurr = mfspr(SPRN_SPURR); 825 local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr; 826 local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr; 827 vcpu->arch.purr = purr; 828 vcpu->arch.spurr = spurr; 829 830 vcpu->arch.ic = mfspr(SPRN_IC); 831 vcpu->arch.pid = mfspr(SPRN_PID); 832 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); 833 834 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); 835 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); 836 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); 837 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); 838 839 dpdes = mfspr(SPRN_DPDES); 840 if (dpdes) 841 vcpu->arch.doorbell_request = 1; 842 843 vc->vtb = mfspr(SPRN_VTB); 844 845 dec = mfspr(SPRN_DEC); 846 if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ 847 dec = (s32) dec; 848 *tb = mftb(); 849 vcpu->arch.dec_expires = dec + *tb; 850 851 if (vc->tb_offset_applied) { 852 u64 new_tb = *tb - vc->tb_offset_applied; 853 mtspr(SPRN_TBU40, new_tb); 854 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 855 new_tb += 0x1000000; 856 mtspr(SPRN_TBU40, new_tb); 857 } 858 *tb = new_tb; 859 vc->tb_offset_applied = 0; 860 } 861 862 save_clear_guest_mmu(kvm, vcpu); 863 switch_mmu_to_host(kvm, host_pidr); 864 865 /* 866 * Enable MSR here in order to have facilities enabled to save 867 * guest registers. This enables MMU (if we were in realmode), so 868 * only switch MMU on after the MMU is switched to host, to avoid 869 * the P9_RADIX_PREFETCH_BUG or hash guest context. 870 */ 871 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && 872 vcpu->arch.shregs.msr & MSR_TS_MASK) 873 msr |= MSR_TS_S; 874 __mtmsrd(msr, 0); 875 876 store_vcpu_state(vcpu); 877 878 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr); 879 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr); 880 881 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 882 /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ 883 mtspr(SPRN_PSSCR, host_hpsscr | 884 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 885 } 886 887 mtspr(SPRN_HFSCR, host_hfscr); 888 if (vcpu->arch.ciabr != host_ciabr) 889 mtspr(SPRN_CIABR, host_ciabr); 890 891 if (dawr_enabled()) { 892 if (vcpu->arch.dawr0 != host_dawr0) 893 mtspr(SPRN_DAWR0, host_dawr0); 894 if (vcpu->arch.dawrx0 != host_dawrx0) 895 mtspr(SPRN_DAWRX0, host_dawrx0); 896 if (cpu_has_feature(CPU_FTR_DAWR1)) { 897 if (vcpu->arch.dawr1 != host_dawr1) 898 mtspr(SPRN_DAWR1, host_dawr1); 899 if (vcpu->arch.dawrx1 != host_dawrx1) 900 mtspr(SPRN_DAWRX1, host_dawrx1); 901 } 902 } 903 904 if (dpdes) 905 mtspr(SPRN_DPDES, 0); 906 if (vc->pcr) 907 mtspr(SPRN_PCR, PCR_MASK); 908 909 /* HDEC must be at least as large as DEC, so decrementer_max fits */ 910 mtspr(SPRN_HDEC, decrementer_max); 911 912 timer_rearm_host_dec(*tb); 913 914 restore_p9_host_os_sprs(vcpu, &host_os_sprs); 915 916 barrier(); /* Close in_guest critical section */ 917 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE); 918 /* Interrupts are recoverable at this point */ 919 920 /* 921 * cp_abort is required if the processor supports local copy-paste 922 * to clear the copy buffer that was under control of the guest. 923 */ 924 if (cpu_has_feature(CPU_FTR_ARCH_31)) 925 asm volatile(PPC_CP_ABORT); 926 927 out: 928 return trap; 929 } 930 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9); 931