1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/kvm_host.h> 4 #include <asm/asm-prototypes.h> 5 #include <asm/dbell.h> 6 #include <asm/kvm_ppc.h> 7 #include <asm/pmc.h> 8 #include <asm/ppc-opcode.h> 9 10 #include "book3s_hv.h" 11 12 static void freeze_pmu(unsigned long mmcr0, unsigned long mmcra) 13 { 14 if (!(mmcr0 & MMCR0_FC)) 15 goto do_freeze; 16 if (mmcra & MMCRA_SAMPLE_ENABLE) 17 goto do_freeze; 18 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 19 if (!(mmcr0 & MMCR0_PMCCEXT)) 20 goto do_freeze; 21 if (!(mmcra & MMCRA_BHRB_DISABLE)) 22 goto do_freeze; 23 } 24 return; 25 26 do_freeze: 27 mmcr0 = MMCR0_FC; 28 mmcra = 0; 29 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 30 mmcr0 |= MMCR0_PMCCEXT; 31 mmcra = MMCRA_BHRB_DISABLE; 32 } 33 34 mtspr(SPRN_MMCR0, mmcr0); 35 mtspr(SPRN_MMCRA, mmcra); 36 isync(); 37 } 38 39 void switch_pmu_to_guest(struct kvm_vcpu *vcpu, 40 struct p9_host_os_sprs *host_os_sprs) 41 { 42 struct lppaca *lp; 43 int load_pmu = 1; 44 45 lp = vcpu->arch.vpa.pinned_addr; 46 if (lp) 47 load_pmu = lp->pmcregs_in_use; 48 49 /* Save host */ 50 if (ppc_get_pmu_inuse()) { 51 /* 52 * It might be better to put PMU handling (at least for the 53 * host) in the perf subsystem because it knows more about what 54 * is being used. 55 */ 56 57 /* POWER9, POWER10 do not implement HPMC or SPMC */ 58 59 host_os_sprs->mmcr0 = mfspr(SPRN_MMCR0); 60 host_os_sprs->mmcra = mfspr(SPRN_MMCRA); 61 62 freeze_pmu(host_os_sprs->mmcr0, host_os_sprs->mmcra); 63 64 host_os_sprs->pmc1 = mfspr(SPRN_PMC1); 65 host_os_sprs->pmc2 = mfspr(SPRN_PMC2); 66 host_os_sprs->pmc3 = mfspr(SPRN_PMC3); 67 host_os_sprs->pmc4 = mfspr(SPRN_PMC4); 68 host_os_sprs->pmc5 = mfspr(SPRN_PMC5); 69 host_os_sprs->pmc6 = mfspr(SPRN_PMC6); 70 host_os_sprs->mmcr1 = mfspr(SPRN_MMCR1); 71 host_os_sprs->mmcr2 = mfspr(SPRN_MMCR2); 72 host_os_sprs->sdar = mfspr(SPRN_SDAR); 73 host_os_sprs->siar = mfspr(SPRN_SIAR); 74 host_os_sprs->sier1 = mfspr(SPRN_SIER); 75 76 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 77 host_os_sprs->mmcr3 = mfspr(SPRN_MMCR3); 78 host_os_sprs->sier2 = mfspr(SPRN_SIER2); 79 host_os_sprs->sier3 = mfspr(SPRN_SIER3); 80 } 81 } 82 83 #ifdef CONFIG_PPC_PSERIES 84 /* After saving PMU, before loading guest PMU, flip pmcregs_in_use */ 85 if (kvmhv_on_pseries()) { 86 barrier(); 87 get_lppaca()->pmcregs_in_use = load_pmu; 88 barrier(); 89 } 90 #endif 91 92 /* 93 * Load guest. If the VPA said the PMCs are not in use but the guest 94 * tried to access them anyway, HFSCR[PM] will be set by the HFAC 95 * fault so we can make forward progress. 96 */ 97 if (load_pmu || (vcpu->arch.hfscr & HFSCR_PM)) { 98 mtspr(SPRN_PMC1, vcpu->arch.pmc[0]); 99 mtspr(SPRN_PMC2, vcpu->arch.pmc[1]); 100 mtspr(SPRN_PMC3, vcpu->arch.pmc[2]); 101 mtspr(SPRN_PMC4, vcpu->arch.pmc[3]); 102 mtspr(SPRN_PMC5, vcpu->arch.pmc[4]); 103 mtspr(SPRN_PMC6, vcpu->arch.pmc[5]); 104 mtspr(SPRN_MMCR1, vcpu->arch.mmcr[1]); 105 mtspr(SPRN_MMCR2, vcpu->arch.mmcr[2]); 106 mtspr(SPRN_SDAR, vcpu->arch.sdar); 107 mtspr(SPRN_SIAR, vcpu->arch.siar); 108 mtspr(SPRN_SIER, vcpu->arch.sier[0]); 109 110 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 111 mtspr(SPRN_MMCR3, vcpu->arch.mmcr[3]); 112 mtspr(SPRN_SIER2, vcpu->arch.sier[1]); 113 mtspr(SPRN_SIER3, vcpu->arch.sier[2]); 114 } 115 116 /* Set MMCRA then MMCR0 last */ 117 mtspr(SPRN_MMCRA, vcpu->arch.mmcra); 118 mtspr(SPRN_MMCR0, vcpu->arch.mmcr[0]); 119 /* No isync necessary because we're starting counters */ 120 121 if (!vcpu->arch.nested && 122 (vcpu->arch.hfscr_permitted & HFSCR_PM)) 123 vcpu->arch.hfscr |= HFSCR_PM; 124 } 125 } 126 EXPORT_SYMBOL_GPL(switch_pmu_to_guest); 127 128 void switch_pmu_to_host(struct kvm_vcpu *vcpu, 129 struct p9_host_os_sprs *host_os_sprs) 130 { 131 struct lppaca *lp; 132 int save_pmu = 1; 133 134 lp = vcpu->arch.vpa.pinned_addr; 135 if (lp) 136 save_pmu = lp->pmcregs_in_use; 137 if (IS_ENABLED(CONFIG_KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND)) { 138 /* 139 * Save pmu if this guest is capable of running nested guests. 140 * This is option is for old L1s that do not set their 141 * lppaca->pmcregs_in_use properly when entering their L2. 142 */ 143 save_pmu |= nesting_enabled(vcpu->kvm); 144 } 145 146 if (save_pmu) { 147 vcpu->arch.mmcr[0] = mfspr(SPRN_MMCR0); 148 vcpu->arch.mmcra = mfspr(SPRN_MMCRA); 149 150 freeze_pmu(vcpu->arch.mmcr[0], vcpu->arch.mmcra); 151 152 vcpu->arch.pmc[0] = mfspr(SPRN_PMC1); 153 vcpu->arch.pmc[1] = mfspr(SPRN_PMC2); 154 vcpu->arch.pmc[2] = mfspr(SPRN_PMC3); 155 vcpu->arch.pmc[3] = mfspr(SPRN_PMC4); 156 vcpu->arch.pmc[4] = mfspr(SPRN_PMC5); 157 vcpu->arch.pmc[5] = mfspr(SPRN_PMC6); 158 vcpu->arch.mmcr[1] = mfspr(SPRN_MMCR1); 159 vcpu->arch.mmcr[2] = mfspr(SPRN_MMCR2); 160 vcpu->arch.sdar = mfspr(SPRN_SDAR); 161 vcpu->arch.siar = mfspr(SPRN_SIAR); 162 vcpu->arch.sier[0] = mfspr(SPRN_SIER); 163 164 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 165 vcpu->arch.mmcr[3] = mfspr(SPRN_MMCR3); 166 vcpu->arch.sier[1] = mfspr(SPRN_SIER2); 167 vcpu->arch.sier[2] = mfspr(SPRN_SIER3); 168 } 169 170 } else if (vcpu->arch.hfscr & HFSCR_PM) { 171 /* 172 * The guest accessed PMC SPRs without specifying they should 173 * be preserved, or it cleared pmcregs_in_use after the last 174 * access. Just ensure they are frozen. 175 */ 176 freeze_pmu(mfspr(SPRN_MMCR0), mfspr(SPRN_MMCRA)); 177 178 /* 179 * Demand-fault PMU register access in the guest. 180 * 181 * This is used to grab the guest's VPA pmcregs_in_use value 182 * and reflect it into the host's VPA in the case of a nested 183 * hypervisor. 184 * 185 * It also avoids having to zero-out SPRs after each guest 186 * exit to avoid side-channels when. 187 * 188 * This is cleared here when we exit the guest, so later HFSCR 189 * interrupt handling can add it back to run the guest with 190 * PM enabled next time. 191 */ 192 if (!vcpu->arch.nested) 193 vcpu->arch.hfscr &= ~HFSCR_PM; 194 } /* otherwise the PMU should still be frozen */ 195 196 #ifdef CONFIG_PPC_PSERIES 197 if (kvmhv_on_pseries()) { 198 barrier(); 199 get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse(); 200 barrier(); 201 } 202 #endif 203 204 if (ppc_get_pmu_inuse()) { 205 mtspr(SPRN_PMC1, host_os_sprs->pmc1); 206 mtspr(SPRN_PMC2, host_os_sprs->pmc2); 207 mtspr(SPRN_PMC3, host_os_sprs->pmc3); 208 mtspr(SPRN_PMC4, host_os_sprs->pmc4); 209 mtspr(SPRN_PMC5, host_os_sprs->pmc5); 210 mtspr(SPRN_PMC6, host_os_sprs->pmc6); 211 mtspr(SPRN_MMCR1, host_os_sprs->mmcr1); 212 mtspr(SPRN_MMCR2, host_os_sprs->mmcr2); 213 mtspr(SPRN_SDAR, host_os_sprs->sdar); 214 mtspr(SPRN_SIAR, host_os_sprs->siar); 215 mtspr(SPRN_SIER, host_os_sprs->sier1); 216 217 if (cpu_has_feature(CPU_FTR_ARCH_31)) { 218 mtspr(SPRN_MMCR3, host_os_sprs->mmcr3); 219 mtspr(SPRN_SIER2, host_os_sprs->sier2); 220 mtspr(SPRN_SIER3, host_os_sprs->sier3); 221 } 222 223 /* Set MMCRA then MMCR0 last */ 224 mtspr(SPRN_MMCRA, host_os_sprs->mmcra); 225 mtspr(SPRN_MMCR0, host_os_sprs->mmcr0); 226 isync(); 227 } 228 } 229 EXPORT_SYMBOL_GPL(switch_pmu_to_host); 230 231 static void load_spr_state(struct kvm_vcpu *vcpu, 232 struct p9_host_os_sprs *host_os_sprs) 233 { 234 /* TAR is very fast */ 235 mtspr(SPRN_TAR, vcpu->arch.tar); 236 237 #ifdef CONFIG_ALTIVEC 238 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 239 current->thread.vrsave != vcpu->arch.vrsave) 240 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); 241 #endif 242 243 if (vcpu->arch.hfscr & HFSCR_EBB) { 244 if (current->thread.ebbhr != vcpu->arch.ebbhr) 245 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); 246 if (current->thread.ebbrr != vcpu->arch.ebbrr) 247 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); 248 if (current->thread.bescr != vcpu->arch.bescr) 249 mtspr(SPRN_BESCR, vcpu->arch.bescr); 250 } 251 252 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 253 current->thread.tidr != vcpu->arch.tid) 254 mtspr(SPRN_TIDR, vcpu->arch.tid); 255 if (host_os_sprs->iamr != vcpu->arch.iamr) 256 mtspr(SPRN_IAMR, vcpu->arch.iamr); 257 if (host_os_sprs->amr != vcpu->arch.amr) 258 mtspr(SPRN_AMR, vcpu->arch.amr); 259 if (vcpu->arch.uamor != 0) 260 mtspr(SPRN_UAMOR, vcpu->arch.uamor); 261 if (current->thread.fscr != vcpu->arch.fscr) 262 mtspr(SPRN_FSCR, vcpu->arch.fscr); 263 if (current->thread.dscr != vcpu->arch.dscr) 264 mtspr(SPRN_DSCR, vcpu->arch.dscr); 265 if (vcpu->arch.pspb != 0) 266 mtspr(SPRN_PSPB, vcpu->arch.pspb); 267 268 /* 269 * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] 270 * clear (or hstate set appropriately to catch those registers 271 * being clobbered if we take a MCE or SRESET), so those are done 272 * later. 273 */ 274 275 if (!(vcpu->arch.ctrl & 1)) 276 mtspr(SPRN_CTRLT, 0); 277 } 278 279 static void store_spr_state(struct kvm_vcpu *vcpu) 280 { 281 vcpu->arch.tar = mfspr(SPRN_TAR); 282 283 #ifdef CONFIG_ALTIVEC 284 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 285 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); 286 #endif 287 288 if (vcpu->arch.hfscr & HFSCR_EBB) { 289 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); 290 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); 291 vcpu->arch.bescr = mfspr(SPRN_BESCR); 292 } 293 294 if (cpu_has_feature(CPU_FTR_P9_TIDR)) 295 vcpu->arch.tid = mfspr(SPRN_TIDR); 296 vcpu->arch.iamr = mfspr(SPRN_IAMR); 297 vcpu->arch.amr = mfspr(SPRN_AMR); 298 vcpu->arch.uamor = mfspr(SPRN_UAMOR); 299 vcpu->arch.fscr = mfspr(SPRN_FSCR); 300 vcpu->arch.dscr = mfspr(SPRN_DSCR); 301 vcpu->arch.pspb = mfspr(SPRN_PSPB); 302 303 vcpu->arch.ctrl = mfspr(SPRN_CTRLF); 304 } 305 306 /* Returns true if current MSR and/or guest MSR may have changed */ 307 bool load_vcpu_state(struct kvm_vcpu *vcpu, 308 struct p9_host_os_sprs *host_os_sprs) 309 { 310 bool ret = false; 311 312 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 313 if (cpu_has_feature(CPU_FTR_TM) || 314 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 315 unsigned long guest_msr = vcpu->arch.shregs.msr; 316 if (MSR_TM_ACTIVE(guest_msr)) { 317 kvmppc_restore_tm_hv(vcpu, guest_msr, true); 318 ret = true; 319 } else if (vcpu->arch.hfscr & HFSCR_TM) { 320 mtspr(SPRN_TEXASR, vcpu->arch.texasr); 321 mtspr(SPRN_TFHAR, vcpu->arch.tfhar); 322 mtspr(SPRN_TFIAR, vcpu->arch.tfiar); 323 } 324 } 325 #endif 326 327 load_spr_state(vcpu, host_os_sprs); 328 329 load_fp_state(&vcpu->arch.fp); 330 #ifdef CONFIG_ALTIVEC 331 load_vr_state(&vcpu->arch.vr); 332 #endif 333 334 return ret; 335 } 336 EXPORT_SYMBOL_GPL(load_vcpu_state); 337 338 void store_vcpu_state(struct kvm_vcpu *vcpu) 339 { 340 store_spr_state(vcpu); 341 342 store_fp_state(&vcpu->arch.fp); 343 #ifdef CONFIG_ALTIVEC 344 store_vr_state(&vcpu->arch.vr); 345 #endif 346 347 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 348 if (cpu_has_feature(CPU_FTR_TM) || 349 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 350 unsigned long guest_msr = vcpu->arch.shregs.msr; 351 if (MSR_TM_ACTIVE(guest_msr)) { 352 kvmppc_save_tm_hv(vcpu, guest_msr, true); 353 } else if (vcpu->arch.hfscr & HFSCR_TM) { 354 vcpu->arch.texasr = mfspr(SPRN_TEXASR); 355 vcpu->arch.tfhar = mfspr(SPRN_TFHAR); 356 vcpu->arch.tfiar = mfspr(SPRN_TFIAR); 357 358 if (!vcpu->arch.nested) { 359 vcpu->arch.load_tm++; /* see load_ebb comment */ 360 if (!vcpu->arch.load_tm) 361 vcpu->arch.hfscr &= ~HFSCR_TM; 362 } 363 } 364 } 365 #endif 366 } 367 EXPORT_SYMBOL_GPL(store_vcpu_state); 368 369 void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) 370 { 371 host_os_sprs->iamr = mfspr(SPRN_IAMR); 372 host_os_sprs->amr = mfspr(SPRN_AMR); 373 } 374 EXPORT_SYMBOL_GPL(save_p9_host_os_sprs); 375 376 /* vcpu guest regs must already be saved */ 377 void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, 378 struct p9_host_os_sprs *host_os_sprs) 379 { 380 /* 381 * current->thread.xxx registers must all be restored to host 382 * values before a potential context switch, otherwise the context 383 * switch itself will overwrite current->thread.xxx with the values 384 * from the guest SPRs. 385 */ 386 387 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); 388 389 if (cpu_has_feature(CPU_FTR_P9_TIDR) && 390 current->thread.tidr != vcpu->arch.tid) 391 mtspr(SPRN_TIDR, current->thread.tidr); 392 if (host_os_sprs->iamr != vcpu->arch.iamr) 393 mtspr(SPRN_IAMR, host_os_sprs->iamr); 394 if (vcpu->arch.uamor != 0) 395 mtspr(SPRN_UAMOR, 0); 396 if (host_os_sprs->amr != vcpu->arch.amr) 397 mtspr(SPRN_AMR, host_os_sprs->amr); 398 if (current->thread.fscr != vcpu->arch.fscr) 399 mtspr(SPRN_FSCR, current->thread.fscr); 400 if (current->thread.dscr != vcpu->arch.dscr) 401 mtspr(SPRN_DSCR, current->thread.dscr); 402 if (vcpu->arch.pspb != 0) 403 mtspr(SPRN_PSPB, 0); 404 405 /* Save guest CTRL register, set runlatch to 1 */ 406 if (!(vcpu->arch.ctrl & 1)) 407 mtspr(SPRN_CTRLT, 1); 408 409 #ifdef CONFIG_ALTIVEC 410 if (cpu_has_feature(CPU_FTR_ALTIVEC) && 411 vcpu->arch.vrsave != current->thread.vrsave) 412 mtspr(SPRN_VRSAVE, current->thread.vrsave); 413 #endif 414 if (vcpu->arch.hfscr & HFSCR_EBB) { 415 if (vcpu->arch.bescr != current->thread.bescr) 416 mtspr(SPRN_BESCR, current->thread.bescr); 417 if (vcpu->arch.ebbhr != current->thread.ebbhr) 418 mtspr(SPRN_EBBHR, current->thread.ebbhr); 419 if (vcpu->arch.ebbrr != current->thread.ebbrr) 420 mtspr(SPRN_EBBRR, current->thread.ebbrr); 421 422 if (!vcpu->arch.nested) { 423 /* 424 * This is like load_fp in context switching, turn off 425 * the facility after it wraps the u8 to try avoiding 426 * saving and restoring the registers each partition 427 * switch. 428 */ 429 vcpu->arch.load_ebb++; 430 if (!vcpu->arch.load_ebb) 431 vcpu->arch.hfscr &= ~HFSCR_EBB; 432 } 433 } 434 435 if (vcpu->arch.tar != current->thread.tar) 436 mtspr(SPRN_TAR, current->thread.tar); 437 } 438 EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs); 439 440 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 441 static void __start_timing(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) 442 { 443 struct kvmppc_vcore *vc = vcpu->arch.vcore; 444 u64 tb = mftb() - vc->tb_offset_applied; 445 446 vcpu->arch.cur_activity = next; 447 vcpu->arch.cur_tb_start = tb; 448 } 449 450 static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) 451 { 452 struct kvmppc_vcore *vc = vcpu->arch.vcore; 453 struct kvmhv_tb_accumulator *curr; 454 u64 tb = mftb() - vc->tb_offset_applied; 455 u64 prev_tb; 456 u64 delta; 457 u64 seq; 458 459 curr = vcpu->arch.cur_activity; 460 vcpu->arch.cur_activity = next; 461 prev_tb = vcpu->arch.cur_tb_start; 462 vcpu->arch.cur_tb_start = tb; 463 464 if (!curr) 465 return; 466 467 delta = tb - prev_tb; 468 469 seq = curr->seqcount; 470 curr->seqcount = seq + 1; 471 smp_wmb(); 472 curr->tb_total += delta; 473 if (seq == 0 || delta < curr->tb_min) 474 curr->tb_min = delta; 475 if (delta > curr->tb_max) 476 curr->tb_max = delta; 477 smp_wmb(); 478 curr->seqcount = seq + 2; 479 } 480 481 #define start_timing(vcpu, next) __start_timing(vcpu, next) 482 #define end_timing(vcpu) __start_timing(vcpu, NULL) 483 #define accumulate_time(vcpu, next) __accumulate_time(vcpu, next) 484 #else 485 #define start_timing(vcpu, next) do {} while (0) 486 #define end_timing(vcpu) do {} while (0) 487 #define accumulate_time(vcpu, next) do {} while (0) 488 #endif 489 490 static inline u64 mfslbv(unsigned int idx) 491 { 492 u64 slbev; 493 494 asm volatile("slbmfev %0,%1" : "=r" (slbev) : "r" (idx)); 495 496 return slbev; 497 } 498 499 static inline u64 mfslbe(unsigned int idx) 500 { 501 u64 slbee; 502 503 asm volatile("slbmfee %0,%1" : "=r" (slbee) : "r" (idx)); 504 505 return slbee; 506 } 507 508 static inline void mtslb(u64 slbee, u64 slbev) 509 { 510 asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee)); 511 } 512 513 static inline void clear_slb_entry(unsigned int idx) 514 { 515 mtslb(idx, 0); 516 } 517 518 static inline void slb_clear_invalidate_partition(void) 519 { 520 clear_slb_entry(0); 521 asm volatile(PPC_SLBIA(6)); 522 } 523 524 /* 525 * Malicious or buggy radix guests may have inserted SLB entries 526 * (only 0..3 because radix always runs with UPRT=1), so these must 527 * be cleared here to avoid side-channels. slbmte is used rather 528 * than slbia, as it won't clear cached translations. 529 */ 530 static void radix_clear_slb(void) 531 { 532 int i; 533 534 for (i = 0; i < 4; i++) 535 clear_slb_entry(i); 536 } 537 538 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 539 { 540 struct kvm_nested_guest *nested = vcpu->arch.nested; 541 u32 lpid; 542 u32 pid; 543 544 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; 545 pid = vcpu->arch.pid; 546 547 /* 548 * Prior memory accesses to host PID Q3 must be completed before we 549 * start switching, and stores must be drained to avoid not-my-LPAR 550 * logic (see switch_mmu_to_host). 551 */ 552 asm volatile("hwsync" ::: "memory"); 553 isync(); 554 mtspr(SPRN_LPID, lpid); 555 mtspr(SPRN_LPCR, lpcr); 556 mtspr(SPRN_PID, pid); 557 /* 558 * isync not required here because we are HRFID'ing to guest before 559 * any guest context access, which is context synchronising. 560 */ 561 } 562 563 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) 564 { 565 u32 lpid; 566 u32 pid; 567 int i; 568 569 lpid = kvm->arch.lpid; 570 pid = vcpu->arch.pid; 571 572 /* 573 * See switch_mmu_to_guest_radix. ptesync should not be required here 574 * even if the host is in HPT mode because speculative accesses would 575 * not cause RC updates (we are in real mode). 576 */ 577 asm volatile("hwsync" ::: "memory"); 578 isync(); 579 mtspr(SPRN_LPID, lpid); 580 mtspr(SPRN_LPCR, lpcr); 581 mtspr(SPRN_PID, pid); 582 583 for (i = 0; i < vcpu->arch.slb_max; i++) 584 mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv); 585 /* 586 * isync not required here, see switch_mmu_to_guest_radix. 587 */ 588 } 589 590 static void switch_mmu_to_host(struct kvm *kvm, u32 pid) 591 { 592 u32 lpid = kvm->arch.host_lpid; 593 u64 lpcr = kvm->arch.host_lpcr; 594 595 /* 596 * The guest has exited, so guest MMU context is no longer being 597 * non-speculatively accessed, but a hwsync is needed before the 598 * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained, 599 * so the not-my-LPAR tlbie logic does not overlook them. 600 */ 601 asm volatile("hwsync" ::: "memory"); 602 isync(); 603 mtspr(SPRN_PID, pid); 604 mtspr(SPRN_LPID, lpid); 605 mtspr(SPRN_LPCR, lpcr); 606 /* 607 * isync is not required after the switch, because mtmsrd with L=0 608 * is performed after this switch, which is context synchronising. 609 */ 610 611 if (!radix_enabled()) 612 slb_restore_bolted_realmode(); 613 } 614 615 static void save_clear_host_mmu(struct kvm *kvm) 616 { 617 if (!radix_enabled()) { 618 /* 619 * Hash host could save and restore host SLB entries to 620 * reduce SLB fault overheads of VM exits, but for now the 621 * existing code clears all entries and restores just the 622 * bolted ones when switching back to host. 623 */ 624 slb_clear_invalidate_partition(); 625 } 626 } 627 628 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) 629 { 630 if (kvm_is_radix(kvm)) { 631 radix_clear_slb(); 632 } else { 633 int i; 634 int nr = 0; 635 636 /* 637 * This must run before switching to host (radix host can't 638 * access all SLBs). 639 */ 640 for (i = 0; i < vcpu->arch.slb_nr; i++) { 641 u64 slbee, slbev; 642 643 slbee = mfslbe(i); 644 if (slbee & SLB_ESID_V) { 645 slbev = mfslbv(i); 646 vcpu->arch.slb[nr].orige = slbee | i; 647 vcpu->arch.slb[nr].origv = slbev; 648 nr++; 649 } 650 } 651 vcpu->arch.slb_max = nr; 652 slb_clear_invalidate_partition(); 653 } 654 } 655 656 static void flush_guest_tlb(struct kvm *kvm) 657 { 658 unsigned long rb, set; 659 660 rb = PPC_BIT(52); /* IS = 2 */ 661 if (kvm_is_radix(kvm)) { 662 /* R=1 PRS=1 RIC=2 */ 663 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 664 : : "r" (rb), "i" (1), "i" (1), "i" (2), 665 "r" (0) : "memory"); 666 for (set = 1; set < kvm->arch.tlb_sets; ++set) { 667 rb += PPC_BIT(51); /* increment set number */ 668 /* R=1 PRS=1 RIC=0 */ 669 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 670 : : "r" (rb), "i" (1), "i" (1), "i" (0), 671 "r" (0) : "memory"); 672 } 673 asm volatile("ptesync": : :"memory"); 674 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 675 asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); 676 } else { 677 for (set = 0; set < kvm->arch.tlb_sets; ++set) { 678 /* R=0 PRS=0 RIC=0 */ 679 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) 680 : : "r" (rb), "i" (0), "i" (0), "i" (0), 681 "r" (0) : "memory"); 682 rb += PPC_BIT(51); /* increment set number */ 683 } 684 asm volatile("ptesync": : :"memory"); 685 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. 686 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); 687 } 688 } 689 690 static void check_need_tlb_flush(struct kvm *kvm, int pcpu, 691 struct kvm_nested_guest *nested) 692 { 693 cpumask_t *need_tlb_flush; 694 bool all_set = true; 695 int i; 696 697 if (nested) 698 need_tlb_flush = &nested->need_tlb_flush; 699 else 700 need_tlb_flush = &kvm->arch.need_tlb_flush; 701 702 if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush))) 703 return; 704 705 /* 706 * Individual threads can come in here, but the TLB is shared between 707 * the 4 threads in a core, hence invalidating on one thread 708 * invalidates for all, so only invalidate the first time (if all bits 709 * were set. The others must still execute a ptesync. 710 * 711 * If a race occurs and two threads do the TLB flush, that is not a 712 * problem, just sub-optimal. 713 */ 714 for (i = cpu_first_tlb_thread_sibling(pcpu); 715 i <= cpu_last_tlb_thread_sibling(pcpu); 716 i += cpu_tlb_thread_sibling_step()) { 717 if (!cpumask_test_cpu(i, need_tlb_flush)) { 718 all_set = false; 719 break; 720 } 721 } 722 if (all_set) 723 flush_guest_tlb(kvm); 724 else 725 asm volatile("ptesync" ::: "memory"); 726 727 /* Clear the bit after the TLB flush */ 728 cpumask_clear_cpu(pcpu, need_tlb_flush); 729 } 730 731 unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr) 732 { 733 unsigned long msr_needed = 0; 734 735 msr &= ~MSR_EE; 736 737 /* MSR bits may have been cleared by context switch so must recheck */ 738 if (IS_ENABLED(CONFIG_PPC_FPU)) 739 msr_needed |= MSR_FP; 740 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 741 msr_needed |= MSR_VEC; 742 if (cpu_has_feature(CPU_FTR_VSX)) 743 msr_needed |= MSR_VSX; 744 if ((cpu_has_feature(CPU_FTR_TM) || 745 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && 746 (vcpu->arch.hfscr & HFSCR_TM)) 747 msr_needed |= MSR_TM; 748 749 /* 750 * This could be combined with MSR[RI] clearing, but that expands 751 * the unrecoverable window. It would be better to cover unrecoverable 752 * with KVM bad interrupt handling rather than use MSR[RI] at all. 753 * 754 * Much more difficult and less worthwhile to combine with IR/DR 755 * disable. 756 */ 757 if ((msr & msr_needed) != msr_needed) { 758 msr |= msr_needed; 759 __mtmsrd(msr, 0); 760 } else { 761 __hard_irq_disable(); 762 } 763 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 764 765 return msr; 766 } 767 EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities); 768 769 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) 770 { 771 struct p9_host_os_sprs host_os_sprs; 772 struct kvm *kvm = vcpu->kvm; 773 struct kvm_nested_guest *nested = vcpu->arch.nested; 774 struct kvmppc_vcore *vc = vcpu->arch.vcore; 775 s64 hdec, dec; 776 u64 purr, spurr; 777 u64 *exsave; 778 int trap; 779 unsigned long msr; 780 unsigned long host_hfscr; 781 unsigned long host_ciabr; 782 unsigned long host_dawr0; 783 unsigned long host_dawrx0; 784 unsigned long host_psscr; 785 unsigned long host_hpsscr; 786 unsigned long host_pidr; 787 unsigned long host_dawr1; 788 unsigned long host_dawrx1; 789 unsigned long dpdes; 790 791 hdec = time_limit - *tb; 792 if (hdec < 0) 793 return BOOK3S_INTERRUPT_HV_DECREMENTER; 794 795 WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV); 796 WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME)); 797 798 start_timing(vcpu, &vcpu->arch.rm_entry); 799 800 vcpu->arch.ceded = 0; 801 802 /* Save MSR for restore, with EE clear. */ 803 msr = mfmsr() & ~MSR_EE; 804 805 host_hfscr = mfspr(SPRN_HFSCR); 806 host_ciabr = mfspr(SPRN_CIABR); 807 host_psscr = mfspr(SPRN_PSSCR_PR); 808 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) 809 host_hpsscr = mfspr(SPRN_PSSCR); 810 host_pidr = mfspr(SPRN_PID); 811 812 if (dawr_enabled()) { 813 host_dawr0 = mfspr(SPRN_DAWR0); 814 host_dawrx0 = mfspr(SPRN_DAWRX0); 815 if (cpu_has_feature(CPU_FTR_DAWR1)) { 816 host_dawr1 = mfspr(SPRN_DAWR1); 817 host_dawrx1 = mfspr(SPRN_DAWRX1); 818 } 819 } 820 821 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); 822 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); 823 824 save_p9_host_os_sprs(&host_os_sprs); 825 826 msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr); 827 if (lazy_irq_pending()) { 828 trap = 0; 829 goto out; 830 } 831 832 if (unlikely(load_vcpu_state(vcpu, &host_os_sprs))) 833 msr = mfmsr(); /* MSR may have been updated */ 834 835 if (vc->tb_offset) { 836 u64 new_tb = *tb + vc->tb_offset; 837 mtspr(SPRN_TBU40, new_tb); 838 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 839 new_tb += 0x1000000; 840 mtspr(SPRN_TBU40, new_tb); 841 } 842 *tb = new_tb; 843 vc->tb_offset_applied = vc->tb_offset; 844 } 845 846 mtspr(SPRN_VTB, vc->vtb); 847 mtspr(SPRN_PURR, vcpu->arch.purr); 848 mtspr(SPRN_SPURR, vcpu->arch.spurr); 849 850 if (vc->pcr) 851 mtspr(SPRN_PCR, vc->pcr | PCR_MASK); 852 if (vcpu->arch.doorbell_request) { 853 vcpu->arch.doorbell_request = 0; 854 mtspr(SPRN_DPDES, 1); 855 } 856 857 if (dawr_enabled()) { 858 if (vcpu->arch.dawr0 != host_dawr0) 859 mtspr(SPRN_DAWR0, vcpu->arch.dawr0); 860 if (vcpu->arch.dawrx0 != host_dawrx0) 861 mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); 862 if (cpu_has_feature(CPU_FTR_DAWR1)) { 863 if (vcpu->arch.dawr1 != host_dawr1) 864 mtspr(SPRN_DAWR1, vcpu->arch.dawr1); 865 if (vcpu->arch.dawrx1 != host_dawrx1) 866 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); 867 } 868 } 869 if (vcpu->arch.ciabr != host_ciabr) 870 mtspr(SPRN_CIABR, vcpu->arch.ciabr); 871 872 873 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 874 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | 875 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 876 } else { 877 if (vcpu->arch.psscr != host_psscr) 878 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); 879 } 880 881 mtspr(SPRN_HFSCR, vcpu->arch.hfscr); 882 883 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 884 mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME); 885 886 /* 887 * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage 888 * Interrupt (HDSI) the HDSISR is not be updated at all. 889 * 890 * To work around this we put a canary value into the HDSISR before 891 * returning to a guest and then check for this canary when we take a 892 * HDSI. If we find the canary on a HDSI, we know the hardware didn't 893 * update the HDSISR. In this case we return to the guest to retake the 894 * HDSI which should correctly update the HDSISR the second time HDSI 895 * entry. 896 * 897 * The "radix prefetch bug" test can be used to test for this bug, as 898 * it also exists fo DD2.1 and below. 899 */ 900 if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 901 mtspr(SPRN_HDSISR, HDSISR_CANARY); 902 903 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); 904 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); 905 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); 906 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); 907 908 /* 909 * It might be preferable to load_vcpu_state here, in order to get the 910 * GPR/FP register loads executing in parallel with the previous mtSPR 911 * instructions, but for now that can't be done because the TM handling 912 * in load_vcpu_state can change some SPRs and vcpu state (nip, msr). 913 * But TM could be split out if this would be a significant benefit. 914 */ 915 916 /* 917 * MSR[RI] does not need to be cleared (and is not, for radix guests 918 * with no prefetch bug), because in_guest is set. If we take a SRESET 919 * or MCE with in_guest set but still in HV mode, then 920 * kvmppc_p9_bad_interrupt handles the interrupt, which effectively 921 * clears MSR[RI] and doesn't return. 922 */ 923 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9); 924 barrier(); /* Open in_guest critical section */ 925 926 /* 927 * Hash host, hash guest, or radix guest with prefetch bug, all have 928 * to disable the MMU before switching to guest MMU state. 929 */ 930 if (!radix_enabled() || !kvm_is_radix(kvm) || 931 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) 932 __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0); 933 934 save_clear_host_mmu(kvm); 935 936 if (kvm_is_radix(kvm)) 937 switch_mmu_to_guest_radix(kvm, vcpu, lpcr); 938 else 939 switch_mmu_to_guest_hpt(kvm, vcpu, lpcr); 940 941 /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */ 942 check_need_tlb_flush(kvm, vc->pcpu, nested); 943 944 /* 945 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0, 946 * so set guest LPCR (with HDICE) before writing HDEC. 947 */ 948 mtspr(SPRN_HDEC, hdec); 949 950 mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb); 951 952 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 953 tm_return_to_guest: 954 #endif 955 mtspr(SPRN_DAR, vcpu->arch.shregs.dar); 956 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); 957 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0); 958 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1); 959 960 accumulate_time(vcpu, &vcpu->arch.guest_time); 961 962 switch_pmu_to_guest(vcpu, &host_os_sprs); 963 kvmppc_p9_enter_guest(vcpu); 964 switch_pmu_to_host(vcpu, &host_os_sprs); 965 966 accumulate_time(vcpu, &vcpu->arch.rm_intr); 967 968 /* XXX: Could get these from r11/12 and paca exsave instead */ 969 vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0); 970 vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1); 971 vcpu->arch.shregs.dar = mfspr(SPRN_DAR); 972 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); 973 974 /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */ 975 trap = local_paca->kvm_hstate.scratch0 & ~0x2; 976 977 if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) 978 exsave = local_paca->exgen; 979 else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) 980 exsave = local_paca->exnmi; 981 else /* trap == 0x200 */ 982 exsave = local_paca->exmc; 983 984 vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1; 985 vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2; 986 987 /* 988 * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate 989 * scratch (which we need to move into exsave to make re-entrant vs 990 * SRESET/MCE), register state is protected from reentrancy. However 991 * timebase, MMU, among other state is still set to guest, so don't 992 * enable MSR[RI] here. It gets enabled at the end, after in_guest 993 * is cleared. 994 * 995 * It is possible an NMI could come in here, which is why it is 996 * important to save the above state early so it can be debugged. 997 */ 998 999 vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)]; 1000 vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)]; 1001 vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)]; 1002 vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)]; 1003 vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)]; 1004 vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)]; 1005 vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)]; 1006 vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)]; 1007 1008 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; 1009 1010 if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) { 1011 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 1012 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 1013 kvmppc_realmode_machine_check(vcpu); 1014 1015 } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) { 1016 kvmppc_p9_realmode_hmi_handler(vcpu); 1017 1018 } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) { 1019 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 1020 1021 } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) { 1022 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; 1023 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; 1024 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 1025 1026 } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) { 1027 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); 1028 1029 } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) { 1030 vcpu->arch.hfscr = mfspr(SPRN_HFSCR); 1031 1032 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1033 /* 1034 * Softpatch interrupt for transactional memory emulation cases 1035 * on POWER9 DD2.2. This is early in the guest exit path - we 1036 * haven't saved registers or done a treclaim yet. 1037 */ 1038 } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) { 1039 vcpu->arch.emul_inst = mfspr(SPRN_HEIR); 1040 1041 /* 1042 * The cases we want to handle here are those where the guest 1043 * is in real suspend mode and is trying to transition to 1044 * transactional mode. 1045 */ 1046 if (!local_paca->kvm_hstate.fake_suspend && 1047 (vcpu->arch.shregs.msr & MSR_TS_S)) { 1048 if (kvmhv_p9_tm_emulation_early(vcpu)) { 1049 /* 1050 * Go straight back into the guest with the 1051 * new NIP/MSR as set by TM emulation. 1052 */ 1053 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); 1054 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr); 1055 goto tm_return_to_guest; 1056 } 1057 } 1058 #endif 1059 } 1060 1061 accumulate_time(vcpu, &vcpu->arch.rm_exit); 1062 1063 /* Advance host PURR/SPURR by the amount used by guest */ 1064 purr = mfspr(SPRN_PURR); 1065 spurr = mfspr(SPRN_SPURR); 1066 local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr; 1067 local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr; 1068 vcpu->arch.purr = purr; 1069 vcpu->arch.spurr = spurr; 1070 1071 vcpu->arch.ic = mfspr(SPRN_IC); 1072 vcpu->arch.pid = mfspr(SPRN_PID); 1073 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); 1074 1075 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); 1076 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); 1077 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); 1078 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); 1079 1080 dpdes = mfspr(SPRN_DPDES); 1081 if (dpdes) 1082 vcpu->arch.doorbell_request = 1; 1083 1084 vc->vtb = mfspr(SPRN_VTB); 1085 1086 dec = mfspr(SPRN_DEC); 1087 if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ 1088 dec = (s32) dec; 1089 *tb = mftb(); 1090 vcpu->arch.dec_expires = dec + *tb; 1091 1092 if (vc->tb_offset_applied) { 1093 u64 new_tb = *tb - vc->tb_offset_applied; 1094 mtspr(SPRN_TBU40, new_tb); 1095 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { 1096 new_tb += 0x1000000; 1097 mtspr(SPRN_TBU40, new_tb); 1098 } 1099 *tb = new_tb; 1100 vc->tb_offset_applied = 0; 1101 } 1102 1103 save_clear_guest_mmu(kvm, vcpu); 1104 switch_mmu_to_host(kvm, host_pidr); 1105 1106 /* 1107 * Enable MSR here in order to have facilities enabled to save 1108 * guest registers. This enables MMU (if we were in realmode), so 1109 * only switch MMU on after the MMU is switched to host, to avoid 1110 * the P9_RADIX_PREFETCH_BUG or hash guest context. 1111 */ 1112 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && 1113 vcpu->arch.shregs.msr & MSR_TS_MASK) 1114 msr |= MSR_TS_S; 1115 __mtmsrd(msr, 0); 1116 1117 store_vcpu_state(vcpu); 1118 1119 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr); 1120 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr); 1121 1122 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { 1123 /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ 1124 mtspr(SPRN_PSSCR, host_hpsscr | 1125 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); 1126 } 1127 1128 mtspr(SPRN_HFSCR, host_hfscr); 1129 if (vcpu->arch.ciabr != host_ciabr) 1130 mtspr(SPRN_CIABR, host_ciabr); 1131 1132 if (dawr_enabled()) { 1133 if (vcpu->arch.dawr0 != host_dawr0) 1134 mtspr(SPRN_DAWR0, host_dawr0); 1135 if (vcpu->arch.dawrx0 != host_dawrx0) 1136 mtspr(SPRN_DAWRX0, host_dawrx0); 1137 if (cpu_has_feature(CPU_FTR_DAWR1)) { 1138 if (vcpu->arch.dawr1 != host_dawr1) 1139 mtspr(SPRN_DAWR1, host_dawr1); 1140 if (vcpu->arch.dawrx1 != host_dawrx1) 1141 mtspr(SPRN_DAWRX1, host_dawrx1); 1142 } 1143 } 1144 1145 if (dpdes) 1146 mtspr(SPRN_DPDES, 0); 1147 if (vc->pcr) 1148 mtspr(SPRN_PCR, PCR_MASK); 1149 1150 /* HDEC must be at least as large as DEC, so decrementer_max fits */ 1151 mtspr(SPRN_HDEC, decrementer_max); 1152 1153 timer_rearm_host_dec(*tb); 1154 1155 restore_p9_host_os_sprs(vcpu, &host_os_sprs); 1156 1157 barrier(); /* Close in_guest critical section */ 1158 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE); 1159 /* Interrupts are recoverable at this point */ 1160 1161 /* 1162 * cp_abort is required if the processor supports local copy-paste 1163 * to clear the copy buffer that was under control of the guest. 1164 */ 1165 if (cpu_has_feature(CPU_FTR_ARCH_31)) 1166 asm volatile(PPC_CP_ABORT); 1167 1168 out: 1169 end_timing(vcpu); 1170 1171 return trap; 1172 } 1173 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9); 1174