1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <kvm/arm_hypercalls.h> 8 #include <kvm/arm_psci.h> 9 10 #include <linux/kvm_host.h> 11 #include <linux/mm.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/spectre.h> 15 16 #include <nvhe/alloc.h> 17 #include <nvhe/mem_protect.h> 18 #include <nvhe/memory.h> 19 #include <nvhe/pkvm.h> 20 #include <nvhe/trap_handler.h> 21 22 /* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */ 23 enum mitigation_state spectre_v2_state; 24 enum mitigation_state spectre_v4_state; 25 enum mitigation_state spectre_bhb_state; 26 27 /* Used by icache_is_aliasing(). */ 28 unsigned long __icache_flags; 29 30 /* Used by kvm_get_vttbr(). */ 31 unsigned int kvm_arm_vmid_bits; 32 33 unsigned int kvm_host_sve_max_vl; 34 35 /* 36 * The currently loaded hyp vCPU for each physical CPU. Used in protected mode 37 * for both protected and non-protected VMs. 38 */ 39 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); 40 41 /* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */ 42 static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) 43 { 44 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; 45 46 if (has_hvhe()) 47 vcpu->arch.hcr_el2 |= HCR_E2H; 48 49 if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { 50 /* route synchronous external abort exceptions to EL2 */ 51 vcpu->arch.hcr_el2 |= HCR_TEA; 52 /* trap error record accesses */ 53 vcpu->arch.hcr_el2 |= HCR_TERR; 54 } 55 56 if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 57 vcpu->arch.hcr_el2 |= HCR_FWB; 58 59 /* 60 * Without AArch32 EL1, leave RW set and let the entry fail with an 61 * illegal exception return: the *32_EL2 registers EL2 would otherwise 62 * switch are UNDEFINED there. 63 */ 64 if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT) && 65 cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) 66 vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; 67 68 if (vcpu_has_ptrauth(vcpu)) 69 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); 70 } 71 72 static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) 73 { 74 struct kvm *kvm = vcpu->kvm; 75 u64 val = vcpu->arch.hcr_el2; 76 77 /* No support for AArch32. */ 78 val |= HCR_RW; 79 80 /* 81 * Always trap: 82 * - Feature id registers: to control features exposed to guests 83 * - Implementation-defined features 84 */ 85 val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1; 86 87 if (cpus_have_final_cap(ARM64_HAS_EVT) && 88 !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && 89 kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) 90 val |= HCR_EL2_TID4; 91 else 92 val |= HCR_EL2_TID2; 93 94 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { 95 val |= HCR_TERR | HCR_TEA; 96 val &= ~(HCR_FIEN); 97 } 98 99 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP)) 100 val &= ~(HCR_AMVOFFEN); 101 102 if (!kvm_has_mte(kvm)) { 103 val |= HCR_TID5; 104 val &= ~(HCR_DCT | HCR_ATA); 105 } 106 107 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 108 val |= HCR_TLOR; 109 110 vcpu->arch.hcr_el2 = val; 111 } 112 113 static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu) 114 { 115 struct kvm *kvm = vcpu->kvm; 116 u64 val = vcpu->arch.mdcr_el2; 117 118 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) { 119 val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; 120 val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME); 121 } 122 123 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP)) 124 val |= MDCR_EL2_TDRA | MDCR_EL2_TDA; 125 126 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 127 val |= MDCR_EL2_TDOSA; 128 129 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) { 130 val |= MDCR_EL2_TPMS; 131 val &= ~MDCR_EL2_E2PB_MASK; 132 } 133 134 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 135 val |= MDCR_EL2_TTRF; 136 137 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 138 val &= ~MDCR_EL2_E2TB_MASK; 139 140 /* Trap Debug Communications Channel registers */ 141 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP)) 142 val |= MDCR_EL2_TDCC; 143 144 vcpu->arch.mdcr_el2 = val; 145 } 146 147 /* 148 * Check that cpu features that are neither trapped nor supported are not 149 * enabled for protected VMs. 150 */ 151 static int pkvm_check_pvm_cpu_features(struct kvm_vcpu *vcpu) 152 { 153 struct kvm *kvm = vcpu->kvm; 154 155 /* No AArch32 support for protected guests. */ 156 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL0, AARCH32) || 157 kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL1, AARCH32)) 158 return -EINVAL; 159 160 /* 161 * Linux guests assume support for floating-point and Advanced SIMD. Do 162 * not change the trapping behavior for these from the KVM default. 163 */ 164 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, FP, IMP) || 165 !kvm_has_feat(kvm, ID_AA64PFR0_EL1, AdvSIMD, IMP)) 166 return -EINVAL; 167 168 /* No SME support in KVM right now. Check to catch if it changes. */ 169 if (kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 170 return -EINVAL; 171 172 return 0; 173 } 174 175 /* 176 * Initialize trap register values in protected mode. 177 */ 178 static int pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu) 179 { 180 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 181 int ret; 182 183 vcpu->arch.mdcr_el2 = 0; 184 185 pkvm_vcpu_reset_hcr(vcpu); 186 187 if ((!pkvm_hyp_vcpu_is_protected(hyp_vcpu))) { 188 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 189 190 /* Trust the host for non-protected vcpu features. */ 191 vcpu->arch.hcrx_el2 = host_vcpu->arch.hcrx_el2; 192 return 0; 193 } 194 195 ret = pkvm_check_pvm_cpu_features(vcpu); 196 if (ret) 197 return ret; 198 199 pvm_init_traps_hcr(vcpu); 200 pvm_init_traps_mdcr(vcpu); 201 vcpu_set_hcrx(vcpu); 202 203 return 0; 204 } 205 206 /* 207 * Start the VM table handle at the offset defined instead of at 0. 208 * Mainly for sanity checking and debugging. 209 */ 210 #define HANDLE_OFFSET 0x1000 211 212 /* 213 * Marks a reserved but not yet used entry in the VM table. 214 */ 215 #define RESERVED_ENTRY ((void *)0xa110ca7ed) 216 217 static unsigned int vm_handle_to_idx(pkvm_handle_t handle) 218 { 219 return handle - HANDLE_OFFSET; 220 } 221 222 static pkvm_handle_t idx_to_vm_handle(unsigned int idx) 223 { 224 return idx + HANDLE_OFFSET; 225 } 226 227 /* 228 * Spinlock for protecting state related to the VM table. Protects writes 229 * to 'vm_table', 'nr_table_entries', and other per-vm state on initialization. 230 * Also protects reads and writes to 'last_hyp_vcpu_lookup'. 231 */ 232 DEFINE_HYP_SPINLOCK(vm_table_lock); 233 234 /* 235 * A table that tracks all VMs in protected mode. 236 * Allocated during hyp initialization and setup. 237 */ 238 static struct pkvm_hyp_vm **vm_table; 239 240 void pkvm_hyp_vm_table_init(void *tbl) 241 { 242 BUILD_BUG_ON((u64)HANDLE_OFFSET + KVM_MAX_PVMS > (pkvm_handle_t)-1); 243 WARN_ON(vm_table); 244 vm_table = tbl; 245 } 246 247 /* 248 * Return the hyp vm structure corresponding to the handle. 249 */ 250 struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle) 251 { 252 unsigned int idx = vm_handle_to_idx(handle); 253 254 hyp_assert_lock_held(&vm_table_lock); 255 256 if (unlikely(idx >= KVM_MAX_PVMS)) 257 return NULL; 258 259 /* A reserved entry doesn't represent an initialized VM. */ 260 if (unlikely(vm_table[idx] == RESERVED_ENTRY)) 261 return NULL; 262 263 return vm_table[idx]; 264 } 265 266 struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, 267 unsigned int vcpu_idx) 268 { 269 struct pkvm_hyp_vcpu *hyp_vcpu = NULL; 270 struct pkvm_hyp_vm *hyp_vm; 271 272 /* Cannot load a new vcpu without putting the old one first. */ 273 if (__this_cpu_read(loaded_hyp_vcpu)) 274 return NULL; 275 276 hyp_spin_lock(&vm_table_lock); 277 hyp_vm = get_vm_by_handle(handle); 278 if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying) 279 goto unlock; 280 281 if (hyp_vm->kvm.created_vcpus <= vcpu_idx) 282 goto unlock; 283 284 /* Pairs with smp_store_release() in register_hyp_vcpu(). */ 285 hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[vcpu_idx]); 286 if (!hyp_vcpu) 287 goto unlock; 288 289 /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */ 290 if (unlikely(hyp_vcpu->loaded_hyp_vcpu)) { 291 hyp_vcpu = NULL; 292 goto unlock; 293 } 294 295 hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu); 296 pkvm_hyp_vm_ref_inc(hyp_vm); 297 unlock: 298 hyp_spin_unlock(&vm_table_lock); 299 300 if (hyp_vcpu) 301 __this_cpu_write(loaded_hyp_vcpu, hyp_vcpu); 302 return hyp_vcpu; 303 } 304 305 void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 306 { 307 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 308 309 hyp_spin_lock(&vm_table_lock); 310 hyp_vcpu->loaded_hyp_vcpu = NULL; 311 __this_cpu_write(loaded_hyp_vcpu, NULL); 312 pkvm_hyp_vm_ref_dec(hyp_vm); 313 hyp_spin_unlock(&vm_table_lock); 314 } 315 316 struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void) 317 { 318 return __this_cpu_read(loaded_hyp_vcpu); 319 320 } 321 322 static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu) 323 { 324 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 325 326 if (hyp_vcpu && 327 (vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu)) 328 return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 329 330 return NULL; 331 } 332 333 /* 334 * A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at 335 * EL2 only while pinned, so it's read once and pinned; the pin fails for 336 * memory the host isn't sharing. 337 */ 338 struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu) 339 { 340 struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu); 341 struct kvm *kvm; 342 343 if (hyp_vm) 344 return &hyp_vm->kvm; 345 346 kvm = kern_hyp_va(READ_ONCE(vcpu->kvm)); 347 if (hyp_pin_shared_mem(kvm, kvm + 1)) 348 return NULL; 349 350 return kvm; 351 } 352 353 void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) 354 { 355 if (kvm && !pkvm_get_loaded_hyp_vm(vcpu)) 356 hyp_unpin_shared_mem(kvm, kvm + 1); 357 } 358 359 struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) 360 { 361 struct pkvm_hyp_vm *hyp_vm; 362 363 hyp_spin_lock(&vm_table_lock); 364 hyp_vm = get_vm_by_handle(handle); 365 if (hyp_vm) 366 pkvm_hyp_vm_ref_inc(hyp_vm); 367 hyp_spin_unlock(&vm_table_lock); 368 369 return hyp_vm; 370 } 371 372 void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm) 373 { 374 hyp_spin_lock(&vm_table_lock); 375 pkvm_hyp_vm_ref_dec(hyp_vm); 376 hyp_spin_unlock(&vm_table_lock); 377 } 378 379 struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle) 380 { 381 struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); 382 383 if (hyp_vm && pkvm_hyp_vm_is_protected(hyp_vm)) { 384 put_pkvm_hyp_vm(hyp_vm); 385 hyp_vm = NULL; 386 } 387 388 return hyp_vm; 389 } 390 391 static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struct kvm *host_kvm) 392 { 393 struct kvm *kvm = &hyp_vm->kvm; 394 unsigned long host_arch_flags = READ_ONCE(host_kvm->arch.flags); 395 DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES); 396 397 /* CTR_EL0 is always under host control, even for protected VMs. */ 398 hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0; 399 400 /* Preserve the vgic model so that GICv3 emulation works */ 401 hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model; 402 403 /* No restrictions for non-protected VMs. */ 404 if (!kvm_vm_is_protected(kvm)) { 405 hyp_vm->kvm.arch.flags = host_arch_flags; 406 hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED); 407 408 bitmap_copy(kvm->arch.vcpu_features, 409 host_kvm->arch.vcpu_features, 410 KVM_VCPU_MAX_FEATURES); 411 412 if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) 413 hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; 414 415 goto out; 416 } 417 418 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) 419 kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED); 420 421 kvm_pkvm_vcpu_allowed_features(kvm, allowed_features); 422 423 bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, 424 allowed_features, KVM_VCPU_MAX_FEATURES); 425 out: 426 __assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags, 427 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); 428 } 429 430 static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state) 431 { 432 struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; 433 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 434 struct kvm_vcpu *host_vcpu; 435 436 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 437 /* The host manages a non-protected vCPU: always ON at EL2. */ 438 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON; 439 return 0; 440 } 441 442 if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) 443 return -EINVAL; 444 445 if (mp_state == KVM_MP_STATE_STOPPED) { 446 reset_state->reset = false; 447 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; 448 return 0; 449 } 450 451 hyp_assert_lock_held(&vm_table_lock); 452 if (hyp_vm->primary_vcpu) 453 return -EINVAL; 454 hyp_vm->primary_vcpu = hyp_vcpu; 455 456 host_vcpu = hyp_vcpu->host_vcpu; 457 reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu)); 458 reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); 459 reset_state->reset = true; 460 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; 461 return 0; 462 } 463 464 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) 465 { 466 if (host_vcpu) 467 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); 468 } 469 470 static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu) 471 { 472 void *sve_state; 473 474 sve_state = hyp_vcpu->vcpu.arch.sve_state; 475 if (!sve_state) 476 return; 477 478 hyp_unpin_shared_mem(sve_state, 479 sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu)); 480 } 481 482 static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], 483 unsigned int nr_vcpus) 484 { 485 int i; 486 487 for (i = 0; i < nr_vcpus; i++) { 488 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i]; 489 490 if (!hyp_vcpu) 491 continue; 492 493 unpin_host_vcpu(hyp_vcpu->host_vcpu); 494 unpin_host_sve_state(hyp_vcpu); 495 } 496 } 497 498 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, 499 unsigned int nr_vcpus, pkvm_handle_t handle) 500 { 501 struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 502 int idx = vm_handle_to_idx(handle); 503 504 hyp_vm->kvm.arch.pkvm.handle = handle; 505 506 hyp_vm->host_kvm = host_kvm; 507 hyp_vm->kvm.created_vcpus = nr_vcpus; 508 if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) 509 hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; 510 else 511 hyp_vm->kvm.arch.vm_flavor = VM_PKVM; 512 513 hyp_vm->kvm.arch.flags = 0; 514 pkvm_init_features_from_host(hyp_vm, host_kvm); 515 516 /* VMID 0 is reserved for the host */ 517 atomic64_set(&mmu->vmid.id, idx + 1); 518 519 mmu->vtcr = host_mmu.arch.mmu.vtcr; 520 mmu->arch = &hyp_vm->kvm.arch; 521 mmu->pgt = &hyp_vm->pgt; 522 } 523 524 static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) 525 { 526 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 527 unsigned int sve_max_vl; 528 size_t sve_state_size; 529 void *sve_state; 530 int ret; 531 532 if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) { 533 vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED); 534 return 0; 535 } 536 537 /* Limit guest vector length to the maximum supported by the host. */ 538 sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); 539 sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); 540 541 if (!sve_vl_valid(sve_max_vl) || !sve_state) 542 return -EINVAL; 543 544 sve_state_size = sve_state_size_from_vl(sve_max_vl); 545 546 ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); 547 if (ret) 548 return ret; 549 550 vcpu->arch.sve_state = sve_state; 551 vcpu->arch.sve_max_vl = sve_max_vl; 552 553 return 0; 554 } 555 556 static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) 557 { 558 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 559 const struct kvm *host_kvm = hyp_vm->host_kvm; 560 struct kvm *kvm = &hyp_vm->kvm; 561 562 if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &host_kvm->arch.flags)) 563 return -EINVAL; 564 565 if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) 566 return 0; 567 568 memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs)); 569 570 return 0; 571 } 572 573 static int pkvm_vcpu_init_sysregs(struct pkvm_hyp_vcpu *hyp_vcpu) 574 { 575 int ret = 0; 576 577 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 578 kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); 579 else 580 ret = vm_copy_id_regs(hyp_vcpu); 581 582 return ret; 583 } 584 585 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, 586 struct pkvm_hyp_vm *hyp_vm, 587 struct kvm_vcpu *host_vcpu) 588 { 589 int ret = 0; 590 u32 mp_state; 591 592 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) 593 return -EBUSY; 594 595 mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); 596 hyp_vcpu->host_vcpu = host_vcpu; 597 598 hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; 599 hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); 600 hyp_vcpu->vcpu.vcpu_idx = READ_ONCE(host_vcpu->vcpu_idx); 601 602 hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; 603 hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); 604 605 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 606 /* 607 * Timer offsets are pointing to the untrusted KVM copy, 608 * which is pinned in __pkvm_init_vm() for the VM life time. 609 * It is worth noting that hyp_vm->host_kvm points to an EL2 610 * linear map address and timer_get_offset() will use 611 * kern_hyp_va() which is safe as it is idempotent. 612 */ 613 vcpu_vtimer(&hyp_vcpu->vcpu)->offset.vm_offset = 614 &hyp_vm->host_kvm->arch.timer_data.voffset; 615 vcpu_ptimer(&hyp_vcpu->vcpu)->offset.vm_offset = 616 &hyp_vm->host_kvm->arch.timer_data.poffset; 617 } 618 619 ret = pkvm_vcpu_init_sysregs(hyp_vcpu); 620 if (ret) 621 goto done; 622 623 ret = pkvm_vcpu_init_traps(hyp_vcpu); 624 if (ret) 625 goto done; 626 627 ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu); 628 if (ret) 629 goto done; 630 631 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 632 kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); 633 ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state); 634 done: 635 if (ret) { 636 unpin_host_vcpu(host_vcpu); 637 unpin_host_sve_state(hyp_vcpu); 638 } 639 return ret; 640 } 641 642 static int find_free_vm_table_entry(void) 643 { 644 int i; 645 646 for (i = 0; i < KVM_MAX_PVMS; ++i) { 647 if (!vm_table[i]) 648 return i; 649 } 650 651 return -ENOMEM; 652 } 653 654 /* 655 * Reserve a VM table entry. 656 * 657 * Return a unique handle to the VM on success, 658 * negative error code on failure. 659 */ 660 static int allocate_vm_table_entry(void) 661 { 662 int idx; 663 664 hyp_assert_lock_held(&vm_table_lock); 665 666 /* 667 * Initializing protected state might have failed, yet a malicious 668 * host could trigger this function. Thus, ensure that 'vm_table' 669 * exists. 670 */ 671 if (unlikely(!vm_table)) 672 return -EINVAL; 673 674 idx = find_free_vm_table_entry(); 675 if (unlikely(idx < 0)) 676 return idx; 677 678 vm_table[idx] = RESERVED_ENTRY; 679 680 return idx; 681 } 682 683 static int __insert_vm_table_entry(pkvm_handle_t handle, 684 struct pkvm_hyp_vm *hyp_vm) 685 { 686 unsigned int idx; 687 688 hyp_assert_lock_held(&vm_table_lock); 689 690 /* 691 * Initializing protected state might have failed, yet a malicious 692 * host could trigger this function. Thus, ensure that 'vm_table' 693 * exists. 694 */ 695 if (unlikely(!vm_table)) 696 return -EINVAL; 697 698 idx = vm_handle_to_idx(handle); 699 if (unlikely(idx >= KVM_MAX_PVMS)) 700 return -EINVAL; 701 702 if (unlikely(vm_table[idx] != RESERVED_ENTRY)) 703 return -EINVAL; 704 705 vm_table[idx] = hyp_vm; 706 707 return 0; 708 } 709 710 /* 711 * Insert a pointer to the initialized VM into the VM table. 712 * 713 * Return 0 on success, or negative error code on failure. 714 */ 715 static int insert_vm_table_entry(pkvm_handle_t handle, 716 struct pkvm_hyp_vm *hyp_vm) 717 { 718 int ret; 719 720 hyp_spin_lock(&vm_table_lock); 721 ret = __insert_vm_table_entry(handle, hyp_vm); 722 hyp_spin_unlock(&vm_table_lock); 723 724 return ret; 725 } 726 727 /* 728 * Deallocate and remove the VM table entry corresponding to the handle. 729 */ 730 static void remove_vm_table_entry(pkvm_handle_t handle) 731 { 732 hyp_assert_lock_held(&vm_table_lock); 733 vm_table[vm_handle_to_idx(handle)] = NULL; 734 } 735 736 static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) 737 { 738 return size_add(sizeof(struct pkvm_hyp_vm), 739 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); 740 } 741 742 static void *map_donated_memory_noclear(void __kern *host_va, size_t size) 743 { 744 void *va = kern_hyp_va_host(host_va); 745 746 if (!PAGE_ALIGNED(va)) 747 return NULL; 748 749 if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va), 750 PAGE_ALIGN(size) >> PAGE_SHIFT)) 751 return NULL; 752 753 return va; 754 } 755 756 static void __unmap_donated_memory(void *va, size_t size) 757 { 758 kvm_flush_dcache_to_poc(va, size); 759 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va), 760 PAGE_ALIGN(size) >> PAGE_SHIFT)); 761 } 762 763 static void unmap_donated_memory(void *va, size_t size) 764 { 765 if (!va) 766 return; 767 768 memset(va, 0, size); 769 __unmap_donated_memory(va, size); 770 } 771 772 static void unmap_donated_memory_noclear(void *va, size_t size) 773 { 774 if (!va) 775 return; 776 777 __unmap_donated_memory(va, size); 778 } 779 780 /* 781 * Reserves an entry in the hypervisor for a new VM in protected mode. 782 * 783 * Return a unique handle to the VM on success, negative error code on failure. 784 */ 785 int __pkvm_reserve_vm(void) 786 { 787 int ret; 788 789 hyp_spin_lock(&vm_table_lock); 790 ret = allocate_vm_table_entry(); 791 hyp_spin_unlock(&vm_table_lock); 792 793 if (ret < 0) 794 return ret; 795 796 return idx_to_vm_handle(ret); 797 } 798 799 /* 800 * Removes a reserved entry, but only if is hasn't been used yet. 801 * Otherwise, the VM needs to be destroyed. 802 */ 803 void __pkvm_unreserve_vm(pkvm_handle_t handle) 804 { 805 unsigned int idx = vm_handle_to_idx(handle); 806 807 if (unlikely(!vm_table)) 808 return; 809 810 hyp_spin_lock(&vm_table_lock); 811 if (likely(idx < KVM_MAX_PVMS && vm_table[idx] == RESERVED_ENTRY)) 812 remove_vm_table_entry(handle); 813 hyp_spin_unlock(&vm_table_lock); 814 } 815 816 #ifdef CONFIG_NVHE_EL2_DEBUG 817 static struct pkvm_hyp_vm selftest_vm = { 818 .kvm = { 819 .arch = { 820 .mmu = { 821 .arch = &selftest_vm.kvm.arch, 822 .pgt = &selftest_vm.pgt, 823 }, 824 }, 825 }, 826 }; 827 828 static struct pkvm_hyp_vcpu selftest_vcpu = { 829 .vcpu = { 830 .arch = { 831 .hw_mmu = &selftest_vm.kvm.arch.mmu, 832 }, 833 .kvm = &selftest_vm.kvm, 834 }, 835 }; 836 837 struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) 838 { 839 struct hyp_page *p = hyp_virt_to_page(virt); 840 unsigned long min_pages, seeded = 0; 841 int i; 842 843 selftest_vm.kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr; 844 WARN_ON(kvm_guest_prepare_stage2(&selftest_vm, virt)); 845 846 /* 847 * Mirror pkvm_refill_memcache() for the share/donate pre-checks; 848 * the selftest invokes those functions directly and would 849 * otherwise see an empty memcache. 850 */ 851 min_pages = kvm_mmu_cache_min_pages(&selftest_vm.kvm.arch.mmu); 852 853 for (i = 0; i < pkvm_selftest_pages(); i++) { 854 if (p[i].refcount) 855 continue; 856 p[i].refcount = 1; 857 if (seeded < min_pages) { 858 push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc, 859 hyp_page_to_virt(&p[i]), hyp_virt_to_phys); 860 seeded++; 861 } else { 862 hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i])); 863 } 864 } 865 866 selftest_vm.kvm.arch.pkvm.handle = __pkvm_reserve_vm(); 867 insert_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle, &selftest_vm); 868 return &selftest_vcpu; 869 } 870 871 void teardown_selftest_vm(void) 872 { 873 hyp_spin_lock(&vm_table_lock); 874 remove_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle); 875 hyp_spin_unlock(&vm_table_lock); 876 } 877 #endif /* CONFIG_NVHE_EL2_DEBUG */ 878 879 /* 880 * Initialize the hypervisor copy of the VM state using host-donated memory. 881 * 882 * Unmap the donated memory from the host at stage 2. 883 * 884 * host_kvm: A pointer to the host's struct kvm. 885 * pgd_hva: The host va of the area being donated for the stage-2 PGD for 886 * the VM. Must be page aligned. Its size is implied by the VM's 887 * VTCR. 888 * 889 * Return 0 success, negative error code on failure. 890 */ 891 int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva) 892 { 893 struct pkvm_hyp_vm *hyp_vm = NULL; 894 size_t vm_size, pgd_size; 895 unsigned int nr_vcpus; 896 pkvm_handle_t handle; 897 void *pgd = NULL; 898 int ret; 899 900 ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1); 901 if (ret) 902 return ret; 903 904 nr_vcpus = READ_ONCE(host_kvm->created_vcpus); 905 if (nr_vcpus < 1) { 906 ret = -EINVAL; 907 goto err_unpin_kvm; 908 } 909 910 handle = READ_ONCE(host_kvm->arch.pkvm.handle); 911 if (unlikely(handle < HANDLE_OFFSET)) { 912 ret = -EINVAL; 913 goto err_unpin_kvm; 914 } 915 916 vm_size = pkvm_get_hyp_vm_size(nr_vcpus); 917 pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); 918 919 hyp_vm = hyp_alloc(vm_size); 920 if (!hyp_vm) { 921 ret = hyp_alloc_errno(); 922 goto err_unpin_kvm; 923 } 924 925 ret = -ENOMEM; 926 927 pgd = map_donated_memory_noclear(pgd_hva, pgd_size); 928 if (!pgd) 929 goto err_free_hyp_vm; 930 931 init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); 932 933 ret = kvm_guest_prepare_stage2(hyp_vm, pgd); 934 if (ret) 935 goto err_remove_mappings; 936 937 /* Must be called last since this publishes the VM. */ 938 ret = insert_vm_table_entry(handle, hyp_vm); 939 if (ret) 940 goto err_destroy_stage2; 941 942 return 0; 943 944 err_destroy_stage2: 945 kvm_guest_destroy_stage2(hyp_vm); 946 err_remove_mappings: 947 unmap_donated_memory(pgd, pgd_size); 948 err_free_hyp_vm: 949 hyp_free(hyp_vm); 950 err_unpin_kvm: 951 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 952 return ret; 953 } 954 955 /* 956 * Initialize the hypervisor copy of the vCPU state using host-donated memory. 957 * 958 * handle: The hypervisor handle for the vm. 959 * host_vcpu: A pointer to the corresponding host vcpu. 960 * vcpu_hva: The host va of the area being donated for the vcpu state. 961 * Must be page aligned. The size of the area must be equal to 962 * the page-aligned size of 'struct pkvm_hyp_vcpu'. 963 * Return 0 on success, negative error code on failure. 964 */ 965 static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, 966 struct pkvm_hyp_vcpu *hyp_vcpu) 967 { 968 unsigned int idx = hyp_vcpu->vcpu.vcpu_idx; 969 970 if (idx >= hyp_vm->kvm.created_vcpus) 971 return -EINVAL; 972 973 if (hyp_vm->vcpus[idx]) 974 return -EINVAL; 975 976 /* 977 * Ensure the hyp_vcpu is initialised before publishing it to 978 * the vCPU-load path via 'hyp_vm->vcpus[]'. 979 */ 980 smp_store_release(&hyp_vm->vcpus[idx], hyp_vcpu); 981 return 0; 982 } 983 984 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) 985 { 986 struct pkvm_hyp_vcpu *hyp_vcpu; 987 struct pkvm_hyp_vm *hyp_vm; 988 int ret; 989 990 hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu)); 991 if (!hyp_vcpu) 992 return hyp_alloc_errno(); 993 994 hyp_spin_lock(&vm_table_lock); 995 996 hyp_vm = get_vm_by_handle(handle); 997 if (!hyp_vm) { 998 ret = -ENOENT; 999 goto unlock; 1000 } 1001 1002 ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); 1003 if (ret) 1004 goto unclaim; 1005 1006 ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); 1007 if (ret) { 1008 unpin_host_vcpu(host_vcpu); 1009 unpin_host_sve_state(hyp_vcpu); 1010 goto unclaim; 1011 } 1012 goto unlock; 1013 unclaim: 1014 /* 1015 * Under vm_table_lock, so no other claim can have landed: undo 1016 * this one. 1017 */ 1018 if (hyp_vm->primary_vcpu == hyp_vcpu) 1019 hyp_vm->primary_vcpu = NULL; 1020 unlock: 1021 hyp_spin_unlock(&vm_table_lock); 1022 if (ret) 1023 hyp_free(hyp_vcpu); 1024 1025 return ret; 1026 } 1027 1028 int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn) 1029 { 1030 struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); 1031 int ret = -EINVAL; 1032 1033 if (!hyp_vm) 1034 return ret; 1035 1036 if (hyp_vm->kvm.arch.pkvm.is_dying) 1037 ret = __pkvm_host_reclaim_page_guest(gfn, hyp_vm); 1038 1039 put_pkvm_hyp_vm(hyp_vm); 1040 return ret; 1041 } 1042 1043 static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle) 1044 { 1045 struct pkvm_hyp_vm *hyp_vm; 1046 1047 hyp_assert_lock_held(&vm_table_lock); 1048 1049 hyp_vm = get_vm_by_handle(handle); 1050 if (!hyp_vm || hyp_vm->refcount) 1051 return NULL; 1052 1053 return hyp_vm; 1054 } 1055 1056 int __pkvm_start_teardown_vm(pkvm_handle_t handle) 1057 { 1058 struct pkvm_hyp_vm *hyp_vm; 1059 int ret = 0; 1060 1061 hyp_spin_lock(&vm_table_lock); 1062 hyp_vm = get_pkvm_unref_hyp_vm_locked(handle); 1063 if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying) { 1064 ret = -EINVAL; 1065 goto unlock; 1066 } 1067 1068 hyp_vm->kvm.arch.pkvm.is_dying = true; 1069 unlock: 1070 hyp_spin_unlock(&vm_table_lock); 1071 1072 return ret; 1073 } 1074 1075 int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) 1076 { 1077 struct kvm_hyp_memcache *stage2_mc; 1078 struct pkvm_hyp_vm *hyp_vm; 1079 struct kvm *host_kvm; 1080 unsigned int idx; 1081 int err; 1082 1083 hyp_spin_lock(&vm_table_lock); 1084 hyp_vm = get_pkvm_unref_hyp_vm_locked(handle); 1085 if (!hyp_vm || !hyp_vm->kvm.arch.pkvm.is_dying) { 1086 err = -EINVAL; 1087 goto err_unlock; 1088 } 1089 1090 host_kvm = hyp_vm->host_kvm; 1091 1092 /* Ensure the VMID is clean before it can be reallocated */ 1093 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 1094 remove_vm_table_entry(handle); 1095 hyp_spin_unlock(&vm_table_lock); 1096 1097 /* Reclaim guest pages (including page-table pages) */ 1098 stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc; 1099 reclaim_pgtable_pages(hyp_vm, stage2_mc); 1100 unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); 1101 1102 /* Push the stage-2 pages to the teardown memcache */ 1103 for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { 1104 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; 1105 struct kvm_hyp_memcache *vcpu_mc; 1106 1107 if (!hyp_vcpu) 1108 continue; 1109 1110 vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; 1111 1112 while (vcpu_mc->nr_pages) { 1113 void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt); 1114 1115 push_hyp_memcache(stage2_mc, addr, hyp_virt_to_phys); 1116 unmap_donated_memory_noclear(addr, PAGE_SIZE); 1117 } 1118 1119 hyp_free(hyp_vcpu); 1120 } 1121 1122 hyp_free(hyp_vm); 1123 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 1124 return 0; 1125 1126 err_unlock: 1127 hyp_spin_unlock(&vm_table_lock); 1128 return err; 1129 } 1130 1131 static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa) 1132 { 1133 u64 elr; 1134 1135 /* Fake up a data abort (level 3 translation fault on write) */ 1136 vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | 1137 ESR_ELx_IL | ESR_ELx_WNR | 1138 ESR_ELx_FSC_FAULT | 1139 FIELD_PREP(ESR_ELx_FSC_LEVEL, 3); 1140 1141 /* Shuffle the IPA around into the HPFAR */ 1142 vcpu->arch.fault.hpfar_el2 = (HPFAR_EL2_NS | (ipa >> 8)) & HPFAR_MASK; 1143 1144 /* This is a virtual address. 0's good. Let's go with 0. */ 1145 vcpu->arch.fault.far_el2 = 0; 1146 1147 /* Rewind the ELR so we return to the HVC once the IPA is mapped */ 1148 elr = read_sysreg(elr_el2); 1149 elr -= 4; 1150 write_sysreg(elr, elr_el2); 1151 1152 return ARM_EXCEPTION_TRAP; 1153 } 1154 1155 static bool pkvm_memshare_call(u64 *ret, struct kvm_vcpu *vcpu, u64 *exit_code) 1156 { 1157 struct pkvm_hyp_vcpu *hyp_vcpu; 1158 u64 ipa = smccc_get_arg1(vcpu); 1159 1160 if (!PAGE_ALIGNED(ipa)) 1161 goto out_guest; 1162 1163 hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1164 switch (__pkvm_guest_share_host(hyp_vcpu, hyp_phys_to_pfn(ipa))) { 1165 case 0: 1166 ret[0] = SMCCC_RET_SUCCESS; 1167 goto out_guest; 1168 case -ENOENT: 1169 /* 1170 * Convert the exception into a data abort so that the page 1171 * being shared is mapped into the guest next time. 1172 */ 1173 *exit_code = __pkvm_memshare_page_req(vcpu, ipa); 1174 goto out_host; 1175 } 1176 1177 out_guest: 1178 return true; 1179 out_host: 1180 return false; 1181 } 1182 1183 static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) 1184 { 1185 struct pkvm_hyp_vcpu *hyp_vcpu; 1186 u64 ipa = smccc_get_arg1(vcpu); 1187 1188 if (!PAGE_ALIGNED(ipa)) 1189 return; 1190 1191 hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1192 if (!__pkvm_guest_unshare_host(hyp_vcpu, hyp_phys_to_pfn(ipa))) 1193 ret[0] = SMCCC_RET_SUCCESS; 1194 } 1195 1196 /* 1197 * Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the 1198 * target's own CPU. Returns -ECANCELED, with no side effects, if a rollback 1199 * raced the reset. 1200 * 1201 * The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on(): 1202 * 1203 * cmpxchg_relaxed(power_state): OFF -> ON_PENDING 1204 * <ctrl> 1205 * write_reset_state(); 1206 * smp_store_release(reset_state->reset, true); 1207 * 1208 * The target coming online here, running, then its PSCI CPU_OFF in 1209 * pvm_psci_vcpu_off(): 1210 * 1211 * smp_load_acquire(reset_state->reset) == true; 1212 * cmpxchg_relaxed(power_state): ON_PENDING -> ON; 1213 * read_reset_state(); 1214 * 1215 * <run guest> 1216 * 1217 * WRITE_ONCE(reset_state->reset, false); 1218 * smp_store_release(power_state, OFF); 1219 */ 1220 int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 1221 { 1222 struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; 1223 1224 /* 1225 * Pairs with smp_store_release(&reset_state->reset, true) in 1226 * pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a 1227 * winning cmpxchg with a false acquire would leave power_state == ON 1228 * with the reset skipped. 1229 */ 1230 if (!smp_load_acquire(&reset_state->reset)) 1231 return -ECANCELED; 1232 1233 if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, 1234 PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) 1235 return -ECANCELED; 1236 1237 kvm_reset_vcpu_core(&hyp_vcpu->vcpu); 1238 kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); 1239 1240 /* Must be done after resetting sys registers. */ 1241 kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state); 1242 1243 hyp_vcpu->exit_code = 0; 1244 return 0; 1245 } 1246 1247 struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr) 1248 { 1249 struct pkvm_hyp_vcpu *hyp_vcpu; 1250 int i; 1251 1252 mpidr &= MPIDR_HWID_BITMASK; 1253 1254 for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { 1255 /* Pairs with smp_store_release() in register_hyp_vcpu(). */ 1256 hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]); 1257 1258 if (!hyp_vcpu) 1259 continue; 1260 1261 if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) 1262 return hyp_vcpu; 1263 } 1264 1265 return NULL; 1266 } 1267 1268 /* 1269 * Returns true when handled at EL2, false when the host must wake the target 1270 * vCPU. 1271 */ 1272 static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu) 1273 { 1274 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 1275 struct vcpu_reset_state *reset_state; 1276 struct pkvm_hyp_vcpu *target; 1277 unsigned long cpu_id, ret; 1278 int power_state; 1279 1280 cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu); 1281 if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) { 1282 ret = PSCI_RET_INVALID_PARAMS; 1283 goto error; 1284 } 1285 1286 target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); 1287 if (!target) { 1288 ret = PSCI_RET_INVALID_PARAMS; 1289 goto error; 1290 } 1291 1292 /* 1293 * vCPUs race to power on the same target. The writes below depend on 1294 * the cmpxchg reading OFF, and that control dependency with the release 1295 * of OFF in pvm_psci_vcpu_off() orders them after the target's accesses 1296 * to reset_state, so relaxed suffices. 1297 */ 1298 power_state = cmpxchg_relaxed(&target->power_state, 1299 PSCI_0_2_AFFINITY_LEVEL_OFF, 1300 PSCI_0_2_AFFINITY_LEVEL_ON_PENDING); 1301 switch (power_state) { 1302 case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: 1303 ret = PSCI_RET_ON_PENDING; 1304 goto error; 1305 case PSCI_0_2_AFFINITY_LEVEL_ON: 1306 ret = PSCI_RET_ALREADY_ON; 1307 goto error; 1308 case PSCI_0_2_AFFINITY_LEVEL_OFF: 1309 break; 1310 default: 1311 ret = PSCI_RET_INTERNAL_FAILURE; 1312 goto error; 1313 } 1314 1315 reset_state = &target->vcpu.arch.reset_state; 1316 WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu)); 1317 WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu)); 1318 WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu)); 1319 /* 1320 * Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with 1321 * smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu(). 1322 */ 1323 smp_store_release(&reset_state->reset, true); 1324 1325 /* The host requests KVM_REQ_VCPU_RESET and wakes the target. */ 1326 return false; 1327 1328 error: 1329 smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0); 1330 return true; 1331 } 1332 1333 /* 1334 * Returns true when handled at EL2, false when the host must stop scheduling 1335 * the vCPU. 1336 */ 1337 static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu) 1338 { 1339 /* No other writer runs while this vCPU is ON and executing. */ 1340 WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON); 1341 1342 WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false); 1343 1344 /* 1345 * Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear 1346 * above, before OFF. Pairs with the control dependency in 1347 * pvm_psci_vcpu_on(). 1348 */ 1349 smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF); 1350 1351 /* Return to the host so that it can finish powering off the vcpu. */ 1352 return false; 1353 } 1354 1355 static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu) 1356 { 1357 /* Nothing to be handled by the host. Go back to the guest. */ 1358 smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0); 1359 return true; 1360 } 1361 1362 static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu) 1363 { 1364 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 1365 u32 feature = smccc_get_arg1(vcpu); 1366 unsigned long val; 1367 1368 switch (feature) { 1369 case PSCI_0_2_FN_PSCI_VERSION: 1370 case PSCI_0_2_FN_CPU_SUSPEND: 1371 case PSCI_0_2_FN64_CPU_SUSPEND: 1372 case PSCI_0_2_FN_CPU_OFF: 1373 case PSCI_0_2_FN_CPU_ON: 1374 case PSCI_0_2_FN64_CPU_ON: 1375 case PSCI_0_2_FN_AFFINITY_INFO: 1376 case PSCI_0_2_FN64_AFFINITY_INFO: 1377 case PSCI_0_2_FN_SYSTEM_OFF: 1378 case PSCI_0_2_FN_SYSTEM_RESET: 1379 case PSCI_1_0_FN_PSCI_FEATURES: 1380 case PSCI_1_1_FN_SYSTEM_RESET2: 1381 case PSCI_1_1_FN64_SYSTEM_RESET2: 1382 case ARM_SMCCC_VERSION_FUNC_ID: 1383 val = PSCI_RET_SUCCESS; 1384 break; 1385 default: 1386 val = PSCI_RET_NOT_SUPPORTED; 1387 break; 1388 } 1389 1390 /* Nothing to be handled by the host. Go back to the guest. */ 1391 smccc_set_retval(vcpu, val, 0, 0, 0); 1392 return true; 1393 } 1394 1395 static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu) 1396 { 1397 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 1398 u32 psci_fn = smccc_get_function(vcpu); 1399 1400 switch (psci_fn) { 1401 case PSCI_0_2_FN_CPU_ON: 1402 kvm_psci_narrow_to_32bit(vcpu); 1403 fallthrough; 1404 case PSCI_0_2_FN64_CPU_ON: 1405 return pvm_psci_vcpu_on(hyp_vcpu); 1406 case PSCI_0_2_FN_CPU_OFF: 1407 return pvm_psci_vcpu_off(hyp_vcpu); 1408 case PSCI_0_2_FN_PSCI_VERSION: 1409 return pvm_psci_version(hyp_vcpu); 1410 case PSCI_1_0_FN_PSCI_FEATURES: 1411 return pvm_psci_features(hyp_vcpu); 1412 case PSCI_0_2_FN_AFFINITY_INFO: 1413 case PSCI_0_2_FN64_AFFINITY_INFO: 1414 case PSCI_0_2_FN_SYSTEM_RESET: 1415 case PSCI_0_2_FN_CPU_SUSPEND: 1416 case PSCI_0_2_FN64_CPU_SUSPEND: 1417 case PSCI_0_2_FN_SYSTEM_OFF: 1418 case PSCI_1_1_FN_SYSTEM_RESET2: 1419 case PSCI_1_1_FN64_SYSTEM_RESET2: 1420 return false; /* Handled by the host. */ 1421 default: 1422 /* Unknown PSCI calls are handled here, not forwarded. */ 1423 smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0); 1424 return true; 1425 } 1426 } 1427 1428 /* 1429 * Handler for protected VM HVC calls. 1430 * 1431 * Returns true if the hypervisor has handled the exit (and control 1432 * should return to the guest) or false if it hasn't (and the handling 1433 * should be performed by the host). 1434 */ 1435 bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) 1436 { 1437 struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1438 u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; 1439 bool handled = true; 1440 u32 feature; 1441 uuid_t uuid; 1442 1443 switch (smccc_get_function(vcpu)) { 1444 case ARM_SMCCC_VERSION_FUNC_ID: 1445 /* Nothing to be handled by the host. Go back to the guest. */ 1446 val[0] = ARM_SMCCC_VERSION_1_1; 1447 break; 1448 case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: 1449 /* The workaround calls themselves are handled on hyp entry. */ 1450 val[0] = SMCCC_RET_NOT_SUPPORTED; 1451 feature = smccc_get_arg1(vcpu); 1452 switch (feature) { 1453 case ARM_SMCCC_VERSION_FUNC_ID: 1454 case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: 1455 val[0] = SMCCC_RET_SUCCESS; 1456 break; 1457 case ARM_SMCCC_ARCH_WORKAROUND_1: 1458 switch (spectre_v2_state) { 1459 case SPECTRE_VULNERABLE: 1460 break; 1461 case SPECTRE_MITIGATED: 1462 val[0] = SMCCC_RET_SUCCESS; 1463 break; 1464 case SPECTRE_UNAFFECTED: 1465 val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; 1466 break; 1467 } 1468 break; 1469 case ARM_SMCCC_ARCH_WORKAROUND_2: 1470 switch (spectre_v4_state) { 1471 case SPECTRE_VULNERABLE: 1472 break; 1473 case SPECTRE_MITIGATED: 1474 /* With SSBS advertised the guest's default is safe. */ 1475 if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP)) 1476 break; 1477 fallthrough; 1478 case SPECTRE_UNAFFECTED: 1479 val[0] = SMCCC_RET_NOT_REQUIRED; 1480 break; 1481 } 1482 break; 1483 case ARM_SMCCC_ARCH_WORKAROUND_3: 1484 switch (spectre_bhb_state) { 1485 case SPECTRE_VULNERABLE: 1486 break; 1487 case SPECTRE_MITIGATED: 1488 val[0] = SMCCC_RET_SUCCESS; 1489 break; 1490 case SPECTRE_UNAFFECTED: 1491 val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; 1492 break; 1493 } 1494 break; 1495 } 1496 break; 1497 case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: 1498 uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM; 1499 val[0] = smccc_uuid_to_reg(&uuid, 0); 1500 val[1] = smccc_uuid_to_reg(&uuid, 1); 1501 val[2] = smccc_uuid_to_reg(&uuid, 2); 1502 val[3] = smccc_uuid_to_reg(&uuid, 3); 1503 break; 1504 case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: 1505 val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); 1506 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); 1507 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_SHARE); 1508 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_UNSHARE); 1509 break; 1510 case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID: 1511 if (smccc_get_arg1(vcpu) || 1512 smccc_get_arg2(vcpu) || 1513 smccc_get_arg3(vcpu)) { 1514 break; 1515 } 1516 1517 val[0] = PAGE_SIZE; 1518 break; 1519 case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID: 1520 if (smccc_get_arg2(vcpu) || 1521 smccc_get_arg3(vcpu)) { 1522 break; 1523 } 1524 1525 handled = pkvm_memshare_call(val, vcpu, exit_code); 1526 break; 1527 case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID: 1528 if (smccc_get_arg2(vcpu) || 1529 smccc_get_arg3(vcpu)) { 1530 break; 1531 } 1532 1533 pkvm_memunshare_call(val, vcpu); 1534 break; 1535 default: 1536 return pkvm_handle_psci(hyp_vcpu); 1537 } 1538 1539 if (handled) 1540 smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]); 1541 return handled; 1542 } 1543