1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7 #include <kvm/arm_hypercalls.h> 8 #include <kvm/arm_psci.h> 9 10 #include <linux/kvm_host.h> 11 #include <linux/mm.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/spectre.h> 15 16 #include <nvhe/alloc.h> 17 #include <nvhe/mem_protect.h> 18 #include <nvhe/memory.h> 19 #include <nvhe/pkvm.h> 20 #include <nvhe/trap_handler.h> 21 22 /* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */ 23 enum mitigation_state spectre_v2_state; 24 enum mitigation_state spectre_v4_state; 25 enum mitigation_state spectre_bhb_state; 26 27 /* Used by icache_is_aliasing(). */ 28 unsigned long __icache_flags; 29 30 /* Used by kvm_get_vttbr(). */ 31 unsigned int kvm_arm_vmid_bits; 32 33 unsigned int kvm_host_sve_max_vl; 34 35 /* 36 * The currently loaded hyp vCPU for each physical CPU. Used in protected mode 37 * for both protected and non-protected VMs. 38 */ 39 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); 40 41 /* 42 * The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on 43 * each entry, and a protected vCPU's TWI and TWE at vCPU load. 44 */ 45 static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) 46 { 47 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; 48 49 if (has_hvhe()) 50 vcpu->arch.hcr_el2 |= HCR_EL2_E2H; 51 52 if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { 53 /* route synchronous external abort exceptions to EL2 */ 54 vcpu->arch.hcr_el2 |= HCR_EL2_TEA; 55 /* trap error record accesses */ 56 vcpu->arch.hcr_el2 |= HCR_EL2_TERR; 57 } 58 59 if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 60 vcpu->arch.hcr_el2 |= HCR_EL2_FWB; 61 62 /* 63 * Without AArch32 EL1, leave RW set and let the entry fail with an 64 * illegal exception return: the *32_EL2 registers EL2 would otherwise 65 * switch are UNDEFINED there. 66 */ 67 if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT) && 68 cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) 69 vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; 70 71 if (vcpu_has_ptrauth(vcpu)) 72 vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK); 73 } 74 75 static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) 76 { 77 struct kvm *kvm = vcpu->kvm; 78 u64 val = vcpu->arch.hcr_el2; 79 80 /* No support for AArch32. */ 81 val |= HCR_EL2_RW; 82 83 /* 84 * Always trap: 85 * - Feature id registers: to control features exposed to guests 86 * - Implementation-defined features 87 */ 88 val |= HCR_EL2_TACR | HCR_EL2_TIDCP | HCR_EL2_TID3 | HCR_EL2_TID1; 89 90 if (cpus_have_final_cap(ARM64_HAS_EVT) && 91 !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && 92 kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) 93 val |= HCR_EL2_TID4; 94 else 95 val |= HCR_EL2_TID2; 96 97 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { 98 val |= HCR_EL2_TERR | HCR_EL2_TEA; 99 val &= ~(HCR_EL2_FIEN); 100 } 101 102 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP)) 103 val &= ~(HCR_EL2_AMVOFFEN); 104 105 if (!kvm_has_mte(kvm)) { 106 val |= HCR_EL2_TID5; 107 val &= ~(HCR_EL2_DCT | HCR_EL2_ATA); 108 } 109 110 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 111 val |= HCR_EL2_TLOR; 112 113 vcpu->arch.hcr_el2 = val; 114 } 115 116 static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu) 117 { 118 struct kvm *kvm = vcpu->kvm; 119 u64 val = vcpu->arch.mdcr_el2; 120 121 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) { 122 val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; 123 val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME); 124 } 125 126 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP)) 127 val |= MDCR_EL2_TDRA | MDCR_EL2_TDA; 128 129 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 130 val |= MDCR_EL2_TDOSA; 131 132 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) { 133 val |= MDCR_EL2_TPMS; 134 val &= ~MDCR_EL2_E2PB_MASK; 135 } 136 137 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 138 val |= MDCR_EL2_TTRF; 139 140 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 141 val &= ~MDCR_EL2_E2TB_MASK; 142 143 /* Trap Debug Communications Channel registers */ 144 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP)) 145 val |= MDCR_EL2_TDCC; 146 147 vcpu->arch.mdcr_el2 = val; 148 } 149 150 /* 151 * Check that cpu features that are neither trapped nor supported are not 152 * enabled for protected VMs. 153 */ 154 static int pkvm_check_pvm_cpu_features(struct kvm_vcpu *vcpu) 155 { 156 struct kvm *kvm = vcpu->kvm; 157 158 /* No AArch32 support for protected guests. */ 159 if (kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL0, AARCH32) || 160 kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL1, AARCH32)) 161 return -EINVAL; 162 163 /* 164 * Linux guests assume support for floating-point and Advanced SIMD. Do 165 * not change the trapping behavior for these from the KVM default. 166 */ 167 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, FP, IMP) || 168 !kvm_has_feat(kvm, ID_AA64PFR0_EL1, AdvSIMD, IMP)) 169 return -EINVAL; 170 171 /* No SME support in KVM right now. Check to catch if it changes. */ 172 if (kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 173 return -EINVAL; 174 175 return 0; 176 } 177 178 /* 179 * Initialize trap register values in protected mode. 180 */ 181 static int pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu) 182 { 183 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 184 int ret; 185 186 vcpu->arch.mdcr_el2 = 0; 187 188 pkvm_vcpu_reset_hcr(vcpu); 189 190 if ((!pkvm_hyp_vcpu_is_protected(hyp_vcpu))) { 191 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 192 193 /* Trust the host for non-protected vcpu features. */ 194 vcpu->arch.hcrx_el2 = host_vcpu->arch.hcrx_el2; 195 return 0; 196 } 197 198 ret = pkvm_check_pvm_cpu_features(vcpu); 199 if (ret) 200 return ret; 201 202 pvm_init_traps_hcr(vcpu); 203 pvm_init_traps_mdcr(vcpu); 204 vcpu_set_hcrx(vcpu); 205 206 return 0; 207 } 208 209 /* 210 * Start the VM table handle at the offset defined instead of at 0. 211 * Mainly for sanity checking and debugging. 212 */ 213 #define HANDLE_OFFSET 0x1000 214 215 /* 216 * Marks a reserved but not yet used entry in the VM table. 217 */ 218 #define RESERVED_ENTRY ((void *)0xa110ca7ed) 219 220 static unsigned int vm_handle_to_idx(pkvm_handle_t handle) 221 { 222 return handle - HANDLE_OFFSET; 223 } 224 225 static pkvm_handle_t idx_to_vm_handle(unsigned int idx) 226 { 227 return idx + HANDLE_OFFSET; 228 } 229 230 /* 231 * Spinlock for protecting state related to the VM table. Protects writes 232 * to 'vm_table', 'nr_table_entries', and other per-vm state on initialization. 233 * Also protects reads and writes to 'last_hyp_vcpu_lookup'. 234 */ 235 DEFINE_HYP_SPINLOCK(vm_table_lock); 236 237 /* 238 * A table that tracks all VMs in protected mode. 239 * Allocated during hyp initialization and setup. 240 */ 241 static struct pkvm_hyp_vm **vm_table; 242 243 void pkvm_hyp_vm_table_init(void *tbl) 244 { 245 BUILD_BUG_ON((u64)HANDLE_OFFSET + KVM_MAX_PVMS > (pkvm_handle_t)-1); 246 WARN_ON(vm_table); 247 vm_table = tbl; 248 } 249 250 /* 251 * Return the hyp vm structure corresponding to the handle. 252 */ 253 struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle) 254 { 255 unsigned int idx = vm_handle_to_idx(handle); 256 257 hyp_assert_lock_held(&vm_table_lock); 258 259 if (unlikely(idx >= KVM_MAX_PVMS)) 260 return NULL; 261 262 /* A reserved entry doesn't represent an initialized VM. */ 263 if (unlikely(vm_table[idx] == RESERVED_ENTRY)) 264 return NULL; 265 266 return vm_table[idx]; 267 } 268 269 struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, 270 unsigned int vcpu_idx) 271 { 272 struct pkvm_hyp_vcpu *hyp_vcpu = NULL; 273 struct pkvm_hyp_vm *hyp_vm; 274 275 /* Cannot load a new vcpu without putting the old one first. */ 276 if (__this_cpu_read(loaded_hyp_vcpu)) 277 return NULL; 278 279 hyp_spin_lock(&vm_table_lock); 280 hyp_vm = get_vm_by_handle(handle); 281 if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying) 282 goto unlock; 283 284 if (hyp_vm->kvm.created_vcpus <= vcpu_idx) 285 goto unlock; 286 287 /* Pairs with smp_store_release() in register_hyp_vcpu(). */ 288 hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[vcpu_idx]); 289 if (!hyp_vcpu) 290 goto unlock; 291 292 /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */ 293 if (unlikely(hyp_vcpu->loaded_hyp_vcpu)) { 294 hyp_vcpu = NULL; 295 goto unlock; 296 } 297 298 hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu); 299 pkvm_hyp_vm_ref_inc(hyp_vm); 300 unlock: 301 hyp_spin_unlock(&vm_table_lock); 302 303 if (hyp_vcpu) 304 __this_cpu_write(loaded_hyp_vcpu, hyp_vcpu); 305 return hyp_vcpu; 306 } 307 308 void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 309 { 310 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 311 312 hyp_spin_lock(&vm_table_lock); 313 hyp_vcpu->loaded_hyp_vcpu = NULL; 314 __this_cpu_write(loaded_hyp_vcpu, NULL); 315 pkvm_hyp_vm_ref_dec(hyp_vm); 316 hyp_spin_unlock(&vm_table_lock); 317 } 318 319 struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void) 320 { 321 return __this_cpu_read(loaded_hyp_vcpu); 322 323 } 324 325 static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu) 326 { 327 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 328 329 if (hyp_vcpu && 330 (vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu)) 331 return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 332 333 return NULL; 334 } 335 336 /* 337 * A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at 338 * EL2 only while pinned, so it's read once and pinned; the pin fails for 339 * memory the host isn't sharing. 340 */ 341 struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu) 342 { 343 struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu); 344 struct kvm *kvm; 345 346 if (hyp_vm) 347 return &hyp_vm->kvm; 348 349 kvm = kern_hyp_va(READ_ONCE(vcpu->kvm)); 350 if (hyp_pin_shared_mem(kvm, kvm + 1)) 351 return NULL; 352 353 return kvm; 354 } 355 356 void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) 357 { 358 if (kvm && !pkvm_get_loaded_hyp_vm(vcpu)) 359 hyp_unpin_shared_mem(kvm, kvm + 1); 360 } 361 362 struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) 363 { 364 struct pkvm_hyp_vm *hyp_vm; 365 366 hyp_spin_lock(&vm_table_lock); 367 hyp_vm = get_vm_by_handle(handle); 368 if (hyp_vm) 369 pkvm_hyp_vm_ref_inc(hyp_vm); 370 hyp_spin_unlock(&vm_table_lock); 371 372 return hyp_vm; 373 } 374 375 void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm) 376 { 377 hyp_spin_lock(&vm_table_lock); 378 pkvm_hyp_vm_ref_dec(hyp_vm); 379 hyp_spin_unlock(&vm_table_lock); 380 } 381 382 struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle) 383 { 384 struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); 385 386 if (hyp_vm && pkvm_hyp_vm_is_protected(hyp_vm)) { 387 put_pkvm_hyp_vm(hyp_vm); 388 hyp_vm = NULL; 389 } 390 391 return hyp_vm; 392 } 393 394 static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struct kvm *host_kvm) 395 { 396 struct kvm *kvm = &hyp_vm->kvm; 397 unsigned long host_arch_flags = READ_ONCE(host_kvm->arch.flags); 398 u32 vgic_model = READ_ONCE(host_kvm->arch.vgic.vgic_model); 399 DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES); 400 401 /* CTR_EL0 is always under host control, even for protected VMs. */ 402 hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0; 403 404 /* 405 * pKVM runs only GICv3. Anything else becomes 0, the model of a VM 406 * with no in-kernel irqchip, which keeps EL2 off the GICv5 registers. 407 */ 408 if (vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) 409 vgic_model = 0; 410 hyp_vm->kvm.arch.vgic.vgic_model = vgic_model; 411 412 /* No restrictions for non-protected VMs. */ 413 if (!kvm_vm_is_protected(kvm)) { 414 hyp_vm->kvm.arch.flags = host_arch_flags; 415 hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED); 416 417 bitmap_copy(kvm->arch.vcpu_features, 418 host_kvm->arch.vcpu_features, 419 KVM_VCPU_MAX_FEATURES); 420 421 if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) 422 hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; 423 424 goto out; 425 } 426 427 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) 428 kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED); 429 430 kvm_pkvm_vcpu_allowed_features(kvm, allowed_features); 431 432 bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, 433 allowed_features, KVM_VCPU_MAX_FEATURES); 434 out: 435 __assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags, 436 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); 437 } 438 439 static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state) 440 { 441 struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; 442 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 443 struct kvm_vcpu *host_vcpu; 444 445 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 446 /* The host manages a non-protected vCPU: always ON at EL2. */ 447 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON; 448 return 0; 449 } 450 451 if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) 452 return -EINVAL; 453 454 if (mp_state == KVM_MP_STATE_STOPPED) { 455 reset_state->reset = false; 456 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; 457 return 0; 458 } 459 460 hyp_assert_lock_held(&vm_table_lock); 461 if (hyp_vm->primary_vcpu) 462 return -EINVAL; 463 hyp_vm->primary_vcpu = hyp_vcpu; 464 465 host_vcpu = hyp_vcpu->host_vcpu; 466 reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu)); 467 reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); 468 reset_state->reset = true; 469 hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; 470 return 0; 471 } 472 473 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) 474 { 475 if (host_vcpu) 476 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); 477 } 478 479 static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu) 480 { 481 void *sve_state; 482 483 sve_state = hyp_vcpu->vcpu.arch.sve_state; 484 if (!sve_state) 485 return; 486 487 hyp_unpin_shared_mem(sve_state, 488 sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu)); 489 } 490 491 static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], 492 unsigned int nr_vcpus) 493 { 494 int i; 495 496 for (i = 0; i < nr_vcpus; i++) { 497 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i]; 498 499 if (!hyp_vcpu) 500 continue; 501 502 unpin_host_vcpu(hyp_vcpu->host_vcpu); 503 unpin_host_sve_state(hyp_vcpu); 504 } 505 } 506 507 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, 508 unsigned int nr_vcpus, pkvm_handle_t handle) 509 { 510 struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 511 int idx = vm_handle_to_idx(handle); 512 513 hyp_vm->kvm.arch.pkvm.handle = handle; 514 515 hyp_vm->host_kvm = host_kvm; 516 hyp_vm->kvm.created_vcpus = nr_vcpus; 517 if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) 518 hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; 519 else 520 hyp_vm->kvm.arch.vm_flavor = VM_PKVM; 521 522 hyp_vm->kvm.arch.flags = 0; 523 pkvm_init_features_from_host(hyp_vm, host_kvm); 524 525 /* VMID 0 is reserved for the host */ 526 atomic64_set(&mmu->vmid.id, idx + 1); 527 528 mmu->vtcr = host_mmu.arch.mmu.vtcr; 529 mmu->arch = &hyp_vm->kvm.arch; 530 mmu->pgt = &hyp_vm->pgt; 531 } 532 533 static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) 534 { 535 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 536 unsigned int sve_max_vl; 537 size_t sve_state_size; 538 void *sve_state; 539 int ret; 540 541 if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) { 542 vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED); 543 return 0; 544 } 545 546 /* Limit guest vector length to the maximum supported by the host. */ 547 sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); 548 sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); 549 550 if (!sve_vl_valid(sve_max_vl) || !sve_state) 551 return -EINVAL; 552 553 sve_state_size = sve_state_size_from_vl(sve_max_vl); 554 555 ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); 556 if (ret) 557 return ret; 558 559 vcpu->arch.sve_state = sve_state; 560 vcpu->arch.sve_max_vl = sve_max_vl; 561 562 return 0; 563 } 564 565 static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) 566 { 567 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 568 const struct kvm *host_kvm = hyp_vm->host_kvm; 569 struct kvm *kvm = &hyp_vm->kvm; 570 571 if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &host_kvm->arch.flags)) 572 return -EINVAL; 573 574 if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) 575 return 0; 576 577 memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs)); 578 579 return 0; 580 } 581 582 static int pkvm_vcpu_init_sysregs(struct pkvm_hyp_vcpu *hyp_vcpu) 583 { 584 int ret = 0; 585 586 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 587 kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); 588 else 589 ret = vm_copy_id_regs(hyp_vcpu); 590 591 return ret; 592 } 593 594 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, 595 struct pkvm_hyp_vm *hyp_vm, 596 struct kvm_vcpu *host_vcpu) 597 { 598 int ret = 0; 599 u32 mp_state; 600 601 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) 602 return -EBUSY; 603 604 mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); 605 hyp_vcpu->host_vcpu = host_vcpu; 606 607 hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; 608 hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); 609 hyp_vcpu->vcpu.vcpu_idx = READ_ONCE(host_vcpu->vcpu_idx); 610 611 hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; 612 hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); 613 614 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 615 /* 616 * Timer offsets are pointing to the untrusted KVM copy, 617 * which is pinned in __pkvm_init_vm() for the VM life time. 618 * It is worth noting that hyp_vm->host_kvm points to an EL2 619 * linear map address and timer_get_offset() will use 620 * kern_hyp_va() which is safe as it is idempotent. 621 */ 622 vcpu_vtimer(&hyp_vcpu->vcpu)->offset.vm_offset = 623 &hyp_vm->host_kvm->arch.timer_data.voffset; 624 vcpu_ptimer(&hyp_vcpu->vcpu)->offset.vm_offset = 625 &hyp_vm->host_kvm->arch.timer_data.poffset; 626 } 627 628 ret = pkvm_vcpu_init_sysregs(hyp_vcpu); 629 if (ret) 630 goto done; 631 632 ret = pkvm_vcpu_init_traps(hyp_vcpu); 633 if (ret) 634 goto done; 635 636 ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu); 637 if (ret) 638 goto done; 639 640 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 641 kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); 642 ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state); 643 done: 644 if (ret) { 645 unpin_host_vcpu(host_vcpu); 646 unpin_host_sve_state(hyp_vcpu); 647 } 648 return ret; 649 } 650 651 static int find_free_vm_table_entry(void) 652 { 653 int i; 654 655 for (i = 0; i < KVM_MAX_PVMS; ++i) { 656 if (!vm_table[i]) 657 return i; 658 } 659 660 return -ENOMEM; 661 } 662 663 /* 664 * Reserve a VM table entry. 665 * 666 * Return a unique handle to the VM on success, 667 * negative error code on failure. 668 */ 669 static int allocate_vm_table_entry(void) 670 { 671 int idx; 672 673 hyp_assert_lock_held(&vm_table_lock); 674 675 /* 676 * Initializing protected state might have failed, yet a malicious 677 * host could trigger this function. Thus, ensure that 'vm_table' 678 * exists. 679 */ 680 if (unlikely(!vm_table)) 681 return -EINVAL; 682 683 idx = find_free_vm_table_entry(); 684 if (unlikely(idx < 0)) 685 return idx; 686 687 vm_table[idx] = RESERVED_ENTRY; 688 689 return idx; 690 } 691 692 static int __insert_vm_table_entry(pkvm_handle_t handle, 693 struct pkvm_hyp_vm *hyp_vm) 694 { 695 unsigned int idx; 696 697 hyp_assert_lock_held(&vm_table_lock); 698 699 /* 700 * Initializing protected state might have failed, yet a malicious 701 * host could trigger this function. Thus, ensure that 'vm_table' 702 * exists. 703 */ 704 if (unlikely(!vm_table)) 705 return -EINVAL; 706 707 idx = vm_handle_to_idx(handle); 708 if (unlikely(idx >= KVM_MAX_PVMS)) 709 return -EINVAL; 710 711 if (unlikely(vm_table[idx] != RESERVED_ENTRY)) 712 return -EINVAL; 713 714 vm_table[idx] = hyp_vm; 715 716 return 0; 717 } 718 719 /* 720 * Insert a pointer to the initialized VM into the VM table. 721 * 722 * Return 0 on success, or negative error code on failure. 723 */ 724 static int insert_vm_table_entry(pkvm_handle_t handle, 725 struct pkvm_hyp_vm *hyp_vm) 726 { 727 int ret; 728 729 hyp_spin_lock(&vm_table_lock); 730 ret = __insert_vm_table_entry(handle, hyp_vm); 731 hyp_spin_unlock(&vm_table_lock); 732 733 return ret; 734 } 735 736 /* 737 * Deallocate and remove the VM table entry corresponding to the handle. 738 */ 739 static void remove_vm_table_entry(pkvm_handle_t handle) 740 { 741 hyp_assert_lock_held(&vm_table_lock); 742 vm_table[vm_handle_to_idx(handle)] = NULL; 743 } 744 745 static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) 746 { 747 return size_add(sizeof(struct pkvm_hyp_vm), 748 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); 749 } 750 751 static void *map_donated_memory_noclear(void __kern *host_va, size_t size) 752 { 753 void *va = kern_hyp_va_host(host_va); 754 755 if (!PAGE_ALIGNED(va)) 756 return NULL; 757 758 if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va), 759 PAGE_ALIGN(size) >> PAGE_SHIFT)) 760 return NULL; 761 762 return va; 763 } 764 765 static void __unmap_donated_memory(void *va, size_t size) 766 { 767 kvm_flush_dcache_to_poc(va, size); 768 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va), 769 PAGE_ALIGN(size) >> PAGE_SHIFT)); 770 } 771 772 static void unmap_donated_memory(void *va, size_t size) 773 { 774 if (!va) 775 return; 776 777 memset(va, 0, size); 778 __unmap_donated_memory(va, size); 779 } 780 781 static void unmap_donated_memory_noclear(void *va, size_t size) 782 { 783 if (!va) 784 return; 785 786 __unmap_donated_memory(va, size); 787 } 788 789 /* 790 * Reserves an entry in the hypervisor for a new VM in protected mode. 791 * 792 * Return a unique handle to the VM on success, negative error code on failure. 793 */ 794 int __pkvm_reserve_vm(void) 795 { 796 int ret; 797 798 hyp_spin_lock(&vm_table_lock); 799 ret = allocate_vm_table_entry(); 800 hyp_spin_unlock(&vm_table_lock); 801 802 if (ret < 0) 803 return ret; 804 805 return idx_to_vm_handle(ret); 806 } 807 808 /* 809 * Removes a reserved entry, but only if is hasn't been used yet. 810 * Otherwise, the VM needs to be destroyed. 811 */ 812 void __pkvm_unreserve_vm(pkvm_handle_t handle) 813 { 814 unsigned int idx = vm_handle_to_idx(handle); 815 816 if (unlikely(!vm_table)) 817 return; 818 819 hyp_spin_lock(&vm_table_lock); 820 if (likely(idx < KVM_MAX_PVMS && vm_table[idx] == RESERVED_ENTRY)) 821 remove_vm_table_entry(handle); 822 hyp_spin_unlock(&vm_table_lock); 823 } 824 825 #ifdef CONFIG_NVHE_EL2_DEBUG 826 static struct pkvm_hyp_vm selftest_vm = { 827 .kvm = { 828 .arch = { 829 .mmu = { 830 .arch = &selftest_vm.kvm.arch, 831 .pgt = &selftest_vm.pgt, 832 }, 833 }, 834 }, 835 }; 836 837 static struct pkvm_hyp_vcpu selftest_vcpu = { 838 .vcpu = { 839 .arch = { 840 .hw_mmu = &selftest_vm.kvm.arch.mmu, 841 }, 842 .kvm = &selftest_vm.kvm, 843 }, 844 }; 845 846 struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) 847 { 848 struct hyp_page *p = hyp_virt_to_page(virt); 849 unsigned long min_pages, seeded = 0; 850 int i; 851 852 selftest_vm.kvm.arch.mmu.vtcr = host_mmu.arch.mmu.vtcr; 853 WARN_ON(kvm_guest_prepare_stage2(&selftest_vm, virt)); 854 855 /* 856 * Mirror pkvm_refill_memcache() for the share/donate pre-checks; 857 * the selftest invokes those functions directly and would 858 * otherwise see an empty memcache. 859 */ 860 min_pages = kvm_mmu_cache_min_pages(&selftest_vm.kvm.arch.mmu); 861 862 for (i = 0; i < pkvm_selftest_pages(); i++) { 863 if (p[i].refcount) 864 continue; 865 p[i].refcount = 1; 866 if (seeded < min_pages) { 867 push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc, 868 hyp_page_to_virt(&p[i]), hyp_virt_to_phys); 869 seeded++; 870 } else { 871 hyp_put_page(&selftest_vm.pool, hyp_page_to_virt(&p[i])); 872 } 873 } 874 875 selftest_vm.kvm.arch.pkvm.handle = __pkvm_reserve_vm(); 876 insert_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle, &selftest_vm); 877 return &selftest_vcpu; 878 } 879 880 void teardown_selftest_vm(void) 881 { 882 hyp_spin_lock(&vm_table_lock); 883 remove_vm_table_entry(selftest_vm.kvm.arch.pkvm.handle); 884 hyp_spin_unlock(&vm_table_lock); 885 } 886 #endif /* CONFIG_NVHE_EL2_DEBUG */ 887 888 /* 889 * Initialize the hypervisor copy of the VM state using host-donated memory. 890 * 891 * Unmap the donated memory from the host at stage 2. 892 * 893 * host_kvm: A pointer to the host's struct kvm. 894 * pgd_hva: The host va of the area being donated for the stage-2 PGD for 895 * the VM. Must be page aligned. Its size is implied by the VM's 896 * VTCR. 897 * 898 * Return 0 success, negative error code on failure. 899 */ 900 int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva) 901 { 902 struct pkvm_hyp_vm *hyp_vm = NULL; 903 size_t vm_size, pgd_size; 904 unsigned int nr_vcpus; 905 pkvm_handle_t handle; 906 void *pgd = NULL; 907 int ret; 908 909 ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1); 910 if (ret) 911 return ret; 912 913 nr_vcpus = READ_ONCE(host_kvm->created_vcpus); 914 if (nr_vcpus < 1) { 915 ret = -EINVAL; 916 goto err_unpin_kvm; 917 } 918 919 handle = READ_ONCE(host_kvm->arch.pkvm.handle); 920 if (unlikely(handle < HANDLE_OFFSET)) { 921 ret = -EINVAL; 922 goto err_unpin_kvm; 923 } 924 925 vm_size = pkvm_get_hyp_vm_size(nr_vcpus); 926 pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); 927 928 hyp_vm = hyp_alloc(vm_size); 929 if (!hyp_vm) { 930 ret = hyp_alloc_errno(); 931 goto err_unpin_kvm; 932 } 933 934 ret = -ENOMEM; 935 936 pgd = map_donated_memory_noclear(pgd_hva, pgd_size); 937 if (!pgd) 938 goto err_free_hyp_vm; 939 940 init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); 941 942 ret = kvm_guest_prepare_stage2(hyp_vm, pgd); 943 if (ret) 944 goto err_remove_mappings; 945 946 /* Must be called last since this publishes the VM. */ 947 ret = insert_vm_table_entry(handle, hyp_vm); 948 if (ret) 949 goto err_destroy_stage2; 950 951 return 0; 952 953 err_destroy_stage2: 954 kvm_guest_destroy_stage2(hyp_vm); 955 err_remove_mappings: 956 unmap_donated_memory(pgd, pgd_size); 957 err_free_hyp_vm: 958 hyp_free(hyp_vm); 959 err_unpin_kvm: 960 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 961 return ret; 962 } 963 964 /* 965 * Initialize the hypervisor copy of the vCPU state using host-donated memory. 966 * 967 * handle: The hypervisor handle for the vm. 968 * host_vcpu: A pointer to the corresponding host vcpu. 969 * vcpu_hva: The host va of the area being donated for the vcpu state. 970 * Must be page aligned. The size of the area must be equal to 971 * the page-aligned size of 'struct pkvm_hyp_vcpu'. 972 * Return 0 on success, negative error code on failure. 973 */ 974 static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, 975 struct pkvm_hyp_vcpu *hyp_vcpu) 976 { 977 unsigned int idx = hyp_vcpu->vcpu.vcpu_idx; 978 979 if (idx >= hyp_vm->kvm.created_vcpus) 980 return -EINVAL; 981 982 if (hyp_vm->vcpus[idx]) 983 return -EINVAL; 984 985 /* 986 * Ensure the hyp_vcpu is initialised before publishing it to 987 * the vCPU-load path via 'hyp_vm->vcpus[]'. 988 */ 989 smp_store_release(&hyp_vm->vcpus[idx], hyp_vcpu); 990 return 0; 991 } 992 993 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) 994 { 995 struct pkvm_hyp_vcpu *hyp_vcpu; 996 struct pkvm_hyp_vm *hyp_vm; 997 int ret; 998 999 hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu)); 1000 if (!hyp_vcpu) 1001 return hyp_alloc_errno(); 1002 1003 hyp_spin_lock(&vm_table_lock); 1004 1005 hyp_vm = get_vm_by_handle(handle); 1006 if (!hyp_vm) { 1007 ret = -ENOENT; 1008 goto unlock; 1009 } 1010 1011 ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); 1012 if (ret) 1013 goto unclaim; 1014 1015 ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); 1016 if (ret) { 1017 unpin_host_vcpu(host_vcpu); 1018 unpin_host_sve_state(hyp_vcpu); 1019 goto unclaim; 1020 } 1021 goto unlock; 1022 unclaim: 1023 /* 1024 * Under vm_table_lock, so no other claim can have landed: undo 1025 * this one. 1026 */ 1027 if (hyp_vm->primary_vcpu == hyp_vcpu) 1028 hyp_vm->primary_vcpu = NULL; 1029 unlock: 1030 hyp_spin_unlock(&vm_table_lock); 1031 if (ret) 1032 hyp_free(hyp_vcpu); 1033 1034 return ret; 1035 } 1036 1037 int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn) 1038 { 1039 struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); 1040 int ret = -EINVAL; 1041 1042 if (!hyp_vm) 1043 return ret; 1044 1045 if (hyp_vm->kvm.arch.pkvm.is_dying) 1046 ret = __pkvm_host_reclaim_page_guest(gfn, hyp_vm); 1047 1048 put_pkvm_hyp_vm(hyp_vm); 1049 return ret; 1050 } 1051 1052 static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle) 1053 { 1054 struct pkvm_hyp_vm *hyp_vm; 1055 1056 hyp_assert_lock_held(&vm_table_lock); 1057 1058 hyp_vm = get_vm_by_handle(handle); 1059 if (!hyp_vm || hyp_vm->refcount) 1060 return NULL; 1061 1062 return hyp_vm; 1063 } 1064 1065 int __pkvm_start_teardown_vm(pkvm_handle_t handle) 1066 { 1067 struct pkvm_hyp_vm *hyp_vm; 1068 int ret = 0; 1069 1070 hyp_spin_lock(&vm_table_lock); 1071 hyp_vm = get_pkvm_unref_hyp_vm_locked(handle); 1072 if (!hyp_vm || hyp_vm->kvm.arch.pkvm.is_dying) { 1073 ret = -EINVAL; 1074 goto unlock; 1075 } 1076 1077 hyp_vm->kvm.arch.pkvm.is_dying = true; 1078 unlock: 1079 hyp_spin_unlock(&vm_table_lock); 1080 1081 return ret; 1082 } 1083 1084 int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) 1085 { 1086 struct kvm_hyp_memcache *stage2_mc; 1087 struct pkvm_hyp_vm *hyp_vm; 1088 struct kvm *host_kvm; 1089 unsigned int idx; 1090 int err; 1091 1092 hyp_spin_lock(&vm_table_lock); 1093 hyp_vm = get_pkvm_unref_hyp_vm_locked(handle); 1094 if (!hyp_vm || !hyp_vm->kvm.arch.pkvm.is_dying) { 1095 err = -EINVAL; 1096 goto err_unlock; 1097 } 1098 1099 host_kvm = hyp_vm->host_kvm; 1100 1101 /* Ensure the VMID is clean before it can be reallocated */ 1102 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 1103 remove_vm_table_entry(handle); 1104 hyp_spin_unlock(&vm_table_lock); 1105 1106 /* Reclaim guest pages (including page-table pages) */ 1107 stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc; 1108 reclaim_pgtable_pages(hyp_vm, stage2_mc); 1109 unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); 1110 1111 /* Push the stage-2 pages to the teardown memcache */ 1112 for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { 1113 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; 1114 struct kvm_hyp_memcache *vcpu_mc; 1115 1116 if (!hyp_vcpu) 1117 continue; 1118 1119 vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; 1120 1121 while (vcpu_mc->nr_pages) { 1122 void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt); 1123 1124 push_hyp_memcache(stage2_mc, addr, hyp_virt_to_phys); 1125 unmap_donated_memory_noclear(addr, PAGE_SIZE); 1126 } 1127 1128 hyp_free(hyp_vcpu); 1129 } 1130 1131 hyp_free(hyp_vm); 1132 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 1133 return 0; 1134 1135 err_unlock: 1136 hyp_spin_unlock(&vm_table_lock); 1137 return err; 1138 } 1139 1140 static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa) 1141 { 1142 u64 elr; 1143 1144 /* Fake up a data abort (level 3 translation fault on write) */ 1145 vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | 1146 ESR_ELx_IL | ESR_ELx_WNR | 1147 ESR_ELx_FSC_FAULT | 1148 FIELD_PREP(ESR_ELx_FSC_LEVEL, 3); 1149 1150 /* Shuffle the IPA around into the HPFAR */ 1151 vcpu->arch.fault.hpfar_el2 = (HPFAR_EL2_NS | (ipa >> 8)) & HPFAR_MASK; 1152 1153 /* This is a virtual address. 0's good. Let's go with 0. */ 1154 vcpu->arch.fault.far_el2 = 0; 1155 1156 /* Rewind the ELR so we return to the HVC once the IPA is mapped */ 1157 elr = read_sysreg(elr_el2); 1158 elr -= 4; 1159 write_sysreg(elr, elr_el2); 1160 1161 return ARM_EXCEPTION_TRAP; 1162 } 1163 1164 static bool pkvm_memshare_call(u64 *ret, struct kvm_vcpu *vcpu, u64 *exit_code) 1165 { 1166 struct pkvm_hyp_vcpu *hyp_vcpu; 1167 u64 ipa = smccc_get_arg1(vcpu); 1168 1169 if (!PAGE_ALIGNED(ipa)) 1170 goto out_guest; 1171 1172 hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1173 switch (__pkvm_guest_share_host(hyp_vcpu, hyp_phys_to_pfn(ipa))) { 1174 case 0: 1175 ret[0] = SMCCC_RET_SUCCESS; 1176 goto out_guest; 1177 case -ENOENT: 1178 /* 1179 * Convert the exception into a data abort so that the page 1180 * being shared is mapped into the guest next time. 1181 */ 1182 *exit_code = __pkvm_memshare_page_req(vcpu, ipa); 1183 goto out_host; 1184 } 1185 1186 out_guest: 1187 return true; 1188 out_host: 1189 return false; 1190 } 1191 1192 static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) 1193 { 1194 struct pkvm_hyp_vcpu *hyp_vcpu; 1195 u64 ipa = smccc_get_arg1(vcpu); 1196 1197 if (!PAGE_ALIGNED(ipa)) 1198 return; 1199 1200 hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1201 if (!__pkvm_guest_unshare_host(hyp_vcpu, hyp_phys_to_pfn(ipa))) 1202 ret[0] = SMCCC_RET_SUCCESS; 1203 } 1204 1205 /* 1206 * Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the 1207 * target's own CPU. Returns -ECANCELED, with no side effects, if a rollback 1208 * raced the reset. 1209 * 1210 * The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on(): 1211 * 1212 * cmpxchg_relaxed(power_state): OFF -> ON_PENDING 1213 * <ctrl> 1214 * write_reset_state(); 1215 * smp_store_release(reset_state->reset, true); 1216 * 1217 * The target coming online here, running, then its PSCI CPU_OFF in 1218 * pvm_psci_vcpu_off(): 1219 * 1220 * smp_load_acquire(reset_state->reset) == true; 1221 * cmpxchg_relaxed(power_state): ON_PENDING -> ON; 1222 * read_reset_state(); 1223 * 1224 * <run guest> 1225 * 1226 * WRITE_ONCE(reset_state->reset, false); 1227 * smp_store_release(power_state, OFF); 1228 */ 1229 int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 1230 { 1231 struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; 1232 1233 /* 1234 * Pairs with smp_store_release(&reset_state->reset, true) in 1235 * pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a 1236 * winning cmpxchg with a false acquire would leave power_state == ON 1237 * with the reset skipped. 1238 */ 1239 if (!smp_load_acquire(&reset_state->reset)) 1240 return -ECANCELED; 1241 1242 if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, 1243 PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) 1244 return -ECANCELED; 1245 1246 kvm_reset_vcpu_core(&hyp_vcpu->vcpu); 1247 kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); 1248 1249 /* Must be done after resetting sys registers. */ 1250 kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state); 1251 1252 hyp_vcpu->exit_code = 0; 1253 return 0; 1254 } 1255 1256 struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr) 1257 { 1258 struct pkvm_hyp_vcpu *hyp_vcpu; 1259 int i; 1260 1261 mpidr &= MPIDR_HWID_BITMASK; 1262 1263 for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { 1264 /* Pairs with smp_store_release() in register_hyp_vcpu(). */ 1265 hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]); 1266 1267 if (!hyp_vcpu) 1268 continue; 1269 1270 if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) 1271 return hyp_vcpu; 1272 } 1273 1274 return NULL; 1275 } 1276 1277 /* 1278 * Returns true when handled at EL2, false when the host must wake the target 1279 * vCPU. 1280 */ 1281 static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu) 1282 { 1283 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 1284 struct vcpu_reset_state *reset_state; 1285 struct pkvm_hyp_vcpu *target; 1286 unsigned long cpu_id, ret; 1287 int power_state; 1288 1289 cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu); 1290 if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) { 1291 ret = PSCI_RET_INVALID_PARAMS; 1292 goto error; 1293 } 1294 1295 target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); 1296 if (!target) { 1297 ret = PSCI_RET_INVALID_PARAMS; 1298 goto error; 1299 } 1300 1301 /* 1302 * vCPUs race to power on the same target. The writes below depend on 1303 * the cmpxchg reading OFF, and that control dependency with the release 1304 * of OFF in pvm_psci_vcpu_off() orders them after the target's accesses 1305 * to reset_state, so relaxed suffices. 1306 */ 1307 power_state = cmpxchg_relaxed(&target->power_state, 1308 PSCI_0_2_AFFINITY_LEVEL_OFF, 1309 PSCI_0_2_AFFINITY_LEVEL_ON_PENDING); 1310 switch (power_state) { 1311 case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: 1312 ret = PSCI_RET_ON_PENDING; 1313 goto error; 1314 case PSCI_0_2_AFFINITY_LEVEL_ON: 1315 ret = PSCI_RET_ALREADY_ON; 1316 goto error; 1317 case PSCI_0_2_AFFINITY_LEVEL_OFF: 1318 break; 1319 default: 1320 ret = PSCI_RET_INTERNAL_FAILURE; 1321 goto error; 1322 } 1323 1324 reset_state = &target->vcpu.arch.reset_state; 1325 WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu)); 1326 WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu)); 1327 WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu)); 1328 /* 1329 * Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with 1330 * smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu(). 1331 */ 1332 smp_store_release(&reset_state->reset, true); 1333 1334 /* The host requests KVM_REQ_VCPU_RESET and wakes the target. */ 1335 return false; 1336 1337 error: 1338 smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0); 1339 return true; 1340 } 1341 1342 /* 1343 * Returns true when handled at EL2, false when the host must stop scheduling 1344 * the vCPU. 1345 */ 1346 static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu) 1347 { 1348 /* No other writer runs while this vCPU is ON and executing. */ 1349 WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON); 1350 1351 WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false); 1352 1353 /* 1354 * Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear 1355 * above, before OFF. Pairs with the control dependency in 1356 * pvm_psci_vcpu_on(). 1357 */ 1358 smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF); 1359 1360 /* Return to the host so that it can finish powering off the vcpu. */ 1361 return false; 1362 } 1363 1364 static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu) 1365 { 1366 /* Nothing to be handled by the host. Go back to the guest. */ 1367 smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0); 1368 return true; 1369 } 1370 1371 static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu) 1372 { 1373 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 1374 u32 feature = smccc_get_arg1(vcpu); 1375 unsigned long val; 1376 1377 switch (feature) { 1378 case PSCI_0_2_FN_PSCI_VERSION: 1379 case PSCI_0_2_FN_CPU_SUSPEND: 1380 case PSCI_0_2_FN64_CPU_SUSPEND: 1381 case PSCI_0_2_FN_CPU_OFF: 1382 case PSCI_0_2_FN_CPU_ON: 1383 case PSCI_0_2_FN64_CPU_ON: 1384 case PSCI_0_2_FN_AFFINITY_INFO: 1385 case PSCI_0_2_FN64_AFFINITY_INFO: 1386 case PSCI_0_2_FN_SYSTEM_OFF: 1387 case PSCI_0_2_FN_SYSTEM_RESET: 1388 case PSCI_1_0_FN_PSCI_FEATURES: 1389 case PSCI_1_1_FN_SYSTEM_RESET2: 1390 case PSCI_1_1_FN64_SYSTEM_RESET2: 1391 case ARM_SMCCC_VERSION_FUNC_ID: 1392 val = PSCI_RET_SUCCESS; 1393 break; 1394 default: 1395 val = PSCI_RET_NOT_SUPPORTED; 1396 break; 1397 } 1398 1399 /* Nothing to be handled by the host. Go back to the guest. */ 1400 smccc_set_retval(vcpu, val, 0, 0, 0); 1401 return true; 1402 } 1403 1404 static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu) 1405 { 1406 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 1407 u32 psci_fn = smccc_get_function(vcpu); 1408 1409 switch (psci_fn) { 1410 case PSCI_0_2_FN_CPU_ON: 1411 kvm_psci_narrow_to_32bit(vcpu); 1412 fallthrough; 1413 case PSCI_0_2_FN64_CPU_ON: 1414 return pvm_psci_vcpu_on(hyp_vcpu); 1415 case PSCI_0_2_FN_CPU_OFF: 1416 return pvm_psci_vcpu_off(hyp_vcpu); 1417 case PSCI_0_2_FN_PSCI_VERSION: 1418 return pvm_psci_version(hyp_vcpu); 1419 case PSCI_1_0_FN_PSCI_FEATURES: 1420 return pvm_psci_features(hyp_vcpu); 1421 case PSCI_0_2_FN_AFFINITY_INFO: 1422 case PSCI_0_2_FN64_AFFINITY_INFO: 1423 case PSCI_0_2_FN_SYSTEM_RESET: 1424 case PSCI_0_2_FN_CPU_SUSPEND: 1425 case PSCI_0_2_FN64_CPU_SUSPEND: 1426 case PSCI_0_2_FN_SYSTEM_OFF: 1427 case PSCI_1_1_FN_SYSTEM_RESET2: 1428 case PSCI_1_1_FN64_SYSTEM_RESET2: 1429 return false; /* Handled by the host. */ 1430 default: 1431 /* Unknown PSCI calls are handled here, not forwarded. */ 1432 smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0); 1433 return true; 1434 } 1435 } 1436 1437 /* 1438 * Handler for protected VM HVC calls. 1439 * 1440 * Returns true if the hypervisor has handled the exit (and control 1441 * should return to the guest) or false if it hasn't (and the handling 1442 * should be performed by the host). 1443 */ 1444 bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) 1445 { 1446 struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); 1447 u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; 1448 bool handled = true; 1449 u32 feature; 1450 uuid_t uuid; 1451 1452 switch (smccc_get_function(vcpu)) { 1453 case ARM_SMCCC_VERSION_FUNC_ID: 1454 /* Nothing to be handled by the host. Go back to the guest. */ 1455 val[0] = ARM_SMCCC_VERSION_1_1; 1456 break; 1457 case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: 1458 /* The workaround calls themselves are handled on hyp entry. */ 1459 val[0] = SMCCC_RET_NOT_SUPPORTED; 1460 feature = smccc_get_arg1(vcpu); 1461 switch (feature) { 1462 case ARM_SMCCC_VERSION_FUNC_ID: 1463 case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: 1464 val[0] = SMCCC_RET_SUCCESS; 1465 break; 1466 case ARM_SMCCC_ARCH_WORKAROUND_1: 1467 switch (spectre_v2_state) { 1468 case SPECTRE_VULNERABLE: 1469 break; 1470 case SPECTRE_MITIGATED: 1471 val[0] = SMCCC_RET_SUCCESS; 1472 break; 1473 case SPECTRE_UNAFFECTED: 1474 val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; 1475 break; 1476 } 1477 break; 1478 case ARM_SMCCC_ARCH_WORKAROUND_2: 1479 switch (spectre_v4_state) { 1480 case SPECTRE_VULNERABLE: 1481 break; 1482 case SPECTRE_MITIGATED: 1483 /* With SSBS advertised the guest's default is safe. */ 1484 if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP)) 1485 break; 1486 fallthrough; 1487 case SPECTRE_UNAFFECTED: 1488 val[0] = SMCCC_RET_NOT_REQUIRED; 1489 break; 1490 } 1491 break; 1492 case ARM_SMCCC_ARCH_WORKAROUND_3: 1493 switch (spectre_bhb_state) { 1494 case SPECTRE_VULNERABLE: 1495 break; 1496 case SPECTRE_MITIGATED: 1497 val[0] = SMCCC_RET_SUCCESS; 1498 break; 1499 case SPECTRE_UNAFFECTED: 1500 val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; 1501 break; 1502 } 1503 break; 1504 } 1505 break; 1506 case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: 1507 uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM; 1508 val[0] = smccc_uuid_to_reg(&uuid, 0); 1509 val[1] = smccc_uuid_to_reg(&uuid, 1); 1510 val[2] = smccc_uuid_to_reg(&uuid, 2); 1511 val[3] = smccc_uuid_to_reg(&uuid, 3); 1512 break; 1513 case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: 1514 val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); 1515 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); 1516 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_SHARE); 1517 val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_UNSHARE); 1518 break; 1519 case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID: 1520 if (smccc_get_arg1(vcpu) || 1521 smccc_get_arg2(vcpu) || 1522 smccc_get_arg3(vcpu)) { 1523 break; 1524 } 1525 1526 val[0] = PAGE_SIZE; 1527 break; 1528 case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID: 1529 if (smccc_get_arg2(vcpu) || 1530 smccc_get_arg3(vcpu)) { 1531 break; 1532 } 1533 1534 handled = pkvm_memshare_call(val, vcpu, exit_code); 1535 break; 1536 case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID: 1537 if (smccc_get_arg2(vcpu) || 1538 smccc_get_arg3(vcpu)) { 1539 break; 1540 } 1541 1542 pkvm_memunshare_call(val, vcpu); 1543 break; 1544 default: 1545 return pkvm_handle_psci(hyp_vcpu); 1546 } 1547 1548 if (handled) 1549 smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]); 1550 return handled; 1551 } 1552