1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 4 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/cpu_pm.h> 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/kvm_host.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/vmalloc.h> 15 #include <linux/fs.h> 16 #include <linux/mman.h> 17 #include <linux/sched.h> 18 #include <linux/kvm.h> 19 #include <linux/kvm_irqfd.h> 20 #include <linux/irqbypass.h> 21 #include <linux/sched/stat.h> 22 #include <linux/psci.h> 23 #include <trace/events/kvm.h> 24 25 #define CREATE_TRACE_POINTS 26 #include "trace_arm.h" 27 28 #include <linux/uaccess.h> 29 #include <asm/ptrace.h> 30 #include <asm/mman.h> 31 #include <asm/tlbflush.h> 32 #include <asm/cacheflush.h> 33 #include <asm/cpufeature.h> 34 #include <asm/virt.h> 35 #include <asm/kvm_arm.h> 36 #include <asm/kvm_asm.h> 37 #include <asm/kvm_emulate.h> 38 #include <asm/kvm_mmu.h> 39 #include <asm/kvm_nested.h> 40 #include <asm/kvm_pkvm.h> 41 #include <asm/kvm_ptrauth.h> 42 #include <asm/sections.h> 43 #include <asm/stacktrace/nvhe.h> 44 45 #include <kvm/arm_hypercalls.h> 46 #include <kvm/arm_pmu.h> 47 #include <kvm/arm_psci.h> 48 #include <kvm/arm_vgic.h> 49 50 #include <linux/irqchip/arm-gic-v5.h> 51 52 #include "sys_regs.h" 53 54 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; 55 56 enum kvm_wfx_trap_policy { 57 KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */ 58 KVM_WFX_NOTRAP, 59 KVM_WFX_TRAP, 60 }; 61 62 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK; 63 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK; 64 65 /* 66 * Tracks KVM IOCTLs and their associated KVM capabilities. 67 */ 68 struct kvm_ioctl_cap_map { 69 unsigned int ioctl; 70 long ext; 71 }; 72 73 /* Make KVM_CAP_NR_VCPUS the reference for features we always supported */ 74 #define KVM_CAP_ARM_BASIC KVM_CAP_NR_VCPUS 75 76 /* 77 * Sorted by ioctl to allow for potential binary search, 78 * though linear scan is sufficient for this size. 79 */ 80 static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = { 81 { KVM_CREATE_IRQCHIP, KVM_CAP_IRQCHIP }, 82 { KVM_ARM_SET_DEVICE_ADDR, KVM_CAP_ARM_SET_DEVICE_ADDR }, 83 { KVM_ARM_MTE_COPY_TAGS, KVM_CAP_ARM_MTE }, 84 { KVM_SET_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL }, 85 { KVM_GET_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL }, 86 { KVM_HAS_DEVICE_ATTR, KVM_CAP_DEVICE_CTRL }, 87 { KVM_ARM_SET_COUNTER_OFFSET, KVM_CAP_COUNTER_OFFSET }, 88 { KVM_ARM_GET_REG_WRITABLE_MASKS, KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES }, 89 { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC }, 90 }; 91 92 /* 93 * Set *ext to the capability. 94 * Return 0 if found, or -EINVAL if no IOCTL matches. 95 */ 96 long kvm_get_cap_for_kvm_ioctl(unsigned int ioctl, long *ext) 97 { 98 int i; 99 100 for (i = 0; i < ARRAY_SIZE(vm_ioctl_caps); i++) { 101 if (vm_ioctl_caps[i].ioctl == ioctl) { 102 *ext = vm_ioctl_caps[i].ext; 103 return 0; 104 } 105 } 106 107 return -EINVAL; 108 } 109 110 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); 111 112 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); 113 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 114 115 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); 116 117 static bool vgic_present, kvm_arm_initialised; 118 119 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized); 120 121 bool is_kvm_arm_initialised(void) 122 { 123 return kvm_arm_initialised; 124 } 125 126 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 127 { 128 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 129 } 130 131 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 132 struct kvm_enable_cap *cap) 133 { 134 int r = -EINVAL; 135 136 if (cap->flags) 137 return -EINVAL; 138 139 if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, cap->cap)) 140 return -EINVAL; 141 142 switch (cap->cap) { 143 case KVM_CAP_ARM_NISV_TO_USER: 144 r = 0; 145 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, 146 &kvm->arch.flags); 147 break; 148 case KVM_CAP_ARM_MTE: 149 mutex_lock(&kvm->lock); 150 if (system_supports_mte() && !kvm->created_vcpus) { 151 r = 0; 152 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); 153 } 154 mutex_unlock(&kvm->lock); 155 break; 156 case KVM_CAP_ARM_SYSTEM_SUSPEND: 157 r = 0; 158 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); 159 break; 160 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: 161 mutex_lock(&kvm->slots_lock); 162 /* 163 * To keep things simple, allow changing the chunk 164 * size only when no memory slots have been created. 165 */ 166 if (kvm_are_all_memslots_empty(kvm)) { 167 u64 new_cap = cap->args[0]; 168 169 if (!new_cap || kvm_is_block_size_supported(new_cap)) { 170 r = 0; 171 kvm->arch.mmu.split_page_chunk_size = new_cap; 172 } 173 } 174 mutex_unlock(&kvm->slots_lock); 175 break; 176 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: 177 mutex_lock(&kvm->lock); 178 if (!kvm->created_vcpus) { 179 r = 0; 180 set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags); 181 } 182 mutex_unlock(&kvm->lock); 183 break; 184 case KVM_CAP_ARM_SEA_TO_USER: 185 r = 0; 186 set_bit(KVM_ARCH_FLAG_EXIT_SEA, &kvm->arch.flags); 187 break; 188 default: 189 break; 190 } 191 192 return r; 193 } 194 195 static int kvm_arm_default_max_vcpus(void) 196 { 197 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; 198 } 199 200 /** 201 * kvm_arch_init_vm - initializes a VM data structure 202 * @kvm: pointer to the KVM struct 203 * @type: kvm device type 204 */ 205 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 206 { 207 int ret; 208 209 mutex_init(&kvm->arch.config_lock); 210 211 #ifdef CONFIG_LOCKDEP 212 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */ 213 mutex_lock(&kvm->lock); 214 mutex_lock(&kvm->arch.config_lock); 215 mutex_unlock(&kvm->arch.config_lock); 216 mutex_unlock(&kvm->lock); 217 #endif 218 219 kvm_init_nested(kvm); 220 221 ret = kvm_share_hyp(kvm, kvm + 1); 222 if (ret) 223 return ret; 224 225 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) { 226 ret = -ENOMEM; 227 goto err_unshare_kvm; 228 } 229 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask); 230 231 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type); 232 if (ret) 233 goto err_free_cpumask; 234 235 if (is_protected_kvm_enabled()) { 236 /* 237 * If any failures occur after this is successful, make sure to 238 * call __pkvm_unreserve_vm to unreserve the VM in hyp. 239 */ 240 ret = pkvm_init_host_vm(kvm); 241 if (ret) 242 goto err_free_cpumask; 243 } 244 245 kvm_vgic_early_init(kvm); 246 247 kvm_timer_init_vm(kvm); 248 249 /* The maximum number of VCPUs is limited by the host's GIC model */ 250 kvm->max_vcpus = kvm_arm_default_max_vcpus(); 251 252 kvm_arm_init_hypercalls(kvm); 253 254 bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); 255 256 return 0; 257 258 err_free_cpumask: 259 free_cpumask_var(kvm->arch.supported_cpus); 260 err_unshare_kvm: 261 kvm_unshare_hyp(kvm, kvm + 1); 262 return ret; 263 } 264 265 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 266 { 267 return VM_FAULT_SIGBUS; 268 } 269 270 void kvm_arch_create_vm_debugfs(struct kvm *kvm) 271 { 272 kvm_sys_regs_create_debugfs(kvm); 273 kvm_s2_ptdump_create_debugfs(kvm); 274 } 275 276 static void kvm_destroy_mpidr_data(struct kvm *kvm) 277 { 278 struct kvm_mpidr_data *data; 279 280 mutex_lock(&kvm->arch.config_lock); 281 282 data = rcu_dereference_protected(kvm->arch.mpidr_data, 283 lockdep_is_held(&kvm->arch.config_lock)); 284 if (data) { 285 rcu_assign_pointer(kvm->arch.mpidr_data, NULL); 286 synchronize_rcu(); 287 kfree(data); 288 } 289 290 mutex_unlock(&kvm->arch.config_lock); 291 } 292 293 /** 294 * kvm_arch_destroy_vm - destroy the VM data structure 295 * @kvm: pointer to the KVM struct 296 */ 297 void kvm_arch_destroy_vm(struct kvm *kvm) 298 { 299 bitmap_free(kvm->arch.pmu_filter); 300 free_cpumask_var(kvm->arch.supported_cpus); 301 302 kvm_vgic_destroy(kvm); 303 304 if (is_protected_kvm_enabled()) 305 pkvm_destroy_hyp_vm(kvm); 306 307 kvm_destroy_mpidr_data(kvm); 308 309 kfree(kvm->arch.sysreg_masks); 310 kvm_destroy_vcpus(kvm); 311 312 kvm_unshare_hyp(kvm, kvm + 1); 313 314 kvm_arm_teardown_hypercalls(kvm); 315 } 316 317 static bool kvm_has_full_ptr_auth(void) 318 { 319 bool apa, gpa, api, gpi, apa3, gpa3; 320 u64 isar1, isar2, val; 321 322 /* 323 * Check that: 324 * 325 * - both Address and Generic auth are implemented for a given 326 * algorithm (Q5, IMPDEF or Q3) 327 * - only a single algorithm is implemented. 328 */ 329 if (!system_has_full_ptr_auth()) 330 return false; 331 332 isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); 333 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); 334 335 apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1); 336 val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1); 337 gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP); 338 339 api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1); 340 val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1); 341 gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP); 342 343 apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2); 344 val = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2); 345 gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP); 346 347 return (apa == gpa && api == gpi && apa3 == gpa3 && 348 (apa + api + apa3) == 1); 349 } 350 351 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 352 { 353 int r; 354 355 if (is_protected_kvm_enabled() && !kvm_pkvm_ext_allowed(kvm, ext)) 356 return 0; 357 358 switch (ext) { 359 case KVM_CAP_IRQCHIP: 360 r = vgic_present; 361 break; 362 case KVM_CAP_IOEVENTFD: 363 case KVM_CAP_USER_MEMORY: 364 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 365 case KVM_CAP_ONE_REG: 366 case KVM_CAP_ARM_PSCI: 367 case KVM_CAP_ARM_PSCI_0_2: 368 case KVM_CAP_READONLY_MEM: 369 case KVM_CAP_MP_STATE: 370 case KVM_CAP_IMMEDIATE_EXIT: 371 case KVM_CAP_VCPU_EVENTS: 372 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: 373 case KVM_CAP_ARM_NISV_TO_USER: 374 case KVM_CAP_ARM_INJECT_EXT_DABT: 375 case KVM_CAP_SET_GUEST_DEBUG: 376 case KVM_CAP_VCPU_ATTRIBUTES: 377 case KVM_CAP_PTP_KVM: 378 case KVM_CAP_ARM_SYSTEM_SUSPEND: 379 case KVM_CAP_IRQFD_RESAMPLE: 380 case KVM_CAP_COUNTER_OFFSET: 381 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: 382 case KVM_CAP_ARM_SEA_TO_USER: 383 r = 1; 384 break; 385 case KVM_CAP_SET_GUEST_DEBUG2: 386 return KVM_GUESTDBG_VALID_MASK; 387 case KVM_CAP_ARM_SET_DEVICE_ADDR: 388 r = 1; 389 break; 390 case KVM_CAP_NR_VCPUS: 391 /* 392 * ARM64 treats KVM_CAP_NR_CPUS differently from all other 393 * architectures, as it does not always bound it to 394 * KVM_CAP_MAX_VCPUS. It should not matter much because 395 * this is just an advisory value. 396 */ 397 r = min_t(unsigned int, num_online_cpus(), 398 kvm_arm_default_max_vcpus()); 399 break; 400 case KVM_CAP_MAX_VCPUS: 401 case KVM_CAP_MAX_VCPU_ID: 402 if (kvm) 403 r = kvm->max_vcpus; 404 else 405 r = kvm_arm_default_max_vcpus(); 406 break; 407 case KVM_CAP_MSI_DEVID: 408 if (!kvm) 409 r = -EINVAL; 410 else 411 r = kvm->arch.vgic.msis_require_devid; 412 break; 413 case KVM_CAP_ARM_USER_IRQ: 414 /* 415 * 1: EL1_VTIMER, EL1_PTIMER, and PMU. 416 * (bump this number if adding more devices) 417 */ 418 r = 1; 419 break; 420 case KVM_CAP_ARM_MTE: 421 r = system_supports_mte(); 422 break; 423 case KVM_CAP_STEAL_TIME: 424 r = kvm_arm_pvtime_supported(); 425 break; 426 case KVM_CAP_ARM_EL1_32BIT: 427 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); 428 break; 429 case KVM_CAP_ARM_EL2: 430 r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT); 431 break; 432 case KVM_CAP_ARM_EL2_E2H0: 433 r = cpus_have_final_cap(ARM64_HAS_HCR_NV1); 434 break; 435 case KVM_CAP_GUEST_DEBUG_HW_BPS: 436 r = get_num_brps(); 437 break; 438 case KVM_CAP_GUEST_DEBUG_HW_WPS: 439 r = get_num_wrps(); 440 break; 441 case KVM_CAP_ARM_PMU_V3: 442 r = kvm_supports_guest_pmuv3(); 443 break; 444 case KVM_CAP_ARM_INJECT_SERROR_ESR: 445 r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); 446 break; 447 case KVM_CAP_ARM_VM_IPA_SIZE: 448 r = get_kvm_ipa_limit(); 449 break; 450 case KVM_CAP_ARM_SVE: 451 r = system_supports_sve(); 452 break; 453 case KVM_CAP_ARM_PTRAUTH_ADDRESS: 454 case KVM_CAP_ARM_PTRAUTH_GENERIC: 455 r = kvm_has_full_ptr_auth(); 456 break; 457 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: 458 if (kvm) 459 r = kvm->arch.mmu.split_page_chunk_size; 460 else 461 r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT; 462 break; 463 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: 464 r = kvm_supported_block_sizes(); 465 break; 466 case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES: 467 r = BIT(0); 468 break; 469 case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED: 470 if (!kvm) 471 r = -EINVAL; 472 else 473 r = kvm_supports_cacheable_pfnmap(); 474 break; 475 476 default: 477 r = 0; 478 } 479 480 return r; 481 } 482 483 long kvm_arch_dev_ioctl(struct file *filp, 484 unsigned int ioctl, unsigned long arg) 485 { 486 return -EINVAL; 487 } 488 489 struct kvm *kvm_arch_alloc_vm(void) 490 { 491 size_t sz = sizeof(struct kvm); 492 493 if (!has_vhe()) 494 return kzalloc(sz, GFP_KERNEL_ACCOUNT); 495 496 return kvzalloc(sz, GFP_KERNEL_ACCOUNT); 497 } 498 499 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 500 { 501 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) 502 return -EBUSY; 503 504 if (id >= kvm->max_vcpus) 505 return -EINVAL; 506 507 return 0; 508 } 509 510 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 511 { 512 int err; 513 514 spin_lock_init(&vcpu->arch.mp_state_lock); 515 516 #ifdef CONFIG_LOCKDEP 517 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */ 518 mutex_lock(&vcpu->mutex); 519 mutex_lock(&vcpu->kvm->arch.config_lock); 520 mutex_unlock(&vcpu->kvm->arch.config_lock); 521 mutex_unlock(&vcpu->mutex); 522 #endif 523 524 /* Force users to call KVM_ARM_VCPU_INIT */ 525 vcpu_clear_flag(vcpu, VCPU_INITIALIZED); 526 527 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 528 529 /* Set up the timer */ 530 kvm_timer_vcpu_init(vcpu); 531 532 kvm_pmu_vcpu_init(vcpu); 533 534 kvm_arm_pvtime_vcpu_init(&vcpu->arch); 535 536 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 537 538 /* 539 * This vCPU may have been created after mpidr_data was initialized. 540 * Throw out the pre-computed mappings if that is the case which forces 541 * KVM to fall back to iteratively searching the vCPUs. 542 */ 543 kvm_destroy_mpidr_data(vcpu->kvm); 544 545 err = kvm_vgic_vcpu_init(vcpu); 546 if (err) 547 return err; 548 549 err = kvm_share_hyp(vcpu, vcpu + 1); 550 if (err) 551 kvm_vgic_vcpu_destroy(vcpu); 552 553 return err; 554 } 555 556 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 557 { 558 } 559 560 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 561 { 562 if (!is_protected_kvm_enabled()) 563 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 564 else 565 free_hyp_memcache(&vcpu->arch.pkvm_memcache); 566 kvm_timer_vcpu_terminate(vcpu); 567 kvm_pmu_vcpu_destroy(vcpu); 568 kvm_vgic_vcpu_destroy(vcpu); 569 kvm_arm_vcpu_destroy(vcpu); 570 } 571 572 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 573 { 574 575 } 576 577 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 578 { 579 580 } 581 582 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) 583 { 584 if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { 585 /* 586 * Either we're running an L2 guest, and the API/APK bits come 587 * from L1's HCR_EL2, or API/APK are both set. 588 */ 589 if (unlikely(is_nested_ctxt(vcpu))) { 590 u64 val; 591 592 val = __vcpu_sys_reg(vcpu, HCR_EL2); 593 val &= (HCR_API | HCR_APK); 594 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); 595 vcpu->arch.hcr_el2 |= val; 596 } else { 597 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); 598 } 599 600 /* 601 * Save the host keys if there is any chance for the guest 602 * to use pauth, as the entry code will reload the guest 603 * keys in that case. 604 */ 605 if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { 606 struct kvm_cpu_context *ctxt; 607 608 ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); 609 ptrauth_save_keys(ctxt); 610 } 611 } 612 } 613 614 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu) 615 { 616 if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK)) 617 return kvm_wfi_trap_policy == KVM_WFX_NOTRAP; 618 619 if (vgic_is_v5(vcpu->kvm)) 620 return single_task_running(); 621 622 return single_task_running() && 623 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 && 624 (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) || 625 vcpu->kvm->arch.vgic.nassgireq); 626 } 627 628 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) 629 { 630 if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK)) 631 return kvm_wfe_trap_policy == KVM_WFX_NOTRAP; 632 633 return single_task_running(); 634 } 635 636 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 637 { 638 struct kvm_s2_mmu *mmu; 639 int *last_ran; 640 641 if (is_protected_kvm_enabled()) 642 goto nommu; 643 644 if (vcpu_has_nv(vcpu)) 645 kvm_vcpu_load_hw_mmu(vcpu); 646 647 mmu = vcpu->arch.hw_mmu; 648 last_ran = this_cpu_ptr(mmu->last_vcpu_ran); 649 650 /* 651 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2, 652 * which happens eagerly in VHE. 653 * 654 * Also, the VMID allocator only preserves VMIDs that are active at the 655 * time of rollover, so KVM might need to grab a new VMID for the MMU if 656 * this is called from kvm_sched_in(). 657 */ 658 kvm_arm_vmid_update(&mmu->vmid); 659 660 /* 661 * We guarantee that both TLBs and I-cache are private to each 662 * vcpu. If detecting that a vcpu from the same VM has 663 * previously run on the same physical CPU, call into the 664 * hypervisor code to nuke the relevant contexts. 665 * 666 * We might get preempted before the vCPU actually runs, but 667 * over-invalidation doesn't affect correctness. 668 */ 669 if (*last_ran != vcpu->vcpu_idx) { 670 kvm_call_hyp(__kvm_flush_cpu_context, mmu); 671 *last_ran = vcpu->vcpu_idx; 672 } 673 674 nommu: 675 vcpu->cpu = cpu; 676 677 /* 678 * The timer must be loaded before the vgic to correctly set up physical 679 * interrupt deactivation in nested state (e.g. timer interrupt). 680 */ 681 kvm_timer_vcpu_load(vcpu); 682 kvm_vgic_load(vcpu); 683 kvm_vcpu_load_debug(vcpu); 684 kvm_vcpu_load_fgt(vcpu); 685 if (has_vhe()) 686 kvm_vcpu_load_vhe(vcpu); 687 kvm_arch_vcpu_load_fp(vcpu); 688 kvm_vcpu_pmu_restore_guest(vcpu); 689 if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) 690 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); 691 692 if (kvm_vcpu_should_clear_twe(vcpu)) 693 vcpu->arch.hcr_el2 &= ~HCR_TWE; 694 else 695 vcpu->arch.hcr_el2 |= HCR_TWE; 696 697 if (kvm_vcpu_should_clear_twi(vcpu)) 698 vcpu->arch.hcr_el2 &= ~HCR_TWI; 699 else 700 vcpu->arch.hcr_el2 |= HCR_TWI; 701 702 vcpu_set_pauth_traps(vcpu); 703 704 if (is_protected_kvm_enabled()) { 705 kvm_call_hyp_nvhe(__pkvm_vcpu_load, 706 vcpu->kvm->arch.pkvm.handle, 707 vcpu->vcpu_idx, vcpu->arch.hcr_el2); 708 kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, 709 &vcpu->arch.vgic_cpu.vgic_v3); 710 } 711 712 if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) 713 vcpu_set_on_unsupported_cpu(vcpu); 714 } 715 716 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 717 { 718 if (is_protected_kvm_enabled()) { 719 kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); 720 kvm_call_hyp_nvhe(__pkvm_vcpu_put); 721 } 722 723 kvm_vcpu_put_debug(vcpu); 724 kvm_arch_vcpu_put_fp(vcpu); 725 if (has_vhe()) 726 kvm_vcpu_put_vhe(vcpu); 727 kvm_timer_vcpu_put(vcpu); 728 kvm_vgic_put(vcpu); 729 kvm_vcpu_pmu_restore_host(vcpu); 730 if (vcpu_has_nv(vcpu)) 731 kvm_vcpu_put_hw_mmu(vcpu); 732 kvm_arm_vmid_clear_active(); 733 734 vcpu_clear_on_unsupported_cpu(vcpu); 735 vcpu->cpu = -1; 736 } 737 738 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) 739 { 740 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); 741 kvm_make_request(KVM_REQ_SLEEP, vcpu); 742 kvm_vcpu_kick(vcpu); 743 } 744 745 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) 746 { 747 spin_lock(&vcpu->arch.mp_state_lock); 748 __kvm_arm_vcpu_power_off(vcpu); 749 spin_unlock(&vcpu->arch.mp_state_lock); 750 } 751 752 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) 753 { 754 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; 755 } 756 757 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) 758 { 759 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED); 760 kvm_make_request(KVM_REQ_SUSPEND, vcpu); 761 kvm_vcpu_kick(vcpu); 762 } 763 764 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) 765 { 766 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED; 767 } 768 769 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 770 struct kvm_mp_state *mp_state) 771 { 772 *mp_state = READ_ONCE(vcpu->arch.mp_state); 773 774 return 0; 775 } 776 777 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 778 struct kvm_mp_state *mp_state) 779 { 780 int ret = 0; 781 782 spin_lock(&vcpu->arch.mp_state_lock); 783 784 switch (mp_state->mp_state) { 785 case KVM_MP_STATE_RUNNABLE: 786 WRITE_ONCE(vcpu->arch.mp_state, *mp_state); 787 break; 788 case KVM_MP_STATE_STOPPED: 789 __kvm_arm_vcpu_power_off(vcpu); 790 break; 791 case KVM_MP_STATE_SUSPENDED: 792 kvm_arm_vcpu_suspend(vcpu); 793 break; 794 default: 795 ret = -EINVAL; 796 } 797 798 spin_unlock(&vcpu->arch.mp_state_lock); 799 800 return ret; 801 } 802 803 /** 804 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled 805 * @v: The VCPU pointer 806 * 807 * If the guest CPU is not waiting for interrupts or an interrupt line is 808 * asserted, the CPU is by definition runnable. 809 */ 810 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 811 { 812 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE); 813 814 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) 815 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); 816 } 817 818 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 819 { 820 return vcpu_mode_priv(vcpu); 821 } 822 823 #ifdef CONFIG_GUEST_PERF_EVENTS 824 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 825 { 826 return *vcpu_pc(vcpu); 827 } 828 #endif 829 830 static void kvm_init_mpidr_data(struct kvm *kvm) 831 { 832 struct kvm_mpidr_data *data = NULL; 833 unsigned long c, mask, nr_entries; 834 u64 aff_set = 0, aff_clr = ~0UL; 835 struct kvm_vcpu *vcpu; 836 837 mutex_lock(&kvm->arch.config_lock); 838 839 if (rcu_access_pointer(kvm->arch.mpidr_data) || 840 atomic_read(&kvm->online_vcpus) == 1) 841 goto out; 842 843 kvm_for_each_vcpu(c, vcpu, kvm) { 844 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu); 845 aff_set |= aff; 846 aff_clr &= aff; 847 } 848 849 /* 850 * A significant bit can be either 0 or 1, and will only appear in 851 * aff_set. Use aff_clr to weed out the useless stuff. 852 */ 853 mask = aff_set ^ aff_clr; 854 nr_entries = BIT_ULL(hweight_long(mask)); 855 856 /* 857 * Don't let userspace fool us. If we need more than a single page 858 * to describe the compressed MPIDR array, just fall back to the 859 * iterative method. Single vcpu VMs do not need this either. 860 */ 861 if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE) 862 data = kzalloc_flex(*data, cmpidr_to_idx, nr_entries, 863 GFP_KERNEL_ACCOUNT); 864 865 if (!data) 866 goto out; 867 868 data->mpidr_mask = mask; 869 870 kvm_for_each_vcpu(c, vcpu, kvm) { 871 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu); 872 u16 index = kvm_mpidr_index(data, aff); 873 874 data->cmpidr_to_idx[index] = c; 875 } 876 877 rcu_assign_pointer(kvm->arch.mpidr_data, data); 878 out: 879 mutex_unlock(&kvm->arch.config_lock); 880 } 881 882 /* 883 * Handle both the initialisation that is being done when the vcpu is 884 * run for the first time, as well as the updates that must be 885 * performed each time we get a new thread dealing with this vcpu. 886 */ 887 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 888 { 889 struct kvm *kvm = vcpu->kvm; 890 int ret; 891 892 if (!kvm_vcpu_initialized(vcpu)) 893 return -ENOEXEC; 894 895 if (!kvm_arm_vcpu_is_finalized(vcpu)) 896 return -EPERM; 897 898 if (likely(vcpu_has_run_once(vcpu))) 899 return 0; 900 901 kvm_init_mpidr_data(kvm); 902 903 if (likely(irqchip_in_kernel(kvm))) { 904 /* 905 * Map the VGIC hardware resources before running a vcpu the 906 * first time on this VM. 907 */ 908 ret = kvm_vgic_map_resources(kvm); 909 if (ret) 910 return ret; 911 } 912 913 ret = kvm_finalize_sys_regs(vcpu); 914 if (ret) 915 return ret; 916 917 if (vcpu_has_nv(vcpu)) { 918 ret = kvm_vcpu_allocate_vncr_tlb(vcpu); 919 if (ret) 920 return ret; 921 922 ret = kvm_vgic_vcpu_nv_init(vcpu); 923 if (ret) 924 return ret; 925 } 926 927 /* 928 * This needs to happen after any restriction has been applied 929 * to the feature set. 930 */ 931 kvm_calculate_traps(vcpu); 932 933 ret = kvm_timer_enable(vcpu); 934 if (ret) 935 return ret; 936 937 if (kvm_vcpu_has_pmu(vcpu)) { 938 ret = kvm_arm_pmu_v3_enable(vcpu); 939 if (ret) 940 return ret; 941 } 942 943 ret = vgic_v5_finalize_ppi_state(kvm); 944 if (ret) 945 return ret; 946 947 if (is_protected_kvm_enabled()) { 948 ret = pkvm_create_hyp_vm(kvm); 949 if (ret) 950 return ret; 951 952 ret = pkvm_create_hyp_vcpu(vcpu); 953 if (ret) 954 return ret; 955 } 956 957 mutex_lock(&kvm->arch.config_lock); 958 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags); 959 mutex_unlock(&kvm->arch.config_lock); 960 961 return ret; 962 } 963 964 bool kvm_arch_intc_initialized(struct kvm *kvm) 965 { 966 return vgic_initialized(kvm); 967 } 968 969 void kvm_arm_halt_guest(struct kvm *kvm) 970 { 971 unsigned long i; 972 struct kvm_vcpu *vcpu; 973 974 kvm_for_each_vcpu(i, vcpu, kvm) 975 vcpu->arch.pause = true; 976 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); 977 } 978 979 void kvm_arm_resume_guest(struct kvm *kvm) 980 { 981 unsigned long i; 982 struct kvm_vcpu *vcpu; 983 984 kvm_for_each_vcpu(i, vcpu, kvm) { 985 vcpu->arch.pause = false; 986 __kvm_vcpu_wake_up(vcpu); 987 } 988 } 989 990 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) 991 { 992 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 993 994 rcuwait_wait_event(wait, 995 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), 996 TASK_INTERRUPTIBLE); 997 998 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) { 999 /* Awaken to handle a signal, request we sleep again later. */ 1000 kvm_make_request(KVM_REQ_SLEEP, vcpu); 1001 } 1002 1003 /* 1004 * Make sure we will observe a potential reset request if we've 1005 * observed a change to the power state. Pairs with the smp_wmb() in 1006 * kvm_psci_vcpu_on(). 1007 */ 1008 smp_rmb(); 1009 } 1010 1011 /** 1012 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior 1013 * @vcpu: The VCPU pointer 1014 * 1015 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until 1016 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending 1017 * on when a wake event arrives, e.g. there may already be a pending wake event. 1018 */ 1019 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) 1020 { 1021 /* 1022 * Sync back the state of the GIC CPU interface so that we have 1023 * the latest PMR and group enables. This ensures that 1024 * kvm_arch_vcpu_runnable has up-to-date data to decide whether 1025 * we have pending interrupts, e.g. when determining if the 1026 * vCPU should block. 1027 * 1028 * For the same reason, we want to tell GICv4 that we need 1029 * doorbells to be signalled, should an interrupt become pending. 1030 */ 1031 preempt_disable(); 1032 vcpu_set_flag(vcpu, IN_WFI); 1033 kvm_vgic_put(vcpu); 1034 preempt_enable(); 1035 1036 kvm_vcpu_halt(vcpu); 1037 vcpu_clear_flag(vcpu, IN_WFIT); 1038 1039 preempt_disable(); 1040 vcpu_clear_flag(vcpu, IN_WFI); 1041 kvm_vgic_load(vcpu); 1042 preempt_enable(); 1043 } 1044 1045 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) 1046 { 1047 if (!kvm_arm_vcpu_suspended(vcpu)) 1048 return 1; 1049 1050 kvm_vcpu_wfi(vcpu); 1051 1052 /* 1053 * The suspend state is sticky; we do not leave it until userspace 1054 * explicitly marks the vCPU as runnable. Request that we suspend again 1055 * later. 1056 */ 1057 kvm_make_request(KVM_REQ_SUSPEND, vcpu); 1058 1059 /* 1060 * Check to make sure the vCPU is actually runnable. If so, exit to 1061 * userspace informing it of the wakeup condition. 1062 */ 1063 if (kvm_arch_vcpu_runnable(vcpu)) { 1064 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); 1065 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP; 1066 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 1067 return 0; 1068 } 1069 1070 /* 1071 * Otherwise, we were unblocked to process a different event, such as a 1072 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to 1073 * process the event. 1074 */ 1075 return 1; 1076 } 1077 1078 /** 1079 * check_vcpu_requests - check and handle pending vCPU requests 1080 * @vcpu: the VCPU pointer 1081 * 1082 * Return: 1 if we should enter the guest 1083 * 0 if we should exit to userspace 1084 * < 0 if we should exit to userspace, where the return value indicates 1085 * an error 1086 */ 1087 static int check_vcpu_requests(struct kvm_vcpu *vcpu) 1088 { 1089 if (kvm_request_pending(vcpu)) { 1090 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) 1091 return -EIO; 1092 1093 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) 1094 kvm_vcpu_sleep(vcpu); 1095 1096 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1097 kvm_reset_vcpu(vcpu); 1098 1099 /* 1100 * Clear IRQ_PENDING requests that were made to guarantee 1101 * that a VCPU sees new virtual interrupts. 1102 */ 1103 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); 1104 1105 /* Process interrupts deactivated through a trap */ 1106 if (kvm_check_request(KVM_REQ_VGIC_PROCESS_UPDATE, vcpu)) 1107 kvm_vgic_process_async_update(vcpu); 1108 1109 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) 1110 kvm_update_stolen_time(vcpu); 1111 1112 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { 1113 /* The distributor enable bits were changed */ 1114 preempt_disable(); 1115 vgic_v4_put(vcpu); 1116 vgic_v4_load(vcpu); 1117 preempt_enable(); 1118 } 1119 1120 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) 1121 kvm_vcpu_reload_pmu(vcpu); 1122 1123 if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu)) 1124 kvm_vcpu_pmu_restore_guest(vcpu); 1125 1126 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu)) 1127 return kvm_vcpu_suspend(vcpu); 1128 1129 if (kvm_dirty_ring_check_request(vcpu)) 1130 return 0; 1131 1132 check_nested_vcpu_requests(vcpu); 1133 } 1134 1135 return 1; 1136 } 1137 1138 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) 1139 { 1140 if (likely(!vcpu_mode_is_32bit(vcpu))) 1141 return false; 1142 1143 if (vcpu_has_nv(vcpu)) 1144 return true; 1145 1146 return !kvm_supports_32bit_el0(); 1147 } 1148 1149 /** 1150 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest 1151 * @vcpu: The VCPU pointer 1152 * @ret: Pointer to write optional return code 1153 * 1154 * Returns: true if the VCPU needs to return to a preemptible + interruptible 1155 * and skip guest entry. 1156 * 1157 * This function disambiguates between two different types of exits: exits to a 1158 * preemptible + interruptible kernel context and exits to userspace. For an 1159 * exit to userspace, this function will write the return code to ret and return 1160 * true. For an exit to preemptible + interruptible kernel context (i.e. check 1161 * for pending work and re-enter), return true without writing to ret. 1162 */ 1163 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret) 1164 { 1165 struct kvm_run *run = vcpu->run; 1166 1167 /* 1168 * If we're using a userspace irqchip, then check if we need 1169 * to tell a userspace irqchip about timer or PMU level 1170 * changes and if so, exit to userspace (the actual level 1171 * state gets updated in kvm_timer_update_run and 1172 * kvm_pmu_update_run below). 1173 */ 1174 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 1175 if (kvm_timer_should_notify_user(vcpu) || 1176 kvm_pmu_should_notify_user(vcpu)) { 1177 *ret = -EINTR; 1178 run->exit_reason = KVM_EXIT_INTR; 1179 return true; 1180 } 1181 } 1182 1183 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) { 1184 run->exit_reason = KVM_EXIT_FAIL_ENTRY; 1185 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED; 1186 run->fail_entry.cpu = smp_processor_id(); 1187 *ret = 0; 1188 return true; 1189 } 1190 1191 return kvm_request_pending(vcpu) || 1192 xfer_to_guest_mode_work_pending(); 1193 } 1194 1195 /* 1196 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 1197 * the vCPU is running. 1198 * 1199 * This must be noinstr as instrumentation may make use of RCU, and this is not 1200 * safe during the EQS. 1201 */ 1202 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 1203 { 1204 int ret; 1205 1206 guest_state_enter_irqoff(); 1207 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu); 1208 guest_state_exit_irqoff(); 1209 1210 return ret; 1211 } 1212 1213 /** 1214 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code 1215 * @vcpu: The VCPU pointer 1216 * 1217 * This function is called through the VCPU_RUN ioctl called from user space. It 1218 * will execute VM code in a loop until the time slice for the process is used 1219 * or some emulation is needed from user space in which case the function will 1220 * return with return value 0 and with the kvm_run structure filled in with the 1221 * required data for the requested emulation. 1222 */ 1223 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1224 { 1225 struct kvm_run *run = vcpu->run; 1226 int ret; 1227 1228 if (run->exit_reason == KVM_EXIT_MMIO) { 1229 ret = kvm_handle_mmio_return(vcpu); 1230 if (ret <= 0) 1231 return ret; 1232 } 1233 1234 vcpu_load(vcpu); 1235 1236 if (!vcpu->wants_to_run) { 1237 ret = -EINTR; 1238 goto out; 1239 } 1240 1241 kvm_sigset_activate(vcpu); 1242 1243 ret = 1; 1244 run->exit_reason = KVM_EXIT_UNKNOWN; 1245 run->flags = 0; 1246 while (ret > 0) { 1247 /* 1248 * Check conditions before entering the guest 1249 */ 1250 ret = kvm_xfer_to_guest_mode_handle_work(vcpu); 1251 if (!ret) 1252 ret = 1; 1253 1254 if (ret > 0) 1255 ret = check_vcpu_requests(vcpu); 1256 1257 /* 1258 * Preparing the interrupts to be injected also 1259 * involves poking the GIC, which must be done in a 1260 * non-preemptible context. 1261 */ 1262 preempt_disable(); 1263 1264 kvm_nested_flush_hwstate(vcpu); 1265 1266 if (kvm_vcpu_has_pmu(vcpu)) 1267 kvm_pmu_flush_hwstate(vcpu); 1268 1269 local_irq_disable(); 1270 1271 kvm_vgic_flush_hwstate(vcpu); 1272 1273 kvm_pmu_update_vcpu_events(vcpu); 1274 1275 /* 1276 * Ensure we set mode to IN_GUEST_MODE after we disable 1277 * interrupts and before the final VCPU requests check. 1278 * See the comment in kvm_vcpu_exiting_guest_mode() and 1279 * Documentation/virt/kvm/vcpu-requests.rst 1280 */ 1281 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 1282 1283 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) { 1284 vcpu->mode = OUTSIDE_GUEST_MODE; 1285 isb(); /* Ensure work in x_flush_hwstate is committed */ 1286 if (kvm_vcpu_has_pmu(vcpu)) 1287 kvm_pmu_sync_hwstate(vcpu); 1288 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 1289 kvm_timer_sync_user(vcpu); 1290 kvm_vgic_sync_hwstate(vcpu); 1291 local_irq_enable(); 1292 preempt_enable(); 1293 continue; 1294 } 1295 1296 kvm_arch_vcpu_ctxflush_fp(vcpu); 1297 1298 /************************************************************** 1299 * Enter the guest 1300 */ 1301 trace_kvm_entry(*vcpu_pc(vcpu)); 1302 guest_timing_enter_irqoff(); 1303 1304 ret = kvm_arm_vcpu_enter_exit(vcpu); 1305 1306 vcpu->mode = OUTSIDE_GUEST_MODE; 1307 vcpu->stat.exits++; 1308 /* 1309 * Back from guest 1310 *************************************************************/ 1311 1312 /* 1313 * We must sync the PMU state before the vgic state so 1314 * that the vgic can properly sample the updated state of the 1315 * interrupt line. 1316 */ 1317 if (kvm_vcpu_has_pmu(vcpu)) 1318 kvm_pmu_sync_hwstate(vcpu); 1319 1320 /* 1321 * Sync the vgic state before syncing the timer state because 1322 * the timer code needs to know if the virtual timer 1323 * interrupts are active. 1324 */ 1325 kvm_vgic_sync_hwstate(vcpu); 1326 1327 /* 1328 * Sync the timer hardware state before enabling interrupts as 1329 * we don't want vtimer interrupts to race with syncing the 1330 * timer virtual interrupt state. 1331 */ 1332 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 1333 kvm_timer_sync_user(vcpu); 1334 1335 if (is_hyp_ctxt(vcpu)) 1336 kvm_timer_sync_nested(vcpu); 1337 1338 kvm_arch_vcpu_ctxsync_fp(vcpu); 1339 1340 /* 1341 * We must ensure that any pending interrupts are taken before 1342 * we exit guest timing so that timer ticks are accounted as 1343 * guest time. Transiently unmask interrupts so that any 1344 * pending interrupts are taken. 1345 * 1346 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other 1347 * context synchronization event) is necessary to ensure that 1348 * pending interrupts are taken. 1349 */ 1350 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) { 1351 local_irq_enable(); 1352 isb(); 1353 local_irq_disable(); 1354 } 1355 1356 guest_timing_exit_irqoff(); 1357 1358 local_irq_enable(); 1359 1360 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); 1361 1362 /* Exit types that need handling before we can be preempted */ 1363 handle_exit_early(vcpu, ret); 1364 1365 kvm_nested_sync_hwstate(vcpu); 1366 1367 preempt_enable(); 1368 1369 /* 1370 * The ARMv8 architecture doesn't give the hypervisor 1371 * a mechanism to prevent a guest from dropping to AArch32 EL0 1372 * if implemented by the CPU. If we spot the guest in such 1373 * state and that we decided it wasn't supposed to do so (like 1374 * with the asymmetric AArch32 case), return to userspace with 1375 * a fatal error. 1376 */ 1377 if (vcpu_mode_is_bad_32bit(vcpu)) { 1378 /* 1379 * As we have caught the guest red-handed, decide that 1380 * it isn't fit for purpose anymore by making the vcpu 1381 * invalid. The VMM can try and fix it by issuing a 1382 * KVM_ARM_VCPU_INIT if it really wants to. 1383 */ 1384 vcpu_clear_flag(vcpu, VCPU_INITIALIZED); 1385 ret = ARM_EXCEPTION_IL; 1386 } 1387 1388 ret = handle_exit(vcpu, ret); 1389 } 1390 1391 /* Tell userspace about in-kernel device output levels */ 1392 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 1393 kvm_timer_update_run(vcpu); 1394 kvm_pmu_update_run(vcpu); 1395 } 1396 1397 kvm_sigset_deactivate(vcpu); 1398 1399 out: 1400 /* 1401 * In the unlikely event that we are returning to userspace 1402 * with pending exceptions or PC adjustment, commit these 1403 * adjustments in order to give userspace a consistent view of 1404 * the vcpu state. Note that this relies on __kvm_adjust_pc() 1405 * being preempt-safe on VHE. 1406 */ 1407 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) || 1408 vcpu_get_flag(vcpu, INCREMENT_PC))) 1409 kvm_call_hyp(__kvm_adjust_pc, vcpu); 1410 1411 vcpu_put(vcpu); 1412 return ret; 1413 } 1414 1415 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) 1416 { 1417 int bit_index; 1418 bool set; 1419 unsigned long *hcr; 1420 1421 if (number == KVM_ARM_IRQ_CPU_IRQ) 1422 bit_index = __ffs(HCR_VI); 1423 else /* KVM_ARM_IRQ_CPU_FIQ */ 1424 bit_index = __ffs(HCR_VF); 1425 1426 hcr = vcpu_hcr(vcpu); 1427 if (level) 1428 set = test_and_set_bit(bit_index, hcr); 1429 else 1430 set = test_and_clear_bit(bit_index, hcr); 1431 1432 /* 1433 * If we didn't change anything, no need to wake up or kick other CPUs 1434 */ 1435 if (set == level) 1436 return 0; 1437 1438 /* 1439 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and 1440 * trigger a world-switch round on the running physical CPU to set the 1441 * virtual IRQ/FIQ fields in the HCR appropriately. 1442 */ 1443 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 1444 kvm_vcpu_kick(vcpu); 1445 1446 return 0; 1447 } 1448 1449 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 1450 bool line_status) 1451 { 1452 u32 irq = irq_level->irq; 1453 unsigned int irq_type, vcpu_id, irq_num; 1454 struct kvm_vcpu *vcpu = NULL; 1455 bool level = irq_level->level; 1456 1457 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK; 1458 vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK; 1459 vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1); 1460 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK; 1461 1462 trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level); 1463 1464 switch (irq_type) { 1465 case KVM_ARM_IRQ_TYPE_CPU: 1466 if (irqchip_in_kernel(kvm)) 1467 return -ENXIO; 1468 1469 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 1470 if (!vcpu) 1471 return -EINVAL; 1472 1473 if (irq_num > KVM_ARM_IRQ_CPU_FIQ) 1474 return -EINVAL; 1475 1476 return vcpu_interrupt_line(vcpu, irq_num, level); 1477 case KVM_ARM_IRQ_TYPE_PPI: 1478 if (!irqchip_in_kernel(kvm)) 1479 return -ENXIO; 1480 1481 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 1482 if (!vcpu) 1483 return -EINVAL; 1484 1485 if (vgic_is_v5(kvm)) { 1486 if (irq_num >= VGIC_V5_NR_PRIVATE_IRQS) 1487 return -EINVAL; 1488 1489 /* Build a GICv5-style IntID here */ 1490 irq_num = vgic_v5_make_ppi(irq_num); 1491 } else if (irq_num < VGIC_NR_SGIS || 1492 irq_num >= VGIC_NR_PRIVATE_IRQS) { 1493 return -EINVAL; 1494 } 1495 1496 return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL); 1497 case KVM_ARM_IRQ_TYPE_SPI: 1498 if (!irqchip_in_kernel(kvm)) 1499 return -ENXIO; 1500 1501 if (vgic_is_v5(kvm)) { 1502 /* Build a GICv5-style IntID here */ 1503 irq_num = vgic_v5_make_spi(irq_num); 1504 } else { 1505 if (irq_num < VGIC_NR_PRIVATE_IRQS) 1506 return -EINVAL; 1507 } 1508 1509 return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL); 1510 } 1511 1512 return -EINVAL; 1513 } 1514 1515 static unsigned long system_supported_vcpu_features(void) 1516 { 1517 unsigned long features = KVM_VCPU_VALID_FEATURES; 1518 1519 if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) 1520 clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); 1521 1522 if (!kvm_supports_guest_pmuv3()) 1523 clear_bit(KVM_ARM_VCPU_PMU_V3, &features); 1524 1525 if (!system_supports_sve()) 1526 clear_bit(KVM_ARM_VCPU_SVE, &features); 1527 1528 if (!kvm_has_full_ptr_auth()) { 1529 clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features); 1530 clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features); 1531 } 1532 1533 if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT)) 1534 clear_bit(KVM_ARM_VCPU_HAS_EL2, &features); 1535 1536 return features; 1537 } 1538 1539 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, 1540 const struct kvm_vcpu_init *init) 1541 { 1542 unsigned long features = init->features[0]; 1543 int i; 1544 1545 if (features & ~KVM_VCPU_VALID_FEATURES) 1546 return -ENOENT; 1547 1548 for (i = 1; i < ARRAY_SIZE(init->features); i++) { 1549 if (init->features[i]) 1550 return -ENOENT; 1551 } 1552 1553 if (features & ~system_supported_vcpu_features()) 1554 return -EINVAL; 1555 1556 /* 1557 * For now make sure that both address/generic pointer authentication 1558 * features are requested by the userspace together. 1559 */ 1560 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) != 1561 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features)) 1562 return -EINVAL; 1563 1564 if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) 1565 return 0; 1566 1567 /* MTE is incompatible with AArch32 */ 1568 if (kvm_has_mte(vcpu->kvm)) 1569 return -EINVAL; 1570 1571 /* NV is incompatible with AArch32 */ 1572 if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features)) 1573 return -EINVAL; 1574 1575 return 0; 1576 } 1577 1578 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, 1579 const struct kvm_vcpu_init *init) 1580 { 1581 unsigned long features = init->features[0]; 1582 1583 return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features, 1584 KVM_VCPU_MAX_FEATURES); 1585 } 1586 1587 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) 1588 { 1589 struct kvm *kvm = vcpu->kvm; 1590 int ret = 0; 1591 1592 /* 1593 * When the vCPU has a PMU, but no PMU is set for the guest 1594 * yet, set the default one. 1595 */ 1596 if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu) 1597 ret = kvm_arm_set_default_pmu(kvm); 1598 1599 /* Prepare for nested if required */ 1600 if (!ret && vcpu_has_nv(vcpu)) 1601 ret = kvm_vcpu_init_nested(vcpu); 1602 1603 return ret; 1604 } 1605 1606 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 1607 const struct kvm_vcpu_init *init) 1608 { 1609 unsigned long features = init->features[0]; 1610 struct kvm *kvm = vcpu->kvm; 1611 int ret = -EINVAL; 1612 1613 mutex_lock(&kvm->arch.config_lock); 1614 1615 if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && 1616 kvm_vcpu_init_changed(vcpu, init)) 1617 goto out_unlock; 1618 1619 bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); 1620 1621 ret = kvm_setup_vcpu(vcpu); 1622 if (ret) 1623 goto out_unlock; 1624 1625 /* Now we know what it is, we can reset it. */ 1626 kvm_reset_vcpu(vcpu); 1627 1628 set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); 1629 vcpu_set_flag(vcpu, VCPU_INITIALIZED); 1630 ret = 0; 1631 out_unlock: 1632 mutex_unlock(&kvm->arch.config_lock); 1633 return ret; 1634 } 1635 1636 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 1637 const struct kvm_vcpu_init *init) 1638 { 1639 int ret; 1640 1641 if (init->target != KVM_ARM_TARGET_GENERIC_V8 && 1642 init->target != kvm_target_cpu()) 1643 return -EINVAL; 1644 1645 ret = kvm_vcpu_init_check_features(vcpu, init); 1646 if (ret) 1647 return ret; 1648 1649 if (!kvm_vcpu_initialized(vcpu)) 1650 return __kvm_vcpu_set_target(vcpu, init); 1651 1652 if (kvm_vcpu_init_changed(vcpu, init)) 1653 return -EINVAL; 1654 1655 kvm_reset_vcpu(vcpu); 1656 return 0; 1657 } 1658 1659 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, 1660 struct kvm_vcpu_init *init) 1661 { 1662 bool power_off = false; 1663 int ret; 1664 1665 /* 1666 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid 1667 * reflecting it in the finalized feature set, thus limiting its scope 1668 * to a single KVM_ARM_VCPU_INIT call. 1669 */ 1670 if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) { 1671 init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF); 1672 power_off = true; 1673 } 1674 1675 ret = kvm_vcpu_set_target(vcpu, init); 1676 if (ret) 1677 return ret; 1678 1679 /* 1680 * Ensure a rebooted VM will fault in RAM pages and detect if the 1681 * guest MMU is turned off and flush the caches as needed. 1682 * 1683 * S2FWB enforces all memory accesses to RAM being cacheable, 1684 * ensuring that the data side is always coherent. We still 1685 * need to invalidate the I-cache though, as FWB does *not* 1686 * imply CTR_EL0.DIC. 1687 */ 1688 if (vcpu_has_run_once(vcpu)) { 1689 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 1690 stage2_unmap_vm(vcpu->kvm); 1691 else 1692 icache_inval_all_pou(); 1693 } 1694 1695 vcpu_reset_hcr(vcpu); 1696 1697 /* 1698 * Handle the "start in power-off" case. 1699 */ 1700 spin_lock(&vcpu->arch.mp_state_lock); 1701 1702 if (power_off) 1703 __kvm_arm_vcpu_power_off(vcpu); 1704 else 1705 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); 1706 1707 spin_unlock(&vcpu->arch.mp_state_lock); 1708 1709 return 0; 1710 } 1711 1712 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, 1713 struct kvm_device_attr *attr) 1714 { 1715 int ret = -ENXIO; 1716 1717 switch (attr->group) { 1718 default: 1719 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr); 1720 break; 1721 } 1722 1723 return ret; 1724 } 1725 1726 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, 1727 struct kvm_device_attr *attr) 1728 { 1729 int ret = -ENXIO; 1730 1731 switch (attr->group) { 1732 default: 1733 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr); 1734 break; 1735 } 1736 1737 return ret; 1738 } 1739 1740 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, 1741 struct kvm_device_attr *attr) 1742 { 1743 int ret = -ENXIO; 1744 1745 switch (attr->group) { 1746 default: 1747 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr); 1748 break; 1749 } 1750 1751 return ret; 1752 } 1753 1754 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 1755 struct kvm_vcpu_events *events) 1756 { 1757 memset(events, 0, sizeof(*events)); 1758 1759 return __kvm_arm_vcpu_get_events(vcpu, events); 1760 } 1761 1762 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 1763 struct kvm_vcpu_events *events) 1764 { 1765 int i; 1766 1767 /* check whether the reserved field is zero */ 1768 for (i = 0; i < ARRAY_SIZE(events->reserved); i++) 1769 if (events->reserved[i]) 1770 return -EINVAL; 1771 1772 /* check whether the pad field is zero */ 1773 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++) 1774 if (events->exception.pad[i]) 1775 return -EINVAL; 1776 1777 return __kvm_arm_vcpu_set_events(vcpu, events); 1778 } 1779 1780 long kvm_arch_vcpu_ioctl(struct file *filp, 1781 unsigned int ioctl, unsigned long arg) 1782 { 1783 struct kvm_vcpu *vcpu = filp->private_data; 1784 void __user *argp = (void __user *)arg; 1785 struct kvm_device_attr attr; 1786 long r; 1787 1788 switch (ioctl) { 1789 case KVM_ARM_VCPU_INIT: { 1790 struct kvm_vcpu_init init; 1791 1792 r = -EFAULT; 1793 if (copy_from_user(&init, argp, sizeof(init))) 1794 break; 1795 1796 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); 1797 break; 1798 } 1799 case KVM_SET_ONE_REG: 1800 case KVM_GET_ONE_REG: { 1801 struct kvm_one_reg reg; 1802 1803 r = -ENOEXEC; 1804 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1805 break; 1806 1807 r = -EFAULT; 1808 if (copy_from_user(®, argp, sizeof(reg))) 1809 break; 1810 1811 /* 1812 * We could owe a reset due to PSCI. Handle the pending reset 1813 * here to ensure userspace register accesses are ordered after 1814 * the reset. 1815 */ 1816 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1817 kvm_reset_vcpu(vcpu); 1818 1819 if (ioctl == KVM_SET_ONE_REG) 1820 r = kvm_arm_set_reg(vcpu, ®); 1821 else 1822 r = kvm_arm_get_reg(vcpu, ®); 1823 break; 1824 } 1825 case KVM_GET_REG_LIST: { 1826 struct kvm_reg_list __user *user_list = argp; 1827 struct kvm_reg_list reg_list; 1828 unsigned n; 1829 1830 r = -ENOEXEC; 1831 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1832 break; 1833 1834 r = -EPERM; 1835 if (!kvm_arm_vcpu_is_finalized(vcpu)) 1836 break; 1837 1838 r = -EFAULT; 1839 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 1840 break; 1841 n = reg_list.n; 1842 reg_list.n = kvm_arm_num_regs(vcpu); 1843 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 1844 break; 1845 r = -E2BIG; 1846 if (n < reg_list.n) 1847 break; 1848 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg); 1849 break; 1850 } 1851 case KVM_SET_DEVICE_ATTR: { 1852 r = -EFAULT; 1853 if (copy_from_user(&attr, argp, sizeof(attr))) 1854 break; 1855 r = kvm_arm_vcpu_set_attr(vcpu, &attr); 1856 break; 1857 } 1858 case KVM_GET_DEVICE_ATTR: { 1859 r = -EFAULT; 1860 if (copy_from_user(&attr, argp, sizeof(attr))) 1861 break; 1862 r = kvm_arm_vcpu_get_attr(vcpu, &attr); 1863 break; 1864 } 1865 case KVM_HAS_DEVICE_ATTR: { 1866 r = -EFAULT; 1867 if (copy_from_user(&attr, argp, sizeof(attr))) 1868 break; 1869 r = kvm_arm_vcpu_has_attr(vcpu, &attr); 1870 break; 1871 } 1872 case KVM_GET_VCPU_EVENTS: { 1873 struct kvm_vcpu_events events; 1874 1875 if (!kvm_vcpu_initialized(vcpu)) 1876 return -ENOEXEC; 1877 1878 if (kvm_arm_vcpu_get_events(vcpu, &events)) 1879 return -EINVAL; 1880 1881 if (copy_to_user(argp, &events, sizeof(events))) 1882 return -EFAULT; 1883 1884 return 0; 1885 } 1886 case KVM_SET_VCPU_EVENTS: { 1887 struct kvm_vcpu_events events; 1888 1889 if (!kvm_vcpu_initialized(vcpu)) 1890 return -ENOEXEC; 1891 1892 if (copy_from_user(&events, argp, sizeof(events))) 1893 return -EFAULT; 1894 1895 return kvm_arm_vcpu_set_events(vcpu, &events); 1896 } 1897 case KVM_ARM_VCPU_FINALIZE: { 1898 int what; 1899 1900 if (!kvm_vcpu_initialized(vcpu)) 1901 return -ENOEXEC; 1902 1903 if (get_user(what, (const int __user *)argp)) 1904 return -EFAULT; 1905 1906 return kvm_arm_vcpu_finalize(vcpu, what); 1907 } 1908 default: 1909 r = -EINVAL; 1910 } 1911 1912 return r; 1913 } 1914 1915 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, 1916 unsigned long arg) 1917 { 1918 return -ENOIOCTLCMD; 1919 } 1920 1921 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 1922 { 1923 1924 } 1925 1926 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, 1927 struct kvm_arm_device_addr *dev_addr) 1928 { 1929 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) { 1930 case KVM_ARM_DEVICE_VGIC_V2: 1931 if (!vgic_present) 1932 return -ENXIO; 1933 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr); 1934 default: 1935 return -ENODEV; 1936 } 1937 } 1938 1939 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1940 { 1941 switch (attr->group) { 1942 case KVM_ARM_VM_SMCCC_CTRL: 1943 return kvm_vm_smccc_has_attr(kvm, attr); 1944 default: 1945 return -ENXIO; 1946 } 1947 } 1948 1949 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1950 { 1951 switch (attr->group) { 1952 case KVM_ARM_VM_SMCCC_CTRL: 1953 return kvm_vm_smccc_set_attr(kvm, attr); 1954 default: 1955 return -ENXIO; 1956 } 1957 } 1958 1959 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 1960 { 1961 struct kvm *kvm = filp->private_data; 1962 void __user *argp = (void __user *)arg; 1963 struct kvm_device_attr attr; 1964 1965 if (is_protected_kvm_enabled() && !kvm_pkvm_ioctl_allowed(kvm, ioctl)) 1966 return -EINVAL; 1967 1968 switch (ioctl) { 1969 case KVM_CREATE_IRQCHIP: { 1970 int ret; 1971 if (!vgic_present) 1972 return -ENXIO; 1973 mutex_lock(&kvm->lock); 1974 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); 1975 mutex_unlock(&kvm->lock); 1976 return ret; 1977 } 1978 case KVM_ARM_SET_DEVICE_ADDR: { 1979 struct kvm_arm_device_addr dev_addr; 1980 1981 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr))) 1982 return -EFAULT; 1983 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr); 1984 } 1985 case KVM_ARM_PREFERRED_TARGET: { 1986 struct kvm_vcpu_init init = { 1987 .target = KVM_ARM_TARGET_GENERIC_V8, 1988 }; 1989 1990 if (copy_to_user(argp, &init, sizeof(init))) 1991 return -EFAULT; 1992 1993 return 0; 1994 } 1995 case KVM_ARM_MTE_COPY_TAGS: { 1996 struct kvm_arm_copy_mte_tags copy_tags; 1997 1998 if (copy_from_user(©_tags, argp, sizeof(copy_tags))) 1999 return -EFAULT; 2000 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags); 2001 } 2002 case KVM_ARM_SET_COUNTER_OFFSET: { 2003 struct kvm_arm_counter_offset offset; 2004 2005 if (copy_from_user(&offset, argp, sizeof(offset))) 2006 return -EFAULT; 2007 return kvm_vm_ioctl_set_counter_offset(kvm, &offset); 2008 } 2009 case KVM_HAS_DEVICE_ATTR: { 2010 if (copy_from_user(&attr, argp, sizeof(attr))) 2011 return -EFAULT; 2012 2013 return kvm_vm_has_attr(kvm, &attr); 2014 } 2015 case KVM_SET_DEVICE_ATTR: { 2016 if (copy_from_user(&attr, argp, sizeof(attr))) 2017 return -EFAULT; 2018 2019 return kvm_vm_set_attr(kvm, &attr); 2020 } 2021 case KVM_ARM_GET_REG_WRITABLE_MASKS: { 2022 struct reg_mask_range range; 2023 2024 if (copy_from_user(&range, argp, sizeof(range))) 2025 return -EFAULT; 2026 return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range); 2027 } 2028 default: 2029 return -EINVAL; 2030 } 2031 } 2032 2033 static unsigned long nvhe_percpu_size(void) 2034 { 2035 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - 2036 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start); 2037 } 2038 2039 static unsigned long nvhe_percpu_order(void) 2040 { 2041 unsigned long size = nvhe_percpu_size(); 2042 2043 return size ? get_order(size) : 0; 2044 } 2045 2046 static size_t pkvm_host_sve_state_order(void) 2047 { 2048 return get_order(pkvm_host_sve_state_size()); 2049 } 2050 2051 /* A lookup table holding the hypervisor VA for each vector slot */ 2052 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS]; 2053 2054 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot) 2055 { 2056 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot); 2057 } 2058 2059 static int kvm_init_vector_slots(void) 2060 { 2061 int err; 2062 void *base; 2063 2064 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 2065 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT); 2066 2067 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); 2068 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); 2069 2070 if (kvm_system_needs_idmapped_vectors() && 2071 !is_protected_kvm_enabled()) { 2072 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), 2073 __BP_HARDEN_HYP_VECS_SZ, &base); 2074 if (err) 2075 return err; 2076 } 2077 2078 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT); 2079 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT); 2080 return 0; 2081 } 2082 2083 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) 2084 { 2085 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); 2086 unsigned long tcr; 2087 2088 /* 2089 * Calculate the raw per-cpu offset without a translation from the 2090 * kernel's mapping to the linear mapping, and store it in tpidr_el2 2091 * so that we can use adr_l to access per-cpu variables in EL2. 2092 * Also drop the KASAN tag which gets in the way... 2093 */ 2094 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) - 2095 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); 2096 2097 params->mair_el2 = read_sysreg(mair_el1); 2098 2099 tcr = read_sysreg(tcr_el1); 2100 if (cpus_have_final_cap(ARM64_KVM_HVHE)) { 2101 tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK); 2102 tcr |= TCR_EPD1_MASK; 2103 } else { 2104 unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr); 2105 2106 tcr &= TCR_EL2_MASK; 2107 tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips); 2108 if (lpa2_is_enabled()) 2109 tcr |= TCR_EL2_DS; 2110 } 2111 tcr |= TCR_T0SZ(hyp_va_bits); 2112 params->tcr_el2 = tcr; 2113 2114 params->pgd_pa = kvm_mmu_get_httbr(); 2115 if (is_protected_kvm_enabled()) 2116 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; 2117 else 2118 params->hcr_el2 = HCR_HOST_NVHE_FLAGS; 2119 2120 if (system_supports_mte()) 2121 params->hcr_el2 |= HCR_ATA; 2122 else 2123 params->hcr_el2 |= HCR_TID5; 2124 2125 if (cpus_have_final_cap(ARM64_KVM_HVHE)) 2126 params->hcr_el2 |= HCR_E2H; 2127 params->vttbr = params->vtcr = 0; 2128 2129 /* 2130 * Flush the init params from the data cache because the struct will 2131 * be read while the MMU is off. 2132 */ 2133 kvm_flush_dcache_to_poc(params, sizeof(*params)); 2134 } 2135 2136 static void hyp_install_host_vector(void) 2137 { 2138 struct kvm_nvhe_init_params *params; 2139 struct arm_smccc_res res; 2140 2141 /* Switch from the HYP stub to our own HYP init vector */ 2142 __hyp_set_vectors(kvm_get_idmap_vector()); 2143 2144 /* 2145 * Call initialization code, and switch to the full blown HYP code. 2146 * If the cpucaps haven't been finalized yet, something has gone very 2147 * wrong, and hyp will crash and burn when it uses any 2148 * cpus_have_*_cap() wrapper. 2149 */ 2150 BUG_ON(!system_capabilities_finalized()); 2151 params = this_cpu_ptr_nvhe_sym(kvm_init_params); 2152 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res); 2153 WARN_ON(res.a0 != SMCCC_RET_SUCCESS); 2154 } 2155 2156 static void cpu_init_hyp_mode(void) 2157 { 2158 hyp_install_host_vector(); 2159 2160 /* 2161 * Disabling SSBD on a non-VHE system requires us to enable SSBS 2162 * at EL2. 2163 */ 2164 if (this_cpu_has_cap(ARM64_SSBS) && 2165 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) { 2166 kvm_call_hyp_nvhe(__kvm_enable_ssbs); 2167 } 2168 } 2169 2170 static void cpu_hyp_reset(void) 2171 { 2172 if (!is_kernel_in_hyp_mode()) 2173 __hyp_reset_vectors(); 2174 } 2175 2176 /* 2177 * EL2 vectors can be mapped and rerouted in a number of ways, 2178 * depending on the kernel configuration and CPU present: 2179 * 2180 * - If the CPU is affected by Spectre-v2, the hardening sequence is 2181 * placed in one of the vector slots, which is executed before jumping 2182 * to the real vectors. 2183 * 2184 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot 2185 * containing the hardening sequence is mapped next to the idmap page, 2186 * and executed before jumping to the real vectors. 2187 * 2188 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an 2189 * empty slot is selected, mapped next to the idmap page, and 2190 * executed before jumping to the real vectors. 2191 * 2192 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with 2193 * VHE, as we don't have hypervisor-specific mappings. If the system 2194 * is VHE and yet selects this capability, it will be ignored. 2195 */ 2196 static void cpu_set_hyp_vector(void) 2197 { 2198 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 2199 void *vector = hyp_spectre_vector_selector[data->slot]; 2200 2201 if (!is_protected_kvm_enabled()) 2202 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; 2203 else 2204 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot); 2205 } 2206 2207 static void cpu_hyp_init_context(void) 2208 { 2209 kvm_init_host_cpu_context(host_data_ptr(host_ctxt)); 2210 kvm_init_host_debug_data(); 2211 2212 if (!is_kernel_in_hyp_mode()) 2213 cpu_init_hyp_mode(); 2214 } 2215 2216 static void cpu_hyp_init_features(void) 2217 { 2218 cpu_set_hyp_vector(); 2219 2220 if (is_kernel_in_hyp_mode()) { 2221 kvm_timer_init_vhe(); 2222 kvm_debug_init_vhe(); 2223 } 2224 2225 if (vgic_present) 2226 kvm_vgic_init_cpu_hardware(); 2227 } 2228 2229 static void cpu_hyp_reinit(void) 2230 { 2231 cpu_hyp_reset(); 2232 cpu_hyp_init_context(); 2233 cpu_hyp_init_features(); 2234 } 2235 2236 static void cpu_hyp_init(void *discard) 2237 { 2238 if (!__this_cpu_read(kvm_hyp_initialized)) { 2239 cpu_hyp_reinit(); 2240 __this_cpu_write(kvm_hyp_initialized, 1); 2241 } 2242 } 2243 2244 static void cpu_hyp_uninit(void *discard) 2245 { 2246 if (!is_protected_kvm_enabled() && __this_cpu_read(kvm_hyp_initialized)) { 2247 cpu_hyp_reset(); 2248 __this_cpu_write(kvm_hyp_initialized, 0); 2249 } 2250 } 2251 2252 int kvm_arch_enable_virtualization_cpu(void) 2253 { 2254 /* 2255 * Most calls to this function are made with migration 2256 * disabled, but not with preemption disabled. The former is 2257 * enough to ensure correctness, but most of the helpers 2258 * expect the later and will throw a tantrum otherwise. 2259 */ 2260 preempt_disable(); 2261 2262 cpu_hyp_init(NULL); 2263 2264 kvm_vgic_cpu_up(); 2265 kvm_timer_cpu_up(); 2266 2267 preempt_enable(); 2268 2269 return 0; 2270 } 2271 2272 void kvm_arch_disable_virtualization_cpu(void) 2273 { 2274 kvm_timer_cpu_down(); 2275 kvm_vgic_cpu_down(); 2276 2277 if (!is_protected_kvm_enabled()) 2278 cpu_hyp_uninit(NULL); 2279 } 2280 2281 #ifdef CONFIG_CPU_PM 2282 static int hyp_init_cpu_pm_notifier(struct notifier_block *self, 2283 unsigned long cmd, 2284 void *v) 2285 { 2286 /* 2287 * kvm_hyp_initialized is left with its old value over 2288 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should 2289 * re-enable hyp. 2290 */ 2291 switch (cmd) { 2292 case CPU_PM_ENTER: 2293 if (__this_cpu_read(kvm_hyp_initialized)) 2294 /* 2295 * don't update kvm_hyp_initialized here 2296 * so that the hyp will be re-enabled 2297 * when we resume. See below. 2298 */ 2299 cpu_hyp_reset(); 2300 2301 return NOTIFY_OK; 2302 case CPU_PM_ENTER_FAILED: 2303 case CPU_PM_EXIT: 2304 if (__this_cpu_read(kvm_hyp_initialized)) 2305 /* The hyp was enabled before suspend. */ 2306 cpu_hyp_reinit(); 2307 2308 return NOTIFY_OK; 2309 2310 default: 2311 return NOTIFY_DONE; 2312 } 2313 } 2314 2315 static struct notifier_block hyp_init_cpu_pm_nb = { 2316 .notifier_call = hyp_init_cpu_pm_notifier, 2317 }; 2318 2319 static void __init hyp_cpu_pm_init(void) 2320 { 2321 if (!is_protected_kvm_enabled()) 2322 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb); 2323 } 2324 static void __init hyp_cpu_pm_exit(void) 2325 { 2326 if (!is_protected_kvm_enabled()) 2327 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb); 2328 } 2329 #else 2330 static inline void __init hyp_cpu_pm_init(void) 2331 { 2332 } 2333 static inline void __init hyp_cpu_pm_exit(void) 2334 { 2335 } 2336 #endif 2337 2338 static void __init init_cpu_logical_map(void) 2339 { 2340 unsigned int cpu; 2341 2342 /* 2343 * Copy the MPIDR <-> logical CPU ID mapping to hyp. 2344 * Only copy the set of online CPUs whose features have been checked 2345 * against the finalized system capabilities. The hypervisor will not 2346 * allow any other CPUs from the `possible` set to boot. 2347 */ 2348 for_each_online_cpu(cpu) 2349 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu); 2350 } 2351 2352 #define init_psci_0_1_impl_state(config, what) \ 2353 config.psci_0_1_ ## what ## _implemented = psci_ops.what 2354 2355 static bool __init init_psci_relay(void) 2356 { 2357 /* 2358 * If PSCI has not been initialized, protected KVM cannot install 2359 * itself on newly booted CPUs. 2360 */ 2361 if (!psci_ops.get_version) { 2362 kvm_err("Cannot initialize protected mode without PSCI\n"); 2363 return false; 2364 } 2365 2366 kvm_host_psci_config.version = psci_ops.get_version(); 2367 kvm_host_psci_config.smccc_version = arm_smccc_get_version(); 2368 2369 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) { 2370 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids(); 2371 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend); 2372 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on); 2373 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off); 2374 init_psci_0_1_impl_state(kvm_host_psci_config, migrate); 2375 } 2376 return true; 2377 } 2378 2379 static int __init init_subsystems(void) 2380 { 2381 int err = 0; 2382 2383 /* 2384 * Enable hardware so that subsystem initialisation can access EL2. 2385 */ 2386 on_each_cpu(cpu_hyp_init, NULL, 1); 2387 2388 /* 2389 * Register CPU lower-power notifier 2390 */ 2391 hyp_cpu_pm_init(); 2392 2393 /* 2394 * Init HYP view of VGIC 2395 */ 2396 err = kvm_vgic_hyp_init(); 2397 switch (err) { 2398 case 0: 2399 vgic_present = true; 2400 break; 2401 case -ENODEV: 2402 case -ENXIO: 2403 /* 2404 * No VGIC? No pKVM for you. 2405 * 2406 * Protected mode assumes that VGICv3 is present, so no point 2407 * in trying to hobble along if vgic initialization fails. 2408 */ 2409 if (is_protected_kvm_enabled()) 2410 goto out; 2411 2412 /* 2413 * Otherwise, userspace could choose to implement a GIC for its 2414 * guest on non-cooperative hardware. 2415 */ 2416 vgic_present = false; 2417 err = 0; 2418 break; 2419 default: 2420 goto out; 2421 } 2422 2423 if (kvm_mode == KVM_MODE_NV && 2424 !(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 || 2425 kvm_vgic_global_state.has_gcie_v3_compat))) { 2426 kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n"); 2427 err = -EINVAL; 2428 goto out; 2429 } 2430 2431 /* 2432 * Init HYP architected timer support 2433 */ 2434 err = kvm_timer_hyp_init(vgic_present); 2435 if (err) 2436 goto out; 2437 2438 kvm_register_perf_callbacks(); 2439 2440 out: 2441 if (err) 2442 hyp_cpu_pm_exit(); 2443 2444 if (err || !is_protected_kvm_enabled()) 2445 on_each_cpu(cpu_hyp_uninit, NULL, 1); 2446 2447 return err; 2448 } 2449 2450 static void __init teardown_subsystems(void) 2451 { 2452 kvm_unregister_perf_callbacks(); 2453 hyp_cpu_pm_exit(); 2454 } 2455 2456 static void __init teardown_hyp_mode(void) 2457 { 2458 bool free_sve = system_supports_sve() && is_protected_kvm_enabled(); 2459 int cpu; 2460 2461 free_hyp_pgds(); 2462 for_each_possible_cpu(cpu) { 2463 if (per_cpu(kvm_hyp_initialized, cpu)) 2464 continue; 2465 2466 free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT); 2467 2468 if (!kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]) 2469 continue; 2470 2471 if (free_sve) { 2472 struct cpu_sve_state *sve_state; 2473 2474 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; 2475 free_pages((unsigned long) sve_state, pkvm_host_sve_state_order()); 2476 } 2477 2478 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order()); 2479 2480 } 2481 } 2482 2483 static int __init do_pkvm_init(u32 hyp_va_bits) 2484 { 2485 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)); 2486 int ret; 2487 2488 preempt_disable(); 2489 cpu_hyp_init_context(); 2490 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size, 2491 num_possible_cpus(), kern_hyp_va(per_cpu_base), 2492 hyp_va_bits); 2493 cpu_hyp_init_features(); 2494 2495 /* 2496 * The stub hypercalls are now disabled, so set our local flag to 2497 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu(). 2498 */ 2499 __this_cpu_write(kvm_hyp_initialized, 1); 2500 preempt_enable(); 2501 2502 return ret; 2503 } 2504 2505 static u64 get_hyp_id_aa64pfr0_el1(void) 2506 { 2507 /* 2508 * Track whether the system isn't affected by spectre/meltdown in the 2509 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs. 2510 * Although this is per-CPU, we make it global for simplicity, e.g., not 2511 * to have to worry about vcpu migration. 2512 * 2513 * Unlike for non-protected VMs, userspace cannot override this for 2514 * protected VMs. 2515 */ 2516 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); 2517 2518 val &= ~(ID_AA64PFR0_EL1_CSV2 | 2519 ID_AA64PFR0_EL1_CSV3); 2520 2521 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV2, 2522 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED); 2523 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV3, 2524 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED); 2525 2526 return val; 2527 } 2528 2529 static void kvm_hyp_init_symbols(void) 2530 { 2531 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1(); 2532 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); 2533 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1); 2534 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); 2535 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); 2536 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 2537 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); 2538 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1); 2539 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1); 2540 kvm_nvhe_sym(__icache_flags) = __icache_flags; 2541 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits; 2542 2543 /* Propagate the FGT state to the nVHE side */ 2544 kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks; 2545 kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks; 2546 kvm_nvhe_sym(hfgitr_masks) = hfgitr_masks; 2547 kvm_nvhe_sym(hdfgrtr_masks) = hdfgrtr_masks; 2548 kvm_nvhe_sym(hdfgwtr_masks) = hdfgwtr_masks; 2549 kvm_nvhe_sym(hafgrtr_masks) = hafgrtr_masks; 2550 kvm_nvhe_sym(hfgrtr2_masks) = hfgrtr2_masks; 2551 kvm_nvhe_sym(hfgwtr2_masks) = hfgwtr2_masks; 2552 kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks; 2553 kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks; 2554 kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks; 2555 kvm_nvhe_sym(ich_hfgrtr_masks) = ich_hfgrtr_masks; 2556 kvm_nvhe_sym(ich_hfgwtr_masks) = ich_hfgwtr_masks; 2557 kvm_nvhe_sym(ich_hfgitr_masks) = ich_hfgitr_masks; 2558 2559 /* 2560 * Flush entire BSS since part of its data containing init symbols is read 2561 * while the MMU is off. 2562 */ 2563 kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start), 2564 kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start)); 2565 } 2566 2567 static int __init kvm_hyp_init_protection(u32 hyp_va_bits) 2568 { 2569 void *addr = phys_to_virt(hyp_mem_base); 2570 int ret; 2571 2572 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP); 2573 if (ret) 2574 return ret; 2575 2576 ret = do_pkvm_init(hyp_va_bits); 2577 if (ret) 2578 return ret; 2579 2580 free_hyp_pgds(); 2581 2582 return 0; 2583 } 2584 2585 static int init_pkvm_host_sve_state(void) 2586 { 2587 int cpu; 2588 2589 if (!system_supports_sve()) 2590 return 0; 2591 2592 /* Allocate pages for host sve state in protected mode. */ 2593 for_each_possible_cpu(cpu) { 2594 struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order()); 2595 2596 if (!page) 2597 return -ENOMEM; 2598 2599 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page); 2600 } 2601 2602 /* 2603 * Don't map the pages in hyp since these are only used in protected 2604 * mode, which will (re)create its own mapping when initialized. 2605 */ 2606 2607 return 0; 2608 } 2609 2610 /* 2611 * Finalizes the initialization of hyp mode, once everything else is initialized 2612 * and the initialziation process cannot fail. 2613 */ 2614 static void finalize_init_hyp_mode(void) 2615 { 2616 int cpu; 2617 2618 if (system_supports_sve() && is_protected_kvm_enabled()) { 2619 for_each_possible_cpu(cpu) { 2620 struct cpu_sve_state *sve_state; 2621 2622 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; 2623 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = 2624 kern_hyp_va(sve_state); 2625 } 2626 } 2627 } 2628 2629 static void pkvm_hyp_init_ptrauth(void) 2630 { 2631 struct kvm_cpu_context *hyp_ctxt; 2632 int cpu; 2633 2634 for_each_possible_cpu(cpu) { 2635 hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu); 2636 hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long(); 2637 hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long(); 2638 hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long(); 2639 hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long(); 2640 hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long(); 2641 hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long(); 2642 hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long(); 2643 hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long(); 2644 hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long(); 2645 hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long(); 2646 } 2647 } 2648 2649 /* Inits Hyp-mode on all online CPUs */ 2650 static int __init init_hyp_mode(void) 2651 { 2652 u32 hyp_va_bits = kvm_hyp_va_bits(); 2653 int cpu; 2654 int err = -ENOMEM; 2655 2656 /* 2657 * The protected Hyp-mode cannot be initialized if the memory pool 2658 * allocation has failed. 2659 */ 2660 if (is_protected_kvm_enabled() && !hyp_mem_base) 2661 goto out_err; 2662 2663 /* 2664 * Allocate Hyp PGD and setup Hyp identity mapping 2665 */ 2666 err = kvm_mmu_init(hyp_va_bits); 2667 if (err) 2668 goto out_err; 2669 2670 /* 2671 * Allocate stack pages for Hypervisor-mode 2672 */ 2673 for_each_possible_cpu(cpu) { 2674 unsigned long stack_base; 2675 2676 stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT); 2677 if (!stack_base) { 2678 err = -ENOMEM; 2679 goto out_err; 2680 } 2681 2682 per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base; 2683 } 2684 2685 /* 2686 * Allocate and initialize pages for Hypervisor-mode percpu regions. 2687 */ 2688 for_each_possible_cpu(cpu) { 2689 struct page *page; 2690 void *page_addr; 2691 2692 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order()); 2693 if (!page) { 2694 err = -ENOMEM; 2695 goto out_err; 2696 } 2697 2698 page_addr = page_address(page); 2699 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size()); 2700 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr; 2701 } 2702 2703 /* 2704 * Map the Hyp-code called directly from the host 2705 */ 2706 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), 2707 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC); 2708 if (err) { 2709 kvm_err("Cannot map world-switch code\n"); 2710 goto out_err; 2711 } 2712 2713 err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start), 2714 kvm_ksym_ref(__hyp_data_end), PAGE_HYP); 2715 if (err) { 2716 kvm_err("Cannot map .hyp.data section\n"); 2717 goto out_err; 2718 } 2719 2720 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start), 2721 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO); 2722 if (err) { 2723 kvm_err("Cannot map .hyp.rodata section\n"); 2724 goto out_err; 2725 } 2726 2727 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata), 2728 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO); 2729 if (err) { 2730 kvm_err("Cannot map rodata section\n"); 2731 goto out_err; 2732 } 2733 2734 /* 2735 * .hyp.bss is guaranteed to be placed at the beginning of the .bss 2736 * section thanks to an assertion in the linker script. Map it RW and 2737 * the rest of .bss RO. 2738 */ 2739 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start), 2740 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP); 2741 if (err) { 2742 kvm_err("Cannot map hyp bss section: %d\n", err); 2743 goto out_err; 2744 } 2745 2746 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end), 2747 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); 2748 if (err) { 2749 kvm_err("Cannot map bss section\n"); 2750 goto out_err; 2751 } 2752 2753 /* 2754 * Map the Hyp stack pages 2755 */ 2756 for_each_possible_cpu(cpu) { 2757 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); 2758 char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu); 2759 2760 err = create_hyp_stack(__pa(stack_base), ¶ms->stack_hyp_va); 2761 if (err) { 2762 kvm_err("Cannot map hyp stack\n"); 2763 goto out_err; 2764 } 2765 2766 /* 2767 * Save the stack PA in nvhe_init_params. This will be needed 2768 * to recreate the stack mapping in protected nVHE mode. 2769 * __hyp_pa() won't do the right thing there, since the stack 2770 * has been mapped in the flexible private VA space. 2771 */ 2772 params->stack_pa = __pa(stack_base); 2773 } 2774 2775 for_each_possible_cpu(cpu) { 2776 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; 2777 char *percpu_end = percpu_begin + nvhe_percpu_size(); 2778 2779 /* Map Hyp percpu pages */ 2780 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); 2781 if (err) { 2782 kvm_err("Cannot map hyp percpu region\n"); 2783 goto out_err; 2784 } 2785 2786 /* Prepare the CPU initialization parameters */ 2787 cpu_prepare_hyp_mode(cpu, hyp_va_bits); 2788 } 2789 2790 kvm_hyp_init_symbols(); 2791 2792 if (is_protected_kvm_enabled()) { 2793 if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) && 2794 cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH)) 2795 pkvm_hyp_init_ptrauth(); 2796 2797 init_cpu_logical_map(); 2798 2799 if (!init_psci_relay()) { 2800 err = -ENODEV; 2801 goto out_err; 2802 } 2803 2804 err = init_pkvm_host_sve_state(); 2805 if (err) 2806 goto out_err; 2807 2808 err = kvm_hyp_init_protection(hyp_va_bits); 2809 if (err) { 2810 kvm_err("Failed to init hyp memory protection\n"); 2811 goto out_err; 2812 } 2813 } 2814 2815 return 0; 2816 2817 out_err: 2818 teardown_hyp_mode(); 2819 kvm_err("error initializing Hyp mode: %d\n", err); 2820 return err; 2821 } 2822 2823 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) 2824 { 2825 struct kvm_vcpu *vcpu = NULL; 2826 struct kvm_mpidr_data *data; 2827 unsigned long i; 2828 2829 mpidr &= MPIDR_HWID_BITMASK; 2830 2831 rcu_read_lock(); 2832 data = rcu_dereference(kvm->arch.mpidr_data); 2833 2834 if (data) { 2835 u16 idx = kvm_mpidr_index(data, mpidr); 2836 2837 vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]); 2838 if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu)) 2839 vcpu = NULL; 2840 } 2841 2842 rcu_read_unlock(); 2843 2844 if (vcpu) 2845 return vcpu; 2846 2847 kvm_for_each_vcpu(i, vcpu, kvm) { 2848 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu)) 2849 return vcpu; 2850 } 2851 return NULL; 2852 } 2853 2854 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) 2855 { 2856 return irqchip_in_kernel(kvm); 2857 } 2858 2859 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 2860 struct irq_bypass_producer *prod) 2861 { 2862 struct kvm_kernel_irqfd *irqfd = 2863 container_of(cons, struct kvm_kernel_irqfd, consumer); 2864 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry; 2865 2866 /* 2867 * The only thing we have a chance of directly-injecting is LPIs. Maybe 2868 * one day... 2869 */ 2870 if (irq_entry->type != KVM_IRQ_ROUTING_MSI) 2871 return 0; 2872 2873 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq, 2874 &irqfd->irq_entry); 2875 } 2876 2877 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 2878 struct irq_bypass_producer *prod) 2879 { 2880 struct kvm_kernel_irqfd *irqfd = 2881 container_of(cons, struct kvm_kernel_irqfd, consumer); 2882 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry; 2883 2884 if (irq_entry->type != KVM_IRQ_ROUTING_MSI) 2885 return; 2886 2887 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq); 2888 } 2889 2890 void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, 2891 struct kvm_kernel_irq_routing_entry *old, 2892 struct kvm_kernel_irq_routing_entry *new) 2893 { 2894 if (old->type == KVM_IRQ_ROUTING_MSI && 2895 new->type == KVM_IRQ_ROUTING_MSI && 2896 !memcmp(&old->msi, &new->msi, sizeof(new->msi))) 2897 return; 2898 2899 /* 2900 * Remapping the vLPI requires taking the its_lock mutex to resolve 2901 * the new translation. We're in spinlock land at this point, so no 2902 * chance of resolving the translation. 2903 * 2904 * Unmap the vLPI and fall back to software LPI injection. 2905 */ 2906 return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq); 2907 } 2908 2909 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons) 2910 { 2911 struct kvm_kernel_irqfd *irqfd = 2912 container_of(cons, struct kvm_kernel_irqfd, consumer); 2913 2914 kvm_arm_halt_guest(irqfd->kvm); 2915 } 2916 2917 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons) 2918 { 2919 struct kvm_kernel_irqfd *irqfd = 2920 container_of(cons, struct kvm_kernel_irqfd, consumer); 2921 2922 kvm_arm_resume_guest(irqfd->kvm); 2923 } 2924 2925 /* Initialize Hyp-mode and memory mappings on all CPUs */ 2926 static __init int kvm_arm_init(void) 2927 { 2928 int err; 2929 bool in_hyp_mode; 2930 2931 if (!is_hyp_mode_available()) { 2932 kvm_info("HYP mode not available\n"); 2933 return -ENODEV; 2934 } 2935 2936 if (kvm_get_mode() == KVM_MODE_NONE) { 2937 kvm_info("KVM disabled from command line\n"); 2938 return -ENODEV; 2939 } 2940 2941 err = kvm_sys_reg_table_init(); 2942 if (err) { 2943 kvm_info("Error initializing system register tables"); 2944 return err; 2945 } 2946 2947 in_hyp_mode = is_kernel_in_hyp_mode(); 2948 2949 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || 2950 cpus_have_final_cap(ARM64_WORKAROUND_1508412)) 2951 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ 2952 "Only trusted guests should be used on this system.\n"); 2953 2954 err = kvm_set_ipa_limit(); 2955 if (err) 2956 return err; 2957 2958 err = kvm_arm_init_sve(); 2959 if (err) 2960 return err; 2961 2962 err = kvm_arm_vmid_alloc_init(); 2963 if (err) { 2964 kvm_err("Failed to initialize VMID allocator.\n"); 2965 return err; 2966 } 2967 2968 if (!in_hyp_mode) { 2969 err = init_hyp_mode(); 2970 if (err) 2971 goto out_err; 2972 } 2973 2974 err = kvm_init_vector_slots(); 2975 if (err) { 2976 kvm_err("Cannot initialise vector slots\n"); 2977 goto out_hyp; 2978 } 2979 2980 err = init_subsystems(); 2981 if (err) 2982 goto out_hyp; 2983 2984 kvm_info("%s%sVHE%s mode initialized successfully\n", 2985 in_hyp_mode ? "" : (is_protected_kvm_enabled() ? 2986 "Protected " : "Hyp "), 2987 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ? 2988 "h" : "n"), 2989 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": ""); 2990 2991 /* 2992 * FIXME: Do something reasonable if kvm_init() fails after pKVM 2993 * hypervisor protection is finalized. 2994 */ 2995 err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); 2996 if (err) 2997 goto out_subs; 2998 2999 /* 3000 * This should be called after initialization is done and failure isn't 3001 * possible anymore. 3002 */ 3003 if (!in_hyp_mode) 3004 finalize_init_hyp_mode(); 3005 3006 kvm_arm_initialised = true; 3007 3008 return 0; 3009 3010 out_subs: 3011 teardown_subsystems(); 3012 out_hyp: 3013 if (!in_hyp_mode) 3014 teardown_hyp_mode(); 3015 out_err: 3016 kvm_arm_vmid_alloc_free(); 3017 return err; 3018 } 3019 3020 static int __init early_kvm_mode_cfg(char *arg) 3021 { 3022 if (!arg) 3023 return -EINVAL; 3024 3025 if (strcmp(arg, "none") == 0) { 3026 kvm_mode = KVM_MODE_NONE; 3027 return 0; 3028 } 3029 3030 if (!is_hyp_mode_available()) { 3031 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n"); 3032 return 0; 3033 } 3034 3035 if (strcmp(arg, "protected") == 0) { 3036 if (!is_kernel_in_hyp_mode()) 3037 kvm_mode = KVM_MODE_PROTECTED; 3038 else 3039 pr_warn_once("Protected KVM not available with VHE\n"); 3040 3041 return 0; 3042 } 3043 3044 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) { 3045 kvm_mode = KVM_MODE_DEFAULT; 3046 return 0; 3047 } 3048 3049 if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) { 3050 kvm_mode = KVM_MODE_NV; 3051 return 0; 3052 } 3053 3054 return -EINVAL; 3055 } 3056 early_param("kvm-arm.mode", early_kvm_mode_cfg); 3057 3058 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p) 3059 { 3060 if (!arg) 3061 return -EINVAL; 3062 3063 if (strcmp(arg, "trap") == 0) { 3064 *p = KVM_WFX_TRAP; 3065 return 0; 3066 } 3067 3068 if (strcmp(arg, "notrap") == 0) { 3069 *p = KVM_WFX_NOTRAP; 3070 return 0; 3071 } 3072 3073 return -EINVAL; 3074 } 3075 3076 static int __init early_kvm_wfi_trap_policy_cfg(char *arg) 3077 { 3078 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy); 3079 } 3080 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg); 3081 3082 static int __init early_kvm_wfe_trap_policy_cfg(char *arg) 3083 { 3084 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy); 3085 } 3086 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg); 3087 3088 enum kvm_mode kvm_get_mode(void) 3089 { 3090 return kvm_mode; 3091 } 3092 3093 module_init(kvm_arm_init); 3094