1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 4 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/cpu_pm.h> 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/kvm_host.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/vmalloc.h> 15 #include <linux/fs.h> 16 #include <linux/mman.h> 17 #include <linux/sched.h> 18 #include <linux/kvm.h> 19 #include <linux/kvm_irqfd.h> 20 #include <linux/irqbypass.h> 21 #include <linux/sched/stat.h> 22 #include <linux/psci.h> 23 #include <trace/events/kvm.h> 24 25 #define CREATE_TRACE_POINTS 26 #include "trace_arm.h" 27 28 #include <linux/uaccess.h> 29 #include <asm/ptrace.h> 30 #include <asm/mman.h> 31 #include <asm/tlbflush.h> 32 #include <asm/cacheflush.h> 33 #include <asm/cpufeature.h> 34 #include <asm/virt.h> 35 #include <asm/kvm_arm.h> 36 #include <asm/kvm_asm.h> 37 #include <asm/kvm_mmu.h> 38 #include <asm/kvm_emulate.h> 39 #include <asm/sections.h> 40 41 #include <kvm/arm_hypercalls.h> 42 #include <kvm/arm_pmu.h> 43 #include <kvm/arm_psci.h> 44 45 #ifdef REQUIRES_VIRT 46 __asm__(".arch_extension virt"); 47 #endif 48 49 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; 50 DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); 51 52 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); 53 54 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page); 55 unsigned long kvm_arm_hyp_percpu_base[NR_CPUS]; 56 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 57 58 /* The VMID used in the VTTBR */ 59 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1); 60 static u32 kvm_next_vmid; 61 static DEFINE_SPINLOCK(kvm_vmid_lock); 62 63 static bool vgic_present; 64 65 static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled); 66 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use); 67 68 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 69 { 70 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 71 } 72 73 int kvm_arch_hardware_setup(void *opaque) 74 { 75 return 0; 76 } 77 78 int kvm_arch_check_processor_compat(void *opaque) 79 { 80 return 0; 81 } 82 83 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 84 struct kvm_enable_cap *cap) 85 { 86 int r; 87 88 if (cap->flags) 89 return -EINVAL; 90 91 switch (cap->cap) { 92 case KVM_CAP_ARM_NISV_TO_USER: 93 r = 0; 94 kvm->arch.return_nisv_io_abort_to_user = true; 95 break; 96 default: 97 r = -EINVAL; 98 break; 99 } 100 101 return r; 102 } 103 104 static int kvm_arm_default_max_vcpus(void) 105 { 106 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; 107 } 108 109 static void set_default_spectre(struct kvm *kvm) 110 { 111 /* 112 * The default is to expose CSV2 == 1 if the HW isn't affected. 113 * Although this is a per-CPU feature, we make it global because 114 * asymmetric systems are just a nuisance. 115 * 116 * Userspace can override this as long as it doesn't promise 117 * the impossible. 118 */ 119 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) 120 kvm->arch.pfr0_csv2 = 1; 121 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) 122 kvm->arch.pfr0_csv3 = 1; 123 } 124 125 /** 126 * kvm_arch_init_vm - initializes a VM data structure 127 * @kvm: pointer to the KVM struct 128 */ 129 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 130 { 131 int ret; 132 133 ret = kvm_arm_setup_stage2(kvm, type); 134 if (ret) 135 return ret; 136 137 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu); 138 if (ret) 139 return ret; 140 141 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP); 142 if (ret) 143 goto out_free_stage2_pgd; 144 145 kvm_vgic_early_init(kvm); 146 147 /* The maximum number of VCPUs is limited by the host's GIC model */ 148 kvm->arch.max_vcpus = kvm_arm_default_max_vcpus(); 149 150 set_default_spectre(kvm); 151 152 return ret; 153 out_free_stage2_pgd: 154 kvm_free_stage2_pgd(&kvm->arch.mmu); 155 return ret; 156 } 157 158 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 159 { 160 return VM_FAULT_SIGBUS; 161 } 162 163 164 /** 165 * kvm_arch_destroy_vm - destroy the VM data structure 166 * @kvm: pointer to the KVM struct 167 */ 168 void kvm_arch_destroy_vm(struct kvm *kvm) 169 { 170 int i; 171 172 bitmap_free(kvm->arch.pmu_filter); 173 174 kvm_vgic_destroy(kvm); 175 176 for (i = 0; i < KVM_MAX_VCPUS; ++i) { 177 if (kvm->vcpus[i]) { 178 kvm_vcpu_destroy(kvm->vcpus[i]); 179 kvm->vcpus[i] = NULL; 180 } 181 } 182 atomic_set(&kvm->online_vcpus, 0); 183 } 184 185 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 186 { 187 int r; 188 switch (ext) { 189 case KVM_CAP_IRQCHIP: 190 r = vgic_present; 191 break; 192 case KVM_CAP_IOEVENTFD: 193 case KVM_CAP_DEVICE_CTRL: 194 case KVM_CAP_USER_MEMORY: 195 case KVM_CAP_SYNC_MMU: 196 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 197 case KVM_CAP_ONE_REG: 198 case KVM_CAP_ARM_PSCI: 199 case KVM_CAP_ARM_PSCI_0_2: 200 case KVM_CAP_READONLY_MEM: 201 case KVM_CAP_MP_STATE: 202 case KVM_CAP_IMMEDIATE_EXIT: 203 case KVM_CAP_VCPU_EVENTS: 204 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: 205 case KVM_CAP_ARM_NISV_TO_USER: 206 case KVM_CAP_ARM_INJECT_EXT_DABT: 207 case KVM_CAP_SET_GUEST_DEBUG: 208 case KVM_CAP_VCPU_ATTRIBUTES: 209 r = 1; 210 break; 211 case KVM_CAP_ARM_SET_DEVICE_ADDR: 212 r = 1; 213 break; 214 case KVM_CAP_NR_VCPUS: 215 r = num_online_cpus(); 216 break; 217 case KVM_CAP_MAX_VCPUS: 218 case KVM_CAP_MAX_VCPU_ID: 219 if (kvm) 220 r = kvm->arch.max_vcpus; 221 else 222 r = kvm_arm_default_max_vcpus(); 223 break; 224 case KVM_CAP_MSI_DEVID: 225 if (!kvm) 226 r = -EINVAL; 227 else 228 r = kvm->arch.vgic.msis_require_devid; 229 break; 230 case KVM_CAP_ARM_USER_IRQ: 231 /* 232 * 1: EL1_VTIMER, EL1_PTIMER, and PMU. 233 * (bump this number if adding more devices) 234 */ 235 r = 1; 236 break; 237 case KVM_CAP_STEAL_TIME: 238 r = kvm_arm_pvtime_supported(); 239 break; 240 case KVM_CAP_ARM_EL1_32BIT: 241 r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1); 242 break; 243 case KVM_CAP_GUEST_DEBUG_HW_BPS: 244 r = get_num_brps(); 245 break; 246 case KVM_CAP_GUEST_DEBUG_HW_WPS: 247 r = get_num_wrps(); 248 break; 249 case KVM_CAP_ARM_PMU_V3: 250 r = kvm_arm_support_pmu_v3(); 251 break; 252 case KVM_CAP_ARM_INJECT_SERROR_ESR: 253 r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); 254 break; 255 case KVM_CAP_ARM_VM_IPA_SIZE: 256 r = get_kvm_ipa_limit(); 257 break; 258 case KVM_CAP_ARM_SVE: 259 r = system_supports_sve(); 260 break; 261 case KVM_CAP_ARM_PTRAUTH_ADDRESS: 262 case KVM_CAP_ARM_PTRAUTH_GENERIC: 263 r = system_has_full_ptr_auth(); 264 break; 265 default: 266 r = 0; 267 } 268 269 return r; 270 } 271 272 long kvm_arch_dev_ioctl(struct file *filp, 273 unsigned int ioctl, unsigned long arg) 274 { 275 return -EINVAL; 276 } 277 278 struct kvm *kvm_arch_alloc_vm(void) 279 { 280 if (!has_vhe()) 281 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 282 283 return vzalloc(sizeof(struct kvm)); 284 } 285 286 void kvm_arch_free_vm(struct kvm *kvm) 287 { 288 if (!has_vhe()) 289 kfree(kvm); 290 else 291 vfree(kvm); 292 } 293 294 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 295 { 296 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) 297 return -EBUSY; 298 299 if (id >= kvm->arch.max_vcpus) 300 return -EINVAL; 301 302 return 0; 303 } 304 305 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 306 { 307 int err; 308 309 /* Force users to call KVM_ARM_VCPU_INIT */ 310 vcpu->arch.target = -1; 311 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); 312 313 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 314 315 /* Set up the timer */ 316 kvm_timer_vcpu_init(vcpu); 317 318 kvm_pmu_vcpu_init(vcpu); 319 320 kvm_arm_reset_debug_ptr(vcpu); 321 322 kvm_arm_pvtime_vcpu_init(&vcpu->arch); 323 324 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 325 326 err = kvm_vgic_vcpu_init(vcpu); 327 if (err) 328 return err; 329 330 return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP); 331 } 332 333 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 334 { 335 } 336 337 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 338 { 339 if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm))) 340 static_branch_dec(&userspace_irqchip_in_use); 341 342 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 343 kvm_timer_vcpu_terminate(vcpu); 344 kvm_pmu_vcpu_destroy(vcpu); 345 346 kvm_arm_vcpu_destroy(vcpu); 347 } 348 349 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 350 { 351 return kvm_timer_is_pending(vcpu); 352 } 353 354 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 355 { 356 /* 357 * If we're about to block (most likely because we've just hit a 358 * WFI), we need to sync back the state of the GIC CPU interface 359 * so that we have the latest PMR and group enables. This ensures 360 * that kvm_arch_vcpu_runnable has up-to-date data to decide 361 * whether we have pending interrupts. 362 * 363 * For the same reason, we want to tell GICv4 that we need 364 * doorbells to be signalled, should an interrupt become pending. 365 */ 366 preempt_disable(); 367 kvm_vgic_vmcr_sync(vcpu); 368 vgic_v4_put(vcpu, true); 369 preempt_enable(); 370 } 371 372 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 373 { 374 preempt_disable(); 375 vgic_v4_load(vcpu); 376 preempt_enable(); 377 } 378 379 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 380 { 381 struct kvm_s2_mmu *mmu; 382 int *last_ran; 383 384 mmu = vcpu->arch.hw_mmu; 385 last_ran = this_cpu_ptr(mmu->last_vcpu_ran); 386 387 /* 388 * We guarantee that both TLBs and I-cache are private to each 389 * vcpu. If detecting that a vcpu from the same VM has 390 * previously run on the same physical CPU, call into the 391 * hypervisor code to nuke the relevant contexts. 392 * 393 * We might get preempted before the vCPU actually runs, but 394 * over-invalidation doesn't affect correctness. 395 */ 396 if (*last_ran != vcpu->vcpu_id) { 397 kvm_call_hyp(__kvm_flush_cpu_context, mmu); 398 *last_ran = vcpu->vcpu_id; 399 } 400 401 vcpu->cpu = cpu; 402 403 kvm_vgic_load(vcpu); 404 kvm_timer_vcpu_load(vcpu); 405 if (has_vhe()) 406 kvm_vcpu_load_sysregs_vhe(vcpu); 407 kvm_arch_vcpu_load_fp(vcpu); 408 kvm_vcpu_pmu_restore_guest(vcpu); 409 if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) 410 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); 411 412 if (single_task_running()) 413 vcpu_clear_wfx_traps(vcpu); 414 else 415 vcpu_set_wfx_traps(vcpu); 416 417 if (vcpu_has_ptrauth(vcpu)) 418 vcpu_ptrauth_disable(vcpu); 419 } 420 421 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 422 { 423 kvm_arch_vcpu_put_fp(vcpu); 424 if (has_vhe()) 425 kvm_vcpu_put_sysregs_vhe(vcpu); 426 kvm_timer_vcpu_put(vcpu); 427 kvm_vgic_put(vcpu); 428 kvm_vcpu_pmu_restore_host(vcpu); 429 430 vcpu->cpu = -1; 431 } 432 433 static void vcpu_power_off(struct kvm_vcpu *vcpu) 434 { 435 vcpu->arch.power_off = true; 436 kvm_make_request(KVM_REQ_SLEEP, vcpu); 437 kvm_vcpu_kick(vcpu); 438 } 439 440 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 441 struct kvm_mp_state *mp_state) 442 { 443 if (vcpu->arch.power_off) 444 mp_state->mp_state = KVM_MP_STATE_STOPPED; 445 else 446 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 447 448 return 0; 449 } 450 451 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 452 struct kvm_mp_state *mp_state) 453 { 454 int ret = 0; 455 456 switch (mp_state->mp_state) { 457 case KVM_MP_STATE_RUNNABLE: 458 vcpu->arch.power_off = false; 459 break; 460 case KVM_MP_STATE_STOPPED: 461 vcpu_power_off(vcpu); 462 break; 463 default: 464 ret = -EINVAL; 465 } 466 467 return ret; 468 } 469 470 /** 471 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled 472 * @v: The VCPU pointer 473 * 474 * If the guest CPU is not waiting for interrupts or an interrupt line is 475 * asserted, the CPU is by definition runnable. 476 */ 477 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 478 { 479 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF); 480 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) 481 && !v->arch.power_off && !v->arch.pause); 482 } 483 484 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 485 { 486 return vcpu_mode_priv(vcpu); 487 } 488 489 /* Just ensure a guest exit from a particular CPU */ 490 static void exit_vm_noop(void *info) 491 { 492 } 493 494 void force_vm_exit(const cpumask_t *mask) 495 { 496 preempt_disable(); 497 smp_call_function_many(mask, exit_vm_noop, NULL, true); 498 preempt_enable(); 499 } 500 501 /** 502 * need_new_vmid_gen - check that the VMID is still valid 503 * @vmid: The VMID to check 504 * 505 * return true if there is a new generation of VMIDs being used 506 * 507 * The hardware supports a limited set of values with the value zero reserved 508 * for the host, so we check if an assigned value belongs to a previous 509 * generation, which requires us to assign a new value. If we're the first to 510 * use a VMID for the new generation, we must flush necessary caches and TLBs 511 * on all CPUs. 512 */ 513 static bool need_new_vmid_gen(struct kvm_vmid *vmid) 514 { 515 u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen); 516 smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */ 517 return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen); 518 } 519 520 /** 521 * update_vmid - Update the vmid with a valid VMID for the current generation 522 * @vmid: The stage-2 VMID information struct 523 */ 524 static void update_vmid(struct kvm_vmid *vmid) 525 { 526 if (!need_new_vmid_gen(vmid)) 527 return; 528 529 spin_lock(&kvm_vmid_lock); 530 531 /* 532 * We need to re-check the vmid_gen here to ensure that if another vcpu 533 * already allocated a valid vmid for this vm, then this vcpu should 534 * use the same vmid. 535 */ 536 if (!need_new_vmid_gen(vmid)) { 537 spin_unlock(&kvm_vmid_lock); 538 return; 539 } 540 541 /* First user of a new VMID generation? */ 542 if (unlikely(kvm_next_vmid == 0)) { 543 atomic64_inc(&kvm_vmid_gen); 544 kvm_next_vmid = 1; 545 546 /* 547 * On SMP we know no other CPUs can use this CPU's or each 548 * other's VMID after force_vm_exit returns since the 549 * kvm_vmid_lock blocks them from reentry to the guest. 550 */ 551 force_vm_exit(cpu_all_mask); 552 /* 553 * Now broadcast TLB + ICACHE invalidation over the inner 554 * shareable domain to make sure all data structures are 555 * clean. 556 */ 557 kvm_call_hyp(__kvm_flush_vm_context); 558 } 559 560 vmid->vmid = kvm_next_vmid; 561 kvm_next_vmid++; 562 kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1; 563 564 smp_wmb(); 565 WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen)); 566 567 spin_unlock(&kvm_vmid_lock); 568 } 569 570 static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) 571 { 572 struct kvm *kvm = vcpu->kvm; 573 int ret = 0; 574 575 if (likely(vcpu->arch.has_run_once)) 576 return 0; 577 578 if (!kvm_arm_vcpu_is_finalized(vcpu)) 579 return -EPERM; 580 581 vcpu->arch.has_run_once = true; 582 583 if (likely(irqchip_in_kernel(kvm))) { 584 /* 585 * Map the VGIC hardware resources before running a vcpu the 586 * first time on this VM. 587 */ 588 ret = kvm_vgic_map_resources(kvm); 589 if (ret) 590 return ret; 591 } else { 592 /* 593 * Tell the rest of the code that there are userspace irqchip 594 * VMs in the wild. 595 */ 596 static_branch_inc(&userspace_irqchip_in_use); 597 } 598 599 ret = kvm_timer_enable(vcpu); 600 if (ret) 601 return ret; 602 603 ret = kvm_arm_pmu_v3_enable(vcpu); 604 605 return ret; 606 } 607 608 bool kvm_arch_intc_initialized(struct kvm *kvm) 609 { 610 return vgic_initialized(kvm); 611 } 612 613 void kvm_arm_halt_guest(struct kvm *kvm) 614 { 615 int i; 616 struct kvm_vcpu *vcpu; 617 618 kvm_for_each_vcpu(i, vcpu, kvm) 619 vcpu->arch.pause = true; 620 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); 621 } 622 623 void kvm_arm_resume_guest(struct kvm *kvm) 624 { 625 int i; 626 struct kvm_vcpu *vcpu; 627 628 kvm_for_each_vcpu(i, vcpu, kvm) { 629 vcpu->arch.pause = false; 630 rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); 631 } 632 } 633 634 static void vcpu_req_sleep(struct kvm_vcpu *vcpu) 635 { 636 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 637 638 rcuwait_wait_event(wait, 639 (!vcpu->arch.power_off) &&(!vcpu->arch.pause), 640 TASK_INTERRUPTIBLE); 641 642 if (vcpu->arch.power_off || vcpu->arch.pause) { 643 /* Awaken to handle a signal, request we sleep again later. */ 644 kvm_make_request(KVM_REQ_SLEEP, vcpu); 645 } 646 647 /* 648 * Make sure we will observe a potential reset request if we've 649 * observed a change to the power state. Pairs with the smp_wmb() in 650 * kvm_psci_vcpu_on(). 651 */ 652 smp_rmb(); 653 } 654 655 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) 656 { 657 return vcpu->arch.target >= 0; 658 } 659 660 static void check_vcpu_requests(struct kvm_vcpu *vcpu) 661 { 662 if (kvm_request_pending(vcpu)) { 663 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) 664 vcpu_req_sleep(vcpu); 665 666 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 667 kvm_reset_vcpu(vcpu); 668 669 /* 670 * Clear IRQ_PENDING requests that were made to guarantee 671 * that a VCPU sees new virtual interrupts. 672 */ 673 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); 674 675 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) 676 kvm_update_stolen_time(vcpu); 677 678 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { 679 /* The distributor enable bits were changed */ 680 preempt_disable(); 681 vgic_v4_put(vcpu, false); 682 vgic_v4_load(vcpu); 683 preempt_enable(); 684 } 685 } 686 } 687 688 /** 689 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code 690 * @vcpu: The VCPU pointer 691 * 692 * This function is called through the VCPU_RUN ioctl called from user space. It 693 * will execute VM code in a loop until the time slice for the process is used 694 * or some emulation is needed from user space in which case the function will 695 * return with return value 0 and with the kvm_run structure filled in with the 696 * required data for the requested emulation. 697 */ 698 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 699 { 700 struct kvm_run *run = vcpu->run; 701 int ret; 702 703 if (unlikely(!kvm_vcpu_initialized(vcpu))) 704 return -ENOEXEC; 705 706 ret = kvm_vcpu_first_run_init(vcpu); 707 if (ret) 708 return ret; 709 710 if (run->exit_reason == KVM_EXIT_MMIO) { 711 ret = kvm_handle_mmio_return(vcpu); 712 if (ret) 713 return ret; 714 } 715 716 if (run->immediate_exit) 717 return -EINTR; 718 719 vcpu_load(vcpu); 720 721 kvm_sigset_activate(vcpu); 722 723 ret = 1; 724 run->exit_reason = KVM_EXIT_UNKNOWN; 725 while (ret > 0) { 726 /* 727 * Check conditions before entering the guest 728 */ 729 cond_resched(); 730 731 update_vmid(&vcpu->arch.hw_mmu->vmid); 732 733 check_vcpu_requests(vcpu); 734 735 /* 736 * Preparing the interrupts to be injected also 737 * involves poking the GIC, which must be done in a 738 * non-preemptible context. 739 */ 740 preempt_disable(); 741 742 kvm_pmu_flush_hwstate(vcpu); 743 744 local_irq_disable(); 745 746 kvm_vgic_flush_hwstate(vcpu); 747 748 /* 749 * Exit if we have a signal pending so that we can deliver the 750 * signal to user space. 751 */ 752 if (signal_pending(current)) { 753 ret = -EINTR; 754 run->exit_reason = KVM_EXIT_INTR; 755 } 756 757 /* 758 * If we're using a userspace irqchip, then check if we need 759 * to tell a userspace irqchip about timer or PMU level 760 * changes and if so, exit to userspace (the actual level 761 * state gets updated in kvm_timer_update_run and 762 * kvm_pmu_update_run below). 763 */ 764 if (static_branch_unlikely(&userspace_irqchip_in_use)) { 765 if (kvm_timer_should_notify_user(vcpu) || 766 kvm_pmu_should_notify_user(vcpu)) { 767 ret = -EINTR; 768 run->exit_reason = KVM_EXIT_INTR; 769 } 770 } 771 772 /* 773 * Ensure we set mode to IN_GUEST_MODE after we disable 774 * interrupts and before the final VCPU requests check. 775 * See the comment in kvm_vcpu_exiting_guest_mode() and 776 * Documentation/virt/kvm/vcpu-requests.rst 777 */ 778 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 779 780 if (ret <= 0 || need_new_vmid_gen(&vcpu->arch.hw_mmu->vmid) || 781 kvm_request_pending(vcpu)) { 782 vcpu->mode = OUTSIDE_GUEST_MODE; 783 isb(); /* Ensure work in x_flush_hwstate is committed */ 784 kvm_pmu_sync_hwstate(vcpu); 785 if (static_branch_unlikely(&userspace_irqchip_in_use)) 786 kvm_timer_sync_user(vcpu); 787 kvm_vgic_sync_hwstate(vcpu); 788 local_irq_enable(); 789 preempt_enable(); 790 continue; 791 } 792 793 kvm_arm_setup_debug(vcpu); 794 795 /************************************************************** 796 * Enter the guest 797 */ 798 trace_kvm_entry(*vcpu_pc(vcpu)); 799 guest_enter_irqoff(); 800 801 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu); 802 803 vcpu->mode = OUTSIDE_GUEST_MODE; 804 vcpu->stat.exits++; 805 /* 806 * Back from guest 807 *************************************************************/ 808 809 kvm_arm_clear_debug(vcpu); 810 811 /* 812 * We must sync the PMU state before the vgic state so 813 * that the vgic can properly sample the updated state of the 814 * interrupt line. 815 */ 816 kvm_pmu_sync_hwstate(vcpu); 817 818 /* 819 * Sync the vgic state before syncing the timer state because 820 * the timer code needs to know if the virtual timer 821 * interrupts are active. 822 */ 823 kvm_vgic_sync_hwstate(vcpu); 824 825 /* 826 * Sync the timer hardware state before enabling interrupts as 827 * we don't want vtimer interrupts to race with syncing the 828 * timer virtual interrupt state. 829 */ 830 if (static_branch_unlikely(&userspace_irqchip_in_use)) 831 kvm_timer_sync_user(vcpu); 832 833 kvm_arch_vcpu_ctxsync_fp(vcpu); 834 835 /* 836 * We may have taken a host interrupt in HYP mode (ie 837 * while executing the guest). This interrupt is still 838 * pending, as we haven't serviced it yet! 839 * 840 * We're now back in SVC mode, with interrupts 841 * disabled. Enabling the interrupts now will have 842 * the effect of taking the interrupt again, in SVC 843 * mode this time. 844 */ 845 local_irq_enable(); 846 847 /* 848 * We do local_irq_enable() before calling guest_exit() so 849 * that if a timer interrupt hits while running the guest we 850 * account that tick as being spent in the guest. We enable 851 * preemption after calling guest_exit() so that if we get 852 * preempted we make sure ticks after that is not counted as 853 * guest time. 854 */ 855 guest_exit(); 856 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); 857 858 /* Exit types that need handling before we can be preempted */ 859 handle_exit_early(vcpu, ret); 860 861 preempt_enable(); 862 863 /* 864 * The ARMv8 architecture doesn't give the hypervisor 865 * a mechanism to prevent a guest from dropping to AArch32 EL0 866 * if implemented by the CPU. If we spot the guest in such 867 * state and that we decided it wasn't supposed to do so (like 868 * with the asymmetric AArch32 case), return to userspace with 869 * a fatal error. 870 */ 871 if (!system_supports_32bit_el0() && vcpu_mode_is_32bit(vcpu)) { 872 /* 873 * As we have caught the guest red-handed, decide that 874 * it isn't fit for purpose anymore by making the vcpu 875 * invalid. The VMM can try and fix it by issuing a 876 * KVM_ARM_VCPU_INIT if it really wants to. 877 */ 878 vcpu->arch.target = -1; 879 ret = ARM_EXCEPTION_IL; 880 } 881 882 ret = handle_exit(vcpu, ret); 883 } 884 885 /* Tell userspace about in-kernel device output levels */ 886 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 887 kvm_timer_update_run(vcpu); 888 kvm_pmu_update_run(vcpu); 889 } 890 891 kvm_sigset_deactivate(vcpu); 892 893 vcpu_put(vcpu); 894 return ret; 895 } 896 897 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) 898 { 899 int bit_index; 900 bool set; 901 unsigned long *hcr; 902 903 if (number == KVM_ARM_IRQ_CPU_IRQ) 904 bit_index = __ffs(HCR_VI); 905 else /* KVM_ARM_IRQ_CPU_FIQ */ 906 bit_index = __ffs(HCR_VF); 907 908 hcr = vcpu_hcr(vcpu); 909 if (level) 910 set = test_and_set_bit(bit_index, hcr); 911 else 912 set = test_and_clear_bit(bit_index, hcr); 913 914 /* 915 * If we didn't change anything, no need to wake up or kick other CPUs 916 */ 917 if (set == level) 918 return 0; 919 920 /* 921 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and 922 * trigger a world-switch round on the running physical CPU to set the 923 * virtual IRQ/FIQ fields in the HCR appropriately. 924 */ 925 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 926 kvm_vcpu_kick(vcpu); 927 928 return 0; 929 } 930 931 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 932 bool line_status) 933 { 934 u32 irq = irq_level->irq; 935 unsigned int irq_type, vcpu_idx, irq_num; 936 int nrcpus = atomic_read(&kvm->online_vcpus); 937 struct kvm_vcpu *vcpu = NULL; 938 bool level = irq_level->level; 939 940 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK; 941 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK; 942 vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1); 943 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK; 944 945 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level); 946 947 switch (irq_type) { 948 case KVM_ARM_IRQ_TYPE_CPU: 949 if (irqchip_in_kernel(kvm)) 950 return -ENXIO; 951 952 if (vcpu_idx >= nrcpus) 953 return -EINVAL; 954 955 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 956 if (!vcpu) 957 return -EINVAL; 958 959 if (irq_num > KVM_ARM_IRQ_CPU_FIQ) 960 return -EINVAL; 961 962 return vcpu_interrupt_line(vcpu, irq_num, level); 963 case KVM_ARM_IRQ_TYPE_PPI: 964 if (!irqchip_in_kernel(kvm)) 965 return -ENXIO; 966 967 if (vcpu_idx >= nrcpus) 968 return -EINVAL; 969 970 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 971 if (!vcpu) 972 return -EINVAL; 973 974 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS) 975 return -EINVAL; 976 977 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL); 978 case KVM_ARM_IRQ_TYPE_SPI: 979 if (!irqchip_in_kernel(kvm)) 980 return -ENXIO; 981 982 if (irq_num < VGIC_NR_PRIVATE_IRQS) 983 return -EINVAL; 984 985 return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL); 986 } 987 988 return -EINVAL; 989 } 990 991 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 992 const struct kvm_vcpu_init *init) 993 { 994 unsigned int i, ret; 995 int phys_target = kvm_target_cpu(); 996 997 if (init->target != phys_target) 998 return -EINVAL; 999 1000 /* 1001 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must 1002 * use the same target. 1003 */ 1004 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target) 1005 return -EINVAL; 1006 1007 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */ 1008 for (i = 0; i < sizeof(init->features) * 8; i++) { 1009 bool set = (init->features[i / 32] & (1 << (i % 32))); 1010 1011 if (set && i >= KVM_VCPU_MAX_FEATURES) 1012 return -ENOENT; 1013 1014 /* 1015 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must 1016 * use the same feature set. 1017 */ 1018 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES && 1019 test_bit(i, vcpu->arch.features) != set) 1020 return -EINVAL; 1021 1022 if (set) 1023 set_bit(i, vcpu->arch.features); 1024 } 1025 1026 vcpu->arch.target = phys_target; 1027 1028 /* Now we know what it is, we can reset it. */ 1029 ret = kvm_reset_vcpu(vcpu); 1030 if (ret) { 1031 vcpu->arch.target = -1; 1032 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); 1033 } 1034 1035 return ret; 1036 } 1037 1038 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, 1039 struct kvm_vcpu_init *init) 1040 { 1041 int ret; 1042 1043 ret = kvm_vcpu_set_target(vcpu, init); 1044 if (ret) 1045 return ret; 1046 1047 /* 1048 * Ensure a rebooted VM will fault in RAM pages and detect if the 1049 * guest MMU is turned off and flush the caches as needed. 1050 * 1051 * S2FWB enforces all memory accesses to RAM being cacheable, 1052 * ensuring that the data side is always coherent. We still 1053 * need to invalidate the I-cache though, as FWB does *not* 1054 * imply CTR_EL0.DIC. 1055 */ 1056 if (vcpu->arch.has_run_once) { 1057 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 1058 stage2_unmap_vm(vcpu->kvm); 1059 else 1060 __flush_icache_all(); 1061 } 1062 1063 vcpu_reset_hcr(vcpu); 1064 1065 /* 1066 * Handle the "start in power-off" case. 1067 */ 1068 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) 1069 vcpu_power_off(vcpu); 1070 else 1071 vcpu->arch.power_off = false; 1072 1073 return 0; 1074 } 1075 1076 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, 1077 struct kvm_device_attr *attr) 1078 { 1079 int ret = -ENXIO; 1080 1081 switch (attr->group) { 1082 default: 1083 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr); 1084 break; 1085 } 1086 1087 return ret; 1088 } 1089 1090 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, 1091 struct kvm_device_attr *attr) 1092 { 1093 int ret = -ENXIO; 1094 1095 switch (attr->group) { 1096 default: 1097 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr); 1098 break; 1099 } 1100 1101 return ret; 1102 } 1103 1104 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, 1105 struct kvm_device_attr *attr) 1106 { 1107 int ret = -ENXIO; 1108 1109 switch (attr->group) { 1110 default: 1111 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr); 1112 break; 1113 } 1114 1115 return ret; 1116 } 1117 1118 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 1119 struct kvm_vcpu_events *events) 1120 { 1121 memset(events, 0, sizeof(*events)); 1122 1123 return __kvm_arm_vcpu_get_events(vcpu, events); 1124 } 1125 1126 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 1127 struct kvm_vcpu_events *events) 1128 { 1129 int i; 1130 1131 /* check whether the reserved field is zero */ 1132 for (i = 0; i < ARRAY_SIZE(events->reserved); i++) 1133 if (events->reserved[i]) 1134 return -EINVAL; 1135 1136 /* check whether the pad field is zero */ 1137 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++) 1138 if (events->exception.pad[i]) 1139 return -EINVAL; 1140 1141 return __kvm_arm_vcpu_set_events(vcpu, events); 1142 } 1143 1144 long kvm_arch_vcpu_ioctl(struct file *filp, 1145 unsigned int ioctl, unsigned long arg) 1146 { 1147 struct kvm_vcpu *vcpu = filp->private_data; 1148 void __user *argp = (void __user *)arg; 1149 struct kvm_device_attr attr; 1150 long r; 1151 1152 switch (ioctl) { 1153 case KVM_ARM_VCPU_INIT: { 1154 struct kvm_vcpu_init init; 1155 1156 r = -EFAULT; 1157 if (copy_from_user(&init, argp, sizeof(init))) 1158 break; 1159 1160 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); 1161 break; 1162 } 1163 case KVM_SET_ONE_REG: 1164 case KVM_GET_ONE_REG: { 1165 struct kvm_one_reg reg; 1166 1167 r = -ENOEXEC; 1168 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1169 break; 1170 1171 r = -EFAULT; 1172 if (copy_from_user(®, argp, sizeof(reg))) 1173 break; 1174 1175 if (ioctl == KVM_SET_ONE_REG) 1176 r = kvm_arm_set_reg(vcpu, ®); 1177 else 1178 r = kvm_arm_get_reg(vcpu, ®); 1179 break; 1180 } 1181 case KVM_GET_REG_LIST: { 1182 struct kvm_reg_list __user *user_list = argp; 1183 struct kvm_reg_list reg_list; 1184 unsigned n; 1185 1186 r = -ENOEXEC; 1187 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1188 break; 1189 1190 r = -EPERM; 1191 if (!kvm_arm_vcpu_is_finalized(vcpu)) 1192 break; 1193 1194 r = -EFAULT; 1195 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 1196 break; 1197 n = reg_list.n; 1198 reg_list.n = kvm_arm_num_regs(vcpu); 1199 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 1200 break; 1201 r = -E2BIG; 1202 if (n < reg_list.n) 1203 break; 1204 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg); 1205 break; 1206 } 1207 case KVM_SET_DEVICE_ATTR: { 1208 r = -EFAULT; 1209 if (copy_from_user(&attr, argp, sizeof(attr))) 1210 break; 1211 r = kvm_arm_vcpu_set_attr(vcpu, &attr); 1212 break; 1213 } 1214 case KVM_GET_DEVICE_ATTR: { 1215 r = -EFAULT; 1216 if (copy_from_user(&attr, argp, sizeof(attr))) 1217 break; 1218 r = kvm_arm_vcpu_get_attr(vcpu, &attr); 1219 break; 1220 } 1221 case KVM_HAS_DEVICE_ATTR: { 1222 r = -EFAULT; 1223 if (copy_from_user(&attr, argp, sizeof(attr))) 1224 break; 1225 r = kvm_arm_vcpu_has_attr(vcpu, &attr); 1226 break; 1227 } 1228 case KVM_GET_VCPU_EVENTS: { 1229 struct kvm_vcpu_events events; 1230 1231 if (kvm_arm_vcpu_get_events(vcpu, &events)) 1232 return -EINVAL; 1233 1234 if (copy_to_user(argp, &events, sizeof(events))) 1235 return -EFAULT; 1236 1237 return 0; 1238 } 1239 case KVM_SET_VCPU_EVENTS: { 1240 struct kvm_vcpu_events events; 1241 1242 if (copy_from_user(&events, argp, sizeof(events))) 1243 return -EFAULT; 1244 1245 return kvm_arm_vcpu_set_events(vcpu, &events); 1246 } 1247 case KVM_ARM_VCPU_FINALIZE: { 1248 int what; 1249 1250 if (!kvm_vcpu_initialized(vcpu)) 1251 return -ENOEXEC; 1252 1253 if (get_user(what, (const int __user *)argp)) 1254 return -EFAULT; 1255 1256 return kvm_arm_vcpu_finalize(vcpu, what); 1257 } 1258 default: 1259 r = -EINVAL; 1260 } 1261 1262 return r; 1263 } 1264 1265 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 1266 { 1267 1268 } 1269 1270 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, 1271 struct kvm_memory_slot *memslot) 1272 { 1273 kvm_flush_remote_tlbs(kvm); 1274 } 1275 1276 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, 1277 struct kvm_arm_device_addr *dev_addr) 1278 { 1279 unsigned long dev_id, type; 1280 1281 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >> 1282 KVM_ARM_DEVICE_ID_SHIFT; 1283 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >> 1284 KVM_ARM_DEVICE_TYPE_SHIFT; 1285 1286 switch (dev_id) { 1287 case KVM_ARM_DEVICE_VGIC_V2: 1288 if (!vgic_present) 1289 return -ENXIO; 1290 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true); 1291 default: 1292 return -ENODEV; 1293 } 1294 } 1295 1296 long kvm_arch_vm_ioctl(struct file *filp, 1297 unsigned int ioctl, unsigned long arg) 1298 { 1299 struct kvm *kvm = filp->private_data; 1300 void __user *argp = (void __user *)arg; 1301 1302 switch (ioctl) { 1303 case KVM_CREATE_IRQCHIP: { 1304 int ret; 1305 if (!vgic_present) 1306 return -ENXIO; 1307 mutex_lock(&kvm->lock); 1308 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); 1309 mutex_unlock(&kvm->lock); 1310 return ret; 1311 } 1312 case KVM_ARM_SET_DEVICE_ADDR: { 1313 struct kvm_arm_device_addr dev_addr; 1314 1315 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr))) 1316 return -EFAULT; 1317 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr); 1318 } 1319 case KVM_ARM_PREFERRED_TARGET: { 1320 int err; 1321 struct kvm_vcpu_init init; 1322 1323 err = kvm_vcpu_preferred_target(&init); 1324 if (err) 1325 return err; 1326 1327 if (copy_to_user(argp, &init, sizeof(init))) 1328 return -EFAULT; 1329 1330 return 0; 1331 } 1332 default: 1333 return -EINVAL; 1334 } 1335 } 1336 1337 static unsigned long nvhe_percpu_size(void) 1338 { 1339 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - 1340 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start); 1341 } 1342 1343 static unsigned long nvhe_percpu_order(void) 1344 { 1345 unsigned long size = nvhe_percpu_size(); 1346 1347 return size ? get_order(size) : 0; 1348 } 1349 1350 /* A lookup table holding the hypervisor VA for each vector slot */ 1351 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS]; 1352 1353 static int __kvm_vector_slot2idx(enum arm64_hyp_spectre_vector slot) 1354 { 1355 return slot - (slot != HYP_VECTOR_DIRECT); 1356 } 1357 1358 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot) 1359 { 1360 int idx = __kvm_vector_slot2idx(slot); 1361 1362 hyp_spectre_vector_selector[slot] = base + (idx * SZ_2K); 1363 } 1364 1365 static int kvm_init_vector_slots(void) 1366 { 1367 int err; 1368 void *base; 1369 1370 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 1371 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT); 1372 1373 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); 1374 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); 1375 1376 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) 1377 return 0; 1378 1379 if (!has_vhe()) { 1380 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), 1381 __BP_HARDEN_HYP_VECS_SZ, &base); 1382 if (err) 1383 return err; 1384 } 1385 1386 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT); 1387 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT); 1388 return 0; 1389 } 1390 1391 static void cpu_init_hyp_mode(void) 1392 { 1393 struct kvm_nvhe_init_params *params = this_cpu_ptr_nvhe_sym(kvm_init_params); 1394 struct arm_smccc_res res; 1395 unsigned long tcr; 1396 1397 /* Switch from the HYP stub to our own HYP init vector */ 1398 __hyp_set_vectors(kvm_get_idmap_vector()); 1399 1400 /* 1401 * Calculate the raw per-cpu offset without a translation from the 1402 * kernel's mapping to the linear mapping, and store it in tpidr_el2 1403 * so that we can use adr_l to access per-cpu variables in EL2. 1404 * Also drop the KASAN tag which gets in the way... 1405 */ 1406 params->tpidr_el2 = (unsigned long)kasan_reset_tag(this_cpu_ptr_nvhe_sym(__per_cpu_start)) - 1407 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); 1408 1409 params->mair_el2 = read_sysreg(mair_el1); 1410 1411 /* 1412 * The ID map may be configured to use an extended virtual address 1413 * range. This is only the case if system RAM is out of range for the 1414 * currently configured page size and VA_BITS, in which case we will 1415 * also need the extended virtual range for the HYP ID map, or we won't 1416 * be able to enable the EL2 MMU. 1417 * 1418 * However, at EL2, there is only one TTBR register, and we can't switch 1419 * between translation tables *and* update TCR_EL2.T0SZ at the same 1420 * time. Bottom line: we need to use the extended range with *both* our 1421 * translation tables. 1422 * 1423 * So use the same T0SZ value we use for the ID map. 1424 */ 1425 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1; 1426 tcr &= ~TCR_T0SZ_MASK; 1427 tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET; 1428 params->tcr_el2 = tcr; 1429 1430 params->stack_hyp_va = kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE); 1431 params->pgd_pa = kvm_mmu_get_httbr(); 1432 1433 /* 1434 * Flush the init params from the data cache because the struct will 1435 * be read while the MMU is off. 1436 */ 1437 kvm_flush_dcache_to_poc(params, sizeof(*params)); 1438 1439 /* 1440 * Call initialization code, and switch to the full blown HYP code. 1441 * If the cpucaps haven't been finalized yet, something has gone very 1442 * wrong, and hyp will crash and burn when it uses any 1443 * cpus_have_const_cap() wrapper. 1444 */ 1445 BUG_ON(!system_capabilities_finalized()); 1446 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res); 1447 WARN_ON(res.a0 != SMCCC_RET_SUCCESS); 1448 1449 /* 1450 * Disabling SSBD on a non-VHE system requires us to enable SSBS 1451 * at EL2. 1452 */ 1453 if (this_cpu_has_cap(ARM64_SSBS) && 1454 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) { 1455 kvm_call_hyp_nvhe(__kvm_enable_ssbs); 1456 } 1457 } 1458 1459 static void cpu_hyp_reset(void) 1460 { 1461 if (!is_kernel_in_hyp_mode()) 1462 __hyp_reset_vectors(); 1463 } 1464 1465 /* 1466 * EL2 vectors can be mapped and rerouted in a number of ways, 1467 * depending on the kernel configuration and CPU present: 1468 * 1469 * - If the CPU is affected by Spectre-v2, the hardening sequence is 1470 * placed in one of the vector slots, which is executed before jumping 1471 * to the real vectors. 1472 * 1473 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot 1474 * containing the hardening sequence is mapped next to the idmap page, 1475 * and executed before jumping to the real vectors. 1476 * 1477 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an 1478 * empty slot is selected, mapped next to the idmap page, and 1479 * executed before jumping to the real vectors. 1480 * 1481 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with 1482 * VHE, as we don't have hypervisor-specific mappings. If the system 1483 * is VHE and yet selects this capability, it will be ignored. 1484 */ 1485 static void cpu_set_hyp_vector(void) 1486 { 1487 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 1488 void *vector = hyp_spectre_vector_selector[data->slot]; 1489 1490 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; 1491 } 1492 1493 static void cpu_hyp_reinit(void) 1494 { 1495 kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt); 1496 1497 cpu_hyp_reset(); 1498 cpu_set_hyp_vector(); 1499 1500 if (is_kernel_in_hyp_mode()) 1501 kvm_timer_init_vhe(); 1502 else 1503 cpu_init_hyp_mode(); 1504 1505 kvm_arm_init_debug(); 1506 1507 if (vgic_present) 1508 kvm_vgic_init_cpu_hardware(); 1509 } 1510 1511 static void _kvm_arch_hardware_enable(void *discard) 1512 { 1513 if (!__this_cpu_read(kvm_arm_hardware_enabled)) { 1514 cpu_hyp_reinit(); 1515 __this_cpu_write(kvm_arm_hardware_enabled, 1); 1516 } 1517 } 1518 1519 int kvm_arch_hardware_enable(void) 1520 { 1521 _kvm_arch_hardware_enable(NULL); 1522 return 0; 1523 } 1524 1525 static void _kvm_arch_hardware_disable(void *discard) 1526 { 1527 if (__this_cpu_read(kvm_arm_hardware_enabled)) { 1528 cpu_hyp_reset(); 1529 __this_cpu_write(kvm_arm_hardware_enabled, 0); 1530 } 1531 } 1532 1533 void kvm_arch_hardware_disable(void) 1534 { 1535 if (!is_protected_kvm_enabled()) 1536 _kvm_arch_hardware_disable(NULL); 1537 } 1538 1539 #ifdef CONFIG_CPU_PM 1540 static int hyp_init_cpu_pm_notifier(struct notifier_block *self, 1541 unsigned long cmd, 1542 void *v) 1543 { 1544 /* 1545 * kvm_arm_hardware_enabled is left with its old value over 1546 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should 1547 * re-enable hyp. 1548 */ 1549 switch (cmd) { 1550 case CPU_PM_ENTER: 1551 if (__this_cpu_read(kvm_arm_hardware_enabled)) 1552 /* 1553 * don't update kvm_arm_hardware_enabled here 1554 * so that the hardware will be re-enabled 1555 * when we resume. See below. 1556 */ 1557 cpu_hyp_reset(); 1558 1559 return NOTIFY_OK; 1560 case CPU_PM_ENTER_FAILED: 1561 case CPU_PM_EXIT: 1562 if (__this_cpu_read(kvm_arm_hardware_enabled)) 1563 /* The hardware was enabled before suspend. */ 1564 cpu_hyp_reinit(); 1565 1566 return NOTIFY_OK; 1567 1568 default: 1569 return NOTIFY_DONE; 1570 } 1571 } 1572 1573 static struct notifier_block hyp_init_cpu_pm_nb = { 1574 .notifier_call = hyp_init_cpu_pm_notifier, 1575 }; 1576 1577 static void hyp_cpu_pm_init(void) 1578 { 1579 if (!is_protected_kvm_enabled()) 1580 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb); 1581 } 1582 static void hyp_cpu_pm_exit(void) 1583 { 1584 if (!is_protected_kvm_enabled()) 1585 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb); 1586 } 1587 #else 1588 static inline void hyp_cpu_pm_init(void) 1589 { 1590 } 1591 static inline void hyp_cpu_pm_exit(void) 1592 { 1593 } 1594 #endif 1595 1596 static void init_cpu_logical_map(void) 1597 { 1598 unsigned int cpu; 1599 1600 /* 1601 * Copy the MPIDR <-> logical CPU ID mapping to hyp. 1602 * Only copy the set of online CPUs whose features have been chacked 1603 * against the finalized system capabilities. The hypervisor will not 1604 * allow any other CPUs from the `possible` set to boot. 1605 */ 1606 for_each_online_cpu(cpu) 1607 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu); 1608 } 1609 1610 #define init_psci_0_1_impl_state(config, what) \ 1611 config.psci_0_1_ ## what ## _implemented = psci_ops.what 1612 1613 static bool init_psci_relay(void) 1614 { 1615 /* 1616 * If PSCI has not been initialized, protected KVM cannot install 1617 * itself on newly booted CPUs. 1618 */ 1619 if (!psci_ops.get_version) { 1620 kvm_err("Cannot initialize protected mode without PSCI\n"); 1621 return false; 1622 } 1623 1624 kvm_host_psci_config.version = psci_ops.get_version(); 1625 1626 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) { 1627 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids(); 1628 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend); 1629 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on); 1630 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off); 1631 init_psci_0_1_impl_state(kvm_host_psci_config, migrate); 1632 } 1633 return true; 1634 } 1635 1636 static int init_common_resources(void) 1637 { 1638 return kvm_set_ipa_limit(); 1639 } 1640 1641 static int init_subsystems(void) 1642 { 1643 int err = 0; 1644 1645 /* 1646 * Enable hardware so that subsystem initialisation can access EL2. 1647 */ 1648 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1); 1649 1650 /* 1651 * Register CPU lower-power notifier 1652 */ 1653 hyp_cpu_pm_init(); 1654 1655 /* 1656 * Init HYP view of VGIC 1657 */ 1658 err = kvm_vgic_hyp_init(); 1659 switch (err) { 1660 case 0: 1661 vgic_present = true; 1662 break; 1663 case -ENODEV: 1664 case -ENXIO: 1665 vgic_present = false; 1666 err = 0; 1667 break; 1668 default: 1669 goto out; 1670 } 1671 1672 /* 1673 * Init HYP architected timer support 1674 */ 1675 err = kvm_timer_hyp_init(vgic_present); 1676 if (err) 1677 goto out; 1678 1679 kvm_perf_init(); 1680 kvm_sys_reg_table_init(); 1681 1682 out: 1683 if (err || !is_protected_kvm_enabled()) 1684 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1); 1685 1686 return err; 1687 } 1688 1689 static void teardown_hyp_mode(void) 1690 { 1691 int cpu; 1692 1693 free_hyp_pgds(); 1694 for_each_possible_cpu(cpu) { 1695 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu)); 1696 free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order()); 1697 } 1698 } 1699 1700 /** 1701 * Inits Hyp-mode on all online CPUs 1702 */ 1703 static int init_hyp_mode(void) 1704 { 1705 int cpu; 1706 int err = 0; 1707 1708 /* 1709 * Allocate Hyp PGD and setup Hyp identity mapping 1710 */ 1711 err = kvm_mmu_init(); 1712 if (err) 1713 goto out_err; 1714 1715 /* 1716 * Allocate stack pages for Hypervisor-mode 1717 */ 1718 for_each_possible_cpu(cpu) { 1719 unsigned long stack_page; 1720 1721 stack_page = __get_free_page(GFP_KERNEL); 1722 if (!stack_page) { 1723 err = -ENOMEM; 1724 goto out_err; 1725 } 1726 1727 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page; 1728 } 1729 1730 /* 1731 * Allocate and initialize pages for Hypervisor-mode percpu regions. 1732 */ 1733 for_each_possible_cpu(cpu) { 1734 struct page *page; 1735 void *page_addr; 1736 1737 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order()); 1738 if (!page) { 1739 err = -ENOMEM; 1740 goto out_err; 1741 } 1742 1743 page_addr = page_address(page); 1744 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size()); 1745 kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr; 1746 } 1747 1748 /* 1749 * Map the Hyp-code called directly from the host 1750 */ 1751 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), 1752 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC); 1753 if (err) { 1754 kvm_err("Cannot map world-switch code\n"); 1755 goto out_err; 1756 } 1757 1758 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start), 1759 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO); 1760 if (err) { 1761 kvm_err("Cannot map .hyp.rodata section\n"); 1762 goto out_err; 1763 } 1764 1765 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata), 1766 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO); 1767 if (err) { 1768 kvm_err("Cannot map rodata section\n"); 1769 goto out_err; 1770 } 1771 1772 err = create_hyp_mappings(kvm_ksym_ref(__bss_start), 1773 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); 1774 if (err) { 1775 kvm_err("Cannot map bss section\n"); 1776 goto out_err; 1777 } 1778 1779 /* 1780 * Map the Hyp stack pages 1781 */ 1782 for_each_possible_cpu(cpu) { 1783 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu); 1784 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE, 1785 PAGE_HYP); 1786 1787 if (err) { 1788 kvm_err("Cannot map hyp stack\n"); 1789 goto out_err; 1790 } 1791 } 1792 1793 /* 1794 * Map Hyp percpu pages 1795 */ 1796 for_each_possible_cpu(cpu) { 1797 char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu]; 1798 char *percpu_end = percpu_begin + nvhe_percpu_size(); 1799 1800 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); 1801 1802 if (err) { 1803 kvm_err("Cannot map hyp percpu region\n"); 1804 goto out_err; 1805 } 1806 } 1807 1808 if (is_protected_kvm_enabled()) { 1809 init_cpu_logical_map(); 1810 1811 if (!init_psci_relay()) 1812 goto out_err; 1813 } 1814 1815 return 0; 1816 1817 out_err: 1818 teardown_hyp_mode(); 1819 kvm_err("error initializing Hyp mode: %d\n", err); 1820 return err; 1821 } 1822 1823 static void check_kvm_target_cpu(void *ret) 1824 { 1825 *(int *)ret = kvm_target_cpu(); 1826 } 1827 1828 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) 1829 { 1830 struct kvm_vcpu *vcpu; 1831 int i; 1832 1833 mpidr &= MPIDR_HWID_BITMASK; 1834 kvm_for_each_vcpu(i, vcpu, kvm) { 1835 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu)) 1836 return vcpu; 1837 } 1838 return NULL; 1839 } 1840 1841 bool kvm_arch_has_irq_bypass(void) 1842 { 1843 return true; 1844 } 1845 1846 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 1847 struct irq_bypass_producer *prod) 1848 { 1849 struct kvm_kernel_irqfd *irqfd = 1850 container_of(cons, struct kvm_kernel_irqfd, consumer); 1851 1852 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq, 1853 &irqfd->irq_entry); 1854 } 1855 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 1856 struct irq_bypass_producer *prod) 1857 { 1858 struct kvm_kernel_irqfd *irqfd = 1859 container_of(cons, struct kvm_kernel_irqfd, consumer); 1860 1861 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq, 1862 &irqfd->irq_entry); 1863 } 1864 1865 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons) 1866 { 1867 struct kvm_kernel_irqfd *irqfd = 1868 container_of(cons, struct kvm_kernel_irqfd, consumer); 1869 1870 kvm_arm_halt_guest(irqfd->kvm); 1871 } 1872 1873 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons) 1874 { 1875 struct kvm_kernel_irqfd *irqfd = 1876 container_of(cons, struct kvm_kernel_irqfd, consumer); 1877 1878 kvm_arm_resume_guest(irqfd->kvm); 1879 } 1880 1881 /** 1882 * Initialize Hyp-mode and memory mappings on all CPUs. 1883 */ 1884 int kvm_arch_init(void *opaque) 1885 { 1886 int err; 1887 int ret, cpu; 1888 bool in_hyp_mode; 1889 1890 if (!is_hyp_mode_available()) { 1891 kvm_info("HYP mode not available\n"); 1892 return -ENODEV; 1893 } 1894 1895 in_hyp_mode = is_kernel_in_hyp_mode(); 1896 1897 if (!in_hyp_mode && kvm_arch_requires_vhe()) { 1898 kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n"); 1899 return -ENODEV; 1900 } 1901 1902 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || 1903 cpus_have_final_cap(ARM64_WORKAROUND_1508412)) 1904 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ 1905 "Only trusted guests should be used on this system.\n"); 1906 1907 for_each_online_cpu(cpu) { 1908 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1); 1909 if (ret < 0) { 1910 kvm_err("Error, CPU %d not supported!\n", cpu); 1911 return -ENODEV; 1912 } 1913 } 1914 1915 err = init_common_resources(); 1916 if (err) 1917 return err; 1918 1919 err = kvm_arm_init_sve(); 1920 if (err) 1921 return err; 1922 1923 if (!in_hyp_mode) { 1924 err = init_hyp_mode(); 1925 if (err) 1926 goto out_err; 1927 } 1928 1929 err = kvm_init_vector_slots(); 1930 if (err) { 1931 kvm_err("Cannot initialise vector slots\n"); 1932 goto out_err; 1933 } 1934 1935 err = init_subsystems(); 1936 if (err) 1937 goto out_hyp; 1938 1939 if (is_protected_kvm_enabled()) { 1940 static_branch_enable(&kvm_protected_mode_initialized); 1941 kvm_info("Protected nVHE mode initialized successfully\n"); 1942 } else if (in_hyp_mode) { 1943 kvm_info("VHE mode initialized successfully\n"); 1944 } else { 1945 kvm_info("Hyp mode initialized successfully\n"); 1946 } 1947 1948 return 0; 1949 1950 out_hyp: 1951 hyp_cpu_pm_exit(); 1952 if (!in_hyp_mode) 1953 teardown_hyp_mode(); 1954 out_err: 1955 return err; 1956 } 1957 1958 /* NOP: Compiling as a module not supported */ 1959 void kvm_arch_exit(void) 1960 { 1961 kvm_perf_teardown(); 1962 } 1963 1964 static int __init early_kvm_mode_cfg(char *arg) 1965 { 1966 if (!arg) 1967 return -EINVAL; 1968 1969 if (strcmp(arg, "protected") == 0) { 1970 kvm_mode = KVM_MODE_PROTECTED; 1971 return 0; 1972 } 1973 1974 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) 1975 return 0; 1976 1977 return -EINVAL; 1978 } 1979 early_param("kvm-arm.mode", early_kvm_mode_cfg); 1980 1981 enum kvm_mode kvm_get_mode(void) 1982 { 1983 return kvm_mode; 1984 } 1985 1986 static int arm_init(void) 1987 { 1988 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 1989 return rc; 1990 } 1991 1992 module_init(arm_init); 1993