1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - Google Inc 4 * Author: Andrew Scull <ascull@google.com> 5 */ 6 7 #include <hyp/adjust_pc.h> 8 #include <hyp/switch.h> 9 10 #include <asm/pgtable-types.h> 11 #include <asm/kvm_asm.h> 12 #include <asm/kvm_emulate.h> 13 #include <asm/kvm_host.h> 14 #include <asm/kvm_hyp.h> 15 #include <asm/kvm_hypevents.h> 16 #include <asm/kvm_mmu.h> 17 18 #include <nvhe/ffa.h> 19 #include <nvhe/mem_protect.h> 20 #include <nvhe/mm.h> 21 #include <nvhe/pkvm.h> 22 #include <nvhe/trace.h> 23 #include <nvhe/trap_handler.h> 24 25 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 26 27 void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); 28 29 static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) 30 { 31 __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); 32 /* 33 * On saving/restoring guest sve state, always use the maximum VL for 34 * the guest. The layout of the data when saving the sve state depends 35 * on the VL, so use a consistent (i.e., the maximum) guest VL. 36 */ 37 sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); 38 __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr, true); 39 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 40 } 41 42 static void __hyp_sve_restore_host(void) 43 { 44 struct cpu_sve_state *sve_state = *host_data_ptr(sve_state); 45 46 /* 47 * On saving/restoring host sve state, always use the maximum VL for 48 * the host. The layout of the data when saving the sve state depends 49 * on the VL, so use a consistent (i.e., the maximum) host VL. 50 * 51 * Note that this constrains the PE to the maximum shared VL 52 * that was discovered, if we wish to use larger VLs this will 53 * need to be revisited. 54 */ 55 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 56 __sve_restore_state(sve_state->sve_regs + sve_ffr_offset(kvm_host_sve_max_vl), 57 &sve_state->fpsr, 58 true); 59 write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); 60 } 61 62 static void fpsimd_sve_flush(void) 63 { 64 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; 65 } 66 67 static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) 68 { 69 bool has_fpmr; 70 71 if (!guest_owns_fp_regs()) 72 return; 73 74 /* 75 * Traps have been disabled by __deactivate_cptr_traps(), but there 76 * hasn't necessarily been a context synchronization event yet. 77 */ 78 isb(); 79 80 if (vcpu_has_sve(vcpu)) 81 __hyp_sve_save_guest(vcpu); 82 else 83 __fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); 84 85 has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm)); 86 if (has_fpmr) 87 __vcpu_assign_sys_reg(vcpu, FPMR, read_sysreg_s(SYS_FPMR)); 88 89 if (system_supports_sve()) 90 __hyp_sve_restore_host(); 91 else 92 __fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs)); 93 94 if (has_fpmr) 95 write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR); 96 97 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; 98 } 99 100 static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) 101 { 102 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 103 104 hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; 105 106 if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) 107 hyp_vcpu->vcpu.arch.vcpu_debug_state = host_vcpu->arch.vcpu_debug_state; 108 else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) 109 hyp_vcpu->vcpu.arch.external_debug_state = host_vcpu->arch.external_debug_state; 110 } 111 112 static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) 113 { 114 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 115 116 if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) 117 host_vcpu->arch.vcpu_debug_state = hyp_vcpu->vcpu.arch.vcpu_debug_state; 118 else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) 119 host_vcpu->arch.external_debug_state = hyp_vcpu->vcpu.arch.external_debug_state; 120 } 121 122 static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 123 { 124 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 125 126 fpsimd_sve_flush(); 127 flush_debug_state(hyp_vcpu); 128 129 hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; 130 131 hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; 132 hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); 133 hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & 134 (HCR_TWI | HCR_TWE); 135 136 hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; 137 138 hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; 139 140 hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; 141 142 hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; 143 } 144 145 static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 146 { 147 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 148 struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; 149 struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; 150 unsigned int i; 151 152 fpsimd_sve_sync(&hyp_vcpu->vcpu); 153 sync_debug_state(hyp_vcpu); 154 155 host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; 156 157 host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2; 158 159 host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; 160 161 host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; 162 163 host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; 164 host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; 165 for (i = 0; i < hyp_cpu_if->used_lrs; ++i) 166 host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; 167 } 168 169 static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) 170 { 171 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 172 DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2); 173 DECLARE_REG(u64, hcr_el2, host_ctxt, 3); 174 struct pkvm_hyp_vcpu *hyp_vcpu; 175 176 hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx); 177 if (!hyp_vcpu) 178 return; 179 180 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 181 /* Propagate WFx trapping flags */ 182 hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); 183 hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); 184 } else { 185 memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt, 186 sizeof(hyp_vcpu->vcpu.arch.fgt)); 187 } 188 } 189 190 static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) 191 { 192 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 193 194 if (hyp_vcpu) 195 pkvm_put_hyp_vcpu(hyp_vcpu); 196 } 197 198 static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) 199 { 200 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1); 201 int ret; 202 203 if (unlikely(is_protected_kvm_enabled())) { 204 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 205 206 /* 207 * KVM (and pKVM) doesn't support SME guests for now, and 208 * ensures that SME features aren't enabled in pstate when 209 * loading a vcpu. Therefore, if SME features enabled the host 210 * is misbehaving. 211 */ 212 if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { 213 ret = -EINVAL; 214 goto out; 215 } 216 217 if (!hyp_vcpu) { 218 ret = -EINVAL; 219 goto out; 220 } 221 222 flush_hyp_vcpu(hyp_vcpu); 223 224 ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); 225 226 sync_hyp_vcpu(hyp_vcpu); 227 } else { 228 struct kvm_vcpu *vcpu = kern_hyp_va(host_vcpu); 229 230 /* The host is fully trusted, run its vCPU directly. */ 231 fpsimd_lazy_switch_to_guest(vcpu); 232 ret = __kvm_vcpu_run(vcpu); 233 fpsimd_lazy_switch_to_host(vcpu); 234 } 235 out: 236 cpu_reg(host_ctxt, 1) = ret; 237 } 238 239 static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) 240 { 241 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 242 243 return refill_memcache(&hyp_vcpu->vcpu.arch.pkvm_memcache, 244 host_vcpu->arch.pkvm_memcache.nr_pages, 245 &host_vcpu->arch.pkvm_memcache); 246 } 247 248 static void handle___pkvm_host_donate_guest(struct kvm_cpu_context *host_ctxt) 249 { 250 DECLARE_REG(u64, pfn, host_ctxt, 1); 251 DECLARE_REG(u64, gfn, host_ctxt, 2); 252 struct pkvm_hyp_vcpu *hyp_vcpu; 253 int ret = -EINVAL; 254 255 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 256 if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 257 goto out; 258 259 ret = pkvm_refill_memcache(hyp_vcpu); 260 if (ret) 261 goto out; 262 263 ret = __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); 264 out: 265 cpu_reg(host_ctxt, 1) = ret; 266 } 267 268 static void handle___pkvm_host_share_guest(struct kvm_cpu_context *host_ctxt) 269 { 270 DECLARE_REG(u64, pfn, host_ctxt, 1); 271 DECLARE_REG(u64, gfn, host_ctxt, 2); 272 DECLARE_REG(u64, nr_pages, host_ctxt, 3); 273 DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4); 274 struct pkvm_hyp_vcpu *hyp_vcpu; 275 int ret = -EINVAL; 276 277 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 278 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 279 goto out; 280 281 ret = pkvm_refill_memcache(hyp_vcpu); 282 if (ret) 283 goto out; 284 285 ret = __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); 286 out: 287 cpu_reg(host_ctxt, 1) = ret; 288 } 289 290 static void handle___pkvm_host_unshare_guest(struct kvm_cpu_context *host_ctxt) 291 { 292 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 293 DECLARE_REG(u64, gfn, host_ctxt, 2); 294 DECLARE_REG(u64, nr_pages, host_ctxt, 3); 295 struct pkvm_hyp_vm *hyp_vm; 296 int ret = -EINVAL; 297 298 hyp_vm = get_np_pkvm_hyp_vm(handle); 299 if (!hyp_vm) 300 goto out; 301 302 ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm); 303 put_pkvm_hyp_vm(hyp_vm); 304 out: 305 cpu_reg(host_ctxt, 1) = ret; 306 } 307 308 static void handle___pkvm_host_relax_perms_guest(struct kvm_cpu_context *host_ctxt) 309 { 310 DECLARE_REG(u64, gfn, host_ctxt, 1); 311 DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 2); 312 struct pkvm_hyp_vcpu *hyp_vcpu; 313 int ret = -EINVAL; 314 315 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 316 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 317 goto out; 318 319 ret = __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); 320 out: 321 cpu_reg(host_ctxt, 1) = ret; 322 } 323 324 static void handle___pkvm_host_wrprotect_guest(struct kvm_cpu_context *host_ctxt) 325 { 326 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 327 DECLARE_REG(u64, gfn, host_ctxt, 2); 328 DECLARE_REG(u64, nr_pages, host_ctxt, 3); 329 struct pkvm_hyp_vm *hyp_vm; 330 int ret = -EINVAL; 331 332 hyp_vm = get_np_pkvm_hyp_vm(handle); 333 if (!hyp_vm) 334 goto out; 335 336 ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm); 337 put_pkvm_hyp_vm(hyp_vm); 338 out: 339 cpu_reg(host_ctxt, 1) = ret; 340 } 341 342 static void handle___pkvm_host_test_clear_young_guest(struct kvm_cpu_context *host_ctxt) 343 { 344 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 345 DECLARE_REG(u64, gfn, host_ctxt, 2); 346 DECLARE_REG(u64, nr_pages, host_ctxt, 3); 347 DECLARE_REG(bool, mkold, host_ctxt, 4); 348 struct pkvm_hyp_vm *hyp_vm; 349 int ret = -EINVAL; 350 351 hyp_vm = get_np_pkvm_hyp_vm(handle); 352 if (!hyp_vm) 353 goto out; 354 355 ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm); 356 put_pkvm_hyp_vm(hyp_vm); 357 out: 358 cpu_reg(host_ctxt, 1) = ret; 359 } 360 361 static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) 362 { 363 DECLARE_REG(u64, gfn, host_ctxt, 1); 364 struct pkvm_hyp_vcpu *hyp_vcpu; 365 int ret = -EINVAL; 366 367 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 368 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 369 goto out; 370 371 ret = __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); 372 out: 373 cpu_reg(host_ctxt, 1) = ret; 374 } 375 376 static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) 377 { 378 DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); 379 380 __kvm_adjust_pc(kern_hyp_va(vcpu)); 381 } 382 383 static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt) 384 { 385 __kvm_flush_vm_context(); 386 } 387 388 static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) 389 { 390 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); 391 DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); 392 DECLARE_REG(int, level, host_ctxt, 3); 393 394 __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); 395 } 396 397 static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt) 398 { 399 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); 400 DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); 401 DECLARE_REG(int, level, host_ctxt, 3); 402 403 __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); 404 } 405 406 static void 407 handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) 408 { 409 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); 410 DECLARE_REG(phys_addr_t, start, host_ctxt, 2); 411 DECLARE_REG(unsigned long, pages, host_ctxt, 3); 412 413 __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages); 414 } 415 416 static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) 417 { 418 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); 419 420 __kvm_tlb_flush_vmid(kern_hyp_va(mmu)); 421 } 422 423 static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) 424 { 425 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 426 struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle); 427 428 if (!hyp_vm) 429 return; 430 431 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 432 put_pkvm_hyp_vm(hyp_vm); 433 } 434 435 static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt) 436 { 437 DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); 438 439 __kvm_flush_cpu_context(kern_hyp_va(mmu)); 440 } 441 442 static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt) 443 { 444 __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1)); 445 } 446 447 static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) 448 { 449 u64 tmp; 450 451 tmp = read_sysreg_el2(SYS_SCTLR); 452 tmp |= SCTLR_ELx_DSSBS; 453 write_sysreg_el2(tmp, SYS_SCTLR); 454 } 455 456 static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt) 457 { 458 cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config(); 459 } 460 461 static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) 462 { 463 __vgic_v3_init_lrs(); 464 } 465 466 static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) 467 { 468 DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); 469 470 __vgic_v3_save_aprs(kern_hyp_va(cpu_if)); 471 } 472 473 static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) 474 { 475 DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); 476 477 __vgic_v3_restore_vmcr_aprs(kern_hyp_va(cpu_if)); 478 } 479 480 static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) 481 { 482 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); 483 DECLARE_REG(unsigned long, size, host_ctxt, 2); 484 DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3); 485 DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4); 486 487 /* 488 * __pkvm_init() will return only if an error occurred, otherwise it 489 * will tail-call in __pkvm_init_finalise() which will have to deal 490 * with the host context directly. 491 */ 492 cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); 493 } 494 495 static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt) 496 { 497 DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1); 498 499 cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot); 500 } 501 502 static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) 503 { 504 DECLARE_REG(u64, pfn, host_ctxt, 1); 505 506 cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn); 507 } 508 509 static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt) 510 { 511 DECLARE_REG(u64, pfn, host_ctxt, 1); 512 513 cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn); 514 } 515 516 static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) 517 { 518 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); 519 DECLARE_REG(size_t, size, host_ctxt, 2); 520 DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); 521 522 /* 523 * __pkvm_create_private_mapping() populates a pointer with the 524 * hypervisor start address of the allocation. 525 * 526 * However, handle___pkvm_create_private_mapping() hypercall crosses the 527 * EL1/EL2 boundary so the pointer would not be valid in this context. 528 * 529 * Instead pass the allocation address as the return value (or return 530 * ERR_PTR() on failure). 531 */ 532 unsigned long haddr; 533 int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); 534 535 if (err) 536 haddr = (unsigned long)ERR_PTR(err); 537 538 cpu_reg(host_ctxt, 1) = haddr; 539 } 540 541 static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) 542 { 543 cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); 544 } 545 546 static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt) 547 { 548 cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm(); 549 } 550 551 static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) 552 { 553 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 554 555 __pkvm_unreserve_vm(handle); 556 } 557 558 static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) 559 { 560 DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); 561 DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2); 562 DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3); 563 564 host_kvm = kern_hyp_va(host_kvm); 565 cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); 566 } 567 568 static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) 569 { 570 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 571 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); 572 DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3); 573 574 host_vcpu = kern_hyp_va(host_vcpu); 575 cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva); 576 } 577 578 static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt) 579 { 580 int ret; 581 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 582 583 ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; 584 cpu_reg(host_ctxt, 1) = ret; 585 } 586 587 static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt) 588 { 589 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); 590 591 cpu_reg(host_ctxt, 1) = __pkvm_host_force_reclaim_page_guest(phys); 592 } 593 594 static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ctxt) 595 { 596 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 597 DECLARE_REG(u64, gfn, host_ctxt, 2); 598 599 cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_page(handle, gfn); 600 } 601 602 static void handle___pkvm_start_teardown_vm(struct kvm_cpu_context *host_ctxt) 603 { 604 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 605 606 cpu_reg(host_ctxt, 1) = __pkvm_start_teardown_vm(handle); 607 } 608 609 static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt) 610 { 611 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); 612 613 cpu_reg(host_ctxt, 1) = __pkvm_finalize_teardown_vm(handle); 614 } 615 616 static void handle___tracing_load(struct kvm_cpu_context *host_ctxt) 617 { 618 DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1); 619 DECLARE_REG(size_t, desc_size, host_ctxt, 2); 620 621 cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size); 622 } 623 624 static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt) 625 { 626 __tracing_unload(); 627 } 628 629 static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt) 630 { 631 DECLARE_REG(bool, enable, host_ctxt, 1); 632 633 cpu_reg(host_ctxt, 1) = __tracing_enable(enable); 634 } 635 636 static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt) 637 { 638 DECLARE_REG(unsigned int, cpu, host_ctxt, 1); 639 640 cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu); 641 } 642 643 static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt) 644 { 645 DECLARE_REG(u32, mult, host_ctxt, 1); 646 DECLARE_REG(u32, shift, host_ctxt, 2); 647 DECLARE_REG(u64, epoch_ns, host_ctxt, 3); 648 DECLARE_REG(u64, epoch_cyc, host_ctxt, 4); 649 650 __tracing_update_clock(mult, shift, epoch_ns, epoch_cyc); 651 } 652 653 static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt) 654 { 655 DECLARE_REG(unsigned int, cpu, host_ctxt, 1); 656 657 cpu_reg(host_ctxt, 1) = __tracing_reset(cpu); 658 } 659 660 static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt) 661 { 662 DECLARE_REG(unsigned short, id, host_ctxt, 1); 663 DECLARE_REG(bool, enable, host_ctxt, 2); 664 665 cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable); 666 } 667 668 static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt) 669 { 670 DECLARE_REG(u64, id, host_ctxt, 1); 671 672 trace_selftest(id); 673 } 674 675 static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt) 676 { 677 DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); 678 679 __vgic_v5_save_apr(kern_hyp_va(cpu_if)); 680 } 681 682 static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt) 683 { 684 DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); 685 686 __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); 687 } 688 689 typedef void (*hcall_t)(struct kvm_cpu_context *); 690 691 #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x 692 693 static const hcall_t host_hcall[] = { 694 /* ___kvm_hyp_init */ 695 HANDLE_FUNC(__pkvm_init), 696 HANDLE_FUNC(__pkvm_create_private_mapping), 697 HANDLE_FUNC(__pkvm_cpu_set_vector), 698 HANDLE_FUNC(__kvm_enable_ssbs), 699 HANDLE_FUNC(__vgic_v3_init_lrs), 700 HANDLE_FUNC(__vgic_v3_get_gic_config), 701 HANDLE_FUNC(__pkvm_prot_finalize), 702 703 HANDLE_FUNC(__kvm_adjust_pc), 704 HANDLE_FUNC(__kvm_vcpu_run), 705 HANDLE_FUNC(__kvm_flush_vm_context), 706 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), 707 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa_nsh), 708 HANDLE_FUNC(__kvm_tlb_flush_vmid), 709 HANDLE_FUNC(__kvm_tlb_flush_vmid_range), 710 HANDLE_FUNC(__kvm_flush_cpu_context), 711 HANDLE_FUNC(__kvm_timer_set_cntvoff), 712 HANDLE_FUNC(__tracing_load), 713 HANDLE_FUNC(__tracing_unload), 714 HANDLE_FUNC(__tracing_enable), 715 HANDLE_FUNC(__tracing_swap_reader), 716 HANDLE_FUNC(__tracing_update_clock), 717 HANDLE_FUNC(__tracing_reset), 718 HANDLE_FUNC(__tracing_enable_event), 719 HANDLE_FUNC(__tracing_write_event), 720 HANDLE_FUNC(__vgic_v3_save_aprs), 721 HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), 722 HANDLE_FUNC(__vgic_v5_save_apr), 723 HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), 724 725 HANDLE_FUNC(__pkvm_host_share_hyp), 726 HANDLE_FUNC(__pkvm_host_unshare_hyp), 727 HANDLE_FUNC(__pkvm_host_donate_guest), 728 HANDLE_FUNC(__pkvm_host_share_guest), 729 HANDLE_FUNC(__pkvm_host_unshare_guest), 730 HANDLE_FUNC(__pkvm_host_relax_perms_guest), 731 HANDLE_FUNC(__pkvm_host_wrprotect_guest), 732 HANDLE_FUNC(__pkvm_host_test_clear_young_guest), 733 HANDLE_FUNC(__pkvm_host_mkyoung_guest), 734 HANDLE_FUNC(__pkvm_reserve_vm), 735 HANDLE_FUNC(__pkvm_unreserve_vm), 736 HANDLE_FUNC(__pkvm_init_vm), 737 HANDLE_FUNC(__pkvm_init_vcpu), 738 HANDLE_FUNC(__pkvm_vcpu_in_poison_fault), 739 HANDLE_FUNC(__pkvm_force_reclaim_guest_page), 740 HANDLE_FUNC(__pkvm_reclaim_dying_guest_page), 741 HANDLE_FUNC(__pkvm_start_teardown_vm), 742 HANDLE_FUNC(__pkvm_finalize_teardown_vm), 743 HANDLE_FUNC(__pkvm_vcpu_load), 744 HANDLE_FUNC(__pkvm_vcpu_put), 745 HANDLE_FUNC(__pkvm_tlb_flush_vmid), 746 }; 747 748 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) 749 { 750 DECLARE_REG(unsigned long, id, host_ctxt, 0); 751 unsigned long hcall_min = 0, hcall_max = __KVM_HOST_SMCCC_FUNC_MAX; 752 hcall_t hfn; 753 754 BUILD_BUG_ON(ARRAY_SIZE(host_hcall) != __KVM_HOST_SMCCC_FUNC_MAX); 755 756 /* 757 * If pKVM has been initialised then reject any calls to the 758 * early "privileged" hypercalls. Note that we cannot reject 759 * calls to __pkvm_prot_finalize for two reasons: (1) The static 760 * key used to determine initialisation must be toggled prior to 761 * finalisation and (2) finalisation is performed on a per-CPU 762 * basis. This is all fine, however, since __pkvm_prot_finalize 763 * returns -EPERM after the first call for a given CPU. 764 */ 765 if (static_branch_unlikely(&kvm_protected_mode_initialized)) { 766 hcall_min = __KVM_HOST_SMCCC_FUNC_MIN_PKVM; 767 } else { 768 hcall_max = __KVM_HOST_SMCCC_FUNC_PKVM_ONLY; 769 } 770 771 id &= ~ARM_SMCCC_CALL_HINTS; 772 id -= KVM_HOST_SMCCC_ID(0); 773 774 if (unlikely(id < hcall_min || id >= hcall_max)) 775 goto inval; 776 777 hfn = host_hcall[id]; 778 if (unlikely(!hfn)) 779 goto inval; 780 781 cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS; 782 hfn(host_ctxt); 783 784 return; 785 inval: 786 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 787 } 788 789 static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt) 790 { 791 trace_hyp_exit(host_ctxt, HYP_REASON_SMC); 792 __kvm_hyp_host_forward_smc(host_ctxt); 793 trace_hyp_enter(host_ctxt, HYP_REASON_SMC); 794 } 795 796 static void handle_host_smc(struct kvm_cpu_context *host_ctxt) 797 { 798 DECLARE_REG(u64, func_id, host_ctxt, 0); 799 u64 esr = read_sysreg_el2(SYS_ESR); 800 bool handled; 801 802 if (esr & ESR_ELx_xVC_IMM_MASK) { 803 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 804 goto exit_skip_instr; 805 } 806 807 func_id &= ~ARM_SMCCC_CALL_HINTS; 808 if (upper_32_bits(func_id)) { 809 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 810 goto exit_skip_instr; 811 } 812 813 handled = kvm_host_psci_handler(host_ctxt, func_id); 814 if (!handled) 815 handled = kvm_host_ffa_handler(host_ctxt, func_id); 816 if (!handled) 817 default_host_smc_handler(host_ctxt); 818 819 exit_skip_instr: 820 /* SMC was trapped, move ELR past the current PC. */ 821 kvm_skip_host_instr(); 822 } 823 824 void inject_host_exception(u64 esr) 825 { 826 u64 sctlr, spsr_el1, spsr_el2, exc_offset = except_type_sync; 827 const u64 spsr_mask = PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | 828 PSR_V_BIT | PSR_DIT_BIT | PSR_PAN_BIT; 829 830 spsr_el1 = spsr_el2 = read_sysreg_el2(SYS_SPSR); 831 switch (spsr_el1 & (PSR_MODE_MASK | PSR_MODE32_BIT)) { 832 case PSR_MODE_EL0t: 833 exc_offset += LOWER_EL_AArch64_VECTOR; 834 break; 835 case PSR_MODE_EL0t | PSR_MODE32_BIT: 836 exc_offset += LOWER_EL_AArch32_VECTOR; 837 break; 838 default: 839 exc_offset += CURRENT_EL_SP_ELx_VECTOR; 840 } 841 842 spsr_el2 &= spsr_mask; 843 spsr_el2 |= PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | 844 PSR_MODE_EL1h; 845 846 sctlr = read_sysreg_el1(SYS_SCTLR); 847 if (!(sctlr & SCTLR_EL1_SPAN)) 848 spsr_el2 |= PSR_PAN_BIT; 849 850 if (sctlr & SCTLR_ELx_DSSBS) 851 spsr_el2 |= PSR_SSBS_BIT; 852 853 if (system_supports_mte()) 854 spsr_el2 |= PSR_TCO_BIT; 855 856 if (esr_fsc_is_translation_fault(esr)) 857 write_sysreg_el1(read_sysreg_el2(SYS_FAR), SYS_FAR); 858 859 write_sysreg_el1(esr, SYS_ESR); 860 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); 861 write_sysreg_el1(spsr_el1, SYS_SPSR); 862 write_sysreg_el2(read_sysreg_el1(SYS_VBAR) + exc_offset, SYS_ELR); 863 write_sysreg_el2(spsr_el2, SYS_SPSR); 864 } 865 866 static void inject_host_undef64(void) 867 { 868 inject_host_exception((ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | 869 ESR_ELx_IL); 870 } 871 872 static bool handle_host_mte(u64 esr) 873 { 874 switch (esr_sys64_to_sysreg(esr)) { 875 case SYS_RGSR_EL1: 876 case SYS_GCR_EL1: 877 case SYS_TFSR_EL1: 878 case SYS_TFSRE0_EL1: 879 /* If we're here for any reason other than MTE, it's a bug. */ 880 if (read_sysreg(HCR_EL2) & HCR_ATA) 881 return false; 882 break; 883 case SYS_GMID_EL1: 884 /* If we're here for any reason other than MTE, it's a bug. */ 885 if (!(read_sysreg(HCR_EL2) & HCR_TID5)) 886 return false; 887 break; 888 default: 889 return false; 890 } 891 892 inject_host_undef64(); 893 return true; 894 } 895 896 void handle_trap(struct kvm_cpu_context *host_ctxt) 897 { 898 u64 esr = read_sysreg_el2(SYS_ESR); 899 900 901 switch (ESR_ELx_EC(esr)) { 902 case ESR_ELx_EC_HVC64: 903 trace_hyp_enter(host_ctxt, HYP_REASON_HVC); 904 handle_host_hcall(host_ctxt); 905 break; 906 case ESR_ELx_EC_SMC64: 907 trace_hyp_enter(host_ctxt, HYP_REASON_SMC); 908 handle_host_smc(host_ctxt); 909 break; 910 case ESR_ELx_EC_IABT_LOW: 911 case ESR_ELx_EC_DABT_LOW: 912 trace_hyp_enter(host_ctxt, HYP_REASON_HOST_ABORT); 913 handle_host_mem_abort(host_ctxt); 914 break; 915 case ESR_ELx_EC_SYS64: 916 if (handle_host_mte(esr)) 917 break; 918 fallthrough; 919 default: 920 BUG(); 921 } 922 923 trace_hyp_exit(host_ctxt, HYP_REASON_ERET_HOST); 924 } 925