1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - Google Inc 4 * Author: Andrew Scull <ascull@google.com> 5 */ 6 7 #include <kvm/arm_hypercalls.h> 8 9 #include <hyp/adjust_pc.h> 10 #include <hyp/switch.h> 11 12 #include <linux/irqchip/arm-gic-v3.h> 13 #include <uapi/linux/psci.h> 14 15 #include <asm/pgtable-types.h> 16 #include <asm/kvm_asm.h> 17 #include <asm/kvm_emulate.h> 18 #include <asm/kvm_host.h> 19 #include <asm/kvm_hyp.h> 20 #include <asm/kvm_hypevents.h> 21 #include <asm/kvm_mmu.h> 22 23 #include <nvhe/alloc.h> 24 #include <nvhe/ffa.h> 25 #include <nvhe/mem_protect.h> 26 #include <nvhe/mm.h> 27 #include <nvhe/pkvm.h> 28 #include <nvhe/trace.h> 29 #include <nvhe/trap_handler.h> 30 31 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 32 33 /* 34 * Define a hypercall handler: handle_<name> unmarshals the arguments from 35 * the host context and hands them, correctly typed, to the body that 36 * follows the macro. The parameter list is type-checked against the 37 * signature declared in <asm/kvm_hcall.h>, so the handler cannot drift 38 * from what the typed caller stubs marshal in. Modelled on the syscall 39 * wrappers. 40 */ 41 /* Truncate the fixed list of argument registers to the declared signature. */ 42 #define KVM_HOST_HCALL_REGS(...) \ 43 __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), __KVM_HCALL_ARGS \ 44 ,, cpu_reg(host_ctxt, 1),, cpu_reg(host_ctxt, 2) \ 45 ,, cpu_reg(host_ctxt, 3),, cpu_reg(host_ctxt, 4) \ 46 ,, cpu_reg(host_ctxt, 5),, cpu_reg(host_ctxt, 6)) 47 48 #define set_cpu_reg_ulong(ctxt, r, v) { cpu_reg(ctxt, r) = v; } 49 #define set_cpu_reg_u64(ctxt, r, v) { cpu_reg(ctxt, r) = v; } 50 #define set_cpu_reg_int(ctxt, r, v) { cpu_reg(ctxt, r) = v; } 51 #define set_cpu_reg_void(ctxt, r, v) { v; } 52 #define set_cpu_reg(ctxt, r, t, v) set_cpu_reg_##t(ctxt, r, v) 53 54 #define DEFINE_KVM_HOST_HCALL(ret, name, ...) \ 55 static kvm_host_hcall_sig_##name __do_##name; \ 56 static __always_inline \ 57 ret __se_##name(__KVM_HCALL_MAP(__KVM_HCALL_LONG, __VA_ARGS__)) \ 58 { \ 59 return __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_CAST, __VA_ARGS__)); \ 60 } \ 61 static void handle_##name(struct kvm_cpu_context *host_ctxt) \ 62 { \ 63 set_cpu_reg(host_ctxt, 1, ret, __se_##name(KVM_HOST_HCALL_REGS(__VA_ARGS__))); \ 64 } \ 65 static ret __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) 66 67 #define DEFINE_KVM_HOST_HCALL0(ret, name) \ 68 static kvm_host_hcall_sig_##name __do_##name; \ 69 static void handle_##name(struct kvm_cpu_context *host_ctxt) \ 70 { \ 71 set_cpu_reg(host_ctxt, 1, ret, __do_##name()); \ 72 } \ 73 static ret __do_##name(void) 74 75 /* 76 * Encode a hypervisor request in the host SMCCC return registers for known 77 * error numbers. 78 * 79 * Must be paired with pkvm_call_hyp_req() on the host side. 80 */ 81 static int errno_to_smccc(int ret) 82 { 83 struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; 84 85 switch (ret) { 86 case -ENOMEM: { 87 u32 nr_pages = hyp_alloc_topup_needed(); 88 89 if (nr_pages) { 90 req.type = PKVM_HYP_REQ_HYP_ALLOC; 91 req.mem.nr_pages = nr_pages; 92 } 93 break; 94 } 95 } 96 97 pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); 98 99 return ret; 100 } 101 102 /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ 103 unsigned int hyp_gicv3_nr_lr; 104 105 void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); 106 107 typedef void (*hyp_entry_exit_handler_fn)(struct pkvm_hyp_vcpu *); 108 109 static bool pvm_sys64_is_write(u64 esr) 110 { 111 return (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_WRITE; 112 } 113 114 static void handle_pvm_entry_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) 115 { 116 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 117 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 118 119 /* Exceptions have priority; the host injects none on WFx. */ 120 if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) 121 return; 122 123 if (vcpu_get_flag(host_vcpu, INCREMENT_PC)) { 124 vcpu_clear_flag(vcpu, PC_UPDATE_REQ); 125 kvm_incr_pc(vcpu); 126 } 127 } 128 129 static void handle_pvm_entry_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) 130 { 131 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 132 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 133 bool pc_update; 134 135 /* Exceptions have priority over anything else */ 136 if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { 137 /* A host-requested exception on SYS64 is always an UNDEF. */ 138 u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; 139 140 __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); 141 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); 142 return; 143 } 144 145 /* Handle PC increment on a host-emulated access */ 146 pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC); 147 if (pc_update) { 148 vcpu_clear_flag(vcpu, PC_UPDATE_REQ); 149 kvm_incr_pc(vcpu); 150 } 151 152 /* If the host emulated a read access, update the register */ 153 if (pc_update && !pvm_sys64_is_write(kvm_vcpu_get_esr(vcpu))) { 154 /* r0 as transfer register between the guest and the host. */ 155 u64 rt_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); 156 int rt = kvm_vcpu_sys_get_rt(vcpu); 157 158 vcpu_set_reg(vcpu, rt, rt_val); 159 } 160 } 161 162 static void handle_pvm_entry_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) 163 { 164 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 165 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 166 unsigned long cpsr = *vcpu_cpsr(vcpu); 167 u32 esr = ESR_ELx_IL; 168 169 if (!vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) 170 return; 171 172 /* The host's only IABT injection: an external abort. */ 173 if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) 174 esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); 175 else 176 esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); 177 178 esr |= ESR_ELx_FSC_EXTABT; 179 180 __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); 181 __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); 182 183 /* Injected by __kvm_adjust_pc() on entry. */ 184 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); 185 } 186 187 /* 188 * Clamp MMIO data to the access width, so a write does not leak the 189 * register's upper bits and a read takes no bits beyond the load. The 190 * host applies endianness. 191 */ 192 static inline u64 kvm_mmio_clamp_data(struct kvm_vcpu *vcpu, u64 val) 193 { 194 unsigned int len = kvm_vcpu_dabt_get_as(vcpu); 195 196 return val & GENMASK_U64(len * 8 - 1, 0); 197 } 198 199 /* 200 * Complete an MMIO load: sign-extend from EL2's own syndrome, as the 201 * architecture does. 202 */ 203 static inline u64 kvm_mmio_read_data(struct kvm_vcpu *vcpu, u64 val) 204 { 205 val = kvm_mmio_clamp_data(vcpu, val); 206 207 if (kvm_vcpu_dabt_issext(vcpu)) 208 val = sign_extend64(val, kvm_vcpu_dabt_get_as(vcpu) * 8 - 1); 209 210 if (!kvm_vcpu_dabt_issf(vcpu)) 211 val &= GENMASK_U64(31, 0); 212 213 return val; 214 } 215 216 static void handle_pvm_entry_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) 217 { 218 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 219 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 220 bool pc_update; 221 222 /* Exceptions have priority over anything else */ 223 if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { 224 unsigned long cpsr = *vcpu_cpsr(vcpu); 225 u32 esr = ESR_ELx_IL; 226 227 if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) 228 esr |= (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT); 229 else 230 esr |= (ESR_ELx_EC_DABT_CUR << ESR_ELx_EC_SHIFT); 231 232 esr |= ESR_ELx_FSC_EXTABT; 233 234 __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); 235 __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); 236 237 /* Injected by __kvm_adjust_pc() on entry. */ 238 kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); 239 240 /* Cancel any in-flight MMIO */ 241 vcpu->mmio_needed = false; 242 return; 243 } 244 245 /* Handle PC increment on MMIO, or on a CMO the host skipped */ 246 pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC) && 247 (vcpu->mmio_needed || esr_dabt_is_cm(kvm_vcpu_get_esr(vcpu))); 248 if (pc_update) { 249 vcpu_clear_flag(vcpu, PC_UPDATE_REQ); 250 kvm_incr_pc(vcpu); 251 } 252 253 /* If the host emulated an MMIO read, update the register */ 254 if (pc_update && vcpu->mmio_needed && !kvm_vcpu_dabt_iswrite(vcpu)) { 255 /* r0 as transfer register between the guest and the host. */ 256 u64 rd_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); 257 int rd = kvm_vcpu_dabt_get_rd(vcpu); 258 259 rd_val = kvm_mmio_read_data(vcpu, rd_val); 260 vcpu_set_reg(vcpu, rd, rd_val); 261 } 262 263 vcpu->mmio_needed = false; 264 } 265 266 static void handle_pvm_entry_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) 267 { 268 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 269 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 270 u64 ret = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); 271 u32 psci_fn = smccc_get_function(vcpu); 272 273 switch (psci_fn) { 274 case PSCI_0_2_FN_CPU_ON: 275 case PSCI_0_2_FN64_CPU_ON: 276 /* 277 * Roll back a CPU_ON the host failed, unless the target 278 * already reached ON: it is running, and the guest sees 279 * SUCCESS. 280 */ 281 if (ret != PSCI_RET_SUCCESS) { 282 unsigned long cpu_id = smccc_get_arg1(vcpu); 283 struct pkvm_hyp_vcpu *target_vcpu; 284 struct pkvm_hyp_vm *hyp_vm; 285 int prev; 286 287 hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 288 target_vcpu = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); 289 290 /* 291 * pvm_psci_vcpu_on() resolved this MPIDR and vcpus[] 292 * entries are never removed, so the lookup cannot miss. 293 * The release orders this vCPU's reset_state writes 294 * before OFF, for the next CPU_ON. 295 */ 296 prev = cmpxchg_release(&target_vcpu->power_state, 297 PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, 298 PSCI_0_2_AFFINITY_LEVEL_OFF); 299 switch (prev) { 300 case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: 301 /* 302 * Leave reset_state.reset set: a clear races a 303 * fresh CPU_ON's publish. The stale pc/r0/be are 304 * the guest's own. ALREADY_ON is PSCI's retry 305 * signal for a CPU_ON that raced the CPU_OFF. 306 */ 307 if (ret != PSCI_RET_ALREADY_ON) 308 ret = PSCI_RET_INTERNAL_FAILURE; 309 break; 310 case PSCI_0_2_AFFINITY_LEVEL_ON: 311 case PSCI_0_2_AFFINITY_LEVEL_OFF: 312 /* Target already ran (and may have stopped). */ 313 ret = PSCI_RET_SUCCESS; 314 break; 315 default: 316 ret = PSCI_RET_INTERNAL_FAILURE; 317 break; 318 } 319 } 320 321 break; 322 default: 323 break; 324 } 325 326 vcpu_set_reg(vcpu, 0, ret); 327 } 328 329 /* The host's view of a syndrome: the guest register index is withheld. */ 330 static u64 pvm_host_esr(u64 esr) 331 { 332 switch (ESR_ELx_EC(esr)) { 333 case ESR_ELx_EC_WFx: 334 return esr & ~ESR_ELx_WFx_ISS_RN; 335 case ESR_ELx_EC_SYS64: 336 return esr & ~ESR_ELx_SYS64_ISS_RT_MASK; 337 case ESR_ELx_EC_DABT_LOW: 338 return esr & ~ESR_ELx_SRT_MASK; 339 default: 340 return esr; 341 } 342 } 343 344 /* 345 * The host's view of PSTATE: the mode, with SErrors masked so that the host 346 * pends an SError through HCR_EL2.VSE rather than emulating the entry. 347 */ 348 static unsigned long pvm_host_pstate(unsigned long pstate) 349 { 350 return (pstate & PSR_MODE_MASK) | PSR_A_BIT; 351 } 352 353 static void handle_pvm_exit_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) 354 { 355 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 356 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 357 358 *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); 359 } 360 361 static void handle_pvm_exit_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) 362 { 363 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 364 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 365 u64 esr = kvm_vcpu_get_esr(vcpu); 366 367 /* The mode is required for the host to emulate some sysregs */ 368 *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); 369 370 /* r0 as transfer register between the guest and the host. */ 371 if (pvm_sys64_is_write(esr)) { 372 int rt = kvm_vcpu_sys_get_rt(vcpu); 373 u64 rt_val = vcpu_get_reg(vcpu, rt); 374 375 vcpu_set_reg(host_vcpu, 0, rt_val); 376 } 377 } 378 379 static void handle_pvm_exit_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) 380 { 381 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 382 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 383 384 host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; 385 } 386 387 static void handle_pvm_exit_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) 388 { 389 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 390 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 391 u64 sctlr; 392 393 /* 394 * EL2 has no memslot view: a decodable data abort is prepared as MMIO 395 * for the host to resolve. On unbacked memory the host injects an SEA 396 * for one with ISV clear (LDP/STP, atomics), which EL2 does not 397 * decode, and skips a cache maintenance operation, as for any guest. 398 */ 399 vcpu->mmio_needed = kvm_vcpu_dabt_isvalid(vcpu); 400 401 /* r0 as transfer register between the guest and the host. */ 402 if (vcpu->mmio_needed && kvm_vcpu_dabt_iswrite(vcpu)) { 403 int rt = kvm_vcpu_dabt_get_rd(vcpu); 404 u64 rt_val = vcpu_get_reg(vcpu, rt); 405 406 rt_val = kvm_mmio_clamp_data(vcpu, rt_val); 407 vcpu_set_reg(host_vcpu, 0, rt_val); 408 } 409 410 *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); 411 host_vcpu->arch.fault.far_el2 = vcpu->arch.fault.far_el2 & GENMASK(11, 0); 412 host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; 413 sctlr = __vcpu_sys_reg(vcpu, SCTLR_EL1) & (SCTLR_ELx_EE | SCTLR_EL1_E0E); 414 __vcpu_assign_sys_reg(host_vcpu, SCTLR_EL1, sctlr); 415 } 416 417 static void handle_pvm_exit_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) 418 { 419 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 420 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 421 int n, i; 422 423 switch (smccc_get_function(vcpu)) { 424 /* 425 * CPU_ON: the host uses only the target MPIDR (x1). EL2 resets the 426 * target from its own copy of the entry point and context id. 427 */ 428 case PSCI_0_2_FN_CPU_ON: 429 case PSCI_0_2_FN64_CPU_ON: 430 n = 2; 431 break; 432 433 case PSCI_0_2_FN_CPU_OFF: 434 case PSCI_0_2_FN_SYSTEM_OFF: 435 case PSCI_0_2_FN_SYSTEM_RESET: 436 case PSCI_0_2_FN_CPU_SUSPEND: 437 case PSCI_0_2_FN64_CPU_SUSPEND: 438 n = 1; 439 break; 440 441 case PSCI_0_2_FN_AFFINITY_INFO: 442 case PSCI_0_2_FN64_AFFINITY_INFO: 443 case PSCI_1_1_FN_SYSTEM_RESET2: 444 case PSCI_1_1_FN64_SYSTEM_RESET2: 445 n = 3; 446 break; 447 448 /* Unreachable: kvm_handle_pvm_hvc64() forwards only the calls above. */ 449 default: 450 hyp_panic(); 451 } 452 453 /* Pass the HVC function id (r0) and its arguments. */ 454 for (i = 0; i < n; i++) 455 vcpu_set_reg(host_vcpu, i, vcpu_get_reg(vcpu, i)); 456 } 457 458 static const hyp_entry_exit_handler_fn entry_hyp_pvm_handlers[] = { 459 [0 ... ESR_ELx_EC_MAX] = NULL, 460 [ESR_ELx_EC_WFx] = handle_pvm_entry_wfx, 461 [ESR_ELx_EC_SYS64] = handle_pvm_entry_sys64, 462 [ESR_ELx_EC_IABT_LOW] = handle_pvm_entry_iabt, 463 [ESR_ELx_EC_DABT_LOW] = handle_pvm_entry_dabt, 464 [ESR_ELx_EC_HVC64] = handle_pvm_entry_hvc64, 465 }; 466 467 static const hyp_entry_exit_handler_fn exit_hyp_pvm_handlers[] = { 468 [0 ... ESR_ELx_EC_MAX] = NULL, 469 [ESR_ELx_EC_WFx] = handle_pvm_exit_wfx, 470 [ESR_ELx_EC_SYS64] = handle_pvm_exit_sys64, 471 [ESR_ELx_EC_IABT_LOW] = handle_pvm_exit_iabt, 472 [ESR_ELx_EC_DABT_LOW] = handle_pvm_exit_dabt, 473 [ESR_ELx_EC_HVC64] = handle_pvm_exit_hvc64, 474 }; 475 476 static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) 477 { 478 __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); 479 /* 480 * On saving/restoring guest sve state, always use the maximum VL for 481 * the guest. The layout of the data when saving the sve state depends 482 * on the VL, so use a consistent (i.e., the maximum) guest VL. 483 */ 484 sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); 485 sve_save_state(kern_hyp_va(vcpu->arch.sve_state), true); 486 fpsimd_save_common(&vcpu->arch.ctxt.fp_regs); 487 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 488 } 489 490 static void __hyp_sve_restore_host(void) 491 { 492 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 493 struct arm64_sve_state *sve_regs = *host_data_ptr(sve_regs); 494 495 /* 496 * On saving/restoring host sve state, always use the maximum VL for 497 * the host. The layout of the data when saving the sve state depends 498 * on the VL, so use a consistent (i.e., the maximum) host VL. 499 * 500 * Note that this constrains the PE to the maximum shared VL 501 * that was discovered, if we wish to use larger VLs this will 502 * need to be revisited. 503 */ 504 write_sysreg_s(sve_vq_from_vl(kvm_host_sve_max_vl) - 1, SYS_ZCR_EL2); 505 sve_load_state(sve_regs, true); 506 fpsimd_load_common(&hctxt->fp_regs); 507 write_sysreg_el1(ctxt_sys_reg(hctxt, ZCR_EL1), SYS_ZCR); 508 } 509 510 static void fpsimd_sve_flush(void) 511 { 512 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; 513 } 514 515 static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) 516 { 517 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); 518 bool has_fpmr; 519 520 if (!guest_owns_fp_regs()) 521 return; 522 523 /* 524 * Traps have been disabled by __deactivate_cptr_traps(), but there 525 * hasn't necessarily been a context synchronization event yet. 526 */ 527 isb(); 528 529 if (vcpu_has_sve(vcpu)) 530 __hyp_sve_save_guest(vcpu); 531 else 532 fpsimd_save_state(&vcpu->arch.ctxt.fp_regs); 533 534 has_fpmr = kvm_has_fpmr(kern_hyp_va(vcpu->kvm)); 535 if (has_fpmr) 536 __vcpu_assign_sys_reg(vcpu, FPMR, read_sysreg_s(SYS_FPMR)); 537 538 if (system_supports_sve()) 539 __hyp_sve_restore_host(); 540 else 541 fpsimd_load_state(&hctxt->fp_regs); 542 543 if (has_fpmr) 544 write_sysreg_s(ctxt_sys_reg(hctxt, FPMR), SYS_FPMR); 545 546 *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; 547 } 548 549 static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) 550 { 551 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 552 struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; 553 unsigned int used_lrs, i; 554 555 host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; 556 hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; 557 558 used_lrs = host_cpu_if->used_lrs; 559 used_lrs = min(used_lrs, hyp_gicv3_nr_lr); 560 561 hyp_cpu_if->vgic_hcr = host_cpu_if->vgic_hcr; 562 /* Should be a one-off */ 563 hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | 564 ICC_SRE_EL1_DFB | 565 ICC_SRE_EL1_SRE); 566 hyp_cpu_if->used_lrs = used_lrs; 567 568 for (i = 0; i < used_lrs; i++) 569 hyp_cpu_if->vgic_lr[i] = host_cpu_if->vgic_lr[i]; 570 } 571 572 static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) 573 { 574 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 575 struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; 576 unsigned int i; 577 578 host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; 579 hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; 580 581 host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; 582 host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; 583 584 for (i = 0; i < hyp_cpu_if->used_lrs; i++) 585 host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; 586 } 587 588 static void flush_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) 589 { 590 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 591 592 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 593 return; 594 595 /* A hyp vcpu has no offset, and sees vtime == ptime. */ 596 write_sysreg(0, cntvoff_el2); 597 write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CVAL_EL0), SYS_CNTV_CVAL); 598 isb(); 599 write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CTL_EL0), SYS_CNTV_CTL); 600 } 601 602 static void sync_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) 603 { 604 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; 605 606 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 607 return; 608 609 /* 610 * Preserve the vtimer state so that it is always correct, 611 * even if the host tries to make a mess. 612 */ 613 __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, read_sysreg_el0(SYS_CNTV_CVAL)); 614 __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, read_sysreg_el0(SYS_CNTV_CTL)); 615 } 616 617 static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, 618 struct kvm_vcpu *to_vcpu) 619 { 620 int i; 621 622 to_vcpu->arch.ctxt.regs = from_vcpu->arch.ctxt.regs; 623 to_vcpu->arch.ctxt.spsr_abt = from_vcpu->arch.ctxt.spsr_abt; 624 to_vcpu->arch.ctxt.spsr_und = from_vcpu->arch.ctxt.spsr_und; 625 to_vcpu->arch.ctxt.spsr_irq = from_vcpu->arch.ctxt.spsr_irq; 626 to_vcpu->arch.ctxt.spsr_fiq = from_vcpu->arch.ctxt.spsr_fiq; 627 to_vcpu->arch.ctxt.fp_regs = from_vcpu->arch.ctxt.fp_regs; 628 629 /* 630 * Copy the sysregs, but don't mess with the timer state which 631 * is directly handled by EL1 and is expected to be preserved. 632 * enum vcpu_sysreg is sparse: VNCR-mapped registers take values 633 * derived from their VNCR page offset, so the timer registers do 634 * not form a contiguous numeric range and must be skipped by name. 635 */ 636 for (i = 1; i < NR_SYS_REGS; i++) { 637 switch (i) { 638 case CNTVOFF_EL2: 639 case CNTV_CVAL_EL0: 640 case CNTV_CTL_EL0: 641 case CNTP_CVAL_EL0: 642 case CNTP_CTL_EL0: 643 continue; 644 } 645 to_vcpu->arch.ctxt.sys_regs[i] = from_vcpu->arch.ctxt.sys_regs[i]; 646 } 647 } 648 649 static void sync_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) 650 { 651 __copy_vcpu_state(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu); 652 } 653 654 static void flush_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) 655 { 656 __copy_vcpu_state(hyp_vcpu->host_vcpu, &hyp_vcpu->vcpu); 657 } 658 659 static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) 660 { 661 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 662 663 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 664 return; 665 666 hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; 667 668 if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { 669 hyp_vcpu->vcpu.arch.vcpu_debug_state = host_vcpu->arch.vcpu_debug_state; 670 } else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) { 671 hyp_vcpu->vcpu.arch.external_debug_state = host_vcpu->arch.external_debug_state; 672 /* 673 * The world switch loads MDSCR_EL1 from external_mdscr_el1 674 * (ctxt_mdscr_el1()). 675 */ 676 hyp_vcpu->vcpu.arch.external_mdscr_el1 = host_vcpu->arch.external_mdscr_el1; 677 } 678 } 679 680 static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) 681 { 682 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 683 684 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 685 return; 686 687 if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) 688 host_vcpu->arch.vcpu_debug_state = hyp_vcpu->vcpu.arch.vcpu_debug_state; 689 else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) 690 host_vcpu->arch.external_debug_state = hyp_vcpu->vcpu.arch.external_debug_state; 691 } 692 693 static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 694 { 695 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 696 u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM; 697 hyp_entry_exit_handler_fn ec_handler; 698 u8 esr_ec; 699 700 fpsimd_sve_flush(); 701 flush_debug_state(hyp_vcpu); 702 703 /* 704 * If we deal with a non-protected guest and the state is potentially 705 * dirty (from a host perspective), copy the state back into the hyp 706 * vcpu. 707 */ 708 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 709 if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) 710 flush_hyp_vcpu_state(hyp_vcpu); 711 712 hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; 713 hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; 714 host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM; 715 } 716 717 /* __hyp_running_vcpu must be NULL in a guest context. */ 718 hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; 719 720 /* 721 * A host-injected vSError is masked by the guest's own PSTATE.A, so it 722 * applies to protected guests too. 723 */ 724 hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask; 725 hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask; 726 hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; 727 728 flush_hyp_vgic_state(hyp_vcpu); 729 flush_hyp_timer_state(hyp_vcpu); 730 731 hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; 732 733 switch (ARM_EXCEPTION_CODE(hyp_vcpu->exit_code)) { 734 case ARM_EXCEPTION_IRQ: 735 case ARM_EXCEPTION_EL1_SERROR: 736 case ARM_EXCEPTION_IL: 737 break; 738 case ARM_EXCEPTION_TRAP: 739 /* Nothing was marshalled for this trap, see sync_hyp_vcpu(). */ 740 if (ARM_SERROR_PENDING(hyp_vcpu->exit_code)) 741 break; 742 743 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 744 esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); 745 ec_handler = entry_hyp_pvm_handlers[esr_ec]; 746 if (ec_handler) 747 ec_handler(hyp_vcpu); 748 } 749 break; 750 default: 751 BUG(); 752 } 753 754 hyp_vcpu->exit_code = 0; 755 } 756 757 static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) 758 { 759 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 760 hyp_entry_exit_handler_fn ec_handler; 761 u8 esr_ec; 762 763 fpsimd_sve_sync(&hyp_vcpu->vcpu); 764 sync_debug_state(hyp_vcpu); 765 766 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 767 /* 768 * Protected: the host sees ESR_EL2 as EL2 took it, register 769 * index withheld; the fault addresses stay withheld unless the 770 * EC handler below adds them. 771 */ 772 host_vcpu->arch.fault = (struct kvm_vcpu_fault_info) { 773 .esr_el2 = pvm_host_esr(hyp_vcpu->vcpu.arch.fault.esr_el2), 774 .disr_el1 = hyp_vcpu->vcpu.arch.fault.disr_el1, 775 }; 776 } else { 777 /* Non-protected: the host gets the full fault. */ 778 host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; 779 host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; 780 /* 781 * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step 782 * machine, and both run before the next on-demand ctxt sync. 783 */ 784 host_vcpu->arch.ctxt.regs.pc = hyp_vcpu->vcpu.arch.ctxt.regs.pc; 785 host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; 786 } 787 788 switch (ARM_EXCEPTION_CODE(exit_reason)) { 789 case ARM_EXCEPTION_IRQ: 790 break; 791 case ARM_EXCEPTION_TRAP: 792 /* SError pending: not handled at EL2, the guest replays it. */ 793 if (ARM_SERROR_PENDING(exit_reason)) 794 break; 795 796 /* Per-EC marshalling is for protected guests only. */ 797 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 798 esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); 799 ec_handler = exit_hyp_pvm_handlers[esr_ec]; 800 if (ec_handler) 801 ec_handler(hyp_vcpu); 802 } 803 break; 804 case ARM_EXCEPTION_EL1_SERROR: 805 case ARM_EXCEPTION_IL: 806 break; 807 default: 808 BUG(); 809 } 810 811 /* Cleared by hardware once the guest takes the vSError. */ 812 host_vcpu->arch.hcr_el2 &= ~HCR_VSE; 813 host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; 814 815 sync_hyp_vgic_state(hyp_vcpu); 816 sync_hyp_timer_state(hyp_vcpu); 817 818 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 819 vcpu_clear_flag(host_vcpu, PC_UPDATE_REQ); 820 821 hyp_vcpu->exit_code = exit_reason; 822 } 823 824 DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, 825 pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) 826 { 827 struct pkvm_hyp_vcpu *hyp_vcpu; 828 829 hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx); 830 if (!hyp_vcpu) 831 return; 832 833 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { 834 u64 dfr0 = read_sysreg(id_aa64dfr0_el1); 835 836 /* Propagate WFx trapping flags */ 837 hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); 838 hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); 839 840 /* HPMN == 0 is reserved without FEAT_HPMN0. */ 841 if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) 842 u64p_replace_bits(&hyp_vcpu->vcpu.arch.mdcr_el2, 843 FIELD_GET(ARMV8_PMU_PMCR_N, read_sysreg(pmcr_el0)), 844 MDCR_EL2_HPMN); 845 } else { 846 memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt, 847 sizeof(hyp_vcpu->vcpu.arch.fgt)); 848 } 849 } 850 851 DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) 852 { 853 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 854 855 if (hyp_vcpu) { 856 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 857 858 if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) && 859 !vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) { 860 sync_hyp_vcpu_state(hyp_vcpu); 861 } 862 863 pkvm_put_hyp_vcpu(hyp_vcpu); 864 } 865 } 866 867 DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) 868 { 869 struct pkvm_hyp_vcpu *hyp_vcpu; 870 871 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 872 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 873 return; 874 875 sync_hyp_vcpu_state(hyp_vcpu); 876 } 877 878 static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *host_vcpu, 879 struct pkvm_hyp_vcpu **hyp_vcpup) 880 { 881 struct pkvm_hyp_vcpu *hyp_vcpu = NULL; 882 883 if (unlikely(is_protected_kvm_enabled())) { 884 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 885 886 if (!hyp_vcpu || hyp_vcpu->host_vcpu != host_vcpu) { 887 hyp_vcpu = NULL; 888 host_vcpu = NULL; 889 } 890 } 891 892 *hyp_vcpup = hyp_vcpu; 893 return host_vcpu; 894 } 895 896 static struct kvm_vcpu * 897 __get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if __kern *cpu_if, 898 struct pkvm_hyp_vcpu **hyp_vcpup) 899 { 900 struct vgic_v3_cpu_if *host_cpu_if = kern_hyp_va_host(cpu_if); 901 struct kvm_vcpu *host_vcpu = container_of(host_cpu_if, struct kvm_vcpu, 902 arch.vgic_cpu.vgic_v3); 903 904 return __get_host_hyp_vcpus(host_vcpu, hyp_vcpup); 905 } 906 907 DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run, struct kvm_vcpu __kern *, vcpu) 908 { 909 struct pkvm_hyp_vcpu *hyp_vcpu; 910 struct kvm_vcpu *host_vcpu; 911 int ret = ARM_EXCEPTION_IL; 912 913 host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); 914 if (!host_vcpu) 915 return ret; 916 917 if (unlikely(hyp_vcpu)) { 918 /* 919 * KVM (and pKVM) doesn't support SME guests for now, and 920 * ensures that SME features aren't enabled in pstate when 921 * loading a vcpu. Therefore, if SME features enabled the host 922 * is misbehaving. 923 */ 924 if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) 925 return ret; 926 927 /* 928 * ON has a single writer, pkvm_reset_vcpu() on this CPU, so 929 * READ_ONCE suffices. ON_PENDING takes the reset; -ECANCELED 930 * is a rollback that raced it. 931 */ 932 switch (READ_ONCE(hyp_vcpu->power_state)) { 933 case PSCI_0_2_AFFINITY_LEVEL_ON: 934 break; 935 case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: 936 if (pkvm_reset_vcpu(hyp_vcpu)) 937 return ret; 938 break; 939 default: 940 return ret; 941 } 942 943 flush_hyp_vcpu(hyp_vcpu); 944 945 ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); 946 947 sync_hyp_vcpu(hyp_vcpu, ret); 948 } else { 949 /* The host is fully trusted, run its vCPU directly. */ 950 fpsimd_lazy_switch_to_guest(host_vcpu); 951 ret = __kvm_vcpu_run(host_vcpu); 952 fpsimd_lazy_switch_to_host(host_vcpu); 953 } 954 955 return ret; 956 } 957 958 static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) 959 { 960 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; 961 962 return refill_memcache(&hyp_vcpu->vcpu.arch.stage2_mc, 963 host_vcpu->arch.stage2_mc.nr_pages, 964 &host_vcpu->arch.stage2_mc); 965 } 966 967 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, 968 u64, pfn, u64, gfn) 969 { 970 struct pkvm_hyp_vcpu *hyp_vcpu; 971 int ret; 972 973 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 974 if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 975 return -EINVAL; 976 977 ret = pkvm_refill_memcache(hyp_vcpu); 978 if (ret) 979 return ret; 980 981 return __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); 982 } 983 984 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, 985 u64, pfn, u64, gfn, u64, nr_pages, u64, prot) 986 { 987 struct pkvm_hyp_vcpu *hyp_vcpu; 988 int ret; 989 990 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 991 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 992 return -EINVAL; 993 994 ret = pkvm_refill_memcache(hyp_vcpu); 995 if (ret) 996 return ret; 997 998 return __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); 999 } 1000 1001 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, 1002 pkvm_handle_t, handle, u64, gfn, u64, nr_pages) 1003 { 1004 struct pkvm_hyp_vm *hyp_vm; 1005 int ret; 1006 1007 hyp_vm = get_np_pkvm_hyp_vm(handle); 1008 if (!hyp_vm) 1009 return -EINVAL; 1010 1011 ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm); 1012 put_pkvm_hyp_vm(hyp_vm); 1013 1014 return ret; 1015 } 1016 1017 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, 1018 u64, gfn, u64, prot) 1019 { 1020 struct pkvm_hyp_vcpu *hyp_vcpu; 1021 1022 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 1023 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 1024 return -EINVAL; 1025 1026 return __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); 1027 } 1028 1029 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, 1030 pkvm_handle_t, handle, u64, gfn, u64, nr_pages) 1031 { 1032 struct pkvm_hyp_vm *hyp_vm; 1033 int ret; 1034 1035 hyp_vm = get_np_pkvm_hyp_vm(handle); 1036 if (!hyp_vm) 1037 return -EINVAL; 1038 1039 ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm); 1040 put_pkvm_hyp_vm(hyp_vm); 1041 1042 return ret; 1043 } 1044 1045 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, 1046 pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) 1047 { 1048 struct pkvm_hyp_vm *hyp_vm; 1049 int ret; 1050 1051 hyp_vm = get_np_pkvm_hyp_vm(handle); 1052 if (!hyp_vm) 1053 return -EINVAL; 1054 1055 ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm); 1056 put_pkvm_hyp_vm(hyp_vm); 1057 1058 return ret; 1059 } 1060 1061 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, 1062 u64, gfn) 1063 { 1064 struct pkvm_hyp_vcpu *hyp_vcpu; 1065 1066 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 1067 if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 1068 return -EINVAL; 1069 1070 return __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); 1071 } 1072 1073 /* 1074 * PKVM_HOST_STATE_DIRTY names the authoritative copy, the host's when set. 1075 * A loaded protected vCPU takes the request at its next entry instead. 1076 */ 1077 struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) 1078 { 1079 struct pkvm_hyp_vcpu *hyp_vcpu; 1080 1081 if (!is_protected_kvm_enabled()) 1082 return vcpu; 1083 1084 hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 1085 if (!hyp_vcpu || vcpu == &hyp_vcpu->vcpu) 1086 return vcpu; 1087 1088 if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) 1089 return NULL; 1090 1091 if (vcpu_get_flag(vcpu, PKVM_HOST_STATE_DIRTY)) 1092 return vcpu; 1093 1094 vcpu_copy_flag(&hyp_vcpu->vcpu, vcpu, PC_UPDATE_REQ); 1095 return &hyp_vcpu->vcpu; 1096 } 1097 1098 /* Reflect the consumed request back, otherwise it stays pending. */ 1099 void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target) 1100 { 1101 if (target != vcpu) 1102 vcpu_copy_flag(vcpu, target, PC_UPDATE_REQ); 1103 } 1104 1105 DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc, 1106 struct kvm_vcpu __kern *, vcpu) 1107 { 1108 struct pkvm_hyp_vcpu *hyp_vcpu; 1109 struct kvm_vcpu *host_vcpu; 1110 1111 host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); 1112 if (host_vcpu) { 1113 __kvm_adjust_pc(host_vcpu); 1114 return; 1115 } 1116 1117 /* 1118 * With no hyp vCPU loaded for it, the host vCPU may be unpinned, 1119 * and so unmapped at EL2: its first run pins it. A pin fails only 1120 * for memory the host isn't sharing, a bad pointer, so the request 1121 * is dropped. 1122 */ 1123 host_vcpu = kern_hyp_va_host(vcpu); 1124 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) 1125 return; 1126 1127 __kvm_adjust_pc(host_vcpu); 1128 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); 1129 } 1130 1131 DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) 1132 { 1133 __kvm_flush_vm_context(); 1134 } 1135 1136 DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, 1137 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) 1138 { 1139 __kvm_tlb_flush_vmid_ipa(kern_hyp_va_host(mmu), ipa, level); 1140 } 1141 1142 DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, 1143 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) 1144 { 1145 __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va_host(mmu), ipa, level); 1146 } 1147 1148 DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, 1149 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) 1150 { 1151 __kvm_tlb_flush_vmid_range(kern_hyp_va_host(mmu), start, pages); 1152 } 1153 1154 DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, 1155 struct kvm_s2_mmu __kern *, mmu) 1156 { 1157 __kvm_tlb_flush_vmid(kern_hyp_va_host(mmu)); 1158 } 1159 1160 DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, 1161 pkvm_handle_t, handle) 1162 { 1163 struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle); 1164 1165 if (!hyp_vm) 1166 return; 1167 1168 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 1169 put_pkvm_hyp_vm(hyp_vm); 1170 } 1171 1172 DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, 1173 struct kvm_s2_mmu __kern *, mmu) 1174 { 1175 __kvm_flush_cpu_context(kern_hyp_va_host(mmu)); 1176 } 1177 1178 DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, 1179 u64, cntvoff) 1180 { 1181 __kvm_timer_set_cntvoff(cntvoff); 1182 } 1183 1184 DEFINE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) 1185 { 1186 u64 tmp; 1187 1188 tmp = read_sysreg_el2(SYS_SCTLR); 1189 tmp |= SCTLR_ELx_DSSBS; 1190 write_sysreg_el2(tmp, SYS_SCTLR); 1191 } 1192 1193 DEFINE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) 1194 { 1195 return __vgic_v3_get_gic_config(); 1196 } 1197 1198 DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) 1199 { 1200 __vgic_v3_init_lrs(); 1201 } 1202 1203 DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, 1204 struct vgic_v3_cpu_if __kern *, cpu_if) 1205 { 1206 struct pkvm_hyp_vcpu *hyp_vcpu; 1207 struct kvm_vcpu *host_vcpu; 1208 1209 host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); 1210 if (!host_vcpu) 1211 return; 1212 1213 if (unlikely(hyp_vcpu)) { 1214 struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; 1215 int i; 1216 1217 hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; 1218 __vgic_v3_save_aprs(hyp_cpu_if); 1219 1220 host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; 1221 host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; 1222 for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { 1223 host_cpu_if->vgic_ap0r[i] = hyp_cpu_if->vgic_ap0r[i]; 1224 host_cpu_if->vgic_ap1r[i] = hyp_cpu_if->vgic_ap1r[i]; 1225 } 1226 } else { 1227 __vgic_v3_save_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); 1228 } 1229 } 1230 1231 DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, 1232 struct vgic_v3_cpu_if __kern *, cpu_if) 1233 { 1234 struct pkvm_hyp_vcpu *hyp_vcpu; 1235 struct kvm_vcpu *host_vcpu; 1236 1237 host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); 1238 if (!host_vcpu) 1239 return; 1240 1241 if (unlikely(hyp_vcpu)) { 1242 struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; 1243 int i; 1244 1245 hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; 1246 host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; 1247 1248 hyp_cpu_if->vgic_vmcr = host_cpu_if->vgic_vmcr; 1249 /* Should be a one-off */ 1250 hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | 1251 ICC_SRE_EL1_DFB | 1252 ICC_SRE_EL1_SRE); 1253 for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { 1254 hyp_cpu_if->vgic_ap0r[i] = host_cpu_if->vgic_ap0r[i]; 1255 hyp_cpu_if->vgic_ap1r[i] = host_cpu_if->vgic_ap1r[i]; 1256 } 1257 1258 __vgic_v3_restore_vmcr_aprs(hyp_cpu_if); 1259 } else { 1260 __vgic_v3_restore_vmcr_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); 1261 } 1262 } 1263 1264 DEFINE_KVM_HOST_HCALL(int, __pkvm_init, 1265 phys_addr_t, phys, unsigned long, size, 1266 unsigned long *, per_cpu_base, u32, hyp_va_bits) 1267 { 1268 /* 1269 * __pkvm_init() will return only if an error occurred, otherwise it 1270 * will tail-call in __pkvm_init_finalise() which will have to deal 1271 * with the host context directly. 1272 */ 1273 return __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); 1274 } 1275 1276 DEFINE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, 1277 enum arm64_hyp_spectre_vector, slot) 1278 { 1279 return pkvm_cpu_set_vector(slot); 1280 } 1281 1282 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, 1283 u64, pfn) 1284 { 1285 return __pkvm_host_share_hyp(pfn); 1286 } 1287 1288 DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, 1289 u64, pfn) 1290 { 1291 return __pkvm_host_unshare_hyp(pfn); 1292 } 1293 1294 DEFINE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, 1295 phys_addr_t, phys, size_t, size, u64, prot) 1296 { 1297 /* 1298 * __pkvm_create_private_mapping() populates a pointer with the 1299 * hypervisor start address of the allocation. 1300 * 1301 * However, handle___pkvm_create_private_mapping() hypercall crosses the 1302 * EL1/EL2 boundary so the pointer would not be valid in this context. 1303 * 1304 * Instead pass the allocation address as the return value (or return 1305 * ERR_PTR() on failure). 1306 */ 1307 ulong haddr; 1308 int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); 1309 1310 if (err) 1311 haddr = (ulong)ERR_PTR(err); 1312 1313 return haddr; 1314 } 1315 1316 DEFINE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) 1317 { 1318 return __pkvm_prot_finalize(); 1319 } 1320 1321 DEFINE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) 1322 { 1323 return __pkvm_reserve_vm(); 1324 } 1325 1326 DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, 1327 pkvm_handle_t, handle) 1328 { 1329 __pkvm_unreserve_vm(handle); 1330 } 1331 1332 DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, 1333 struct kvm __kern *, host_kvm, void __kern *, pgd_hva) 1334 { 1335 return errno_to_smccc(__pkvm_init_vm(kern_hyp_va_host(host_kvm), pgd_hva)); 1336 } 1337 1338 DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, 1339 pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) 1340 { 1341 return errno_to_smccc(__pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu))); 1342 } 1343 1344 DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) 1345 { 1346 struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); 1347 1348 return hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; 1349 } 1350 1351 DEFINE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, 1352 phys_addr_t, phys) 1353 { 1354 return __pkvm_host_force_reclaim_page_guest(phys); 1355 } 1356 1357 DEFINE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, 1358 pkvm_handle_t, handle, u64, gfn) 1359 { 1360 return __pkvm_reclaim_dying_guest_page(handle, gfn); 1361 } 1362 1363 DEFINE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, 1364 pkvm_handle_t, handle) 1365 { 1366 return __pkvm_start_teardown_vm(handle); 1367 } 1368 1369 DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, 1370 pkvm_handle_t, handle) 1371 { 1372 return __pkvm_finalize_teardown_vm(handle); 1373 } 1374 1375 DEFINE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) 1376 { 1377 struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; 1378 int ret = -EPERM; 1379 1380 #ifdef CONFIG_NVHE_EL2_DEBUG 1381 ret = hyp_allocator_selftest(); 1382 if (ret == -ENOMEM) { 1383 req.type = PKVM_HYP_REQ_HYP_ALLOC_SELFTEST; 1384 req.mem.nr_pages = hyp_alloc_selftest_topup_needed(); 1385 } 1386 #endif 1387 pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); 1388 1389 return ret; 1390 } 1391 1392 DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, 1393 enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) 1394 { 1395 struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); 1396 struct kvm_hyp_memcache host_mc = { 1397 .head = head, 1398 .nr_pages = nr_pages, 1399 }; 1400 int ret; 1401 1402 switch (id) { 1403 case PKVM_TOPUP_HYP_ALLOC: 1404 ret = hyp_alloc_topup(&host_mc); 1405 break; 1406 case PKVM_TOPUP_HYP_ALLOC_SELFTEST: 1407 ret = hyp_alloc_selftest_topup(&host_mc); 1408 break; 1409 default: 1410 ret = -EINVAL; 1411 } 1412 1413 cpu_reg(host_ctxt, 2) = host_mc.head; 1414 cpu_reg(host_ctxt, 3) = host_mc.nr_pages; 1415 1416 return ret; 1417 } 1418 1419 DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, 1420 enum pkvm_topup_id, id, unsigned long, target) 1421 { 1422 struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); 1423 struct kvm_hyp_memcache host_mc = {}; 1424 int ret = 0; 1425 1426 switch (id) { 1427 case PKVM_TOPUP_HYP_ALLOC: 1428 hyp_alloc_reclaim(&host_mc, target); 1429 break; 1430 case PKVM_TOPUP_HYP_ALLOC_SELFTEST: 1431 hyp_alloc_selftest_reclaim(&host_mc, target); 1432 break; 1433 default: 1434 ret = -EINVAL; 1435 } 1436 1437 cpu_reg(host_ctxt, 2) = host_mc.head; 1438 cpu_reg(host_ctxt, 3) = host_mc.nr_pages; 1439 1440 return ret; 1441 } 1442 1443 DEFINE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, 1444 enum pkvm_topup_id, id) 1445 { 1446 switch (id) { 1447 case PKVM_TOPUP_HYP_ALLOC: 1448 return hyp_alloc_reclaimable(); 1449 default: 1450 return 0; 1451 } 1452 } 1453 1454 DEFINE_KVM_HOST_HCALL(int, __tracing_load, 1455 void __kern *, desc_hva, size_t, desc_size) 1456 { 1457 return errno_to_smccc(__tracing_load(desc_hva, desc_size)); 1458 } 1459 1460 DEFINE_KVM_HOST_HCALL0(void, __tracing_unload) 1461 { 1462 __tracing_unload(); 1463 } 1464 1465 DEFINE_KVM_HOST_HCALL(int, __tracing_enable, 1466 bool, enable) 1467 { 1468 return __tracing_enable(enable); 1469 } 1470 1471 DEFINE_KVM_HOST_HCALL(int, __tracing_swap_reader, 1472 unsigned int, cpu) 1473 { 1474 return __tracing_swap_reader(cpu); 1475 } 1476 1477 DEFINE_KVM_HOST_HCALL(void, __tracing_update_clock, 1478 u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) 1479 { 1480 __tracing_update_clock(mult, shift, epoch_ns, epoch_cyc); 1481 } 1482 1483 DEFINE_KVM_HOST_HCALL(int, __tracing_reset, 1484 unsigned int, cpu) 1485 { 1486 return __tracing_reset(cpu); 1487 } 1488 1489 DEFINE_KVM_HOST_HCALL(int, __tracing_enable_event, 1490 unsigned short, id, bool, enable) 1491 { 1492 return __tracing_enable_event(id, enable); 1493 } 1494 1495 DEFINE_KVM_HOST_HCALL(void, __tracing_write_event, 1496 u64, id) 1497 { 1498 trace_selftest(id); 1499 } 1500 1501 DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, 1502 struct vgic_v5_cpu_if __kern *, cpu_if) 1503 { 1504 if (unlikely(is_protected_kvm_enabled())) 1505 return; 1506 1507 __vgic_v5_make_resident(kern_hyp_va_host(cpu_if)); 1508 } 1509 1510 DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, 1511 struct vgic_v5_cpu_if __kern *, cpu_if) 1512 { 1513 if (unlikely(is_protected_kvm_enabled())) 1514 return; 1515 1516 __vgic_v5_make_non_resident(kern_hyp_va_host(cpu_if)); 1517 } 1518 1519 DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, 1520 struct vgic_v5_cpu_if __kern *, cpu_if) 1521 { 1522 __vgic_v5_save_apr(kern_hyp_va_host(cpu_if)); 1523 } 1524 1525 DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, 1526 struct vgic_v5_cpu_if __kern *, cpu_if) 1527 { 1528 __vgic_v5_restore_vmcr_apr(kern_hyp_va_host(cpu_if)); 1529 } 1530 1531 DEFINE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, 1532 u32, intid, bool, pending, u16, vm) 1533 { 1534 if (unlikely(is_protected_kvm_enabled())) 1535 return; 1536 1537 __vgic_v5_vdpend(intid, pending, vm); 1538 } 1539 1540 typedef void (*hcall_t)(struct kvm_cpu_context *); 1541 1542 #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x 1543 1544 static const hcall_t host_hcall[] = { 1545 /* ___kvm_hyp_init */ 1546 HANDLE_FUNC(__pkvm_init), 1547 HANDLE_FUNC(__pkvm_create_private_mapping), 1548 HANDLE_FUNC(__pkvm_cpu_set_vector), 1549 HANDLE_FUNC(__kvm_enable_ssbs), 1550 HANDLE_FUNC(__vgic_v3_init_lrs), 1551 HANDLE_FUNC(__vgic_v3_get_gic_config), 1552 HANDLE_FUNC(__pkvm_hyp_alloc_selftest), 1553 HANDLE_FUNC(__pkvm_prot_finalize), 1554 1555 HANDLE_FUNC(__kvm_adjust_pc), 1556 HANDLE_FUNC(__kvm_vcpu_run), 1557 HANDLE_FUNC(__kvm_flush_vm_context), 1558 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), 1559 HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa_nsh), 1560 HANDLE_FUNC(__kvm_tlb_flush_vmid), 1561 HANDLE_FUNC(__kvm_tlb_flush_vmid_range), 1562 HANDLE_FUNC(__kvm_flush_cpu_context), 1563 HANDLE_FUNC(__kvm_timer_set_cntvoff), 1564 HANDLE_FUNC(__tracing_load), 1565 HANDLE_FUNC(__tracing_unload), 1566 HANDLE_FUNC(__tracing_enable), 1567 HANDLE_FUNC(__tracing_swap_reader), 1568 HANDLE_FUNC(__tracing_update_clock), 1569 HANDLE_FUNC(__tracing_reset), 1570 HANDLE_FUNC(__tracing_enable_event), 1571 HANDLE_FUNC(__tracing_write_event), 1572 HANDLE_FUNC(__vgic_v3_save_aprs), 1573 HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), 1574 HANDLE_FUNC(__vgic_v5_make_resident), 1575 HANDLE_FUNC(__vgic_v5_make_non_resident), 1576 HANDLE_FUNC(__vgic_v5_vdpend), 1577 HANDLE_FUNC(__vgic_v5_save_apr), 1578 HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), 1579 1580 HANDLE_FUNC(__pkvm_hyp_topup), 1581 HANDLE_FUNC(__pkvm_hyp_reclaim), 1582 HANDLE_FUNC(__pkvm_hyp_reclaimable), 1583 HANDLE_FUNC(__pkvm_host_share_hyp), 1584 HANDLE_FUNC(__pkvm_host_unshare_hyp), 1585 1586 HANDLE_FUNC(__pkvm_host_donate_guest), 1587 HANDLE_FUNC(__pkvm_host_share_guest), 1588 HANDLE_FUNC(__pkvm_host_unshare_guest), 1589 HANDLE_FUNC(__pkvm_host_relax_perms_guest), 1590 HANDLE_FUNC(__pkvm_host_wrprotect_guest), 1591 HANDLE_FUNC(__pkvm_host_test_clear_young_guest), 1592 HANDLE_FUNC(__pkvm_host_mkyoung_guest), 1593 HANDLE_FUNC(__pkvm_reserve_vm), 1594 HANDLE_FUNC(__pkvm_unreserve_vm), 1595 HANDLE_FUNC(__pkvm_init_vm), 1596 HANDLE_FUNC(__pkvm_init_vcpu), 1597 HANDLE_FUNC(__pkvm_vcpu_in_poison_fault), 1598 HANDLE_FUNC(__pkvm_force_reclaim_guest_page), 1599 HANDLE_FUNC(__pkvm_reclaim_dying_guest_page), 1600 HANDLE_FUNC(__pkvm_start_teardown_vm), 1601 HANDLE_FUNC(__pkvm_finalize_teardown_vm), 1602 HANDLE_FUNC(__pkvm_vcpu_load), 1603 HANDLE_FUNC(__pkvm_vcpu_put), 1604 HANDLE_FUNC(__pkvm_vcpu_sync_state), 1605 HANDLE_FUNC(__pkvm_tlb_flush_vmid), 1606 }; 1607 1608 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) 1609 { 1610 DECLARE_REG(unsigned long, id, host_ctxt, 0); 1611 unsigned long hcall_min = 0, hcall_max = __KVM_HOST_SMCCC_FUNC_MAX; 1612 hcall_t hfn; 1613 1614 BUILD_BUG_ON(ARRAY_SIZE(host_hcall) != __KVM_HOST_SMCCC_FUNC_MAX); 1615 1616 /* 1617 * If pKVM has been initialised then reject any calls to the 1618 * early "privileged" hypercalls. Note that we cannot reject 1619 * calls to __pkvm_prot_finalize for two reasons: (1) The static 1620 * key used to determine initialisation must be toggled prior to 1621 * finalisation and (2) finalisation is performed on a per-CPU 1622 * basis. This is all fine, however, since __pkvm_prot_finalize 1623 * returns -EPERM after the first call for a given CPU. 1624 */ 1625 if (static_branch_unlikely(&kvm_protected_mode_initialized)) { 1626 hcall_min = __KVM_HOST_SMCCC_FUNC_MIN_PKVM; 1627 } else { 1628 hcall_max = __KVM_HOST_SMCCC_FUNC_PKVM_ONLY; 1629 } 1630 1631 id &= ~ARM_SMCCC_CALL_HINTS; 1632 id -= KVM_HOST_SMCCC_ID(0); 1633 1634 if (unlikely(id < hcall_min || id >= hcall_max)) 1635 goto inval; 1636 1637 hfn = host_hcall[id]; 1638 if (unlikely(!hfn)) 1639 goto inval; 1640 1641 cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS; 1642 hfn(host_ctxt); 1643 1644 return; 1645 inval: 1646 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 1647 } 1648 1649 static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt) 1650 { 1651 trace_hyp_exit(host_ctxt, HYP_REASON_SMC); 1652 __kvm_hyp_host_forward_smc(host_ctxt); 1653 trace_hyp_enter(host_ctxt, HYP_REASON_SMC); 1654 } 1655 1656 static void handle_host_smc(struct kvm_cpu_context *host_ctxt) 1657 { 1658 DECLARE_REG(u64, func_id, host_ctxt, 0); 1659 u64 esr = read_sysreg_el2(SYS_ESR); 1660 bool handled; 1661 1662 if (esr & ESR_ELx_xVC_IMM_MASK) { 1663 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 1664 goto exit_skip_instr; 1665 } 1666 1667 func_id &= ~ARM_SMCCC_CALL_HINTS; 1668 if (upper_32_bits(func_id)) { 1669 cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; 1670 goto exit_skip_instr; 1671 } 1672 1673 handled = kvm_host_psci_handler(host_ctxt, func_id); 1674 if (!handled) 1675 handled = kvm_host_ffa_handler(host_ctxt, func_id); 1676 if (!handled) 1677 default_host_smc_handler(host_ctxt); 1678 1679 exit_skip_instr: 1680 /* SMC was trapped, move ELR past the current PC. */ 1681 kvm_skip_host_instr(); 1682 } 1683 1684 void inject_host_exception(u64 esr) 1685 { 1686 u64 sctlr, spsr_el1, spsr_el2, exc_offset = except_type_sync; 1687 const u64 spsr_mask = PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | 1688 PSR_V_BIT | PSR_DIT_BIT | PSR_PAN_BIT; 1689 1690 spsr_el1 = spsr_el2 = read_sysreg_el2(SYS_SPSR); 1691 switch (spsr_el1 & (PSR_MODE_MASK | PSR_MODE32_BIT)) { 1692 case PSR_MODE_EL0t: 1693 exc_offset += LOWER_EL_AArch64_VECTOR; 1694 break; 1695 case PSR_MODE_EL0t | PSR_MODE32_BIT: 1696 exc_offset += LOWER_EL_AArch32_VECTOR; 1697 break; 1698 default: 1699 exc_offset += CURRENT_EL_SP_ELx_VECTOR; 1700 } 1701 1702 spsr_el2 &= spsr_mask; 1703 spsr_el2 |= PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | 1704 PSR_MODE_EL1h; 1705 1706 sctlr = read_sysreg_el1(SYS_SCTLR); 1707 if (!(sctlr & SCTLR_EL1_SPAN)) 1708 spsr_el2 |= PSR_PAN_BIT; 1709 1710 if (sctlr & SCTLR_ELx_DSSBS) 1711 spsr_el2 |= PSR_SSBS_BIT; 1712 1713 if (system_supports_mte()) 1714 spsr_el2 |= PSR_TCO_BIT; 1715 1716 if (esr_fsc_is_translation_fault(esr)) 1717 write_sysreg_el1(read_sysreg_el2(SYS_FAR), SYS_FAR); 1718 1719 write_sysreg_el1(esr, SYS_ESR); 1720 write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); 1721 write_sysreg_el1(spsr_el1, SYS_SPSR); 1722 write_sysreg_el2(read_sysreg_el1(SYS_VBAR) + exc_offset, SYS_ELR); 1723 write_sysreg_el2(spsr_el2, SYS_SPSR); 1724 } 1725 1726 static void inject_host_undef64(void) 1727 { 1728 inject_host_exception((ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | 1729 ESR_ELx_IL); 1730 } 1731 1732 static bool handle_host_mte(u64 esr) 1733 { 1734 switch (esr_sys64_to_sysreg(esr)) { 1735 case SYS_RGSR_EL1: 1736 case SYS_GCR_EL1: 1737 case SYS_TFSR_EL1: 1738 case SYS_TFSRE0_EL1: 1739 /* If we're here for any reason other than MTE, it's a bug. */ 1740 if (read_sysreg(HCR_EL2) & HCR_ATA) 1741 return false; 1742 break; 1743 case SYS_GMID_EL1: 1744 /* If we're here for any reason other than MTE, it's a bug. */ 1745 if (!(read_sysreg(HCR_EL2) & HCR_TID5)) 1746 return false; 1747 break; 1748 default: 1749 return false; 1750 } 1751 1752 inject_host_undef64(); 1753 return true; 1754 } 1755 1756 void handle_trap(struct kvm_cpu_context *host_ctxt) 1757 { 1758 u64 esr = read_sysreg_el2(SYS_ESR); 1759 1760 1761 switch (ESR_ELx_EC(esr)) { 1762 case ESR_ELx_EC_HVC64: 1763 trace_hyp_enter(host_ctxt, HYP_REASON_HVC); 1764 handle_host_hcall(host_ctxt); 1765 break; 1766 case ESR_ELx_EC_SMC64: 1767 trace_hyp_enter(host_ctxt, HYP_REASON_SMC); 1768 handle_host_smc(host_ctxt); 1769 break; 1770 case ESR_ELx_EC_IABT_LOW: 1771 case ESR_ELx_EC_DABT_LOW: 1772 trace_hyp_enter(host_ctxt, HYP_REASON_HOST_ABORT); 1773 handle_host_mem_abort(host_ctxt); 1774 break; 1775 case ESR_ELx_EC_SYS64: 1776 trace_hyp_enter(host_ctxt, HYP_REASON_SYS); 1777 if (handle_host_mte(esr)) 1778 break; 1779 fallthrough; 1780 default: 1781 BUG(); 1782 } 1783 1784 trace_hyp_exit(host_ctxt, HYP_REASON_ERET_HOST); 1785 } 1786