1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * The host<->hyp hypercall interface. 4 * 5 * Copyright (C) 2026 Google LLC 6 * Author: Fuad Tabba <fuad.tabba@linux.dev> 7 */ 8 9 #ifndef __ARM64_KVM_HCALL_H__ 10 #define __ARM64_KVM_HCALL_H__ 11 12 #include <linux/args.h> 13 #include <linux/arm-smccc.h> 14 #include <linux/bug.h> 15 #include <linux/errno.h> 16 #include <linux/types.h> 17 18 #include <asm/barrier.h> 19 #include <asm/kvm_asm.h> 20 #include <asm/spectre.h> 21 #include <asm/virt.h> 22 23 typedef u16 pkvm_handle_t; 24 25 enum pkvm_topup_id { 26 PKVM_TOPUP_HYP_ALLOC, 27 NR_PKVM_TOPUP_HYP_IDS, 28 PKVM_TOPUP_HYP_ALLOC_SELFTEST, 29 }; 30 31 struct kvm; 32 struct kvm_s2_mmu; 33 struct kvm_vcpu; 34 struct vgic_v3_cpu_if; 35 struct vgic_v5_cpu_if; 36 37 /* 38 * A host VA carried by a hypercall argument. At EL2 such a pointer must not 39 * be dereferenced until it is translated with kern_hyp_va_host(); sparse 40 * flags any use that skips the translation. The tag describes the EL2 view 41 * only: the host dereferences its own VAs freely. 42 */ 43 #if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__) 44 #define __kern __attribute__((noderef, address_space(__kern))) 45 #else 46 #define __kern 47 #endif 48 49 /* 50 * Hypercall signatures are declared as (type, name) argument pairs. 51 * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP() 52 * in <linux/syscalls.h>. The ladder is indexed by list entries, two per 53 * argument; __KVM_HCALL_MAP_N() takes that count explicitly. 54 */ 55 #define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a) 56 #define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__) 57 #define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__) 58 #define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__) 59 #define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__) 60 #define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__) 61 #define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__) 62 #define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__) 63 64 #define __KVM_HCALL_DECL(t, a) t a 65 #define __KVM_HCALL_LONG(t, a) unsigned long a 66 #define __KVM_HCALL_CAST(t, a) (__force t) a 67 #define __KVM_HCALL_ARGS(t, a) a 68 69 #ifndef __KVM_NVHE_HYPERVISOR__ 70 #define __kvm_call_hyp_nvhe(f, ...) \ 71 ({ \ 72 struct arm_smccc_res res; \ 73 \ 74 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ 75 ##__VA_ARGS__, &res); \ 76 if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ 77 res.a1 = -EOPNOTSUPP; \ 78 \ 79 res.a1; \ 80 }) 81 82 /* 83 * Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe() 84 * resolves to the stub, so a call with the wrong argument count or types 85 * fails to compile instead of being silently truncated to an SMCCC function 86 * number and a pile of registers. The stub inlines to the same SMCCC call 87 * the untyped macro used to make. 88 */ 89 #define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ 90 static __always_inline \ 91 ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \ 92 { \ 93 return (ret)__kvm_call_hyp_nvhe(name, \ 94 __KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\ 95 } 96 97 #define DECLARE_KVM_HOST_HCALL0(ret, name) \ 98 static __always_inline ret nvhe_hvc_##name(void) \ 99 { \ 100 return (ret)__kvm_call_hyp_nvhe(name); \ 101 } 102 103 #define kvm_call_hyp_nvhe(f, ...) nvhe_hvc_##f(__VA_ARGS__) 104 105 /* 106 * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, 107 * where the eret to EL1 acts as a context synchronization event. 108 */ 109 #define kvm_call_hyp(f, ...) \ 110 do { \ 111 if (has_vhe()) { \ 112 f(__VA_ARGS__); \ 113 isb(); \ 114 } else { \ 115 kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ 116 } \ 117 } while (0) 118 119 #define kvm_call_hyp_ret(f, ...) \ 120 ({ \ 121 typeof(f(__VA_ARGS__)) ret; \ 122 \ 123 if (has_vhe()) { \ 124 ret = f(__VA_ARGS__); \ 125 } else { \ 126 ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ 127 } \ 128 \ 129 ret; \ 130 }) 131 #else /* __KVM_NVHE_HYPERVISOR__ */ 132 #define kvm_call_hyp(f, ...) f(__VA_ARGS__) 133 #define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) 134 #define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) 135 136 /* 137 * At EL2 each declaration emits the canonical signature of the hypercall, 138 * which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler 139 * definition against. 140 */ 141 #define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ 142 typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)); 143 #define DECLARE_KVM_HOST_HCALL0(ret, name) \ 144 typedef ret kvm_host_hcall_sig_##name(void); 145 #endif /* __KVM_NVHE_HYPERVISOR__ */ 146 147 /* Hypercalls that are unavailable once pKVM has finalised. */ 148 DECLARE_KVM_HOST_HCALL(int, __pkvm_init, 149 phys_addr_t, phys, unsigned long, size, 150 unsigned long *, per_cpu_base, u32, hyp_va_bits) 151 DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, 152 phys_addr_t, phys, size_t, size, u64, prot) 153 DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, 154 enum arm64_hyp_spectre_vector, slot) 155 DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) 156 DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) 157 DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) 158 DECLARE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) 159 160 DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) 161 162 /* Hypercalls that are always available and common to [nh]VHE/pKVM. */ 163 DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc, 164 struct kvm_vcpu __kern *, vcpu) 165 DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run, 166 struct kvm_vcpu __kern *, vcpu) 167 DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) 168 DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, 169 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) 170 DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, 171 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) 172 DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, 173 struct kvm_s2_mmu __kern *, mmu) 174 DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, 175 struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) 176 DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, 177 struct kvm_s2_mmu __kern *, mmu) 178 DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, 179 u64, cntvoff) 180 DECLARE_KVM_HOST_HCALL(int, __tracing_load, 181 void __kern *, desc_hva, size_t, desc_size) 182 DECLARE_KVM_HOST_HCALL0(void, __tracing_unload) 183 DECLARE_KVM_HOST_HCALL(int, __tracing_enable, 184 bool, enable) 185 DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader, 186 unsigned int, cpu) 187 DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock, 188 u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) 189 DECLARE_KVM_HOST_HCALL(int, __tracing_reset, 190 unsigned int, cpu) 191 DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event, 192 unsigned short, id, bool, enable) 193 DECLARE_KVM_HOST_HCALL(void, __tracing_write_event, 194 u64, id) 195 DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, 196 struct vgic_v5_cpu_if __kern *, cpu_if); 197 DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, 198 struct vgic_v5_cpu_if __kern *, cpu_if); 199 DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, 200 struct vgic_v3_cpu_if __kern *, cpu_if) 201 DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, 202 struct vgic_v3_cpu_if __kern *, cpu_if) 203 DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, 204 struct vgic_v5_cpu_if __kern *, cpu_if) 205 DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, 206 struct vgic_v5_cpu_if __kern *, cpu_if) 207 DECLARE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, 208 u32, intid, bool, pending, u16, vm); 209 DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, 210 enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) 211 DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, 212 enum pkvm_topup_id, id, unsigned long, target) 213 DECLARE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, 214 enum pkvm_topup_id, id) 215 216 /* Hypercalls that are available only when pKVM has finalised. */ 217 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, 218 u64, pfn) 219 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, 220 u64, pfn) 221 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, 222 u64, pfn, u64, gfn) 223 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, 224 u64, pfn, u64, gfn, u64, nr_pages, u64, prot) 225 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, 226 pkvm_handle_t, handle, u64, gfn, u64, nr_pages) 227 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, 228 u64, gfn, u64, prot) 229 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, 230 pkvm_handle_t, handle, u64, gfn, u64, nr_pages) 231 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, 232 pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) 233 DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, 234 u64, gfn) 235 DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) 236 DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, 237 pkvm_handle_t, handle) 238 DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, 239 struct kvm __kern *, host_kvm, void __kern *, pgd_hva) 240 DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, 241 pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) 242 DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) 243 DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, 244 phys_addr_t, phys) 245 DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, 246 pkvm_handle_t, handle, u64, gfn) 247 DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, 248 pkvm_handle_t, handle) 249 DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, 250 pkvm_handle_t, handle) 251 DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, 252 pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) 253 DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) 254 DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) 255 DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, 256 pkvm_handle_t, handle) 257 258 #endif /* __ARM64_KVM_HCALL_H__ */ 259