1 /* SPDX-License-Identifier: GPL-2.0-only */
3  * Copyright (C) 2012,2013 - ARM Ltd
7  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
14 #include <linux/arm-smccc.h>
43 #define KVM_VCPU_VALID_FEATURES	(BIT(KVM_VCPU_MAX_FEATURES) - 1)
64  * Mode of operation configurable with kvm-arm.mode early param.
65  * See Documentation/admin-guide/kernel-parameters.txt for more information.
100 	*p = mc->head;  in push_hyp_memcache()
101 	mc->head = to_pa(p);  in push_hyp_memcache()
102 	mc->nr_pages++;  in push_hyp_memcache()
108 	phys_addr_t *p = to_va(mc->head & PAGE_MASK);  in pop_hyp_memcache()
110 	if (!mc->nr_pages)  in pop_hyp_memcache()
113 	mc->head = *p;  in pop_hyp_memcache()
114 	mc->nr_pages--;  in pop_hyp_memcache()
125 	while (mc->nr_pages < min_pages) {  in __topup_hyp_memcache()
129 			return -ENOMEM;  in __topup_hyp_memcache()
141 	while (mc->nr_pages)  in __free_hyp_memcache()
160 	 * translation regime that isn't affected by its own stage-2
161 	 * translation, such as a non-VHE hypervisor running at vEL2, or
163 	 * canonical stage-2 page tables.
169 	 * VTCR value used on the host. For a non-NV guest (or a NV
171 	 * apply), its T0SZ value reflects that of the IPA size.
191 	 * Protected by kvm->slots_lock.
199 	 * For a shadow stage-2 MMU, the virtual vttbr used by the
202 	 * - the virtual VTTBR programmed by the guest hypervisor with
204 	 * - The value 1 (VMID=0, BADDR=0, CnP=1) if invalid
245 	unsigned long vendor_hyp_bmap; /* Function numbers 0-63 */
246 	unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */
265 	unsigned long index = 0, mask = data->mpidr_mask;  in kvm_mpidr_index()
297 	 * Fine-Grained UNDEF, mimicking the FGT layout defined by the
299 	 * same feature-set to all vcpus.
322 	/* Protects VM-scoped configuration data */
346 	/* Fine-Grained UNDEF initialised */
354 	/* VM-wide vCPU feature set */
361 	 * VM-wide PMU filter, implemented as a bitmap and big enough for
384 	 * These emulated idregs are VM-wide, but accessed from the context of a vCPU.
387 #define IDREG_IDX(id)		(((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id))
396 	/* Masks for VNCR-backed and general EL2 sysregs */
412 	u64 hpfar_el2;		/* Hyp IPA Fault Address Register */
418  * __VNCR_START__, and the value (after correction) to be an 8-byte offset
427 #define __MAX__(x,y)	((x) ^ (((x) ^ (y)) & -((x) < (y))))
431 	__after_##r = __MAX__(__before_##r - 1, r)
434 	m, __after_##m = m - 1
453 	PMEVCNTR0_EL0,	/* Event Counter Register (0-30) */
456 	PMEVTYPER0_EL0,	/* Event Type Register (0-30) */
490 	FPEXC32_EL2,	/* Floating-Point Exception Control Register */
511 	HPFAR_EL2,	/* Hypervisor IPA Fault Address Register */
527 	CNTHCTL_EL2,	/* Counter-timer Hypervisor Control register */
529 	/* Any VNCR-capable reg goes after this point */
623 	} mask[NR_SYS_REGS - __SANITISED_REG_START__];
690  * This structure is instantiated on a per-CPU basis, and contains
693  * - tied to a single physical CPU, and
694  * - either have a lifetime that does not extend past vcpu_put()
695  * - or is an invariant for the lifetime of the system
736 		/* Self-hosted trace */
862 	/* Pages to top-up the pKVM/EL2 guest pool */
877 	/* Per-vcpu CCSIDR override or NULL */
880 	/* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */
885  * Each 'flag' is composed of a comma-separated triplet:
887  * - the flag-set it belongs to in the vcpu->arch structure
888  * - the value for that flag
889  * - the mask for that flag
891  *  __vcpu_single_flag() builds such a triplet for a single-bit flag.
902 		typeof(v->arch.flagset) *_fset;			\
914 		READ_ONCE(v->arch.flagset) & (m);		\
918  * Note that the set/clear accessors must be preempt-safe in order to
922 /* the nVHE hypervisor is always non-preemptible */
932 		typeof(v->arch.flagset) *fset;			\
936 		fset = &v->arch.flagset;			\
946 		typeof(v->arch.flagset) *fset;			\
950 		fset = &v->arch.flagset;			\
1008 /* Software step state is Active-pending for external debug */
1021 #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) +	\
1022 			     sve_ffr_offset((vcpu)->arch.sve_max_vl))
1024 #define vcpu_sve_max_vq(vcpu)	sve_vq_from_vl((vcpu)->arch.sve_max_vl)
1043 #define vcpu_sve_state_size(vcpu) sve_state_size_from_vl((vcpu)->arch.sve_max_vl)
1051 				 test_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &(kvm)->arch.flags))
1054 #define vcpu_has_sve(vcpu)	kvm_has_sve(kern_hyp_va((vcpu)->kvm))
1056 #define vcpu_has_sve(vcpu)	kvm_has_sve((vcpu)->kvm)
1078 #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.regs)
1087  * Don't bother with VNCR-based accesses in the nVHE code, it has no
1094 		     r >= __VNCR_START__ && ctxt->vncr_array))  in ___ctxt_sys_reg()
1095 		return &ctxt->vncr_array[r - __VNCR_START__];  in ___ctxt_sys_reg()
1097 	return (u64 *)&ctxt->sys_regs[r];  in ___ctxt_sys_reg()
1113 		const struct kvm_cpu_context *ctxt = &(v)->arch.ctxt;	\
1123 		const struct kvm_cpu_context *ctxt = &(v)->arch.ctxt;	\
1134 		const struct kvm_cpu_context *ctxt = &(v)->arch.ctxt;	\
1155 	 * thread when emulating cross-VCPU communication.  in __vcpu_read_sys_reg_from_cpu()
1277 #define vcpu_has_run_once(vcpu)	(!!READ_ONCE((vcpu)->pid))
1382 	vcpu_arch->steal.base = INVALID_GPA;  in kvm_arm_pvtime_vcpu_init()
1387 	return (vcpu_arch->steal.base != INVALID_GPA);  in kvm_arm_is_pvtime_enabled()
1397  * How we access per-CPU host data depends on the where we access it from,
1400  * - VHE and nVHE hypervisor bits use their locally defined instance
1402  * - the rest of the kernel use either the VHE or nVHE one, depending on
1406  *   per-CPU stuff is exclusively accessible to the protected EL2 code.
1408  *   (which makes sense in a way as there shouldn't be any shared state
1414 #define host_data_ptr(f)	(&this_cpu_ptr(&kvm_host_data)->f)
1418 	 &this_cpu_ptr(&kvm_host_data)->f :				\
1419 	 &this_cpu_ptr_hyp_sym(kvm_host_data)->f)
1462 	((vcpu)->arch.debug_owner != VCPU_DEBUG_FREE)
1464 	((vcpu)->arch.debug_owner == VCPU_DEBUG_HOST_OWNED)
1466 	((vcpu)->arch.debug_owner == VCPU_DEBUG_GUEST_OWNED)
1491 	return (!has_vhe() && attr->exclude_host);  in kvm_pmu_counter_deferred()
1526 #define kvm_vm_is_protected(kvm)	(is_protected_kvm_enabled() && (kvm)->arch.pkvm.enabled)
1528 #define vcpu_is_protected(vcpu)		kvm_vm_is_protected((vcpu)->kvm)
1537 	 test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags))
1544 	(test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &(kvm)->arch.flags))
1548 	return test_bit(feature, ka->vcpu_features);  in __vcpu_has_feature()
1551 #define kvm_vcpu_has_feature(k, f)	__vcpu_has_feature(&(k)->arch, (f))
1552 #define vcpu_has_feature(v, f)	__vcpu_has_feature(&(v)->kvm->arch, (f))
1572 		return &ka->id_regs[IDREG_IDX(reg)];  in __vm_id_reg()
1574 		return &ka->ctr_el0;  in __vm_id_reg()
1576 		return &ka->midr_el1;  in __vm_id_reg()
1578 		return &ka->revidr_el1;  in __vm_id_reg()
1580 		return &ka->aidr_el1;  in __vm_id_reg()
1588 	({ u64 __val = *__vm_id_reg(&(kvm)->arch, reg); __val; })
1598 		sign_extend64(__val, id##_##fld##_WIDTH - 1);		\
1610 		sign_extend64(__val, id##_##fld##_WIDTH - 1);		\