1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4 */
5
6 #ifndef __ASM_LOONGARCH_KVM_HOST_H__
7 #define __ASM_LOONGARCH_KVM_HOST_H__
8
9 #include <linux/cpumask.h>
10 #include <linux/hrtimer.h>
11 #include <linux/interrupt.h>
12 #include <linux/kvm.h>
13 #include <linux/kvm_types.h>
14 #include <linux/mutex.h>
15 #include <linux/perf_event.h>
16 #include <linux/spinlock.h>
17 #include <linux/threads.h>
18 #include <linux/types.h>
19
20 #include <asm/inst.h>
21 #include <asm/kvm_mmu.h>
22 #include <asm/kvm_ipi.h>
23 #include <asm/kvm_dmsintc.h>
24 #include <asm/kvm_eiointc.h>
25 #include <asm/kvm_pch_pic.h>
26 #include <asm/loongarch.h>
27
28 #define __KVM_HAVE_ARCH_INTC_INITIALIZED
29
30 /* Loongarch KVM register ids */
31 #define KVM_GET_IOC_CSR_IDX(id) ((id & KVM_CSR_IDX_MASK) >> LOONGARCH_REG_SHIFT)
32 #define KVM_GET_IOC_CPUCFG_IDX(id) ((id & KVM_CPUCFG_IDX_MASK) >> LOONGARCH_REG_SHIFT)
33
34 #define KVM_MAX_VCPUS 256
35 #define KVM_MAX_CPUCFG_REGS 21
36
37 #define KVM_HALT_POLL_NS_DEFAULT 500000
38 #define KVM_REQ_TLB_FLUSH_GPA KVM_ARCH_REQ(0)
39 #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(1)
40 #define KVM_REQ_PMU KVM_ARCH_REQ(2)
41 #define KVM_REQ_AUX_LOAD KVM_ARCH_REQ(3)
42
43 #define KVM_GUESTDBG_SW_BP_MASK \
44 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)
45 #define KVM_GUESTDBG_VALID_MASK \
46 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP | KVM_GUESTDBG_SINGLESTEP)
47
48 #define KVM_DIRTY_LOG_MANUAL_CAPS \
49 (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | KVM_DIRTY_LOG_INITIALLY_SET)
50
51 struct kvm_vm_stat {
52 struct kvm_vm_stat_generic generic;
53 u64 pages;
54 u64 hugepages;
55 };
56
57 struct kvm_vcpu_stat {
58 struct kvm_vcpu_stat_generic generic;
59 u64 int_exits;
60 u64 idle_exits;
61 u64 cpucfg_exits;
62 u64 signal_exits;
63 u64 hypercall_exits;
64 u64 ipi_read_exits;
65 u64 ipi_write_exits;
66 u64 eiointc_read_exits;
67 u64 eiointc_write_exits;
68 u64 pch_pic_read_exits;
69 u64 pch_pic_write_exits;
70 };
71
72 #define KVM_MEM_HUGEPAGE_CAPABLE (1UL << 0)
73 #define KVM_MEM_HUGEPAGE_INCAPABLE (1UL << 1)
74 struct kvm_arch_memory_slot {
75 unsigned long flags;
76 };
77
78 #define HOST_MAX_PMNUM 16
79 struct kvm_context {
80 unsigned long vpid_cache;
81 struct kvm_vcpu *last_vcpu;
82 /* Host PMU CSR */
83 u64 perf_ctrl[HOST_MAX_PMNUM];
84 u64 perf_cntr[HOST_MAX_PMNUM];
85 };
86
87 struct kvm_world_switch {
88 int (*exc_entry)(void);
89 int (*enter_guest)(struct kvm_run *run, struct kvm_vcpu *vcpu);
90 };
91
92 #define MAX_PGTABLE_LEVELS 4
93
94 /*
95 * Physical CPUID is used for interrupt routing, there are different
96 * definitions about physical cpuid on different hardwares.
97 *
98 * For LOONGARCH_CSR_CPUID register, max CPUID size if 512
99 * For IPI hardware, max destination CPUID size 1024
100 * For eiointc interrupt controller, max destination CPUID size is 256
101 * For msgint interrupt controller, max supported CPUID size is 65536
102 *
103 * Currently max CPUID is defined as 256 for KVM hypervisor, in future
104 * it will be expanded to 4096, including 16 packages at most. And every
105 * package supports at most 256 vcpus
106 */
107 #define KVM_MAX_PHYID 256
108
109 struct kvm_phyid_info {
110 struct kvm_vcpu *vcpu;
111 bool enabled;
112 };
113
114 struct kvm_phyid_map {
115 int max_phyid;
116 struct kvm_phyid_info phys_map[KVM_MAX_PHYID];
117 };
118
119 struct kvm_arch {
120 /* Guest physical mm */
121 kvm_pte_t *pgd;
122 unsigned long gpa_size;
123 unsigned long invalid_ptes[MAX_PGTABLE_LEVELS];
124 unsigned int pte_shifts[MAX_PGTABLE_LEVELS];
125 unsigned int root_level;
126 spinlock_t phyid_map_lock;
127 struct kvm_phyid_map *phyid_map;
128 /* Enabled PV features */
129 unsigned long pv_features;
130 /* Supported KVM features */
131 unsigned long kvm_features;
132
133 s64 time_offset;
134 struct kvm_context __percpu *vmcs;
135 struct loongarch_ipi *ipi;
136 struct loongarch_dmsintc *dmsintc;
137 struct loongarch_eiointc *eiointc;
138 struct loongarch_pch_pic *pch_pic;
139 };
140
141 #define CSR_MAX_NUMS 0x800
142
143 struct loongarch_csrs {
144 unsigned long csrs[CSR_MAX_NUMS];
145 };
146
147 /* Resume Flags */
148 #define RESUME_HOST 0
149 #define RESUME_GUEST 1
150
151 enum emulation_result {
152 EMULATE_DONE, /* no further processing */
153 EMULATE_DO_MMIO, /* kvm_run filled with MMIO request */
154 EMULATE_DO_IOCSR, /* handle IOCSR request */
155 EMULATE_FAIL, /* can't emulate this instruction */
156 EMULATE_EXCEPT, /* A guest exception has been generated */
157 };
158
159 #define KVM_LARCH_FPU (0x1 << 0)
160 #define KVM_LARCH_LSX (0x1 << 1)
161 #define KVM_LARCH_LASX (0x1 << 2)
162 #define KVM_LARCH_LBT (0x1 << 3)
163 #define KVM_LARCH_PMU (0x1 << 4)
164 #define KVM_LARCH_SWCSR_LATEST (0x1 << 5)
165 #define KVM_LARCH_HWCSR_USABLE (0x1 << 6)
166
167 #define LOONGARCH_PV_FEAT_UPDATED BIT_ULL(63)
168 #define LOONGARCH_PV_FEAT_MASK (BIT(KVM_FEATURE_IPI) | \
169 BIT(KVM_FEATURE_PREEMPT) | \
170 BIT(KVM_FEATURE_STEAL_TIME) | \
171 BIT(KVM_FEATURE_USER_HCALL) | \
172 BIT(KVM_FEATURE_VIRT_EXTIOI))
173
174 struct kvm_vcpu_arch {
175 /*
176 * Switch pointer-to-function type to unsigned long
177 * for loading the value into register directly.
178 */
179 unsigned long host_eentry;
180 unsigned long guest_eentry;
181
182 /* Pointers stored here for easy accessing from assembly code */
183 int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu);
184
185 /* GPA (=HVA) of PGD for secondary mmu */
186 unsigned long kvm_pgd;
187
188 /* Host registers preserved across guest mode execution */
189 unsigned long host_sp;
190 unsigned long host_tp;
191 unsigned long host_pgd;
192
193 /* Host CSRs are used when handling exits from guest */
194 unsigned long badi;
195 unsigned long badv;
196 unsigned long host_ecfg;
197 unsigned long host_estat;
198 unsigned long host_percpu;
199
200 /* GPRs */
201 unsigned long gprs[32];
202 unsigned long pc;
203
204 /* Which auxiliary state is loaded (KVM_LARCH_*) */
205 unsigned int aux_inuse;
206 unsigned int aux_ldtype;
207
208 /* FPU state */
209 struct loongarch_fpu fpu FPU_ALIGN;
210 struct loongarch_lbt lbt;
211
212 /* CSR state */
213 struct loongarch_csrs *csr;
214
215 /* Guest max PMU CSR id */
216 int max_pmu_csrid;
217
218 /* GPR used as IO source/target */
219 u32 io_gpr;
220
221 /* KVM register to control count timer */
222 u32 count_ctl;
223 struct hrtimer swtimer;
224
225 /* Bitmask of intr that are pending */
226 unsigned long irq_pending;
227 /* Bitmask of pending intr to be cleared */
228 unsigned long irq_clear;
229
230 /* Bitmask of exceptions that are pending */
231 unsigned long exception_pending;
232 unsigned int esubcode;
233
234 /* Cache for pages needed inside spinlock regions */
235 struct kvm_mmu_memory_cache mmu_page_cache;
236
237 /* vcpu's vpid */
238 u64 vpid;
239 gpa_t flush_gpa;
240
241 /* Frequency of stable timer in Hz */
242 u64 timer_mhz;
243 ktime_t expire;
244
245 /* Last CPU the vCPU state was loaded on */
246 int last_sched_cpu;
247 /* mp state */
248 struct kvm_mp_state mp_state;
249 /* ipi state */
250 struct ipi_state ipi_state;
251 struct dmsintc_state dmsintc_state;
252 /* cpucfg */
253 u32 cpucfg[KVM_MAX_CPUCFG_REGS];
254
255 /* paravirt steal time */
256 struct {
257 u64 guest_addr;
258 u64 last_steal;
259 struct gfn_to_hva_cache cache;
260 u8 preempted;
261 } st;
262 };
263
readl_sw_gcsr(struct loongarch_csrs * csr,int reg)264 static inline unsigned long readl_sw_gcsr(struct loongarch_csrs *csr, int reg)
265 {
266 return csr->csrs[reg];
267 }
268
writel_sw_gcsr(struct loongarch_csrs * csr,int reg,unsigned long val)269 static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, unsigned long val)
270 {
271 csr->csrs[reg] = val;
272 }
273
kvm_guest_has_msgint(struct kvm_vcpu_arch * arch)274 static inline bool kvm_guest_has_msgint(struct kvm_vcpu_arch *arch)
275 {
276 return arch->cpucfg[1] & CPUCFG1_MSGINT;
277 }
278
kvm_guest_has_fpu(struct kvm_vcpu_arch * arch)279 static inline bool kvm_guest_has_fpu(struct kvm_vcpu_arch *arch)
280 {
281 return arch->cpucfg[2] & CPUCFG2_FP;
282 }
283
kvm_guest_has_lsx(struct kvm_vcpu_arch * arch)284 static inline bool kvm_guest_has_lsx(struct kvm_vcpu_arch *arch)
285 {
286 return arch->cpucfg[2] & CPUCFG2_LSX;
287 }
288
kvm_guest_has_lasx(struct kvm_vcpu_arch * arch)289 static inline bool kvm_guest_has_lasx(struct kvm_vcpu_arch *arch)
290 {
291 return arch->cpucfg[2] & CPUCFG2_LASX;
292 }
293
kvm_guest_has_lbt(struct kvm_vcpu_arch * arch)294 static inline bool kvm_guest_has_lbt(struct kvm_vcpu_arch *arch)
295 {
296 return arch->cpucfg[2] & (CPUCFG2_X86BT | CPUCFG2_ARMBT | CPUCFG2_MIPSBT);
297 }
298
kvm_guest_has_pmu(struct kvm_vcpu_arch * arch)299 static inline bool kvm_guest_has_pmu(struct kvm_vcpu_arch *arch)
300 {
301 return arch->cpucfg[6] & CPUCFG6_PMP;
302 }
303
kvm_get_pmu_num(struct kvm_vcpu_arch * arch)304 static inline int kvm_get_pmu_num(struct kvm_vcpu_arch *arch)
305 {
306 return (arch->cpucfg[6] & CPUCFG6_PMNUM) >> CPUCFG6_PMNUM_SHIFT;
307 }
308
309 /* Check whether KVM support this feature (VMM may disable it) */
kvm_vm_support(struct kvm_arch * arch,int feature)310 static inline bool kvm_vm_support(struct kvm_arch *arch, int feature)
311 {
312 return !!(arch->kvm_features & BIT_ULL(feature));
313 }
314
315 bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu);
316
317 /* Debug: dump vcpu state */
318 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
319
320 /* MMU handling */
321 void kvm_flush_tlb_all(void);
322 void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa);
323 int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write, int ecode);
324
325 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable);
326 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
327 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
328
update_pc(struct kvm_vcpu_arch * arch)329 static inline void update_pc(struct kvm_vcpu_arch *arch)
330 {
331 arch->pc += 4;
332 }
333
334 /*
335 * kvm_is_ifetch_fault() - Find whether a TLBL exception is due to ifetch fault.
336 * @vcpu: Virtual CPU.
337 *
338 * Returns: Whether the TLBL exception was likely due to an instruction
339 * fetch fault rather than a data load fault.
340 */
kvm_is_ifetch_fault(struct kvm_vcpu_arch * arch)341 static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *arch)
342 {
343 return arch->pc == arch->badv;
344 }
345
346 /* Misc */
kvm_arch_hardware_unsetup(void)347 static inline void kvm_arch_hardware_unsetup(void) {}
kvm_arch_memslots_updated(struct kvm * kvm,u64 gen)348 static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)349 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)350 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
kvm_arch_vcpu_block_finish(struct kvm_vcpu * vcpu)351 static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
kvm_arch_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)352 static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {}
353 void kvm_check_vpid(struct kvm_vcpu *vcpu);
354 enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer);
355 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, const struct kvm_memory_slot *memslot);
356 void kvm_init_vmcs(struct kvm *kvm);
357 void kvm_exc_entry(void);
358 int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
359
360 extern unsigned long vpid_mask;
361 extern struct kvm_world_switch *kvm_loongarch_ops;
362
363 #define SW_GCSR (1 << 0)
364 #define HW_GCSR (1 << 1)
365 #define INVALID_GCSR (1 << 2)
366
367 int get_gcsr_flag(int csr);
368 void set_hw_gcsr(int csr_id, unsigned long val);
369
370 #endif /* __ASM_LOONGARCH_KVM_HOST_H__ */
371