1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4 */
5
6 #include <linux/err.h>
7 #include <linux/module.h>
8 #include <linux/kvm_host.h>
9 #include <asm/cacheflush.h>
10 #include <asm/cpufeature.h>
11 #include <asm/kvm_csr.h>
12 #include <asm/kvm_eiointc.h>
13 #include <asm/kvm_pch_pic.h>
14 #include "trace.h"
15
16 unsigned long vpid_mask;
17 struct kvm_world_switch *kvm_loongarch_ops;
18 static int gcsr_flag[CSR_MAX_NUMS];
19 static struct kvm_context __percpu *vmcs;
20
get_gcsr_flag(int csr)21 int get_gcsr_flag(int csr)
22 {
23 if (csr < CSR_MAX_NUMS)
24 return gcsr_flag[csr];
25
26 return INVALID_GCSR;
27 }
28
set_gcsr_sw_flag(int csr)29 static inline void set_gcsr_sw_flag(int csr)
30 {
31 if (csr < CSR_MAX_NUMS)
32 gcsr_flag[csr] |= SW_GCSR;
33 }
34
set_gcsr_hw_flag(int csr)35 static inline void set_gcsr_hw_flag(int csr)
36 {
37 if (csr < CSR_MAX_NUMS)
38 gcsr_flag[csr] |= HW_GCSR;
39 }
40
41 /*
42 * The default value of gcsr_flag[CSR] is 0, and we use this
43 * function to set the flag to 1 (SW_GCSR) or 2 (HW_GCSR) if the
44 * gcsr is software or hardware. It will be used by get/set_gcsr,
45 * if gcsr_flag is HW we should use gcsrrd/gcsrwr to access it,
46 * else use software csr to emulate it.
47 */
kvm_init_gcsr_flag(void)48 static void kvm_init_gcsr_flag(void)
49 {
50 set_gcsr_hw_flag(LOONGARCH_CSR_CRMD);
51 set_gcsr_hw_flag(LOONGARCH_CSR_PRMD);
52 set_gcsr_hw_flag(LOONGARCH_CSR_EUEN);
53 set_gcsr_hw_flag(LOONGARCH_CSR_MISC);
54 set_gcsr_hw_flag(LOONGARCH_CSR_ECFG);
55 set_gcsr_hw_flag(LOONGARCH_CSR_ESTAT);
56 set_gcsr_hw_flag(LOONGARCH_CSR_ERA);
57 set_gcsr_hw_flag(LOONGARCH_CSR_BADV);
58 set_gcsr_hw_flag(LOONGARCH_CSR_BADI);
59 set_gcsr_hw_flag(LOONGARCH_CSR_EENTRY);
60 set_gcsr_hw_flag(LOONGARCH_CSR_TLBIDX);
61 set_gcsr_hw_flag(LOONGARCH_CSR_TLBEHI);
62 set_gcsr_hw_flag(LOONGARCH_CSR_TLBELO0);
63 set_gcsr_hw_flag(LOONGARCH_CSR_TLBELO1);
64 set_gcsr_hw_flag(LOONGARCH_CSR_ASID);
65 set_gcsr_hw_flag(LOONGARCH_CSR_PGDL);
66 set_gcsr_hw_flag(LOONGARCH_CSR_PGDH);
67 set_gcsr_hw_flag(LOONGARCH_CSR_PGD);
68 set_gcsr_hw_flag(LOONGARCH_CSR_PWCTL0);
69 set_gcsr_hw_flag(LOONGARCH_CSR_PWCTL1);
70 set_gcsr_hw_flag(LOONGARCH_CSR_STLBPGSIZE);
71 set_gcsr_hw_flag(LOONGARCH_CSR_RVACFG);
72 set_gcsr_hw_flag(LOONGARCH_CSR_CPUID);
73 set_gcsr_hw_flag(LOONGARCH_CSR_PRCFG1);
74 set_gcsr_hw_flag(LOONGARCH_CSR_PRCFG2);
75 set_gcsr_hw_flag(LOONGARCH_CSR_PRCFG3);
76 set_gcsr_hw_flag(LOONGARCH_CSR_KS0);
77 set_gcsr_hw_flag(LOONGARCH_CSR_KS1);
78 set_gcsr_hw_flag(LOONGARCH_CSR_KS2);
79 set_gcsr_hw_flag(LOONGARCH_CSR_KS3);
80 set_gcsr_hw_flag(LOONGARCH_CSR_KS4);
81 set_gcsr_hw_flag(LOONGARCH_CSR_KS5);
82 set_gcsr_hw_flag(LOONGARCH_CSR_KS6);
83 set_gcsr_hw_flag(LOONGARCH_CSR_KS7);
84 set_gcsr_hw_flag(LOONGARCH_CSR_TMID);
85 set_gcsr_hw_flag(LOONGARCH_CSR_TCFG);
86 set_gcsr_hw_flag(LOONGARCH_CSR_TVAL);
87 set_gcsr_hw_flag(LOONGARCH_CSR_TINTCLR);
88 set_gcsr_hw_flag(LOONGARCH_CSR_CNTC);
89 set_gcsr_hw_flag(LOONGARCH_CSR_LLBCTL);
90 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRENTRY);
91 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRBADV);
92 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRERA);
93 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRSAVE);
94 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRELO0);
95 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRELO1);
96 set_gcsr_hw_flag(LOONGARCH_CSR_TLBREHI);
97 set_gcsr_hw_flag(LOONGARCH_CSR_TLBRPRMD);
98 set_gcsr_hw_flag(LOONGARCH_CSR_DMWIN0);
99 set_gcsr_hw_flag(LOONGARCH_CSR_DMWIN1);
100 set_gcsr_hw_flag(LOONGARCH_CSR_DMWIN2);
101 set_gcsr_hw_flag(LOONGARCH_CSR_DMWIN3);
102
103 set_gcsr_sw_flag(LOONGARCH_CSR_IMPCTL1);
104 set_gcsr_sw_flag(LOONGARCH_CSR_IMPCTL2);
105 set_gcsr_sw_flag(LOONGARCH_CSR_MERRCTL);
106 set_gcsr_sw_flag(LOONGARCH_CSR_MERRINFO1);
107 set_gcsr_sw_flag(LOONGARCH_CSR_MERRINFO2);
108 set_gcsr_sw_flag(LOONGARCH_CSR_MERRENTRY);
109 set_gcsr_sw_flag(LOONGARCH_CSR_MERRERA);
110 set_gcsr_sw_flag(LOONGARCH_CSR_MERRSAVE);
111 set_gcsr_sw_flag(LOONGARCH_CSR_CTAG);
112 set_gcsr_sw_flag(LOONGARCH_CSR_DEBUG);
113 set_gcsr_sw_flag(LOONGARCH_CSR_DERA);
114 set_gcsr_sw_flag(LOONGARCH_CSR_DESAVE);
115
116 set_gcsr_sw_flag(LOONGARCH_CSR_FWPC);
117 set_gcsr_sw_flag(LOONGARCH_CSR_FWPS);
118 set_gcsr_sw_flag(LOONGARCH_CSR_MWPC);
119 set_gcsr_sw_flag(LOONGARCH_CSR_MWPS);
120
121 set_gcsr_sw_flag(LOONGARCH_CSR_DB0ADDR);
122 set_gcsr_sw_flag(LOONGARCH_CSR_DB0MASK);
123 set_gcsr_sw_flag(LOONGARCH_CSR_DB0CTRL);
124 set_gcsr_sw_flag(LOONGARCH_CSR_DB0ASID);
125 set_gcsr_sw_flag(LOONGARCH_CSR_DB1ADDR);
126 set_gcsr_sw_flag(LOONGARCH_CSR_DB1MASK);
127 set_gcsr_sw_flag(LOONGARCH_CSR_DB1CTRL);
128 set_gcsr_sw_flag(LOONGARCH_CSR_DB1ASID);
129 set_gcsr_sw_flag(LOONGARCH_CSR_DB2ADDR);
130 set_gcsr_sw_flag(LOONGARCH_CSR_DB2MASK);
131 set_gcsr_sw_flag(LOONGARCH_CSR_DB2CTRL);
132 set_gcsr_sw_flag(LOONGARCH_CSR_DB2ASID);
133 set_gcsr_sw_flag(LOONGARCH_CSR_DB3ADDR);
134 set_gcsr_sw_flag(LOONGARCH_CSR_DB3MASK);
135 set_gcsr_sw_flag(LOONGARCH_CSR_DB3CTRL);
136 set_gcsr_sw_flag(LOONGARCH_CSR_DB3ASID);
137 set_gcsr_sw_flag(LOONGARCH_CSR_DB4ADDR);
138 set_gcsr_sw_flag(LOONGARCH_CSR_DB4MASK);
139 set_gcsr_sw_flag(LOONGARCH_CSR_DB4CTRL);
140 set_gcsr_sw_flag(LOONGARCH_CSR_DB4ASID);
141 set_gcsr_sw_flag(LOONGARCH_CSR_DB5ADDR);
142 set_gcsr_sw_flag(LOONGARCH_CSR_DB5MASK);
143 set_gcsr_sw_flag(LOONGARCH_CSR_DB5CTRL);
144 set_gcsr_sw_flag(LOONGARCH_CSR_DB5ASID);
145 set_gcsr_sw_flag(LOONGARCH_CSR_DB6ADDR);
146 set_gcsr_sw_flag(LOONGARCH_CSR_DB6MASK);
147 set_gcsr_sw_flag(LOONGARCH_CSR_DB6CTRL);
148 set_gcsr_sw_flag(LOONGARCH_CSR_DB6ASID);
149 set_gcsr_sw_flag(LOONGARCH_CSR_DB7ADDR);
150 set_gcsr_sw_flag(LOONGARCH_CSR_DB7MASK);
151 set_gcsr_sw_flag(LOONGARCH_CSR_DB7CTRL);
152 set_gcsr_sw_flag(LOONGARCH_CSR_DB7ASID);
153
154 set_gcsr_sw_flag(LOONGARCH_CSR_IB0ADDR);
155 set_gcsr_sw_flag(LOONGARCH_CSR_IB0MASK);
156 set_gcsr_sw_flag(LOONGARCH_CSR_IB0CTRL);
157 set_gcsr_sw_flag(LOONGARCH_CSR_IB0ASID);
158 set_gcsr_sw_flag(LOONGARCH_CSR_IB1ADDR);
159 set_gcsr_sw_flag(LOONGARCH_CSR_IB1MASK);
160 set_gcsr_sw_flag(LOONGARCH_CSR_IB1CTRL);
161 set_gcsr_sw_flag(LOONGARCH_CSR_IB1ASID);
162 set_gcsr_sw_flag(LOONGARCH_CSR_IB2ADDR);
163 set_gcsr_sw_flag(LOONGARCH_CSR_IB2MASK);
164 set_gcsr_sw_flag(LOONGARCH_CSR_IB2CTRL);
165 set_gcsr_sw_flag(LOONGARCH_CSR_IB2ASID);
166 set_gcsr_sw_flag(LOONGARCH_CSR_IB3ADDR);
167 set_gcsr_sw_flag(LOONGARCH_CSR_IB3MASK);
168 set_gcsr_sw_flag(LOONGARCH_CSR_IB3CTRL);
169 set_gcsr_sw_flag(LOONGARCH_CSR_IB3ASID);
170 set_gcsr_sw_flag(LOONGARCH_CSR_IB4ADDR);
171 set_gcsr_sw_flag(LOONGARCH_CSR_IB4MASK);
172 set_gcsr_sw_flag(LOONGARCH_CSR_IB4CTRL);
173 set_gcsr_sw_flag(LOONGARCH_CSR_IB4ASID);
174 set_gcsr_sw_flag(LOONGARCH_CSR_IB5ADDR);
175 set_gcsr_sw_flag(LOONGARCH_CSR_IB5MASK);
176 set_gcsr_sw_flag(LOONGARCH_CSR_IB5CTRL);
177 set_gcsr_sw_flag(LOONGARCH_CSR_IB5ASID);
178 set_gcsr_sw_flag(LOONGARCH_CSR_IB6ADDR);
179 set_gcsr_sw_flag(LOONGARCH_CSR_IB6MASK);
180 set_gcsr_sw_flag(LOONGARCH_CSR_IB6CTRL);
181 set_gcsr_sw_flag(LOONGARCH_CSR_IB6ASID);
182 set_gcsr_sw_flag(LOONGARCH_CSR_IB7ADDR);
183 set_gcsr_sw_flag(LOONGARCH_CSR_IB7MASK);
184 set_gcsr_sw_flag(LOONGARCH_CSR_IB7CTRL);
185 set_gcsr_sw_flag(LOONGARCH_CSR_IB7ASID);
186
187 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCTRL0);
188 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR0);
189 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCTRL1);
190 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR1);
191 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCTRL2);
192 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR2);
193 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCTRL3);
194 set_gcsr_sw_flag(LOONGARCH_CSR_PERFCNTR3);
195
196 if (cpu_has_msgint) {
197 set_gcsr_hw_flag(LOONGARCH_CSR_IPR);
198 set_gcsr_hw_flag(LOONGARCH_CSR_ISR0);
199 set_gcsr_hw_flag(LOONGARCH_CSR_ISR1);
200 set_gcsr_hw_flag(LOONGARCH_CSR_ISR2);
201 set_gcsr_hw_flag(LOONGARCH_CSR_ISR3);
202 }
203 }
204
kvm_update_vpid(struct kvm_vcpu * vcpu,int cpu)205 static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu)
206 {
207 unsigned long vpid;
208 struct kvm_context *context;
209
210 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
211 vpid = context->vpid_cache + 1;
212 if (!(vpid & vpid_mask)) {
213 /* finish round of vpid loop */
214 if (unlikely(!vpid))
215 vpid = vpid_mask + 1;
216
217 ++vpid; /* vpid 0 reserved for root */
218
219 /* start new vpid cycle */
220 kvm_flush_tlb_all();
221 }
222
223 context->vpid_cache = vpid;
224 vcpu->arch.vpid = vpid;
225 }
226
kvm_check_vpid(struct kvm_vcpu * vcpu)227 void kvm_check_vpid(struct kvm_vcpu *vcpu)
228 {
229 int cpu;
230 bool migrated;
231 unsigned long ver, old, vpid;
232 struct kvm_context *context;
233
234 cpu = smp_processor_id();
235 /*
236 * Are we entering guest context on a different CPU to last time?
237 * If so, the vCPU's guest TLB state on this CPU may be stale.
238 */
239 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
240 migrated = (vcpu->cpu != cpu);
241
242 /*
243 * Check if our vpid is of an older version
244 *
245 * We also discard the stored vpid if we've executed on
246 * another CPU, as the guest mappings may have changed without
247 * hypervisor knowledge.
248 */
249 ver = vcpu->arch.vpid & ~vpid_mask;
250 old = context->vpid_cache & ~vpid_mask;
251 if (migrated || (ver != old)) {
252 kvm_update_vpid(vcpu, cpu);
253 trace_kvm_vpid_change(vcpu, vcpu->arch.vpid);
254 vcpu->cpu = cpu;
255 kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu);
256
257 /*
258 * LLBCTL is a separated guest CSR register from host, a general
259 * exception ERET instruction clears the host LLBCTL register in
260 * host mode, and clears the guest LLBCTL register in guest mode.
261 * ERET in tlb refill exception does not clear LLBCTL register.
262 *
263 * When secondary mmu mapping is changed, guest OS does not know
264 * even if the content is changed after mapping is changed.
265 *
266 * Here clear WCLLB of the guest LLBCTL register when mapping is
267 * changed. Otherwise, if mmu mapping is changed while guest is
268 * executing LL/SC pair, LL loads with the old address and set
269 * the LLBCTL flag, SC checks the LLBCTL flag and will store the
270 * new address successfully since LLBCTL_WCLLB is on, even if
271 * memory with new address is changed on other VCPUs.
272 */
273 set_gcsr_llbctl(CSR_LLBCTL_WCLLB);
274
275 /* Restore GSTAT(0x50).vpid */
276 vpid = (vcpu->arch.vpid & vpid_mask) << CSR_GSTAT_GID_SHIFT;
277 change_csr_gstat(vpid_mask << CSR_GSTAT_GID_SHIFT, vpid);
278 }
279 }
280
kvm_init_vmcs(struct kvm * kvm)281 void kvm_init_vmcs(struct kvm *kvm)
282 {
283 kvm->arch.vmcs = vmcs;
284 }
285
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)286 long kvm_arch_dev_ioctl(struct file *filp,
287 unsigned int ioctl, unsigned long arg)
288 {
289 return -ENOIOCTLCMD;
290 }
291
kvm_arch_enable_virtualization_cpu(void)292 int kvm_arch_enable_virtualization_cpu(void)
293 {
294 unsigned long env, gcfg = 0;
295
296 env = read_csr_gcfg();
297
298 /* First init gcfg, gstat, gintc, gtlbc. All guest use the same config */
299 write_csr_gcfg(0);
300 write_csr_gstat(0);
301 write_csr_gintc(0);
302 clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI);
303
304 /*
305 * Enable virtualization features granting guest direct control of
306 * certain features:
307 * GCI=2: Trap on init or unimplemented cache instruction.
308 * TORU=0: Trap on Root Unimplement.
309 * CACTRL=1: Root control cache.
310 * TOP=0: Trap on Privilege.
311 * TOE=0: Trap on Exception.
312 * TIT=0: Trap on Timer.
313 */
314 if (env & CSR_GCFG_GCIP_SECURE)
315 gcfg |= CSR_GCFG_GCI_SECURE;
316 if (env & CSR_GCFG_MATP_ROOT)
317 gcfg |= CSR_GCFG_MATC_ROOT;
318
319 write_csr_gcfg(gcfg);
320
321 kvm_flush_tlb_all();
322
323 /* Enable using TGID */
324 set_csr_gtlbc(CSR_GTLBC_USETGID);
325 kvm_debug("GCFG:%lx GSTAT:%lx GINTC:%lx GTLBC:%lx",
326 read_csr_gcfg(), read_csr_gstat(), read_csr_gintc(), read_csr_gtlbc());
327
328 /*
329 * HW Guest CSR registers are lost after CPU suspend and resume.
330 * Clear last_vcpu so that Guest CSR registers forced to reload
331 * from vCPU SW state.
332 */
333 this_cpu_ptr(vmcs)->last_vcpu = NULL;
334
335 return 0;
336 }
337
kvm_arch_disable_virtualization_cpu(void)338 void kvm_arch_disable_virtualization_cpu(void)
339 {
340 write_csr_gcfg(0);
341 write_csr_gstat(0);
342 write_csr_gintc(0);
343 clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI);
344
345 /* Flush any remaining guest TLB entries */
346 kvm_flush_tlb_all();
347 }
348
kvm_loongarch_env_init(void)349 static int kvm_loongarch_env_init(void)
350 {
351 int cpu, ret;
352 struct kvm_context *context;
353
354 vmcs = alloc_percpu(struct kvm_context);
355 if (!vmcs) {
356 pr_err("kvm: failed to allocate percpu kvm_context\n");
357 return -ENOMEM;
358 }
359
360 kvm_loongarch_ops = kzalloc_obj(*kvm_loongarch_ops);
361 if (!kvm_loongarch_ops) {
362 free_percpu(vmcs);
363 vmcs = NULL;
364 return -ENOMEM;
365 }
366
367 kvm_loongarch_ops->exc_entry = (void *)kvm_exc_entry;
368 kvm_loongarch_ops->enter_guest = (void *)kvm_enter_guest;
369
370 vpid_mask = read_csr_gstat();
371 vpid_mask = (vpid_mask & CSR_GSTAT_GIDBIT) >> CSR_GSTAT_GIDBIT_SHIFT;
372 if (vpid_mask)
373 vpid_mask = GENMASK(vpid_mask - 1, 0);
374
375 for_each_possible_cpu(cpu) {
376 context = per_cpu_ptr(vmcs, cpu);
377 context->vpid_cache = vpid_mask + 1;
378 context->last_vcpu = NULL;
379 }
380
381 kvm_init_gcsr_flag();
382 kvm_register_perf_callbacks();
383
384 /* Register LoongArch IPI interrupt controller interface. */
385 ret = kvm_loongarch_register_ipi_device();
386 if (ret)
387 return ret;
388
389 /* Register LoongArch EIOINTC interrupt controller interface. */
390 ret = kvm_loongarch_register_eiointc_device();
391 if (ret)
392 return ret;
393
394 /* Register LoongArch PCH-PIC interrupt controller interface. */
395 ret = kvm_loongarch_register_pch_pic_device();
396 if (ret)
397 return ret;
398
399 /* Register LoongArch DMSINTC interrupt contrroller interface */
400 if (cpu_has_msgint)
401 ret = kvm_loongarch_register_dmsintc_device();
402
403 return ret;
404 }
405
kvm_loongarch_env_exit(void)406 static void kvm_loongarch_env_exit(void)
407 {
408 if (vmcs)
409 free_percpu(vmcs);
410
411 if (kvm_loongarch_ops) {
412 kfree(kvm_loongarch_ops);
413 }
414
415 kvm_unregister_perf_callbacks();
416 }
417
kvm_loongarch_init(void)418 static int kvm_loongarch_init(void)
419 {
420 int r;
421
422 if (!cpu_has_lvz) {
423 kvm_info("Hardware virtualization not available\n");
424 return -ENODEV;
425 }
426 r = kvm_loongarch_env_init();
427 if (r)
428 return r;
429
430 return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
431 }
432
kvm_loongarch_exit(void)433 static void kvm_loongarch_exit(void)
434 {
435 kvm_exit();
436 kvm_loongarch_env_exit();
437 }
438
439 module_init(kvm_loongarch_init);
440 module_exit(kvm_loongarch_exit);
441
442 #ifdef MODULE
443 static const struct cpu_feature kvm_feature[] = {
444 { .feature = cpu_feature(LOONGARCH_LVZ) },
445 {},
446 };
447 MODULE_DEVICE_TABLE(cpu, kvm_feature);
448 #endif
449