1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/types.h>
3 #include <linux/interrupt.h>
4 #include <linux/irq_work.h>
5 #include <linux/jump_label.h>
6 #include <linux/kvm_para.h>
7 #include <linux/reboot.h>
8 #include <linux/static_call.h>
9 #include <asm/paravirt.h>
10
11 static int has_steal_clock;
12 struct static_key paravirt_steal_enabled;
13 struct static_key paravirt_steal_rq_enabled;
14 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
15 DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key);
16
native_steal_clock(int cpu)17 static u64 native_steal_clock(int cpu)
18 {
19 return 0;
20 }
21
22 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
23
24 static bool steal_acc = true;
25
parse_no_stealacc(char * arg)26 static int __init parse_no_stealacc(char *arg)
27 {
28 steal_acc = false;
29 return 0;
30 }
31 early_param("no-steal-acc", parse_no_stealacc);
32
paravt_steal_clock(int cpu)33 static u64 paravt_steal_clock(int cpu)
34 {
35 int version;
36 u64 steal;
37 struct kvm_steal_time *src;
38
39 src = &per_cpu(steal_time, cpu);
40 do {
41
42 version = src->version;
43 virt_rmb(); /* Make sure that the version is read before the steal */
44 steal = src->steal;
45 virt_rmb(); /* Make sure that the steal is read before the next version */
46
47 } while ((version & 1) || (version != src->version));
48
49 return steal;
50 }
51
52 #ifdef CONFIG_SMP
53 static struct smp_ops native_ops;
54
pv_send_ipi_single(int cpu,unsigned int action)55 static void pv_send_ipi_single(int cpu, unsigned int action)
56 {
57 int min, old;
58 irq_cpustat_t *info = &per_cpu(irq_stat, cpu);
59
60 if (unlikely(action == ACTION_BOOT_CPU)) {
61 native_ops.send_ipi_single(cpu, action);
62 return;
63 }
64
65 old = atomic_fetch_or(BIT(action), &info->message);
66 if (old)
67 return;
68
69 min = cpu_logical_map(cpu);
70 kvm_hypercall3(KVM_HCALL_FUNC_IPI, 1, 0, min);
71 }
72
73 #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG)
74
pv_send_ipi_mask(const struct cpumask * mask,unsigned int action)75 static void pv_send_ipi_mask(const struct cpumask *mask, unsigned int action)
76 {
77 int i, cpu, min = 0, max = 0, old;
78 __uint128_t bitmap = 0;
79 irq_cpustat_t *info;
80
81 if (cpumask_empty(mask))
82 return;
83
84 if (unlikely(action == ACTION_BOOT_CPU)) {
85 native_ops.send_ipi_mask(mask, action);
86 return;
87 }
88
89 action = BIT(action);
90 for_each_cpu(i, mask) {
91 info = &per_cpu(irq_stat, i);
92 old = atomic_fetch_or(action, &info->message);
93 if (old)
94 continue;
95
96 cpu = cpu_logical_map(i);
97 if (!bitmap) {
98 min = max = cpu;
99 } else if (cpu < min && cpu > (max - KVM_IPI_CLUSTER_SIZE)) {
100 /* cpu < min, and bitmap still enough */
101 bitmap <<= min - cpu;
102 min = cpu;
103 } else if (cpu > min && cpu < (min + KVM_IPI_CLUSTER_SIZE)) {
104 /* cpu > min, and bitmap still enough */
105 max = cpu > max ? cpu : max;
106 } else {
107 /*
108 * With cpu, bitmap will exceed KVM_IPI_CLUSTER_SIZE,
109 * send IPI here directly and skip the remaining CPUs.
110 */
111 kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
112 (unsigned long)(bitmap >> BITS_PER_LONG), min);
113 min = max = cpu;
114 bitmap = 0;
115 }
116 __set_bit(cpu - min, (unsigned long *)&bitmap);
117 }
118
119 if (bitmap)
120 kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
121 (unsigned long)(bitmap >> BITS_PER_LONG), min);
122 }
123
pv_ipi_interrupt(int irq,void * dev)124 static irqreturn_t pv_ipi_interrupt(int irq, void *dev)
125 {
126 u32 action;
127 irq_cpustat_t *info;
128
129 /* Clear SWI interrupt */
130 clear_csr_estat(1 << INT_SWI0);
131 info = this_cpu_ptr(&irq_stat);
132 action = atomic_xchg(&info->message, 0);
133
134 if (action & SMP_RESCHEDULE) {
135 scheduler_ipi();
136 info->ipi_irqs[IPI_RESCHEDULE]++;
137 }
138
139 if (action & SMP_CALL_FUNCTION) {
140 generic_smp_call_function_interrupt();
141 info->ipi_irqs[IPI_CALL_FUNCTION]++;
142 }
143
144 if (action & SMP_IRQ_WORK) {
145 irq_work_run();
146 info->ipi_irqs[IPI_IRQ_WORK]++;
147 }
148
149 if (action & SMP_CLEAR_VECTOR) {
150 complete_irq_moving();
151 info->ipi_irqs[IPI_CLEAR_VECTOR]++;
152 }
153
154 return IRQ_HANDLED;
155 }
156
pv_init_ipi(void)157 static void pv_init_ipi(void)
158 {
159 int r, swi;
160
161 /* Init native ipi irq for ACTION_BOOT_CPU */
162 native_ops.init_ipi();
163 swi = get_percpu_irq(INT_SWI0);
164 if (swi < 0)
165 panic("SWI0 IRQ mapping failed\n");
166 irq_set_percpu_devid(swi);
167 r = request_percpu_irq(swi, pv_ipi_interrupt, "SWI0-IPI", &irq_stat);
168 if (r < 0)
169 panic("SWI0 IRQ request failed\n");
170 }
171 #endif
172
kvm_para_available(void)173 bool kvm_para_available(void)
174 {
175 int config;
176 static int hypervisor_type;
177
178 if (!cpu_has_hypervisor)
179 return false;
180
181 if (!hypervisor_type) {
182 config = read_cpucfg(CPUCFG_KVM_SIG);
183 if (!memcmp(&config, KVM_SIGNATURE, 4))
184 hypervisor_type = HYPERVISOR_KVM;
185 }
186
187 return hypervisor_type == HYPERVISOR_KVM;
188 }
189
kvm_arch_para_features(void)190 unsigned int kvm_arch_para_features(void)
191 {
192 static unsigned int feature;
193
194 if (!kvm_para_available())
195 return 0;
196
197 if (!feature)
198 feature = read_cpucfg(CPUCFG_KVM_FEATURE);
199
200 return feature;
201 }
202
pv_ipi_init(void)203 int __init pv_ipi_init(void)
204 {
205 if (!kvm_para_has_feature(KVM_FEATURE_IPI))
206 return 0;
207
208 #ifdef CONFIG_SMP
209 native_ops = mp_ops;
210 mp_ops.init_ipi = pv_init_ipi;
211 mp_ops.send_ipi_single = pv_send_ipi_single;
212 mp_ops.send_ipi_mask = pv_send_ipi_mask;
213 #endif
214
215 return 0;
216 }
217
pv_enable_steal_time(void)218 static int pv_enable_steal_time(void)
219 {
220 int cpu = smp_processor_id();
221 unsigned long addr;
222 struct kvm_steal_time *st;
223
224 if (!has_steal_clock)
225 return -EPERM;
226
227 st = &per_cpu(steal_time, cpu);
228 addr = per_cpu_ptr_to_phys(st);
229
230 /* The whole structure kvm_steal_time should be in one page */
231 if (PFN_DOWN(addr) != PFN_DOWN(addr + sizeof(*st))) {
232 pr_warn("Illegal PV steal time addr %lx\n", addr);
233 return -EFAULT;
234 }
235
236 addr |= KVM_STEAL_PHYS_VALID;
237 kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), addr);
238
239 return 0;
240 }
241
pv_disable_steal_time(void)242 static void pv_disable_steal_time(void)
243 {
244 if (has_steal_clock)
245 kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), 0);
246 }
247
248 #ifdef CONFIG_SMP
pv_time_cpu_online(unsigned int cpu)249 static int pv_time_cpu_online(unsigned int cpu)
250 {
251 unsigned long flags;
252
253 local_irq_save(flags);
254 pv_enable_steal_time();
255 local_irq_restore(flags);
256
257 return 0;
258 }
259
pv_time_cpu_down_prepare(unsigned int cpu)260 static int pv_time_cpu_down_prepare(unsigned int cpu)
261 {
262 unsigned long flags;
263
264 local_irq_save(flags);
265 pv_disable_steal_time();
266 local_irq_restore(flags);
267
268 return 0;
269 }
270 #endif
271
pv_cpu_reboot(void * unused)272 static void pv_cpu_reboot(void *unused)
273 {
274 pv_disable_steal_time();
275 }
276
pv_reboot_notify(struct notifier_block * nb,unsigned long code,void * unused)277 static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, void *unused)
278 {
279 on_each_cpu(pv_cpu_reboot, NULL, 1);
280 return NOTIFY_DONE;
281 }
282
283 static struct notifier_block pv_reboot_nb = {
284 .notifier_call = pv_reboot_notify,
285 };
286
pv_time_init(void)287 int __init pv_time_init(void)
288 {
289 int r;
290
291 if (!kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
292 return 0;
293
294 has_steal_clock = 1;
295 r = pv_enable_steal_time();
296 if (r < 0) {
297 has_steal_clock = 0;
298 return 0;
299 }
300 register_reboot_notifier(&pv_reboot_nb);
301
302 #ifdef CONFIG_SMP
303 r = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
304 "loongarch/pv_time:online",
305 pv_time_cpu_online, pv_time_cpu_down_prepare);
306 if (r < 0) {
307 has_steal_clock = 0;
308 pr_err("Failed to install cpu hotplug callbacks\n");
309 return r;
310 }
311 #endif
312
313 static_call_update(pv_steal_clock, paravt_steal_clock);
314
315 static_key_slow_inc(¶virt_steal_enabled);
316 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
317 if (steal_acc)
318 static_key_slow_inc(¶virt_steal_rq_enabled);
319 #endif
320
321 pr_info("Using paravirt steal-time\n");
322
323 return 0;
324 }
325
pv_spinlock_init(void)326 int __init pv_spinlock_init(void)
327 {
328 if (!cpu_has_hypervisor)
329 return 0;
330
331 static_branch_enable(&virt_spin_lock_key);
332
333 return 0;
334 }
335