1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2024 Loongson Technology Corporation Limited
4 */
5
6 #include <linux/kvm_host.h>
7 #include <asm/kvm_ipi.h>
8 #include <asm/kvm_vcpu.h>
9
ipi_set(struct kvm_vcpu * vcpu,uint32_t data)10 static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data)
11 {
12 uint32_t status;
13 struct kvm_interrupt irq;
14
15 spin_lock(&vcpu->arch.ipi_state.lock);
16 status = vcpu->arch.ipi_state.status;
17 vcpu->arch.ipi_state.status |= data;
18 spin_unlock(&vcpu->arch.ipi_state.lock);
19 if ((status == 0) && data) {
20 irq.irq = LARCH_INT_IPI;
21 kvm_vcpu_ioctl_interrupt(vcpu, &irq);
22 }
23 }
24
ipi_send(struct kvm * kvm,uint64_t data)25 static void ipi_send(struct kvm *kvm, uint64_t data)
26 {
27 int cpu;
28 struct kvm_vcpu *vcpu;
29
30 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
31 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
32 if (unlikely(vcpu == NULL)) {
33 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
34 return;
35 }
36
37 ipi_set(vcpu, BIT(data & 0x1f));
38 }
39
ipi_clear(struct kvm_vcpu * vcpu,uint64_t data)40 static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data)
41 {
42 uint32_t status;
43 struct kvm_interrupt irq;
44
45 spin_lock(&vcpu->arch.ipi_state.lock);
46 vcpu->arch.ipi_state.status &= ~data;
47 status = vcpu->arch.ipi_state.status;
48 spin_unlock(&vcpu->arch.ipi_state.lock);
49 if (status == 0) {
50 irq.irq = -LARCH_INT_IPI;
51 kvm_vcpu_ioctl_interrupt(vcpu, &irq);
52 }
53 }
54
read_mailbox(struct kvm_vcpu * vcpu,int offset,int len)55 static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len)
56 {
57 uint64_t data = 0;
58
59 spin_lock(&vcpu->arch.ipi_state.lock);
60 data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20));
61 spin_unlock(&vcpu->arch.ipi_state.lock);
62
63 switch (len) {
64 case 1:
65 return data & 0xff;
66 case 2:
67 return data & 0xffff;
68 case 4:
69 return data & 0xffffffff;
70 case 8:
71 return data;
72 default:
73 kvm_err("%s: unknown data len: %d\n", __func__, len);
74 return 0;
75 }
76 }
77
write_mailbox(struct kvm_vcpu * vcpu,int offset,uint64_t data,int len)78 static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len)
79 {
80 void *pbuf;
81
82 spin_lock(&vcpu->arch.ipi_state.lock);
83 pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20);
84
85 switch (len) {
86 case 1:
87 *(unsigned char *)pbuf = (unsigned char)data;
88 break;
89 case 2:
90 *(unsigned short *)pbuf = (unsigned short)data;
91 break;
92 case 4:
93 *(unsigned int *)pbuf = (unsigned int)data;
94 break;
95 case 8:
96 *(unsigned long *)pbuf = (unsigned long)data;
97 break;
98 default:
99 kvm_err("%s: unknown data len: %d\n", __func__, len);
100 }
101 spin_unlock(&vcpu->arch.ipi_state.lock);
102 }
103
mail_send(struct kvm * kvm,uint64_t data)104 static int mail_send(struct kvm *kvm, uint64_t data)
105 {
106 int i, cpu, mailbox, offset;
107 uint32_t val = 0, mask = 0;
108 struct kvm_vcpu *vcpu;
109
110 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
111 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
112 if (unlikely(vcpu == NULL)) {
113 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
114 return 0;
115 }
116 mailbox = ((data & 0xffffffff) >> 2) & 0x7;
117 offset = IOCSR_IPI_BUF_20 + mailbox * 4;
118 if ((data >> 27) & 0xf) {
119 val = read_mailbox(vcpu, offset, 4);
120 for (i = 0; i < 4; i++)
121 if (data & (BIT(27 + i)))
122 mask |= (0xff << (i * 8));
123 val &= mask;
124 }
125
126 val |= ((uint32_t)(data >> 32) & ~mask);
127 write_mailbox(vcpu, offset, val, 4);
128
129 return 0;
130 }
131
send_ipi_data(struct kvm_vcpu * vcpu,gpa_t addr,uint64_t data)132 static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data)
133 {
134 int i, idx, ret;
135 uint64_t val = 0, mask = 0;
136
137 /*
138 * Bit 27-30 is mask for byte writing.
139 * If the mask is 0, we need not to do anything.
140 */
141 if ((data >> 27) & 0xf) {
142 /* Read the old val */
143 idx = srcu_read_lock(&vcpu->kvm->srcu);
144 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
145 srcu_read_unlock(&vcpu->kvm->srcu, idx);
146 if (unlikely(ret)) {
147 kvm_err("%s: : read data from addr %llx failed\n", __func__, addr);
148 return 0;
149 }
150 /* Construct the mask by scanning the bit 27-30 */
151 for (i = 0; i < 4; i++) {
152 if (data & (BIT(27 + i)))
153 mask |= (0xff << (i * 8));
154 }
155 /* Save the old part of val */
156 val &= mask;
157 }
158 val |= ((uint32_t)(data >> 32) & ~mask);
159 idx = srcu_read_lock(&vcpu->kvm->srcu);
160 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
161 srcu_read_unlock(&vcpu->kvm->srcu, idx);
162 if (unlikely(ret))
163 kvm_err("%s: : write data to addr %llx failed\n", __func__, addr);
164
165 return 0;
166 }
167
any_send(struct kvm * kvm,uint64_t data)168 static int any_send(struct kvm *kvm, uint64_t data)
169 {
170 int cpu, offset;
171 struct kvm_vcpu *vcpu;
172
173 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
174 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
175 if (unlikely(vcpu == NULL)) {
176 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
177 return 0;
178 }
179 offset = data & 0xffff;
180
181 return send_ipi_data(vcpu, offset, data);
182 }
183
loongarch_ipi_readl(struct kvm_vcpu * vcpu,gpa_t addr,int len,void * val)184 static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val)
185 {
186 uint32_t offset;
187 uint64_t res = 0;
188
189 offset = (uint32_t)(addr & 0x1ff);
190 WARN_ON_ONCE(offset & (len - 1));
191
192 switch (offset) {
193 case IOCSR_IPI_STATUS:
194 spin_lock(&vcpu->arch.ipi_state.lock);
195 res = vcpu->arch.ipi_state.status;
196 spin_unlock(&vcpu->arch.ipi_state.lock);
197 break;
198 case IOCSR_IPI_EN:
199 spin_lock(&vcpu->arch.ipi_state.lock);
200 res = vcpu->arch.ipi_state.en;
201 spin_unlock(&vcpu->arch.ipi_state.lock);
202 break;
203 case IOCSR_IPI_SET:
204 case IOCSR_IPI_CLEAR:
205 break;
206 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
207 if (offset + len > IOCSR_IPI_BUF_38 + 8) {
208 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
209 __func__, offset, len);
210 break;
211 }
212 res = read_mailbox(vcpu, offset, len);
213 break;
214 default:
215 kvm_err("%s: unknown addr: %llx\n", __func__, addr);
216 break;
217 }
218 *(uint64_t *)val = res;
219
220 return 0;
221 }
222
loongarch_ipi_writel(struct kvm_vcpu * vcpu,gpa_t addr,int len,const void * val)223 static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val)
224 {
225 uint64_t data;
226 uint32_t offset;
227
228 data = *(uint64_t *)val;
229
230 offset = (uint32_t)(addr & 0x1ff);
231 WARN_ON_ONCE(offset & (len - 1));
232
233 switch (offset) {
234 case IOCSR_IPI_STATUS:
235 break;
236 case IOCSR_IPI_EN:
237 spin_lock(&vcpu->arch.ipi_state.lock);
238 vcpu->arch.ipi_state.en = data;
239 spin_unlock(&vcpu->arch.ipi_state.lock);
240 break;
241 case IOCSR_IPI_SET:
242 ipi_set(vcpu, data);
243 break;
244 case IOCSR_IPI_CLEAR:
245 /* Just clear the status of the current vcpu */
246 ipi_clear(vcpu, data);
247 break;
248 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
249 if (offset + len > IOCSR_IPI_BUF_38 + 8) {
250 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n",
251 __func__, offset, len);
252 break;
253 }
254 write_mailbox(vcpu, offset, data, len);
255 break;
256 case IOCSR_IPI_SEND:
257 ipi_send(vcpu->kvm, data);
258 break;
259 case IOCSR_MAIL_SEND:
260 mail_send(vcpu->kvm, data);
261 break;
262 case IOCSR_ANY_SEND:
263 any_send(vcpu->kvm, data);
264 break;
265 default:
266 kvm_err("%s: unknown addr: %llx\n", __func__, addr);
267 break;
268 }
269
270 return 0;
271 }
272
kvm_ipi_read(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,void * val)273 static int kvm_ipi_read(struct kvm_vcpu *vcpu,
274 struct kvm_io_device *dev,
275 gpa_t addr, int len, void *val)
276 {
277 vcpu->stat.ipi_read_exits++;
278 return loongarch_ipi_readl(vcpu, addr, len, val);
279 }
280
kvm_ipi_write(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,const void * val)281 static int kvm_ipi_write(struct kvm_vcpu *vcpu,
282 struct kvm_io_device *dev,
283 gpa_t addr, int len, const void *val)
284 {
285 vcpu->stat.ipi_write_exits++;
286 return loongarch_ipi_writel(vcpu, addr, len, val);
287 }
288
289 static const struct kvm_io_device_ops kvm_ipi_ops = {
290 .read = kvm_ipi_read,
291 .write = kvm_ipi_write,
292 };
293
kvm_ipi_regs_access(struct kvm_device * dev,struct kvm_device_attr * attr,bool is_write)294 static int kvm_ipi_regs_access(struct kvm_device *dev,
295 struct kvm_device_attr *attr,
296 bool is_write)
297 {
298 int len = 4;
299 int cpu, addr;
300 uint64_t val;
301 void *p = NULL;
302 struct kvm_vcpu *vcpu;
303
304 cpu = (attr->attr >> 16) & 0x3ff;
305 addr = attr->attr & 0xff;
306
307 vcpu = kvm_get_vcpu_by_id(dev->kvm, cpu);
308 if (unlikely(vcpu == NULL)) {
309 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu);
310 return -EINVAL;
311 }
312
313 switch (addr) {
314 case IOCSR_IPI_STATUS:
315 p = &vcpu->arch.ipi_state.status;
316 break;
317 case IOCSR_IPI_EN:
318 p = &vcpu->arch.ipi_state.en;
319 break;
320 case IOCSR_IPI_SET:
321 p = &vcpu->arch.ipi_state.set;
322 break;
323 case IOCSR_IPI_CLEAR:
324 p = &vcpu->arch.ipi_state.clear;
325 break;
326 case IOCSR_IPI_BUF_20:
327 p = &vcpu->arch.ipi_state.buf[0];
328 len = 8;
329 break;
330 case IOCSR_IPI_BUF_28:
331 p = &vcpu->arch.ipi_state.buf[1];
332 len = 8;
333 break;
334 case IOCSR_IPI_BUF_30:
335 p = &vcpu->arch.ipi_state.buf[2];
336 len = 8;
337 break;
338 case IOCSR_IPI_BUF_38:
339 p = &vcpu->arch.ipi_state.buf[3];
340 len = 8;
341 break;
342 default:
343 kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr);
344 return -EINVAL;
345 }
346
347 if (is_write) {
348 if (len == 4) {
349 if (get_user(val, (uint32_t __user *)attr->addr))
350 return -EFAULT;
351 *(uint32_t *)p = (uint32_t)val;
352 } else if (len == 8) {
353 if (get_user(val, (uint64_t __user *)attr->addr))
354 return -EFAULT;
355 *(uint64_t *)p = val;
356 }
357 } else {
358 if (len == 4) {
359 val = *(uint32_t *)p;
360 return put_user(val, (uint32_t __user *)attr->addr);
361 } else if (len == 8) {
362 val = *(uint64_t *)p;
363 return put_user(val, (uint64_t __user *)attr->addr);
364 }
365 }
366
367 return 0;
368 }
369
kvm_ipi_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)370 static int kvm_ipi_get_attr(struct kvm_device *dev,
371 struct kvm_device_attr *attr)
372 {
373 switch (attr->group) {
374 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
375 return kvm_ipi_regs_access(dev, attr, false);
376 default:
377 kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
378 return -EINVAL;
379 }
380 }
381
kvm_ipi_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)382 static int kvm_ipi_set_attr(struct kvm_device *dev,
383 struct kvm_device_attr *attr)
384 {
385 switch (attr->group) {
386 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
387 return kvm_ipi_regs_access(dev, attr, true);
388 default:
389 kvm_err("%s: unknown group (%d)\n", __func__, attr->group);
390 return -EINVAL;
391 }
392 }
393
kvm_ipi_create(struct kvm_device * dev,u32 type)394 static int kvm_ipi_create(struct kvm_device *dev, u32 type)
395 {
396 int ret;
397 struct kvm *kvm;
398 struct kvm_io_device *device;
399 struct loongarch_ipi *s;
400
401 if (!dev) {
402 kvm_err("%s: kvm_device ptr is invalid!\n", __func__);
403 return -EINVAL;
404 }
405
406 kvm = dev->kvm;
407 if (kvm->arch.ipi) {
408 kvm_err("%s: LoongArch IPI has already been created!\n", __func__);
409 return -EINVAL;
410 }
411
412 s = kzalloc_obj(struct loongarch_ipi);
413 if (!s)
414 return -ENOMEM;
415
416 spin_lock_init(&s->lock);
417 s->kvm = kvm;
418
419 /*
420 * Initialize IOCSR device
421 */
422 device = &s->device;
423 kvm_iodevice_init(device, &kvm_ipi_ops);
424 mutex_lock(&kvm->slots_lock);
425 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
426 mutex_unlock(&kvm->slots_lock);
427 if (ret < 0) {
428 kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret);
429 goto err;
430 }
431
432 kvm->arch.ipi = s;
433 return 0;
434
435 err:
436 kfree(s);
437 return -EFAULT;
438 }
439
kvm_ipi_destroy(struct kvm_device * dev)440 static void kvm_ipi_destroy(struct kvm_device *dev)
441 {
442 struct kvm *kvm;
443 struct loongarch_ipi *ipi;
444
445 if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
446 return;
447
448 kvm = dev->kvm;
449 ipi = kvm->arch.ipi;
450 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device);
451 kfree(ipi);
452 kfree(dev);
453 }
454
455 static struct kvm_device_ops kvm_ipi_dev_ops = {
456 .name = "kvm-loongarch-ipi",
457 .create = kvm_ipi_create,
458 .destroy = kvm_ipi_destroy,
459 .set_attr = kvm_ipi_set_attr,
460 .get_attr = kvm_ipi_get_attr,
461 };
462
kvm_loongarch_register_ipi_device(void)463 int kvm_loongarch_register_ipi_device(void)
464 {
465 return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI);
466 }
467