1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2024 Loongson Technology Corporation Limited
4 */
5
6 #include <linux/kvm_host.h>
7 #include <asm/kvm_ipi.h>
8 #include <asm/kvm_vcpu.h>
9
ipi_set(struct kvm_vcpu * vcpu,uint32_t data)10 static void ipi_set(struct kvm_vcpu *vcpu, uint32_t data)
11 {
12 uint32_t status;
13
14 spin_lock(&vcpu->arch.ipi_state.lock);
15 status = vcpu->arch.ipi_state.status;
16 vcpu->arch.ipi_state.status |= data;
17 if ((status == 0) && data) {
18 kvm_queue_irq(vcpu, LARCH_INT_IPI);
19 kvm_vcpu_kick(vcpu);
20 }
21 spin_unlock(&vcpu->arch.ipi_state.lock);
22 }
23
ipi_send(struct kvm * kvm,uint64_t data)24 static void ipi_send(struct kvm *kvm, uint64_t data)
25 {
26 int cpu;
27 struct kvm_vcpu *vcpu;
28
29 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
30 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
31 if (unlikely(vcpu == NULL)) {
32 kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu);
33 return;
34 }
35
36 ipi_set(vcpu, BIT(data & 0x1f));
37 }
38
ipi_clear(struct kvm_vcpu * vcpu,uint64_t data)39 static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data)
40 {
41 uint32_t status;
42
43 spin_lock(&vcpu->arch.ipi_state.lock);
44 vcpu->arch.ipi_state.status &= ~data;
45 status = vcpu->arch.ipi_state.status;
46 if (status == 0) {
47 kvm_dequeue_irq(vcpu, LARCH_INT_IPI);
48 kvm_vcpu_kick(vcpu);
49 }
50 spin_unlock(&vcpu->arch.ipi_state.lock);
51 }
52
read_mailbox(struct kvm_vcpu * vcpu,int offset,int len)53 static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len)
54 {
55 uint64_t data = 0;
56
57 spin_lock(&vcpu->arch.ipi_state.lock);
58 data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20));
59 spin_unlock(&vcpu->arch.ipi_state.lock);
60
61 switch (len) {
62 case 1:
63 return data & 0xff;
64 case 2:
65 return data & 0xffff;
66 case 4:
67 return data & 0xffffffff;
68 default:
69 return data;
70 }
71 }
72
write_mailbox(struct kvm_vcpu * vcpu,int offset,uint64_t data,int len)73 static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len)
74 {
75 void *pbuf;
76
77 spin_lock(&vcpu->arch.ipi_state.lock);
78 pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20);
79
80 switch (len) {
81 case 1:
82 *(unsigned char *)pbuf = (unsigned char)data;
83 break;
84 case 2:
85 *(unsigned short *)pbuf = (unsigned short)data;
86 break;
87 case 4:
88 *(unsigned int *)pbuf = (unsigned int)data;
89 break;
90 default:
91 *(unsigned long *)pbuf = (unsigned long)data;
92 break;
93 }
94 spin_unlock(&vcpu->arch.ipi_state.lock);
95 }
96
mail_send(struct kvm * kvm,uint64_t data)97 static int mail_send(struct kvm *kvm, uint64_t data)
98 {
99 int i, cpu, mailbox, offset;
100 uint32_t val = 0, mask = 0;
101 struct kvm_vcpu *vcpu;
102
103 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
104 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
105 if (unlikely(vcpu == NULL)) {
106 kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu);
107 return 0;
108 }
109 mailbox = ((data & 0xffffffff) >> 2) & 0x7;
110 offset = IOCSR_IPI_BUF_20 + mailbox * 4;
111 if ((data >> 27) & 0xf) {
112 val = read_mailbox(vcpu, offset, 4);
113 for (i = 0; i < 4; i++)
114 if (data & (BIT(27 + i)))
115 mask |= (0xff << (i * 8));
116 val &= mask;
117 }
118
119 val |= ((uint32_t)(data >> 32) & ~mask);
120 write_mailbox(vcpu, offset, val, 4);
121
122 return 0;
123 }
124
send_ipi_data(struct kvm_vcpu * vcpu,gpa_t addr,uint64_t data)125 static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data)
126 {
127 int i, idx, ret;
128 uint64_t val = 0, mask = 0;
129
130 /*
131 * Bit 27-30 is mask for byte writing.
132 * If the mask is 0, we need not to do anything.
133 */
134 if ((data >> 27) & 0xf) {
135 /* Read the old val */
136 idx = srcu_read_lock(&vcpu->kvm->srcu);
137 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
138 srcu_read_unlock(&vcpu->kvm->srcu, idx);
139 if (unlikely(ret)) {
140 kvm_pr_unimpl("%s: : read data from addr %llx failed\n", __func__, addr);
141 return 0;
142 }
143 /* Construct the mask by scanning the bit 27-30 */
144 for (i = 0; i < 4; i++) {
145 if (data & (BIT(27 + i)))
146 mask |= (0xff << (i * 8));
147 }
148 /* Save the old part of val */
149 val &= mask;
150 }
151 val |= ((uint32_t)(data >> 32) & ~mask);
152 idx = srcu_read_lock(&vcpu->kvm->srcu);
153 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, 4, &val);
154 srcu_read_unlock(&vcpu->kvm->srcu, idx);
155 if (unlikely(ret))
156 kvm_pr_unimpl("%s: : write data to addr %llx failed\n", __func__, addr);
157
158 return 0;
159 }
160
any_send(struct kvm * kvm,uint64_t data)161 static int any_send(struct kvm *kvm, uint64_t data)
162 {
163 int cpu, offset;
164 struct kvm_vcpu *vcpu;
165
166 cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
167 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu);
168 if (unlikely(vcpu == NULL)) {
169 kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu);
170 return 0;
171 }
172 offset = data & 0xffff;
173
174 return send_ipi_data(vcpu, offset, data);
175 }
176
loongarch_ipi_readl(struct kvm_vcpu * vcpu,gpa_t addr,int len,void * val)177 static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val)
178 {
179 uint32_t offset;
180 uint64_t res = 0;
181
182 if (addr & (len - 1)) {
183 *(uint64_t *)val = res;
184 kvm_pr_unimpl("%s: ipi not aligned addr %llx len %d\n", __func__, addr, len);
185 return 0;
186 }
187
188 offset = addr - IOCSR_IPI_BASE;
189
190 switch (offset) {
191 case IOCSR_IPI_STATUS:
192 spin_lock(&vcpu->arch.ipi_state.lock);
193 res = vcpu->arch.ipi_state.status;
194 spin_unlock(&vcpu->arch.ipi_state.lock);
195 break;
196 case IOCSR_IPI_EN:
197 spin_lock(&vcpu->arch.ipi_state.lock);
198 res = vcpu->arch.ipi_state.en;
199 spin_unlock(&vcpu->arch.ipi_state.lock);
200 break;
201 case IOCSR_IPI_SET:
202 case IOCSR_IPI_CLEAR:
203 break;
204 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
205 res = read_mailbox(vcpu, offset, len);
206 break;
207 default:
208 kvm_pr_unimpl("%s: unknown addr: %llx\n", __func__, addr);
209 break;
210 }
211 *(uint64_t *)val = res;
212
213 return 0;
214 }
215
loongarch_ipi_writel(struct kvm_vcpu * vcpu,gpa_t addr,int len,const void * val)216 static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val)
217 {
218 uint64_t data;
219 uint32_t offset;
220
221 data = *(uint64_t *)val;
222
223 if (addr & (len - 1)) {
224 kvm_pr_unimpl("%s: ipi not aligned addr %llx len %d\n", __func__, addr, len);
225 return 0;
226 }
227
228 offset = addr - IOCSR_IPI_BASE;
229
230 switch (offset) {
231 case IOCSR_IPI_STATUS:
232 break;
233 case IOCSR_IPI_EN:
234 spin_lock(&vcpu->arch.ipi_state.lock);
235 vcpu->arch.ipi_state.en = data;
236 spin_unlock(&vcpu->arch.ipi_state.lock);
237 break;
238 case IOCSR_IPI_SET:
239 ipi_set(vcpu, data);
240 break;
241 case IOCSR_IPI_CLEAR:
242 /* Just clear the status of the current vcpu */
243 ipi_clear(vcpu, data);
244 break;
245 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7:
246 if (offset + len > IOCSR_IPI_BUF_38 + 8) {
247 kvm_pr_unimpl("%s: invalid offset or len: offset = %d, len = %d\n",
248 __func__, offset, len);
249 break;
250 }
251 write_mailbox(vcpu, offset, data, len);
252 break;
253 case IOCSR_IPI_SEND:
254 ipi_send(vcpu->kvm, data);
255 break;
256 case IOCSR_MAIL_SEND:
257 mail_send(vcpu->kvm, data);
258 break;
259 case IOCSR_ANY_SEND:
260 any_send(vcpu->kvm, data);
261 break;
262 default:
263 kvm_pr_unimpl("%s: unknown addr: %llx\n", __func__, addr);
264 break;
265 }
266
267 return 0;
268 }
269
kvm_ipi_read(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,void * val)270 static int kvm_ipi_read(struct kvm_vcpu *vcpu,
271 struct kvm_io_device *dev,
272 gpa_t addr, int len, void *val)
273 {
274 vcpu->stat.ipi_read_exits++;
275 return loongarch_ipi_readl(vcpu, addr, len, val);
276 }
277
kvm_ipi_write(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,const void * val)278 static int kvm_ipi_write(struct kvm_vcpu *vcpu,
279 struct kvm_io_device *dev,
280 gpa_t addr, int len, const void *val)
281 {
282 vcpu->stat.ipi_write_exits++;
283 return loongarch_ipi_writel(vcpu, addr, len, val);
284 }
285
286 static const struct kvm_io_device_ops kvm_ipi_ops = {
287 .read = kvm_ipi_read,
288 .write = kvm_ipi_write,
289 };
290
kvm_ipi_regs_access(struct kvm_device * dev,struct kvm_device_attr * attr,bool is_write)291 static int kvm_ipi_regs_access(struct kvm_device *dev,
292 struct kvm_device_attr *attr,
293 bool is_write)
294 {
295 int len = 4;
296 int cpu, addr;
297 uint64_t val;
298 void *p = NULL;
299 struct kvm_vcpu *vcpu;
300
301 cpu = (attr->attr >> 16) & 0x3ff;
302 addr = attr->attr & 0xff;
303
304 vcpu = kvm_get_vcpu_by_id(dev->kvm, cpu);
305 if (unlikely(vcpu == NULL)) {
306 kvm_pr_unimpl("%s: invalid target cpu: %d\n", __func__, cpu);
307 return -EINVAL;
308 }
309
310 switch (addr) {
311 case IOCSR_IPI_STATUS:
312 p = &vcpu->arch.ipi_state.status;
313 break;
314 case IOCSR_IPI_EN:
315 p = &vcpu->arch.ipi_state.en;
316 break;
317 case IOCSR_IPI_SET:
318 p = &vcpu->arch.ipi_state.set;
319 break;
320 case IOCSR_IPI_CLEAR:
321 p = &vcpu->arch.ipi_state.clear;
322 break;
323 case IOCSR_IPI_BUF_20:
324 p = &vcpu->arch.ipi_state.buf[0];
325 len = 8;
326 break;
327 case IOCSR_IPI_BUF_28:
328 p = &vcpu->arch.ipi_state.buf[1];
329 len = 8;
330 break;
331 case IOCSR_IPI_BUF_30:
332 p = &vcpu->arch.ipi_state.buf[2];
333 len = 8;
334 break;
335 case IOCSR_IPI_BUF_38:
336 p = &vcpu->arch.ipi_state.buf[3];
337 len = 8;
338 break;
339 default:
340 kvm_pr_unimpl("%s: unknown ipi register, addr = %d\n", __func__, addr);
341 return -EINVAL;
342 }
343
344 if (is_write) {
345 if (len == 4) {
346 if (get_user(val, (uint32_t __user *)attr->addr))
347 return -EFAULT;
348 *(uint32_t *)p = (uint32_t)val;
349 } else if (len == 8) {
350 if (get_user(val, (uint64_t __user *)attr->addr))
351 return -EFAULT;
352 *(uint64_t *)p = val;
353 }
354 } else {
355 if (len == 4) {
356 val = *(uint32_t *)p;
357 return put_user(val, (uint32_t __user *)attr->addr);
358 } else if (len == 8) {
359 val = *(uint64_t *)p;
360 return put_user(val, (uint64_t __user *)attr->addr);
361 }
362 }
363
364 return 0;
365 }
366
kvm_ipi_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)367 static int kvm_ipi_get_attr(struct kvm_device *dev,
368 struct kvm_device_attr *attr)
369 {
370 switch (attr->group) {
371 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
372 return kvm_ipi_regs_access(dev, attr, false);
373 default:
374 kvm_pr_unimpl("%s: unknown group (%d)\n", __func__, attr->group);
375 return -EINVAL;
376 }
377 }
378
kvm_ipi_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)379 static int kvm_ipi_set_attr(struct kvm_device *dev,
380 struct kvm_device_attr *attr)
381 {
382 switch (attr->group) {
383 case KVM_DEV_LOONGARCH_IPI_GRP_REGS:
384 return kvm_ipi_regs_access(dev, attr, true);
385 default:
386 kvm_pr_unimpl("%s: unknown group (%d)\n", __func__, attr->group);
387 return -EINVAL;
388 }
389 }
390
kvm_ipi_create(struct kvm_device * dev,u32 type)391 static int kvm_ipi_create(struct kvm_device *dev, u32 type)
392 {
393 int ret;
394 struct kvm *kvm;
395 struct kvm_io_device *device;
396 struct loongarch_ipi *s;
397
398 if (!dev) {
399 kvm_pr_unimpl("%s: kvm_device ptr is invalid!\n", __func__);
400 return -EINVAL;
401 }
402
403 kvm = dev->kvm;
404 if (kvm->arch.ipi) {
405 kvm_pr_unimpl("%s: LoongArch IPI has already been created!\n", __func__);
406 return -EINVAL;
407 }
408
409 s = kzalloc_obj(struct loongarch_ipi);
410 if (!s)
411 return -ENOMEM;
412
413 spin_lock_init(&s->lock);
414 s->kvm = kvm;
415
416 /*
417 * Initialize IOCSR device
418 */
419 device = &s->device;
420 kvm_iodevice_init(device, &kvm_ipi_ops);
421 mutex_lock(&kvm->slots_lock);
422 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device);
423 mutex_unlock(&kvm->slots_lock);
424 if (ret < 0) {
425 kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret);
426 goto err;
427 }
428
429 kvm->arch.ipi = s;
430 return 0;
431
432 err:
433 kfree(s);
434 return -EFAULT;
435 }
436
kvm_ipi_destroy(struct kvm_device * dev)437 static void kvm_ipi_destroy(struct kvm_device *dev)
438 {
439 struct kvm *kvm;
440 struct loongarch_ipi *ipi;
441
442 if (!dev || !dev->kvm || !dev->kvm->arch.ipi)
443 return;
444
445 kvm = dev->kvm;
446 ipi = kvm->arch.ipi;
447 mutex_lock(&kvm->slots_lock);
448 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device);
449 mutex_unlock(&kvm->slots_lock);
450 kfree(ipi);
451 kfree(dev);
452 }
453
454 static struct kvm_device_ops kvm_ipi_dev_ops = {
455 .name = "kvm-loongarch-ipi",
456 .create = kvm_ipi_create,
457 .destroy = kvm_ipi_destroy,
458 .set_attr = kvm_ipi_set_attr,
459 .get_attr = kvm_ipi_get_attr,
460 };
461
kvm_loongarch_register_ipi_device(void)462 int kvm_loongarch_register_ipi_device(void)
463 {
464 return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI);
465 }
466
kvm_loongarch_unregister_ipi_device(void)467 void kvm_loongarch_unregister_ipi_device(void)
468 {
469 kvm_unregister_device_ops(KVM_DEV_TYPE_LOONGARCH_IPI);
470 }
471