1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_ipi.h> 8 #include <asm/kvm_vcpu.h> 9 10 static void ipi_send(struct kvm *kvm, uint64_t data) 11 { 12 int cpu, action; 13 uint32_t status; 14 struct kvm_vcpu *vcpu; 15 struct kvm_interrupt irq; 16 17 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 18 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 19 if (unlikely(vcpu == NULL)) { 20 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 21 return; 22 } 23 24 action = BIT(data & 0x1f); 25 spin_lock(&vcpu->arch.ipi_state.lock); 26 status = vcpu->arch.ipi_state.status; 27 vcpu->arch.ipi_state.status |= action; 28 spin_unlock(&vcpu->arch.ipi_state.lock); 29 if (status == 0) { 30 irq.irq = LARCH_INT_IPI; 31 kvm_vcpu_ioctl_interrupt(vcpu, &irq); 32 } 33 } 34 35 static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data) 36 { 37 uint32_t status; 38 struct kvm_interrupt irq; 39 40 spin_lock(&vcpu->arch.ipi_state.lock); 41 vcpu->arch.ipi_state.status &= ~data; 42 status = vcpu->arch.ipi_state.status; 43 spin_unlock(&vcpu->arch.ipi_state.lock); 44 if (status == 0) { 45 irq.irq = -LARCH_INT_IPI; 46 kvm_vcpu_ioctl_interrupt(vcpu, &irq); 47 } 48 } 49 50 static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len) 51 { 52 uint64_t data = 0; 53 54 spin_lock(&vcpu->arch.ipi_state.lock); 55 data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20)); 56 spin_unlock(&vcpu->arch.ipi_state.lock); 57 58 switch (len) { 59 case 1: 60 return data & 0xff; 61 case 2: 62 return data & 0xffff; 63 case 4: 64 return data & 0xffffffff; 65 case 8: 66 return data; 67 default: 68 kvm_err("%s: unknown data len: %d\n", __func__, len); 69 return 0; 70 } 71 } 72 73 static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len) 74 { 75 void *pbuf; 76 77 spin_lock(&vcpu->arch.ipi_state.lock); 78 pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20); 79 80 switch (len) { 81 case 1: 82 *(unsigned char *)pbuf = (unsigned char)data; 83 break; 84 case 2: 85 *(unsigned short *)pbuf = (unsigned short)data; 86 break; 87 case 4: 88 *(unsigned int *)pbuf = (unsigned int)data; 89 break; 90 case 8: 91 *(unsigned long *)pbuf = (unsigned long)data; 92 break; 93 default: 94 kvm_err("%s: unknown data len: %d\n", __func__, len); 95 } 96 spin_unlock(&vcpu->arch.ipi_state.lock); 97 } 98 99 static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data) 100 { 101 int i, idx, ret; 102 uint32_t val = 0, mask = 0; 103 104 /* 105 * Bit 27-30 is mask for byte writing. 106 * If the mask is 0, we need not to do anything. 107 */ 108 if ((data >> 27) & 0xf) { 109 /* Read the old val */ 110 idx = srcu_read_lock(&vcpu->kvm->srcu); 111 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); 112 srcu_read_unlock(&vcpu->kvm->srcu, idx); 113 if (unlikely(ret)) { 114 kvm_err("%s: : read date from addr %llx failed\n", __func__, addr); 115 return ret; 116 } 117 /* Construct the mask by scanning the bit 27-30 */ 118 for (i = 0; i < 4; i++) { 119 if (data & (BIT(27 + i))) 120 mask |= (0xff << (i * 8)); 121 } 122 /* Save the old part of val */ 123 val &= mask; 124 } 125 val |= ((uint32_t)(data >> 32) & ~mask); 126 idx = srcu_read_lock(&vcpu->kvm->srcu); 127 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); 128 srcu_read_unlock(&vcpu->kvm->srcu, idx); 129 if (unlikely(ret)) 130 kvm_err("%s: : write date to addr %llx failed\n", __func__, addr); 131 132 return ret; 133 } 134 135 static int mail_send(struct kvm *kvm, uint64_t data) 136 { 137 int cpu, mailbox, offset; 138 struct kvm_vcpu *vcpu; 139 140 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 141 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 142 if (unlikely(vcpu == NULL)) { 143 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 144 return -EINVAL; 145 } 146 mailbox = ((data & 0xffffffff) >> 2) & 0x7; 147 offset = IOCSR_IPI_BASE + IOCSR_IPI_BUF_20 + mailbox * 4; 148 149 return send_ipi_data(vcpu, offset, data); 150 } 151 152 static int any_send(struct kvm *kvm, uint64_t data) 153 { 154 int cpu, offset; 155 struct kvm_vcpu *vcpu; 156 157 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 158 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 159 if (unlikely(vcpu == NULL)) { 160 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 161 return -EINVAL; 162 } 163 offset = data & 0xffff; 164 165 return send_ipi_data(vcpu, offset, data); 166 } 167 168 static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val) 169 { 170 int ret = 0; 171 uint32_t offset; 172 uint64_t res = 0; 173 174 offset = (uint32_t)(addr & 0x1ff); 175 WARN_ON_ONCE(offset & (len - 1)); 176 177 switch (offset) { 178 case IOCSR_IPI_STATUS: 179 spin_lock(&vcpu->arch.ipi_state.lock); 180 res = vcpu->arch.ipi_state.status; 181 spin_unlock(&vcpu->arch.ipi_state.lock); 182 break; 183 case IOCSR_IPI_EN: 184 spin_lock(&vcpu->arch.ipi_state.lock); 185 res = vcpu->arch.ipi_state.en; 186 spin_unlock(&vcpu->arch.ipi_state.lock); 187 break; 188 case IOCSR_IPI_SET: 189 res = 0; 190 break; 191 case IOCSR_IPI_CLEAR: 192 res = 0; 193 break; 194 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: 195 if (offset + len > IOCSR_IPI_BUF_38 + 8) { 196 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", 197 __func__, offset, len); 198 ret = -EINVAL; 199 break; 200 } 201 res = read_mailbox(vcpu, offset, len); 202 break; 203 default: 204 kvm_err("%s: unknown addr: %llx\n", __func__, addr); 205 ret = -EINVAL; 206 break; 207 } 208 *(uint64_t *)val = res; 209 210 return ret; 211 } 212 213 static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val) 214 { 215 int ret = 0; 216 uint64_t data; 217 uint32_t offset; 218 219 data = *(uint64_t *)val; 220 221 offset = (uint32_t)(addr & 0x1ff); 222 WARN_ON_ONCE(offset & (len - 1)); 223 224 switch (offset) { 225 case IOCSR_IPI_STATUS: 226 ret = -EINVAL; 227 break; 228 case IOCSR_IPI_EN: 229 spin_lock(&vcpu->arch.ipi_state.lock); 230 vcpu->arch.ipi_state.en = data; 231 spin_unlock(&vcpu->arch.ipi_state.lock); 232 break; 233 case IOCSR_IPI_SET: 234 ret = -EINVAL; 235 break; 236 case IOCSR_IPI_CLEAR: 237 /* Just clear the status of the current vcpu */ 238 ipi_clear(vcpu, data); 239 break; 240 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: 241 if (offset + len > IOCSR_IPI_BUF_38 + 8) { 242 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", 243 __func__, offset, len); 244 ret = -EINVAL; 245 break; 246 } 247 write_mailbox(vcpu, offset, data, len); 248 break; 249 case IOCSR_IPI_SEND: 250 ipi_send(vcpu->kvm, data); 251 break; 252 case IOCSR_MAIL_SEND: 253 ret = mail_send(vcpu->kvm, *(uint64_t *)val); 254 break; 255 case IOCSR_ANY_SEND: 256 ret = any_send(vcpu->kvm, *(uint64_t *)val); 257 break; 258 default: 259 kvm_err("%s: unknown addr: %llx\n", __func__, addr); 260 ret = -EINVAL; 261 break; 262 } 263 264 return ret; 265 } 266 267 static int kvm_ipi_read(struct kvm_vcpu *vcpu, 268 struct kvm_io_device *dev, 269 gpa_t addr, int len, void *val) 270 { 271 int ret; 272 struct loongarch_ipi *ipi; 273 274 ipi = vcpu->kvm->arch.ipi; 275 if (!ipi) { 276 kvm_err("%s: ipi irqchip not valid!\n", __func__); 277 return -EINVAL; 278 } 279 ipi->kvm->stat.ipi_read_exits++; 280 ret = loongarch_ipi_readl(vcpu, addr, len, val); 281 282 return ret; 283 } 284 285 static int kvm_ipi_write(struct kvm_vcpu *vcpu, 286 struct kvm_io_device *dev, 287 gpa_t addr, int len, const void *val) 288 { 289 int ret; 290 struct loongarch_ipi *ipi; 291 292 ipi = vcpu->kvm->arch.ipi; 293 if (!ipi) { 294 kvm_err("%s: ipi irqchip not valid!\n", __func__); 295 return -EINVAL; 296 } 297 ipi->kvm->stat.ipi_write_exits++; 298 ret = loongarch_ipi_writel(vcpu, addr, len, val); 299 300 return ret; 301 } 302 303 static const struct kvm_io_device_ops kvm_ipi_ops = { 304 .read = kvm_ipi_read, 305 .write = kvm_ipi_write, 306 }; 307 308 static int kvm_ipi_regs_access(struct kvm_device *dev, 309 struct kvm_device_attr *attr, 310 bool is_write) 311 { 312 int len = 4; 313 int cpu, addr; 314 uint64_t val; 315 void *p = NULL; 316 struct kvm_vcpu *vcpu; 317 318 cpu = (attr->attr >> 16) & 0x3ff; 319 addr = attr->attr & 0xff; 320 321 vcpu = kvm_get_vcpu(dev->kvm, cpu); 322 if (unlikely(vcpu == NULL)) { 323 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 324 return -EINVAL; 325 } 326 327 switch (addr) { 328 case IOCSR_IPI_STATUS: 329 p = &vcpu->arch.ipi_state.status; 330 break; 331 case IOCSR_IPI_EN: 332 p = &vcpu->arch.ipi_state.en; 333 break; 334 case IOCSR_IPI_SET: 335 p = &vcpu->arch.ipi_state.set; 336 break; 337 case IOCSR_IPI_CLEAR: 338 p = &vcpu->arch.ipi_state.clear; 339 break; 340 case IOCSR_IPI_BUF_20: 341 p = &vcpu->arch.ipi_state.buf[0]; 342 len = 8; 343 break; 344 case IOCSR_IPI_BUF_28: 345 p = &vcpu->arch.ipi_state.buf[1]; 346 len = 8; 347 break; 348 case IOCSR_IPI_BUF_30: 349 p = &vcpu->arch.ipi_state.buf[2]; 350 len = 8; 351 break; 352 case IOCSR_IPI_BUF_38: 353 p = &vcpu->arch.ipi_state.buf[3]; 354 len = 8; 355 break; 356 default: 357 kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr); 358 return -EINVAL; 359 } 360 361 if (is_write) { 362 if (len == 4) { 363 if (get_user(val, (uint32_t __user *)attr->addr)) 364 return -EFAULT; 365 *(uint32_t *)p = (uint32_t)val; 366 } else if (len == 8) { 367 if (get_user(val, (uint64_t __user *)attr->addr)) 368 return -EFAULT; 369 *(uint64_t *)p = val; 370 } 371 } else { 372 if (len == 4) { 373 val = *(uint32_t *)p; 374 return put_user(val, (uint32_t __user *)attr->addr); 375 } else if (len == 8) { 376 val = *(uint64_t *)p; 377 return put_user(val, (uint64_t __user *)attr->addr); 378 } 379 } 380 381 return 0; 382 } 383 384 static int kvm_ipi_get_attr(struct kvm_device *dev, 385 struct kvm_device_attr *attr) 386 { 387 switch (attr->group) { 388 case KVM_DEV_LOONGARCH_IPI_GRP_REGS: 389 return kvm_ipi_regs_access(dev, attr, false); 390 default: 391 kvm_err("%s: unknown group (%d)\n", __func__, attr->group); 392 return -EINVAL; 393 } 394 } 395 396 static int kvm_ipi_set_attr(struct kvm_device *dev, 397 struct kvm_device_attr *attr) 398 { 399 switch (attr->group) { 400 case KVM_DEV_LOONGARCH_IPI_GRP_REGS: 401 return kvm_ipi_regs_access(dev, attr, true); 402 default: 403 kvm_err("%s: unknown group (%d)\n", __func__, attr->group); 404 return -EINVAL; 405 } 406 } 407 408 static int kvm_ipi_create(struct kvm_device *dev, u32 type) 409 { 410 int ret; 411 struct kvm *kvm; 412 struct kvm_io_device *device; 413 struct loongarch_ipi *s; 414 415 if (!dev) { 416 kvm_err("%s: kvm_device ptr is invalid!\n", __func__); 417 return -EINVAL; 418 } 419 420 kvm = dev->kvm; 421 if (kvm->arch.ipi) { 422 kvm_err("%s: LoongArch IPI has already been created!\n", __func__); 423 return -EINVAL; 424 } 425 426 s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL); 427 if (!s) 428 return -ENOMEM; 429 430 spin_lock_init(&s->lock); 431 s->kvm = kvm; 432 433 /* 434 * Initialize IOCSR device 435 */ 436 device = &s->device; 437 kvm_iodevice_init(device, &kvm_ipi_ops); 438 mutex_lock(&kvm->slots_lock); 439 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device); 440 mutex_unlock(&kvm->slots_lock); 441 if (ret < 0) { 442 kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); 443 goto err; 444 } 445 446 kvm->arch.ipi = s; 447 return 0; 448 449 err: 450 kfree(s); 451 return -EFAULT; 452 } 453 454 static void kvm_ipi_destroy(struct kvm_device *dev) 455 { 456 struct kvm *kvm; 457 struct loongarch_ipi *ipi; 458 459 if (!dev || !dev->kvm || !dev->kvm->arch.ipi) 460 return; 461 462 kvm = dev->kvm; 463 ipi = kvm->arch.ipi; 464 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device); 465 kfree(ipi); 466 } 467 468 static struct kvm_device_ops kvm_ipi_dev_ops = { 469 .name = "kvm-loongarch-ipi", 470 .create = kvm_ipi_create, 471 .destroy = kvm_ipi_destroy, 472 .set_attr = kvm_ipi_set_attr, 473 .get_attr = kvm_ipi_get_attr, 474 }; 475 476 int kvm_loongarch_register_ipi_device(void) 477 { 478 return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI); 479 } 480