1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_ipi.h> 8 #include <asm/kvm_vcpu.h> 9 10 static void ipi_send(struct kvm *kvm, uint64_t data) 11 { 12 int cpu, action; 13 uint32_t status; 14 struct kvm_vcpu *vcpu; 15 struct kvm_interrupt irq; 16 17 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 18 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 19 if (unlikely(vcpu == NULL)) { 20 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 21 return; 22 } 23 24 action = BIT(data & 0x1f); 25 spin_lock(&vcpu->arch.ipi_state.lock); 26 status = vcpu->arch.ipi_state.status; 27 vcpu->arch.ipi_state.status |= action; 28 spin_unlock(&vcpu->arch.ipi_state.lock); 29 if (status == 0) { 30 irq.irq = LARCH_INT_IPI; 31 kvm_vcpu_ioctl_interrupt(vcpu, &irq); 32 } 33 } 34 35 static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data) 36 { 37 uint32_t status; 38 struct kvm_interrupt irq; 39 40 spin_lock(&vcpu->arch.ipi_state.lock); 41 vcpu->arch.ipi_state.status &= ~data; 42 status = vcpu->arch.ipi_state.status; 43 spin_unlock(&vcpu->arch.ipi_state.lock); 44 if (status == 0) { 45 irq.irq = -LARCH_INT_IPI; 46 kvm_vcpu_ioctl_interrupt(vcpu, &irq); 47 } 48 } 49 50 static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len) 51 { 52 uint64_t data = 0; 53 54 spin_lock(&vcpu->arch.ipi_state.lock); 55 data = *(ulong *)((void *)vcpu->arch.ipi_state.buf + (offset - 0x20)); 56 spin_unlock(&vcpu->arch.ipi_state.lock); 57 58 switch (len) { 59 case 1: 60 return data & 0xff; 61 case 2: 62 return data & 0xffff; 63 case 4: 64 return data & 0xffffffff; 65 case 8: 66 return data; 67 default: 68 kvm_err("%s: unknown data len: %d\n", __func__, len); 69 return 0; 70 } 71 } 72 73 static void write_mailbox(struct kvm_vcpu *vcpu, int offset, uint64_t data, int len) 74 { 75 void *pbuf; 76 77 spin_lock(&vcpu->arch.ipi_state.lock); 78 pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20); 79 80 switch (len) { 81 case 1: 82 *(unsigned char *)pbuf = (unsigned char)data; 83 break; 84 case 2: 85 *(unsigned short *)pbuf = (unsigned short)data; 86 break; 87 case 4: 88 *(unsigned int *)pbuf = (unsigned int)data; 89 break; 90 case 8: 91 *(unsigned long *)pbuf = (unsigned long)data; 92 break; 93 default: 94 kvm_err("%s: unknown data len: %d\n", __func__, len); 95 } 96 spin_unlock(&vcpu->arch.ipi_state.lock); 97 } 98 99 static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data) 100 { 101 int i, ret; 102 uint32_t val = 0, mask = 0; 103 104 /* 105 * Bit 27-30 is mask for byte writing. 106 * If the mask is 0, we need not to do anything. 107 */ 108 if ((data >> 27) & 0xf) { 109 /* Read the old val */ 110 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); 111 if (unlikely(ret)) { 112 kvm_err("%s: : read date from addr %llx failed\n", __func__, addr); 113 return ret; 114 } 115 /* Construct the mask by scanning the bit 27-30 */ 116 for (i = 0; i < 4; i++) { 117 if (data & (BIT(27 + i))) 118 mask |= (0xff << (i * 8)); 119 } 120 /* Save the old part of val */ 121 val &= mask; 122 } 123 val |= ((uint32_t)(data >> 32) & ~mask); 124 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); 125 if (unlikely(ret)) 126 kvm_err("%s: : write date to addr %llx failed\n", __func__, addr); 127 128 return ret; 129 } 130 131 static int mail_send(struct kvm *kvm, uint64_t data) 132 { 133 int cpu, mailbox, offset; 134 struct kvm_vcpu *vcpu; 135 136 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 137 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 138 if (unlikely(vcpu == NULL)) { 139 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 140 return -EINVAL; 141 } 142 mailbox = ((data & 0xffffffff) >> 2) & 0x7; 143 offset = IOCSR_IPI_BASE + IOCSR_IPI_BUF_20 + mailbox * 4; 144 145 return send_ipi_data(vcpu, offset, data); 146 } 147 148 static int any_send(struct kvm *kvm, uint64_t data) 149 { 150 int cpu, offset; 151 struct kvm_vcpu *vcpu; 152 153 cpu = ((data & 0xffffffff) >> 16) & 0x3ff; 154 vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); 155 if (unlikely(vcpu == NULL)) { 156 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 157 return -EINVAL; 158 } 159 offset = data & 0xffff; 160 161 return send_ipi_data(vcpu, offset, data); 162 } 163 164 static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *val) 165 { 166 int ret = 0; 167 uint32_t offset; 168 uint64_t res = 0; 169 170 offset = (uint32_t)(addr & 0x1ff); 171 WARN_ON_ONCE(offset & (len - 1)); 172 173 switch (offset) { 174 case IOCSR_IPI_STATUS: 175 spin_lock(&vcpu->arch.ipi_state.lock); 176 res = vcpu->arch.ipi_state.status; 177 spin_unlock(&vcpu->arch.ipi_state.lock); 178 break; 179 case IOCSR_IPI_EN: 180 spin_lock(&vcpu->arch.ipi_state.lock); 181 res = vcpu->arch.ipi_state.en; 182 spin_unlock(&vcpu->arch.ipi_state.lock); 183 break; 184 case IOCSR_IPI_SET: 185 res = 0; 186 break; 187 case IOCSR_IPI_CLEAR: 188 res = 0; 189 break; 190 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: 191 if (offset + len > IOCSR_IPI_BUF_38 + 8) { 192 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", 193 __func__, offset, len); 194 ret = -EINVAL; 195 break; 196 } 197 res = read_mailbox(vcpu, offset, len); 198 break; 199 default: 200 kvm_err("%s: unknown addr: %llx\n", __func__, addr); 201 ret = -EINVAL; 202 break; 203 } 204 *(uint64_t *)val = res; 205 206 return ret; 207 } 208 209 static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, int len, const void *val) 210 { 211 int ret = 0; 212 uint64_t data; 213 uint32_t offset; 214 215 data = *(uint64_t *)val; 216 217 offset = (uint32_t)(addr & 0x1ff); 218 WARN_ON_ONCE(offset & (len - 1)); 219 220 switch (offset) { 221 case IOCSR_IPI_STATUS: 222 ret = -EINVAL; 223 break; 224 case IOCSR_IPI_EN: 225 spin_lock(&vcpu->arch.ipi_state.lock); 226 vcpu->arch.ipi_state.en = data; 227 spin_unlock(&vcpu->arch.ipi_state.lock); 228 break; 229 case IOCSR_IPI_SET: 230 ret = -EINVAL; 231 break; 232 case IOCSR_IPI_CLEAR: 233 /* Just clear the status of the current vcpu */ 234 ipi_clear(vcpu, data); 235 break; 236 case IOCSR_IPI_BUF_20 ... IOCSR_IPI_BUF_38 + 7: 237 if (offset + len > IOCSR_IPI_BUF_38 + 8) { 238 kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", 239 __func__, offset, len); 240 ret = -EINVAL; 241 break; 242 } 243 write_mailbox(vcpu, offset, data, len); 244 break; 245 case IOCSR_IPI_SEND: 246 ipi_send(vcpu->kvm, data); 247 break; 248 case IOCSR_MAIL_SEND: 249 ret = mail_send(vcpu->kvm, *(uint64_t *)val); 250 break; 251 case IOCSR_ANY_SEND: 252 ret = any_send(vcpu->kvm, *(uint64_t *)val); 253 break; 254 default: 255 kvm_err("%s: unknown addr: %llx\n", __func__, addr); 256 ret = -EINVAL; 257 break; 258 } 259 260 return ret; 261 } 262 263 static int kvm_ipi_read(struct kvm_vcpu *vcpu, 264 struct kvm_io_device *dev, 265 gpa_t addr, int len, void *val) 266 { 267 int ret; 268 struct loongarch_ipi *ipi; 269 270 ipi = vcpu->kvm->arch.ipi; 271 if (!ipi) { 272 kvm_err("%s: ipi irqchip not valid!\n", __func__); 273 return -EINVAL; 274 } 275 ipi->kvm->stat.ipi_read_exits++; 276 ret = loongarch_ipi_readl(vcpu, addr, len, val); 277 278 return ret; 279 } 280 281 static int kvm_ipi_write(struct kvm_vcpu *vcpu, 282 struct kvm_io_device *dev, 283 gpa_t addr, int len, const void *val) 284 { 285 int ret; 286 struct loongarch_ipi *ipi; 287 288 ipi = vcpu->kvm->arch.ipi; 289 if (!ipi) { 290 kvm_err("%s: ipi irqchip not valid!\n", __func__); 291 return -EINVAL; 292 } 293 ipi->kvm->stat.ipi_write_exits++; 294 ret = loongarch_ipi_writel(vcpu, addr, len, val); 295 296 return ret; 297 } 298 299 static const struct kvm_io_device_ops kvm_ipi_ops = { 300 .read = kvm_ipi_read, 301 .write = kvm_ipi_write, 302 }; 303 304 static int kvm_ipi_regs_access(struct kvm_device *dev, 305 struct kvm_device_attr *attr, 306 bool is_write) 307 { 308 int len = 4; 309 int cpu, addr; 310 uint64_t val; 311 void *p = NULL; 312 struct kvm_vcpu *vcpu; 313 314 cpu = (attr->attr >> 16) & 0x3ff; 315 addr = attr->attr & 0xff; 316 317 vcpu = kvm_get_vcpu(dev->kvm, cpu); 318 if (unlikely(vcpu == NULL)) { 319 kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); 320 return -EINVAL; 321 } 322 323 switch (addr) { 324 case IOCSR_IPI_STATUS: 325 p = &vcpu->arch.ipi_state.status; 326 break; 327 case IOCSR_IPI_EN: 328 p = &vcpu->arch.ipi_state.en; 329 break; 330 case IOCSR_IPI_SET: 331 p = &vcpu->arch.ipi_state.set; 332 break; 333 case IOCSR_IPI_CLEAR: 334 p = &vcpu->arch.ipi_state.clear; 335 break; 336 case IOCSR_IPI_BUF_20: 337 p = &vcpu->arch.ipi_state.buf[0]; 338 len = 8; 339 break; 340 case IOCSR_IPI_BUF_28: 341 p = &vcpu->arch.ipi_state.buf[1]; 342 len = 8; 343 break; 344 case IOCSR_IPI_BUF_30: 345 p = &vcpu->arch.ipi_state.buf[2]; 346 len = 8; 347 break; 348 case IOCSR_IPI_BUF_38: 349 p = &vcpu->arch.ipi_state.buf[3]; 350 len = 8; 351 break; 352 default: 353 kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr); 354 return -EINVAL; 355 } 356 357 if (is_write) { 358 if (len == 4) { 359 if (get_user(val, (uint32_t __user *)attr->addr)) 360 return -EFAULT; 361 *(uint32_t *)p = (uint32_t)val; 362 } else if (len == 8) { 363 if (get_user(val, (uint64_t __user *)attr->addr)) 364 return -EFAULT; 365 *(uint64_t *)p = val; 366 } 367 } else { 368 if (len == 4) { 369 val = *(uint32_t *)p; 370 return put_user(val, (uint32_t __user *)attr->addr); 371 } else if (len == 8) { 372 val = *(uint64_t *)p; 373 return put_user(val, (uint64_t __user *)attr->addr); 374 } 375 } 376 377 return 0; 378 } 379 380 static int kvm_ipi_get_attr(struct kvm_device *dev, 381 struct kvm_device_attr *attr) 382 { 383 switch (attr->group) { 384 case KVM_DEV_LOONGARCH_IPI_GRP_REGS: 385 return kvm_ipi_regs_access(dev, attr, false); 386 default: 387 kvm_err("%s: unknown group (%d)\n", __func__, attr->group); 388 return -EINVAL; 389 } 390 } 391 392 static int kvm_ipi_set_attr(struct kvm_device *dev, 393 struct kvm_device_attr *attr) 394 { 395 switch (attr->group) { 396 case KVM_DEV_LOONGARCH_IPI_GRP_REGS: 397 return kvm_ipi_regs_access(dev, attr, true); 398 default: 399 kvm_err("%s: unknown group (%d)\n", __func__, attr->group); 400 return -EINVAL; 401 } 402 } 403 404 static int kvm_ipi_create(struct kvm_device *dev, u32 type) 405 { 406 int ret; 407 struct kvm *kvm; 408 struct kvm_io_device *device; 409 struct loongarch_ipi *s; 410 411 if (!dev) { 412 kvm_err("%s: kvm_device ptr is invalid!\n", __func__); 413 return -EINVAL; 414 } 415 416 kvm = dev->kvm; 417 if (kvm->arch.ipi) { 418 kvm_err("%s: LoongArch IPI has already been created!\n", __func__); 419 return -EINVAL; 420 } 421 422 s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL); 423 if (!s) 424 return -ENOMEM; 425 426 spin_lock_init(&s->lock); 427 s->kvm = kvm; 428 429 /* 430 * Initialize IOCSR device 431 */ 432 device = &s->device; 433 kvm_iodevice_init(device, &kvm_ipi_ops); 434 mutex_lock(&kvm->slots_lock); 435 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device); 436 mutex_unlock(&kvm->slots_lock); 437 if (ret < 0) { 438 kvm_err("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); 439 goto err; 440 } 441 442 kvm->arch.ipi = s; 443 return 0; 444 445 err: 446 kfree(s); 447 return -EFAULT; 448 } 449 450 static void kvm_ipi_destroy(struct kvm_device *dev) 451 { 452 struct kvm *kvm; 453 struct loongarch_ipi *ipi; 454 455 if (!dev || !dev->kvm || !dev->kvm->arch.ipi) 456 return; 457 458 kvm = dev->kvm; 459 ipi = kvm->arch.ipi; 460 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &ipi->device); 461 kfree(ipi); 462 } 463 464 static struct kvm_device_ops kvm_ipi_dev_ops = { 465 .name = "kvm-loongarch-ipi", 466 .create = kvm_ipi_create, 467 .destroy = kvm_ipi_destroy, 468 .set_attr = kvm_ipi_set_attr, 469 .get_attr = kvm_ipi_get_attr, 470 }; 471 472 int kvm_loongarch_register_ipi_device(void) 473 { 474 return kvm_register_device_ops(&kvm_ipi_dev_ops, KVM_DEV_TYPE_LOONGARCH_IPI); 475 } 476