1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024 Loongson Technology Corporation Limited 4 */ 5 6 #include <asm/kvm_eiointc.h> 7 #include <asm/kvm_vcpu.h> 8 #include <linux/count_zeros.h> 9 10 static void eiointc_set_sw_coreisr(struct loongarch_eiointc *s) 11 { 12 int ipnum, cpu, cpuid, irq_index, irq_mask, irq; 13 struct kvm_vcpu *vcpu; 14 15 for (irq = 0; irq < EIOINTC_IRQS; irq++) { 16 ipnum = s->ipmap.reg_u8[irq / 32]; 17 if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) { 18 ipnum = count_trailing_zeros(ipnum); 19 ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0; 20 } 21 irq_index = irq / 32; 22 irq_mask = BIT(irq & 0x1f); 23 24 cpuid = s->coremap.reg_u8[irq]; 25 vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); 26 if (!vcpu) 27 continue; 28 29 cpu = vcpu->vcpu_id; 30 if (!!(s->coreisr.reg_u32[cpu][irq_index] & irq_mask)) 31 set_bit(irq, s->sw_coreisr[cpu][ipnum]); 32 else 33 clear_bit(irq, s->sw_coreisr[cpu][ipnum]); 34 } 35 } 36 37 static void eiointc_update_irq(struct loongarch_eiointc *s, int irq, int level) 38 { 39 int ipnum, cpu, found, irq_index, irq_mask; 40 struct kvm_vcpu *vcpu; 41 struct kvm_interrupt vcpu_irq; 42 43 ipnum = s->ipmap.reg_u8[irq / 32]; 44 if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) { 45 ipnum = count_trailing_zeros(ipnum); 46 ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0; 47 } 48 49 cpu = s->sw_coremap[irq]; 50 vcpu = kvm_get_vcpu(s->kvm, cpu); 51 irq_index = irq / 32; 52 irq_mask = BIT(irq & 0x1f); 53 54 if (level) { 55 /* if not enable return false */ 56 if (((s->enable.reg_u32[irq_index]) & irq_mask) == 0) 57 return; 58 s->coreisr.reg_u32[cpu][irq_index] |= irq_mask; 59 found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS); 60 set_bit(irq, s->sw_coreisr[cpu][ipnum]); 61 } else { 62 s->coreisr.reg_u32[cpu][irq_index] &= ~irq_mask; 63 clear_bit(irq, s->sw_coreisr[cpu][ipnum]); 64 found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS); 65 } 66 67 if (found < EIOINTC_IRQS) 68 return; /* other irq is handling, needn't update parent irq */ 69 70 vcpu_irq.irq = level ? (INT_HWI0 + ipnum) : -(INT_HWI0 + ipnum); 71 kvm_vcpu_ioctl_interrupt(vcpu, &vcpu_irq); 72 } 73 74 static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, 75 int irq, u64 val, u32 len, bool notify) 76 { 77 int i, cpu, cpuid; 78 struct kvm_vcpu *vcpu; 79 80 for (i = 0; i < len; i++) { 81 cpuid = val & 0xff; 82 val = val >> 8; 83 84 if (!(s->status & BIT(EIOINTC_ENABLE_CPU_ENCODE))) { 85 cpuid = ffs(cpuid) - 1; 86 cpuid = (cpuid >= 4) ? 0 : cpuid; 87 } 88 89 vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); 90 if (!vcpu) 91 continue; 92 93 cpu = vcpu->vcpu_id; 94 if (s->sw_coremap[irq + i] == cpu) 95 continue; 96 97 if (notify && test_bit(irq + i, (unsigned long *)s->isr.reg_u8)) { 98 /* lower irq at old cpu and raise irq at new cpu */ 99 eiointc_update_irq(s, irq + i, 0); 100 s->sw_coremap[irq + i] = cpu; 101 eiointc_update_irq(s, irq + i, 1); 102 } else { 103 s->sw_coremap[irq + i] = cpu; 104 } 105 } 106 } 107 108 void eiointc_set_irq(struct loongarch_eiointc *s, int irq, int level) 109 { 110 unsigned long flags; 111 unsigned long *isr = (unsigned long *)s->isr.reg_u8; 112 113 level ? set_bit(irq, isr) : clear_bit(irq, isr); 114 spin_lock_irqsave(&s->lock, flags); 115 eiointc_update_irq(s, irq, level); 116 spin_unlock_irqrestore(&s->lock, flags); 117 } 118 119 static inline void eiointc_enable_irq(struct kvm_vcpu *vcpu, 120 struct loongarch_eiointc *s, int index, u8 mask, int level) 121 { 122 u8 val; 123 int irq; 124 125 val = mask & s->isr.reg_u8[index]; 126 irq = ffs(val); 127 while (irq != 0) { 128 /* 129 * enable bit change from 0 to 1, 130 * need to update irq by pending bits 131 */ 132 eiointc_update_irq(s, irq - 1 + index * 8, level); 133 val &= ~BIT(irq - 1); 134 irq = ffs(val); 135 } 136 } 137 138 static int loongarch_eiointc_readb(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s, 139 gpa_t addr, int len, void *val) 140 { 141 int index, ret = 0; 142 u8 data = 0; 143 gpa_t offset; 144 145 offset = addr - EIOINTC_BASE; 146 switch (offset) { 147 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 148 index = offset - EIOINTC_NODETYPE_START; 149 data = s->nodetype.reg_u8[index]; 150 break; 151 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 152 index = offset - EIOINTC_IPMAP_START; 153 data = s->ipmap.reg_u8[index]; 154 break; 155 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 156 index = offset - EIOINTC_ENABLE_START; 157 data = s->enable.reg_u8[index]; 158 break; 159 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 160 index = offset - EIOINTC_BOUNCE_START; 161 data = s->bounce.reg_u8[index]; 162 break; 163 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 164 index = offset - EIOINTC_COREISR_START; 165 data = s->coreisr.reg_u8[vcpu->vcpu_id][index]; 166 break; 167 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 168 index = offset - EIOINTC_COREMAP_START; 169 data = s->coremap.reg_u8[index]; 170 break; 171 default: 172 ret = -EINVAL; 173 break; 174 } 175 *(u8 *)val = data; 176 177 return ret; 178 } 179 180 static int loongarch_eiointc_readw(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s, 181 gpa_t addr, int len, void *val) 182 { 183 int index, ret = 0; 184 u16 data = 0; 185 gpa_t offset; 186 187 offset = addr - EIOINTC_BASE; 188 switch (offset) { 189 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 190 index = (offset - EIOINTC_NODETYPE_START) >> 1; 191 data = s->nodetype.reg_u16[index]; 192 break; 193 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 194 index = (offset - EIOINTC_IPMAP_START) >> 1; 195 data = s->ipmap.reg_u16[index]; 196 break; 197 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 198 index = (offset - EIOINTC_ENABLE_START) >> 1; 199 data = s->enable.reg_u16[index]; 200 break; 201 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 202 index = (offset - EIOINTC_BOUNCE_START) >> 1; 203 data = s->bounce.reg_u16[index]; 204 break; 205 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 206 index = (offset - EIOINTC_COREISR_START) >> 1; 207 data = s->coreisr.reg_u16[vcpu->vcpu_id][index]; 208 break; 209 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 210 index = (offset - EIOINTC_COREMAP_START) >> 1; 211 data = s->coremap.reg_u16[index]; 212 break; 213 default: 214 ret = -EINVAL; 215 break; 216 } 217 *(u16 *)val = data; 218 219 return ret; 220 } 221 222 static int loongarch_eiointc_readl(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s, 223 gpa_t addr, int len, void *val) 224 { 225 int index, ret = 0; 226 u32 data = 0; 227 gpa_t offset; 228 229 offset = addr - EIOINTC_BASE; 230 switch (offset) { 231 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 232 index = (offset - EIOINTC_NODETYPE_START) >> 2; 233 data = s->nodetype.reg_u32[index]; 234 break; 235 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 236 index = (offset - EIOINTC_IPMAP_START) >> 2; 237 data = s->ipmap.reg_u32[index]; 238 break; 239 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 240 index = (offset - EIOINTC_ENABLE_START) >> 2; 241 data = s->enable.reg_u32[index]; 242 break; 243 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 244 index = (offset - EIOINTC_BOUNCE_START) >> 2; 245 data = s->bounce.reg_u32[index]; 246 break; 247 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 248 index = (offset - EIOINTC_COREISR_START) >> 2; 249 data = s->coreisr.reg_u32[vcpu->vcpu_id][index]; 250 break; 251 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 252 index = (offset - EIOINTC_COREMAP_START) >> 2; 253 data = s->coremap.reg_u32[index]; 254 break; 255 default: 256 ret = -EINVAL; 257 break; 258 } 259 *(u32 *)val = data; 260 261 return ret; 262 } 263 264 static int loongarch_eiointc_readq(struct kvm_vcpu *vcpu, struct loongarch_eiointc *s, 265 gpa_t addr, int len, void *val) 266 { 267 int index, ret = 0; 268 u64 data = 0; 269 gpa_t offset; 270 271 offset = addr - EIOINTC_BASE; 272 switch (offset) { 273 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 274 index = (offset - EIOINTC_NODETYPE_START) >> 3; 275 data = s->nodetype.reg_u64[index]; 276 break; 277 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 278 index = (offset - EIOINTC_IPMAP_START) >> 3; 279 data = s->ipmap.reg_u64; 280 break; 281 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 282 index = (offset - EIOINTC_ENABLE_START) >> 3; 283 data = s->enable.reg_u64[index]; 284 break; 285 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 286 index = (offset - EIOINTC_BOUNCE_START) >> 3; 287 data = s->bounce.reg_u64[index]; 288 break; 289 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 290 index = (offset - EIOINTC_COREISR_START) >> 3; 291 data = s->coreisr.reg_u64[vcpu->vcpu_id][index]; 292 break; 293 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 294 index = (offset - EIOINTC_COREMAP_START) >> 3; 295 data = s->coremap.reg_u64[index]; 296 break; 297 default: 298 ret = -EINVAL; 299 break; 300 } 301 *(u64 *)val = data; 302 303 return ret; 304 } 305 306 static int kvm_eiointc_read(struct kvm_vcpu *vcpu, 307 struct kvm_io_device *dev, 308 gpa_t addr, int len, void *val) 309 { 310 int ret = -EINVAL; 311 unsigned long flags; 312 struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc; 313 314 if (!eiointc) { 315 kvm_err("%s: eiointc irqchip not valid!\n", __func__); 316 return -EINVAL; 317 } 318 319 if (addr & (len - 1)) { 320 kvm_err("%s: eiointc not aligned addr %llx len %d\n", __func__, addr, len); 321 return -EINVAL; 322 } 323 324 vcpu->kvm->stat.eiointc_read_exits++; 325 spin_lock_irqsave(&eiointc->lock, flags); 326 switch (len) { 327 case 1: 328 ret = loongarch_eiointc_readb(vcpu, eiointc, addr, len, val); 329 break; 330 case 2: 331 ret = loongarch_eiointc_readw(vcpu, eiointc, addr, len, val); 332 break; 333 case 4: 334 ret = loongarch_eiointc_readl(vcpu, eiointc, addr, len, val); 335 break; 336 case 8: 337 ret = loongarch_eiointc_readq(vcpu, eiointc, addr, len, val); 338 break; 339 default: 340 WARN_ONCE(1, "%s: Abnormal address access: addr 0x%llx, size %d\n", 341 __func__, addr, len); 342 } 343 spin_unlock_irqrestore(&eiointc->lock, flags); 344 345 return ret; 346 } 347 348 static int loongarch_eiointc_writeb(struct kvm_vcpu *vcpu, 349 struct loongarch_eiointc *s, 350 gpa_t addr, int len, const void *val) 351 { 352 int index, irq, bits, ret = 0; 353 u8 cpu; 354 u8 data, old_data; 355 u8 coreisr, old_coreisr; 356 gpa_t offset; 357 358 data = *(u8 *)val; 359 offset = addr - EIOINTC_BASE; 360 361 switch (offset) { 362 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 363 index = (offset - EIOINTC_NODETYPE_START); 364 s->nodetype.reg_u8[index] = data; 365 break; 366 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 367 /* 368 * ipmap cannot be set at runtime, can be set only at the beginning 369 * of irqchip driver, need not update upper irq level 370 */ 371 index = (offset - EIOINTC_IPMAP_START); 372 s->ipmap.reg_u8[index] = data; 373 break; 374 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 375 index = (offset - EIOINTC_ENABLE_START); 376 old_data = s->enable.reg_u8[index]; 377 s->enable.reg_u8[index] = data; 378 /* 379 * 1: enable irq. 380 * update irq when isr is set. 381 */ 382 data = s->enable.reg_u8[index] & ~old_data & s->isr.reg_u8[index]; 383 eiointc_enable_irq(vcpu, s, index, data, 1); 384 /* 385 * 0: disable irq. 386 * update irq when isr is set. 387 */ 388 data = ~s->enable.reg_u8[index] & old_data & s->isr.reg_u8[index]; 389 eiointc_enable_irq(vcpu, s, index, data, 0); 390 break; 391 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 392 /* do not emulate hw bounced irq routing */ 393 index = offset - EIOINTC_BOUNCE_START; 394 s->bounce.reg_u8[index] = data; 395 break; 396 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 397 index = (offset - EIOINTC_COREISR_START); 398 /* use attrs to get current cpu index */ 399 cpu = vcpu->vcpu_id; 400 coreisr = data; 401 old_coreisr = s->coreisr.reg_u8[cpu][index]; 402 /* write 1 to clear interrupt */ 403 s->coreisr.reg_u8[cpu][index] = old_coreisr & ~coreisr; 404 coreisr &= old_coreisr; 405 bits = sizeof(data) * 8; 406 irq = find_first_bit((void *)&coreisr, bits); 407 while (irq < bits) { 408 eiointc_update_irq(s, irq + index * bits, 0); 409 bitmap_clear((void *)&coreisr, irq, 1); 410 irq = find_first_bit((void *)&coreisr, bits); 411 } 412 break; 413 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 414 irq = offset - EIOINTC_COREMAP_START; 415 index = irq; 416 s->coremap.reg_u8[index] = data; 417 eiointc_update_sw_coremap(s, irq, data, sizeof(data), true); 418 break; 419 default: 420 ret = -EINVAL; 421 break; 422 } 423 424 return ret; 425 } 426 427 static int loongarch_eiointc_writew(struct kvm_vcpu *vcpu, 428 struct loongarch_eiointc *s, 429 gpa_t addr, int len, const void *val) 430 { 431 int i, index, irq, bits, ret = 0; 432 u8 cpu; 433 u16 data, old_data; 434 u16 coreisr, old_coreisr; 435 gpa_t offset; 436 437 data = *(u16 *)val; 438 offset = addr - EIOINTC_BASE; 439 440 switch (offset) { 441 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 442 index = (offset - EIOINTC_NODETYPE_START) >> 1; 443 s->nodetype.reg_u16[index] = data; 444 break; 445 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 446 /* 447 * ipmap cannot be set at runtime, can be set only at the beginning 448 * of irqchip driver, need not update upper irq level 449 */ 450 index = (offset - EIOINTC_IPMAP_START) >> 1; 451 s->ipmap.reg_u16[index] = data; 452 break; 453 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 454 index = (offset - EIOINTC_ENABLE_START) >> 1; 455 old_data = s->enable.reg_u16[index]; 456 s->enable.reg_u16[index] = data; 457 /* 458 * 1: enable irq. 459 * update irq when isr is set. 460 */ 461 data = s->enable.reg_u16[index] & ~old_data & s->isr.reg_u16[index]; 462 for (i = 0; i < sizeof(data); i++) { 463 u8 mask = (data >> (i * 8)) & 0xff; 464 eiointc_enable_irq(vcpu, s, index * 2 + i, mask, 1); 465 } 466 /* 467 * 0: disable irq. 468 * update irq when isr is set. 469 */ 470 data = ~s->enable.reg_u16[index] & old_data & s->isr.reg_u16[index]; 471 for (i = 0; i < sizeof(data); i++) { 472 u8 mask = (data >> (i * 8)) & 0xff; 473 eiointc_enable_irq(vcpu, s, index * 2 + i, mask, 0); 474 } 475 break; 476 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 477 /* do not emulate hw bounced irq routing */ 478 index = (offset - EIOINTC_BOUNCE_START) >> 1; 479 s->bounce.reg_u16[index] = data; 480 break; 481 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 482 index = (offset - EIOINTC_COREISR_START) >> 1; 483 /* use attrs to get current cpu index */ 484 cpu = vcpu->vcpu_id; 485 coreisr = data; 486 old_coreisr = s->coreisr.reg_u16[cpu][index]; 487 /* write 1 to clear interrupt */ 488 s->coreisr.reg_u16[cpu][index] = old_coreisr & ~coreisr; 489 coreisr &= old_coreisr; 490 bits = sizeof(data) * 8; 491 irq = find_first_bit((void *)&coreisr, bits); 492 while (irq < bits) { 493 eiointc_update_irq(s, irq + index * bits, 0); 494 bitmap_clear((void *)&coreisr, irq, 1); 495 irq = find_first_bit((void *)&coreisr, bits); 496 } 497 break; 498 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 499 irq = offset - EIOINTC_COREMAP_START; 500 index = irq >> 1; 501 s->coremap.reg_u16[index] = data; 502 eiointc_update_sw_coremap(s, irq, data, sizeof(data), true); 503 break; 504 default: 505 ret = -EINVAL; 506 break; 507 } 508 509 return ret; 510 } 511 512 static int loongarch_eiointc_writel(struct kvm_vcpu *vcpu, 513 struct loongarch_eiointc *s, 514 gpa_t addr, int len, const void *val) 515 { 516 int i, index, irq, bits, ret = 0; 517 u8 cpu; 518 u32 data, old_data; 519 u32 coreisr, old_coreisr; 520 gpa_t offset; 521 522 data = *(u32 *)val; 523 offset = addr - EIOINTC_BASE; 524 525 switch (offset) { 526 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 527 index = (offset - EIOINTC_NODETYPE_START) >> 2; 528 s->nodetype.reg_u32[index] = data; 529 break; 530 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 531 /* 532 * ipmap cannot be set at runtime, can be set only at the beginning 533 * of irqchip driver, need not update upper irq level 534 */ 535 index = (offset - EIOINTC_IPMAP_START) >> 2; 536 s->ipmap.reg_u32[index] = data; 537 break; 538 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 539 index = (offset - EIOINTC_ENABLE_START) >> 2; 540 old_data = s->enable.reg_u32[index]; 541 s->enable.reg_u32[index] = data; 542 /* 543 * 1: enable irq. 544 * update irq when isr is set. 545 */ 546 data = s->enable.reg_u32[index] & ~old_data & s->isr.reg_u32[index]; 547 for (i = 0; i < sizeof(data); i++) { 548 u8 mask = (data >> (i * 8)) & 0xff; 549 eiointc_enable_irq(vcpu, s, index * 4 + i, mask, 1); 550 } 551 /* 552 * 0: disable irq. 553 * update irq when isr is set. 554 */ 555 data = ~s->enable.reg_u32[index] & old_data & s->isr.reg_u32[index]; 556 for (i = 0; i < sizeof(data); i++) { 557 u8 mask = (data >> (i * 8)) & 0xff; 558 eiointc_enable_irq(vcpu, s, index * 4 + i, mask, 0); 559 } 560 break; 561 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 562 /* do not emulate hw bounced irq routing */ 563 index = (offset - EIOINTC_BOUNCE_START) >> 2; 564 s->bounce.reg_u32[index] = data; 565 break; 566 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 567 index = (offset - EIOINTC_COREISR_START) >> 2; 568 /* use attrs to get current cpu index */ 569 cpu = vcpu->vcpu_id; 570 coreisr = data; 571 old_coreisr = s->coreisr.reg_u32[cpu][index]; 572 /* write 1 to clear interrupt */ 573 s->coreisr.reg_u32[cpu][index] = old_coreisr & ~coreisr; 574 coreisr &= old_coreisr; 575 bits = sizeof(data) * 8; 576 irq = find_first_bit((void *)&coreisr, bits); 577 while (irq < bits) { 578 eiointc_update_irq(s, irq + index * bits, 0); 579 bitmap_clear((void *)&coreisr, irq, 1); 580 irq = find_first_bit((void *)&coreisr, bits); 581 } 582 break; 583 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 584 irq = offset - EIOINTC_COREMAP_START; 585 index = irq >> 2; 586 s->coremap.reg_u32[index] = data; 587 eiointc_update_sw_coremap(s, irq, data, sizeof(data), true); 588 break; 589 default: 590 ret = -EINVAL; 591 break; 592 } 593 594 return ret; 595 } 596 597 static int loongarch_eiointc_writeq(struct kvm_vcpu *vcpu, 598 struct loongarch_eiointc *s, 599 gpa_t addr, int len, const void *val) 600 { 601 int i, index, irq, bits, ret = 0; 602 u8 cpu; 603 u64 data, old_data; 604 u64 coreisr, old_coreisr; 605 gpa_t offset; 606 607 data = *(u64 *)val; 608 offset = addr - EIOINTC_BASE; 609 610 switch (offset) { 611 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 612 index = (offset - EIOINTC_NODETYPE_START) >> 3; 613 s->nodetype.reg_u64[index] = data; 614 break; 615 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 616 /* 617 * ipmap cannot be set at runtime, can be set only at the beginning 618 * of irqchip driver, need not update upper irq level 619 */ 620 index = (offset - EIOINTC_IPMAP_START) >> 3; 621 s->ipmap.reg_u64 = data; 622 break; 623 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 624 index = (offset - EIOINTC_ENABLE_START) >> 3; 625 old_data = s->enable.reg_u64[index]; 626 s->enable.reg_u64[index] = data; 627 /* 628 * 1: enable irq. 629 * update irq when isr is set. 630 */ 631 data = s->enable.reg_u64[index] & ~old_data & s->isr.reg_u64[index]; 632 for (i = 0; i < sizeof(data); i++) { 633 u8 mask = (data >> (i * 8)) & 0xff; 634 eiointc_enable_irq(vcpu, s, index * 8 + i, mask, 1); 635 } 636 /* 637 * 0: disable irq. 638 * update irq when isr is set. 639 */ 640 data = ~s->enable.reg_u64[index] & old_data & s->isr.reg_u64[index]; 641 for (i = 0; i < sizeof(data); i++) { 642 u8 mask = (data >> (i * 8)) & 0xff; 643 eiointc_enable_irq(vcpu, s, index * 8 + i, mask, 0); 644 } 645 break; 646 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 647 /* do not emulate hw bounced irq routing */ 648 index = (offset - EIOINTC_BOUNCE_START) >> 3; 649 s->bounce.reg_u64[index] = data; 650 break; 651 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 652 index = (offset - EIOINTC_COREISR_START) >> 3; 653 /* use attrs to get current cpu index */ 654 cpu = vcpu->vcpu_id; 655 coreisr = data; 656 old_coreisr = s->coreisr.reg_u64[cpu][index]; 657 /* write 1 to clear interrupt */ 658 s->coreisr.reg_u64[cpu][index] = old_coreisr & ~coreisr; 659 coreisr &= old_coreisr; 660 bits = sizeof(data) * 8; 661 irq = find_first_bit((void *)&coreisr, bits); 662 while (irq < bits) { 663 eiointc_update_irq(s, irq + index * bits, 0); 664 bitmap_clear((void *)&coreisr, irq, 1); 665 irq = find_first_bit((void *)&coreisr, bits); 666 } 667 break; 668 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 669 irq = offset - EIOINTC_COREMAP_START; 670 index = irq >> 3; 671 s->coremap.reg_u64[index] = data; 672 eiointc_update_sw_coremap(s, irq, data, sizeof(data), true); 673 break; 674 default: 675 ret = -EINVAL; 676 break; 677 } 678 679 return ret; 680 } 681 682 static int kvm_eiointc_write(struct kvm_vcpu *vcpu, 683 struct kvm_io_device *dev, 684 gpa_t addr, int len, const void *val) 685 { 686 int ret = -EINVAL; 687 unsigned long flags; 688 struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc; 689 690 if (!eiointc) { 691 kvm_err("%s: eiointc irqchip not valid!\n", __func__); 692 return -EINVAL; 693 } 694 695 if (addr & (len - 1)) { 696 kvm_err("%s: eiointc not aligned addr %llx len %d\n", __func__, addr, len); 697 return -EINVAL; 698 } 699 700 vcpu->kvm->stat.eiointc_write_exits++; 701 spin_lock_irqsave(&eiointc->lock, flags); 702 switch (len) { 703 case 1: 704 ret = loongarch_eiointc_writeb(vcpu, eiointc, addr, len, val); 705 break; 706 case 2: 707 ret = loongarch_eiointc_writew(vcpu, eiointc, addr, len, val); 708 break; 709 case 4: 710 ret = loongarch_eiointc_writel(vcpu, eiointc, addr, len, val); 711 break; 712 case 8: 713 ret = loongarch_eiointc_writeq(vcpu, eiointc, addr, len, val); 714 break; 715 default: 716 WARN_ONCE(1, "%s: Abnormal address access: addr 0x%llx, size %d\n", 717 __func__, addr, len); 718 } 719 spin_unlock_irqrestore(&eiointc->lock, flags); 720 721 return ret; 722 } 723 724 static const struct kvm_io_device_ops kvm_eiointc_ops = { 725 .read = kvm_eiointc_read, 726 .write = kvm_eiointc_write, 727 }; 728 729 static int kvm_eiointc_virt_read(struct kvm_vcpu *vcpu, 730 struct kvm_io_device *dev, 731 gpa_t addr, int len, void *val) 732 { 733 unsigned long flags; 734 u32 *data = val; 735 struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc; 736 737 if (!eiointc) { 738 kvm_err("%s: eiointc irqchip not valid!\n", __func__); 739 return -EINVAL; 740 } 741 742 addr -= EIOINTC_VIRT_BASE; 743 spin_lock_irqsave(&eiointc->lock, flags); 744 switch (addr) { 745 case EIOINTC_VIRT_FEATURES: 746 *data = eiointc->features; 747 break; 748 case EIOINTC_VIRT_CONFIG: 749 *data = eiointc->status; 750 break; 751 default: 752 break; 753 } 754 spin_unlock_irqrestore(&eiointc->lock, flags); 755 756 return 0; 757 } 758 759 static int kvm_eiointc_virt_write(struct kvm_vcpu *vcpu, 760 struct kvm_io_device *dev, 761 gpa_t addr, int len, const void *val) 762 { 763 int ret = 0; 764 unsigned long flags; 765 u32 value = *(u32 *)val; 766 struct loongarch_eiointc *eiointc = vcpu->kvm->arch.eiointc; 767 768 if (!eiointc) { 769 kvm_err("%s: eiointc irqchip not valid!\n", __func__); 770 return -EINVAL; 771 } 772 773 addr -= EIOINTC_VIRT_BASE; 774 spin_lock_irqsave(&eiointc->lock, flags); 775 switch (addr) { 776 case EIOINTC_VIRT_FEATURES: 777 ret = -EPERM; 778 break; 779 case EIOINTC_VIRT_CONFIG: 780 /* 781 * eiointc features can only be set at disabled status 782 */ 783 if ((eiointc->status & BIT(EIOINTC_ENABLE)) && value) { 784 ret = -EPERM; 785 break; 786 } 787 eiointc->status = value & eiointc->features; 788 break; 789 default: 790 break; 791 } 792 spin_unlock_irqrestore(&eiointc->lock, flags); 793 794 return ret; 795 } 796 797 static const struct kvm_io_device_ops kvm_eiointc_virt_ops = { 798 .read = kvm_eiointc_virt_read, 799 .write = kvm_eiointc_virt_write, 800 }; 801 802 static int kvm_eiointc_ctrl_access(struct kvm_device *dev, 803 struct kvm_device_attr *attr) 804 { 805 int ret = 0; 806 unsigned long flags; 807 unsigned long type = (unsigned long)attr->attr; 808 u32 i, start_irq, val; 809 void __user *data; 810 struct loongarch_eiointc *s = dev->kvm->arch.eiointc; 811 812 data = (void __user *)attr->addr; 813 spin_lock_irqsave(&s->lock, flags); 814 switch (type) { 815 case KVM_DEV_LOONGARCH_EXTIOI_CTRL_INIT_NUM_CPU: 816 if (copy_from_user(&val, data, 4)) 817 ret = -EFAULT; 818 else { 819 if (val >= EIOINTC_ROUTE_MAX_VCPUS) 820 ret = -EINVAL; 821 else 822 s->num_cpu = val; 823 } 824 break; 825 case KVM_DEV_LOONGARCH_EXTIOI_CTRL_INIT_FEATURE: 826 if (copy_from_user(&s->features, data, 4)) 827 ret = -EFAULT; 828 if (!(s->features & BIT(EIOINTC_HAS_VIRT_EXTENSION))) 829 s->status |= BIT(EIOINTC_ENABLE); 830 break; 831 case KVM_DEV_LOONGARCH_EXTIOI_CTRL_LOAD_FINISHED: 832 eiointc_set_sw_coreisr(s); 833 for (i = 0; i < (EIOINTC_IRQS / 4); i++) { 834 start_irq = i * 4; 835 eiointc_update_sw_coremap(s, start_irq, 836 s->coremap.reg_u32[i], sizeof(u32), false); 837 } 838 break; 839 default: 840 break; 841 } 842 spin_unlock_irqrestore(&s->lock, flags); 843 844 return ret; 845 } 846 847 static int kvm_eiointc_regs_access(struct kvm_device *dev, 848 struct kvm_device_attr *attr, 849 bool is_write) 850 { 851 int addr, cpu, offset, ret = 0; 852 unsigned long flags; 853 void *p = NULL; 854 void __user *data; 855 struct loongarch_eiointc *s; 856 857 s = dev->kvm->arch.eiointc; 858 addr = attr->attr; 859 cpu = addr >> 16; 860 addr &= 0xffff; 861 data = (void __user *)attr->addr; 862 switch (addr) { 863 case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: 864 offset = (addr - EIOINTC_NODETYPE_START) / 4; 865 p = &s->nodetype.reg_u32[offset]; 866 break; 867 case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: 868 offset = (addr - EIOINTC_IPMAP_START) / 4; 869 p = &s->ipmap.reg_u32[offset]; 870 break; 871 case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: 872 offset = (addr - EIOINTC_ENABLE_START) / 4; 873 p = &s->enable.reg_u32[offset]; 874 break; 875 case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: 876 offset = (addr - EIOINTC_BOUNCE_START) / 4; 877 p = &s->bounce.reg_u32[offset]; 878 break; 879 case EIOINTC_ISR_START ... EIOINTC_ISR_END: 880 offset = (addr - EIOINTC_ISR_START) / 4; 881 p = &s->isr.reg_u32[offset]; 882 break; 883 case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: 884 if (cpu >= s->num_cpu) 885 return -EINVAL; 886 887 offset = (addr - EIOINTC_COREISR_START) / 4; 888 p = &s->coreisr.reg_u32[cpu][offset]; 889 break; 890 case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: 891 offset = (addr - EIOINTC_COREMAP_START) / 4; 892 p = &s->coremap.reg_u32[offset]; 893 break; 894 default: 895 kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr); 896 return -EINVAL; 897 } 898 899 spin_lock_irqsave(&s->lock, flags); 900 if (is_write) { 901 if (copy_from_user(p, data, 4)) 902 ret = -EFAULT; 903 } else { 904 if (copy_to_user(data, p, 4)) 905 ret = -EFAULT; 906 } 907 spin_unlock_irqrestore(&s->lock, flags); 908 909 return ret; 910 } 911 912 static int kvm_eiointc_sw_status_access(struct kvm_device *dev, 913 struct kvm_device_attr *attr, 914 bool is_write) 915 { 916 int addr, ret = 0; 917 unsigned long flags; 918 void *p = NULL; 919 void __user *data; 920 struct loongarch_eiointc *s; 921 922 s = dev->kvm->arch.eiointc; 923 addr = attr->attr; 924 addr &= 0xffff; 925 926 data = (void __user *)attr->addr; 927 switch (addr) { 928 case KVM_DEV_LOONGARCH_EXTIOI_SW_STATUS_NUM_CPU: 929 if (is_write) 930 return ret; 931 932 p = &s->num_cpu; 933 break; 934 case KVM_DEV_LOONGARCH_EXTIOI_SW_STATUS_FEATURE: 935 if (is_write) 936 return ret; 937 938 p = &s->features; 939 break; 940 case KVM_DEV_LOONGARCH_EXTIOI_SW_STATUS_STATE: 941 p = &s->status; 942 break; 943 default: 944 kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr); 945 return -EINVAL; 946 } 947 spin_lock_irqsave(&s->lock, flags); 948 if (is_write) { 949 if (copy_from_user(p, data, 4)) 950 ret = -EFAULT; 951 } else { 952 if (copy_to_user(data, p, 4)) 953 ret = -EFAULT; 954 } 955 spin_unlock_irqrestore(&s->lock, flags); 956 957 return ret; 958 } 959 960 static int kvm_eiointc_get_attr(struct kvm_device *dev, 961 struct kvm_device_attr *attr) 962 { 963 switch (attr->group) { 964 case KVM_DEV_LOONGARCH_EXTIOI_GRP_REGS: 965 return kvm_eiointc_regs_access(dev, attr, false); 966 case KVM_DEV_LOONGARCH_EXTIOI_GRP_SW_STATUS: 967 return kvm_eiointc_sw_status_access(dev, attr, false); 968 default: 969 return -EINVAL; 970 } 971 } 972 973 static int kvm_eiointc_set_attr(struct kvm_device *dev, 974 struct kvm_device_attr *attr) 975 { 976 switch (attr->group) { 977 case KVM_DEV_LOONGARCH_EXTIOI_GRP_CTRL: 978 return kvm_eiointc_ctrl_access(dev, attr); 979 case KVM_DEV_LOONGARCH_EXTIOI_GRP_REGS: 980 return kvm_eiointc_regs_access(dev, attr, true); 981 case KVM_DEV_LOONGARCH_EXTIOI_GRP_SW_STATUS: 982 return kvm_eiointc_sw_status_access(dev, attr, true); 983 default: 984 return -EINVAL; 985 } 986 } 987 988 static int kvm_eiointc_create(struct kvm_device *dev, u32 type) 989 { 990 int ret; 991 struct loongarch_eiointc *s; 992 struct kvm_io_device *device, *device1; 993 struct kvm *kvm = dev->kvm; 994 995 /* eiointc has been created */ 996 if (kvm->arch.eiointc) 997 return -EINVAL; 998 999 s = kzalloc(sizeof(struct loongarch_eiointc), GFP_KERNEL); 1000 if (!s) 1001 return -ENOMEM; 1002 1003 spin_lock_init(&s->lock); 1004 s->kvm = kvm; 1005 1006 /* 1007 * Initialize IOCSR device 1008 */ 1009 device = &s->device; 1010 kvm_iodevice_init(device, &kvm_eiointc_ops); 1011 mutex_lock(&kvm->slots_lock); 1012 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, 1013 EIOINTC_BASE, EIOINTC_SIZE, device); 1014 mutex_unlock(&kvm->slots_lock); 1015 if (ret < 0) { 1016 kfree(s); 1017 return ret; 1018 } 1019 1020 device1 = &s->device_vext; 1021 kvm_iodevice_init(device1, &kvm_eiointc_virt_ops); 1022 ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, 1023 EIOINTC_VIRT_BASE, EIOINTC_VIRT_SIZE, device1); 1024 if (ret < 0) { 1025 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &s->device); 1026 kfree(s); 1027 return ret; 1028 } 1029 kvm->arch.eiointc = s; 1030 1031 return 0; 1032 } 1033 1034 static void kvm_eiointc_destroy(struct kvm_device *dev) 1035 { 1036 struct kvm *kvm; 1037 struct loongarch_eiointc *eiointc; 1038 1039 if (!dev || !dev->kvm || !dev->kvm->arch.eiointc) 1040 return; 1041 1042 kvm = dev->kvm; 1043 eiointc = kvm->arch.eiointc; 1044 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device); 1045 kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, &eiointc->device_vext); 1046 kfree(eiointc); 1047 } 1048 1049 static struct kvm_device_ops kvm_eiointc_dev_ops = { 1050 .name = "kvm-loongarch-eiointc", 1051 .create = kvm_eiointc_create, 1052 .destroy = kvm_eiointc_destroy, 1053 .set_attr = kvm_eiointc_set_attr, 1054 .get_attr = kvm_eiointc_get_attr, 1055 }; 1056 1057 int kvm_loongarch_register_eiointc_device(void) 1058 { 1059 return kvm_register_device_ops(&kvm_eiointc_dev_ops, KVM_DEV_TYPE_LOONGARCH_EIOINTC); 1060 } 1061