1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/bitops.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/riscv-imsic.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kvm_host.h> 16 #include <linux/percpu.h> 17 #include <linux/spinlock.h> 18 #include <asm/cpufeature.h> 19 #include <asm/kvm_nacl.h> 20 21 struct aia_hgei_control { 22 raw_spinlock_t lock; 23 unsigned long free_bitmap; 24 struct kvm_vcpu *owners[BITS_PER_LONG]; 25 }; 26 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); 27 static int hgei_parent_irq; 28 29 unsigned int kvm_riscv_aia_nr_hgei; 30 unsigned int kvm_riscv_aia_max_ids; 31 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); 32 33 static int aia_find_hgei(struct kvm_vcpu *owner) 34 { 35 int i, hgei; 36 unsigned long flags; 37 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 38 39 raw_spin_lock_irqsave(&hgctrl->lock, flags); 40 41 hgei = -1; 42 for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) { 43 if (hgctrl->owners[i] == owner) { 44 hgei = i; 45 break; 46 } 47 } 48 49 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 50 51 put_cpu_ptr(&aia_hgei); 52 return hgei; 53 } 54 55 static inline unsigned long aia_hvictl_value(bool ext_irq_pending) 56 { 57 unsigned long hvictl; 58 59 /* 60 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents 61 * no interrupt in HVICTL. 62 */ 63 64 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; 65 hvictl |= ext_irq_pending; 66 return hvictl; 67 } 68 69 #ifdef CONFIG_32BIT 70 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) 71 { 72 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 73 unsigned long mask, val; 74 75 if (!kvm_riscv_aia_available()) 76 return; 77 78 if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { 79 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); 80 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; 81 82 csr->hviph &= ~mask; 83 csr->hviph |= val; 84 } 85 } 86 87 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) 88 { 89 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 90 91 if (kvm_riscv_aia_available()) 92 csr->vsieh = ncsr_read(CSR_VSIEH); 93 } 94 #endif 95 96 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 97 { 98 int hgei; 99 unsigned long seip; 100 101 if (!kvm_riscv_aia_available()) 102 return false; 103 104 #ifdef CONFIG_32BIT 105 if (READ_ONCE(vcpu->arch.irqs_pending[1]) & 106 (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) 107 return true; 108 #endif 109 110 seip = vcpu->arch.guest_csr.vsie; 111 seip &= (unsigned long)mask; 112 seip &= BIT(IRQ_S_EXT); 113 114 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) 115 return false; 116 117 hgei = aia_find_hgei(vcpu); 118 if (hgei > 0) 119 return !!(ncsr_read(CSR_HGEIP) & BIT(hgei)); 120 121 return false; 122 } 123 124 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) 125 { 126 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 127 128 if (!kvm_riscv_aia_available()) 129 return; 130 131 #ifdef CONFIG_32BIT 132 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); 133 #endif 134 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT)))); 135 } 136 137 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) 138 { 139 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 140 void *nsh; 141 142 if (!kvm_riscv_aia_available()) 143 return; 144 145 if (kvm_riscv_nacl_sync_csr_available()) { 146 nsh = nacl_shmem(); 147 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect); 148 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1); 149 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2); 150 #ifdef CONFIG_32BIT 151 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh); 152 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph); 153 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h); 154 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h); 155 #endif 156 } else { 157 csr_write(CSR_VSISELECT, csr->vsiselect); 158 csr_write(CSR_HVIPRIO1, csr->hviprio1); 159 csr_write(CSR_HVIPRIO2, csr->hviprio2); 160 #ifdef CONFIG_32BIT 161 csr_write(CSR_VSIEH, csr->vsieh); 162 csr_write(CSR_HVIPH, csr->hviph); 163 csr_write(CSR_HVIPRIO1H, csr->hviprio1h); 164 csr_write(CSR_HVIPRIO2H, csr->hviprio2h); 165 #endif 166 } 167 } 168 169 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) 170 { 171 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 172 void *nsh; 173 174 if (!kvm_riscv_aia_available()) 175 return; 176 177 if (kvm_riscv_nacl_available()) { 178 nsh = nacl_shmem(); 179 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); 180 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1); 181 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2); 182 #ifdef CONFIG_32BIT 183 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH); 184 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH); 185 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H); 186 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H); 187 #endif 188 } else { 189 csr->vsiselect = csr_read(CSR_VSISELECT); 190 csr->hviprio1 = csr_read(CSR_HVIPRIO1); 191 csr->hviprio2 = csr_read(CSR_HVIPRIO2); 192 #ifdef CONFIG_32BIT 193 csr->vsieh = csr_read(CSR_VSIEH); 194 csr->hviph = csr_read(CSR_HVIPH); 195 csr->hviprio1h = csr_read(CSR_HVIPRIO1H); 196 csr->hviprio2h = csr_read(CSR_HVIPRIO2H); 197 #endif 198 } 199 } 200 201 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, 202 unsigned long reg_num, 203 unsigned long *out_val) 204 { 205 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 206 207 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) 208 return -ENOENT; 209 210 *out_val = 0; 211 if (kvm_riscv_aia_available()) 212 *out_val = ((unsigned long *)csr)[reg_num]; 213 214 return 0; 215 } 216 217 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, 218 unsigned long reg_num, 219 unsigned long val) 220 { 221 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 222 223 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) 224 return -ENOENT; 225 226 if (kvm_riscv_aia_available()) { 227 ((unsigned long *)csr)[reg_num] = val; 228 229 #ifdef CONFIG_32BIT 230 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) 231 WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); 232 #endif 233 } 234 235 return 0; 236 } 237 238 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, 239 unsigned int csr_num, 240 unsigned long *val, 241 unsigned long new_val, 242 unsigned long wr_mask) 243 { 244 /* If AIA not available then redirect trap */ 245 if (!kvm_riscv_aia_available()) 246 return KVM_INSN_ILLEGAL_TRAP; 247 248 /* If AIA not initialized then forward to user space */ 249 if (!kvm_riscv_aia_initialized(vcpu->kvm)) 250 return KVM_INSN_EXIT_TO_USER_SPACE; 251 252 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, 253 val, new_val, wr_mask); 254 } 255 256 /* 257 * External IRQ priority always read-only zero. This means default 258 * priority order is always preferred for external IRQs unless 259 * HVICTL.IID == 9 and HVICTL.IPRIO != 0 260 */ 261 static int aia_irq2bitpos[] = { 262 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ 263 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ 264 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ 265 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ 266 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ 267 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ 268 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ 269 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ 270 }; 271 272 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) 273 { 274 unsigned long hviprio; 275 int bitpos = aia_irq2bitpos[irq]; 276 277 if (bitpos < 0) 278 return 0; 279 280 switch (bitpos / BITS_PER_LONG) { 281 case 0: 282 hviprio = ncsr_read(CSR_HVIPRIO1); 283 break; 284 case 1: 285 #ifndef CONFIG_32BIT 286 hviprio = ncsr_read(CSR_HVIPRIO2); 287 break; 288 #else 289 hviprio = ncsr_read(CSR_HVIPRIO1H); 290 break; 291 case 2: 292 hviprio = ncsr_read(CSR_HVIPRIO2); 293 break; 294 case 3: 295 hviprio = ncsr_read(CSR_HVIPRIO2H); 296 break; 297 #endif 298 default: 299 return 0; 300 } 301 302 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; 303 } 304 305 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) 306 { 307 unsigned long hviprio; 308 int bitpos = aia_irq2bitpos[irq]; 309 310 if (bitpos < 0) 311 return; 312 313 switch (bitpos / BITS_PER_LONG) { 314 case 0: 315 hviprio = ncsr_read(CSR_HVIPRIO1); 316 break; 317 case 1: 318 #ifndef CONFIG_32BIT 319 hviprio = ncsr_read(CSR_HVIPRIO2); 320 break; 321 #else 322 hviprio = ncsr_read(CSR_HVIPRIO1H); 323 break; 324 case 2: 325 hviprio = ncsr_read(CSR_HVIPRIO2); 326 break; 327 case 3: 328 hviprio = ncsr_read(CSR_HVIPRIO2H); 329 break; 330 #endif 331 default: 332 return; 333 } 334 335 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); 336 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); 337 338 switch (bitpos / BITS_PER_LONG) { 339 case 0: 340 ncsr_write(CSR_HVIPRIO1, hviprio); 341 break; 342 case 1: 343 #ifndef CONFIG_32BIT 344 ncsr_write(CSR_HVIPRIO2, hviprio); 345 break; 346 #else 347 ncsr_write(CSR_HVIPRIO1H, hviprio); 348 break; 349 case 2: 350 ncsr_write(CSR_HVIPRIO2, hviprio); 351 break; 352 case 3: 353 ncsr_write(CSR_HVIPRIO2H, hviprio); 354 break; 355 #endif 356 default: 357 return; 358 } 359 } 360 361 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, 362 unsigned long *val, unsigned long new_val, 363 unsigned long wr_mask) 364 { 365 int i, first_irq, nirqs; 366 unsigned long old_val; 367 u8 prio; 368 369 #ifndef CONFIG_32BIT 370 if (isel & 0x1) 371 return KVM_INSN_ILLEGAL_TRAP; 372 #endif 373 374 nirqs = 4 * (BITS_PER_LONG / 32); 375 first_irq = (isel - ISELECT_IPRIO0) * 4; 376 377 old_val = 0; 378 for (i = 0; i < nirqs; i++) { 379 prio = aia_get_iprio8(vcpu, first_irq + i); 380 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); 381 } 382 383 if (val) 384 *val = old_val; 385 386 if (wr_mask) { 387 new_val = (old_val & ~wr_mask) | (new_val & wr_mask); 388 for (i = 0; i < nirqs; i++) { 389 prio = (new_val >> (TOPI_IPRIO_BITS * i)) & 390 TOPI_IPRIO_MASK; 391 aia_set_iprio8(vcpu, first_irq + i, prio); 392 } 393 } 394 395 return KVM_INSN_CONTINUE_NEXT_SEPC; 396 } 397 398 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, 399 unsigned long *val, unsigned long new_val, 400 unsigned long wr_mask) 401 { 402 unsigned int isel; 403 404 /* If AIA not available then redirect trap */ 405 if (!kvm_riscv_aia_available()) 406 return KVM_INSN_ILLEGAL_TRAP; 407 408 /* First try to emulate in kernel space */ 409 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; 410 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) 411 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); 412 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && 413 kvm_riscv_aia_initialized(vcpu->kvm)) 414 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, 415 wr_mask); 416 417 /* We can't handle it here so redirect to user space */ 418 return KVM_INSN_EXIT_TO_USER_SPACE; 419 } 420 421 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, 422 void __iomem **hgei_va, phys_addr_t *hgei_pa) 423 { 424 int ret = -ENOENT; 425 unsigned long flags; 426 const struct imsic_global_config *gc; 427 const struct imsic_local_config *lc; 428 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 429 430 if (!kvm_riscv_aia_available() || !hgctrl) 431 return -ENODEV; 432 433 raw_spin_lock_irqsave(&hgctrl->lock, flags); 434 435 if (hgctrl->free_bitmap) { 436 ret = __ffs(hgctrl->free_bitmap); 437 hgctrl->free_bitmap &= ~BIT(ret); 438 hgctrl->owners[ret] = owner; 439 } 440 441 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 442 443 gc = imsic_get_global_config(); 444 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; 445 if (lc && ret > 0) { 446 if (hgei_va) 447 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); 448 if (hgei_pa) 449 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); 450 } 451 452 return ret; 453 } 454 455 void kvm_riscv_aia_free_hgei(int cpu, int hgei) 456 { 457 unsigned long flags; 458 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 459 460 if (!kvm_riscv_aia_available() || !hgctrl) 461 return; 462 463 raw_spin_lock_irqsave(&hgctrl->lock, flags); 464 465 if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) { 466 if (!(hgctrl->free_bitmap & BIT(hgei))) { 467 hgctrl->free_bitmap |= BIT(hgei); 468 hgctrl->owners[hgei] = NULL; 469 } 470 } 471 472 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 473 } 474 475 void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable) 476 { 477 int hgei; 478 479 if (!kvm_riscv_aia_available()) 480 return; 481 482 hgei = aia_find_hgei(owner); 483 if (hgei > 0) { 484 if (enable) 485 csr_set(CSR_HGEIE, BIT(hgei)); 486 else 487 csr_clear(CSR_HGEIE, BIT(hgei)); 488 } 489 } 490 491 static irqreturn_t hgei_interrupt(int irq, void *dev_id) 492 { 493 int i; 494 unsigned long hgei_mask, flags; 495 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 496 497 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); 498 csr_clear(CSR_HGEIE, hgei_mask); 499 500 raw_spin_lock_irqsave(&hgctrl->lock, flags); 501 502 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { 503 if (hgctrl->owners[i]) 504 kvm_vcpu_kick(hgctrl->owners[i]); 505 } 506 507 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 508 509 put_cpu_ptr(&aia_hgei); 510 return IRQ_HANDLED; 511 } 512 513 static int aia_hgei_init(void) 514 { 515 int cpu, rc; 516 struct irq_domain *domain; 517 struct aia_hgei_control *hgctrl; 518 519 /* Initialize per-CPU guest external interrupt line management */ 520 for_each_possible_cpu(cpu) { 521 hgctrl = per_cpu_ptr(&aia_hgei, cpu); 522 raw_spin_lock_init(&hgctrl->lock); 523 if (kvm_riscv_aia_nr_hgei) { 524 hgctrl->free_bitmap = 525 BIT(kvm_riscv_aia_nr_hgei + 1) - 1; 526 hgctrl->free_bitmap &= ~BIT(0); 527 } else 528 hgctrl->free_bitmap = 0; 529 } 530 531 /* Skip SGEI interrupt setup for zero guest external interrupts */ 532 if (!kvm_riscv_aia_nr_hgei) 533 goto skip_sgei_interrupt; 534 535 /* Find INTC irq domain */ 536 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 537 DOMAIN_BUS_ANY); 538 if (!domain) { 539 kvm_err("unable to find INTC domain\n"); 540 return -ENOENT; 541 } 542 543 /* Map per-CPU SGEI interrupt from INTC domain */ 544 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); 545 if (!hgei_parent_irq) { 546 kvm_err("unable to map SGEI IRQ\n"); 547 return -ENOMEM; 548 } 549 550 /* Request per-CPU SGEI interrupt */ 551 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, 552 "riscv-kvm", &aia_hgei); 553 if (rc) { 554 kvm_err("failed to request SGEI IRQ\n"); 555 return rc; 556 } 557 558 skip_sgei_interrupt: 559 return 0; 560 } 561 562 static void aia_hgei_exit(void) 563 { 564 /* Do nothing for zero guest external interrupts */ 565 if (!kvm_riscv_aia_nr_hgei) 566 return; 567 568 /* Free per-CPU SGEI interrupt */ 569 free_percpu_irq(hgei_parent_irq, &aia_hgei); 570 } 571 572 void kvm_riscv_aia_enable(void) 573 { 574 if (!kvm_riscv_aia_available()) 575 return; 576 577 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 578 csr_write(CSR_HVIPRIO1, 0x0); 579 csr_write(CSR_HVIPRIO2, 0x0); 580 #ifdef CONFIG_32BIT 581 csr_write(CSR_HVIPH, 0x0); 582 csr_write(CSR_HIDELEGH, 0x0); 583 csr_write(CSR_HVIPRIO1H, 0x0); 584 csr_write(CSR_HVIPRIO2H, 0x0); 585 #endif 586 587 /* Enable per-CPU SGEI interrupt */ 588 enable_percpu_irq(hgei_parent_irq, 589 irq_get_trigger_type(hgei_parent_irq)); 590 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 591 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 592 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 593 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 594 } 595 596 void kvm_riscv_aia_disable(void) 597 { 598 int i; 599 unsigned long flags; 600 struct kvm_vcpu *vcpu; 601 struct aia_hgei_control *hgctrl; 602 603 if (!kvm_riscv_aia_available()) 604 return; 605 hgctrl = get_cpu_ptr(&aia_hgei); 606 607 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 608 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 609 /* Disable per-CPU SGEI interrupt */ 610 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); 611 disable_percpu_irq(hgei_parent_irq); 612 613 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 614 615 raw_spin_lock_irqsave(&hgctrl->lock, flags); 616 617 for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) { 618 vcpu = hgctrl->owners[i]; 619 if (!vcpu) 620 continue; 621 622 /* 623 * We release hgctrl->lock before notifying IMSIC 624 * so that we don't have lock ordering issues. 625 */ 626 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 627 628 /* Notify IMSIC */ 629 kvm_riscv_vcpu_aia_imsic_release(vcpu); 630 631 /* 632 * Wakeup VCPU if it was blocked so that it can 633 * run on other HARTs 634 */ 635 if (csr_read(CSR_HGEIE) & BIT(i)) { 636 csr_clear(CSR_HGEIE, BIT(i)); 637 kvm_vcpu_kick(vcpu); 638 } 639 640 raw_spin_lock_irqsave(&hgctrl->lock, flags); 641 } 642 643 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 644 645 put_cpu_ptr(&aia_hgei); 646 } 647 648 int kvm_riscv_aia_init(void) 649 { 650 int rc; 651 const struct imsic_global_config *gc; 652 653 if (!riscv_isa_extension_available(NULL, SxAIA)) 654 return -ENODEV; 655 gc = imsic_get_global_config(); 656 657 /* Figure-out number of bits in HGEIE */ 658 csr_write(CSR_HGEIE, -1UL); 659 kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE)); 660 csr_write(CSR_HGEIE, 0); 661 if (kvm_riscv_aia_nr_hgei) 662 kvm_riscv_aia_nr_hgei--; 663 664 /* 665 * Number of usable HGEI lines should be minimum of per-HART 666 * IMSIC guest files and number of bits in HGEIE 667 */ 668 if (gc) 669 kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei, 670 BIT(gc->guest_index_bits) - 1); 671 else 672 kvm_riscv_aia_nr_hgei = 0; 673 674 /* Find number of guest MSI IDs */ 675 kvm_riscv_aia_max_ids = IMSIC_MAX_ID; 676 if (gc && kvm_riscv_aia_nr_hgei) 677 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; 678 679 /* Initialize guest external interrupt line management */ 680 rc = aia_hgei_init(); 681 if (rc) 682 return rc; 683 684 /* Register device operations */ 685 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, 686 KVM_DEV_TYPE_RISCV_AIA); 687 if (rc) { 688 aia_hgei_exit(); 689 return rc; 690 } 691 692 /* Enable KVM AIA support */ 693 static_branch_enable(&kvm_riscv_aia_available); 694 695 return 0; 696 } 697 698 void kvm_riscv_aia_exit(void) 699 { 700 if (!kvm_riscv_aia_available()) 701 return; 702 703 /* Unregister device operations */ 704 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); 705 706 /* Cleanup the HGEI state */ 707 aia_hgei_exit(); 708 } 709