1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/bitops.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/riscv-imsic.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kvm_host.h> 16 #include <linux/percpu.h> 17 #include <linux/spinlock.h> 18 #include <asm/cpufeature.h> 19 #include <asm/kvm_nacl.h> 20 21 struct aia_hgei_control { 22 raw_spinlock_t lock; 23 unsigned long free_bitmap; 24 struct kvm_vcpu *owners[BITS_PER_LONG]; 25 }; 26 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); 27 static int hgei_parent_irq; 28 29 unsigned int kvm_riscv_aia_nr_hgei; 30 unsigned int kvm_riscv_aia_max_ids; 31 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); 32 33 static inline unsigned long aia_hvictl_value(bool ext_irq_pending) 34 { 35 unsigned long hvictl; 36 37 /* 38 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents 39 * no interrupt in HVICTL. 40 */ 41 42 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; 43 hvictl |= ext_irq_pending; 44 return hvictl; 45 } 46 47 #ifdef CONFIG_32BIT 48 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) 49 { 50 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 51 unsigned long mask, val; 52 53 if (!kvm_riscv_aia_available()) 54 return; 55 56 if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { 57 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); 58 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; 59 60 csr->hviph &= ~mask; 61 csr->hviph |= val; 62 } 63 } 64 65 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) 66 { 67 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 68 69 if (kvm_riscv_aia_available()) 70 csr->vsieh = ncsr_read(CSR_VSIEH); 71 } 72 #endif 73 74 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 75 { 76 unsigned long seip; 77 78 if (!kvm_riscv_aia_available()) 79 return false; 80 81 #ifdef CONFIG_32BIT 82 if (READ_ONCE(vcpu->arch.irqs_pending[1]) & 83 (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) 84 return true; 85 #endif 86 87 seip = vcpu->arch.guest_csr.vsie; 88 seip &= (unsigned long)mask; 89 seip &= BIT(IRQ_S_EXT); 90 91 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) 92 return false; 93 94 return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu); 95 } 96 97 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) 98 { 99 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 100 101 if (!kvm_riscv_aia_available()) 102 return; 103 104 #ifdef CONFIG_32BIT 105 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); 106 #endif 107 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT)))); 108 } 109 110 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) 111 { 112 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 113 void *nsh; 114 115 if (!kvm_riscv_aia_available()) 116 return; 117 118 if (kvm_riscv_nacl_sync_csr_available()) { 119 nsh = nacl_shmem(); 120 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect); 121 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1); 122 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2); 123 #ifdef CONFIG_32BIT 124 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh); 125 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph); 126 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h); 127 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h); 128 #endif 129 } else { 130 csr_write(CSR_VSISELECT, csr->vsiselect); 131 csr_write(CSR_HVIPRIO1, csr->hviprio1); 132 csr_write(CSR_HVIPRIO2, csr->hviprio2); 133 #ifdef CONFIG_32BIT 134 csr_write(CSR_VSIEH, csr->vsieh); 135 csr_write(CSR_HVIPH, csr->hviph); 136 csr_write(CSR_HVIPRIO1H, csr->hviprio1h); 137 csr_write(CSR_HVIPRIO2H, csr->hviprio2h); 138 #endif 139 } 140 141 if (kvm_riscv_aia_initialized(vcpu->kvm)) 142 kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu); 143 } 144 145 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) 146 { 147 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 148 void *nsh; 149 150 if (!kvm_riscv_aia_available()) 151 return; 152 153 if (kvm_riscv_aia_initialized(vcpu->kvm)) 154 kvm_riscv_vcpu_aia_imsic_put(vcpu); 155 156 if (kvm_riscv_nacl_available()) { 157 nsh = nacl_shmem(); 158 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); 159 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1); 160 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2); 161 #ifdef CONFIG_32BIT 162 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH); 163 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH); 164 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H); 165 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H); 166 #endif 167 } else { 168 csr->vsiselect = csr_read(CSR_VSISELECT); 169 csr->hviprio1 = csr_read(CSR_HVIPRIO1); 170 csr->hviprio2 = csr_read(CSR_HVIPRIO2); 171 #ifdef CONFIG_32BIT 172 csr->vsieh = csr_read(CSR_VSIEH); 173 csr->hviph = csr_read(CSR_HVIPH); 174 csr->hviprio1h = csr_read(CSR_HVIPRIO1H); 175 csr->hviprio2h = csr_read(CSR_HVIPRIO2H); 176 #endif 177 } 178 } 179 180 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, 181 unsigned long reg_num, 182 unsigned long *out_val) 183 { 184 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 185 186 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) 187 return -ENOENT; 188 189 *out_val = 0; 190 if (kvm_riscv_aia_available()) 191 *out_val = ((unsigned long *)csr)[reg_num]; 192 193 return 0; 194 } 195 196 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, 197 unsigned long reg_num, 198 unsigned long val) 199 { 200 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 201 202 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) 203 return -ENOENT; 204 205 if (kvm_riscv_aia_available()) { 206 ((unsigned long *)csr)[reg_num] = val; 207 208 #ifdef CONFIG_32BIT 209 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) 210 WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); 211 #endif 212 } 213 214 return 0; 215 } 216 217 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, 218 unsigned int csr_num, 219 unsigned long *val, 220 unsigned long new_val, 221 unsigned long wr_mask) 222 { 223 /* If AIA not available then redirect trap */ 224 if (!kvm_riscv_aia_available()) 225 return KVM_INSN_ILLEGAL_TRAP; 226 227 /* If AIA not initialized then forward to user space */ 228 if (!kvm_riscv_aia_initialized(vcpu->kvm)) 229 return KVM_INSN_EXIT_TO_USER_SPACE; 230 231 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, 232 val, new_val, wr_mask); 233 } 234 235 /* 236 * External IRQ priority always read-only zero. This means default 237 * priority order is always preferred for external IRQs unless 238 * HVICTL.IID == 9 and HVICTL.IPRIO != 0 239 */ 240 static int aia_irq2bitpos[] = { 241 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ 242 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ 243 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ 244 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ 245 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ 246 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ 247 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ 248 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ 249 }; 250 251 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) 252 { 253 unsigned long hviprio; 254 int bitpos = aia_irq2bitpos[irq]; 255 256 if (bitpos < 0) 257 return 0; 258 259 switch (bitpos / BITS_PER_LONG) { 260 case 0: 261 hviprio = ncsr_read(CSR_HVIPRIO1); 262 break; 263 case 1: 264 #ifndef CONFIG_32BIT 265 hviprio = ncsr_read(CSR_HVIPRIO2); 266 break; 267 #else 268 hviprio = ncsr_read(CSR_HVIPRIO1H); 269 break; 270 case 2: 271 hviprio = ncsr_read(CSR_HVIPRIO2); 272 break; 273 case 3: 274 hviprio = ncsr_read(CSR_HVIPRIO2H); 275 break; 276 #endif 277 default: 278 return 0; 279 } 280 281 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; 282 } 283 284 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) 285 { 286 unsigned long hviprio; 287 int bitpos = aia_irq2bitpos[irq]; 288 289 if (bitpos < 0) 290 return; 291 292 switch (bitpos / BITS_PER_LONG) { 293 case 0: 294 hviprio = ncsr_read(CSR_HVIPRIO1); 295 break; 296 case 1: 297 #ifndef CONFIG_32BIT 298 hviprio = ncsr_read(CSR_HVIPRIO2); 299 break; 300 #else 301 hviprio = ncsr_read(CSR_HVIPRIO1H); 302 break; 303 case 2: 304 hviprio = ncsr_read(CSR_HVIPRIO2); 305 break; 306 case 3: 307 hviprio = ncsr_read(CSR_HVIPRIO2H); 308 break; 309 #endif 310 default: 311 return; 312 } 313 314 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); 315 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); 316 317 switch (bitpos / BITS_PER_LONG) { 318 case 0: 319 ncsr_write(CSR_HVIPRIO1, hviprio); 320 break; 321 case 1: 322 #ifndef CONFIG_32BIT 323 ncsr_write(CSR_HVIPRIO2, hviprio); 324 break; 325 #else 326 ncsr_write(CSR_HVIPRIO1H, hviprio); 327 break; 328 case 2: 329 ncsr_write(CSR_HVIPRIO2, hviprio); 330 break; 331 case 3: 332 ncsr_write(CSR_HVIPRIO2H, hviprio); 333 break; 334 #endif 335 default: 336 return; 337 } 338 } 339 340 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, 341 unsigned long *val, unsigned long new_val, 342 unsigned long wr_mask) 343 { 344 int i, first_irq, nirqs; 345 unsigned long old_val; 346 u8 prio; 347 348 #ifndef CONFIG_32BIT 349 if (isel & 0x1) 350 return KVM_INSN_ILLEGAL_TRAP; 351 #endif 352 353 nirqs = 4 * (BITS_PER_LONG / 32); 354 first_irq = (isel - ISELECT_IPRIO0) * 4; 355 356 old_val = 0; 357 for (i = 0; i < nirqs; i++) { 358 prio = aia_get_iprio8(vcpu, first_irq + i); 359 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); 360 } 361 362 if (val) 363 *val = old_val; 364 365 if (wr_mask) { 366 new_val = (old_val & ~wr_mask) | (new_val & wr_mask); 367 for (i = 0; i < nirqs; i++) { 368 prio = (new_val >> (TOPI_IPRIO_BITS * i)) & 369 TOPI_IPRIO_MASK; 370 aia_set_iprio8(vcpu, first_irq + i, prio); 371 } 372 } 373 374 return KVM_INSN_CONTINUE_NEXT_SEPC; 375 } 376 377 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, 378 unsigned long *val, unsigned long new_val, 379 unsigned long wr_mask) 380 { 381 unsigned int isel; 382 383 /* If AIA not available then redirect trap */ 384 if (!kvm_riscv_aia_available()) 385 return KVM_INSN_ILLEGAL_TRAP; 386 387 /* First try to emulate in kernel space */ 388 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; 389 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) 390 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); 391 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && 392 kvm_riscv_aia_initialized(vcpu->kvm)) 393 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, 394 wr_mask); 395 396 /* We can't handle it here so redirect to user space */ 397 return KVM_INSN_EXIT_TO_USER_SPACE; 398 } 399 400 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, 401 void __iomem **hgei_va, phys_addr_t *hgei_pa) 402 { 403 int ret = -ENOENT; 404 unsigned long flags; 405 const struct imsic_global_config *gc; 406 const struct imsic_local_config *lc; 407 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 408 409 if (!kvm_riscv_aia_available() || !hgctrl) 410 return -ENODEV; 411 412 raw_spin_lock_irqsave(&hgctrl->lock, flags); 413 414 if (hgctrl->free_bitmap) { 415 ret = __ffs(hgctrl->free_bitmap); 416 hgctrl->free_bitmap &= ~BIT(ret); 417 hgctrl->owners[ret] = owner; 418 } 419 420 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 421 422 gc = imsic_get_global_config(); 423 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; 424 if (lc && ret > 0) { 425 if (hgei_va) 426 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); 427 if (hgei_pa) 428 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); 429 } 430 431 return ret; 432 } 433 434 void kvm_riscv_aia_free_hgei(int cpu, int hgei) 435 { 436 unsigned long flags; 437 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 438 439 if (!kvm_riscv_aia_available() || !hgctrl) 440 return; 441 442 raw_spin_lock_irqsave(&hgctrl->lock, flags); 443 444 if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) { 445 if (!(hgctrl->free_bitmap & BIT(hgei))) { 446 hgctrl->free_bitmap |= BIT(hgei); 447 hgctrl->owners[hgei] = NULL; 448 } 449 } 450 451 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 452 } 453 454 static irqreturn_t hgei_interrupt(int irq, void *dev_id) 455 { 456 int i; 457 unsigned long hgei_mask, flags; 458 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 459 460 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); 461 csr_clear(CSR_HGEIE, hgei_mask); 462 463 raw_spin_lock_irqsave(&hgctrl->lock, flags); 464 465 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { 466 if (hgctrl->owners[i]) 467 kvm_vcpu_kick(hgctrl->owners[i]); 468 } 469 470 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 471 472 put_cpu_ptr(&aia_hgei); 473 return IRQ_HANDLED; 474 } 475 476 static int aia_hgei_init(void) 477 { 478 int cpu, rc; 479 struct irq_domain *domain; 480 struct aia_hgei_control *hgctrl; 481 482 /* Initialize per-CPU guest external interrupt line management */ 483 for_each_possible_cpu(cpu) { 484 hgctrl = per_cpu_ptr(&aia_hgei, cpu); 485 raw_spin_lock_init(&hgctrl->lock); 486 if (kvm_riscv_aia_nr_hgei) { 487 hgctrl->free_bitmap = 488 BIT(kvm_riscv_aia_nr_hgei + 1) - 1; 489 hgctrl->free_bitmap &= ~BIT(0); 490 } else 491 hgctrl->free_bitmap = 0; 492 } 493 494 /* Skip SGEI interrupt setup for zero guest external interrupts */ 495 if (!kvm_riscv_aia_nr_hgei) 496 goto skip_sgei_interrupt; 497 498 /* Find INTC irq domain */ 499 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 500 DOMAIN_BUS_ANY); 501 if (!domain) { 502 kvm_err("unable to find INTC domain\n"); 503 return -ENOENT; 504 } 505 506 /* Map per-CPU SGEI interrupt from INTC domain */ 507 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); 508 if (!hgei_parent_irq) { 509 kvm_err("unable to map SGEI IRQ\n"); 510 return -ENOMEM; 511 } 512 513 /* Request per-CPU SGEI interrupt */ 514 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, 515 "riscv-kvm", &aia_hgei); 516 if (rc) { 517 kvm_err("failed to request SGEI IRQ\n"); 518 return rc; 519 } 520 521 skip_sgei_interrupt: 522 return 0; 523 } 524 525 static void aia_hgei_exit(void) 526 { 527 /* Do nothing for zero guest external interrupts */ 528 if (!kvm_riscv_aia_nr_hgei) 529 return; 530 531 /* Free per-CPU SGEI interrupt */ 532 free_percpu_irq(hgei_parent_irq, &aia_hgei); 533 } 534 535 void kvm_riscv_aia_enable(void) 536 { 537 if (!kvm_riscv_aia_available()) 538 return; 539 540 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 541 csr_write(CSR_HVIPRIO1, 0x0); 542 csr_write(CSR_HVIPRIO2, 0x0); 543 #ifdef CONFIG_32BIT 544 csr_write(CSR_HVIPH, 0x0); 545 csr_write(CSR_HIDELEGH, 0x0); 546 csr_write(CSR_HVIPRIO1H, 0x0); 547 csr_write(CSR_HVIPRIO2H, 0x0); 548 #endif 549 550 /* Enable per-CPU SGEI interrupt */ 551 enable_percpu_irq(hgei_parent_irq, 552 irq_get_trigger_type(hgei_parent_irq)); 553 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 554 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 555 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 556 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 557 } 558 559 void kvm_riscv_aia_disable(void) 560 { 561 int i; 562 unsigned long flags; 563 struct kvm_vcpu *vcpu; 564 struct aia_hgei_control *hgctrl; 565 566 if (!kvm_riscv_aia_available()) 567 return; 568 hgctrl = get_cpu_ptr(&aia_hgei); 569 570 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 571 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 572 /* Disable per-CPU SGEI interrupt */ 573 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); 574 disable_percpu_irq(hgei_parent_irq); 575 576 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 577 578 raw_spin_lock_irqsave(&hgctrl->lock, flags); 579 580 for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) { 581 vcpu = hgctrl->owners[i]; 582 if (!vcpu) 583 continue; 584 585 /* 586 * We release hgctrl->lock before notifying IMSIC 587 * so that we don't have lock ordering issues. 588 */ 589 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 590 591 /* Notify IMSIC */ 592 kvm_riscv_vcpu_aia_imsic_release(vcpu); 593 594 /* 595 * Wakeup VCPU if it was blocked so that it can 596 * run on other HARTs 597 */ 598 if (csr_read(CSR_HGEIE) & BIT(i)) { 599 csr_clear(CSR_HGEIE, BIT(i)); 600 kvm_vcpu_kick(vcpu); 601 } 602 603 raw_spin_lock_irqsave(&hgctrl->lock, flags); 604 } 605 606 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 607 608 put_cpu_ptr(&aia_hgei); 609 } 610 611 int kvm_riscv_aia_init(void) 612 { 613 int rc; 614 const struct imsic_global_config *gc; 615 616 if (!riscv_isa_extension_available(NULL, SxAIA)) 617 return -ENODEV; 618 gc = imsic_get_global_config(); 619 620 /* Figure-out number of bits in HGEIE */ 621 csr_write(CSR_HGEIE, -1UL); 622 kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE)); 623 csr_write(CSR_HGEIE, 0); 624 if (kvm_riscv_aia_nr_hgei) 625 kvm_riscv_aia_nr_hgei--; 626 627 /* 628 * Number of usable HGEI lines should be minimum of per-HART 629 * IMSIC guest files and number of bits in HGEIE 630 */ 631 if (gc) 632 kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei, 633 BIT(gc->guest_index_bits) - 1); 634 else 635 kvm_riscv_aia_nr_hgei = 0; 636 637 /* Find number of guest MSI IDs */ 638 kvm_riscv_aia_max_ids = IMSIC_MAX_ID; 639 if (gc && kvm_riscv_aia_nr_hgei) 640 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; 641 642 /* Initialize guest external interrupt line management */ 643 rc = aia_hgei_init(); 644 if (rc) 645 return rc; 646 647 /* Register device operations */ 648 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, 649 KVM_DEV_TYPE_RISCV_AIA); 650 if (rc) { 651 aia_hgei_exit(); 652 return rc; 653 } 654 655 /* Enable KVM AIA support */ 656 static_branch_enable(&kvm_riscv_aia_available); 657 658 return 0; 659 } 660 661 void kvm_riscv_aia_exit(void) 662 { 663 if (!kvm_riscv_aia_available()) 664 return; 665 666 /* Unregister device operations */ 667 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); 668 669 /* Cleanup the HGEI state */ 670 aia_hgei_exit(); 671 } 672