1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/bitops.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/riscv-imsic.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kvm_host.h> 16 #include <linux/nospec.h> 17 #include <linux/percpu.h> 18 #include <linux/spinlock.h> 19 #include <asm/cpufeature.h> 20 #include <asm/kvm_nacl.h> 21 22 struct aia_hgei_control { 23 raw_spinlock_t lock; 24 bool free_bitmap_initialized; 25 unsigned long free_bitmap; 26 struct kvm_vcpu *owners[BITS_PER_LONG]; 27 unsigned int nr_hgei; 28 unsigned long saved_hgeie; 29 }; 30 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); 31 static int hgei_parent_irq; 32 33 atomic_t kvm_riscv_aia_nr_hgei; 34 unsigned int kvm_riscv_aia_max_ids; 35 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); 36 37 static inline unsigned long aia_hvictl_value(bool ext_irq_pending) 38 { 39 unsigned long hvictl; 40 41 /* 42 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents 43 * no interrupt in HVICTL. 44 */ 45 46 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; 47 hvictl |= ext_irq_pending; 48 return hvictl; 49 } 50 51 #ifdef CONFIG_32BIT 52 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) 53 { 54 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 55 unsigned long mask, val; 56 57 lockdep_assert_held(&vcpu->arch.irqs_pending_lock); 58 59 if (!kvm_riscv_aia_available()) 60 return; 61 62 mask = vcpu->arch.irqs_pending_mask[1]; 63 if (mask) { 64 vcpu->arch.irqs_pending_mask[1] = 0; 65 val = vcpu->arch.irqs_pending[1] & mask; 66 67 csr->hviph &= ~mask; 68 csr->hviph |= val; 69 } 70 } 71 72 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) 73 { 74 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 75 76 lockdep_assert_held(&vcpu->arch.irqs_pending_lock); 77 78 if (kvm_riscv_aia_available()) 79 csr->vsieh = ncsr_read(CSR_VSIEH); 80 } 81 #endif 82 83 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 84 { 85 unsigned long seip; 86 #ifdef CONFIG_32BIT 87 unsigned long flags; 88 bool pending; 89 #endif 90 91 if (!kvm_riscv_aia_available()) 92 return false; 93 94 #ifdef CONFIG_32BIT 95 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 96 pending = vcpu->arch.irqs_pending[1] & 97 (vcpu->arch.aia_context.guest_csr.vsieh & 98 upper_32_bits(mask)); 99 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); 100 101 if (pending) 102 return true; 103 #endif 104 105 seip = vcpu->arch.guest_csr.vsie; 106 seip &= (unsigned long)mask; 107 seip &= BIT(IRQ_S_EXT); 108 109 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) 110 return false; 111 112 return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu); 113 } 114 115 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) 116 { 117 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 118 119 if (!kvm_riscv_aia_available()) 120 return; 121 122 #ifdef CONFIG_32BIT 123 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); 124 #endif 125 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT)))); 126 } 127 128 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) 129 { 130 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 131 void *nsh; 132 133 if (!kvm_riscv_aia_available()) 134 return; 135 136 if (kvm_riscv_nacl_sync_csr_available()) { 137 nsh = nacl_shmem(); 138 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect); 139 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1); 140 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2); 141 #ifdef CONFIG_32BIT 142 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh); 143 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph); 144 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h); 145 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h); 146 #endif 147 } else { 148 csr_write(CSR_VSISELECT, csr->vsiselect); 149 csr_write(CSR_HVIPRIO1, csr->hviprio1); 150 csr_write(CSR_HVIPRIO2, csr->hviprio2); 151 #ifdef CONFIG_32BIT 152 csr_write(CSR_VSIEH, csr->vsieh); 153 csr_write(CSR_HVIPH, csr->hviph); 154 csr_write(CSR_HVIPRIO1H, csr->hviprio1h); 155 csr_write(CSR_HVIPRIO2H, csr->hviprio2h); 156 #endif 157 } 158 159 if (kvm_riscv_aia_initialized(vcpu->kvm)) 160 kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu); 161 } 162 163 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) 164 { 165 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 166 void *nsh; 167 168 if (!kvm_riscv_aia_available()) 169 return; 170 171 if (kvm_riscv_aia_initialized(vcpu->kvm)) 172 kvm_riscv_vcpu_aia_imsic_put(vcpu); 173 174 if (kvm_riscv_nacl_available()) { 175 nsh = nacl_shmem(); 176 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); 177 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1); 178 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2); 179 #ifdef CONFIG_32BIT 180 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH); 181 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH); 182 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H); 183 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H); 184 #endif 185 } else { 186 csr->vsiselect = csr_read(CSR_VSISELECT); 187 csr->hviprio1 = csr_read(CSR_HVIPRIO1); 188 csr->hviprio2 = csr_read(CSR_HVIPRIO2); 189 #ifdef CONFIG_32BIT 190 csr->vsieh = csr_read(CSR_VSIEH); 191 csr->hviph = csr_read(CSR_HVIPH); 192 csr->hviprio1h = csr_read(CSR_HVIPRIO1H); 193 csr->hviprio2h = csr_read(CSR_HVIPRIO2H); 194 #endif 195 } 196 } 197 198 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, 199 unsigned long reg_num, 200 unsigned long *out_val) 201 { 202 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 203 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 204 205 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 206 return -ENOENT; 207 if (reg_num >= regs_max) 208 return -ENOENT; 209 210 reg_num = array_index_nospec(reg_num, regs_max); 211 212 *out_val = 0; 213 if (kvm_riscv_aia_available()) 214 *out_val = ((unsigned long *)csr)[reg_num]; 215 216 return 0; 217 } 218 219 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, 220 unsigned long reg_num, 221 unsigned long val) 222 { 223 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 224 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 225 #ifdef CONFIG_32BIT 226 unsigned long flags; 227 #endif 228 229 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 230 return -ENOENT; 231 if (reg_num >= regs_max) 232 return -ENOENT; 233 234 reg_num = array_index_nospec(reg_num, regs_max); 235 236 if (kvm_riscv_aia_available()) { 237 ((unsigned long *)csr)[reg_num] = val; 238 239 #ifdef CONFIG_32BIT 240 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { 241 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 242 vcpu->arch.irqs_pending_mask[1] = 0; 243 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, 244 flags); 245 } 246 #endif 247 } 248 249 return 0; 250 } 251 252 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, 253 unsigned int csr_num, 254 unsigned long *val, 255 unsigned long new_val, 256 unsigned long wr_mask) 257 { 258 /* If AIA not available then redirect trap */ 259 if (!kvm_riscv_aia_available()) 260 return KVM_INSN_ILLEGAL_TRAP; 261 262 /* If AIA not initialized then forward to user space */ 263 if (!kvm_riscv_aia_initialized(vcpu->kvm)) 264 return KVM_INSN_EXIT_TO_USER_SPACE; 265 266 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, 267 val, new_val, wr_mask); 268 } 269 270 /* 271 * External IRQ priority always read-only zero. This means default 272 * priority order is always preferred for external IRQs unless 273 * HVICTL.IID == 9 and HVICTL.IPRIO != 0 274 */ 275 static int aia_irq2bitpos[] = { 276 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ 277 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ 278 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ 279 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ 280 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ 281 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ 282 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ 283 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ 284 }; 285 286 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) 287 { 288 unsigned long hviprio; 289 int bitpos = aia_irq2bitpos[irq]; 290 291 if (bitpos < 0) 292 return 0; 293 294 switch (bitpos / BITS_PER_LONG) { 295 case 0: 296 hviprio = ncsr_read(CSR_HVIPRIO1); 297 break; 298 case 1: 299 #ifndef CONFIG_32BIT 300 hviprio = ncsr_read(CSR_HVIPRIO2); 301 break; 302 #else 303 hviprio = ncsr_read(CSR_HVIPRIO1H); 304 break; 305 case 2: 306 hviprio = ncsr_read(CSR_HVIPRIO2); 307 break; 308 case 3: 309 hviprio = ncsr_read(CSR_HVIPRIO2H); 310 break; 311 #endif 312 default: 313 return 0; 314 } 315 316 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; 317 } 318 319 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) 320 { 321 unsigned long hviprio; 322 int bitpos = aia_irq2bitpos[irq]; 323 324 if (bitpos < 0) 325 return; 326 327 switch (bitpos / BITS_PER_LONG) { 328 case 0: 329 hviprio = ncsr_read(CSR_HVIPRIO1); 330 break; 331 case 1: 332 #ifndef CONFIG_32BIT 333 hviprio = ncsr_read(CSR_HVIPRIO2); 334 break; 335 #else 336 hviprio = ncsr_read(CSR_HVIPRIO1H); 337 break; 338 case 2: 339 hviprio = ncsr_read(CSR_HVIPRIO2); 340 break; 341 case 3: 342 hviprio = ncsr_read(CSR_HVIPRIO2H); 343 break; 344 #endif 345 default: 346 return; 347 } 348 349 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); 350 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); 351 352 switch (bitpos / BITS_PER_LONG) { 353 case 0: 354 ncsr_write(CSR_HVIPRIO1, hviprio); 355 break; 356 case 1: 357 #ifndef CONFIG_32BIT 358 ncsr_write(CSR_HVIPRIO2, hviprio); 359 break; 360 #else 361 ncsr_write(CSR_HVIPRIO1H, hviprio); 362 break; 363 case 2: 364 ncsr_write(CSR_HVIPRIO2, hviprio); 365 break; 366 case 3: 367 ncsr_write(CSR_HVIPRIO2H, hviprio); 368 break; 369 #endif 370 default: 371 return; 372 } 373 } 374 375 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, 376 unsigned long *val, unsigned long new_val, 377 unsigned long wr_mask) 378 { 379 int i, first_irq, nirqs; 380 unsigned long old_val; 381 u8 prio; 382 383 #ifndef CONFIG_32BIT 384 if (isel & 0x1) 385 return KVM_INSN_ILLEGAL_TRAP; 386 #endif 387 388 nirqs = 4 * (BITS_PER_LONG / 32); 389 first_irq = (isel - ISELECT_IPRIO0) * 4; 390 391 old_val = 0; 392 for (i = 0; i < nirqs; i++) { 393 prio = aia_get_iprio8(vcpu, first_irq + i); 394 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); 395 } 396 397 if (val) 398 *val = old_val; 399 400 if (wr_mask) { 401 new_val = (old_val & ~wr_mask) | (new_val & wr_mask); 402 for (i = 0; i < nirqs; i++) { 403 prio = (new_val >> (TOPI_IPRIO_BITS * i)) & 404 TOPI_IPRIO_MASK; 405 aia_set_iprio8(vcpu, first_irq + i, prio); 406 } 407 } 408 409 return KVM_INSN_CONTINUE_NEXT_SEPC; 410 } 411 412 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, 413 unsigned long *val, unsigned long new_val, 414 unsigned long wr_mask) 415 { 416 unsigned int isel; 417 418 /* If AIA not available then redirect trap */ 419 if (!kvm_riscv_aia_available()) 420 return KVM_INSN_ILLEGAL_TRAP; 421 422 /* First try to emulate in kernel space */ 423 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; 424 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) 425 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); 426 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && 427 kvm_riscv_aia_initialized(vcpu->kvm)) 428 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, 429 wr_mask); 430 431 /* We can't handle it here so redirect to user space */ 432 return KVM_INSN_EXIT_TO_USER_SPACE; 433 } 434 435 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, 436 void __iomem **hgei_va, phys_addr_t *hgei_pa) 437 { 438 int ret = -ENOENT; 439 unsigned long flags; 440 const struct imsic_global_config *gc; 441 const struct imsic_local_config *lc; 442 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 443 444 if (!kvm_riscv_aia_available() || !hgctrl) 445 return -ENODEV; 446 447 raw_spin_lock_irqsave(&hgctrl->lock, flags); 448 449 if (hgctrl->free_bitmap) { 450 ret = __ffs(hgctrl->free_bitmap); 451 hgctrl->free_bitmap &= ~BIT(ret); 452 hgctrl->owners[ret] = owner; 453 } 454 455 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 456 457 gc = imsic_get_global_config(); 458 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; 459 if (lc && ret > 0) { 460 if (hgei_va) 461 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); 462 if (hgei_pa) 463 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); 464 } 465 466 return ret; 467 } 468 469 void kvm_riscv_aia_free_hgei(int cpu, int hgei) 470 { 471 unsigned long flags; 472 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 473 474 if (!kvm_riscv_aia_available() || !hgctrl) 475 return; 476 477 raw_spin_lock_irqsave(&hgctrl->lock, flags); 478 479 if (hgei > 0 && hgei <= hgctrl->nr_hgei) { 480 if (!(hgctrl->free_bitmap & BIT(hgei))) { 481 hgctrl->free_bitmap |= BIT(hgei); 482 hgctrl->owners[hgei] = NULL; 483 } 484 } 485 486 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 487 } 488 489 static irqreturn_t hgei_interrupt(int irq, void *dev_id) 490 { 491 int i; 492 unsigned long hgei_mask, flags; 493 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 494 495 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); 496 csr_clear(CSR_HGEIE, hgei_mask); 497 498 raw_spin_lock_irqsave(&hgctrl->lock, flags); 499 500 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { 501 if (hgctrl->owners[i]) 502 kvm_vcpu_kick(hgctrl->owners[i]); 503 } 504 505 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 506 507 put_cpu_ptr(&aia_hgei); 508 return IRQ_HANDLED; 509 } 510 511 static int aia_hgei_init(void) 512 { 513 struct aia_hgei_control *hgctrl; 514 struct irq_domain *domain; 515 int cpu, rc; 516 517 /* Initialize per-CPU guest external interrupt line management */ 518 for_each_possible_cpu(cpu) { 519 hgctrl = per_cpu_ptr(&aia_hgei, cpu); 520 raw_spin_lock_init(&hgctrl->lock); 521 hgctrl->free_bitmap_initialized = false; 522 hgctrl->free_bitmap = 0; 523 } 524 525 /* Find INTC irq domain */ 526 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 527 DOMAIN_BUS_ANY); 528 if (!domain) { 529 kvm_err("unable to find INTC domain\n"); 530 return -ENOENT; 531 } 532 533 /* Map per-CPU SGEI interrupt from INTC domain */ 534 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); 535 if (!hgei_parent_irq) { 536 kvm_err("unable to map SGEI IRQ\n"); 537 return -ENOMEM; 538 } 539 540 /* Request per-CPU SGEI interrupt */ 541 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, 542 "riscv-kvm", &aia_hgei); 543 if (rc) { 544 kvm_err("failed to request SGEI IRQ\n"); 545 return rc; 546 } 547 548 return 0; 549 } 550 551 static void aia_hgei_exit(void) 552 { 553 /* Free per-CPU SGEI interrupt */ 554 free_percpu_irq(hgei_parent_irq, &aia_hgei); 555 } 556 557 void kvm_riscv_aia_pm_exit(void) 558 { 559 struct aia_hgei_control *hgctrl; 560 561 if (!kvm_riscv_aia_available()) 562 return; 563 564 hgctrl = this_cpu_ptr(&aia_hgei); 565 csr_write(CSR_HGEIE, hgctrl->saved_hgeie); 566 567 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 568 csr_write(CSR_HVIPRIO1, 0x0); 569 csr_write(CSR_HVIPRIO2, 0x0); 570 #ifdef CONFIG_32BIT 571 csr_write(CSR_HVIPH, 0x0); 572 csr_write(CSR_HIDELEGH, 0x0); 573 csr_write(CSR_HVIPRIO1H, 0x0); 574 csr_write(CSR_HVIPRIO2H, 0x0); 575 #endif 576 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 577 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 578 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 579 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 580 } 581 582 void kvm_riscv_aia_pm_enter(void) 583 { 584 struct aia_hgei_control *hgctrl; 585 586 if (!kvm_riscv_aia_available()) 587 return; 588 589 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 590 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 591 592 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 593 594 hgctrl = this_cpu_ptr(&aia_hgei); 595 hgctrl->saved_hgeie = csr_read(CSR_HGEIE); 596 } 597 598 void kvm_riscv_aia_enable(void) 599 { 600 const struct imsic_global_config *gc; 601 const struct imsic_local_config *lc; 602 struct aia_hgei_control *hgctrl; 603 unsigned long flags; 604 int aia_nr_hgei; 605 606 if (!kvm_riscv_aia_available()) 607 return; 608 609 gc = imsic_get_global_config(); 610 lc = (gc) ? this_cpu_ptr(gc->local) : NULL; 611 hgctrl = this_cpu_ptr(&aia_hgei); 612 613 /* Figure-out number of bits in HGEIE */ 614 csr_write(CSR_HGEIE, -1UL); 615 hgctrl->nr_hgei = fls_long(csr_read(CSR_HGEIE)); 616 csr_write(CSR_HGEIE, 0); 617 if (hgctrl->nr_hgei) 618 hgctrl->nr_hgei--; 619 620 /* 621 * Number of usable per-HART HGEI lines should be minimum of 622 * per-HART IMSIC guest files and number of bits in HGEIE. 623 */ 624 if (lc) 625 hgctrl->nr_hgei = min((ulong)hgctrl->nr_hgei, lc->nr_guest_files); 626 else 627 hgctrl->nr_hgei = 0; 628 629 /* Update the number of IMSIC guest files across all HARTs */ 630 aia_nr_hgei = atomic_read(&kvm_riscv_aia_nr_hgei); 631 do { 632 if (aia_nr_hgei <= hgctrl->nr_hgei) 633 break; 634 } while (!atomic_try_cmpxchg(&kvm_riscv_aia_nr_hgei, &aia_nr_hgei, hgctrl->nr_hgei)); 635 636 raw_spin_lock_irqsave(&hgctrl->lock, flags); 637 if (!hgctrl->free_bitmap_initialized) { 638 hgctrl->free_bitmap = (hgctrl->nr_hgei) ? GENMASK_ULL(hgctrl->nr_hgei, 1) : 0; 639 hgctrl->free_bitmap_initialized = true; 640 } 641 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 642 643 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 644 csr_write(CSR_HVIPRIO1, 0x0); 645 csr_write(CSR_HVIPRIO2, 0x0); 646 #ifdef CONFIG_32BIT 647 csr_write(CSR_HVIPH, 0x0); 648 csr_write(CSR_HIDELEGH, 0x0); 649 csr_write(CSR_HVIPRIO1H, 0x0); 650 csr_write(CSR_HVIPRIO2H, 0x0); 651 #endif 652 653 /* Enable per-CPU SGEI interrupt */ 654 enable_percpu_irq(hgei_parent_irq, 655 irq_get_trigger_type(hgei_parent_irq)); 656 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 657 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 658 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 659 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 660 } 661 662 void kvm_riscv_aia_disable(void) 663 { 664 int i; 665 unsigned long flags; 666 struct kvm_vcpu *vcpu; 667 struct aia_hgei_control *hgctrl; 668 669 if (!kvm_riscv_aia_available()) 670 return; 671 hgctrl = get_cpu_ptr(&aia_hgei); 672 673 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 674 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 675 /* Disable per-CPU SGEI interrupt */ 676 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); 677 disable_percpu_irq(hgei_parent_irq); 678 679 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 680 681 raw_spin_lock_irqsave(&hgctrl->lock, flags); 682 683 for (i = 0; i <= hgctrl->nr_hgei; i++) { 684 vcpu = hgctrl->owners[i]; 685 if (!vcpu) 686 continue; 687 688 /* 689 * We release hgctrl->lock before notifying IMSIC 690 * so that we don't have lock ordering issues. 691 */ 692 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 693 694 /* Notify IMSIC */ 695 kvm_riscv_vcpu_aia_imsic_release(vcpu); 696 697 /* 698 * Wakeup VCPU if it was blocked so that it can 699 * run on other HARTs 700 */ 701 if (csr_read(CSR_HGEIE) & BIT(i)) { 702 csr_clear(CSR_HGEIE, BIT(i)); 703 kvm_vcpu_kick(vcpu); 704 } 705 706 raw_spin_lock_irqsave(&hgctrl->lock, flags); 707 } 708 709 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 710 711 put_cpu_ptr(&aia_hgei); 712 } 713 714 int kvm_riscv_aia_init(void) 715 { 716 int rc; 717 const struct imsic_global_config *gc; 718 719 if (!riscv_isa_extension_available(NULL, SxAIA)) 720 return -ENODEV; 721 gc = imsic_get_global_config(); 722 723 /* Set initial value of IMSIC guest files across all HARTs */ 724 if (gc) 725 atomic_set(&kvm_riscv_aia_nr_hgei, gc->nr_guest_files); 726 else 727 atomic_set(&kvm_riscv_aia_nr_hgei, 0); 728 729 /* Find number of guest MSI IDs */ 730 kvm_riscv_aia_max_ids = IMSIC_MAX_ID; 731 if (gc) 732 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; 733 734 /* Initialize guest external interrupt line management */ 735 rc = aia_hgei_init(); 736 if (rc) 737 return rc; 738 739 /* Register device operations */ 740 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, 741 KVM_DEV_TYPE_RISCV_AIA); 742 if (rc) { 743 aia_hgei_exit(); 744 return rc; 745 } 746 747 /* Enable KVM AIA support */ 748 static_branch_enable(&kvm_riscv_aia_available); 749 750 return 0; 751 } 752 753 void kvm_riscv_aia_exit(void) 754 { 755 if (!kvm_riscv_aia_available()) 756 return; 757 758 /* Unregister device operations */ 759 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); 760 761 /* Cleanup the HGEI state */ 762 aia_hgei_exit(); 763 } 764