1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/bitops.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/riscv-imsic.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kvm_host.h> 16 #include <linux/nospec.h> 17 #include <linux/percpu.h> 18 #include <linux/spinlock.h> 19 #include <asm/cpufeature.h> 20 #include <asm/kvm_nacl.h> 21 22 struct aia_hgei_control { 23 raw_spinlock_t lock; 24 bool free_bitmap_initialized; 25 unsigned long free_bitmap; 26 struct kvm_vcpu *owners[BITS_PER_LONG]; 27 unsigned int nr_hgei; 28 }; 29 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); 30 static int hgei_parent_irq; 31 32 atomic_t kvm_riscv_aia_nr_hgei; 33 unsigned int kvm_riscv_aia_max_ids; 34 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); 35 36 static inline unsigned long aia_hvictl_value(bool ext_irq_pending) 37 { 38 unsigned long hvictl; 39 40 /* 41 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents 42 * no interrupt in HVICTL. 43 */ 44 45 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; 46 hvictl |= ext_irq_pending; 47 return hvictl; 48 } 49 50 #ifdef CONFIG_32BIT 51 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) 52 { 53 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 54 unsigned long mask, val; 55 56 lockdep_assert_held(&vcpu->arch.irqs_pending_lock); 57 58 if (!kvm_riscv_aia_available()) 59 return; 60 61 mask = vcpu->arch.irqs_pending_mask[1]; 62 if (mask) { 63 vcpu->arch.irqs_pending_mask[1] = 0; 64 val = vcpu->arch.irqs_pending[1] & mask; 65 66 csr->hviph &= ~mask; 67 csr->hviph |= val; 68 } 69 } 70 71 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) 72 { 73 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 74 75 lockdep_assert_held(&vcpu->arch.irqs_pending_lock); 76 77 if (kvm_riscv_aia_available()) 78 csr->vsieh = ncsr_read(CSR_VSIEH); 79 } 80 #endif 81 82 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 83 { 84 unsigned long seip; 85 #ifdef CONFIG_32BIT 86 unsigned long flags; 87 bool pending; 88 #endif 89 90 if (!kvm_riscv_aia_available()) 91 return false; 92 93 #ifdef CONFIG_32BIT 94 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 95 pending = vcpu->arch.irqs_pending[1] & 96 (vcpu->arch.aia_context.guest_csr.vsieh & 97 upper_32_bits(mask)); 98 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); 99 100 if (pending) 101 return true; 102 #endif 103 104 seip = vcpu->arch.guest_csr.vsie; 105 seip &= (unsigned long)mask; 106 seip &= BIT(IRQ_S_EXT); 107 108 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) 109 return false; 110 111 return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu); 112 } 113 114 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) 115 { 116 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 117 118 if (!kvm_riscv_aia_available()) 119 return; 120 121 #ifdef CONFIG_32BIT 122 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); 123 #endif 124 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT)))); 125 } 126 127 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) 128 { 129 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 130 void *nsh; 131 132 if (!kvm_riscv_aia_available()) 133 return; 134 135 if (kvm_riscv_nacl_sync_csr_available()) { 136 nsh = nacl_shmem(); 137 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect); 138 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1); 139 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2); 140 #ifdef CONFIG_32BIT 141 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh); 142 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph); 143 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h); 144 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h); 145 #endif 146 } else { 147 csr_write(CSR_VSISELECT, csr->vsiselect); 148 csr_write(CSR_HVIPRIO1, csr->hviprio1); 149 csr_write(CSR_HVIPRIO2, csr->hviprio2); 150 #ifdef CONFIG_32BIT 151 csr_write(CSR_VSIEH, csr->vsieh); 152 csr_write(CSR_HVIPH, csr->hviph); 153 csr_write(CSR_HVIPRIO1H, csr->hviprio1h); 154 csr_write(CSR_HVIPRIO2H, csr->hviprio2h); 155 #endif 156 } 157 158 if (kvm_riscv_aia_initialized(vcpu->kvm)) 159 kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu); 160 } 161 162 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) 163 { 164 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 165 void *nsh; 166 167 if (!kvm_riscv_aia_available()) 168 return; 169 170 if (kvm_riscv_aia_initialized(vcpu->kvm)) 171 kvm_riscv_vcpu_aia_imsic_put(vcpu); 172 173 if (kvm_riscv_nacl_available()) { 174 nsh = nacl_shmem(); 175 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); 176 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1); 177 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2); 178 #ifdef CONFIG_32BIT 179 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH); 180 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH); 181 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H); 182 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H); 183 #endif 184 } else { 185 csr->vsiselect = csr_read(CSR_VSISELECT); 186 csr->hviprio1 = csr_read(CSR_HVIPRIO1); 187 csr->hviprio2 = csr_read(CSR_HVIPRIO2); 188 #ifdef CONFIG_32BIT 189 csr->vsieh = csr_read(CSR_VSIEH); 190 csr->hviph = csr_read(CSR_HVIPH); 191 csr->hviprio1h = csr_read(CSR_HVIPRIO1H); 192 csr->hviprio2h = csr_read(CSR_HVIPRIO2H); 193 #endif 194 } 195 } 196 197 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, 198 unsigned long reg_num, 199 unsigned long *out_val) 200 { 201 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 202 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 203 204 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 205 return -ENOENT; 206 if (reg_num >= regs_max) 207 return -ENOENT; 208 209 reg_num = array_index_nospec(reg_num, regs_max); 210 211 *out_val = 0; 212 if (kvm_riscv_aia_available()) 213 *out_val = ((unsigned long *)csr)[reg_num]; 214 215 return 0; 216 } 217 218 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, 219 unsigned long reg_num, 220 unsigned long val) 221 { 222 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 223 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 224 #ifdef CONFIG_32BIT 225 unsigned long flags; 226 #endif 227 228 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 229 return -ENOENT; 230 if (reg_num >= regs_max) 231 return -ENOENT; 232 233 reg_num = array_index_nospec(reg_num, regs_max); 234 235 if (kvm_riscv_aia_available()) { 236 ((unsigned long *)csr)[reg_num] = val; 237 238 #ifdef CONFIG_32BIT 239 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { 240 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 241 vcpu->arch.irqs_pending_mask[1] = 0; 242 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, 243 flags); 244 } 245 #endif 246 } 247 248 return 0; 249 } 250 251 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, 252 unsigned int csr_num, 253 unsigned long *val, 254 unsigned long new_val, 255 unsigned long wr_mask) 256 { 257 /* If AIA not available then redirect trap */ 258 if (!kvm_riscv_aia_available()) 259 return KVM_INSN_ILLEGAL_TRAP; 260 261 /* If AIA not initialized then forward to user space */ 262 if (!kvm_riscv_aia_initialized(vcpu->kvm)) 263 return KVM_INSN_EXIT_TO_USER_SPACE; 264 265 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, 266 val, new_val, wr_mask); 267 } 268 269 /* 270 * External IRQ priority always read-only zero. This means default 271 * priority order is always preferred for external IRQs unless 272 * HVICTL.IID == 9 and HVICTL.IPRIO != 0 273 */ 274 static int aia_irq2bitpos[] = { 275 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ 276 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ 277 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ 278 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ 279 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ 280 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ 281 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ 282 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ 283 }; 284 285 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) 286 { 287 unsigned long hviprio; 288 int bitpos = aia_irq2bitpos[irq]; 289 290 if (bitpos < 0) 291 return 0; 292 293 switch (bitpos / BITS_PER_LONG) { 294 case 0: 295 hviprio = ncsr_read(CSR_HVIPRIO1); 296 break; 297 case 1: 298 #ifndef CONFIG_32BIT 299 hviprio = ncsr_read(CSR_HVIPRIO2); 300 break; 301 #else 302 hviprio = ncsr_read(CSR_HVIPRIO1H); 303 break; 304 case 2: 305 hviprio = ncsr_read(CSR_HVIPRIO2); 306 break; 307 case 3: 308 hviprio = ncsr_read(CSR_HVIPRIO2H); 309 break; 310 #endif 311 default: 312 return 0; 313 } 314 315 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; 316 } 317 318 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) 319 { 320 unsigned long hviprio; 321 int bitpos = aia_irq2bitpos[irq]; 322 323 if (bitpos < 0) 324 return; 325 326 switch (bitpos / BITS_PER_LONG) { 327 case 0: 328 hviprio = ncsr_read(CSR_HVIPRIO1); 329 break; 330 case 1: 331 #ifndef CONFIG_32BIT 332 hviprio = ncsr_read(CSR_HVIPRIO2); 333 break; 334 #else 335 hviprio = ncsr_read(CSR_HVIPRIO1H); 336 break; 337 case 2: 338 hviprio = ncsr_read(CSR_HVIPRIO2); 339 break; 340 case 3: 341 hviprio = ncsr_read(CSR_HVIPRIO2H); 342 break; 343 #endif 344 default: 345 return; 346 } 347 348 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); 349 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); 350 351 switch (bitpos / BITS_PER_LONG) { 352 case 0: 353 ncsr_write(CSR_HVIPRIO1, hviprio); 354 break; 355 case 1: 356 #ifndef CONFIG_32BIT 357 ncsr_write(CSR_HVIPRIO2, hviprio); 358 break; 359 #else 360 ncsr_write(CSR_HVIPRIO1H, hviprio); 361 break; 362 case 2: 363 ncsr_write(CSR_HVIPRIO2, hviprio); 364 break; 365 case 3: 366 ncsr_write(CSR_HVIPRIO2H, hviprio); 367 break; 368 #endif 369 default: 370 return; 371 } 372 } 373 374 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, 375 unsigned long *val, unsigned long new_val, 376 unsigned long wr_mask) 377 { 378 int i, first_irq, nirqs; 379 unsigned long old_val; 380 u8 prio; 381 382 #ifndef CONFIG_32BIT 383 if (isel & 0x1) 384 return KVM_INSN_ILLEGAL_TRAP; 385 #endif 386 387 nirqs = 4 * (BITS_PER_LONG / 32); 388 first_irq = (isel - ISELECT_IPRIO0) * 4; 389 390 old_val = 0; 391 for (i = 0; i < nirqs; i++) { 392 prio = aia_get_iprio8(vcpu, first_irq + i); 393 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); 394 } 395 396 if (val) 397 *val = old_val; 398 399 if (wr_mask) { 400 new_val = (old_val & ~wr_mask) | (new_val & wr_mask); 401 for (i = 0; i < nirqs; i++) { 402 prio = (new_val >> (TOPI_IPRIO_BITS * i)) & 403 TOPI_IPRIO_MASK; 404 aia_set_iprio8(vcpu, first_irq + i, prio); 405 } 406 } 407 408 return KVM_INSN_CONTINUE_NEXT_SEPC; 409 } 410 411 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, 412 unsigned long *val, unsigned long new_val, 413 unsigned long wr_mask) 414 { 415 unsigned int isel; 416 417 /* If AIA not available then redirect trap */ 418 if (!kvm_riscv_aia_available()) 419 return KVM_INSN_ILLEGAL_TRAP; 420 421 /* First try to emulate in kernel space */ 422 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; 423 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) 424 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); 425 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && 426 kvm_riscv_aia_initialized(vcpu->kvm)) 427 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, 428 wr_mask); 429 430 /* We can't handle it here so redirect to user space */ 431 return KVM_INSN_EXIT_TO_USER_SPACE; 432 } 433 434 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, 435 void __iomem **hgei_va, phys_addr_t *hgei_pa) 436 { 437 int ret = -ENOENT; 438 unsigned long flags; 439 const struct imsic_global_config *gc; 440 const struct imsic_local_config *lc; 441 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 442 443 if (!kvm_riscv_aia_available() || !hgctrl) 444 return -ENODEV; 445 446 raw_spin_lock_irqsave(&hgctrl->lock, flags); 447 448 if (hgctrl->free_bitmap) { 449 ret = __ffs(hgctrl->free_bitmap); 450 hgctrl->free_bitmap &= ~BIT(ret); 451 hgctrl->owners[ret] = owner; 452 } 453 454 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 455 456 gc = imsic_get_global_config(); 457 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; 458 if (lc && ret > 0) { 459 if (hgei_va) 460 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); 461 if (hgei_pa) 462 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); 463 } 464 465 return ret; 466 } 467 468 void kvm_riscv_aia_free_hgei(int cpu, int hgei) 469 { 470 unsigned long flags; 471 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 472 473 if (!kvm_riscv_aia_available() || !hgctrl) 474 return; 475 476 raw_spin_lock_irqsave(&hgctrl->lock, flags); 477 478 if (hgei > 0 && hgei <= hgctrl->nr_hgei) { 479 if (!(hgctrl->free_bitmap & BIT(hgei))) { 480 hgctrl->free_bitmap |= BIT(hgei); 481 hgctrl->owners[hgei] = NULL; 482 } 483 } 484 485 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 486 } 487 488 static irqreturn_t hgei_interrupt(int irq, void *dev_id) 489 { 490 int i; 491 unsigned long hgei_mask, flags; 492 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 493 494 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); 495 csr_clear(CSR_HGEIE, hgei_mask); 496 497 raw_spin_lock_irqsave(&hgctrl->lock, flags); 498 499 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { 500 if (hgctrl->owners[i]) 501 kvm_vcpu_kick(hgctrl->owners[i]); 502 } 503 504 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 505 506 put_cpu_ptr(&aia_hgei); 507 return IRQ_HANDLED; 508 } 509 510 static int aia_hgei_init(void) 511 { 512 struct aia_hgei_control *hgctrl; 513 struct irq_domain *domain; 514 int cpu, rc; 515 516 /* Initialize per-CPU guest external interrupt line management */ 517 for_each_possible_cpu(cpu) { 518 hgctrl = per_cpu_ptr(&aia_hgei, cpu); 519 raw_spin_lock_init(&hgctrl->lock); 520 hgctrl->free_bitmap_initialized = false; 521 hgctrl->free_bitmap = 0; 522 } 523 524 /* Find INTC irq domain */ 525 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 526 DOMAIN_BUS_ANY); 527 if (!domain) { 528 kvm_err("unable to find INTC domain\n"); 529 return -ENOENT; 530 } 531 532 /* Map per-CPU SGEI interrupt from INTC domain */ 533 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); 534 if (!hgei_parent_irq) { 535 kvm_err("unable to map SGEI IRQ\n"); 536 return -ENOMEM; 537 } 538 539 /* Request per-CPU SGEI interrupt */ 540 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, 541 "riscv-kvm", &aia_hgei); 542 if (rc) { 543 kvm_err("failed to request SGEI IRQ\n"); 544 return rc; 545 } 546 547 return 0; 548 } 549 550 static void aia_hgei_exit(void) 551 { 552 /* Free per-CPU SGEI interrupt */ 553 free_percpu_irq(hgei_parent_irq, &aia_hgei); 554 } 555 556 void kvm_riscv_aia_enable(void) 557 { 558 const struct imsic_global_config *gc; 559 const struct imsic_local_config *lc; 560 struct aia_hgei_control *hgctrl; 561 unsigned long flags; 562 int aia_nr_hgei; 563 564 if (!kvm_riscv_aia_available()) 565 return; 566 567 gc = imsic_get_global_config(); 568 lc = (gc) ? this_cpu_ptr(gc->local) : NULL; 569 hgctrl = this_cpu_ptr(&aia_hgei); 570 571 /* Figure-out number of bits in HGEIE */ 572 csr_write(CSR_HGEIE, -1UL); 573 hgctrl->nr_hgei = fls_long(csr_read(CSR_HGEIE)); 574 csr_write(CSR_HGEIE, 0); 575 if (hgctrl->nr_hgei) 576 hgctrl->nr_hgei--; 577 578 /* 579 * Number of usable per-HART HGEI lines should be minimum of 580 * per-HART IMSIC guest files and number of bits in HGEIE. 581 */ 582 if (lc) 583 hgctrl->nr_hgei = min((ulong)hgctrl->nr_hgei, lc->nr_guest_files); 584 else 585 hgctrl->nr_hgei = 0; 586 587 /* Update the number of IMSIC guest files across all HARTs */ 588 aia_nr_hgei = atomic_read(&kvm_riscv_aia_nr_hgei); 589 do { 590 if (aia_nr_hgei <= hgctrl->nr_hgei) 591 break; 592 } while (!atomic_try_cmpxchg(&kvm_riscv_aia_nr_hgei, &aia_nr_hgei, hgctrl->nr_hgei)); 593 594 raw_spin_lock_irqsave(&hgctrl->lock, flags); 595 if (!hgctrl->free_bitmap_initialized) { 596 hgctrl->free_bitmap = (hgctrl->nr_hgei) ? GENMASK_ULL(hgctrl->nr_hgei, 1) : 0; 597 hgctrl->free_bitmap_initialized = true; 598 } 599 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 600 601 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 602 csr_write(CSR_HVIPRIO1, 0x0); 603 csr_write(CSR_HVIPRIO2, 0x0); 604 #ifdef CONFIG_32BIT 605 csr_write(CSR_HVIPH, 0x0); 606 csr_write(CSR_HIDELEGH, 0x0); 607 csr_write(CSR_HVIPRIO1H, 0x0); 608 csr_write(CSR_HVIPRIO2H, 0x0); 609 #endif 610 611 /* Enable per-CPU SGEI interrupt */ 612 enable_percpu_irq(hgei_parent_irq, 613 irq_get_trigger_type(hgei_parent_irq)); 614 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 615 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 616 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 617 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 618 } 619 620 void kvm_riscv_aia_disable(void) 621 { 622 int i; 623 unsigned long flags; 624 struct kvm_vcpu *vcpu; 625 struct aia_hgei_control *hgctrl; 626 627 if (!kvm_riscv_aia_available()) 628 return; 629 hgctrl = get_cpu_ptr(&aia_hgei); 630 631 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 632 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 633 /* Disable per-CPU SGEI interrupt */ 634 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); 635 disable_percpu_irq(hgei_parent_irq); 636 637 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 638 639 raw_spin_lock_irqsave(&hgctrl->lock, flags); 640 641 for (i = 0; i <= hgctrl->nr_hgei; i++) { 642 vcpu = hgctrl->owners[i]; 643 if (!vcpu) 644 continue; 645 646 /* 647 * We release hgctrl->lock before notifying IMSIC 648 * so that we don't have lock ordering issues. 649 */ 650 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 651 652 /* Notify IMSIC */ 653 kvm_riscv_vcpu_aia_imsic_release(vcpu); 654 655 /* 656 * Wakeup VCPU if it was blocked so that it can 657 * run on other HARTs 658 */ 659 if (csr_read(CSR_HGEIE) & BIT(i)) { 660 csr_clear(CSR_HGEIE, BIT(i)); 661 kvm_vcpu_kick(vcpu); 662 } 663 664 raw_spin_lock_irqsave(&hgctrl->lock, flags); 665 } 666 667 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 668 669 put_cpu_ptr(&aia_hgei); 670 } 671 672 int kvm_riscv_aia_init(void) 673 { 674 int rc; 675 const struct imsic_global_config *gc; 676 677 if (!riscv_isa_extension_available(NULL, SxAIA)) 678 return -ENODEV; 679 gc = imsic_get_global_config(); 680 681 /* Set initial value of IMSIC guest files across all HARTs */ 682 if (gc) 683 atomic_set(&kvm_riscv_aia_nr_hgei, gc->nr_guest_files); 684 else 685 atomic_set(&kvm_riscv_aia_nr_hgei, 0); 686 687 /* Find number of guest MSI IDs */ 688 kvm_riscv_aia_max_ids = IMSIC_MAX_ID; 689 if (gc) 690 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; 691 692 /* Initialize guest external interrupt line management */ 693 rc = aia_hgei_init(); 694 if (rc) 695 return rc; 696 697 /* Register device operations */ 698 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, 699 KVM_DEV_TYPE_RISCV_AIA); 700 if (rc) { 701 aia_hgei_exit(); 702 return rc; 703 } 704 705 /* Enable KVM AIA support */ 706 static_branch_enable(&kvm_riscv_aia_available); 707 708 return 0; 709 } 710 711 void kvm_riscv_aia_exit(void) 712 { 713 if (!kvm_riscv_aia_available()) 714 return; 715 716 /* Unregister device operations */ 717 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); 718 719 /* Cleanup the HGEI state */ 720 aia_hgei_exit(); 721 } 722