1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/bitops.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip/riscv-imsic.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kvm_host.h> 16 #include <linux/nospec.h> 17 #include <linux/percpu.h> 18 #include <linux/spinlock.h> 19 #include <asm/cpufeature.h> 20 #include <asm/kvm_nacl.h> 21 22 struct aia_hgei_control { 23 raw_spinlock_t lock; 24 bool free_bitmap_initialized; 25 unsigned long free_bitmap; 26 struct kvm_vcpu *owners[BITS_PER_LONG]; 27 unsigned int nr_hgei; 28 }; 29 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); 30 static int hgei_parent_irq; 31 32 atomic_t kvm_riscv_aia_nr_hgei; 33 unsigned int kvm_riscv_aia_max_ids; 34 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); 35 36 static inline unsigned long aia_hvictl_value(bool ext_irq_pending) 37 { 38 unsigned long hvictl; 39 40 /* 41 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents 42 * no interrupt in HVICTL. 43 */ 44 45 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; 46 hvictl |= ext_irq_pending; 47 return hvictl; 48 } 49 50 #ifdef CONFIG_32BIT 51 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) 52 { 53 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 54 unsigned long mask, val; 55 56 if (!kvm_riscv_aia_available()) 57 return; 58 59 if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { 60 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); 61 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; 62 63 csr->hviph &= ~mask; 64 csr->hviph |= val; 65 } 66 } 67 68 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) 69 { 70 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 71 72 if (kvm_riscv_aia_available()) 73 csr->vsieh = ncsr_read(CSR_VSIEH); 74 } 75 #endif 76 77 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 78 { 79 unsigned long seip; 80 81 if (!kvm_riscv_aia_available()) 82 return false; 83 84 #ifdef CONFIG_32BIT 85 if (READ_ONCE(vcpu->arch.irqs_pending[1]) & 86 (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) 87 return true; 88 #endif 89 90 seip = vcpu->arch.guest_csr.vsie; 91 seip &= (unsigned long)mask; 92 seip &= BIT(IRQ_S_EXT); 93 94 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) 95 return false; 96 97 return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu); 98 } 99 100 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) 101 { 102 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 103 104 if (!kvm_riscv_aia_available()) 105 return; 106 107 #ifdef CONFIG_32BIT 108 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); 109 #endif 110 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT)))); 111 } 112 113 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) 114 { 115 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 116 void *nsh; 117 118 if (!kvm_riscv_aia_available()) 119 return; 120 121 if (kvm_riscv_nacl_sync_csr_available()) { 122 nsh = nacl_shmem(); 123 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect); 124 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1); 125 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2); 126 #ifdef CONFIG_32BIT 127 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh); 128 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph); 129 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h); 130 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h); 131 #endif 132 } else { 133 csr_write(CSR_VSISELECT, csr->vsiselect); 134 csr_write(CSR_HVIPRIO1, csr->hviprio1); 135 csr_write(CSR_HVIPRIO2, csr->hviprio2); 136 #ifdef CONFIG_32BIT 137 csr_write(CSR_VSIEH, csr->vsieh); 138 csr_write(CSR_HVIPH, csr->hviph); 139 csr_write(CSR_HVIPRIO1H, csr->hviprio1h); 140 csr_write(CSR_HVIPRIO2H, csr->hviprio2h); 141 #endif 142 } 143 144 if (kvm_riscv_aia_initialized(vcpu->kvm)) 145 kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu); 146 } 147 148 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) 149 { 150 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 151 void *nsh; 152 153 if (!kvm_riscv_aia_available()) 154 return; 155 156 if (kvm_riscv_aia_initialized(vcpu->kvm)) 157 kvm_riscv_vcpu_aia_imsic_put(vcpu); 158 159 if (kvm_riscv_nacl_available()) { 160 nsh = nacl_shmem(); 161 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); 162 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1); 163 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2); 164 #ifdef CONFIG_32BIT 165 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH); 166 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH); 167 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H); 168 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H); 169 #endif 170 } else { 171 csr->vsiselect = csr_read(CSR_VSISELECT); 172 csr->hviprio1 = csr_read(CSR_HVIPRIO1); 173 csr->hviprio2 = csr_read(CSR_HVIPRIO2); 174 #ifdef CONFIG_32BIT 175 csr->vsieh = csr_read(CSR_VSIEH); 176 csr->hviph = csr_read(CSR_HVIPH); 177 csr->hviprio1h = csr_read(CSR_HVIPRIO1H); 178 csr->hviprio2h = csr_read(CSR_HVIPRIO2H); 179 #endif 180 } 181 } 182 183 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, 184 unsigned long reg_num, 185 unsigned long *out_val) 186 { 187 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 188 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 189 190 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 191 return -ENOENT; 192 if (reg_num >= regs_max) 193 return -ENOENT; 194 195 reg_num = array_index_nospec(reg_num, regs_max); 196 197 *out_val = 0; 198 if (kvm_riscv_aia_available()) 199 *out_val = ((unsigned long *)csr)[reg_num]; 200 201 return 0; 202 } 203 204 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, 205 unsigned long reg_num, 206 unsigned long val) 207 { 208 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; 209 unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 210 211 if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 212 return -ENOENT; 213 if (reg_num >= regs_max) 214 return -ENOENT; 215 216 reg_num = array_index_nospec(reg_num, regs_max); 217 218 if (kvm_riscv_aia_available()) { 219 ((unsigned long *)csr)[reg_num] = val; 220 221 #ifdef CONFIG_32BIT 222 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) 223 WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); 224 #endif 225 } 226 227 return 0; 228 } 229 230 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, 231 unsigned int csr_num, 232 unsigned long *val, 233 unsigned long new_val, 234 unsigned long wr_mask) 235 { 236 /* If AIA not available then redirect trap */ 237 if (!kvm_riscv_aia_available()) 238 return KVM_INSN_ILLEGAL_TRAP; 239 240 /* If AIA not initialized then forward to user space */ 241 if (!kvm_riscv_aia_initialized(vcpu->kvm)) 242 return KVM_INSN_EXIT_TO_USER_SPACE; 243 244 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, 245 val, new_val, wr_mask); 246 } 247 248 /* 249 * External IRQ priority always read-only zero. This means default 250 * priority order is always preferred for external IRQs unless 251 * HVICTL.IID == 9 and HVICTL.IPRIO != 0 252 */ 253 static int aia_irq2bitpos[] = { 254 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ 255 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ 256 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ 257 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ 258 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ 259 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ 260 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ 261 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ 262 }; 263 264 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) 265 { 266 unsigned long hviprio; 267 int bitpos = aia_irq2bitpos[irq]; 268 269 if (bitpos < 0) 270 return 0; 271 272 switch (bitpos / BITS_PER_LONG) { 273 case 0: 274 hviprio = ncsr_read(CSR_HVIPRIO1); 275 break; 276 case 1: 277 #ifndef CONFIG_32BIT 278 hviprio = ncsr_read(CSR_HVIPRIO2); 279 break; 280 #else 281 hviprio = ncsr_read(CSR_HVIPRIO1H); 282 break; 283 case 2: 284 hviprio = ncsr_read(CSR_HVIPRIO2); 285 break; 286 case 3: 287 hviprio = ncsr_read(CSR_HVIPRIO2H); 288 break; 289 #endif 290 default: 291 return 0; 292 } 293 294 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; 295 } 296 297 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) 298 { 299 unsigned long hviprio; 300 int bitpos = aia_irq2bitpos[irq]; 301 302 if (bitpos < 0) 303 return; 304 305 switch (bitpos / BITS_PER_LONG) { 306 case 0: 307 hviprio = ncsr_read(CSR_HVIPRIO1); 308 break; 309 case 1: 310 #ifndef CONFIG_32BIT 311 hviprio = ncsr_read(CSR_HVIPRIO2); 312 break; 313 #else 314 hviprio = ncsr_read(CSR_HVIPRIO1H); 315 break; 316 case 2: 317 hviprio = ncsr_read(CSR_HVIPRIO2); 318 break; 319 case 3: 320 hviprio = ncsr_read(CSR_HVIPRIO2H); 321 break; 322 #endif 323 default: 324 return; 325 } 326 327 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); 328 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); 329 330 switch (bitpos / BITS_PER_LONG) { 331 case 0: 332 ncsr_write(CSR_HVIPRIO1, hviprio); 333 break; 334 case 1: 335 #ifndef CONFIG_32BIT 336 ncsr_write(CSR_HVIPRIO2, hviprio); 337 break; 338 #else 339 ncsr_write(CSR_HVIPRIO1H, hviprio); 340 break; 341 case 2: 342 ncsr_write(CSR_HVIPRIO2, hviprio); 343 break; 344 case 3: 345 ncsr_write(CSR_HVIPRIO2H, hviprio); 346 break; 347 #endif 348 default: 349 return; 350 } 351 } 352 353 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, 354 unsigned long *val, unsigned long new_val, 355 unsigned long wr_mask) 356 { 357 int i, first_irq, nirqs; 358 unsigned long old_val; 359 u8 prio; 360 361 #ifndef CONFIG_32BIT 362 if (isel & 0x1) 363 return KVM_INSN_ILLEGAL_TRAP; 364 #endif 365 366 nirqs = 4 * (BITS_PER_LONG / 32); 367 first_irq = (isel - ISELECT_IPRIO0) * 4; 368 369 old_val = 0; 370 for (i = 0; i < nirqs; i++) { 371 prio = aia_get_iprio8(vcpu, first_irq + i); 372 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); 373 } 374 375 if (val) 376 *val = old_val; 377 378 if (wr_mask) { 379 new_val = (old_val & ~wr_mask) | (new_val & wr_mask); 380 for (i = 0; i < nirqs; i++) { 381 prio = (new_val >> (TOPI_IPRIO_BITS * i)) & 382 TOPI_IPRIO_MASK; 383 aia_set_iprio8(vcpu, first_irq + i, prio); 384 } 385 } 386 387 return KVM_INSN_CONTINUE_NEXT_SEPC; 388 } 389 390 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, 391 unsigned long *val, unsigned long new_val, 392 unsigned long wr_mask) 393 { 394 unsigned int isel; 395 396 /* If AIA not available then redirect trap */ 397 if (!kvm_riscv_aia_available()) 398 return KVM_INSN_ILLEGAL_TRAP; 399 400 /* First try to emulate in kernel space */ 401 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK; 402 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) 403 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); 404 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && 405 kvm_riscv_aia_initialized(vcpu->kvm)) 406 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, 407 wr_mask); 408 409 /* We can't handle it here so redirect to user space */ 410 return KVM_INSN_EXIT_TO_USER_SPACE; 411 } 412 413 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, 414 void __iomem **hgei_va, phys_addr_t *hgei_pa) 415 { 416 int ret = -ENOENT; 417 unsigned long flags; 418 const struct imsic_global_config *gc; 419 const struct imsic_local_config *lc; 420 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 421 422 if (!kvm_riscv_aia_available() || !hgctrl) 423 return -ENODEV; 424 425 raw_spin_lock_irqsave(&hgctrl->lock, flags); 426 427 if (hgctrl->free_bitmap) { 428 ret = __ffs(hgctrl->free_bitmap); 429 hgctrl->free_bitmap &= ~BIT(ret); 430 hgctrl->owners[ret] = owner; 431 } 432 433 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 434 435 gc = imsic_get_global_config(); 436 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL; 437 if (lc && ret > 0) { 438 if (hgei_va) 439 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ); 440 if (hgei_pa) 441 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ); 442 } 443 444 return ret; 445 } 446 447 void kvm_riscv_aia_free_hgei(int cpu, int hgei) 448 { 449 unsigned long flags; 450 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); 451 452 if (!kvm_riscv_aia_available() || !hgctrl) 453 return; 454 455 raw_spin_lock_irqsave(&hgctrl->lock, flags); 456 457 if (hgei > 0 && hgei <= hgctrl->nr_hgei) { 458 if (!(hgctrl->free_bitmap & BIT(hgei))) { 459 hgctrl->free_bitmap |= BIT(hgei); 460 hgctrl->owners[hgei] = NULL; 461 } 462 } 463 464 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 465 } 466 467 static irqreturn_t hgei_interrupt(int irq, void *dev_id) 468 { 469 int i; 470 unsigned long hgei_mask, flags; 471 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); 472 473 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); 474 csr_clear(CSR_HGEIE, hgei_mask); 475 476 raw_spin_lock_irqsave(&hgctrl->lock, flags); 477 478 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { 479 if (hgctrl->owners[i]) 480 kvm_vcpu_kick(hgctrl->owners[i]); 481 } 482 483 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 484 485 put_cpu_ptr(&aia_hgei); 486 return IRQ_HANDLED; 487 } 488 489 static int aia_hgei_init(void) 490 { 491 struct aia_hgei_control *hgctrl; 492 struct irq_domain *domain; 493 int cpu, rc; 494 495 /* Initialize per-CPU guest external interrupt line management */ 496 for_each_possible_cpu(cpu) { 497 hgctrl = per_cpu_ptr(&aia_hgei, cpu); 498 raw_spin_lock_init(&hgctrl->lock); 499 hgctrl->free_bitmap_initialized = false; 500 hgctrl->free_bitmap = 0; 501 } 502 503 /* Find INTC irq domain */ 504 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), 505 DOMAIN_BUS_ANY); 506 if (!domain) { 507 kvm_err("unable to find INTC domain\n"); 508 return -ENOENT; 509 } 510 511 /* Map per-CPU SGEI interrupt from INTC domain */ 512 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); 513 if (!hgei_parent_irq) { 514 kvm_err("unable to map SGEI IRQ\n"); 515 return -ENOMEM; 516 } 517 518 /* Request per-CPU SGEI interrupt */ 519 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt, 520 "riscv-kvm", &aia_hgei); 521 if (rc) { 522 kvm_err("failed to request SGEI IRQ\n"); 523 return rc; 524 } 525 526 return 0; 527 } 528 529 static void aia_hgei_exit(void) 530 { 531 /* Free per-CPU SGEI interrupt */ 532 free_percpu_irq(hgei_parent_irq, &aia_hgei); 533 } 534 535 void kvm_riscv_aia_enable(void) 536 { 537 const struct imsic_global_config *gc; 538 const struct imsic_local_config *lc; 539 struct aia_hgei_control *hgctrl; 540 unsigned long flags; 541 int aia_nr_hgei; 542 543 if (!kvm_riscv_aia_available()) 544 return; 545 546 gc = imsic_get_global_config(); 547 lc = (gc) ? this_cpu_ptr(gc->local) : NULL; 548 hgctrl = this_cpu_ptr(&aia_hgei); 549 550 /* Figure-out number of bits in HGEIE */ 551 csr_write(CSR_HGEIE, -1UL); 552 hgctrl->nr_hgei = fls_long(csr_read(CSR_HGEIE)); 553 csr_write(CSR_HGEIE, 0); 554 if (hgctrl->nr_hgei) 555 hgctrl->nr_hgei--; 556 557 /* 558 * Number of usable per-HART HGEI lines should be minimum of 559 * per-HART IMSIC guest files and number of bits in HGEIE. 560 */ 561 if (lc) 562 hgctrl->nr_hgei = min((ulong)hgctrl->nr_hgei, lc->nr_guest_files); 563 else 564 hgctrl->nr_hgei = 0; 565 566 /* Update the number of IMSIC guest files across all HARTs */ 567 aia_nr_hgei = atomic_read(&kvm_riscv_aia_nr_hgei); 568 do { 569 if (aia_nr_hgei <= hgctrl->nr_hgei) 570 break; 571 } while (!atomic_try_cmpxchg(&kvm_riscv_aia_nr_hgei, &aia_nr_hgei, hgctrl->nr_hgei)); 572 573 raw_spin_lock_irqsave(&hgctrl->lock, flags); 574 if (!hgctrl->free_bitmap_initialized) { 575 hgctrl->free_bitmap = (hgctrl->nr_hgei) ? GENMASK_ULL(hgctrl->nr_hgei, 1) : 0; 576 hgctrl->free_bitmap_initialized = true; 577 } 578 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 579 580 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 581 csr_write(CSR_HVIPRIO1, 0x0); 582 csr_write(CSR_HVIPRIO2, 0x0); 583 #ifdef CONFIG_32BIT 584 csr_write(CSR_HVIPH, 0x0); 585 csr_write(CSR_HIDELEGH, 0x0); 586 csr_write(CSR_HVIPRIO1H, 0x0); 587 csr_write(CSR_HVIPRIO2H, 0x0); 588 #endif 589 590 /* Enable per-CPU SGEI interrupt */ 591 enable_percpu_irq(hgei_parent_irq, 592 irq_get_trigger_type(hgei_parent_irq)); 593 csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); 594 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */ 595 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 596 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 597 } 598 599 void kvm_riscv_aia_disable(void) 600 { 601 int i; 602 unsigned long flags; 603 struct kvm_vcpu *vcpu; 604 struct aia_hgei_control *hgctrl; 605 606 if (!kvm_riscv_aia_available()) 607 return; 608 hgctrl = get_cpu_ptr(&aia_hgei); 609 610 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF)) 611 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF)); 612 /* Disable per-CPU SGEI interrupt */ 613 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); 614 disable_percpu_irq(hgei_parent_irq); 615 616 csr_write(CSR_HVICTL, aia_hvictl_value(false)); 617 618 raw_spin_lock_irqsave(&hgctrl->lock, flags); 619 620 for (i = 0; i <= hgctrl->nr_hgei; i++) { 621 vcpu = hgctrl->owners[i]; 622 if (!vcpu) 623 continue; 624 625 /* 626 * We release hgctrl->lock before notifying IMSIC 627 * so that we don't have lock ordering issues. 628 */ 629 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 630 631 /* Notify IMSIC */ 632 kvm_riscv_vcpu_aia_imsic_release(vcpu); 633 634 /* 635 * Wakeup VCPU if it was blocked so that it can 636 * run on other HARTs 637 */ 638 if (csr_read(CSR_HGEIE) & BIT(i)) { 639 csr_clear(CSR_HGEIE, BIT(i)); 640 kvm_vcpu_kick(vcpu); 641 } 642 643 raw_spin_lock_irqsave(&hgctrl->lock, flags); 644 } 645 646 raw_spin_unlock_irqrestore(&hgctrl->lock, flags); 647 648 put_cpu_ptr(&aia_hgei); 649 } 650 651 int kvm_riscv_aia_init(void) 652 { 653 int rc; 654 const struct imsic_global_config *gc; 655 656 if (!riscv_isa_extension_available(NULL, SxAIA)) 657 return -ENODEV; 658 gc = imsic_get_global_config(); 659 660 /* Set initial value of IMSIC guest files across all HARTs */ 661 if (gc) 662 atomic_set(&kvm_riscv_aia_nr_hgei, gc->nr_guest_files); 663 else 664 atomic_set(&kvm_riscv_aia_nr_hgei, 0); 665 666 /* Find number of guest MSI IDs */ 667 kvm_riscv_aia_max_ids = IMSIC_MAX_ID; 668 if (gc) 669 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1; 670 671 /* Initialize guest external interrupt line management */ 672 rc = aia_hgei_init(); 673 if (rc) 674 return rc; 675 676 /* Register device operations */ 677 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, 678 KVM_DEV_TYPE_RISCV_AIA); 679 if (rc) { 680 aia_hgei_exit(); 681 return rc; 682 } 683 684 /* Enable KVM AIA support */ 685 static_branch_enable(&kvm_riscv_aia_available); 686 687 return 0; 688 } 689 690 void kvm_riscv_aia_exit(void) 691 { 692 if (!kvm_riscv_aia_available()) 693 return; 694 695 /* Unregister device operations */ 696 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); 697 698 /* Cleanup the HGEI state */ 699 aia_hgei_exit(); 700 } 701