Lines Matching +full:buffered +full:- +full:positive
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
5 * Copyright (C) 2020-2022 Andrew Turner
79 #define VGIC_SGI_NUM (GIC_LAST_SGI - GIC_FIRST_SGI + 1)
80 #define VGIC_PPI_NUM (GIC_LAST_PPI - GIC_FIRST_PPI + 1)
81 #define VGIC_SPI_NUM (GIC_LAST_SPI - GIC_FIRST_SPI + 1)
127 /* Per-CPU data not needed by EL2 */
209 /* GICD_STATUSR - RAZ/WI as we don't report errors (yet) */
212 /* GICD_SETSPI_SR - RAZ/WI */
213 /* GICD_CLRSPI_SR - RAZ/WI */
214 /* GICD_IGROUPR - RAZ/WI as GICD_CTLR.ARE == 1 */
236 /* GICD_ITARGETSR - RAZ/WI as GICD_CTLR.ARE == 1 */
240 /* GICD_IGRPMODR - RAZ/WI from non-secure mode */
241 /* GICD_NSACR - RAZ/WI from non-secure mode */
242 /* GICD_SGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
243 /* GICD_CPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
244 /* GICD_SPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
327 /* GICR_CTLR - Ignore writes as no bits can be set */
333 /* GICR_STATUSR - RAZ/WI as we don't report errors (yet) */
334 /* GICR_WAKER - RAZ/WI from non-secure mode */
335 /* GICR_SETLPIR - RAZ/WI as no LPIs are supported */
336 /* GICR_CLRLPIR - RAZ/WI as no LPIs are supported */
337 /* GICR_PROPBASER - RAZ/WI as no LPIs are supported */
338 /* GICR_PENDBASER - RAZ/WI as no LPIs are supported */
339 /* GICR_INVLPIR - RAZ/WI as no LPIs are supported */
340 /* GICR_INVALLR - RAZ/WI as no LPIs are supported */
341 /* GICR_SYNCR - RAZ/WI as no LPIs are supported */
368 /* GICR_IGROUPR0 - RAZ/WI from non-secure mode */
387 /* GICR_ICFGR0 - RAZ/WI from non-secure mode */
391 /* GICR_IGRPMODR0 - RAZ/WI from non-secure mode */
392 /* GICR_NSCAR - RAZ/WI from non-secure mode */
430 vm = hyp->vm; in mpidr_to_vcpu()
432 hypctx = hyp->ctx[i]; in mpidr_to_vcpu()
433 if (hypctx != NULL && (hypctx->vmpidr_el2 & GICD_AFF) == mpidr) in mpidr_to_vcpu()
436 return (-1); in mpidr_to_vcpu()
444 hyp->vgic = malloc(sizeof(*hyp->vgic), M_VGIC_V3, in vgic_v3_vminit()
446 vgic = hyp->vgic; in vgic_v3_vminit()
455 * GICv3 and GICv4, p. 4-464) in vgic_v3_vminit()
457 vgic->gicd_ctlr = 0; in vgic_v3_vminit()
459 mtx_init(&vgic->dist_mtx, "VGICv3 Distributor lock", NULL, in vgic_v3_vminit()
470 hypctx->vgic_cpu = malloc(sizeof(*hypctx->vgic_cpu), in vgic_v3_cpuinit()
472 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_cpuinit()
474 mtx_init(&vgic_cpu->lr_mtx, "VGICv3 ICH_LR_EL2 lock", NULL, MTX_SPIN); in vgic_v3_cpuinit()
478 irq = &vgic_cpu->private_irqs[irqid]; in vgic_v3_cpuinit()
480 mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL, in vgic_v3_cpuinit()
482 irq->irq = irqid; in vgic_v3_cpuinit()
483 irq->mpidr = hypctx->vmpidr_el2 & GICD_AFF; in vgic_v3_cpuinit()
484 irq->target_vcpu = vcpu_vcpuid(hypctx->vcpu); in vgic_v3_cpuinit()
485 MPASS(irq->target_vcpu >= 0); in vgic_v3_cpuinit()
489 irq->enabled = true; in vgic_v3_cpuinit()
490 irq->config = VGIC_CONFIG_EDGE; in vgic_v3_cpuinit()
493 irq->config = VGIC_CONFIG_LEVEL; in vgic_v3_cpuinit()
495 irq->priority = 0; in vgic_v3_cpuinit()
505 hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En; in vgic_v3_cpuinit()
520 hypctx->vgic_v3_regs.ich_vmcr_el2 = in vgic_v3_cpuinit()
523 hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM; in vgic_v3_cpuinit()
524 hypctx->vgic_v3_regs.ich_vmcr_el2 |= ICH_VMCR_EL2_VENG0 | in vgic_v3_cpuinit()
527 hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num; in vgic_v3_cpuinit()
528 for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++) in vgic_v3_cpuinit()
529 hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL; in vgic_v3_cpuinit()
530 vgic_cpu->ich_lr_used = 0; in vgic_v3_cpuinit()
531 TAILQ_INIT(&vgic_cpu->irq_act_pend); in vgic_v3_cpuinit()
533 hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num; in vgic_v3_cpuinit()
543 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_cpucleanup()
545 irq = &vgic_cpu->private_irqs[irqid]; in vgic_v3_cpucleanup()
546 mtx_destroy(&irq->irq_spinmtx); in vgic_v3_cpucleanup()
549 mtx_destroy(&vgic_cpu->lr_mtx); in vgic_v3_cpucleanup()
550 free(hypctx->vgic_cpu, M_VGIC_V3); in vgic_v3_cpucleanup()
556 mtx_destroy(&hyp->vgic->dist_mtx); in vgic_v3_vmcleanup()
557 free(hyp->vgic, M_VGIC_V3); in vgic_v3_vmcleanup()
567 vgic = hyp->vgic; in vgic_v3_max_cpu_count()
568 max_count = vm_get_maxcpus(hyp->vm); in vgic_v3_max_cpu_count()
571 if (vgic->redist_start == 0 && vgic->redist_end == 0) in vgic_v3_max_cpu_count()
574 count = (vgic->redist_end - vgic->redist_start) / in vgic_v3_max_cpu_count()
579 * to a positive integer value. in vgic_v3_max_cpu_count()
590 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL) { in vgic_v3_irq_pending()
591 return (irq->pending || irq->level); in vgic_v3_irq_pending()
593 return (irq->pending); in vgic_v3_irq_pending()
602 MPASS(vcpuid < vm_get_maxcpus(hyp->vm)); in vgic_v3_queue_irq()
604 mtx_assert(&vgic_cpu->lr_mtx, MA_OWNED); in vgic_v3_queue_irq()
605 mtx_assert(&irq->irq_spinmtx, MA_OWNED); in vgic_v3_queue_irq()
608 if (!irq->level && !irq->pending) in vgic_v3_queue_irq()
611 if (!irq->on_aplist) { in vgic_v3_queue_irq()
612 irq->on_aplist = true; in vgic_v3_queue_irq()
613 TAILQ_INSERT_TAIL(&vgic_cpu->irq_act_pend, irq, act_pend_list); in vgic_v3_queue_irq()
624 mask = ((1ul << (size * 8)) - 1) << (offset * 8); in gic_reg_value_64()
643 /* Common read-only/write-ignored helpers */
669 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in read_enabler()
674 if (!irq->enabled) in read_enabler()
696 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in write_enabler()
701 irq->enabled = set; in write_enabler()
717 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in read_pendr()
742 hyp = hypctx->hyp; in write_pendr()
750 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in write_pendr()
756 target_vcpu = irq->target_vcpu; in write_pendr()
759 target_hypctx = hyp->ctx[target_vcpu]; in write_pendr()
762 vgic_cpu = target_hypctx->vgic_cpu; in write_pendr()
765 /* pending -> not pending */ in write_pendr()
766 irq->pending = false; in write_pendr()
768 irq->pending = true; in write_pendr()
769 mtx_lock_spin(&vgic_cpu->lr_mtx); in write_pendr()
772 mtx_unlock_spin(&vgic_cpu->lr_mtx); in write_pendr()
778 vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu)); in write_pendr()
795 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in read_activer()
800 if (irq->active) in read_activer()
819 hyp = hypctx->hyp; in write_activer()
826 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in write_activer()
832 target_vcpu = irq->target_vcpu; in write_activer()
835 target_hypctx = hyp->ctx[target_vcpu]; in write_activer()
838 vgic_cpu = target_hypctx->vgic_cpu; in write_activer()
841 /* active -> not active */ in write_activer()
842 irq->active = false; in write_activer()
844 /* not active -> active */ in write_activer()
845 irq->active = true; in write_activer()
846 mtx_lock_spin(&vgic_cpu->lr_mtx); in write_activer()
849 mtx_unlock_spin(&vgic_cpu->lr_mtx); in write_activer()
855 vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu)); in write_activer()
870 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in read_priorityr()
875 ret |= ((uint64_t)irq->priority) << (i * 8); in read_priorityr()
889 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in write_priorityr()
895 irq->priority = (val >> (i * 8)) & 0xf8; in write_priorityr()
911 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in read_config()
916 ret |= ((uint64_t)irq->config) << (i * 2); in read_config()
934 * an edge-triggered behaviour, and the register is in write_config()
935 * implementation defined to be read-only for PPIs. in write_config()
940 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in write_config()
946 irq->config = (val >> (i * 2)) & VGIC_CONFIG_MASK; in write_config()
957 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n); in read_route()
961 mpidr = irq->mpidr; in read_route()
973 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n); in write_route()
977 irq->mpidr = gic_reg_value_64(irq->mpidr, val, offset, size) & GICD_AFF; in write_route()
978 irq->target_vcpu = mpidr_to_vcpu(hypctx->hyp, irq->mpidr); in write_route()
999 hyp = hypctx->hyp; in dist_ctlr_read()
1000 vgic = hyp->vgic; in dist_ctlr_read()
1002 mtx_lock_spin(&vgic->dist_mtx); in dist_ctlr_read()
1003 *rval = vgic->gicd_ctlr; in dist_ctlr_read()
1004 mtx_unlock_spin(&vgic->dist_mtx); in dist_ctlr_read()
1018 vgic = hypctx->hyp->vgic; in dist_ctlr_write()
1024 * EnableGrp1A is supported, and RWP is read-only. in dist_ctlr_write()
1026 * All other bits are RES0 from non-secure mode as we in dist_ctlr_write()
1032 mtx_lock_spin(&vgic->dist_mtx); in dist_ctlr_write()
1033 vgic->gicd_ctlr = wval; in dist_ctlr_write()
1035 mtx_unlock_spin(&vgic->dist_mtx); in dist_ctlr_write()
1045 typer = (10 - 1) << GICD_TYPER_IDBITS_SHIFT; in dist_typer_read()
1048 typer |= howmany(VGIC_NIRQS + 1, 32) - 1; in dist_typer_read()
1070 INJECT_IRQ(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), irqid, in dist_setclrspi_nsr_write()
1080 n = (reg - GICD_ISENABLER(0)) / 4; in dist_isenabler_read()
1094 n = (reg - GICD_ISENABLER(0)) / 4; in dist_isenabler_write()
1106 n = (reg - GICD_ICENABLER(0)) / 4; in dist_icenabler_read()
1120 n = (reg - GICD_ISENABLER(0)) / 4; in dist_icenabler_write()
1132 n = (reg - GICD_ISPENDR(0)) / 4; in dist_ispendr_read()
1146 n = (reg - GICD_ISPENDR(0)) / 4; in dist_ispendr_write()
1158 n = (reg - GICD_ICPENDR(0)) / 4; in dist_icpendr_read()
1172 n = (reg - GICD_ICPENDR(0)) / 4; in dist_icpendr_write()
1185 n = (reg - GICD_ISACTIVER(0)) / 4; in dist_isactiver_read()
1199 n = (reg - GICD_ISACTIVER(0)) / 4; in dist_isactiver_write()
1212 n = (reg - GICD_ICACTIVER(0)) / 4; in dist_icactiver_read()
1226 n = (reg - GICD_ICACTIVER(0)) / 4; in dist_icactiver_write()
1233 /* Affinity routing is enabled so ipriorityr0-7 is RAZ/WI */
1240 n = (reg - GICD_IPRIORITYR(0)) / 4; in dist_ipriorityr_read()
1241 /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */ in dist_ipriorityr_read()
1252 irq_base = (reg - GICD_IPRIORITYR(0)) + offset; in dist_ipriorityr_write()
1253 /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */ in dist_ipriorityr_write()
1264 n = (reg - GICD_ICFGR(0)) / 4; in dist_icfgr_read()
1265 /* GICD_ICFGR0-1 are RAZ/WI so handled separately */ in dist_icfgr_read()
1278 n = (reg - GICD_ICFGR(0)) / 4; in dist_icfgr_write()
1279 /* GICD_ICFGR0-1 are RAZ/WI so handled separately */ in dist_icfgr_write()
1290 n = (reg - GICD_IROUTER(0)) / 8; in dist_irouter_read()
1291 /* GICD_IROUTER0-31 don't exist */ in dist_irouter_read()
1302 n = (reg - GICD_IROUTER(0)) / 8; in dist_irouter_write()
1303 /* GICD_IROUTER0-31 don't exist */ in dist_irouter_write()
1316 offset = reg & (reg_list[i].size - 1); in vgic_register_read()
1317 reg -= offset; in vgic_register_read()
1324 *rval &= (1ul << (size * 8)) - 1; in vgic_register_read()
1329 * 12.1.3 "GIC memory-mapped register access" in vgic_register_read()
1351 offset = reg & (reg_list[i].size - 1); in vgic_register_write()
1352 reg -= offset; in vgic_register_write()
1379 hyp = hypctx->hyp; in dist_read()
1380 vgic = hyp->vgic; in dist_read()
1383 if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) { in dist_read()
1387 reg = fault_ipa - vgic->dist_start; in dist_read()
1392 if ((reg & (size - 1)) != 0) { in dist_read()
1417 hyp = hypctx->hyp; in dist_write()
1418 vgic = hyp->vgic; in dist_write()
1421 if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) { in dist_write()
1425 reg = fault_ipa - vgic->dist_start; in dist_write()
1430 if ((reg & (size - 1)) != 0) in dist_write()
1469 if (vcpu_vcpuid(hypctx->vcpu) == (vgic_max_cpu_count(hypctx->hyp) - 1)) in redist_typer_read()
1472 vmpidr_el2 = hypctx->vmpidr_el2; in redist_typer_read()
1486 (uint64_t)vcpu_vcpuid(hypctx->vcpu) << GICR_TYPER_CPUNUM_SHIFT; in redist_typer_read()
1582 n = (reg - GICR_IPRIORITYR(0)) / 4; in redist_ipriorityr_read()
1592 irq_base = (reg - GICR_IPRIORITYR(0)) + offset; in redist_ipriorityr_write()
1624 hyp = hypctx->hyp; in redist_read()
1625 vgic = hyp->vgic; in redist_read()
1628 if (fault_ipa < vgic->redist_start || in redist_read()
1629 fault_ipa + size > vgic->redist_end) { in redist_read()
1633 vcpuid = (fault_ipa - vgic->redist_start) / in redist_read()
1635 if (vcpuid >= vm_get_maxcpus(hyp->vm)) { in redist_read()
1638 * does we don't panic a non-INVARIANTS kernel. in redist_read()
1649 target_hypctx = hyp->ctx[vcpuid]; in redist_read()
1662 reg = (fault_ipa - vgic->redist_start) % in redist_read()
1669 if ((reg & (size - 1)) != 0) { in redist_read()
1680 nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size, in redist_read()
1702 hyp = hypctx->hyp; in redist_write()
1703 vgic = hyp->vgic; in redist_write()
1706 if (fault_ipa < vgic->redist_start || in redist_write()
1707 fault_ipa + size > vgic->redist_end) { in redist_write()
1711 vcpuid = (fault_ipa - vgic->redist_start) / in redist_write()
1713 if (vcpuid >= vm_get_maxcpus(hyp->vm)) { in redist_write()
1716 * does we don't panic a non-INVARIANTS kernel. in redist_write()
1726 target_hypctx = hyp->ctx[vcpuid]; in redist_write()
1739 reg = (fault_ipa - vgic->redist_start) % in redist_write()
1746 if ((reg & (size - 1)) != 0) in redist_write()
1755 nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size, in redist_write()
1791 /* Non-zero points at no vcpus */ in vgic_v3_icc_sgi1r_write()
1838 vgic = hyp->vgic; in vgic_v3_mmio_init()
1839 vgic->irqs = malloc((VGIC_NIRQS - VGIC_PRV_I_NUM) * in vgic_v3_mmio_init()
1840 sizeof(*vgic->irqs), M_VGIC_V3, M_WAITOK | M_ZERO); in vgic_v3_mmio_init()
1842 for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) { in vgic_v3_mmio_init()
1843 irq = &vgic->irqs[i]; in vgic_v3_mmio_init()
1845 mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL, in vgic_v3_mmio_init()
1848 irq->irq = i + VGIC_PRV_I_NUM; in vgic_v3_mmio_init()
1859 vgic = hyp->vgic; in vgic_v3_mmio_destroy()
1860 for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) { in vgic_v3_mmio_destroy()
1861 irq = &vgic->irqs[i]; in vgic_v3_mmio_destroy()
1863 mtx_destroy(&irq->irq_spinmtx); in vgic_v3_mmio_destroy()
1866 free(vgic->irqs, M_VGIC_V3); in vgic_v3_mmio_destroy()
1876 if (descr->ver.version != 3) in vgic_v3_attach_to_vm()
1883 if (!__is_aligned(descr->v3_regs.dist_start, PAGE_SIZE_64K) || in vgic_v3_attach_to_vm()
1884 !__is_aligned(descr->v3_regs.redist_start, PAGE_SIZE_64K) || in vgic_v3_attach_to_vm()
1885 !__is_aligned(descr->v3_regs.redist_size, in vgic_v3_attach_to_vm()
1890 if (descr->v3_regs.dist_size != PAGE_SIZE_64K) in vgic_v3_attach_to_vm()
1893 vm = hyp->vm; in vgic_v3_attach_to_vm()
1899 cpu_count = descr->v3_regs.redist_size / in vgic_v3_attach_to_vm()
1904 vgic = hyp->vgic; in vgic_v3_attach_to_vm()
1907 vgic->dist_start = descr->v3_regs.dist_start; in vgic_v3_attach_to_vm()
1908 vgic->dist_end = descr->v3_regs.dist_start + descr->v3_regs.dist_size; in vgic_v3_attach_to_vm()
1910 vgic->redist_start = descr->v3_regs.redist_start; in vgic_v3_attach_to_vm()
1911 vgic->redist_end = descr->v3_regs.redist_start + in vgic_v3_attach_to_vm()
1912 descr->v3_regs.redist_size; in vgic_v3_attach_to_vm()
1914 vm_register_inst_handler(vm, descr->v3_regs.dist_start, in vgic_v3_attach_to_vm()
1915 descr->v3_regs.dist_size, dist_read, dist_write); in vgic_v3_attach_to_vm()
1916 vm_register_inst_handler(vm, descr->v3_regs.redist_start, in vgic_v3_attach_to_vm()
1917 descr->v3_regs.redist_size, redist_read, redist_write); in vgic_v3_attach_to_vm()
1925 hyp->vgic_attached = true; in vgic_v3_attach_to_vm()
1933 if (hyp->vgic_attached) { in vgic_v3_detach_from_vm()
1934 hyp->vgic_attached = false; in vgic_v3_detach_from_vm()
1947 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm)) in vgic_v3_get_irq()
1949 hypctx = hyp->ctx[vcpuid]; in vgic_v3_get_irq()
1952 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_get_irq()
1953 irq = &vgic_cpu->private_irqs[irqid]; in vgic_v3_get_irq()
1955 irqid -= VGIC_PRV_I_NUM; in vgic_v3_get_irq()
1958 irq = &hyp->vgic->irqs[irqid]; in vgic_v3_get_irq()
1966 mtx_lock_spin(&irq->irq_spinmtx); in vgic_v3_get_irq()
1974 mtx_unlock_spin(&irq->irq_spinmtx); in vgic_v3_release_irq()
1983 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_has_pending_irq()
1984 mtx_lock_spin(&vgic_cpu->lr_mtx); in vgic_v3_has_pending_irq()
1985 empty = TAILQ_EMPTY(&vgic_cpu->irq_act_pend); in vgic_v3_has_pending_irq()
1986 mtx_unlock_spin(&vgic_cpu->lr_mtx); in vgic_v3_has_pending_irq()
1996 * - Level-triggered IRQ: level changes low -> high in vgic_v3_check_irq()
1997 * - Edge-triggered IRQ: level is high in vgic_v3_check_irq()
1999 switch (irq->config & VGIC_CONFIG_MASK) { in vgic_v3_check_irq()
2001 return (level != irq->level); in vgic_v3_check_irq()
2021 if (!hyp->vgic_attached) in vgic_v3_inject_irq()
2024 KASSERT(vcpuid == -1 || irqid < VGIC_PRV_I_NUM, in vgic_v3_inject_irq()
2034 target_vcpu = irq->target_vcpu; in vgic_v3_inject_irq()
2035 KASSERT(vcpuid == -1 || vcpuid == target_vcpu, in vgic_v3_inject_irq()
2038 KASSERT(target_vcpu >= 0 && target_vcpu < vm_get_maxcpus(hyp->vm), in vgic_v3_inject_irq()
2042 if (vcpuid == -1) in vgic_v3_inject_irq()
2044 /* TODO: Check from 0 to vm->maxcpus */ in vgic_v3_inject_irq()
2045 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm)) { in vgic_v3_inject_irq()
2050 hypctx = hyp->ctx[vcpuid]; in vgic_v3_inject_irq()
2057 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_inject_irq()
2059 mtx_lock_spin(&vgic_cpu->lr_mtx); in vgic_v3_inject_irq()
2065 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL) in vgic_v3_inject_irq()
2066 irq->level = level; in vgic_v3_inject_irq()
2068 irq->pending = true; in vgic_v3_inject_irq()
2073 mtx_unlock_spin(&vgic_cpu->lr_mtx); in vgic_v3_inject_irq()
2077 vcpu_notify_event(vm_vcpu(hyp->vm, vcpuid)); in vgic_v3_inject_irq()
2088 vgic = hyp->vgic; in vgic_v3_inject_msi()
2091 if (addr < vgic->dist_start || addr + 4 > vgic->dist_end) { in vgic_v3_inject_msi()
2095 reg = addr - vgic->dist_start; in vgic_v3_inject_msi()
2099 return (INJECT_IRQ(hyp, -1, msg, true)); in vgic_v3_inject_msi()
2109 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_flush_hwstate()
2118 mtx_lock_spin(&vgic_cpu->lr_mtx); in vgic_v3_flush_hwstate()
2120 hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE; in vgic_v3_flush_hwstate()
2122 /* Exit early if there are no buffered interrupts */ in vgic_v3_flush_hwstate()
2123 if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend)) in vgic_v3_flush_hwstate()
2126 KASSERT(vgic_cpu->ich_lr_used == 0, ("%s: Used LR count not zero %u", in vgic_v3_flush_hwstate()
2127 __func__, vgic_cpu->ich_lr_used)); in vgic_v3_flush_hwstate()
2130 hypctx->vgic_v3_regs.ich_elrsr_el2 = in vgic_v3_flush_hwstate()
2131 (1u << hypctx->vgic_v3_regs.ich_lr_num) - 1; in vgic_v3_flush_hwstate()
2132 TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) { in vgic_v3_flush_hwstate()
2134 if (i == hypctx->vgic_v3_regs.ich_lr_num) in vgic_v3_flush_hwstate()
2137 if (!irq->enabled) in vgic_v3_flush_hwstate()
2140 hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 | in vgic_v3_flush_hwstate()
2141 ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) | in vgic_v3_flush_hwstate()
2142 irq->irq; in vgic_v3_flush_hwstate()
2144 if (irq->active) { in vgic_v3_flush_hwstate()
2145 hypctx->vgic_v3_regs.ich_lr_el2[i] |= in vgic_v3_flush_hwstate()
2151 if ((irq->config & _MASK) == LEVEL) in vgic_v3_flush_hwstate()
2152 hypctx->vgic_v3_regs.ich_lr_el2[i] |= ICH_LR_EL2_EOI; in vgic_v3_flush_hwstate()
2155 if (!irq->active && vgic_v3_irq_pending(irq)) { in vgic_v3_flush_hwstate()
2156 hypctx->vgic_v3_regs.ich_lr_el2[i] |= in vgic_v3_flush_hwstate()
2164 if ((irq->config & VGIC_CONFIG_MASK) == in vgic_v3_flush_hwstate()
2166 irq->pending = false; in vgic_v3_flush_hwstate()
2172 vgic_cpu->ich_lr_used = i; in vgic_v3_flush_hwstate()
2175 mtx_unlock_spin(&vgic_cpu->lr_mtx); in vgic_v3_flush_hwstate()
2186 vgic_cpu = hypctx->vgic_cpu; in vgic_v3_sync_hwstate()
2188 /* Exit early if there are no buffered interrupts */ in vgic_v3_sync_hwstate()
2189 if (vgic_cpu->ich_lr_used == 0) in vgic_v3_sync_hwstate()
2197 for (i = 0; i < vgic_cpu->ich_lr_used; i++) { in vgic_v3_sync_hwstate()
2198 lr = hypctx->vgic_v3_regs.ich_lr_el2[i]; in vgic_v3_sync_hwstate()
2199 hypctx->vgic_v3_regs.ich_lr_el2[i] = 0; in vgic_v3_sync_hwstate()
2201 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), in vgic_v3_sync_hwstate()
2206 irq->active = (lr & ICH_LR_EL2_STATE_ACTIVE) != 0; in vgic_v3_sync_hwstate()
2208 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_EDGE) { in vgic_v3_sync_hwstate()
2214 irq->pending = true; in vgic_v3_sync_hwstate()
2224 irq->pending = false; in vgic_v3_sync_hwstate()
2229 mtx_lock_spin(&vgic_cpu->lr_mtx); in vgic_v3_sync_hwstate()
2230 if (irq->active) { in vgic_v3_sync_hwstate()
2232 TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq, in vgic_v3_sync_hwstate()
2234 TAILQ_INSERT_HEAD(&vgic_cpu->irq_act_pend, irq, in vgic_v3_sync_hwstate()
2238 TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq, in vgic_v3_sync_hwstate()
2240 irq->on_aplist = false; in vgic_v3_sync_hwstate()
2242 mtx_unlock_spin(&vgic_cpu->lr_mtx); in vgic_v3_sync_hwstate()
2246 hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK; in vgic_v3_sync_hwstate()
2247 vgic_cpu->ich_lr_used = 0; in vgic_v3_sync_hwstate()