10a110d5bSKonstantin Belousov /*- 20a110d5bSKonstantin Belousov * Copyright (c) 2015 The FreeBSD Foundation 30a110d5bSKonstantin Belousov * All rights reserved. 40a110d5bSKonstantin Belousov * 50a110d5bSKonstantin Belousov * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 60a110d5bSKonstantin Belousov * under sponsorship from the FreeBSD Foundation. 70a110d5bSKonstantin Belousov * 80a110d5bSKonstantin Belousov * Redistribution and use in source and binary forms, with or without 90a110d5bSKonstantin Belousov * modification, are permitted provided that the following conditions 100a110d5bSKonstantin Belousov * are met: 110a110d5bSKonstantin Belousov * 1. Redistributions of source code must retain the above copyright 120a110d5bSKonstantin Belousov * notice, this list of conditions and the following disclaimer. 130a110d5bSKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright 140a110d5bSKonstantin Belousov * notice, this list of conditions and the following disclaimer in the 150a110d5bSKonstantin Belousov * documentation and/or other materials provided with the distribution. 160a110d5bSKonstantin Belousov * 170a110d5bSKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 180a110d5bSKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 190a110d5bSKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 200a110d5bSKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 210a110d5bSKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 220a110d5bSKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 230a110d5bSKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 240a110d5bSKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 250a110d5bSKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 260a110d5bSKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 270a110d5bSKonstantin Belousov * SUCH DAMAGE. 280a110d5bSKonstantin Belousov */ 290a110d5bSKonstantin Belousov 300a110d5bSKonstantin Belousov #include <sys/cdefs.h> 310a110d5bSKonstantin Belousov __FBSDID("$FreeBSD$"); 320a110d5bSKonstantin Belousov 330a110d5bSKonstantin Belousov #include <sys/param.h> 340a110d5bSKonstantin Belousov #include <sys/systm.h> 350a110d5bSKonstantin Belousov #include <sys/bus.h> 360a110d5bSKonstantin Belousov #include <sys/kernel.h> 370a110d5bSKonstantin Belousov #include <sys/lock.h> 380a110d5bSKonstantin Belousov #include <sys/malloc.h> 390a110d5bSKonstantin Belousov #include <sys/memdesc.h> 40e2e050c8SConrad Meyer #include <sys/mutex.h> 410a110d5bSKonstantin Belousov #include <sys/rman.h> 420a110d5bSKonstantin Belousov #include <sys/rwlock.h> 430a110d5bSKonstantin Belousov #include <sys/taskqueue.h> 440a110d5bSKonstantin Belousov #include <sys/tree.h> 450a110d5bSKonstantin Belousov #include <sys/vmem.h> 460a110d5bSKonstantin Belousov #include <machine/bus.h> 470a110d5bSKonstantin Belousov #include <machine/intr_machdep.h> 480a110d5bSKonstantin Belousov #include <vm/vm.h> 490a110d5bSKonstantin Belousov #include <vm/vm_extern.h> 500a110d5bSKonstantin Belousov #include <vm/vm_kern.h> 510a110d5bSKonstantin Belousov #include <vm/vm_object.h> 520a110d5bSKonstantin Belousov #include <vm/vm_page.h> 530a110d5bSKonstantin Belousov #include <x86/include/apicreg.h> 540a110d5bSKonstantin Belousov #include <x86/include/apicvar.h> 550a110d5bSKonstantin Belousov #include <x86/include/busdma_impl.h> 560a110d5bSKonstantin Belousov #include <x86/iommu/intel_reg.h> 57*f2b2f317SRuslan Bukin #include <dev/iommu/busdma_iommu.h> 58685666aaSKonstantin Belousov #include <dev/pci/pcireg.h> 590a110d5bSKonstantin Belousov #include <x86/iommu/intel_dmar.h> 600a110d5bSKonstantin Belousov #include <dev/pci/pcivar.h> 610a110d5bSKonstantin Belousov #include <x86/iommu/iommu_intrmap.h> 620a110d5bSKonstantin Belousov 630a110d5bSKonstantin Belousov static struct dmar_unit *dmar_ir_find(device_t src, uint16_t *rid, 640a110d5bSKonstantin Belousov int *is_dmar); 650a110d5bSKonstantin Belousov static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, 660a110d5bSKonstantin Belousov uint64_t low, uint16_t rid); 670a110d5bSKonstantin Belousov static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie); 680a110d5bSKonstantin Belousov 690a110d5bSKonstantin Belousov int 700a110d5bSKonstantin Belousov iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) 710a110d5bSKonstantin Belousov { 720a110d5bSKonstantin Belousov struct dmar_unit *unit; 730a110d5bSKonstantin Belousov vmem_addr_t vmem_res; 740a110d5bSKonstantin Belousov u_int idx, i; 750a110d5bSKonstantin Belousov int error; 760a110d5bSKonstantin Belousov 770a110d5bSKonstantin Belousov unit = dmar_ir_find(src, NULL, NULL); 780a110d5bSKonstantin Belousov if (unit == NULL || !unit->ir_enabled) { 790a110d5bSKonstantin Belousov for (i = 0; i < count; i++) 800a110d5bSKonstantin Belousov cookies[i] = -1; 810a110d5bSKonstantin Belousov return (EOPNOTSUPP); 820a110d5bSKonstantin Belousov } 830a110d5bSKonstantin Belousov 840a110d5bSKonstantin Belousov error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT, 850a110d5bSKonstantin Belousov &vmem_res); 860a110d5bSKonstantin Belousov if (error != 0) { 870a110d5bSKonstantin Belousov KASSERT(error != EOPNOTSUPP, 880a110d5bSKonstantin Belousov ("impossible EOPNOTSUPP from vmem")); 890a110d5bSKonstantin Belousov return (error); 900a110d5bSKonstantin Belousov } 910a110d5bSKonstantin Belousov idx = vmem_res; 920a110d5bSKonstantin Belousov for (i = 0; i < count; i++) 930a110d5bSKonstantin Belousov cookies[i] = idx + i; 940a110d5bSKonstantin Belousov return (0); 950a110d5bSKonstantin Belousov } 960a110d5bSKonstantin Belousov 970a110d5bSKonstantin Belousov int 980a110d5bSKonstantin Belousov iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, 990a110d5bSKonstantin Belousov uint64_t *addr, uint32_t *data) 1000a110d5bSKonstantin Belousov { 1010a110d5bSKonstantin Belousov struct dmar_unit *unit; 1020a110d5bSKonstantin Belousov uint64_t low; 1030a110d5bSKonstantin Belousov uint16_t rid; 1040a110d5bSKonstantin Belousov int is_dmar; 1050a110d5bSKonstantin Belousov 1060a110d5bSKonstantin Belousov unit = dmar_ir_find(src, &rid, &is_dmar); 1070a110d5bSKonstantin Belousov if (is_dmar) { 1080a110d5bSKonstantin Belousov KASSERT(unit == NULL, ("DMAR cannot translate itself")); 1090a110d5bSKonstantin Belousov 1100a110d5bSKonstantin Belousov /* 1110a110d5bSKonstantin Belousov * See VT-d specification, 5.1.6 Remapping Hardware - 1120a110d5bSKonstantin Belousov * Interrupt Programming. 1130a110d5bSKonstantin Belousov */ 1140a110d5bSKonstantin Belousov *data = vector; 1150a110d5bSKonstantin Belousov *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 1160a110d5bSKonstantin Belousov if (x2apic_mode) 1170a110d5bSKonstantin Belousov *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 1180a110d5bSKonstantin Belousov else 1190a110d5bSKonstantin Belousov KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu)); 1200a110d5bSKonstantin Belousov return (0); 1210a110d5bSKonstantin Belousov } 1220a110d5bSKonstantin Belousov if (unit == NULL || !unit->ir_enabled || cookie == -1) 1230a110d5bSKonstantin Belousov return (EOPNOTSUPP); 1240a110d5bSKonstantin Belousov 1250a110d5bSKonstantin Belousov low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : 1260a110d5bSKonstantin Belousov DMAR_IRTE1_DST_xAPIC(cpu)) | DMAR_IRTE1_V(vector) | 1270a110d5bSKonstantin Belousov DMAR_IRTE1_DLM_FM | DMAR_IRTE1_TM_EDGE | DMAR_IRTE1_RH_DIRECT | 1280a110d5bSKonstantin Belousov DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; 1290a110d5bSKonstantin Belousov dmar_ir_program_irte(unit, cookie, low, rid); 1300a110d5bSKonstantin Belousov 1310a110d5bSKonstantin Belousov if (addr != NULL) { 1320a110d5bSKonstantin Belousov /* 1330a110d5bSKonstantin Belousov * See VT-d specification, 5.1.5.2 MSI and MSI-X 1340a110d5bSKonstantin Belousov * Register Programming. 1350a110d5bSKonstantin Belousov */ 1360a110d5bSKonstantin Belousov *addr = MSI_INTEL_ADDR_BASE | ((cookie & 0x7fff) << 5) | 1370a110d5bSKonstantin Belousov ((cookie & 0x8000) << 2) | 0x18; 1380a110d5bSKonstantin Belousov *data = 0; 1390a110d5bSKonstantin Belousov } 1400a110d5bSKonstantin Belousov return (0); 1410a110d5bSKonstantin Belousov } 1420a110d5bSKonstantin Belousov 1430a110d5bSKonstantin Belousov int 1440a110d5bSKonstantin Belousov iommu_unmap_msi_intr(device_t src, u_int cookie) 1450a110d5bSKonstantin Belousov { 1460a110d5bSKonstantin Belousov struct dmar_unit *unit; 1470a110d5bSKonstantin Belousov 1480a110d5bSKonstantin Belousov if (cookie == -1) 1490a110d5bSKonstantin Belousov return (0); 1500a110d5bSKonstantin Belousov unit = dmar_ir_find(src, NULL, NULL); 1510a110d5bSKonstantin Belousov return (dmar_ir_free_irte(unit, cookie)); 1520a110d5bSKonstantin Belousov } 1530a110d5bSKonstantin Belousov 1540a110d5bSKonstantin Belousov int 1550a110d5bSKonstantin Belousov iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, 1560a110d5bSKonstantin Belousov bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo) 1570a110d5bSKonstantin Belousov { 1580a110d5bSKonstantin Belousov struct dmar_unit *unit; 1590a110d5bSKonstantin Belousov vmem_addr_t vmem_res; 1600a110d5bSKonstantin Belousov uint64_t low, iorte; 1610a110d5bSKonstantin Belousov u_int idx; 1620a110d5bSKonstantin Belousov int error; 1630a110d5bSKonstantin Belousov uint16_t rid; 1640a110d5bSKonstantin Belousov 1650a110d5bSKonstantin Belousov unit = dmar_find_ioapic(ioapic_id, &rid); 1660a110d5bSKonstantin Belousov if (unit == NULL || !unit->ir_enabled) { 1670a110d5bSKonstantin Belousov *cookie = -1; 1680a110d5bSKonstantin Belousov return (EOPNOTSUPP); 1690a110d5bSKonstantin Belousov } 1700a110d5bSKonstantin Belousov 1710a110d5bSKonstantin Belousov error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res); 1720a110d5bSKonstantin Belousov if (error != 0) { 1730a110d5bSKonstantin Belousov KASSERT(error != EOPNOTSUPP, 1740a110d5bSKonstantin Belousov ("impossible EOPNOTSUPP from vmem")); 1750a110d5bSKonstantin Belousov return (error); 1760a110d5bSKonstantin Belousov } 1770a110d5bSKonstantin Belousov idx = vmem_res; 1780a110d5bSKonstantin Belousov low = 0; 1790a110d5bSKonstantin Belousov switch (irq) { 1800a110d5bSKonstantin Belousov case IRQ_EXTINT: 1810a110d5bSKonstantin Belousov low |= DMAR_IRTE1_DLM_ExtINT; 1820a110d5bSKonstantin Belousov break; 1830a110d5bSKonstantin Belousov case IRQ_NMI: 1840a110d5bSKonstantin Belousov low |= DMAR_IRTE1_DLM_NMI; 1850a110d5bSKonstantin Belousov break; 1860a110d5bSKonstantin Belousov case IRQ_SMI: 1870a110d5bSKonstantin Belousov low |= DMAR_IRTE1_DLM_SMI; 1880a110d5bSKonstantin Belousov break; 1890a110d5bSKonstantin Belousov default: 1900a110d5bSKonstantin Belousov KASSERT(vector != 0, ("No vector for IRQ %u", irq)); 1910a110d5bSKonstantin Belousov low |= DMAR_IRTE1_DLM_FM | DMAR_IRTE1_V(vector); 1920a110d5bSKonstantin Belousov break; 1930a110d5bSKonstantin Belousov } 1940a110d5bSKonstantin Belousov low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : 1950a110d5bSKonstantin Belousov DMAR_IRTE1_DST_xAPIC(cpu)) | 1960a110d5bSKonstantin Belousov (edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) | 1970a110d5bSKonstantin Belousov DMAR_IRTE1_RH_DIRECT | DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; 1980a110d5bSKonstantin Belousov dmar_ir_program_irte(unit, idx, low, rid); 1990a110d5bSKonstantin Belousov 2000a110d5bSKonstantin Belousov if (hi != NULL) { 2010a110d5bSKonstantin Belousov /* 2020a110d5bSKonstantin Belousov * See VT-d specification, 5.1.5.1 I/OxAPIC 2030a110d5bSKonstantin Belousov * Programming. 2040a110d5bSKonstantin Belousov */ 2050a110d5bSKonstantin Belousov iorte = (1ULL << 48) | ((uint64_t)(idx & 0x7fff) << 49) | 2060a110d5bSKonstantin Belousov ((idx & 0x8000) != 0 ? (1 << 11) : 0) | 2070a110d5bSKonstantin Belousov (edge ? IOART_TRGREDG : IOART_TRGRLVL) | 2080a110d5bSKonstantin Belousov (activehi ? IOART_INTAHI : IOART_INTALO) | 2090a110d5bSKonstantin Belousov IOART_DELFIXED | vector; 2100a110d5bSKonstantin Belousov *hi = iorte >> 32; 2110a110d5bSKonstantin Belousov *lo = iorte; 2120a110d5bSKonstantin Belousov } 2130a110d5bSKonstantin Belousov *cookie = idx; 2140a110d5bSKonstantin Belousov return (0); 2150a110d5bSKonstantin Belousov } 2160a110d5bSKonstantin Belousov 2170a110d5bSKonstantin Belousov int 2180a110d5bSKonstantin Belousov iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) 2190a110d5bSKonstantin Belousov { 2200a110d5bSKonstantin Belousov struct dmar_unit *unit; 2210a110d5bSKonstantin Belousov u_int idx; 2220a110d5bSKonstantin Belousov 2230a110d5bSKonstantin Belousov idx = *cookie; 2240a110d5bSKonstantin Belousov if (idx == -1) 2250a110d5bSKonstantin Belousov return (0); 2260a110d5bSKonstantin Belousov *cookie = -1; 2270a110d5bSKonstantin Belousov unit = dmar_find_ioapic(ioapic_id, NULL); 2280a110d5bSKonstantin Belousov KASSERT(unit != NULL && unit->ir_enabled, 2290a110d5bSKonstantin Belousov ("unmap: cookie %d unit %p", idx, unit)); 2300a110d5bSKonstantin Belousov return (dmar_ir_free_irte(unit, idx)); 2310a110d5bSKonstantin Belousov } 2320a110d5bSKonstantin Belousov 2330a110d5bSKonstantin Belousov static struct dmar_unit * 2340a110d5bSKonstantin Belousov dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar) 2350a110d5bSKonstantin Belousov { 2360a110d5bSKonstantin Belousov devclass_t src_class; 2370a110d5bSKonstantin Belousov struct dmar_unit *unit; 2380a110d5bSKonstantin Belousov 2390a110d5bSKonstantin Belousov /* 2400a110d5bSKonstantin Belousov * We need to determine if the interrupt source generates FSB 2410a110d5bSKonstantin Belousov * interrupts. If yes, it is either DMAR, in which case 2420a110d5bSKonstantin Belousov * interrupts are not remapped. Or it is HPET, and interrupts 2430a110d5bSKonstantin Belousov * are remapped. For HPET, source id is reported by HPET 2440a110d5bSKonstantin Belousov * record in DMAR ACPI table. 2450a110d5bSKonstantin Belousov */ 2460a110d5bSKonstantin Belousov if (is_dmar != NULL) 2470a110d5bSKonstantin Belousov *is_dmar = FALSE; 2480a110d5bSKonstantin Belousov src_class = device_get_devclass(src); 2490a110d5bSKonstantin Belousov if (src_class == devclass_find("dmar")) { 2500a110d5bSKonstantin Belousov unit = NULL; 2510a110d5bSKonstantin Belousov if (is_dmar != NULL) 2520a110d5bSKonstantin Belousov *is_dmar = TRUE; 2530a110d5bSKonstantin Belousov } else if (src_class == devclass_find("hpet")) { 2540a110d5bSKonstantin Belousov unit = dmar_find_hpet(src, rid); 2550a110d5bSKonstantin Belousov } else { 256f9feb091SKonstantin Belousov unit = dmar_find(src, bootverbose); 2570a110d5bSKonstantin Belousov if (unit != NULL && rid != NULL) 25859e37c8aSRuslan Bukin iommu_get_requester(src, rid); 2590a110d5bSKonstantin Belousov } 2600a110d5bSKonstantin Belousov return (unit); 2610a110d5bSKonstantin Belousov } 2620a110d5bSKonstantin Belousov 2630a110d5bSKonstantin Belousov static void 2640a110d5bSKonstantin Belousov dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low, 2650a110d5bSKonstantin Belousov uint16_t rid) 2660a110d5bSKonstantin Belousov { 2670a110d5bSKonstantin Belousov dmar_irte_t *irte; 2680a110d5bSKonstantin Belousov uint64_t high; 2690a110d5bSKonstantin Belousov 2700a110d5bSKonstantin Belousov KASSERT(idx < unit->irte_cnt, 2710a110d5bSKonstantin Belousov ("bad cookie %d %d", idx, unit->irte_cnt)); 2720a110d5bSKonstantin Belousov irte = &(unit->irt[idx]); 2730a110d5bSKonstantin Belousov high = DMAR_IRTE2_SVT_RID | DMAR_IRTE2_SQ_RID | 2740a110d5bSKonstantin Belousov DMAR_IRTE2_SID_RID(rid); 2751ad4a031SKonstantin Belousov if (bootverbose) { 2760a110d5bSKonstantin Belousov device_printf(unit->dev, 2770a110d5bSKonstantin Belousov "programming irte[%d] rid %#x high %#jx low %#jx\n", 2780a110d5bSKonstantin Belousov idx, rid, (uintmax_t)high, (uintmax_t)low); 2791ad4a031SKonstantin Belousov } 2800a110d5bSKonstantin Belousov DMAR_LOCK(unit); 2810a110d5bSKonstantin Belousov if ((irte->irte1 & DMAR_IRTE1_P) != 0) { 2820a110d5bSKonstantin Belousov /* 2830a110d5bSKonstantin Belousov * The rte is already valid. Assume that the request 2840a110d5bSKonstantin Belousov * is to remap the interrupt for balancing. Only low 2850a110d5bSKonstantin Belousov * word of rte needs to be changed. Assert that the 2860a110d5bSKonstantin Belousov * high word contains expected value. 2870a110d5bSKonstantin Belousov */ 2880a110d5bSKonstantin Belousov KASSERT(irte->irte2 == high, 2890a110d5bSKonstantin Belousov ("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2, 2900a110d5bSKonstantin Belousov (uintmax_t)high)); 2910a110d5bSKonstantin Belousov dmar_pte_update(&irte->irte1, low); 2920a110d5bSKonstantin Belousov } else { 2930a110d5bSKonstantin Belousov dmar_pte_store(&irte->irte2, high); 2940a110d5bSKonstantin Belousov dmar_pte_store(&irte->irte1, low); 2950a110d5bSKonstantin Belousov } 2960a110d5bSKonstantin Belousov dmar_qi_invalidate_iec(unit, idx, 1); 2970a110d5bSKonstantin Belousov DMAR_UNLOCK(unit); 2980a110d5bSKonstantin Belousov 2990a110d5bSKonstantin Belousov } 3000a110d5bSKonstantin Belousov 3010a110d5bSKonstantin Belousov static int 3020a110d5bSKonstantin Belousov dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie) 3030a110d5bSKonstantin Belousov { 3040a110d5bSKonstantin Belousov dmar_irte_t *irte; 3050a110d5bSKonstantin Belousov 3060a110d5bSKonstantin Belousov KASSERT(unit != NULL && unit->ir_enabled, 3070a110d5bSKonstantin Belousov ("unmap: cookie %d unit %p", cookie, unit)); 3080a110d5bSKonstantin Belousov KASSERT(cookie < unit->irte_cnt, 3090a110d5bSKonstantin Belousov ("bad cookie %u %u", cookie, unit->irte_cnt)); 3100a110d5bSKonstantin Belousov irte = &(unit->irt[cookie]); 3110a110d5bSKonstantin Belousov dmar_pte_clear(&irte->irte1); 3120a110d5bSKonstantin Belousov dmar_pte_clear(&irte->irte2); 3130a110d5bSKonstantin Belousov DMAR_LOCK(unit); 3140a110d5bSKonstantin Belousov dmar_qi_invalidate_iec(unit, cookie, 1); 3150a110d5bSKonstantin Belousov DMAR_UNLOCK(unit); 3160a110d5bSKonstantin Belousov vmem_free(unit->irtids, cookie, 1); 3170a110d5bSKonstantin Belousov return (0); 3180a110d5bSKonstantin Belousov } 3190a110d5bSKonstantin Belousov 3200a110d5bSKonstantin Belousov static u_int 3210a110d5bSKonstantin Belousov clp2(u_int v) 3220a110d5bSKonstantin Belousov { 3230a110d5bSKonstantin Belousov 3240a110d5bSKonstantin Belousov return (powerof2(v) ? v : 1 << fls(v)); 3250a110d5bSKonstantin Belousov } 3260a110d5bSKonstantin Belousov 3270a110d5bSKonstantin Belousov int 3280a110d5bSKonstantin Belousov dmar_init_irt(struct dmar_unit *unit) 3290a110d5bSKonstantin Belousov { 3300a110d5bSKonstantin Belousov 3310a110d5bSKonstantin Belousov if ((unit->hw_ecap & DMAR_ECAP_IR) == 0) 3320a110d5bSKonstantin Belousov return (0); 3330a110d5bSKonstantin Belousov unit->ir_enabled = 1; 3340a110d5bSKonstantin Belousov TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled); 3350a110d5bSKonstantin Belousov if (!unit->ir_enabled) 3360a110d5bSKonstantin Belousov return (0); 3370a110d5bSKonstantin Belousov if (!unit->qi_enabled) { 3380a110d5bSKonstantin Belousov unit->ir_enabled = 0; 3390a110d5bSKonstantin Belousov if (bootverbose) 3400a110d5bSKonstantin Belousov device_printf(unit->dev, 3410a110d5bSKonstantin Belousov "QI disabled, disabling interrupt remapping\n"); 3420a110d5bSKonstantin Belousov return (0); 3430a110d5bSKonstantin Belousov } 344fd036deaSJohn Baldwin unit->irte_cnt = clp2(num_io_irqs); 34544d0efb2SAlan Cox unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig( 3460a110d5bSKonstantin Belousov unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO | M_WAITOK, 0, 3470a110d5bSKonstantin Belousov dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ? 3480a110d5bSKonstantin Belousov VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE); 3490a110d5bSKonstantin Belousov if (unit->irt == NULL) 3500a110d5bSKonstantin Belousov return (ENOMEM); 3510a110d5bSKonstantin Belousov unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt); 3520a110d5bSKonstantin Belousov unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0, 3530a110d5bSKonstantin Belousov M_FIRSTFIT | M_NOWAIT); 3540a110d5bSKonstantin Belousov DMAR_LOCK(unit); 3550a110d5bSKonstantin Belousov dmar_load_irt_ptr(unit); 3560a110d5bSKonstantin Belousov dmar_qi_invalidate_iec_glob(unit); 3570a110d5bSKonstantin Belousov DMAR_UNLOCK(unit); 3580a110d5bSKonstantin Belousov 3590a110d5bSKonstantin Belousov /* 3600a110d5bSKonstantin Belousov * Initialize mappings for already configured interrupt pins. 3610a110d5bSKonstantin Belousov * Required, because otherwise the interrupts fault without 3620a110d5bSKonstantin Belousov * irtes. 3630a110d5bSKonstantin Belousov */ 3640a110d5bSKonstantin Belousov intr_reprogram(); 3650a110d5bSKonstantin Belousov 3660a110d5bSKonstantin Belousov DMAR_LOCK(unit); 3670a110d5bSKonstantin Belousov dmar_enable_ir(unit); 3680a110d5bSKonstantin Belousov DMAR_UNLOCK(unit); 3690a110d5bSKonstantin Belousov return (0); 3700a110d5bSKonstantin Belousov } 3710a110d5bSKonstantin Belousov 3720a110d5bSKonstantin Belousov void 3730a110d5bSKonstantin Belousov dmar_fini_irt(struct dmar_unit *unit) 3740a110d5bSKonstantin Belousov { 3750a110d5bSKonstantin Belousov 3760a110d5bSKonstantin Belousov unit->ir_enabled = 0; 3770a110d5bSKonstantin Belousov if (unit->irt != NULL) { 3780a110d5bSKonstantin Belousov dmar_disable_ir(unit); 3790a110d5bSKonstantin Belousov dmar_qi_invalidate_iec_glob(unit); 3800a110d5bSKonstantin Belousov vmem_destroy(unit->irtids); 38149bfa624SAlan Cox kmem_free((vm_offset_t)unit->irt, unit->irte_cnt * 38249bfa624SAlan Cox sizeof(dmar_irte_t)); 3830a110d5bSKonstantin Belousov } 3840a110d5bSKonstantin Belousov } 385