64 static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
66 static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);
71 	struct dmar_unit *unit;  in dmar_alloc_msi_intr()  local
76 	unit = dmar_ir_find(src, NULL, NULL);  in dmar_alloc_msi_intr()
77 	if (unit == NULL || !unit->ir_enabled) {  in dmar_alloc_msi_intr()
83 	error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT,  in dmar_alloc_msi_intr()
100 	struct dmar_unit *unit;  in dmar_map_msi_intr()  local
105 	unit = dmar_ir_find(src, &rid, &is_dmar);  in dmar_map_msi_intr()
107 		KASSERT(unit == NULL, ("DMAR cannot translate itself"));  in dmar_map_msi_intr()
121 	if (unit == NULL || !unit->ir_enabled || cookie == -1)  in dmar_map_msi_intr()
124 	low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :  in dmar_map_msi_intr()
128 	dmar_ir_program_irte(unit, cookie, low, rid);  in dmar_map_msi_intr()
145 	struct dmar_unit *unit;  in dmar_unmap_msi_intr()  local
149 	unit = dmar_ir_find(src, NULL, NULL);  in dmar_unmap_msi_intr()
150 	return (dmar_ir_free_irte(unit, cookie));  in dmar_unmap_msi_intr()
157 	struct dmar_unit *unit;  in dmar_map_ioapic_intr()  local
164 	unit = dmar_find_ioapic(ioapic_id, &rid);  in dmar_map_ioapic_intr()
165 	if (unit == NULL || !unit->ir_enabled) {  in dmar_map_ioapic_intr()
170 	error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res);  in dmar_map_ioapic_intr()
193 	low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :  in dmar_map_ioapic_intr()
197 	dmar_ir_program_irte(unit, idx, low, rid);  in dmar_map_ioapic_intr()
219 	struct dmar_unit *unit;  in dmar_unmap_ioapic_intr()  local
226 	unit = dmar_find_ioapic(ioapic_id, NULL);  in dmar_unmap_ioapic_intr()
227 	KASSERT(unit != NULL && unit->ir_enabled,  in dmar_unmap_ioapic_intr()
228 	    ("unmap: cookie %d unit %p", idx, unit));  in dmar_unmap_ioapic_intr()
229 	return (dmar_ir_free_irte(unit, idx));  in dmar_unmap_ioapic_intr()
236 	struct dmar_unit *unit;  in dmar_ir_find()  local
249 		unit = NULL;  in dmar_ir_find()
253 		unit = dmar_find_hpet(src, rid);  in dmar_ir_find()
255 		unit = dmar_find(src, bootverbose);  in dmar_ir_find()
256 		if (unit != NULL && rid != NULL)  in dmar_ir_find()
259 	return (unit);  in dmar_ir_find()
263 dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,  in dmar_ir_program_irte()  argument
269 	KASSERT(idx < unit->irte_cnt,  in dmar_ir_program_irte()
270 	    ("bad cookie %d %d", idx, unit->irte_cnt));  in dmar_ir_program_irte()
271 	irte = &(unit->irt[idx]);  in dmar_ir_program_irte()
275 		device_printf(unit->iommu.dev,  in dmar_ir_program_irte()
279 	DMAR_LOCK(unit);  in dmar_ir_program_irte()
295 	dmar_qi_invalidate_iec(unit, idx, 1);  in dmar_ir_program_irte()
296 	DMAR_UNLOCK(unit);  in dmar_ir_program_irte()
301 dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)  in dmar_ir_free_irte()  argument
305 	KASSERT(unit != NULL && unit->ir_enabled,  in dmar_ir_free_irte()
306 	    ("unmap: cookie %d unit %p", cookie, unit));  in dmar_ir_free_irte()
307 	KASSERT(cookie < unit->irte_cnt,  in dmar_ir_free_irte()
308 	    ("bad cookie %u %u", cookie, unit->irte_cnt));  in dmar_ir_free_irte()
309 	irte = &(unit->irt[cookie]);  in dmar_ir_free_irte()
312 	DMAR_LOCK(unit);  in dmar_ir_free_irte()
313 	dmar_qi_invalidate_iec(unit, cookie, 1);  in dmar_ir_free_irte()
314 	DMAR_UNLOCK(unit);  in dmar_ir_free_irte()
315 	vmem_free(unit->irtids, cookie, 1);  in dmar_ir_free_irte()
320 dmar_init_irt(struct dmar_unit *unit)  in dmar_init_irt()  argument
322 	SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx,  in dmar_init_irt()
323 	    SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)),  in dmar_init_irt()
324 	    OID_AUTO, "ir", CTLFLAG_RD, &unit->ir_enabled, 0,  in dmar_init_irt()
326 	if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)  in dmar_init_irt()
328 	unit->ir_enabled = 1;  in dmar_init_irt()
329 	TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);  in dmar_init_irt()
330 	TUNABLE_INT_FETCH("hw.iommu.ir", &unit->ir_enabled);  in dmar_init_irt()
331 	if (!unit->ir_enabled)  in dmar_init_irt()
333 	if (!unit->qi_enabled) {  in dmar_init_irt()
334 		unit->ir_enabled = 0;  in dmar_init_irt()
336 			device_printf(unit->iommu.dev,  in dmar_init_irt()
340 	unit->irte_cnt = roundup_pow_of_two(num_io_irqs);  in dmar_init_irt()
341 	if (unit->memdomain == -1) {  in dmar_init_irt()
342 		unit->irt = kmem_alloc_contig(  in dmar_init_irt()
343 		    unit->irte_cnt * sizeof(dmar_irte_t),  in dmar_init_irt()
345 		    DMAR_IS_COHERENT(unit) ?  in dmar_init_irt()
348 		unit->irt = kmem_alloc_contig_domainset(  in dmar_init_irt()
349 		    DOMAINSET_PREF(unit->memdomain),  in dmar_init_irt()
350 		    unit->irte_cnt * sizeof(dmar_irte_t),  in dmar_init_irt()
352 		    DMAR_IS_COHERENT(unit) ?  in dmar_init_irt()
355 	if (unit->irt == NULL)  in dmar_init_irt()
357 	unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);  in dmar_init_irt()
358 	unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,  in dmar_init_irt()
360 	DMAR_LOCK(unit);  in dmar_init_irt()
361 	dmar_load_irt_ptr(unit);  in dmar_init_irt()
362 	dmar_qi_invalidate_iec_glob(unit);  in dmar_init_irt()
363 	DMAR_UNLOCK(unit);  in dmar_init_irt()
372 	DMAR_LOCK(unit);  in dmar_init_irt()
373 	dmar_enable_ir(unit);  in dmar_init_irt()
374 	DMAR_UNLOCK(unit);  in dmar_init_irt()
379 dmar_fini_irt(struct dmar_unit *unit)  in dmar_fini_irt()  argument
382 	unit->ir_enabled = 0;  in dmar_fini_irt()
383 	if (unit->irt != NULL) {  in dmar_fini_irt()
384 		dmar_disable_ir(unit);  in dmar_fini_irt()
385 		dmar_qi_invalidate_iec_glob(unit);  in dmar_fini_irt()
386 		vmem_destroy(unit->irtids);  in dmar_fini_irt()
387 		kmem_free(unit->irt, unit->irte_cnt * sizeof(dmar_irte_t));  in dmar_fini_irt()