64 static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
66 static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);
71 	struct dmar_unit *unit;  in dmar_alloc_msi_intr()  local
76 	unit = dmar_ir_find(src, NULL, NULL);  in dmar_alloc_msi_intr()
77 	if (unit == NULL || !unit->ir_enabled) {  in dmar_alloc_msi_intr()
83 	error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT,  in dmar_alloc_msi_intr()
100 	struct dmar_unit *unit;  in dmar_map_msi_intr()  local
105 	unit = dmar_ir_find(src, &rid, &is_dmar);  in dmar_map_msi_intr()
107 		KASSERT(unit == NULL, ("DMAR cannot translate itself"));  in dmar_map_msi_intr()
121 	if (unit == NULL || !unit->ir_enabled || cookie == -1)  in dmar_map_msi_intr()
124 	low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :  in dmar_map_msi_intr()
128 	dmar_ir_program_irte(unit, cookie, low, rid);  in dmar_map_msi_intr()
145 	struct dmar_unit *unit;  in dmar_unmap_msi_intr()  local
149 	unit = dmar_ir_find(src, NULL, NULL);  in dmar_unmap_msi_intr()
150 	return (dmar_ir_free_irte(unit, cookie));  in dmar_unmap_msi_intr()
157 	struct dmar_unit *unit;  in dmar_map_ioapic_intr()  local
164 	unit = dmar_find_ioapic(ioapic_id, &rid);  in dmar_map_ioapic_intr()
165 	if (unit == NULL || !unit->ir_enabled) {  in dmar_map_ioapic_intr()
170 	error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res);  in dmar_map_ioapic_intr()
193 	low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :  in dmar_map_ioapic_intr()
197 	dmar_ir_program_irte(unit, idx, low, rid);  in dmar_map_ioapic_intr()
219 	struct dmar_unit *unit;  in dmar_unmap_ioapic_intr()  local
226 	unit = dmar_find_ioapic(ioapic_id, NULL);  in dmar_unmap_ioapic_intr()
227 	KASSERT(unit != NULL && unit->ir_enabled,  in dmar_unmap_ioapic_intr()
228 	    ("unmap: cookie %d unit %p", idx, unit));  in dmar_unmap_ioapic_intr()
229 	return (dmar_ir_free_irte(unit, idx));  in dmar_unmap_ioapic_intr()
236 	struct dmar_unit *unit;  in dmar_ir_find()  local
251 		unit = NULL;  in dmar_ir_find()
255 		unit = dmar_find_hpet(src, rid);  in dmar_ir_find()
257 		unit = dmar_find(src, bootverbose);  in dmar_ir_find()
258 		if (unit != NULL && rid != NULL) {  in dmar_ir_find()
263 	return (unit);  in dmar_ir_find()
267 dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,  in dmar_ir_program_irte()  argument
273 	KASSERT(idx < unit->irte_cnt,  in dmar_ir_program_irte()
274 	    ("bad cookie %d %d", idx, unit->irte_cnt));  in dmar_ir_program_irte()
275 	irte = &(unit->irt[idx]);  in dmar_ir_program_irte()
279 		device_printf(unit->iommu.dev,  in dmar_ir_program_irte()
283 	DMAR_LOCK(unit);  in dmar_ir_program_irte()
299 	dmar_qi_invalidate_iec(unit, idx, 1);  in dmar_ir_program_irte()
300 	DMAR_UNLOCK(unit);  in dmar_ir_program_irte()
305 dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)  in dmar_ir_free_irte()  argument
309 	KASSERT(unit != NULL && unit->ir_enabled,  in dmar_ir_free_irte()
310 	    ("unmap: cookie %d unit %p", cookie, unit));  in dmar_ir_free_irte()
311 	KASSERT(cookie < unit->irte_cnt,  in dmar_ir_free_irte()
312 	    ("bad cookie %u %u", cookie, unit->irte_cnt));  in dmar_ir_free_irte()
313 	irte = &(unit->irt[cookie]);  in dmar_ir_free_irte()
316 	DMAR_LOCK(unit);  in dmar_ir_free_irte()
317 	dmar_qi_invalidate_iec(unit, cookie, 1);  in dmar_ir_free_irte()
318 	DMAR_UNLOCK(unit);  in dmar_ir_free_irte()
319 	vmem_free(unit->irtids, cookie, 1);  in dmar_ir_free_irte()
324 dmar_init_irt(struct dmar_unit *unit)  in dmar_init_irt()  argument
326 	SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx,  in dmar_init_irt()
327 	    SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)),  in dmar_init_irt()
328 	    OID_AUTO, "ir", CTLFLAG_RD, &unit->ir_enabled, 0,  in dmar_init_irt()
330 	if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)  in dmar_init_irt()
332 	unit->ir_enabled = 1;  in dmar_init_irt()
333 	TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);  in dmar_init_irt()
334 	TUNABLE_INT_FETCH("hw.iommu.ir", &unit->ir_enabled);  in dmar_init_irt()
335 	if (!unit->ir_enabled)  in dmar_init_irt()
337 	if (!unit->qi_enabled) {  in dmar_init_irt()
338 		unit->ir_enabled = 0;  in dmar_init_irt()
340 			device_printf(unit->iommu.dev,  in dmar_init_irt()
344 	unit->irte_cnt = roundup_pow_of_two(num_io_irqs);  in dmar_init_irt()
345 	if (unit->memdomain == -1) {  in dmar_init_irt()
346 		unit->irt = kmem_alloc_contig(  in dmar_init_irt()
347 		    unit->irte_cnt * sizeof(dmar_irte_t),  in dmar_init_irt()
349 		    DMAR_IS_COHERENT(unit) ?  in dmar_init_irt()
352 		unit->irt = kmem_alloc_contig_domainset(  in dmar_init_irt()
353 		    DOMAINSET_PREF(unit->memdomain),  in dmar_init_irt()
354 		    unit->irte_cnt * sizeof(dmar_irte_t),  in dmar_init_irt()
356 		    DMAR_IS_COHERENT(unit) ?  in dmar_init_irt()
359 	if (unit->irt == NULL)  in dmar_init_irt()
361 	unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);  in dmar_init_irt()
362 	unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,  in dmar_init_irt()
364 	DMAR_LOCK(unit);  in dmar_init_irt()
365 	dmar_load_irt_ptr(unit);  in dmar_init_irt()
366 	dmar_qi_invalidate_iec_glob(unit);  in dmar_init_irt()
367 	DMAR_UNLOCK(unit);  in dmar_init_irt()
376 	DMAR_LOCK(unit);  in dmar_init_irt()
377 	dmar_enable_ir(unit);  in dmar_init_irt()
378 	DMAR_UNLOCK(unit);  in dmar_init_irt()
383 dmar_fini_irt(struct dmar_unit *unit)  in dmar_fini_irt()  argument
386 	unit->ir_enabled = 0;  in dmar_fini_irt()
387 	if (unit->irt != NULL) {  in dmar_fini_irt()
388 		dmar_disable_ir(unit);  in dmar_fini_irt()
389 		dmar_qi_invalidate_iec_glob(unit);  in dmar_fini_irt()
390 		vmem_destroy(unit->irtids);  in dmar_fini_irt()
391 		kmem_free(unit->irt, unit->irte_cnt * sizeof(dmar_irte_t));  in dmar_fini_irt()