265 dmar_release_resources(device_t dev, struct dmar_unit *unit)  in dmar_release_resources()  argument
269 	iommu_fini_busdma(&unit->iommu);  in dmar_release_resources()
270 	dmar_fini_irt(unit);  in dmar_release_resources()
271 	dmar_fini_qi(unit);  in dmar_release_resources()
272 	dmar_fini_fault_log(unit);  in dmar_release_resources()
274 		iommu_release_intr(DMAR2IOMMU(unit), i);  in dmar_release_resources()
275 	if (unit->regs != NULL) {  in dmar_release_resources()
276 		bus_deactivate_resource(dev, SYS_RES_MEMORY, unit->reg_rid,  in dmar_release_resources()
277 		    unit->regs);  in dmar_release_resources()
278 		bus_release_resource(dev, SYS_RES_MEMORY, unit->reg_rid,  in dmar_release_resources()
279 		    unit->regs);  in dmar_release_resources()
280 		unit->regs = NULL;  in dmar_release_resources()
282 	if (unit->domids != NULL) {  in dmar_release_resources()
283 		delete_unrhdr(unit->domids);  in dmar_release_resources()
284 		unit->domids = NULL;  in dmar_release_resources()
286 	if (unit->ctx_obj != NULL) {  in dmar_release_resources()
287 		vm_object_deallocate(unit->ctx_obj);  in dmar_release_resources()
288 		unit->ctx_obj = NULL;  in dmar_release_resources()
290 	sysctl_ctx_free(&unit->iommu.sysctl_ctx);  in dmar_release_resources()
297 	struct dmar_unit *unit;  in dmar_remap_intr()  local
303 	unit = device_get_softc(dev);  in dmar_remap_intr()
305 		dmd = &unit->x86c.intrs[i];  in dmar_remap_intr()
312 			DMAR_LOCK(unit);  in dmar_remap_intr()
315 			(dmd->disable_intr)(DMAR2IOMMU(unit));  in dmar_remap_intr()
316 			dmar_write4(unit, dmd->msi_data_reg, dmd->msi_data);  in dmar_remap_intr()
317 			dmar_write4(unit, dmd->msi_addr_reg, dmd->msi_addr);  in dmar_remap_intr()
318 			dmar_write4(unit, dmd->msi_uaddr_reg,  in dmar_remap_intr()
320 			(dmd->enable_intr)(DMAR2IOMMU(unit));  in dmar_remap_intr()
321 			DMAR_UNLOCK(unit);  in dmar_remap_intr()
330 dmar_print_caps(device_t dev, struct dmar_unit *unit,  in dmar_print_caps()  argument
336 	    (uintmax_t)dmaru->Address, DMAR_MAJOR_VER(unit->hw_ver),  in dmar_print_caps()
337 	    DMAR_MINOR_VER(unit->hw_ver), dmaru->Segment,  in dmar_print_caps()
339 	caphi = unit->hw_cap >> 32;  in dmar_print_caps()
340 	device_printf(dev, "cap=%b,", (u_int)unit->hw_cap,  in dmar_print_caps()
344 	    DMAR_CAP_ND(unit->hw_cap), DMAR_CAP_SAGAW(unit->hw_cap),  in dmar_print_caps()
345 	    DMAR_CAP_MGAW(unit->hw_cap), DMAR_CAP_FRO(unit->hw_cap),  in dmar_print_caps()
346 	    DMAR_CAP_NFR(unit->hw_cap), DMAR_CAP_SPS(unit->hw_cap));  in dmar_print_caps()
347 	if ((unit->hw_cap & DMAR_CAP_PSI) != 0)  in dmar_print_caps()
348 		printf(", mamv=%d", DMAR_CAP_MAMV(unit->hw_cap));  in dmar_print_caps()
350 	ecaphi = unit->hw_ecap >> 32;  in dmar_print_caps()
351 	device_printf(dev, "ecap=%b,", (u_int)unit->hw_ecap,  in dmar_print_caps()
355 	printf("mhmw=%d, iro=%d\n", DMAR_ECAP_MHMV(unit->hw_ecap),  in dmar_print_caps()
356 	    DMAR_ECAP_IRO(unit->hw_ecap));  in dmar_print_caps()
362 	struct dmar_unit *unit;  in dmar_attach()  local
369 	unit = device_get_softc(dev);  in dmar_attach()
370 	unit->iommu.unit = device_get_unit(dev);  in dmar_attach()
371 	unit->iommu.dev = dev;  in dmar_attach()
372 	sysctl_ctx_init(&unit->iommu.sysctl_ctx);  in dmar_attach()
373 	dmaru = dmar_find_by_index(unit->iommu.unit);  in dmar_attach()
376 	unit->segment = dmaru->Segment;  in dmar_attach()
377 	unit->base = dmaru->Address;  in dmar_attach()
378 	unit->reg_rid = DMAR_REG_RID;  in dmar_attach()
379 	unit->regs = bus_alloc_resource_any(dev, SYS_RES_MEMORY,  in dmar_attach()
380 	    &unit->reg_rid, RF_ACTIVE);  in dmar_attach()
381 	if (unit->regs == NULL) {  in dmar_attach()
383 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
386 	unit->hw_ver = dmar_read4(unit, DMAR_VER_REG);  in dmar_attach()
387 	unit->hw_cap = dmar_read8(unit, DMAR_CAP_REG);  in dmar_attach()
388 	unit->hw_ecap = dmar_read8(unit, DMAR_ECAP_REG);  in dmar_attach()
390 		dmar_print_caps(dev, unit, dmaru);  in dmar_attach()
391 	dmar_quirks_post_ident(unit);  in dmar_attach()
392 	unit->memdomain = acpi_get_domain(dev);  in dmar_attach()
398 		unit->x86c.intrs[i].irq = -1;  in dmar_attach()
400 	dmd = &unit->x86c.intrs[DMAR_INTR_FAULT];  in dmar_attach()
409 	error = iommu_alloc_irq(DMAR2IOMMU(unit), DMAR_INTR_FAULT);  in dmar_attach()
411 		dmar_release_resources(dev, unit);  in dmar_attach()
412 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
415 	dmar_write4(unit, dmd->msi_data_reg, dmd->msi_data);  in dmar_attach()
416 	dmar_write4(unit, dmd->msi_addr_reg, dmd->msi_addr);  in dmar_attach()
417 	dmar_write4(unit, dmd->msi_uaddr_reg, dmd->msi_addr >> 32);  in dmar_attach()
419 	if (DMAR_HAS_QI(unit)) {  in dmar_attach()
420 		dmd = &unit->x86c.intrs[DMAR_INTR_QI];  in dmar_attach()
429 		error = iommu_alloc_irq(DMAR2IOMMU(unit), DMAR_INTR_QI);  in dmar_attach()
431 			dmar_release_resources(dev, unit);  in dmar_attach()
432 			dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
436 		dmar_write4(unit, dmd->msi_data_reg, dmd->msi_data);  in dmar_attach()
437 		dmar_write4(unit, dmd->msi_addr_reg, dmd->msi_addr);  in dmar_attach()
438 		dmar_write4(unit, dmd->msi_uaddr_reg, dmd->msi_addr >> 32);  in dmar_attach()
441 	mtx_init(&unit->iommu.lock, "dmarhw", NULL, MTX_DEF);  in dmar_attach()
442 	unit->domids = new_unrhdr(0, dmar_nd2mask(DMAR_CAP_ND(unit->hw_cap)),  in dmar_attach()
443 	    &unit->iommu.lock);  in dmar_attach()
444 	LIST_INIT(&unit->domains);  in dmar_attach()
453 	if ((unit->hw_cap & DMAR_CAP_CM) != 0)  in dmar_attach()
454 		alloc_unr_specific(unit->domids, 0);  in dmar_attach()
456 	unit->ctx_obj = vm_pager_allocate(OBJT_PHYS, NULL, IDX_TO_OFF(1 +  in dmar_attach()
458 	if (unit->memdomain != -1) {  in dmar_attach()
459 		unit->ctx_obj->domain.dr_policy = DOMAINSET_PREF(  in dmar_attach()
460 		    unit->memdomain);  in dmar_attach()
468 	iommu_pgalloc(unit->ctx_obj, 0, IOMMU_PGF_WAITOK | IOMMU_PGF_ZERO);  in dmar_attach()
469 	DMAR_LOCK(unit);  in dmar_attach()
470 	error = dmar_load_root_entry_ptr(unit);  in dmar_attach()
472 		DMAR_UNLOCK(unit);  in dmar_attach()
473 		dmar_release_resources(dev, unit);  in dmar_attach()
474 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
477 	error = dmar_inv_ctx_glob(unit);  in dmar_attach()
479 		DMAR_UNLOCK(unit);  in dmar_attach()
480 		dmar_release_resources(dev, unit);  in dmar_attach()
481 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
484 	if ((unit->hw_ecap & DMAR_ECAP_DI) != 0) {  in dmar_attach()
485 		error = dmar_inv_iotlb_glob(unit);  in dmar_attach()
487 			DMAR_UNLOCK(unit);  in dmar_attach()
488 			dmar_release_resources(dev, unit);  in dmar_attach()
489 			dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
494 	DMAR_UNLOCK(unit);  in dmar_attach()
495 	error = dmar_init_fault_log(unit);  in dmar_attach()
497 		dmar_release_resources(dev, unit);  in dmar_attach()
498 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
501 	error = dmar_init_qi(unit);  in dmar_attach()
503 		dmar_release_resources(dev, unit);  in dmar_attach()
504 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
507 	error = dmar_init_irt(unit);  in dmar_attach()
509 		dmar_release_resources(dev, unit);  in dmar_attach()
510 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
517 		error = dmar_disable_protected_regions(unit);  in dmar_attach()
523 	error = iommu_init_busdma(&unit->iommu);  in dmar_attach()
525 		dmar_release_resources(dev, unit);  in dmar_attach()
526 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
531 	DMAR_LOCK(unit);  in dmar_attach()
532 	error = dmar_enable_translation(unit);  in dmar_attach()
534 		DMAR_UNLOCK(unit);  in dmar_attach()
535 		dmar_release_resources(dev, unit);  in dmar_attach()
536 		dmar_devs[unit->iommu.unit] = NULL;  in dmar_attach()
539 	DMAR_UNLOCK(unit);  in dmar_attach()
702 dmar_match_by_path(struct dmar_unit *unit, int dev_domain, int dev_busno,  in dmar_match_by_path()  argument
710 	dmarh = dmar_find_by_index(unit->iommu.unit);  in dmar_match_by_path()
742 	struct dmar_unit *unit;  in dmar_find_by_scope()  local
748 		unit = device_get_softc(dmar_devs[i]);  in dmar_find_by_scope()
749 		if (dmar_match_by_path(unit, dev_domain, dev_busno, dev_path,  in dmar_find_by_scope()
751 			return (unit);  in dmar_find_by_scope()
759 	struct dmar_unit *unit;  in dmar_find()  local
779 		unit = device_get_softc(dmar_devs[i]);  in dmar_find()
780 		if (dmar_match_by_path(unit, dev_domain, dev_busno,  in dmar_find()
790 		    pci_get_function(dev), unit->iommu.unit, banner);  in dmar_find()
795 	iommu_device_set_iommu_prop(dev, unit->iommu.dev);  in dmar_find()
796 	return (unit);  in dmar_find()
803 	struct dmar_unit *unit;  in dmar_find_nonpci()  local
817 		unit = (struct dmar_unit *)device_get_softc(dmar_dev);  in dmar_find_nonpci()
840 					return (unit);  in dmar_find_nonpci()
851 				return (unit);  in dmar_find_nonpci()
865 	struct dmar_unit *unit;  in dmar_find_hpet()  local
867 	unit = dmar_find_nonpci(hpet_get_uid(dev), ACPI_DMAR_SCOPE_TYPE_HPET,  in dmar_find_hpet()
869 	if (unit != NULL)  in dmar_find_hpet()
870 		iommu_device_set_iommu_prop(dev, unit->iommu.dev);  in dmar_find_hpet()
871 	return (unit);  in dmar_find_hpet()
877 	struct dmar_unit *unit;  in dmar_find_ioapic()  local
880 	unit = dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid);  in dmar_find_ioapic()
881 	if (unit != NULL) {  in dmar_find_ioapic()
884 			iommu_device_set_iommu_prop(apic_dev, unit->iommu.dev);  in dmar_find_ioapic()
886 	return (unit);  in dmar_find_ioapic()
991 	struct dmar_unit *unit;  in dmar_inst_rmrr_iter()  local
1027 				    iria->dmar->iommu.unit,  in dmar_inst_rmrr_iter()
1035 			unit = dmar_find_by_scope(resmem->Segment,  in dmar_inst_rmrr_iter()
1039 			if (iria->dmar != unit)  in dmar_inst_rmrr_iter()
1046 			unit = dmar_find(dev, false);  in dmar_inst_rmrr_iter()
1047 			if (iria->dmar != unit)  in dmar_inst_rmrr_iter()
1062 dmar_instantiate_rmrr_ctxs(struct iommu_unit *unit)  in dmar_instantiate_rmrr_ctxs()  argument
1068 	dmar = IOMMU2DMAR(unit);  in dmar_instantiate_rmrr_ctxs()
1080 		    dmar->iommu.unit));  in dmar_instantiate_rmrr_ctxs()
1084 			    dmar->iommu.unit);  in dmar_instantiate_rmrr_ctxs()
1089 				    dmar->iommu.unit);  in dmar_instantiate_rmrr_ctxs()
1092 				    "error %d\n", dmar->iommu.unit, error);  in dmar_instantiate_rmrr_ctxs()
1126 	struct dmar_unit *unit;  in DB_SHOW_COMMAND_FLAGS()  local
1174 		unit = device_get_softc(dmar_devs[i]);  in DB_SHOW_COMMAND_FLAGS()
1175 		LIST_FOREACH(domain, &unit->domains, link) {  in DB_SHOW_COMMAND_FLAGS()
1177 				if (pci_domain == unit->segment &&   in DB_SHOW_COMMAND_FLAGS()
1195 	struct dmar_unit *unit;  in dmar_print_one()  local
1199 	unit = device_get_softc(dmar_devs[idx]);  in dmar_print_one()
1200 	db_printf("dmar%d at %p, root at 0x%jx, ver 0x%x\n", unit->iommu.unit,  in dmar_print_one()
1201 	    unit, dmar_read8(unit, DMAR_RTADDR_REG),  in dmar_print_one()
1202 	    dmar_read4(unit, DMAR_VER_REG));  in dmar_print_one()
1204 	    (uintmax_t)dmar_read8(unit, DMAR_CAP_REG),  in dmar_print_one()
1205 	    (uintmax_t)dmar_read8(unit, DMAR_ECAP_REG),  in dmar_print_one()
1206 	    dmar_read4(unit, DMAR_GSTS_REG),  in dmar_print_one()
1207 	    dmar_read4(unit, DMAR_FSTS_REG),  in dmar_print_one()
1208 	    dmar_read4(unit, DMAR_FECTL_REG));  in dmar_print_one()
1209 	if (unit->ir_enabled) {  in dmar_print_one()
1211 		    unit->irt, (uintmax_t)unit->irt_phys, unit->irte_cnt);  in dmar_print_one()
1214 	    dmar_read4(unit, DMAR_FEDATA_REG),  in dmar_print_one()
1215 	    dmar_read4(unit, DMAR_FEADDR_REG),  in dmar_print_one()
1216 	    dmar_read4(unit, DMAR_FEUADDR_REG));  in dmar_print_one()
1218 	for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {  in dmar_print_one()
1219 		frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;  in dmar_print_one()
1221 		    (uintmax_t)dmar_read8(unit, frir),  in dmar_print_one()
1222 		    (uintmax_t)dmar_read8(unit, frir + 8));  in dmar_print_one()
1224 	if (DMAR_HAS_QI(unit)) {  in dmar_print_one()
1226 		    dmar_read4(unit, DMAR_IEDATA_REG),  in dmar_print_one()
1227 		    dmar_read4(unit, DMAR_IEADDR_REG),  in dmar_print_one()
1228 		    dmar_read4(unit, DMAR_IEUADDR_REG));  in dmar_print_one()
1229 		if (unit->qi_enabled) {  in dmar_print_one()
1234 			    (uintmax_t)unit->x86c.inv_queue,  in dmar_print_one()
1235 			    (uintmax_t)dmar_read8(unit, DMAR_IQA_REG),  in dmar_print_one()
1236 			    (uintmax_t)unit->x86c.inv_queue_size,  in dmar_print_one()
1237 			    dmar_read4(unit, DMAR_IQH_REG),  in dmar_print_one()
1238 			    dmar_read4(unit, DMAR_IQT_REG),  in dmar_print_one()
1239 			    unit->x86c.inv_queue_avail,  in dmar_print_one()
1240 			    dmar_read4(unit, DMAR_ICS_REG),  in dmar_print_one()
1241 			    dmar_read4(unit, DMAR_IECTL_REG),  in dmar_print_one()
1242 			    (uintmax_t)unit->x86c.inv_waitd_seq_hw,  in dmar_print_one()
1243 			    &unit->x86c.inv_waitd_seq_hw,  in dmar_print_one()
1244 			    (uintmax_t)unit->x86c.inv_waitd_seq_hw_phys,  in dmar_print_one()
1245 			    unit->x86c.inv_waitd_seq,  in dmar_print_one()
1246 			    unit->x86c.inv_waitd_gen);  in dmar_print_one()
1253 		LIST_FOREACH(domain, &unit->domains, link) {  in dmar_print_one()
1300 dmar_get_x86_common(struct iommu_unit *unit)  in dmar_get_x86_common()  argument
1304 	dmar = IOMMU2DMAR(unit);  in dmar_get_x86_common()
1309 dmar_unit_pre_instantiate_ctx(struct iommu_unit *unit)  in dmar_unit_pre_instantiate_ctx()  argument
1311 	dmar_quirks_pre_use(unit);  in dmar_unit_pre_instantiate_ctx()
1312 	dmar_instantiate_rmrr_ctxs(unit);  in dmar_unit_pre_instantiate_ctx()