74 static void amdiommu_unref_domain_locked(struct amdiommu_unit *unit,
88 	struct amdiommu_unit *unit;  in amdiommu_domain_unload_entry()  local
91 	unit = DOM2AMD(domain);  in amdiommu_domain_unload_entry()
99 		AMDIOMMU_LOCK(unit);  in amdiommu_domain_unload_entry()
101 		AMDIOMMU_UNLOCK(unit);  in amdiommu_domain_unload_entry()
121 	struct amdiommu_unit *unit;  in amdiommu_domain_unload()  local
126 	unit = DOM2AMD(domain);  in amdiommu_domain_unload()
138 	AMDIOMMU_LOCK(unit);  in amdiommu_domain_unload()
144 	AMDIOMMU_UNLOCK(unit);  in amdiommu_domain_unload()
151 	struct amdiommu_unit *unit;  in amdiommu_domain_destroy()  local
175 	unit = DOM2AMD(domain);  in amdiommu_domain_destroy()
176 	free_unr(unit->domids, domain->domain);  in amdiommu_domain_destroy()
193 amdiommu_domain_init_pglvl(struct amdiommu_unit *unit,  in amdiommu_domain_init_pglvl()  argument
207 	efr_hats = unit->efr & AMDIOMMU_EFR_HATS_MASK;  in amdiommu_domain_init_pglvl()
220 		    unit->iommu.unit, (uintmax_t)efr_hats);  in amdiommu_domain_init_pglvl()
227 	    unit->iommu.unit, domain->domain, hats, domain->pglvl);  in amdiommu_domain_init_pglvl()
233 amdiommu_domain_alloc(struct amdiommu_unit *unit, bool id_mapped)  in amdiommu_domain_alloc()  argument
239 	id = alloc_unr(unit->domids);  in amdiommu_domain_alloc()
246 	iommu_domain_init(AMD2IOMMU(unit), iodom, &amdiommu_domain_map_ops);  in amdiommu_domain_alloc()
248 	domain->unit = unit;  in amdiommu_domain_alloc()
251 	amdiommu_domain_init_pglvl(unit, domain);  in amdiommu_domain_alloc()
321 amdiommu_find_ctx_locked(struct amdiommu_unit *unit, uint16_t rid)  in amdiommu_find_ctx_locked()  argument
326 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_find_ctx_locked()
328 	LIST_FOREACH(domain, &unit->domains, link) {  in amdiommu_find_ctx_locked()
338 amdiommu_find_domain(struct amdiommu_unit *unit, uint16_t rid)  in amdiommu_find_domain()  argument
343 	AMDIOMMU_LOCK(unit);  in amdiommu_find_domain()
344 	LIST_FOREACH(domain, &unit->domains, link) {  in amdiommu_find_domain()
350 	AMDIOMMU_UNLOCK(unit);  in amdiommu_find_domain()
355 amdiommu_free_ctx_locked(struct amdiommu_unit *unit, struct amdiommu_ctx *ctx)  in amdiommu_free_ctx_locked()  argument
360 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_free_ctx_locked()
362 	    ("amdiommu %p ctx %p refs %u", unit, ctx, ctx->context.refs));  in amdiommu_free_ctx_locked()
370 		AMDIOMMU_UNLOCK(unit);  in amdiommu_free_ctx_locked()
388 	amdiommu_qi_invalidate_ir_locked_nowait(unit, ctx->context.rid);  in amdiommu_free_ctx_locked()
392 	if (unit->irte_enabled)  in amdiommu_free_ctx_locked()
398 	amdiommu_unref_domain_locked(unit, domain);  in amdiommu_free_ctx_locked()
402 amdiommu_unref_domain_locked(struct amdiommu_unit *unit,  in amdiommu_unref_domain_locked()  argument
405 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_unref_domain_locked()
407 	    ("amdiommu%d domain %p refs %u", unit->iommu.unit, domain,  in amdiommu_unref_domain_locked()
410 	    ("amdiommu%d domain %p refs %d ctx_cnt %d", unit->iommu.unit,  in amdiommu_unref_domain_locked()
415 		AMDIOMMU_UNLOCK(unit);  in amdiommu_unref_domain_locked()
420 	AMDIOMMU_UNLOCK(unit);  in amdiommu_unref_domain_locked()
422 	taskqueue_drain(unit->iommu.delayed_taskqueue,  in amdiommu_unref_domain_locked()
432 	struct amdiommu_unit *unit;  in dte_entry_init_one()  local
435 	unit = DOM2AMD(domain);  in dte_entry_init_one()
452 	if (unit->irte_enabled) {  in dte_entry_init_one()
455 		dtep->inttablen = ilog2(unit->irte_nentries);  in dte_entry_init_one()
456 		dtep->intrroot = pmap_kextract(unit->irte_x2apic ?  in dte_entry_init_one()
480 	struct amdiommu_unit *unit;  in dte_entry_init()  local
485 	unit = DOM2AMD(domain);  in dte_entry_init()
490 	    CTX2AMD(ctx)->iommu.unit, dtep, (uintmax_t)(*(uint64_t *)dtep)));  in dte_entry_init()
492 	if (iommu_is_buswide_ctx(AMD2IOMMU(unit),  in dte_entry_init()
505 amdiommu_get_ctx_for_dev(struct amdiommu_unit *unit, device_t dev, uint16_t rid,  in amdiommu_get_ctx_for_dev()  argument
521 	AMDIOMMU_LOCK(unit);  in amdiommu_get_ctx_for_dev()
522 	KASSERT(!iommu_is_buswide_ctx(AMD2IOMMU(unit), bus) ||  in amdiommu_get_ctx_for_dev()
524 	    ("iommu%d pci%d:%d:%d get_ctx for buswide", AMD2IOMMU(unit)->unit,  in amdiommu_get_ctx_for_dev()
526 	ctx = amdiommu_find_ctx_locked(unit, rid);  in amdiommu_get_ctx_for_dev()
532 		AMDIOMMU_UNLOCK(unit);  in amdiommu_get_ctx_for_dev()
533 		domain1 = amdiommu_domain_alloc(unit, id_mapped);  in amdiommu_get_ctx_for_dev()
545 		AMDIOMMU_LOCK(unit);  in amdiommu_get_ctx_for_dev()
551 		ctx = amdiommu_find_ctx_locked(unit, rid);  in amdiommu_get_ctx_for_dev()
559 			LIST_INSERT_HEAD(&unit->domains, domain, link);  in amdiommu_get_ctx_for_dev()
566 				    AMD2IOMMU(unit)->unit, unit->unit_dom,  in amdiommu_get_ctx_for_dev()
583 	AMDIOMMU_UNLOCK(unit);  in amdiommu_get_ctx_for_dev()
592 	struct amdiommu_unit *unit;  in amdiommu_get_ctx()  local
599 	error = amdiommu_find_unit(dev, &unit,  &rid1,  &dte, &edte,  in amdiommu_get_ctx()
603 	if (AMD2IOMMU(unit) != iommu)	/* XXX complain loudly */  in amdiommu_get_ctx()
605 	ret = amdiommu_get_ctx_for_dev(unit, dev, rid1, pci_get_domain(dev),  in amdiommu_get_ctx()
614 	struct amdiommu_unit *unit;  in amdiommu_free_ctx_locked_method()  local
617 	unit = IOMMU2AMD(iommu);  in amdiommu_free_ctx_locked_method()
619 	amdiommu_free_ctx_locked(unit, ctx);  in amdiommu_free_ctx_locked_method()