72 static MALLOC_DEFINE(M_AMDIOMMU_DOMAIN, "amdiommu_dom", "AMD IOMMU Domain");
75     struct amdiommu_domain *domain);
87 	struct amdiommu_domain *domain;  in amdiommu_domain_unload_entry()  local
90 	domain = IODOM2DOM(entry->domain);  in amdiommu_domain_unload_entry()
91 	unit = DOM2AMD(domain);  in amdiommu_domain_unload_entry()
100 		iommu_qi_invalidate_locked(&domain->iodom, entry, true);  in amdiommu_domain_unload_entry()
103 		iommu_qi_invalidate_sync(&domain->iodom, entry->start,  in amdiommu_domain_unload_entry()
110 amdiommu_domain_unload_emit_wait(struct amdiommu_domain *domain,  in amdiommu_domain_unload_emit_wait()  argument
120 	struct amdiommu_domain *domain;  in amdiommu_domain_unload()  local
125 	domain = IODOM2DOM(iodom);  in amdiommu_domain_unload()
126 	unit = DOM2AMD(domain);  in amdiommu_domain_unload()
130 		    ("not mapped entry %p %p", domain, entry));  in amdiommu_domain_unload()
133 		KASSERT(error == 0, ("unmap %p error %d", domain, error));  in amdiommu_domain_unload()
141 		iommu_qi_invalidate_locked(&domain->iodom, entry,  in amdiommu_domain_unload()
142 		    amdiommu_domain_unload_emit_wait(domain, entry));  in amdiommu_domain_unload()
148 amdiommu_domain_destroy(struct amdiommu_domain *domain)  in amdiommu_domain_destroy()  argument
153 	iodom = DOM2IODOM(domain);  in amdiommu_domain_destroy()
155 	KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),  in amdiommu_domain_destroy()
156 	    ("unfinished unloads %p", domain));  in amdiommu_domain_destroy()
158 	    ("destroying dom %p with contexts", domain));  in amdiommu_domain_destroy()
159 	KASSERT(domain->ctx_cnt == 0,  in amdiommu_domain_destroy()
160 	    ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));  in amdiommu_domain_destroy()
161 	KASSERT(domain->refs == 0,  in amdiommu_domain_destroy()
162 	    ("destroying dom %p with refs %d", domain, domain->refs));  in amdiommu_domain_destroy()
164 	if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) {  in amdiommu_domain_destroy()
165 		AMDIOMMU_DOMAIN_LOCK(domain);  in amdiommu_domain_destroy()
167 		AMDIOMMU_DOMAIN_UNLOCK(domain);  in amdiommu_domain_destroy()
169 	if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) {  in amdiommu_domain_destroy()
170 		if (domain->pgtbl_obj != NULL)  in amdiommu_domain_destroy()
171 			AMDIOMMU_DOMAIN_PGLOCK(domain);  in amdiommu_domain_destroy()
172 		amdiommu_domain_free_pgtbl(domain);  in amdiommu_domain_destroy()
175 	unit = DOM2AMD(domain);  in amdiommu_domain_destroy()
176 	free_unr(unit->domids, domain->domain);  in amdiommu_domain_destroy()
177 	free(domain, M_AMDIOMMU_DOMAIN);  in amdiommu_domain_destroy()
194     struct amdiommu_domain *domain)  in amdiommu_domain_init_pglvl()  argument
200 	end = DOM2IODOM(domain)->end;  in amdiommu_domain_init_pglvl()
205 	domain->pglvl = i;  in amdiommu_domain_init_pglvl()
223 	if (hats >= domain->pglvl)  in amdiommu_domain_init_pglvl()
226 	printf("amdiommu%d: domain %d HATS %d pglvl %d reducing to HATS\n",  in amdiommu_domain_init_pglvl()
227 	    unit->iommu.unit, domain->domain, hats, domain->pglvl);  in amdiommu_domain_init_pglvl()
228 	domain->pglvl = hats;  in amdiommu_domain_init_pglvl()
229 	domain->iodom.end = lvl2addr(hats);  in amdiommu_domain_init_pglvl()
235 	struct amdiommu_domain *domain;  in amdiommu_domain_alloc()  local
242 	domain = malloc(sizeof(*domain), M_AMDIOMMU_DOMAIN, M_WAITOK | M_ZERO);  in amdiommu_domain_alloc()
243 	iodom = DOM2IODOM(domain);  in amdiommu_domain_alloc()
244 	domain->domain = id;  in amdiommu_domain_alloc()
248 	domain->unit = unit;  in amdiommu_domain_alloc()
250 	domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;  in amdiommu_domain_alloc()
251 	amdiommu_domain_init_pglvl(unit, domain);  in amdiommu_domain_alloc()
252 	iommu_gas_init_domain(DOM2IODOM(domain));  in amdiommu_domain_alloc()
255 		domain->iodom.flags |= IOMMU_DOMAIN_IDMAP;  in amdiommu_domain_alloc()
257 		error = amdiommu_domain_alloc_pgtbl(domain);  in amdiommu_domain_alloc()
267 	return (domain);  in amdiommu_domain_alloc()
270 	amdiommu_domain_destroy(domain);  in amdiommu_domain_alloc()
275 amdiommu_ctx_alloc(struct amdiommu_domain *domain, uint16_t rid)  in amdiommu_ctx_alloc()  argument
280 	ctx->context.domain = DOM2IODOM(domain);  in amdiommu_ctx_alloc()
291 	struct amdiommu_domain *domain;  in amdiommu_ctx_link()  local
293 	domain = CTX2DOM(ctx);  in amdiommu_ctx_link()
294 	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);  in amdiommu_ctx_link()
295 	KASSERT(domain->refs >= domain->ctx_cnt,  in amdiommu_ctx_link()
296 	    ("dom %p ref underflow %d %d", domain, domain->refs,  in amdiommu_ctx_link()
297 	    domain->ctx_cnt));  in amdiommu_ctx_link()
298 	domain->refs++;  in amdiommu_ctx_link()
299 	domain->ctx_cnt++;  in amdiommu_ctx_link()
300 	LIST_INSERT_HEAD(&domain->iodom.contexts, &ctx->context, link);  in amdiommu_ctx_link()
306 	struct amdiommu_domain *domain;  in amdiommu_ctx_unlink()  local
308 	domain = CTX2DOM(ctx);  in amdiommu_ctx_unlink()
309 	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);  in amdiommu_ctx_unlink()
310 	KASSERT(domain->refs > 0,  in amdiommu_ctx_unlink()
311 	    ("domain %p ctx dtr refs %d", domain, domain->refs));  in amdiommu_ctx_unlink()
312 	KASSERT(domain->ctx_cnt >= domain->refs,  in amdiommu_ctx_unlink()
313 	    ("domain %p ctx dtr refs %d ctx_cnt %d", domain,  in amdiommu_ctx_unlink()
314 	    domain->refs, domain->ctx_cnt));  in amdiommu_ctx_unlink()
315 	domain->refs--;  in amdiommu_ctx_unlink()
316 	domain->ctx_cnt--;  in amdiommu_ctx_unlink()
323 	struct amdiommu_domain *domain;  in amdiommu_find_ctx_locked()  local
328 	LIST_FOREACH(domain, &unit->domains, link) {  in amdiommu_find_ctx_locked()
329 		LIST_FOREACH(ctx, &domain->iodom.contexts, link) {  in amdiommu_find_ctx_locked()
340 	struct amdiommu_domain *domain;  in amdiommu_find_domain()  local
344 	LIST_FOREACH(domain, &unit->domains, link) {  in amdiommu_find_domain()
345 		LIST_FOREACH(ctx, &domain->iodom.contexts, link) {  in amdiommu_find_domain()
351 	return (domain);  in amdiommu_find_domain()
358 	struct amdiommu_domain *domain;  in amdiommu_free_ctx_locked()  local
386 	domain = CTX2DOM(ctx);  in amdiommu_free_ctx_locked()
389 	amdiommu_qi_invalidate_all_pages_locked_nowait(domain);  in amdiommu_free_ctx_locked()
398 	amdiommu_unref_domain_locked(unit, domain);  in amdiommu_free_ctx_locked()
403     struct amdiommu_domain *domain)  in amdiommu_unref_domain_locked()  argument
406 	KASSERT(domain->refs >= 1,  in amdiommu_unref_domain_locked()
407 	    ("amdiommu%d domain %p refs %u", unit->iommu.unit, domain,  in amdiommu_unref_domain_locked()
408 	    domain->refs));  in amdiommu_unref_domain_locked()
409 	KASSERT(domain->refs > domain->ctx_cnt,  in amdiommu_unref_domain_locked()
410 	    ("amdiommu%d domain %p refs %d ctx_cnt %d", unit->iommu.unit,  in amdiommu_unref_domain_locked()
411 	    domain, domain->refs, domain->ctx_cnt));  in amdiommu_unref_domain_locked()
413 	if (domain->refs > 1) {  in amdiommu_unref_domain_locked()
414 		domain->refs--;  in amdiommu_unref_domain_locked()
419 	LIST_REMOVE(domain, link);  in amdiommu_unref_domain_locked()
423 	    &domain->iodom.unload_task);  in amdiommu_unref_domain_locked()
424 	amdiommu_domain_destroy(domain);  in amdiommu_unref_domain_locked()
431 	struct amdiommu_domain *domain;  in dte_entry_init_one()  local
434 	domain = CTX2DOM(ctx);  in dte_entry_init_one()
435 	unit = DOM2AMD(domain);  in dte_entry_init_one()
441 	dtep->domainid = domain->domain;  in dte_entry_init_one()
463 	if ((DOM2IODOM(domain)->flags & IOMMU_DOMAIN_IDMAP) != 0) {  in dte_entry_init_one()
466 		MPASS(domain->pglvl > 0 && domain->pglvl <=  in dte_entry_init_one()
468 		dtep->pgmode = domain->pglvl;  in dte_entry_init_one()
481 	struct amdiommu_domain *domain;  in dte_entry_init()  local
484 	domain = CTX2DOM(ctx);  in dte_entry_init()
485 	unit = DOM2AMD(domain);  in dte_entry_init()
496 			dte_entry_init_one(&dtep[i], ctx, domain->pgtblr,  in dte_entry_init()
500 		dte_entry_init_one(dtep, ctx, domain->pgtblr, dte, edte);  in dte_entry_init()
508 	struct amdiommu_domain *domain, *domain1;  in amdiommu_get_ctx_for_dev()  local
553 			domain = domain1;  in amdiommu_get_ctx_for_dev()
559 			LIST_INSERT_HEAD(&unit->domains, domain, link);  in amdiommu_get_ctx_for_dev()
564 			    "amdiommu%d pci%d:%d:%d:%d rid %x domain %d "  in amdiommu_get_ctx_for_dev()
567 				    bus, slot, func, rid, domain->domain,  in amdiommu_get_ctx_for_dev()
574 			domain = CTX2DOM(ctx);  in amdiommu_get_ctx_for_dev()
578 		domain = CTX2DOM(ctx);  in amdiommu_get_ctx_for_dev()