72 static int dmar_unmap_buf_locked(struct dmar_domain *domain,
167 dmar_get_idmap_pgtbl(struct dmar_domain *domain, iommu_gaddr_t maxaddr)  in dmar_get_idmap_pgtbl()  argument
181 	for (i = 0; i < domain->pglvl; i++) {  in dmar_get_idmap_pgtbl()
182 		if (i == domain->pglvl - 1 || domain_is_sp_lvl(domain, i)) {  in dmar_get_idmap_pgtbl()
199 		    dmar_pglvl_supported(domain->dmar, tbl->pglvl) &&  in dmar_get_idmap_pgtbl()
204 			domain->pglvl = tbl->pglvl; /* XXXKIB ? */  in dmar_get_idmap_pgtbl()
218 		    dmar_pglvl_supported(domain->dmar, tbl->pglvl) &&  in dmar_get_idmap_pgtbl()
223 			domain->pglvl = tbl->pglvl; /* XXXKIB ? */  in dmar_get_idmap_pgtbl()
232 	tbl->pglvl = domain->pglvl;  in dmar_get_idmap_pgtbl()
263 	unit = domain->dmar;  in dmar_get_idmap_pgtbl()
328 dmar_pgtbl_map_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl,  in dmar_pgtbl_map_pte()  argument
336 	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);  in dmar_pgtbl_map_pte()
339 	idx = pglvl_pgtbl_get_pindex(domain->pglvl, base, lvl);  in dmar_pgtbl_map_pte()
347 		pte = iommu_map_pgtbl(domain->pgtbl_obj, idx, flags, sf);  in dmar_pgtbl_map_pte()
350 			    ("lost root page table page %p", domain));  in dmar_pgtbl_map_pte()
356 			m = iommu_pgalloc(domain->pgtbl_obj, idx, flags |  in dmar_pgtbl_map_pte()
371 			ptep = dmar_pgtbl_map_pte(domain, base, lvl - 1,  in dmar_pgtbl_map_pte()
375 				    ("loosing root page %p", domain));  in dmar_pgtbl_map_pte()
377 				iommu_pgfree(domain->pgtbl_obj, m->pindex,  in dmar_pgtbl_map_pte()
383 			dmar_flush_pte_to_ram(domain->dmar, ptep);  in dmar_pgtbl_map_pte()
391 	pte += pglvl_pgtbl_pte_off(domain->pglvl, base, lvl);  in dmar_pgtbl_map_pte()
396 dmar_map_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base,  in dmar_map_buf_locked()  argument
407 	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);  in dmar_map_buf_locked()
416 			pg_sz = domain_page_size(domain, lvl);  in dmar_map_buf_locked()
418 			if (lvl == domain->pglvl - 1)  in dmar_map_buf_locked()
424 			if (!domain_is_sp_lvl(domain, lvl))  in dmar_map_buf_locked()
448 		    ("mapping loop overflow %p %jx %jx %jx", domain,  in dmar_map_buf_locked()
451 		pte = dmar_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf);  in dmar_map_buf_locked()
454 			    ("failed waitable pte alloc %p", domain));  in dmar_map_buf_locked()
457 			dmar_unmap_buf_locked(domain, base1, base - base1,  in dmar_map_buf_locked()
464 		dmar_flush_pte_to_ram(domain->dmar, pte);  in dmar_map_buf_locked()
477 	struct dmar_domain *domain;  in dmar_map_buf()  local
491 	domain = IODOM2DOM(iodom);  in dmar_map_buf()
492 	unit = domain->dmar;  in dmar_map_buf()
495 	    ("modifying idmap pagetable domain %p", domain));  in dmar_map_buf()
497 	    ("non-aligned base %p %jx %jx", domain, (uintmax_t)base,  in dmar_map_buf()
500 	    ("non-aligned size %p %jx %jx", domain, (uintmax_t)base,  in dmar_map_buf()
502 	KASSERT(size > 0, ("zero size %p %jx %jx", domain, (uintmax_t)base,  in dmar_map_buf()
504 	KASSERT(base < (1ULL << domain->agaw),  in dmar_map_buf()
505 	    ("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base,  in dmar_map_buf()
506 	    (uintmax_t)size, domain->agaw));  in dmar_map_buf()
507 	KASSERT(base + size < (1ULL << domain->agaw),  in dmar_map_buf()
508 	    ("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base,  in dmar_map_buf()
509 	    (uintmax_t)size, domain->agaw));  in dmar_map_buf()
511 	    ("size overflow %p %jx %jx", domain, (uintmax_t)base,  in dmar_map_buf()
521 	    domain, (uintmax_t)pflags));  in dmar_map_buf()
525 	    domain, (uintmax_t)pflags));  in dmar_map_buf()
528 	DMAR_DOMAIN_PGLOCK(domain);  in dmar_map_buf()
529 	error = dmar_map_buf_locked(domain, base, size, ma, pflags, flags,  in dmar_map_buf()
531 	DMAR_DOMAIN_PGUNLOCK(domain);  in dmar_map_buf()
536 		dmar_flush_iotlb_sync(domain, base, size);  in dmar_map_buf()
546 static void dmar_unmap_clear_pte(struct dmar_domain *domain,
551 dmar_free_pgtbl_pde(struct dmar_domain *domain, iommu_gaddr_t base,  in dmar_free_pgtbl_pde()  argument
559 	pde = dmar_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf);  in dmar_free_pgtbl_pde()
560 	dmar_unmap_clear_pte(domain, base, lvl, flags, pde, &sf,  in dmar_free_pgtbl_pde()
565 dmar_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl,  in dmar_unmap_clear_pte()  argument
572 	dmar_flush_pte_to_ram(domain->dmar, pte);  in dmar_unmap_clear_pte()
581 	    ("lost reference (lvl) on root pg domain %p base %jx lvl %d",  in dmar_unmap_clear_pte()
582 	    domain, (uintmax_t)base, lvl));  in dmar_unmap_clear_pte()
584 	    ("lost reference (idx) on root pg domain %p base %jx lvl %d",  in dmar_unmap_clear_pte()
585 	    domain, (uintmax_t)base, lvl));  in dmar_unmap_clear_pte()
586 	iommu_pgfree(domain->pgtbl_obj, m->pindex, flags, entry);  in dmar_unmap_clear_pte()
587 	dmar_free_pgtbl_pde(domain, base, lvl - 1, flags, entry);  in dmar_unmap_clear_pte()
594 dmar_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base,  in dmar_unmap_buf_locked()  argument
603 	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);  in dmar_unmap_buf_locked()
607 	KASSERT((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) == 0,  in dmar_unmap_buf_locked()
608 	    ("modifying idmap pagetable domain %p", domain));  in dmar_unmap_buf_locked()
610 	    ("non-aligned base %p %jx %jx", domain, (uintmax_t)base,  in dmar_unmap_buf_locked()
613 	    ("non-aligned size %p %jx %jx", domain, (uintmax_t)base,  in dmar_unmap_buf_locked()
615 	KASSERT(base < (1ULL << domain->agaw),  in dmar_unmap_buf_locked()
616 	    ("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base,  in dmar_unmap_buf_locked()
617 	    (uintmax_t)size, domain->agaw));  in dmar_unmap_buf_locked()
618 	KASSERT(base + size < (1ULL << domain->agaw),  in dmar_unmap_buf_locked()
619 	    ("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base,  in dmar_unmap_buf_locked()
620 	    (uintmax_t)size, domain->agaw));  in dmar_unmap_buf_locked()
622 	    ("size overflow %p %jx %jx", domain, (uintmax_t)base,  in dmar_unmap_buf_locked()
631 		for (lvl = 0; lvl < domain->pglvl; lvl++) {  in dmar_unmap_buf_locked()
632 			if (lvl != domain->pglvl - 1 &&  in dmar_unmap_buf_locked()
633 			    !domain_is_sp_lvl(domain, lvl))  in dmar_unmap_buf_locked()
635 			pg_sz = domain_page_size(domain, lvl);  in dmar_unmap_buf_locked()
638 			pte = dmar_pgtbl_map_pte(domain, base, lvl, flags,  in dmar_unmap_buf_locked()
642 			    domain, (uintmax_t)base, lvl, flags));  in dmar_unmap_buf_locked()
644 			    lvl == domain->pglvl - 1) {  in dmar_unmap_buf_locked()
645 				dmar_unmap_clear_pte(domain, base, lvl,  in dmar_unmap_buf_locked()
651 		    ("unmapping loop overflow %p %jx %jx %jx", domain,  in dmar_unmap_buf_locked()
669 	struct dmar_domain *domain;  in dmar_unmap_buf()  local
672 	domain = IODOM2DOM(iodom);  in dmar_unmap_buf()
674 	DMAR_DOMAIN_PGLOCK(domain);  in dmar_unmap_buf()
675 	error = dmar_unmap_buf_locked(domain, entry->start, entry->end -  in dmar_unmap_buf()
677 	DMAR_DOMAIN_PGUNLOCK(domain);  in dmar_unmap_buf()
682 dmar_domain_alloc_pgtbl(struct dmar_domain *domain)  in dmar_domain_alloc_pgtbl()  argument
687 	KASSERT(domain->pgtbl_obj == NULL,  in dmar_domain_alloc_pgtbl()
688 	    ("already initialized %p", domain));  in dmar_domain_alloc_pgtbl()
690 	unit = domain->dmar;  in dmar_domain_alloc_pgtbl()
691 	domain->pgtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL,  in dmar_domain_alloc_pgtbl()
692 	    IDX_TO_OFF(pglvl_max_pages(domain->pglvl)), 0, 0, NULL);  in dmar_domain_alloc_pgtbl()
694 		domain->pgtbl_obj->domain.dr_policy = DOMAINSET_PREF(  in dmar_domain_alloc_pgtbl()
697 	DMAR_DOMAIN_PGLOCK(domain);  in dmar_domain_alloc_pgtbl()
698 	m = iommu_pgalloc(domain->pgtbl_obj, 0, IOMMU_PGF_WAITOK |  in dmar_domain_alloc_pgtbl()
702 	DMAR_DOMAIN_PGUNLOCK(domain);  in dmar_domain_alloc_pgtbl()
704 	domain->iodom.flags |= IOMMU_DOMAIN_PGTBL_INITED;  in dmar_domain_alloc_pgtbl()
710 dmar_domain_free_pgtbl(struct dmar_domain *domain)  in dmar_domain_free_pgtbl()  argument
716 	obj = domain->pgtbl_obj;  in dmar_domain_free_pgtbl()
718 		KASSERT((domain->dmar->hw_ecap & DMAR_ECAP_PT) != 0 &&  in dmar_domain_free_pgtbl()
719 		    (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0,  in dmar_domain_free_pgtbl()
720 		    ("lost pagetable object domain %p", domain));  in dmar_domain_free_pgtbl()
723 	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);  in dmar_domain_free_pgtbl()
724 	domain->pgtbl_obj = NULL;  in dmar_domain_free_pgtbl()
726 	if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0) {  in dmar_domain_free_pgtbl()
728 		domain->iodom.flags &= ~IOMMU_DOMAIN_IDMAP;  in dmar_domain_free_pgtbl()
760 dmar_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base,  in dmar_flush_iotlb_sync()  argument
768 	unit = domain->dmar;  in dmar_flush_iotlb_sync()
775 		    DMAR_IOTLB_DID(domain->domain), iro);  in dmar_flush_iotlb_sync()
786 			    DMAR_IOTLB_DID(domain->domain), iro);  in dmar_flush_iotlb_sync()
795 			 * address space for the domain.  in dmar_flush_iotlb_sync()