1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
10  * 1. Redistributions of source code must retain the above copyright
73 #define	VTD_GCR_WBF		(1 << 27)
74 #define	VTD_GCR_SRTP		(1 << 30)
75 #define	VTD_GCR_TE		(1U << 31)
77 #define	VTD_GSR_WBFS		(1 << 27)
78 #define	VTD_GSR_RTPS		(1 << 30)
79 #define	VTD_GSR_TES		(1U << 31)
81 #define	VTD_CCR_ICC		(1UL << 63)	/* invalidate context cache */
82 #define	VTD_CCR_CIRG_GLOBAL	(1UL << 61)	/* global invalidation */
84 #define	VTD_IIR_IVT		(1UL << 63)	/* invalidation IOTLB */
85 #define	VTD_IIR_IIRG_GLOBAL	(1ULL << 60)	/* global IOTLB invalidation */
88 #define	VTD_IIR_DRAIN_READS	(1ULL << 49)	/* drain pending DMA reads */
89 #define	VTD_IIR_DRAIN_WRITES	(1ULL << 48)	/* drain pending DMA writes */
94 #define	VTD_CTX_TT_ALL		(1UL << 2)
96 #define	VTD_PTE_RD		(1UL << 0)
97 #define	VTD_PTE_WR		(1UL << 1)
98 #define	VTD_PTE_SUPERPAGE	(1UL << 7)
132 	nd = VTD_CAP_ND(vtdmap->cap);  in vtd_max_domains()
137 	case 1:  in vtd_max_domains()
160 	/* Skip domain id 0 - it is reserved when Caching Mode field is set */  in domain_id()
161 	for (id = 1; id < max_domains; id++) {  in domain_id()
163 			if (dom->id == id)  in domain_id()
189 		if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) {  in vtd_device_scope()
191 			 * From Intel VT-d arch spec, version 3.0:  in vtd_device_scope()
200 		end = (char *)drhd + drhd->Header.Length;  in vtd_device_scope()
201 		remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT);  in vtd_device_scope()
203 			device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining);  in vtd_device_scope()
204 			remaining -= device_scope->Length;  in vtd_device_scope()
206 			switch (device_scope->EntryType){  in vtd_device_scope()
215 			if (PCI_RID2BUS(rid) != device_scope->Bus)  in vtd_device_scope()
218 			pathend = (char *)device_scope + device_scope->Length;  in vtd_device_scope()
219 			pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE);  in vtd_device_scope()
221 				path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining);  in vtd_device_scope()
222 				pathremaining -= sizeof(ACPI_DMAR_PCI_PATH);  in vtd_device_scope()
224 				if (PCI_RID2SLOT(rid) != path->Device)  in vtd_device_scope()
226 				if (PCI_RID2FUNC(rid) != path->Function)  in vtd_device_scope()
243 	if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0)  in vtd_wbflush()
246 	if (VTD_CAP_RWBF(vtdmap->cap)) {  in vtd_wbflush()
247 		vtdmap->gcr = VTD_GCR_WBF;  in vtd_wbflush()
248 		while ((vtdmap->gsr & VTD_GSR_WBFS) != 0)  in vtd_wbflush()
257 	vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL;  in vtd_ctx_global_invalidate()
258 	while ((vtdmap->ccr & VTD_CCR_ICC) != 0)  in vtd_ctx_global_invalidate()
270 	offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16;  in vtd_iotlb_global_invalidate()
276 	while (1) {  in vtd_iotlb_global_invalidate()
287 	vtdmap->gcr = VTD_GCR_TE;  in vtd_translation_enable()
288 	while ((vtdmap->gsr & VTD_GSR_TES) == 0)  in vtd_translation_enable()
296 	vtdmap->gcr = 0;  in vtd_translation_disable()
297 	while ((vtdmap->gsr & VTD_GSR_TES) != 0)  in vtd_translation_disable()
321 	 * set vtd.regmap.1.addr=0xfeda0000  in vtd_init()
338 	end = (char *)dmar + dmar->Header.Length;  in vtd_init()
339 	remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR);  in vtd_init()
341 		hdr = (ACPI_DMAR_HEADER *)(end - remaining);  in vtd_init()
342 		if (hdr->Length > remaining)  in vtd_init()
345 		 * From Intel VT-d arch spec, version 1.3:  in vtd_init()
349 		 * type 1 (RMRR) and so forth.  in vtd_init()
351 		if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT)  in vtd_init()
356 		vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);  in vtd_init()
359 		remaining -= hdr->Length;  in vtd_init()
372 		if (VTD_CAP_CM(vtdmap->cap) != 0)  in vtd_init()
381 	 * Set up the root-table to point to the context-entry tables  in vtd_init()
410 		vtdmap->rta = vtophys(root_table);  in vtd_enable()
411 		vtdmap->gcr = VTD_GCR_SRTP;  in vtd_enable()
412 		while ((vtdmap->gsr & VTD_GSR_RTPS) == 0)  in vtd_enable()
448 	pt_paddr = vtophys(dom->ptp);  in vtd_add_device()
454 		      (uint16_t)(ctxp[idx + 1] >> 8));  in vtd_add_device()
465 	ctxp[idx + 1] = dom->addrwidth | (dom->id << 8);  in vtd_add_device()
467 	if (VTD_ECAP_DI(vtdmap->ext_cap))  in vtd_add_device()
496 	ctxp[idx + 1] = 0;  in vtd_remove_device()
501 	 * XXX use device-selective invalidation for Context Cache  in vtd_remove_device()
502 	 * XXX use domain-selective invalidation for IOTLB  in vtd_remove_device()
512 #define	REMOVE_MAPPING	1
515 vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,  in vtd_update_mapping()  argument
526 	KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__,  in vtd_update_mapping()
527 	    gpa, len));  in vtd_update_mapping()
528 	KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond "  in vtd_update_mapping()
529 	    "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr));  in vtd_update_mapping()
531 	if (gpa & PAGE_MASK)  in vtd_update_mapping()
532 		panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa);  in vtd_update_mapping()
544 	 * - supported super page size  in vtd_update_mapping()
545 	 * - alignment of the region starting at 'gpa' and 'hpa'  in vtd_update_mapping()
546 	 * - length of the region 'len'  in vtd_update_mapping()
549 	for (i = 3; i >= 0; i--) {  in vtd_update_mapping()
550 		spsize = 1UL << spshift;  in vtd_update_mapping()
551 		if ((dom->spsmask & (1 << i)) != 0 &&  in vtd_update_mapping()
552 		    (gpa & (spsize - 1)) == 0 &&  in vtd_update_mapping()
553 		    (hpa & (spsize - 1)) == 0 &&  in vtd_update_mapping()
557 		spshift -= 9;  in vtd_update_mapping()
560 	ptp = dom->ptp;  in vtd_update_mapping()
561 	nlevels = dom->pt_levels;  in vtd_update_mapping()
562 	while (--nlevels >= 0) {  in vtd_update_mapping()
564 		ptpindex = (gpa >> ptpshift) & 0x1FF;  in vtd_update_mapping()
572 		 * We are working on a non-leaf page table page.  in vtd_update_mapping()
585 	if ((gpa & ((1UL << ptpshift) - 1)) != 0)  in vtd_update_mapping()
586 		panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift);  in vtd_update_mapping()
589 	 * Update the 'gpa' -> 'hpa' mapping  in vtd_update_mapping()
600 	return (1UL << ptpshift);  in vtd_update_mapping()
604 vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)  in vtd_create_mapping()  argument
607 	return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING));  in vtd_create_mapping()
611 vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)  in vtd_remove_mapping()  argument
614 	return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING));  in vtd_remove_mapping()
625 	 * XXX use domain-selective invalidation for IOTLB  in vtd_invalidate_tlb()
650 		addr = 1ULL << gaw;  in vtd_create_domain()
652 	res = (gaw - 12) % 9;  in vtd_create_domain()
656 		agaw = gaw + 9 - res;  in vtd_create_domain()
673 		tmp &= VTD_CAP_SAGAW(vtdmap->cap);  in vtd_create_domain()
677 		if ((tmp & (1 << i)) != 0 && sagaw >= agaw)  in vtd_create_domain()
692 	dom->pt_levels = pt_levels;  in vtd_create_domain()
693 	dom->addrwidth = addrwidth;  in vtd_create_domain()
694 	dom->id = domain_id();  in vtd_create_domain()
695 	dom->maxaddr = maxaddr;  in vtd_create_domain()
696 	dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK);  in vtd_create_domain()
697 	if ((uintptr_t)dom->ptp & PAGE_MASK)  in vtd_create_domain()
698 		panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp);  in vtd_create_domain()
715 	dom->spsmask = ~0;  in vtd_create_domain()
719 		dom->spsmask &= VTD_CAP_SPS(vtdmap->cap);  in vtd_create_domain()
734 	if (level > 1) {  in vtd_free_ptp()
741 			vtd_free_ptp(nlp, level - 1);  in vtd_free_ptp()
757 	vtd_free_ptp(dom->ptp, dom->pt_levels);  in vtd_destroy_domain()