161 static void qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc);
162 static void qinv_context_common(immu_t *immu, uint8_t function_mask,
164 static void qinv_iotlb_common(immu_t *immu, uint_t domain_id,
166 static void qinv_iec_common(immu_t *immu, uint_t iidx,
169 static void qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp);
171 static void qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,
177 qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc)  in qinv_submit_inv_dsc()  argument
186 	qinv = (qinv_t *)immu->immu_qinv;  in qinv_submit_inv_dsc()
205 		    immu_regs_get64(immu, IMMU_REG_INVAL_QH));  in qinv_submit_inv_dsc()
214 	immu_regs_put64(immu, IMMU_REG_INVAL_QT,  in qinv_submit_inv_dsc()
222 qinv_context_common(immu_t *immu, uint8_t function_mask,  in qinv_context_common()  argument
230 	qinv_submit_inv_dsc(immu, &dsc);  in qinv_context_common()
235 qinv_iotlb_common(immu_t *immu, uint_t domain_id,  in qinv_iotlb_common()  argument
242 	if (IMMU_CAP_GET_DRD(immu->immu_regs_cap))  in qinv_iotlb_common()
244 	if (IMMU_CAP_GET_DWD(immu->immu_regs_cap))  in qinv_iotlb_common()
249 		if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap) ||  in qinv_iotlb_common()
250 		    am > IMMU_CAP_GET_MAMV(immu->immu_regs_cap) ||  in qinv_iotlb_common()
274 	qinv_submit_inv_dsc(immu, &dsc);  in qinv_iotlb_common()
279 qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,  in qinv_dev_iotlb_common()  argument
287 	qinv_submit_inv_dsc(immu, &dsc);  in qinv_dev_iotlb_common()
292 qinv_iec_common(immu_t *immu, uint_t iidx, uint_t im, uint_t g)  in qinv_iec_common()  argument
299 	qinv_submit_inv_dsc(immu, &dsc);  in qinv_iec_common()
307 qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp)  in qinv_wait_sync()  argument
331 	qinv_submit_inv_dsc(immu, &dsc);  in qinv_wait_sync()
375 qinv_setup(immu_t *immu)  in qinv_setup()  argument
401 	mutex_init(&(immu->immu_qinv_lock), NULL, MUTEX_DRIVER, NULL);  in qinv_setup()
404 	mutex_enter(&(immu->immu_qinv_lock));  in qinv_setup()
406 	immu->immu_qinv = NULL;  in qinv_setup()
407 	if (!IMMU_ECAP_GET_QI(immu->immu_regs_excap) ||  in qinv_setup()
409 		mutex_exit(&(immu->immu_qinv_lock));  in qinv_setup()
489 	immu->immu_qinv = qinv;  in qinv_setup()
491 	mutex_exit(&(immu->immu_qinv_lock));  in qinv_setup()
507 	mutex_exit(&(immu->immu_qinv_lock));  in qinv_setup()
526 	immu_t *immu;  in immu_qinv_setup()  local
534 	immu = list_head(listp);  in immu_qinv_setup()
535 	for (; immu; immu = list_next(listp, immu)) {  in immu_qinv_setup()
536 		if (qinv_setup(immu) == DDI_SUCCESS) {  in immu_qinv_setup()
537 			immu->immu_qinv_setup = B_TRUE;  in immu_qinv_setup()
548 immu_qinv_startup(immu_t *immu)  in immu_qinv_startup()  argument
553 	if (immu->immu_qinv_setup == B_FALSE) {  in immu_qinv_startup()
557 	qinv = (qinv_t *)immu->immu_qinv;  in immu_qinv_startup()
559 	immu_regs_qinv_enable(immu, qinv_reg_value);  in immu_qinv_startup()
560 	immu->immu_flushops = &immu_qinv_flushops;  in immu_qinv_startup()
561 	immu->immu_qinv_running = B_TRUE;  in immu_qinv_startup()
569 immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,  in immu_qinv_context_fsi()  argument
572 	qinv_context_common(immu, function_mask, source_id,  in immu_qinv_context_fsi()
574 	qinv_wait_sync(immu, iwp);  in immu_qinv_context_fsi()
582 immu_qinv_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)  in immu_qinv_context_dsi()  argument
584 	qinv_context_common(immu, 0, 0, domain_id, CTT_INV_G_DOMAIN);  in immu_qinv_context_dsi()
585 	qinv_wait_sync(immu, iwp);  in immu_qinv_context_dsi()
593 immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)  in immu_qinv_context_gbl()  argument
595 	qinv_context_common(immu, 0, 0, 0, CTT_INV_G_GLOBAL);  in immu_qinv_context_gbl()
596 	qinv_wait_sync(immu, iwp);  in immu_qinv_context_gbl()
604 immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,  in immu_qinv_iotlb_psi()  argument
610 	max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);  in immu_qinv_iotlb_psi()
613 	if (IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {  in immu_qinv_iotlb_psi()
617 				qinv_iotlb_common(immu, domain_id,  in immu_qinv_iotlb_psi()
624 			qinv_iotlb_common(immu, domain_id,  in immu_qinv_iotlb_psi()
630 		qinv_iotlb_common(immu, domain_id, dvma,  in immu_qinv_iotlb_psi()
634 	qinv_wait_sync(immu, iwp);  in immu_qinv_iotlb_psi()
642 immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)  in immu_qinv_iotlb_dsi()  argument
644 	qinv_iotlb_common(immu, domain_id, 0, 0, 0, TLB_INV_G_DOMAIN);  in immu_qinv_iotlb_dsi()
645 	qinv_wait_sync(immu, iwp);  in immu_qinv_iotlb_dsi()
653 immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)  in immu_qinv_iotlb_gbl()  argument
655 	qinv_iotlb_common(immu, 0, 0, 0, 0, TLB_INV_G_GLOBAL);  in immu_qinv_iotlb_gbl()
656 	qinv_wait_sync(immu, iwp);  in immu_qinv_iotlb_gbl()
661 immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp)  in immu_qinv_intr_global()  argument
663 	qinv_iec_common(immu, 0, 0, IEC_INV_GLOBAL);  in immu_qinv_intr_global()
664 	qinv_wait_sync(immu, iwp);  in immu_qinv_intr_global()
669 immu_qinv_intr_one_cache(immu_t *immu, uint_t iidx, immu_inv_wait_t *iwp)  in immu_qinv_intr_one_cache()  argument
671 	qinv_iec_common(immu, iidx, 0, IEC_INV_INDEX);  in immu_qinv_intr_one_cache()
672 	qinv_wait_sync(immu, iwp);  in immu_qinv_intr_one_cache()
677 immu_qinv_intr_caches(immu_t *immu, uint_t iidx, uint_t cnt,  in immu_qinv_intr_caches()  argument
687 			qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);  in immu_qinv_intr_caches()
689 		qinv_wait_sync(immu, iwp);  in immu_qinv_intr_caches()
697 	if (mask > IMMU_ECAP_GET_MHMV(immu->immu_regs_excap)) {  in immu_qinv_intr_caches()
699 			qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);  in immu_qinv_intr_caches()
701 		qinv_wait_sync(immu, iwp);  in immu_qinv_intr_caches()
705 	qinv_iec_common(immu, iidx, mask, IEC_INV_INDEX);  in immu_qinv_intr_caches()
707 	qinv_wait_sync(immu, iwp);  in immu_qinv_intr_caches()
711 immu_qinv_report_fault(immu_t *immu)  in immu_qinv_report_fault()  argument
718 	mutex_enter(&(immu->immu_qinv_lock));  in immu_qinv_report_fault()
720 	qinv = (qinv_t *)(immu->immu_qinv);  in immu_qinv_report_fault()
723 	    immu_regs_get64(immu, IMMU_REG_INVAL_QH));  in immu_qinv_report_fault()
729 	ddi_err(DER_WARN, immu->immu_dip,  in immu_qinv_report_fault()
738 	mutex_exit(&(immu->immu_qinv_lock));  in immu_qinv_report_fault()