38 #define	get_reg32(immu, offset)	ddi_get32((immu)->immu_regs_handle, \  argument
39 		(uint32_t *)(immu->immu_regs_addr + (offset)))
40 #define	get_reg64(immu, offset)	ddi_get64((immu)->immu_regs_handle, \  argument
41 		(uint64_t *)(immu->immu_regs_addr + (offset)))
42 #define	put_reg32(immu, offset, val)	ddi_put32\  argument
43 		((immu)->immu_regs_handle, \
44 		(uint32_t *)(immu->immu_regs_addr + (offset)), val)
45 #define	put_reg64(immu, offset, val)	ddi_put64\  argument
46 		((immu)->immu_regs_handle, \
47 		(uint64_t *)(immu->immu_regs_addr + (offset)), val)
65 #define	wait_completion(immu, offset, getf, completion, status) \  argument
71 		status = getf(immu, offset); \
97 iotlb_flush(immu_t *immu, uint_t domain_id,  in iotlb_flush()  argument
105 	iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap);  in iotlb_flush()
111 	if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) {  in iotlb_flush()
115 	if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) {  in iotlb_flush()
138 		    immu->immu_name);  in iotlb_flush()
143 		put_reg64(immu, iva_offset, iva);  in iotlb_flush()
144 	put_reg64(immu, iotlb_offset, command);  in iotlb_flush()
145 	wait_completion(immu, iotlb_offset, get_reg64,  in iotlb_flush()
155 immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages,  in immu_regs_iotlb_psi()  argument
167 	if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {  in immu_regs_iotlb_psi()
168 		immu_regs_iotlb_dsi(immu, did, iwp);  in immu_regs_iotlb_psi()
172 	max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);  in immu_regs_iotlb_psi()
174 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_iotlb_psi()
196 		iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI);  in immu_regs_iotlb_psi()
204 		iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI);  in immu_regs_iotlb_psi()
206 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_iotlb_psi()
215 immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)  in immu_regs_iotlb_dsi()  argument
217 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_iotlb_dsi()
218 	iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI);  in immu_regs_iotlb_dsi()
219 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_iotlb_dsi()
228 immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)  in immu_regs_iotlb_gbl()  argument
230 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_iotlb_gbl()
231 	iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL);  in immu_regs_iotlb_gbl()
232 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_iotlb_gbl()
259 set_agaw(immu_t *immu)  in set_agaw()  argument
273 	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);  in set_agaw()
274 	sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap);  in set_agaw()
293 		    immu->immu_name, sagaw_mask, max_sagaw_mask);  in set_agaw()
325 		    "and magaw",  immu->immu_name, agaw, magaw);  in set_agaw()
332 		    immu->immu_name, nlevels);  in set_agaw()
339 	immu->immu_dvma_nlevels = nlevels;  in set_agaw()
340 	immu->immu_dvma_agaw = agaw;  in set_agaw()
346 setup_regs(immu_t *immu)  in setup_regs()  argument
353 	mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER,  in setup_regs()
359 	error = ddi_regs_map_setup(immu->immu_dip, 0,  in setup_regs()
360 	    (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,  in setup_regs()
362 	    &(immu->immu_regs_handle));  in setup_regs()
366 		    immu->immu_name);  in setup_regs()
367 		mutex_destroy(&(immu->immu_regs_lock));  in setup_regs()
374 	immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP);  in setup_regs()
375 	immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP);  in setup_regs()
380 	if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) {  in setup_regs()
381 		immu->immu_dvma_coherent = B_TRUE;  in setup_regs()
383 		immu->immu_dvma_coherent = B_FALSE;  in setup_regs()
387 			    "missing clflush functionality", immu->immu_name);  in setup_regs()
388 			ddi_regs_map_free(&(immu->immu_regs_handle));  in setup_regs()
389 			mutex_destroy(&(immu->immu_regs_lock));  in setup_regs()
395 	immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu);  in setup_regs()
396 	immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu);  in setup_regs()
398 	if (IMMU_ECAP_GET_CH(immu->immu_regs_excap) && immu_use_tm)  in setup_regs()
399 		immu->immu_ptemask = PDTE_MASK_TM;  in setup_regs()
401 		immu->immu_ptemask = 0;  in setup_regs()
407 	    !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {  in setup_regs()
411 		IMMU_CAP_SET_RWBF(immu->immu_regs_cap);  in setup_regs()
417 	immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap);  in setup_regs()
422 	if (set_agaw(immu) != DDI_SUCCESS) {  in setup_regs()
423 		ddi_regs_map_free(&(immu->immu_regs_handle));  in setup_regs()
424 		mutex_destroy(&(immu->immu_regs_lock));  in setup_regs()
427 	immu->immu_regs_cmdval = 0;  in setup_regs()
429 	immu->immu_flushops = &immu_regs_flushops;  in setup_regs()
445 	immu_t *immu;  in immu_regs_setup()  local
448 		immu = list_head(listp);  in immu_regs_setup()
449 		for (; immu; immu = list_next(listp, immu)) {  in immu_regs_setup()
451 			if (setup_regs(immu) != DDI_SUCCESS) {  in immu_regs_setup()
452 				immu->immu_regs_setup = B_FALSE;  in immu_regs_setup()
454 				immu->immu_regs_setup = B_TRUE;  in immu_regs_setup()
464 immu_regs_resume(immu_t *immu)  in immu_regs_resume()  argument
471 	error = ddi_regs_map_setup(immu->immu_dip, 0,  in immu_regs_resume()
472 	    (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,  in immu_regs_resume()
474 	    &(immu->immu_regs_handle));  in immu_regs_resume()
479 	immu_regs_set_root_table(immu);  in immu_regs_resume()
481 	immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr,  in immu_regs_resume()
482 	    immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr);  in immu_regs_resume()
484 	(void) immu_intr_handler(immu);  in immu_regs_resume()
486 	immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg);  in immu_regs_resume()
488 	immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value);  in immu_regs_resume()
498 immu_regs_suspend(immu_t *immu)  in immu_regs_suspend()  argument
501 	immu->immu_intrmap_running = B_FALSE;  in immu_regs_suspend()
504 	ddi_regs_map_free(&(immu->immu_regs_handle));  in immu_regs_suspend()
512 immu_regs_startup(immu_t *immu)  in immu_regs_startup()  argument
516 	if (immu->immu_regs_setup == B_FALSE) {  in immu_regs_startup()
520 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_startup()
521 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_startup()
522 	    immu->immu_regs_cmdval | IMMU_GCMD_TE);  in immu_regs_startup()
523 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_startup()
525 	immu->immu_regs_cmdval |= IMMU_GCMD_TE;  in immu_regs_startup()
526 	immu->immu_regs_running = B_TRUE;  in immu_regs_startup()
527 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_startup()
529 	ddi_err(DER_NOTE, NULL, "%s running", immu->immu_name);  in immu_regs_startup()
537 immu_regs_shutdown(immu_t *immu)  in immu_regs_shutdown()  argument
541 	if (immu->immu_regs_running == B_FALSE) {  in immu_regs_shutdown()
545 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_shutdown()
546 	immu->immu_regs_cmdval &= ~IMMU_GCMD_TE;  in immu_regs_shutdown()
547 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_shutdown()
548 	    immu->immu_regs_cmdval);  in immu_regs_shutdown()
549 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_shutdown()
551 	immu->immu_regs_running = B_FALSE;  in immu_regs_shutdown()
552 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_shutdown()
554 	ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name);  in immu_regs_shutdown()
563 immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,  in immu_regs_intr_enable()  argument
566 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_intr_enable()
567 	immu->immu_regs_intr_msi_addr = msi_addr;  in immu_regs_intr_enable()
568 	immu->immu_regs_intr_uaddr = uaddr;  in immu_regs_intr_enable()
569 	immu->immu_regs_intr_msi_data = msi_data;  in immu_regs_intr_enable()
570 	put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr);  in immu_regs_intr_enable()
571 	put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr);  in immu_regs_intr_enable()
572 	put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data);  in immu_regs_intr_enable()
573 	put_reg32(immu, IMMU_REG_FEVNT_CON, 0);  in immu_regs_intr_enable()
574 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_intr_enable()
582 immu_regs_passthru_supported(immu_t *immu)  in immu_regs_passthru_supported()  argument
584 	if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) {  in immu_regs_passthru_supported()
597 immu_regs_is_TM_reserved(immu_t *immu)  in immu_regs_is_TM_reserved()  argument
599 	if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) ||  in immu_regs_is_TM_reserved()
600 	    IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {  in immu_regs_is_TM_reserved()
611 immu_regs_is_SNP_reserved(immu_t *immu)  in immu_regs_is_SNP_reserved()  argument
614 	return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE);  in immu_regs_is_SNP_reserved()
623 immu_regs_wbf_flush(immu_t *immu)  in immu_regs_wbf_flush()  argument
627 	if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {  in immu_regs_wbf_flush()
631 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_wbf_flush()
632 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_wbf_flush()
633 	    immu->immu_regs_cmdval | IMMU_GCMD_WBF);  in immu_regs_wbf_flush()
634 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_wbf_flush()
636 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_wbf_flush()
645 immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size)  in immu_regs_cpu_flush()  argument
649 	if (immu->immu_dvma_coherent == B_TRUE)  in immu_regs_cpu_flush()
668 context_flush(immu_t *immu, uint8_t function_mask,  in context_flush()  argument
693 		    immu->immu_name);  in context_flush()
697 	mutex_enter(&(immu->immu_regs_lock));  in context_flush()
698 	put_reg64(immu, IMMU_REG_CONTEXT_CMD, command);  in context_flush()
699 	wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64,  in context_flush()
701 	mutex_exit(&(immu->immu_regs_lock));  in context_flush()
706 immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,  in immu_regs_context_fsi()  argument
709 	context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI);  in immu_regs_context_fsi()
714 immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)  in immu_regs_context_dsi()  argument
716 	context_flush(immu, 0, 0, domain_id, CONTEXT_DSI);  in immu_regs_context_dsi()
721 immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)  in immu_regs_context_gbl()  argument
723 	context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL);  in immu_regs_context_gbl()
736 immu_regs_set_root_table(immu_t *immu)  in immu_regs_set_root_table()  argument
740 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_set_root_table()
741 	put_reg64(immu, IMMU_REG_ROOTENTRY,  in immu_regs_set_root_table()
742 	    immu->immu_ctx_root->hwpg_paddr);  in immu_regs_set_root_table()
743 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_set_root_table()
744 	    immu->immu_regs_cmdval | IMMU_GCMD_SRTP);  in immu_regs_set_root_table()
745 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_set_root_table()
747 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_set_root_table()
753 immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value)  in immu_regs_qinv_enable()  argument
760 	mutex_enter(&immu->immu_regs_lock);  in immu_regs_qinv_enable()
761 	immu->immu_qinv_reg_value = qinv_reg_value;  in immu_regs_qinv_enable()
763 	put_reg64(immu, IMMU_REG_INVAL_QT, 0);  in immu_regs_qinv_enable()
766 	put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value);  in immu_regs_qinv_enable()
769 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_qinv_enable()
770 	    immu->immu_regs_cmdval | IMMU_GCMD_QIE);  in immu_regs_qinv_enable()
771 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_qinv_enable()
773 	mutex_exit(&immu->immu_regs_lock);  in immu_regs_qinv_enable()
775 	immu->immu_regs_cmdval |= IMMU_GCMD_QIE;  in immu_regs_qinv_enable()
776 	immu->immu_qinv_running = B_TRUE;  in immu_regs_qinv_enable()
782 immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg)  in immu_regs_intrmap_enable()  argument
790 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_intrmap_enable()
791 	immu->immu_intrmap_irta_reg = irta_reg;  in immu_regs_intrmap_enable()
792 	put_reg64(immu, IMMU_REG_IRTAR, irta_reg);  in immu_regs_intrmap_enable()
793 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_intrmap_enable()
794 	    immu->immu_regs_cmdval | IMMU_GCMD_SIRTP);  in immu_regs_intrmap_enable()
795 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_intrmap_enable()
797 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_intrmap_enable()
800 	immu_qinv_intr_global(immu, &immu->immu_intrmap_inv_wait);  in immu_regs_intrmap_enable()
803 	mutex_enter(&(immu->immu_regs_lock));  in immu_regs_intrmap_enable()
804 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_intrmap_enable()
805 	    immu->immu_regs_cmdval | IMMU_GCMD_IRE);  in immu_regs_intrmap_enable()
806 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_intrmap_enable()
809 	immu->immu_regs_cmdval |= IMMU_GCMD_IRE;  in immu_regs_intrmap_enable()
812 	put_reg32(immu, IMMU_REG_GLOBAL_CMD,  in immu_regs_intrmap_enable()
813 	    immu->immu_regs_cmdval | IMMU_GCMD_CFI);  in immu_regs_intrmap_enable()
814 	wait_completion(immu, IMMU_REG_GLOBAL_STS,  in immu_regs_intrmap_enable()
817 	immu->immu_regs_cmdval |= IMMU_GCMD_CFI;  in immu_regs_intrmap_enable()
818 	mutex_exit(&(immu->immu_regs_lock));  in immu_regs_intrmap_enable()
820 	immu->immu_intrmap_running = B_TRUE;  in immu_regs_intrmap_enable()
824 immu_regs_get64(immu_t *immu, uint_t reg)  in immu_regs_get64()  argument
826 	return (get_reg64(immu, reg));  in immu_regs_get64()
830 immu_regs_get32(immu_t *immu, uint_t reg)  in immu_regs_get32()  argument
832 	return (get_reg32(immu, reg));  in immu_regs_get32()
836 immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val)  in immu_regs_put64()  argument
838 	put_reg64(immu, reg, val);  in immu_regs_put64()
842 immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val)  in immu_regs_put32()  argument
844 	put_reg32(immu, reg, val);  in immu_regs_put32()