62 	register ddi_dma_impl_t *mp;  in px_dma_allocmp()  local
66 	if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) {  in px_dma_allocmp()
72 		return (mp);  in px_dma_allocmp()
75 	mp->dmai_rdip = rdip;  in px_dma_allocmp()
76 	mp->dmai_flags = 0;  in px_dma_allocmp()
77 	mp->dmai_pfnlst = NULL;  in px_dma_allocmp()
78 	mp->dmai_winlst = NULL;  in px_dma_allocmp()
79 	mp->dmai_ncookies = 0;  in px_dma_allocmp()
80 	mp->dmai_curcookie = 0;  in px_dma_allocmp()
105 	mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;  in px_dma_allocmp()
106 	mp->dmai_attr.dma_attr_flags = (uint_t)0;  in px_dma_allocmp()
107 	mp->dmai_fault = 0;  in px_dma_allocmp()
108 	mp->dmai_fault_check = NULL;  in px_dma_allocmp()
109 	mp->dmai_fault_notify = NULL;  in px_dma_allocmp()
111 	mp->dmai_error.err_ena = 0;  in px_dma_allocmp()
112 	mp->dmai_error.err_status = DDI_FM_OK;  in px_dma_allocmp()
113 	mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;  in px_dma_allocmp()
114 	mp->dmai_error.err_ontrap = NULL;  in px_dma_allocmp()
115 	mp->dmai_error.err_fep = NULL;  in px_dma_allocmp()
116 	mp->dmai_error.err_cf = NULL;  in px_dma_allocmp()
126 	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ?  in px_dma_allocmp()
129 	ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);  in px_dma_allocmp()
130 	return (mp);  in px_dma_allocmp()
134 px_dma_freemp(ddi_dma_impl_t *mp)  in px_dma_freemp()  argument
136 	ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);  in px_dma_freemp()
137 	if (mp->dmai_ndvmapages > 1)  in px_dma_freemp()
138 		px_dma_freepfn(mp);  in px_dma_freemp()
139 	if (mp->dmai_winlst)  in px_dma_freemp()
140 		px_dma_freewin(mp);  in px_dma_freemp()
141 	kmem_free(mp, sizeof (px_dma_hdl_t));  in px_dma_freemp()
145 px_dma_freepfn(ddi_dma_impl_t *mp)  in px_dma_freepfn()  argument
147 	void *addr = mp->dmai_pfnlst;  in px_dma_freepfn()
149 		size_t npages = mp->dmai_ndvmapages;  in px_dma_freepfn()
152 		mp->dmai_pfnlst = NULL;  in px_dma_freepfn()
154 	mp->dmai_ndvmapages = 0;  in px_dma_freepfn()
181 	ddi_dma_impl_t *mp;  in px_dma_lmts2hdl()  local
197 	if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp,  in px_dma_lmts2hdl()
202 	attr_p = PX_DEV_ATTR(mp);  in px_dma_lmts2hdl()
209 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;  in px_dma_lmts2hdl()
213 		mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT |  in px_dma_lmts2hdl()
217 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;  in px_dma_lmts2hdl()
220 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;  in px_dma_lmts2hdl()
223 	mp->dmai_burstsizes	= lim_p->dlim_burstsizes;  in px_dma_lmts2hdl()
224 	attr_p = &mp->dmai_attr;  in px_dma_lmts2hdl()
227 	return (mp);  in px_dma_lmts2hdl()
269 px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp)  in px_dma_attr2hdl()  argument
274 	ddi_dma_attr_t *attrp		= PX_DEV_ATTR(mp);  in px_dma_attr2hdl()
299 		mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ;  in px_dma_attr2hdl()
314 			dev_info_t *rdip = mp->dmai_rdip;  in px_dma_attr2hdl()
324 		dev_info_t *rdip = mp->dmai_rdip;  in px_dma_attr2hdl()
342 		if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ||  in px_dma_attr2hdl()
346 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;  in px_dma_attr2hdl()
351 		mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT |  in px_dma_attr2hdl()
356 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;  in px_dma_attr2hdl()
358 	if (PX_DMA_NOCTX(mp->dmai_rdip))  in px_dma_attr2hdl()
359 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;  in px_dma_attr2hdl()
361 	mp->dmai_burstsizes	= attrp->dma_attr_burstsizes;  in px_dma_attr2hdl()
362 	attrp = &mp->dmai_attr;  in px_dma_attr2hdl()
388 px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)  in px_dma_type()  argument
397 	mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;  in px_dma_type()
400 		mp->dmai_rflags |= DMP_NOSYNC;  in px_dma_type()
411 			mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;  in px_dma_type()
423 		mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;  in px_dma_type()
430 		    NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);  in px_dma_type()
440 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32;  in px_dma_type()
444 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64;  in px_dma_type()
447 	if (PX_DMA_ISPEERONLY(mp)) {  in px_dma_type()
448 		dev_info_t *rdip = mp->dmai_rdip;  in px_dma_type()
453 	redzone = (mp->dmai_rflags & DDI_DMA_REDZONE) ||  in px_dma_type()
454 	    (mp->dmai_flags & PX_DMAI_FLAGS_MAP_BUFZONE) ?  in px_dma_type()
457 	mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ?  in px_dma_type()
460 	mp->dmai_object	 = *dobj_p;			/* whole object    */  in px_dma_type()
461 	mp->dmai_pfn0	 = (void *)pfn0;		/* cache pfn0	   */  in px_dma_type()
462 	mp->dmai_roffset = offset;			/* win0 pg0 offset */  in px_dma_type()
463 	mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size);  in px_dma_type()
473 px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)  in px_dma_pgpfn()  argument
478 	switch (mp->dmai_object.dmao_type) {  in px_dma_pgpfn()
481 		page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;  in px_dma_pgpfn()
486 			PX_SET_MP_PFN1(mp, i, pfn);  in px_dma_pgpfn()
494 		page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;  in px_dma_pgpfn()
498 			PX_SET_MP_PFN1(mp, i, pfn);  in px_dma_pgpfn()
519 px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)  in px_dma_vapfn()  argument
523 	caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;  in px_dma_vapfn()
526 	vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE;  in px_dma_vapfn()
531 		PX_SET_MP_PFN1(mp, i, pfn);  in px_dma_vapfn()
533 		    mp, i, pfn);  in px_dma_vapfn()
552 px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)  in px_dma_pfn()  argument
554 	uint32_t npages = mp->dmai_ndvmapages;  in px_dma_pfn()
556 	int i, ret, peer = PX_DMA_ISPTP(mp);  in px_dma_pfn()
557 	int peer32 = PX_DMA_ISPTP32(mp);  in px_dma_pfn()
568 	    mp, PX_MP_PFN0(mp) - pfn_adj);  in px_dma_pfn()
571 		PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);  in px_dma_pfn()
575 	if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t),  in px_dma_pfn()
583 	PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);	/* pfnlst[0] */  in px_dma_pfn()
584 	if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) :  in px_dma_pfn()
585 	    px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS)  in px_dma_pfn()
590 		px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i);  in px_dma_pfn()
593 			    NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn);  in px_dma_pfn()
600 			PX_SET_MP_PFN1(mp, i, pfn - pfn_adj);  in px_dma_pfn()
604 	px_dma_freepfn(mp);  in px_dma_pfn()
632 px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)  in px_dvma_win()  argument
634 	uint32_t redzone_sz	= PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0;  in px_dvma_win()
635 	size_t obj_sz		= mp->dmai_object.dmao_size;  in px_dvma_win()
639 	if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {  in px_dvma_win()
640 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;  in px_dvma_win()
641 		mp->dmai_size = obj_sz;  in px_dvma_win()
642 		mp->dmai_winsize = MMU_PAGE_SIZE;  in px_dvma_win()
643 		mp->dmai_nwin = 1;  in px_dvma_win()
647 	pg_off	= mp->dmai_roffset;  in px_dvma_win()
651 		uint64_t nocross = mp->dmai_attr.dma_attr_seg;  in px_dvma_win()
664 		uint32_t count_max = mp->dmai_attr.dma_attr_count_max;  in px_dvma_win()
669 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;  in px_dvma_win()
670 		mp->dmai_size = xfer_sz;  in px_dvma_win()
671 		mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE);  in px_dvma_win()
672 		mp->dmai_nwin = 1;  in px_dvma_win()
682 	mp->dmai_size = xfer_sz - pg_off;	/* 1st window xferrable size */  in px_dvma_win()
683 	mp->dmai_winsize = xfer_sz;		/* redzone not in winsize */  in px_dvma_win()
684 	mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;  in px_dvma_win()
686 	mp->dmai_winlst = NULL;  in px_dvma_win()
687 	px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp);  in px_dvma_win()
702 px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)  in px_dvma_map_fast()  argument
706 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,  in px_dvma_map_fast()
707 	    mp->dmai_attr.dma_attr_flags);  in px_dvma_map_fast()
711 	size_t npages = MMU_BTOP(mp->dmai_winsize);  in px_dvma_map_fast()
715 	ASSERT(MMU_PTOB(npages) == mp->dmai_winsize);  in px_dvma_map_fast()
716 	ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz);  in px_dvma_map_fast()
738 	    PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, 0,  in px_dvma_map_fast()
745 	if (!PX_MAP_BUFZONE(mp))  in px_dvma_map_fast()
750 	ASSERT(PX_HAS_REDZONE(mp));  in px_dvma_map_fast()
753 	    PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, npages - 1,  in px_dvma_map_fast()
766 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);  in px_dvma_map_fast()
767 	mp->dmai_offset = 0;  in px_dvma_map_fast()
768 	mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK;  in px_dvma_map_fast()
769 	PX_SAVE_MP_TTE(mp, attr);	/* save TTE template for unmapping */  in px_dvma_map_fast()
771 		px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping,  in px_dvma_map_fast()
772 		    mp->dmai_size, mp);  in px_dvma_map_fast()
781 px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p)  in px_dvma_map()  argument
783 	uint_t npages = PX_DMA_WINNPGS(mp);  in px_dvma_map()
786 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,  in px_dvma_map()
787 	    mp->dmai_attr.dma_attr_flags);  in px_dvma_map()
789 	dev_info_t *dip = mp->dmai_rdip;  in px_dvma_map()
798 	if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) {  in px_dvma_map()
801 		mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE;  in px_dvma_map()
807 		    MMU_PTOB(npages + PX_HAS_REDZONE(mp)),  in px_dvma_map()
808 		    MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE),  in px_dvma_map()
810 		    mp->dmai_attr.dma_attr_seg + 1,  in px_dvma_map()
811 		    (void *)mp->dmai_attr.dma_attr_addr_lo,  in px_dvma_map()
812 		    (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),  in px_dvma_map()
825 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);  in px_dvma_map()
826 	mp->dmai_offset = 0;  in px_dvma_map()
827 	PX_SAVE_MP_TTE(mp, attr);	/* mp->dmai_tte = tte */  in px_dvma_map()
830 	    mp, dvma_pg, npages, 0)) != DDI_SUCCESS) {  in px_dvma_map()
831 		if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {  in px_dvma_map()
839 			    MMU_PTOB(npages + PX_HAS_REDZONE(mp)));  in px_dvma_map()
858 px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)  in px_dvma_unmap()  argument
860 	px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping;  in px_dvma_unmap()
864 	if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) {  in px_dvma_unmap()
876 	if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {  in px_dvma_unmap()
883 		size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp);  in px_dvma_unmap()
897 px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,  in px_dvma_ctl()  argument
911 px_dma_freewin(ddi_dma_impl_t *mp)  in px_dma_freewin()  argument
913 	px_dma_win_t *win_p = mp->dmai_winlst, *win2_p;  in px_dma_freewin()
919 	mp->dmai_nwin = 0;  in px_dma_freewin()
920 	mp->dmai_winlst = NULL;  in px_dma_freewin()
954 px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,  in px_dma_newwin()  argument
961 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx);  in px_dma_newwin()
967 	io_attributes_t	attr = PX_GET_TTE_ATTR(mp->dmai_rflags,  in px_dma_newwin()
968 	    mp->dmai_attr.dma_attr_flags);  in px_dma_newwin()
982 		pfn = PX_GET_MP_PFN1(mp, start_idx);  in px_dma_newwin()
998 		DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",  in px_dma_newwin()
1016 	DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",  in px_dma_newwin()
1049 px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p)  in px_dma_adjust()  argument
1052 	size_t pg_offset = mp->dmai_roffset;  in px_dma_adjust()
1058 	DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);  in px_dma_adjust()
1060 	mp->dmai_size = win_p->win_size;  in px_dma_adjust()
1061 	mp->dmai_offset = 0;  in px_dma_adjust()
1063 	pg_offset += mp->dmai_object.dmao_size;  in px_dma_adjust()
1067 	DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);  in px_dma_adjust()
1070 		DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);  in px_dma_adjust()
1079 	ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);  in px_dma_adjust()
1128 px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)  in px_dma_physwin()  argument
1130 	uint_t npages = mp->dmai_ndvmapages;  in px_dma_physwin()
1131 	int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;  in px_dma_physwin()
1133 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0);  in px_dma_physwin()
1136 	px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst;  in px_dma_physwin()
1138 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,  in px_dma_physwin()
1139 	    mp->dmai_attr.dma_attr_flags);  in px_dma_physwin()
1142 	ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp));  in px_dma_physwin()
1143 	if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */  in px_dma_physwin()
1144 		ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp);  in px_dma_physwin()
1147 		px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ?  in px_dma_physwin()
1169 		count_max = mp->dmai_attr.dma_attr_count_max;  in px_dma_physwin()
1170 		pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);  in px_dma_physwin()
1171 		pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);  in px_dma_physwin()
1175 			DBG(DBG_BYPASS, mp->dmai_rdip,  in px_dma_physwin()
1185 		pfn = PX_GET_MP_PFN1(mp, i);  in px_dma_physwin()
1206 		DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",  in px_dma_physwin()
1208 		if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no,  in px_dma_physwin()
1222 	DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",  in px_dma_physwin()
1224 	if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index,  in px_dma_physwin()
1228 	px_dma_adjust(dmareq, mp, mp->dmai_winlst);  in px_dma_physwin()
1229 	mp->dmai_nwin = win_no;  in px_dma_physwin()
1230 	mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC;  in px_dma_physwin()
1231 	mp->dmai_rflags &= ~DDI_DMA_REDZONE;  in px_dma_physwin()
1232 	mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC;  in px_dma_physwin()
1233 	cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1);  in px_dma_physwin()
1234 	mp->dmai_cookie = cookie0_p + 1;  in px_dma_physwin()
1235 	mp->dmai_curcookie = 1;  in px_dma_physwin()
1236 	mp->dmai_ncookies = PX_WINLST(mp)->win_ncookies;  in px_dma_physwin()
1237 	mp->dmai_mapping = cookie0_p->dmac_laddress;  in px_dma_physwin()
1239 	px_dma_freepfn(mp);  in px_dma_physwin()
1242 	px_dma_freewin(mp);  in px_dma_physwin()
1247 px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,  in px_dma_ctl()  argument
1311 	ddi_dma_impl_t *mp)  in px_dvma_alloc_debug()  argument
1326 	ptr->mp = mp;  in px_dvma_alloc_debug()
1333 	ptr->mp = mp;  in px_dvma_alloc_debug()
1344     ddi_dma_impl_t *mp)  in px_dvma_free_debug()  argument
1359 	ptr->mp = mp;  in px_dvma_free_debug()