60 extern void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset);
194 	ldc_chan_t 	*ldcp;  in ldc_mem_alloc_handle()  local
202 	ldcp = (ldc_chan_t *)handle;  in ldc_mem_alloc_handle()
204 	mutex_enter(&ldcp->lock);  in ldc_mem_alloc_handle()
207 	if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) {  in ldc_mem_alloc_handle()
208 		DWARN(ldcp->id,  in ldc_mem_alloc_handle()
210 		    ldcp->id);  in ldc_mem_alloc_handle()
211 		mutex_exit(&ldcp->lock);  in ldc_mem_alloc_handle()
223 	mhdl->ldcp = ldcp;  in ldc_mem_alloc_handle()
227 	if (ldcp->mhdl_list == NULL) {  in ldc_mem_alloc_handle()
228 		ldcp->mhdl_list = mhdl;  in ldc_mem_alloc_handle()
232 		mhdl->next = ldcp->mhdl_list;  in ldc_mem_alloc_handle()
233 		ldcp->mhdl_list = mhdl;  in ldc_mem_alloc_handle()
239 	mutex_exit(&ldcp->lock);  in ldc_mem_alloc_handle()
241 	D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n",  in ldc_mem_alloc_handle()
242 	    ldcp->id, mhdl);  in ldc_mem_alloc_handle()
254 	ldc_chan_t 	*ldcp;  in ldc_mem_free_handle()  local
265 	ldcp = mhdl->ldcp;  in ldc_mem_free_handle()
268 		DWARN(ldcp->id,  in ldc_mem_free_handle()
276 	mutex_enter(&ldcp->mlist_lock);  in ldc_mem_free_handle()
278 	phdl = ldcp->mhdl_list;  in ldc_mem_free_handle()
282 		ldcp->mhdl_list = mhdl->next;  in ldc_mem_free_handle()
286 		D1(ldcp->id,  in ldc_mem_free_handle()
288 		    ldcp->id, mhdl);  in ldc_mem_free_handle()
296 				D1(ldcp->id,  in ldc_mem_free_handle()
298 				    "handle 0x%llx\n", ldcp->id, mhdl);  in ldc_mem_free_handle()
306 		DWARN(ldcp->id,  in ldc_mem_free_handle()
308 		mutex_exit(&ldcp->mlist_lock);  in ldc_mem_free_handle()
312 	mutex_exit(&ldcp->mlist_lock);  in ldc_mem_free_handle()
349 	ldc_chan_t 	*ldcp;  in i_ldc_mem_bind_handle()  local
368 	ldcp = mhdl->ldcp;  in i_ldc_mem_bind_handle()
376 		DWARN(ldcp->id,  in i_ldc_mem_bind_handle()
385 		DWARN(ldcp->id,  in i_ldc_mem_bind_handle()
391 	mutex_enter(&ldcp->lock);  in i_ldc_mem_bind_handle()
397 	if ((mtbl = ldcp->mtbl) == NULL) {  in i_ldc_mem_bind_handle()
417 			DWARN(ldcp->id,  in i_ldc_mem_bind_handle()
419 			    "to %lx entries\n", ldcp->id, mtbl->num_entries);  in i_ldc_mem_bind_handle()
429 		rv = hv_ldc_set_map_table(ldcp->id,  in i_ldc_mem_bind_handle()
434 			    ldcp->id, rv);  in i_ldc_mem_bind_handle()
441 			mutex_exit(&ldcp->lock);  in i_ldc_mem_bind_handle()
446 		ldcp->mtbl = mtbl;  in i_ldc_mem_bind_handle()
448 		D1(ldcp->id,  in i_ldc_mem_bind_handle()
450 		    ldcp->id, ldcp->mtbl->table);  in i_ldc_mem_bind_handle()
453 	mutex_exit(&ldcp->lock);  in i_ldc_mem_bind_handle()
461 	D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding "  in i_ldc_mem_bind_handle()
463 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);  in i_ldc_mem_bind_handle()
472 	D1(ldcp->id, "ldc_mem_bind_handle: binding "  in i_ldc_mem_bind_handle()
474 	    ldcp->id, vaddr, v_align, v_offset, npages);  in i_ldc_mem_bind_handle()
480 		D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n",  in i_ldc_mem_bind_handle()
481 		    ldcp->id);  in i_ldc_mem_bind_handle()
496 	D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n",  in i_ldc_mem_bind_handle()
497 	    ldcp->id, npages);  in i_ldc_mem_bind_handle()
539 	D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);  in i_ldc_mem_bind_handle()
555 				    "entry\n", ldcp->id);  in i_ldc_mem_bind_handle()
574 		D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);  in i_ldc_mem_bind_handle()
579 		D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx"  in i_ldc_mem_bind_handle()
580 		    " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index);  in i_ldc_mem_bind_handle()
619 		D1(ldcp->id, "ldc_mem_bind_handle: bound "  in i_ldc_mem_bind_handle()
622 		    ldcp->id, addr, index, raddr, psize, poffset);  in i_ldc_mem_bind_handle()
658 	D1(ldcp->id,  in i_ldc_mem_bind_handle()
661 	    ldcp->id, mhdl, vaddr, npages, memseg->ncookies);  in i_ldc_mem_bind_handle()
674 	ldc_chan_t 	*ldcp;  in ldc_mem_nextcookie()  local
686 	ldcp = mhdl->ldcp;  in ldc_mem_nextcookie()
690 		DWARN(ldcp->id,  in ldc_mem_nextcookie()
692 		    ldcp->id);  in ldc_mem_nextcookie()
705 		DWARN(ldcp->id,  in ldc_mem_nextcookie()
706 		    "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id);  in ldc_mem_nextcookie()
713 	D1(ldcp->id,  in ldc_mem_nextcookie()
715 	    ldcp->id, cookie->addr, cookie->size);  in ldc_mem_nextcookie()
730 	ldc_chan_t 	*ldcp;  in ldc_mem_unbind_handle()  local
754 	ldcp = mhdl->ldcp;  in ldc_mem_unbind_handle()
755 	mtbl = ldcp->mtbl;  in ldc_mem_unbind_handle()
776 			D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke "  in ldc_mem_unbind_handle()
777 			    "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id,  in ldc_mem_unbind_handle()
782 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,  in ldc_mem_unbind_handle()
793 				DWARN(ldcp->id,  in ldc_mem_unbind_handle()
795 				    "revoke mapping, cookie %llx\n", ldcp->id,  in ldc_mem_unbind_handle()
814 	D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n",  in ldc_mem_unbind_handle()
815 	    ldcp->id, mhdl);  in ldc_mem_unbind_handle()
865 	ldc_chan_t 	*ldcp;  in ldc_mem_copy()  local
882 	ldcp = (ldc_chan_t *)handle;  in ldc_mem_copy()
883 	chid = ldcp->id;  in ldc_mem_copy()
886 	if (ldcp->tstate != TS_UP) {  in ldc_mem_copy()
1009 			mutex_enter(&ldcp->lock);  in ldc_mem_copy()
1010 			mutex_enter(&ldcp->tx_lock);  in ldc_mem_copy()
1012 			rv = hv_ldc_rx_get_state(ldcp->id,  in ldc_mem_copy()
1013 			    &rx_hd, &rx_tl, &(ldcp->link_state));  in ldc_mem_copy()
1014 			if (ldcp->link_state == LDC_CHANNEL_DOWN ||  in ldc_mem_copy()
1015 			    ldcp->link_state == LDC_CHANNEL_RESET) {  in ldc_mem_copy()
1016 				i_ldc_reset(ldcp, B_FALSE);  in ldc_mem_copy()
1020 			mutex_exit(&ldcp->tx_lock);  in ldc_mem_copy()
1021 			mutex_exit(&ldcp->lock);  in ldc_mem_copy()
1091 	ldc_chan_t 	*ldcp;  in ldc_mem_rdwr_cookie()  local
1106 	ldcp = (ldc_chan_t *)handle;  in ldc_mem_rdwr_cookie()
1108 	mutex_enter(&ldcp->lock);  in ldc_mem_rdwr_cookie()
1111 	if (ldcp->tstate != TS_UP) {  in ldc_mem_rdwr_cookie()
1112 		DWARN(ldcp->id,  in ldc_mem_rdwr_cookie()
1114 		    ldcp->id);  in ldc_mem_rdwr_cookie()
1115 		mutex_exit(&ldcp->lock);  in ldc_mem_rdwr_cookie()
1121 		DWARN(ldcp->id,  in ldc_mem_rdwr_cookie()
1123 		mutex_exit(&ldcp->lock);  in ldc_mem_rdwr_cookie()
1134 	D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data "  in ldc_mem_rdwr_cookie()
1136 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);  in ldc_mem_rdwr_cookie()
1145 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, "  in ldc_mem_rdwr_cookie()
1147 	    ldcp->id, vaddr, local_valign, local_voff, npages);  in ldc_mem_rdwr_cookie()
1163 		D1(ldcp->id,  in ldc_mem_rdwr_cookie()
1168 		    ldcp->id, direction, target_pa, local_ra, target_poff,  in ldc_mem_rdwr_cookie()
1172 		rv = hv_ldc_copy(ldcp->id, direction,  in ldc_mem_rdwr_cookie()
1179 			    ldcp->id, rv);  in ldc_mem_rdwr_cookie()
1185 			    ldcp->id, direction, target_pa, local_ra,  in ldc_mem_rdwr_cookie()
1190 			mutex_exit(&ldcp->lock);  in ldc_mem_rdwr_cookie()
1194 		D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n",  in ldc_mem_rdwr_cookie()
1225 	mutex_exit(&ldcp->lock);  in ldc_mem_rdwr_cookie()
1227 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n",  in ldc_mem_rdwr_cookie()
1228 	    ldcp->id, *size);  in ldc_mem_rdwr_cookie()
1262 	ldc_chan_t 	*ldcp;  in i_ldc_mem_map()  local
1288 	ldcp = mhdl->ldcp;  in i_ldc_mem_map()
1290 	mutex_enter(&ldcp->lock);  in i_ldc_mem_map()
1292 	if (ldcp->tstate != TS_UP) {  in i_ldc_mem_map()
1293 		DWARN(ldcp->id,  in i_ldc_mem_map()
1295 		    ldcp->id);  in i_ldc_mem_map()
1296 		mutex_exit(&ldcp->lock);  in i_ldc_mem_map()
1302 		DWARN(ldcp->id, "ldc_mem_map: invalid map type\n");  in i_ldc_mem_map()
1303 		mutex_exit(&ldcp->lock);  in i_ldc_mem_map()
1308 	D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n",  in i_ldc_mem_map()
1309 	    ldcp->id, cookie->addr, cookie->size);  in i_ldc_mem_map()
1333 	D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx,"  in i_ldc_mem_map()
1334 	    "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages);  in i_ldc_mem_map()
1349 			    ldcp->id);  in i_ldc_mem_map()
1356 			mutex_exit(&ldcp->lock);  in i_ldc_mem_map()
1382 			D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping "  in i_ldc_mem_map()
1383 			    "cookie 0x%llx, bal=0x%llx\n", ldcp->id,  in i_ldc_mem_map()
1390 				rv = hv_ldc_mapin(ldcp->id, cookie_addr,  in i_ldc_mem_map()
1399 				DWARN(ldcp->id,  in i_ldc_mem_map()
1401 				    ldcp->id, rv);  in i_ldc_mem_map()
1410 						DWARN(ldcp->id,  in i_ldc_mem_map()
1413 						    ldcp->id,  in i_ldc_mem_map()
1428 				D1(ldcp->id,  in i_ldc_mem_map()
1431 				    ldcp->id, tmpaddr, memseg->pages[i].raddr,  in i_ldc_mem_map()
1444 				D1(ldcp->id,  in i_ldc_mem_map()
1446 				    "tte 0x%llx\n", ldcp->id,  in i_ldc_mem_map()
1464 			D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated "  in i_ldc_mem_map()
1465 			    "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr);  in i_ldc_mem_map()
1497 	D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, "  in i_ldc_mem_map()
1499 	    ldcp->id, mhdl, memseg->raddr, memseg->vaddr,  in i_ldc_mem_map()
1509 	mutex_exit(&ldcp->lock);  in i_ldc_mem_map()
1522 	ldc_chan_t 	*ldcp;  in ldc_mem_unmap()  local
1534 	ldcp = mhdl->ldcp;  in ldc_mem_unmap()
1537 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n",  in ldc_mem_unmap()
1538 	    ldcp->id, mhdl);  in ldc_mem_unmap()
1554 				    ldcp->id, rv);  in ldc_mem_unmap()
1571 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n",  in ldc_mem_unmap()
1572 	    ldcp->id, mhdl);  in ldc_mem_unmap()
1591 	ldc_chan_t	*ldcp;  in i_ldc_mem_acquire_release()  local
1605 	if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) {  in i_ldc_mem_acquire_release()
1634 	ldcp = mhdl->ldcp;  in i_ldc_mem_acquire_release()
1643 		err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset,  in i_ldc_mem_acquire_release()
1647 			DWARN(ldcp->id,  in i_ldc_mem_acquire_release()
1825 	ldc_chan_t 	*ldcp;  in ldc_mem_dring_bind()  local
1835 	ldcp = (ldc_chan_t *)handle;  in ldc_mem_dring_bind()
1845 		DWARN(ldcp->id,  in ldc_mem_dring_bind()
1852 		DWARN(ldcp->id, "ldc_mem_dring_bind: invalid map type\n");  in ldc_mem_dring_bind()
1866 		    ldcp->id);  in ldc_mem_dring_bind()
1884 	dringp->ldcp = ldcp;  in ldc_mem_dring_bind()
1891 		    ldcp->id);  in ldc_mem_dring_bind()
1901 		DWARN(ldcp->id,  in ldc_mem_dring_bind()
1903 		    ldcp->id);  in ldc_mem_dring_bind()
1916 		dringp->ldcp = NULL;  in ldc_mem_dring_bind()
1925 	mutex_enter(&ldcp->exp_dlist_lock);  in ldc_mem_dring_bind()
1926 	dringp->ch_next = ldcp->exp_dring_list;  in ldc_mem_dring_bind()
1927 	ldcp->exp_dring_list = dringp;  in ldc_mem_dring_bind()
1928 	mutex_exit(&ldcp->exp_dlist_lock);  in ldc_mem_dring_bind()
1945 	ldc_chan_t	*ldcp;  in ldc_mem_dring_nextcookie()  local
1963 	ldcp = dringp->ldcp;  in ldc_mem_dring_nextcookie()
1966 		DWARN(ldcp->id,  in ldc_mem_dring_nextcookie()
1968 		    ldcp->id);  in ldc_mem_dring_nextcookie()
1987 	ldc_chan_t	*ldcp;  in ldc_mem_dring_unbind()  local
2005 	ldcp = dringp->ldcp;  in ldc_mem_dring_unbind()
2007 	mutex_enter(&ldcp->exp_dlist_lock);  in ldc_mem_dring_unbind()
2009 	tmp_dringp = ldcp->exp_dring_list;  in ldc_mem_dring_unbind()
2011 		ldcp->exp_dring_list = dringp->ch_next;  in ldc_mem_dring_unbind()
2026 			mutex_exit(&ldcp->exp_dlist_lock);  in ldc_mem_dring_unbind()
2032 	mutex_exit(&ldcp->exp_dlist_lock);  in ldc_mem_dring_unbind()
2037 	dringp->ldcp = NULL;  in ldc_mem_dring_unbind()
2048 i_ldc_mem_inject_dring_clear(ldc_chan_t *ldcp)  in i_ldc_mem_inject_dring_clear()  argument
2059 	if ((mtbl = ldcp->mtbl) == NULL)  in i_ldc_mem_inject_dring_clear()
2066 	mutex_enter(&ldcp->exp_dlist_lock);  in i_ldc_mem_inject_dring_clear()
2068 	for (dp = ldcp->exp_dring_list; dp != NULL; dp = dp->ch_next) {  in i_ldc_mem_inject_dring_clear()
2088 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,  in i_ldc_mem_inject_dring_clear()
2099 				DWARN(ldcp->id,  in i_ldc_mem_inject_dring_clear()
2104 				    ldcp->id, cookie_addr,  in i_ldc_mem_inject_dring_clear()
2112 	mutex_exit(&ldcp->exp_dlist_lock);  in i_ldc_mem_inject_dring_clear()
2167 	ldc_chan_t 	*ldcp = (ldc_chan_t *)handle;  in ldc_mem_dring_map()  local
2184 	ldcp = (ldc_chan_t *)handle;  in ldc_mem_dring_map()
2187 		DWARN(ldcp->id,  in ldc_mem_dring_map()
2189 		    ldcp->id);  in ldc_mem_dring_map()
2197 		DWARN(ldcp->id,  in ldc_mem_dring_map()
2199 		    ldcp->id);  in ldc_mem_dring_map()
2205 		DWARN(ldcp->id, "ldc_mem_dring_map: invalid map type\n");  in ldc_mem_dring_map()
2219 	D1(ldcp->id,  in ldc_mem_dring_map()
2233 	dringp->ldcp = ldcp;  in ldc_mem_dring_map()
2263 	mutex_enter(&ldcp->imp_dlist_lock);  in ldc_mem_dring_map()
2264 	dringp->ch_next = ldcp->imp_dring_list;  in ldc_mem_dring_map()
2265 	ldcp->imp_dring_list = dringp;  in ldc_mem_dring_map()
2266 	mutex_exit(&ldcp->imp_dlist_lock);  in ldc_mem_dring_map()
2283 	ldc_chan_t	*ldcp;  in ldc_mem_dring_unmap()  local
2300 	ldcp = dringp->ldcp;  in ldc_mem_dring_unmap()
2302 	mutex_enter(&ldcp->imp_dlist_lock);  in ldc_mem_dring_unmap()
2305 	tmp_dringp = ldcp->imp_dring_list;  in ldc_mem_dring_unmap()
2307 		ldcp->imp_dring_list = dringp->ch_next;  in ldc_mem_dring_unmap()
2322 			mutex_exit(&ldcp->imp_dlist_lock);  in ldc_mem_dring_unmap()
2328 	mutex_exit(&ldcp->imp_dlist_lock);  in ldc_mem_dring_unmap()
2335 	dringp->ldcp = NULL;  in ldc_mem_dring_unmap()
2361 	ldc_chan_t		*ldcp;  in i_ldc_dring_acquire_release()  local
2374 	if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) {  in i_ldc_dring_acquire_release()
2405 	ldcp = dringp->ldcp;  in i_ldc_dring_acquire_release()
2414 	D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n",  in i_ldc_dring_acquire_release()
2419 		DWARN(ldcp->id,  in i_ldc_dring_acquire_release()
2431 		D1(ldcp->id, "i_ldc_dring_acquire_release: c2 "  in i_ldc_dring_acquire_release()
2436 			DWARN(ldcp->id,  in i_ldc_dring_acquire_release()