61 int vgen_create_rx_dring(vgen_ldc_t *ldcp);
62 void vgen_destroy_rx_dring(vgen_ldc_t *ldcp);
63 int vgen_map_tx_dring(vgen_ldc_t *ldcp, void *pkt);
64 void vgen_unmap_tx_dring(vgen_ldc_t *ldcp);
65 int vgen_map_data(vgen_ldc_t *ldcp, void *pkt);
68 mblk_t *vgen_poll_rcv_shm(vgen_ldc_t *ldcp, int bytes_to_pickup);
69 int vgen_send_dringack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp,
73 static int vgen_handle_dringdata_info_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tp);
74 static int vgen_handle_dringdata_ack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp);
75 static int vgen_handle_dringdata_nack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tp);
76 static int vgen_intr_rcv_shm(vgen_ldc_t *ldcp);
77 static int vgen_receive_packet(vgen_ldc_t *ldcp, mblk_t **bp, uint_t *size);
78 static int vgen_send_dringdata_shm(vgen_ldc_t *ldcp, uint32_t start,
80 static int vgen_sendmsg_shm(vgen_ldc_t *ldcp, caddr_t msg,  size_t msglen);
83 extern int vgen_handle_evt_read(vgen_ldc_t *ldcp, vgen_caller_t caller);
84 extern int vgen_handle_evt_reset(vgen_ldc_t *ldcp, vgen_caller_t caller);
104 	vgen_ldc_t *ldcp, const char *fmt, ...);
105 extern boolean_t vgen_inject_error(vgen_ldc_t *ldcp, int error);
114 vgen_create_rx_dring(vgen_ldc_t *ldcp)  in vgen_create_rx_dring()  argument
126 	vgen_t				*vgenp = LDC_TO_VGEN(ldcp);  in vgen_create_rx_dring()
129 	ldcp->num_rxds = vnet_num_descriptors;  in vgen_create_rx_dring()
130 	ldcp->num_rbufs = VGEN_RXDRING_NRBUFS;  in vgen_create_rx_dring()
133 	rv = ldc_mem_dring_create(ldcp->num_rxds, rxdsize,  in vgen_create_rx_dring()
134 	    &ldcp->rx_dring_handle);  in vgen_create_rx_dring()
136 		DWARN(vgenp, ldcp, "ldc_mem_dring_create() failed\n");  in vgen_create_rx_dring()
141 	rv = ldc_mem_dring_info(ldcp->rx_dring_handle, &minfo);  in vgen_create_rx_dring()
143 		DWARN(vgenp, ldcp, "ldc_mem_dring_info() failed\n");  in vgen_create_rx_dring()
146 	ldcp->rxdp = (vnet_rx_dringdata_desc_t *)(minfo.vaddr);  in vgen_create_rx_dring()
147 	bzero(ldcp->rxdp, sizeof (*rxdp) * (ldcp->num_rxds));  in vgen_create_rx_dring()
154 	rxdp_to_vmp = kmem_zalloc(ldcp->num_rxds * sizeof (uintptr_t),  in vgen_create_rx_dring()
156 	ldcp->rxdp_to_vmp = rxdp_to_vmp;  in vgen_create_rx_dring()
178 	ldcp->rx_data_sz = data_sz * ldcp->num_rbufs;  in vgen_create_rx_dring()
179 	ldcp->rx_dblk_sz = data_sz;  in vgen_create_rx_dring()
180 	datap = kmem_zalloc(ldcp->rx_data_sz, KM_SLEEP);  in vgen_create_rx_dring()
181 	ldcp->rx_datap = datap;  in vgen_create_rx_dring()
184 	rv = ldc_mem_alloc_handle(ldcp->ldc_handle, &ldcp->rx_data_handle);  in vgen_create_rx_dring()
186 		ldcp->rx_data_handle = 0;  in vgen_create_rx_dring()
191 	ldcp->rx_data_cookie = kmem_zalloc(VNET_DATA_AREA_COOKIES *  in vgen_create_rx_dring()
198 	rv = ldc_mem_bind_handle(ldcp->rx_data_handle, (caddr_t)datap,  in vgen_create_rx_dring()
199 	    ldcp->rx_data_sz, LDC_DIRECT_MAP, LDC_MEM_W,  in vgen_create_rx_dring()
200 	    ldcp->rx_data_cookie, &ncookies);  in vgen_create_rx_dring()
207 	ldcp->rx_data_ncookies = ncookies;  in vgen_create_rx_dring()
210 		rv = ldc_mem_nextcookie(ldcp->rx_data_handle,  in vgen_create_rx_dring()
211 		    &(ldcp->rx_data_cookie[j]));  in vgen_create_rx_dring()
213 			DERR(vgenp, ldcp, "ldc_mem_nextcookie "  in vgen_create_rx_dring()
228 	rv = vio_create_mblks(ldcp->num_rbufs, data_sz, (uint8_t *)datap,  in vgen_create_rx_dring()
229 	    &ldcp->rx_vmp);  in vgen_create_rx_dring()
234 	for (i = 0; i < ldcp->num_rxds; i++) {  in vgen_create_rx_dring()
235 		rxdp = &(ldcp->rxdp[i]);  in vgen_create_rx_dring()
237 		vmp = vio_allocb(ldcp->rx_vmp);  in vgen_create_rx_dring()
248 	rv = ldc_mem_dring_bind(ldcp->ldc_handle, ldcp->rx_dring_handle,  in vgen_create_rx_dring()
250 	    &ldcp->rx_dring_cookie, &ncookies);  in vgen_create_rx_dring()
252 		DWARN(vgenp, ldcp, "ldc_mem_dring_bind failed "  in vgen_create_rx_dring()
257 	ldcp->rx_dring_ncookies = ncookies;  in vgen_create_rx_dring()
260 	ldcp->next_rxseq = VNET_ISS;  in vgen_create_rx_dring()
261 	ldcp->next_rxi = 0;  in vgen_create_rx_dring()
266 	vgen_destroy_rx_dring(ldcp);  in vgen_create_rx_dring()
274 vgen_destroy_rx_dring(vgen_ldc_t *ldcp)  in vgen_destroy_rx_dring()  argument
276 	vgen_t	*vgenp = LDC_TO_VGEN(ldcp);  in vgen_destroy_rx_dring()
279 	if (ldcp->rx_dring_ncookies != 0) {  in vgen_destroy_rx_dring()
280 		(void) ldc_mem_dring_unbind(ldcp->rx_dring_handle);  in vgen_destroy_rx_dring()
281 		ldcp->rx_dring_ncookies = 0;  in vgen_destroy_rx_dring()
285 	if (ldcp->rx_vmp != NULL) {  in vgen_destroy_rx_dring()
286 		vio_clobber_pool(ldcp->rx_vmp);  in vgen_destroy_rx_dring()
287 		if (vio_destroy_mblks(ldcp->rx_vmp) != 0) {  in vgen_destroy_rx_dring()
297 			    vgen_destroy_rxpools, ldcp->rx_vmp, DDI_SLEEP);  in vgen_destroy_rx_dring()
299 		ldcp->rx_vmp = NULL;  in vgen_destroy_rx_dring()
303 	if (ldcp->rx_data_cookie != NULL) {  in vgen_destroy_rx_dring()
304 		kmem_free(ldcp->rx_data_cookie, VNET_DATA_AREA_COOKIES *  in vgen_destroy_rx_dring()
306 		ldcp->rx_data_cookie = NULL;  in vgen_destroy_rx_dring()
310 	if (ldcp->rx_data_ncookies != 0) {  in vgen_destroy_rx_dring()
311 		(void) ldc_mem_unbind_handle(ldcp->rx_data_handle);  in vgen_destroy_rx_dring()
312 		ldcp->rx_data_ncookies = 0;  in vgen_destroy_rx_dring()
316 	if (ldcp->rx_data_handle != 0) {  in vgen_destroy_rx_dring()
317 		(void) ldc_mem_free_handle(ldcp->rx_data_handle);  in vgen_destroy_rx_dring()
318 		ldcp->rx_data_handle = 0;  in vgen_destroy_rx_dring()
322 	if (ldcp->rx_datap != NULL) {  in vgen_destroy_rx_dring()
324 		kmem_free(ldcp->rx_datap, ldcp->rx_data_sz);  in vgen_destroy_rx_dring()
325 		ldcp->rx_datap = NULL;  in vgen_destroy_rx_dring()
326 		ldcp->rx_data_sz = 0;  in vgen_destroy_rx_dring()
330 	if (ldcp->rx_dring_handle != 0) {  in vgen_destroy_rx_dring()
331 		(void) ldc_mem_dring_destroy(ldcp->rx_dring_handle);  in vgen_destroy_rx_dring()
332 		ldcp->rx_dring_handle = 0;  in vgen_destroy_rx_dring()
333 		ldcp->rxdp = NULL;  in vgen_destroy_rx_dring()
336 	if (ldcp->rxdp_to_vmp != NULL) {  in vgen_destroy_rx_dring()
337 		kmem_free(ldcp->rxdp_to_vmp,  in vgen_destroy_rx_dring()
338 		    ldcp->num_rxds * sizeof (uintptr_t));  in vgen_destroy_rx_dring()
339 		ldcp->rxdp_to_vmp = NULL;  in vgen_destroy_rx_dring()
343 	ldcp->next_rxi = 0;  in vgen_destroy_rx_dring()
344 	ldcp->next_rxseq = VNET_ISS;  in vgen_destroy_rx_dring()
352 vgen_map_tx_dring(vgen_ldc_t *ldcp, void *pkt)  in vgen_map_tx_dring()  argument
381 	rv = ldc_mem_dring_map(ldcp->ldc_handle, &dcookie, ncookies, num_desc,  in vgen_map_tx_dring()
382 	    desc_size, LDC_DIRECT_MAP, &(ldcp->tx_dring_handle));  in vgen_map_tx_dring()
390 	rv = ldc_mem_dring_info(ldcp->tx_dring_handle, &minfo);  in vgen_map_tx_dring()
398 	ldcp->mtxdp = (vnet_rx_dringdata_desc_t *)(minfo.vaddr);  in vgen_map_tx_dring()
399 	bcopy(&dcookie, &(ldcp->tx_dring_cookie), sizeof (dcookie));  in vgen_map_tx_dring()
400 	ldcp->tx_dring_ncookies = ncookies;  in vgen_map_tx_dring()
401 	ldcp->num_txds = num_desc;  in vgen_map_tx_dring()
404 	ldcp->next_txi = ldcp->cur_txi = ldcp->resched_peer_txi = 0;  in vgen_map_tx_dring()
405 	ldcp->next_txseq = VNET_ISS - 1;  in vgen_map_tx_dring()
406 	ldcp->resched_peer = B_TRUE;  in vgen_map_tx_dring()
407 	ldcp->dring_mtype = minfo.mtype;  in vgen_map_tx_dring()
408 	ldcp->dringdata_msgid = 0;  in vgen_map_tx_dring()
411 	bcopy(&dcookie, &(ldcp->peer_hparams.dring_cookie),  in vgen_map_tx_dring()
413 	ldcp->peer_hparams.num_desc = num_desc;  in vgen_map_tx_dring()
414 	ldcp->peer_hparams.desc_size = desc_size;  in vgen_map_tx_dring()
415 	ldcp->peer_hparams.dring_ncookies = ncookies;  in vgen_map_tx_dring()
418 	ldcp->peer_hparams.dring_ident = (uint64_t)ldcp->mtxdp;  in vgen_map_tx_dring()
421 	msg->dring_ident = (uint64_t)ldcp->mtxdp;  in vgen_map_tx_dring()
445 		txdp = &ldcp->mtxdp[i];  in vgen_map_tx_dring()
453 	if (ldcp->tx_dring_handle != 0) {  in vgen_map_tx_dring()
454 		(void) ldc_mem_dring_unmap(ldcp->tx_dring_handle);  in vgen_map_tx_dring()
455 		ldcp->tx_dring_handle = 0;  in vgen_map_tx_dring()
464 vgen_unmap_tx_dring(vgen_ldc_t *ldcp)  in vgen_unmap_tx_dring()  argument
467 	if (ldcp->tx_datap != NULL) {  in vgen_unmap_tx_dring()
468 		(void) ldc_mem_unmap(ldcp->tx_data_handle);  in vgen_unmap_tx_dring()
469 		ldcp->tx_datap = NULL;  in vgen_unmap_tx_dring()
473 	if (ldcp->tx_data_handle != 0) {  in vgen_unmap_tx_dring()
474 		(void) ldc_mem_free_handle(ldcp->tx_data_handle);  in vgen_unmap_tx_dring()
475 		ldcp->tx_data_handle = 0;  in vgen_unmap_tx_dring()
479 	if (ldcp->tx_data_cookie != NULL) {  in vgen_unmap_tx_dring()
480 		kmem_free(ldcp->tx_data_cookie, ldcp->tx_data_ncookies *  in vgen_unmap_tx_dring()
482 		ldcp->tx_data_cookie = NULL;  in vgen_unmap_tx_dring()
483 		ldcp->tx_data_ncookies = 0;  in vgen_unmap_tx_dring()
487 	if (ldcp->tx_dring_handle != 0) {  in vgen_unmap_tx_dring()
488 		(void) ldc_mem_dring_unmap(ldcp->tx_dring_handle);  in vgen_unmap_tx_dring()
489 		ldcp->tx_dring_handle = 0;  in vgen_unmap_tx_dring()
493 	bzero(&ldcp->tx_dring_cookie, sizeof (ldcp->tx_dring_cookie));  in vgen_unmap_tx_dring()
494 	ldcp->mtxdp = NULL;  in vgen_unmap_tx_dring()
495 	ldcp->next_txi = ldcp->cur_txi = ldcp->resched_peer_txi = 0;  in vgen_unmap_tx_dring()
496 	ldcp->num_txds = 0;  in vgen_unmap_tx_dring()
497 	ldcp->next_txseq = VNET_ISS - 1;  in vgen_unmap_tx_dring()
498 	ldcp->resched_peer = B_TRUE;  in vgen_unmap_tx_dring()
505 vgen_map_data(vgen_ldc_t *ldcp, void *pkt)  in vgen_map_data()  argument
511 	vgen_t			*vgenp = LDC_TO_VGEN(ldcp);  in vgen_map_data()
524 	ldcp->tx_data_ncookies = emsg->data_ncookies;  in vgen_map_data()
527 	ldcp->tx_data_sz = emsg->data_area_size;  in vgen_map_data()
530 	rv = ldc_mem_alloc_handle(ldcp->ldc_handle, &ldcp->tx_data_handle);  in vgen_map_data()
532 		DWARN(vgenp, ldcp, "ldc_mem_alloc_handle() failed: %d\n", rv);  in vgen_map_data()
537 	rv = ldc_mem_map(ldcp->tx_data_handle, emsg->data_cookie,  in vgen_map_data()
539 	    (caddr_t *)&ldcp->tx_datap, NULL);  in vgen_map_data()
541 		DWARN(vgenp, ldcp, "ldc_mem_map() failed: %d\n", rv);  in vgen_map_data()
546 	rv = ldc_mem_info(ldcp->tx_data_handle, &minfo);  in vgen_map_data()
548 		DWARN(vgenp, ldcp, "ldc_mem_info() failed: %d\n", rv);  in vgen_map_data()
553 		DWARN(vgenp, ldcp, "mtype(%d) is not direct map\n",  in vgen_map_data()
559 	ldcp->tx_data_cookie = kmem_zalloc(emsg->data_ncookies *  in vgen_map_data()
563 	bcopy(emsg->data_cookie, ldcp->tx_data_cookie,  in vgen_map_data()
591 	vgen_ldc_t			*ldcp = (vgen_ldc_t *)arg;  in vgen_dringsend_shm()  local
592 	vgen_t				*vgenp = LDC_TO_VGEN(ldcp);  in vgen_dringsend_shm()
593 	vgen_stats_t			*statsp = &ldcp->stats;  in vgen_dringsend_shm()
594 	vgen_hparams_t			*lp = &ldcp->local_hparams;  in vgen_dringsend_shm()
599 	if (ldcp->ldc_status != LDC_UP) {  in vgen_dringsend_shm()
600 		DBG2(vgenp, ldcp, "status(%d), dropping packet\n",  in vgen_dringsend_shm()
601 		    ldcp->ldc_status);  in vgen_dringsend_shm()
605 	if (ldcp->hphase != VH_DONE) {  in vgen_dringsend_shm()
606 		DWARN(vgenp, ldcp, "hphase(%x), dropping packet\n",  in vgen_dringsend_shm()
607 		    ldcp->hphase);  in vgen_dringsend_shm()
613 		DWARN(vgenp, ldcp, "invalid size(%d)\n", size);  in vgen_dringsend_shm()
640 		DERR(vgenp, ldcp, "data access fault occured\n");  in vgen_dringsend_shm()
642 		if (mutex_owned(&ldcp->txlock)) {  in vgen_dringsend_shm()
643 			mutex_exit(&ldcp->txlock);  in vgen_dringsend_shm()
645 		if (mutex_owned(&ldcp->wrlock)) {  in vgen_dringsend_shm()
646 			mutex_exit(&ldcp->wrlock);  in vgen_dringsend_shm()
654 	mutex_enter(&ldcp->txlock);  in vgen_dringsend_shm()
655 	txi = next_txi = ldcp->next_txi;  in vgen_dringsend_shm()
656 	INCR_TXI(next_txi, ldcp);  in vgen_dringsend_shm()
657 	txdp = &(ldcp->mtxdp[txi]);  in vgen_dringsend_shm()
659 		if (ldcp->tx_blocked == B_FALSE) {  in vgen_dringsend_shm()
660 			ldcp->tx_blocked_lbolt = ddi_get_lbolt();  in vgen_dringsend_shm()
661 			ldcp->tx_blocked = B_TRUE;  in vgen_dringsend_shm()
664 		mutex_exit(&ldcp->txlock);  in vgen_dringsend_shm()
671 	if (ldcp->tx_blocked == B_TRUE) {  in vgen_dringsend_shm()
672 		ldcp->tx_blocked = B_FALSE;  in vgen_dringsend_shm()
677 	ldcp->next_txi = next_txi;  in vgen_dringsend_shm()
678 	mutex_exit(&ldcp->txlock);  in vgen_dringsend_shm()
682 		    ldcp->portp->vcb.vio_net_tx_update;  in vgen_dringsend_shm()
684 		vtx_update(ldcp->portp->vhp);  in vgen_dringsend_shm()
694 	dst = (caddr_t)ldcp->tx_datap + buf_offset;  in vgen_dringsend_shm()
712 	mutex_enter(&ldcp->wrlock);  in vgen_dringsend_shm()
720 	if (ldcp->resched_peer == B_TRUE && ldcp->resched_peer_txi == txi) {  in vgen_dringsend_shm()
722 		ldcp->resched_peer = B_FALSE;  in vgen_dringsend_shm()
733 	mutex_exit(&ldcp->wrlock);  in vgen_dringsend_shm()
744 		rv = vgen_send_dringdata_shm(ldcp, (uint32_t)txi, -1);  in vgen_dringsend_shm()
747 			DWARN(vgenp, ldcp, "failed sending dringdata msg "  in vgen_dringsend_shm()
749 			mutex_enter(&ldcp->wrlock);  in vgen_dringsend_shm()
751 			ldcp->resched_peer = B_TRUE;  in vgen_dringsend_shm()
752 			mutex_exit(&ldcp->wrlock);  in vgen_dringsend_shm()
758 		(void) vgen_handle_evt_reset(ldcp, VGEN_OTHER);  in vgen_dringsend_shm()
770 	vgen_ldc_t	*ldcp = (vgen_ldc_t *)arg1;  in vgen_handle_dringdata_shm()  local
772 	vgen_t		*vgenp = LDC_TO_VGEN(ldcp);  in vgen_handle_dringdata_shm()
783 		mutex_exit(&ldcp->cblock);  in vgen_handle_dringdata_shm()
784 		mutex_enter(&ldcp->rxlock);  in vgen_handle_dringdata_shm()
785 		rv = vgen_handle_dringdata_info_shm(ldcp, tagp);  in vgen_handle_dringdata_shm()
786 		mutex_exit(&ldcp->rxlock);  in vgen_handle_dringdata_shm()
787 		mutex_enter(&ldcp->cblock);  in vgen_handle_dringdata_shm()
789 			DWARN(vgenp, ldcp, "handle_data_info failed(%d)\n", rv);  in vgen_handle_dringdata_shm()
794 		rv = vgen_handle_dringdata_ack_shm(ldcp, tagp);  in vgen_handle_dringdata_shm()
796 			DWARN(vgenp, ldcp, "handle_data_ack failed(%d)\n", rv);  in vgen_handle_dringdata_shm()
801 		rv = vgen_handle_dringdata_nack_shm(ldcp, tagp);  in vgen_handle_dringdata_shm()
803 			DWARN(vgenp, ldcp, "handle_data_nack failed(%d)\n", rv);  in vgen_handle_dringdata_shm()
812 vgen_handle_dringdata_info_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)  in vgen_handle_dringdata_info_shm()  argument
818 	vgen_t		*vgenp = LDC_TO_VGEN(ldcp);  in vgen_handle_dringdata_info_shm()
819 	vgen_stats_t	*statsp = &ldcp->stats;  in vgen_handle_dringdata_info_shm()
824 	DBG1(vgenp, ldcp, "INFO: start(%d), end(%d)\n",  in vgen_handle_dringdata_info_shm()
827 	if (!(CHECK_RXI(start, ldcp)) ||  in vgen_handle_dringdata_info_shm()
828 	    ((end != -1) && !(CHECK_RXI(end, ldcp)))) {  in vgen_handle_dringdata_info_shm()
829 		DWARN(vgenp, ldcp, "Invalid Rx start(%d) or end(%d)\n",  in vgen_handle_dringdata_info_shm()
836 	if (dringmsg->dring_ident != ldcp->peer_hparams.dring_ident) {  in vgen_handle_dringdata_info_shm()
837 		DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",  in vgen_handle_dringdata_info_shm()
849 	if (ldcp->polling_on == B_TRUE) {  in vgen_handle_dringdata_info_shm()
856 	rv = vgen_intr_rcv_shm(ldcp);  in vgen_handle_dringdata_info_shm()
858 		DWARN(vgenp, ldcp, "vgen_intr_rcv_shm() failed\n");  in vgen_handle_dringdata_info_shm()
868 vgen_intr_rcv_shm(vgen_ldc_t *ldcp)  in vgen_intr_rcv_shm()  argument
879 	vio_net_rx_cb_t vrx_cb = ldcp->portp->vcb.vio_net_rx_cb;  in vgen_intr_rcv_shm()
881 	ASSERT(MUTEX_HELD(&ldcp->rxlock));  in vgen_intr_rcv_shm()
884 		rv = vgen_receive_packet(ldcp, &mp, &mblk_sz);  in vgen_intr_rcv_shm()
890 			DTRACE_PROBE1(vgen_intr_nopkts, vgen_ldc_t *, ldcp);  in vgen_intr_rcv_shm()
916 			DTRACE_PROBE2(vgen_intr_pkts, vgen_ldc_t *, ldcp,  in vgen_intr_rcv_shm()
918 			mutex_exit(&ldcp->rxlock);  in vgen_intr_rcv_shm()
919 			vrx_cb(ldcp->portp->vhp, bp);  in vgen_intr_rcv_shm()
920 			mutex_enter(&ldcp->rxlock);  in vgen_intr_rcv_shm()
929 	} while (total_count < ldcp->num_rxds);  in vgen_intr_rcv_shm()
932 		DTRACE_PROBE2(vgen_intr_pkts, vgen_ldc_t *, ldcp, int, count);  in vgen_intr_rcv_shm()
933 		mutex_exit(&ldcp->rxlock);  in vgen_intr_rcv_shm()
934 		vrx_cb(ldcp->portp->vhp, bp);  in vgen_intr_rcv_shm()
935 		mutex_enter(&ldcp->rxlock);  in vgen_intr_rcv_shm()
938 	if (ldcp->polling_on == B_FALSE) {  in vgen_intr_rcv_shm()
944 		end_ix = ldcp->next_rxi;  in vgen_intr_rcv_shm()
945 		DECR_RXI(end_ix, ldcp);  in vgen_intr_rcv_shm()
946 		msg.dring_ident = ldcp->peer_hparams.dring_ident;  in vgen_intr_rcv_shm()
947 		rv = vgen_send_dringack_shm(ldcp, (vio_msg_tag_t *)&msg,  in vgen_intr_rcv_shm()
961 vgen_poll_rcv_shm(vgen_ldc_t *ldcp, int bytes_to_pickup)  in vgen_poll_rcv_shm()  argument
971 	mutex_enter(&ldcp->rxlock);  in vgen_poll_rcv_shm()
973 	if (ldcp->hphase != VH_DONE) {  in vgen_poll_rcv_shm()
975 		mutex_exit(&ldcp->rxlock);  in vgen_poll_rcv_shm()
980 		rv = vgen_receive_packet(ldcp, &mp, &mblk_sz);  in vgen_poll_rcv_shm()
986 			DTRACE_PROBE1(vgen_poll_nopkts, vgen_ldc_t *, ldcp);  in vgen_poll_rcv_shm()
1021 	if (ldcp->rx_pktdata == vgen_handle_pkt_data &&  in vgen_poll_rcv_shm()
1022 	    ldcp->rx_pri_head != NULL) {  in vgen_poll_rcv_shm()
1023 		ldcp->rx_pri_tail->b_next = bp;  in vgen_poll_rcv_shm()
1024 		bp = ldcp->rx_pri_head;  in vgen_poll_rcv_shm()
1025 		ldcp->rx_pri_head = ldcp->rx_pri_tail = NULL;  in vgen_poll_rcv_shm()
1028 	mutex_exit(&ldcp->rxlock);  in vgen_poll_rcv_shm()
1030 	DTRACE_PROBE2(vgen_poll_pkts, vgen_ldc_t *, ldcp, int, count);  in vgen_poll_rcv_shm()
1031 	DTRACE_PROBE2(vgen_poll_bytes, vgen_ldc_t *, ldcp, uint_t, sz);  in vgen_poll_rcv_shm()
1049 vgen_receive_packet(vgen_ldc_t *ldcp, mblk_t **bp, uint_t *size)  in vgen_receive_packet()  argument
1060 	vgen_stats_t			*statsp = &ldcp->stats;  in vgen_receive_packet()
1061 	vgen_hparams_t			*lp = &ldcp->local_hparams;  in vgen_receive_packet()
1063 	rxi = ldcp->next_rxi;  in vgen_receive_packet()
1064 	rxdp = &(ldcp->rxdp[rxi]);  in vgen_receive_packet()
1065 	vmp = ldcp->rxdp_to_vmp[rxi];  in vgen_receive_packet()
1071 		DTRACE_PROBE1(vgen_noready_rxds, vgen_ldc_t *, ldcp);  in vgen_receive_packet()
1100 	new_vmp = vio_allocb(ldcp->rx_vmp);  in vgen_receive_packet()
1132 		ldcp->rxdp_to_vmp[rxi] = new_vmp;  in vgen_receive_packet()
1157 	INCR_RXI(rxi, ldcp);  in vgen_receive_packet()
1160 	ldcp->next_rxi = rxi;  in vgen_receive_packet()
1169 vgen_handle_dringdata_ack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)  in vgen_handle_dringdata_ack_shm()  argument
1179 	vgen_t				*vgenp = LDC_TO_VGEN(ldcp);  in vgen_handle_dringdata_ack_shm()
1184 	statsp = &ldcp->stats;  in vgen_handle_dringdata_ack_shm()
1191 	DBG2(vgenp, ldcp, "ACK:  start(%d), end(%d)\n", start, end);  in vgen_handle_dringdata_ack_shm()
1198 	if (((start != VNET_START_IDX_UNSPEC) && !(CHECK_TXI(start, ldcp))) ||  in vgen_handle_dringdata_ack_shm()
1199 	    !(CHECK_TXI(end, ldcp))) {  in vgen_handle_dringdata_ack_shm()
1201 		DWARN(vgenp, ldcp, "Invalid Tx ack start(%d) or end(%d)\n",  in vgen_handle_dringdata_ack_shm()
1207 	if (dringmsg->dring_ident != ldcp->local_hparams.dring_ident) {  in vgen_handle_dringdata_ack_shm()
1209 		DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",  in vgen_handle_dringdata_ack_shm()
1219 	mutex_enter(&ldcp->txlock);  in vgen_handle_dringdata_ack_shm()
1220 	if (ldcp->tx_blocked == B_TRUE) {  in vgen_handle_dringdata_ack_shm()
1222 		    ldcp->portp->vcb.vio_net_tx_update;  in vgen_handle_dringdata_ack_shm()
1224 		ldcp->tx_blocked = B_FALSE;  in vgen_handle_dringdata_ack_shm()
1225 		vtx_update(ldcp->portp->vhp);  in vgen_handle_dringdata_ack_shm()
1227 	mutex_exit(&ldcp->txlock);  in vgen_handle_dringdata_ack_shm()
1252 		if (mutex_owned(&ldcp->wrlock)) {  in vgen_handle_dringdata_ack_shm()
1253 			mutex_exit(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1263 	mutex_enter(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1267 	INCR_TXI(txi, ldcp);  in vgen_handle_dringdata_ack_shm()
1268 	txdp = &ldcp->mtxdp[txi];  in vgen_handle_dringdata_ack_shm()
1283 		ldcp->resched_peer = B_TRUE;  in vgen_handle_dringdata_ack_shm()
1284 		ldcp->resched_peer_txi = txi;  in vgen_handle_dringdata_ack_shm()
1285 		mutex_exit(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1293 	ldcp->resched_peer = B_FALSE;  in vgen_handle_dringdata_ack_shm()
1294 	mutex_exit(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1295 	rv = vgen_send_dringdata_shm(ldcp, txi, -1);  in vgen_handle_dringdata_ack_shm()
1297 		mutex_enter(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1298 		ldcp->resched_peer = B_TRUE;  in vgen_handle_dringdata_ack_shm()
1299 		mutex_exit(&ldcp->wrlock);  in vgen_handle_dringdata_ack_shm()
1306 vgen_handle_dringdata_nack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp)  in vgen_handle_dringdata_nack_shm()  argument
1314 	vgen_t				*vgenp = LDC_TO_VGEN(ldcp);  in vgen_handle_dringdata_nack_shm()
1317 	DBG1(vgenp, ldcp, "enter\n");  in vgen_handle_dringdata_nack_shm()
1325 	DWARN(vgenp, ldcp, "NACK: start(%d), end(%d)\n", start, end);  in vgen_handle_dringdata_nack_shm()
1332 	if (((start != VNET_START_IDX_UNSPEC) && !(CHECK_TXI(start, ldcp))) ||  in vgen_handle_dringdata_nack_shm()
1333 	    !(CHECK_TXI(end, ldcp))) {  in vgen_handle_dringdata_nack_shm()
1335 		DWARN(vgenp, ldcp, "Invalid Tx nack start(%d) or end(%d)\n",  in vgen_handle_dringdata_nack_shm()
1341 	if (dringmsg->dring_ident != ldcp->local_hparams.dring_ident) {  in vgen_handle_dringdata_nack_shm()
1343 		DWARN(vgenp, ldcp, "Invalid dring ident 0x%x\n",  in vgen_handle_dringdata_nack_shm()
1358 		mutex_exit(&ldcp->txlock);  in vgen_handle_dringdata_nack_shm()
1363 	mutex_enter(&ldcp->txlock);  in vgen_handle_dringdata_nack_shm()
1365 		txdp = &(ldcp->mtxdp[txi]);  in vgen_handle_dringdata_nack_shm()
1368 		INCR_TXI(txi, ldcp);  in vgen_handle_dringdata_nack_shm()
1376 	mutex_exit(&ldcp->txlock);  in vgen_handle_dringdata_nack_shm()
1385 vgen_send_dringdata_shm(vgen_ldc_t *ldcp, uint32_t start, int32_t end)  in vgen_send_dringdata_shm()  argument
1387 	vgen_t		*vgenp = LDC_TO_VGEN(ldcp);  in vgen_send_dringdata_shm()
1390 	vgen_stats_t	*statsp = &ldcp->stats;  in vgen_send_dringdata_shm()
1394 	if (vgen_inject_error(ldcp, VGEN_ERR_TXTIMEOUT)) {  in vgen_send_dringdata_shm()
1403 	tagp->vio_sid = ldcp->local_sid;  in vgen_send_dringdata_shm()
1405 	msgp->dring_ident = ldcp->local_hparams.dring_ident;  in vgen_send_dringdata_shm()
1408 	msgp->seq_num = atomic_inc_32_nv(&ldcp->dringdata_msgid);  in vgen_send_dringdata_shm()
1410 	rv = vgen_sendmsg_shm(ldcp, (caddr_t)tagp, sizeof (dringmsg));  in vgen_send_dringdata_shm()
1412 		DWARN(vgenp, ldcp, "vgen_sendmsg_shm() failed\n");  in vgen_send_dringdata_shm()
1418 	DBG2(vgenp, ldcp, "DRING_DATA_SENT \n");  in vgen_send_dringdata_shm()
1427 vgen_send_dringack_shm(vgen_ldc_t *ldcp, vio_msg_tag_t *tagp, uint32_t start,  in vgen_send_dringack_shm()  argument
1431 	vgen_t		*vgenp = LDC_TO_VGEN(ldcp);  in vgen_send_dringack_shm()
1433 	vgen_stats_t	*statsp = &ldcp->stats;  in vgen_send_dringack_shm()
1438 	tagp->vio_sid = ldcp->local_sid;  in vgen_send_dringack_shm()
1442 	msgp->seq_num = atomic_inc_32_nv(&ldcp->dringdata_msgid);  in vgen_send_dringack_shm()
1444 	rv = vgen_sendmsg_shm(ldcp, (caddr_t)tagp, sizeof (*msgp));  in vgen_send_dringack_shm()
1446 		DWARN(vgenp, ldcp, "vgen_sendmsg_shm() failed\n");  in vgen_send_dringack_shm()
1461 vgen_sendmsg_shm(vgen_ldc_t *ldcp, caddr_t msg,  size_t msglen)  in vgen_sendmsg_shm()  argument
1466 	vgen_t			*vgenp = LDC_TO_VGEN(ldcp);  in vgen_sendmsg_shm()
1474 		rv = ldc_write(ldcp->ldc_handle, (caddr_t)msg, &len);  in vgen_sendmsg_shm()
1480 		DWARN(vgenp, ldcp, "ldc_write failed: rv(%d) msglen(%d)\n",  in vgen_sendmsg_shm()
1486 		DWARN(vgenp, ldcp, "ldc_write failed: rv(%d) msglen (%d)\n",  in vgen_sendmsg_shm()