91 viona_tx_wait_outstanding(viona_vring_t *ring)  in viona_tx_wait_outstanding()  argument
93 	ASSERT(MUTEX_HELD(&ring->vr_lock));  in viona_tx_wait_outstanding()
95 	while (ring->vr_xfer_outstanding != 0) {  in viona_tx_wait_outstanding()
101 		cv_wait(&ring->vr_cv, &ring->vr_lock);  in viona_tx_wait_outstanding()
147 viona_tx_ring_alloc(viona_vring_t *ring, const uint16_t qsz)  in viona_tx_ring_alloc()  argument
149 	const viona_link_params_t *vlp = &ring->vr_link->l_params;  in viona_tx_ring_alloc()
151 	ring->vr_tx.vrt_header_pad = vlp->vlp_tx_header_pad;  in viona_tx_ring_alloc()
153 	if (!ring->vr_link->l_params.vlp_tx_copy_data) {  in viona_tx_ring_alloc()
156 		ring->vr_tx.vrt_desb = dp;  in viona_tx_ring_alloc()
159 		    VIONA_MAX_HDRS_LEN + ring->vr_tx.vrt_header_pad;  in viona_tx_ring_alloc()
163 			dp->d_ring = ring;  in viona_tx_ring_alloc()
169 	ring->vr_tx.vrt_iov = kmem_alloc(sizeof (struct iovec) * qsz, KM_SLEEP);  in viona_tx_ring_alloc()
170 	ring->vr_tx.vrt_iov_cnt = qsz;  in viona_tx_ring_alloc()
174 viona_tx_ring_free(viona_vring_t *ring, const uint16_t qsz)  in viona_tx_ring_free()  argument
176 	if (ring->vr_tx.vrt_desb != NULL) {  in viona_tx_ring_free()
177 		viona_desb_t *dp = ring->vr_tx.vrt_desb;  in viona_tx_ring_free()
180 		    VIONA_MAX_HDRS_LEN + ring->vr_tx.vrt_header_pad;  in viona_tx_ring_free()
184 		kmem_free(ring->vr_tx.vrt_desb, sizeof (viona_desb_t) * qsz);  in viona_tx_ring_free()
185 		ring->vr_tx.vrt_desb = NULL;  in viona_tx_ring_free()
188 	if (ring->vr_tx.vrt_iov != NULL) {  in viona_tx_ring_free()
189 		ASSERT3U(ring->vr_tx.vrt_iov_cnt, !=, 0);  in viona_tx_ring_free()
191 		kmem_free(ring->vr_tx.vrt_iov,  in viona_tx_ring_free()
192 		    sizeof (struct iovec) * ring->vr_tx.vrt_iov_cnt);  in viona_tx_ring_free()
193 		ring->vr_tx.vrt_iov = NULL;  in viona_tx_ring_free()
194 		ring->vr_tx.vrt_iov_cnt = 0;  in viona_tx_ring_free()
199 viona_tx_done(viona_vring_t *ring, uint32_t len, uint16_t cookie)  in viona_tx_done()  argument
201 	vq_pushchain(ring, len, cookie);  in viona_tx_done()
204 	viona_intr_ring(ring, B_FALSE);  in viona_tx_done()
210 viona_worker_tx(viona_vring_t *ring, viona_link_t *link)  in viona_worker_tx()  argument
212 	(void) thread_vsetname(curthread, "viona_tx_%p", ring);  in viona_worker_tx()
214 	ASSERT(MUTEX_HELD(&ring->vr_lock));  in viona_worker_tx()
215 	ASSERT3U(ring->vr_state, ==, VRS_RUN);  in viona_worker_tx()
217 	mutex_exit(&ring->vr_lock);  in viona_worker_tx()
222 		viona_ring_disable_notify(ring);  in viona_worker_tx()
223 		while (viona_ring_num_avail(ring) != 0) {  in viona_worker_tx()
224 			viona_tx(link, ring);  in viona_worker_tx()
234 				mutex_enter(&ring->vr_lock);  in viona_worker_tx()
235 				const bool need_bail = vring_need_bail(ring);  in viona_worker_tx()
236 				mutex_exit(&ring->vr_lock);  in viona_worker_tx()
254 		viona_ring_enable_notify(ring);  in viona_worker_tx()
257 		if (viona_ring_num_avail(ring) == 0 &&  in viona_worker_tx()
263 			viona_intr_ring(ring, B_TRUE);  in viona_worker_tx()
266 		mutex_enter(&ring->vr_lock);  in viona_worker_tx()
268 			if (vring_need_bail(ring)) {  in viona_worker_tx()
269 				ring->vr_state = VRS_STOP;  in viona_worker_tx()
270 				viona_tx_wait_outstanding(ring);  in viona_worker_tx()
274 			if (vmm_drv_lease_expired(ring->vr_lease)) {  in viona_worker_tx()
275 				ring->vr_state_flags |= VRSF_RENEW;  in viona_worker_tx()
281 				viona_tx_wait_outstanding(ring);  in viona_worker_tx()
284 				    viona_ring_lease_renew(ring);  in viona_worker_tx()
285 				ring->vr_state_flags &= ~VRSF_RENEW;  in viona_worker_tx()
289 					ring->vr_state = VRS_STOP;  in viona_worker_tx()
294 			if (viona_ring_num_avail(ring) != 0) {  in viona_worker_tx()
299 			(void) cv_wait_sig(&ring->vr_cv, &ring->vr_lock);  in viona_worker_tx()
301 		mutex_exit(&ring->vr_lock);  in viona_worker_tx()
309 	viona_vring_t *ring = dp->d_ring;  in viona_desb_release()  local
337 	viona_tx_done(ring, len, cookie);  in viona_desb_release()
339 	mutex_enter(&ring->vr_lock);  in viona_desb_release()
340 	if ((--ring->vr_xfer_outstanding) == 0) {  in viona_desb_release()
341 		cv_broadcast(&ring->vr_cv);  in viona_desb_release()
343 	mutex_exit(&ring->vr_lock);  in viona_desb_release()
406 viona_tx_offloads(viona_vring_t *ring, const struct virtio_net_mrgrxhdr *hdr,  in viona_tx_offloads()  argument
409 	viona_link_t *link = ring->vr_link;  in viona_tx_offloads()
422 		VIONA_RING_STAT_INCR(ring, fail_hcksum);  in viona_tx_offloads()
432 		VIONA_RING_STAT_INCR(ring, fail_hcksum_proto);  in viona_tx_offloads()
442 			VIONA_RING_STAT_INCR(ring, tx_gso_fail);  in viona_tx_offloads()
503 			VIONA_RING_STAT_INCR(ring, tx_gso_fail);  in viona_tx_offloads()
534 		VIONA_RING_STAT_INCR(ring, fail_hcksum);  in viona_tx_offloads()
545 		VIONA_RING_STAT_INCR(ring, fail_hcksum6);  in viona_tx_offloads()
554 	VIONA_RING_STAT_INCR(ring, fail_hcksum_proto);  in viona_tx_offloads()
559 viona_tx_alloc_headers(viona_vring_t *ring, uint16_t cookie, viona_desb_t **dpp,  in viona_tx_alloc_headers()  argument
565 	const size_t header_pad = ring->vr_tx.vrt_header_pad;  in viona_tx_alloc_headers()
567 	if (ring->vr_tx.vrt_desb != NULL) {  in viona_tx_alloc_headers()
568 		viona_desb_t *dp = &ring->vr_tx.vrt_desb[cookie];  in viona_tx_alloc_headers()
613 viona_tx_copy_headers(viona_vring_t *ring, iov_bunch_t *iob, mblk_t *mp,  in viona_tx_copy_headers()  argument
618 	if (ring->vr_tx.vrt_desb == NULL) {  in viona_tx_copy_headers()
682 viona_tx(viona_link_t *link, viona_vring_t *ring)  in viona_tx()  argument
684 	struct iovec		*iov = ring->vr_tx.vrt_iov;  in viona_tx()
685 	const uint_t		max_segs = ring->vr_tx.vrt_iov_cnt;  in viona_tx()
696 	const int n = vq_popchain(ring, iov, max_segs, &cookie, &pages,  in viona_tx()
699 		VIONA_PROBE1(tx_absent, viona_vring_t *, ring);  in viona_tx()
700 		VIONA_RING_STAT_INCR(ring, tx_absent);  in viona_tx()
751 	mp_head = viona_tx_alloc_headers(ring, cookie, &dp, pkt_len);  in viona_tx()
761 	if (!viona_tx_copy_headers(ring, &iob, mp_head, &meoi)) {  in viona_tx()
823 		if (viona_hook(link, ring, &mp, B_TRUE) != 0) {  in viona_tx()
855 		if (!viona_tx_offloads(ring, &hdr, &meoi, mp_head, pkt_len)) {  in viona_tx()
862 			viona_ring_stat_error(ring);  in viona_tx()
873 		mutex_enter(&ring->vr_lock);  in viona_tx()
874 		ring->vr_xfer_outstanding++;  in viona_tx()
875 		mutex_exit(&ring->vr_lock);  in viona_tx()
883 		viona_tx_done(ring, total_len, cookie);  in viona_tx()
890 	viona_ring_stat_accept(ring, pkt_len);  in viona_tx()
913 	viona_ring_stat_error(ring);  in viona_tx()
942 	viona_ring_stat_drop(ring);  in viona_tx()
944 	VIONA_PROBE3(tx_drop, viona_vring_t *, ring, uint32_t, pkt_len,  in viona_tx()
947 	viona_tx_done(ring, total_len, cookie);  in viona_tx()