Lines Matching refs:itrq
705 i40e_alloc_rx_data(i40e_t *i40e, i40e_trqpair_t *itrq) in i40e_alloc_rx_data() argument
712 itrq->itrq_rxdata = rxd; in i40e_alloc_rx_data()
725 itrq->itrq_index); in i40e_alloc_rx_data()
733 "for ring %d", rxd->rxd_free_list_size, itrq->itrq_index); in i40e_alloc_rx_data()
742 itrq->itrq_index); in i40e_alloc_rx_data()
750 itrq->itrq_rxdata = NULL; in i40e_alloc_rx_data()
882 i40e_free_tx_dma(i40e_trqpair_t *itrq) in i40e_free_tx_dma() argument
886 if (itrq->itrq_tcb_area != NULL) { in i40e_free_tx_dma()
888 i40e_tx_control_block_t *tcb = itrq->itrq_tcb_area; in i40e_free_tx_dma()
890 for (i = 0; i < itrq->itrq_tx_free_list_size; i++, tcb++) { in i40e_free_tx_dma()
903 itrq->itrq_tx_free_list_size; in i40e_free_tx_dma()
904 kmem_free(itrq->itrq_tcb_area, fsz); in i40e_free_tx_dma()
905 itrq->itrq_tcb_area = NULL; in i40e_free_tx_dma()
908 if (itrq->itrq_tcb_free_list != NULL) { in i40e_free_tx_dma()
910 itrq->itrq_tx_free_list_size; in i40e_free_tx_dma()
911 kmem_free(itrq->itrq_tcb_free_list, fsz); in i40e_free_tx_dma()
912 itrq->itrq_tcb_free_list = NULL; in i40e_free_tx_dma()
915 if (itrq->itrq_tcb_work_list != NULL) { in i40e_free_tx_dma()
917 itrq->itrq_tx_ring_size; in i40e_free_tx_dma()
918 kmem_free(itrq->itrq_tcb_work_list, fsz); in i40e_free_tx_dma()
919 itrq->itrq_tcb_work_list = NULL; in i40e_free_tx_dma()
922 i40e_free_dma_buffer(&itrq->itrq_desc_area); in i40e_free_tx_dma()
923 itrq->itrq_desc_ring = NULL; in i40e_free_tx_dma()
928 i40e_alloc_tx_dma(i40e_trqpair_t *itrq) in i40e_alloc_tx_dma() argument
933 i40e_t *i40e = itrq->itrq_i40e; in i40e_alloc_tx_dma()
935 itrq->itrq_tx_ring_size = i40e->i40e_tx_ring_size; in i40e_alloc_tx_dma()
936 itrq->itrq_tx_free_list_size = i40e->i40e_tx_ring_size + in i40e_alloc_tx_dma()
942 dmasz = sizeof (i40e_tx_desc_t) * itrq->itrq_tx_ring_size; in i40e_alloc_tx_dma()
946 if (i40e_alloc_dma_buffer(i40e, &itrq->itrq_desc_area, in i40e_alloc_tx_dma()
953 itrq->itrq_desc_ring = in i40e_alloc_tx_dma()
954 (i40e_tx_desc_t *)(uintptr_t)itrq->itrq_desc_area.dmab_address; in i40e_alloc_tx_dma()
955 itrq->itrq_desc_wbhead = (uint32_t *)(itrq->itrq_desc_ring + in i40e_alloc_tx_dma()
956 itrq->itrq_tx_ring_size); in i40e_alloc_tx_dma()
957 itrq->itrq_desc_head = 0; in i40e_alloc_tx_dma()
958 itrq->itrq_desc_tail = 0; in i40e_alloc_tx_dma()
959 itrq->itrq_desc_free = itrq->itrq_tx_ring_size; in i40e_alloc_tx_dma()
961 itrq->itrq_tcb_work_list = kmem_zalloc(itrq->itrq_tx_ring_size * in i40e_alloc_tx_dma()
963 if (itrq->itrq_tcb_work_list == NULL) { in i40e_alloc_tx_dma()
965 "for ring %d", itrq->itrq_tx_ring_size, itrq->itrq_index); in i40e_alloc_tx_dma()
969 itrq->itrq_tcb_free_list = kmem_zalloc(itrq->itrq_tx_free_list_size * in i40e_alloc_tx_dma()
971 if (itrq->itrq_tcb_free_list == NULL) { in i40e_alloc_tx_dma()
973 "for ring %d", itrq->itrq_tx_free_list_size, in i40e_alloc_tx_dma()
974 itrq->itrq_index); in i40e_alloc_tx_dma()
981 itrq->itrq_tcb_area = kmem_zalloc(sizeof (i40e_tx_control_block_t) * in i40e_alloc_tx_dma()
982 itrq->itrq_tx_free_list_size, KM_NOSLEEP); in i40e_alloc_tx_dma()
983 if (itrq->itrq_tcb_area == NULL) { in i40e_alloc_tx_dma()
985 "ring %d", itrq->itrq_tx_free_list_size, itrq->itrq_index); in i40e_alloc_tx_dma()
994 tcb = itrq->itrq_tcb_area; in i40e_alloc_tx_dma()
995 for (i = 0; i < itrq->itrq_tx_free_list_size; i++, tcb++) { in i40e_alloc_tx_dma()
1008 "data binding on ring %d: %d", itrq->itrq_index, in i40e_alloc_tx_dma()
1019 "LSO data binding on ring %d: %d", itrq->itrq_index, in i40e_alloc_tx_dma()
1030 itrq->itrq_index); in i40e_alloc_tx_dma()
1034 itrq->itrq_tcb_free_list[i] = tcb; in i40e_alloc_tx_dma()
1037 itrq->itrq_tcb_free = itrq->itrq_tx_free_list_size; in i40e_alloc_tx_dma()
1042 i40e_free_tx_dma(itrq); in i40e_alloc_tx_dma()
1051 i40e_free_ring_mem(i40e_trqpair_t *itrq, boolean_t failed_init) in i40e_free_ring_mem() argument
1053 i40e_t *i40e = itrq->itrq_i40e; in i40e_free_ring_mem()
1054 i40e_rx_data_t *rxd = itrq->itrq_rxdata; in i40e_free_ring_mem()
1078 itrq->itrq_rxdata = NULL; in i40e_free_ring_mem()
1082 i40e_free_tx_dma(itrq); in i40e_free_ring_mem()
1091 i40e_alloc_ring_mem(i40e_trqpair_t *itrq) in i40e_alloc_ring_mem() argument
1093 if (!i40e_alloc_rx_data(itrq->itrq_i40e, itrq)) in i40e_alloc_ring_mem()
1096 if (!i40e_alloc_rx_dma(itrq->itrq_rxdata)) in i40e_alloc_ring_mem()
1099 if (!i40e_alloc_tx_dma(itrq)) in i40e_alloc_ring_mem()
1105 i40e_free_ring_mem(itrq, B_TRUE); in i40e_alloc_ring_mem()
1236 i40e_rx_bind(i40e_trqpair_t *itrq, i40e_rx_data_t *rxd, uint32_t index, in i40e_rx_bind() argument
1243 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); in i40e_rx_bind()
1246 itrq->itrq_rxstat.irxs_rx_bind_norcb.value.ui64++; in i40e_rx_bind()
1261 itrq->itrq_rxstat.irxs_rx_bind_nomp.value.ui64++; in i40e_rx_bind()
1294 i40e_rx_copy(i40e_trqpair_t *itrq, i40e_rx_data_t *rxd, uint32_t index, in i40e_rx_copy() argument
1314 itrq->itrq_rxstat.irxs_rx_copy_nomem.value.ui64++; in i40e_rx_copy()
1338 i40e_rx_hcksum(i40e_trqpair_t *itrq, mblk_t *mp, uint64_t status, uint32_t err, in i40e_rx_hcksum() argument
1354 itrq->itrq_rxstat.irxs_hck_unknown.value.ui64++; in i40e_rx_hcksum()
1363 itrq->itrq_rxstat.irxs_hck_nol3l4p.value.ui64++; in i40e_rx_hcksum()
1375 itrq->itrq_rxstat.irxs_hck_v6skip.value.ui64++; in i40e_rx_hcksum()
1391 itrq->itrq_rxstat.irxs_hck_iperr.value.ui64++; in i40e_rx_hcksum()
1393 itrq->itrq_rxstat.irxs_hck_v4hdrok.value.ui64++; in i40e_rx_hcksum()
1398 itrq->itrq_rxstat.irxs_hck_eiperr.value.ui64++; in i40e_rx_hcksum()
1400 itrq->itrq_rxstat.irxs_hck_v4hdrok.value.ui64++; in i40e_rx_hcksum()
1420 itrq->itrq_rxstat.irxs_hck_l4err.value.ui64++; in i40e_rx_hcksum()
1422 itrq->itrq_rxstat.irxs_hck_l4hdrok.value.ui64++; in i40e_rx_hcksum()
1428 itrq->itrq_rxstat.irxs_hck_set.value.ui64++; in i40e_rx_hcksum()
1431 itrq->itrq_rxstat.irxs_hck_miss.value.ui64++; in i40e_rx_hcksum()
1436 i40e_ring_rx(i40e_trqpair_t *itrq, int poll_bytes) in i40e_ring_rx() argument
1448 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); in i40e_ring_rx()
1449 rxd = itrq->itrq_rxdata; in i40e_ring_rx()
1450 i40e = itrq->itrq_i40e; in i40e_ring_rx()
1515 itrq->itrq_rxstat.irxs_rx_desc_error.value.ui64++; in i40e_ring_rx()
1549 mp = i40e_rx_bind(itrq, rxd, cur_head, plen); in i40e_ring_rx()
1551 mp = i40e_rx_copy(itrq, rxd, cur_head, plen); in i40e_ring_rx()
1555 i40e_rx_hcksum(itrq, mp, stword, error, ptype); in i40e_ring_rx()
1590 itrq->itrq_rxstat.irxs_rx_intr_limit.value.ui64++; in i40e_ring_rx()
1614 I40E_WRITE_REG(hw, I40E_QRX_TAIL(itrq->itrq_index), tail); in i40e_ring_rx()
1621 itrq->itrq_rxstat.irxs_bytes.value.ui64 += rx_bytes; in i40e_ring_rx()
1622 itrq->itrq_rxstat.irxs_packets.value.ui64 += rx_frames; in i40e_ring_rx()
1642 i40e_trqpair_t *itrq = arg; in i40e_ring_rx_poll() local
1649 mutex_enter(&itrq->itrq_rx_lock); in i40e_ring_rx_poll()
1650 mp = i40e_ring_rx(itrq, poll_bytes); in i40e_ring_rx_poll()
1651 mutex_exit(&itrq->itrq_rx_lock); in i40e_ring_rx_poll()
1669 i40e_tx_context(i40e_t *i40e, i40e_trqpair_t *itrq, mblk_t *mp, in i40e_tx_context() argument
1673 i40e_txq_stat_t *txs = &itrq->itrq_txstat; in i40e_tx_context()
1793 i40e_tcb_free(i40e_trqpair_t *itrq, i40e_tx_control_block_t *tcb) in i40e_tcb_free() argument
1797 mutex_enter(&itrq->itrq_tcb_lock); in i40e_tcb_free()
1798 ASSERT(itrq->itrq_tcb_free < itrq->itrq_tx_free_list_size); in i40e_tcb_free()
1799 itrq->itrq_tcb_free_list[itrq->itrq_tcb_free] = tcb; in i40e_tcb_free()
1800 itrq->itrq_tcb_free++; in i40e_tcb_free()
1801 mutex_exit(&itrq->itrq_tcb_lock); in i40e_tcb_free()
1805 i40e_tcb_alloc(i40e_trqpair_t *itrq) in i40e_tcb_alloc() argument
1809 mutex_enter(&itrq->itrq_tcb_lock); in i40e_tcb_alloc()
1810 if (itrq->itrq_tcb_free == 0) { in i40e_tcb_alloc()
1811 mutex_exit(&itrq->itrq_tcb_lock); in i40e_tcb_alloc()
1815 itrq->itrq_tcb_free--; in i40e_tcb_alloc()
1816 ret = itrq->itrq_tcb_free_list[itrq->itrq_tcb_free]; in i40e_tcb_alloc()
1817 itrq->itrq_tcb_free_list[itrq->itrq_tcb_free] = NULL; in i40e_tcb_alloc()
1818 mutex_exit(&itrq->itrq_tcb_lock); in i40e_tcb_alloc()
1873 i40e_tx_cleanup_ring(i40e_trqpair_t *itrq) in i40e_tx_cleanup_ring() argument
1877 ASSERT(MUTEX_HELD(&itrq->itrq_tx_lock)); in i40e_tx_cleanup_ring()
1878 ASSERT(itrq->itrq_desc_free <= itrq->itrq_tx_ring_size); in i40e_tx_cleanup_ring()
1885 index = I40E_READ_REG(&itrq->itrq_i40e->i40e_hw_space, in i40e_tx_cleanup_ring()
1886 I40E_QTX_ENA(itrq->itrq_index)); in i40e_tx_cleanup_ring()
1891 index = itrq->itrq_desc_head; in i40e_tx_cleanup_ring()
1892 while (itrq->itrq_desc_free < itrq->itrq_tx_ring_size) { in i40e_tx_cleanup_ring()
1895 tcb = itrq->itrq_tcb_work_list[index]; in i40e_tx_cleanup_ring()
1897 itrq->itrq_tcb_work_list[index] = NULL; in i40e_tx_cleanup_ring()
1899 i40e_tcb_free(itrq, tcb); in i40e_tx_cleanup_ring()
1902 bzero(&itrq->itrq_desc_ring[index], sizeof (i40e_tx_desc_t)); in i40e_tx_cleanup_ring()
1903 index = i40e_next_desc(index, 1, itrq->itrq_tx_ring_size); in i40e_tx_cleanup_ring()
1904 itrq->itrq_desc_free++; in i40e_tx_cleanup_ring()
1907 ASSERT(index == itrq->itrq_desc_tail); in i40e_tx_cleanup_ring()
1908 itrq->itrq_desc_head = index; in i40e_tx_cleanup_ring()
1918 i40e_tx_recycle_ring(i40e_trqpair_t *itrq) in i40e_tx_recycle_ring() argument
1922 i40e_t *i40e = itrq->itrq_i40e; in i40e_tx_recycle_ring()
1925 mutex_enter(&itrq->itrq_tx_lock); in i40e_tx_recycle_ring()
1927 ASSERT(itrq->itrq_desc_free <= itrq->itrq_tx_ring_size); in i40e_tx_recycle_ring()
1928 if (itrq->itrq_desc_free == itrq->itrq_tx_ring_size) { in i40e_tx_recycle_ring()
1929 if (itrq->itrq_tx_blocked == B_TRUE) { in i40e_tx_recycle_ring()
1930 itrq->itrq_tx_blocked = B_FALSE; in i40e_tx_recycle_ring()
1932 itrq->itrq_mactxring); in i40e_tx_recycle_ring()
1933 itrq->itrq_txstat.itxs_num_unblocked.value.ui64++; in i40e_tx_recycle_ring()
1935 mutex_exit(&itrq->itrq_tx_lock); in i40e_tx_recycle_ring()
1944 VERIFY0(ddi_dma_sync(itrq->itrq_desc_area.dmab_dma_handle, in i40e_tx_recycle_ring()
1945 (uintptr_t)itrq->itrq_desc_wbhead, in i40e_tx_recycle_ring()
1948 if (i40e_check_dma_handle(itrq->itrq_desc_area.dmab_dma_handle) != in i40e_tx_recycle_ring()
1950 mutex_exit(&itrq->itrq_tx_lock); in i40e_tx_recycle_ring()
1956 wbhead = *itrq->itrq_desc_wbhead; in i40e_tx_recycle_ring()
1957 toclean = itrq->itrq_desc_head; in i40e_tx_recycle_ring()
1964 tcb = itrq->itrq_tcb_work_list[toclean]; in i40e_tx_recycle_ring()
1965 itrq->itrq_tcb_work_list[toclean] = NULL; in i40e_tx_recycle_ring()
1986 bzero(&itrq->itrq_desc_ring[toclean], in i40e_tx_recycle_ring()
1989 itrq->itrq_tx_ring_size); in i40e_tx_recycle_ring()
1994 itrq->itrq_desc_head = wbhead; in i40e_tx_recycle_ring()
1995 itrq->itrq_desc_free += count; in i40e_tx_recycle_ring()
1996 itrq->itrq_txstat.itxs_recycled.value.ui64 += count; in i40e_tx_recycle_ring()
1997 ASSERT(itrq->itrq_desc_free <= itrq->itrq_tx_ring_size); in i40e_tx_recycle_ring()
1999 if (itrq->itrq_tx_blocked == B_TRUE && in i40e_tx_recycle_ring()
2000 itrq->itrq_desc_free > i40e->i40e_tx_block_thresh) { in i40e_tx_recycle_ring()
2001 itrq->itrq_tx_blocked = B_FALSE; in i40e_tx_recycle_ring()
2003 mac_tx_ring_update(i40e->i40e_mac_hdl, itrq->itrq_mactxring); in i40e_tx_recycle_ring()
2004 itrq->itrq_txstat.itxs_num_unblocked.value.ui64++; in i40e_tx_recycle_ring()
2007 mutex_exit(&itrq->itrq_tx_lock); in i40e_tx_recycle_ring()
2017 i40e_tcb_free(itrq, tcb); in i40e_tx_recycle_ring()
2020 DTRACE_PROBE2(i40e__recycle, i40e_trqpair_t *, itrq, uint32_t, count); in i40e_tx_recycle_ring()
2043 i40e_tx_bind_fragment(i40e_trqpair_t *itrq, const mblk_t *mp, in i40e_tx_bind_fragment() argument
2050 i40e_txq_stat_t *txs = &itrq->itrq_txstat; in i40e_tx_bind_fragment()
2052 if ((tcb = i40e_tcb_alloc(itrq)) == NULL) { in i40e_tx_bind_fragment()
2093 i40e_tcb_free(itrq, tcb); in i40e_tx_bind_fragment()
2098 i40e_tx_set_data_desc(i40e_trqpair_t *itrq, i40e_tx_context_t *tctx, in i40e_tx_set_data_desc() argument
2104 ASSERT(MUTEX_HELD(&itrq->itrq_tx_lock)); in i40e_tx_set_data_desc()
2105 itrq->itrq_desc_free--; in i40e_tx_set_data_desc()
2106 txdesc = &itrq->itrq_desc_ring[itrq->itrq_desc_tail]; in i40e_tx_set_data_desc()
2107 itrq->itrq_desc_tail = i40e_next_desc(itrq->itrq_desc_tail, 1, in i40e_tx_set_data_desc()
2108 itrq->itrq_tx_ring_size); in i40e_tx_set_data_desc()
2173 i40e_non_lso_chain(i40e_trqpair_t *itrq, mblk_t *mp, uint_t *ndesc) in i40e_non_lso_chain() argument
2179 i40e_t *i40e = itrq->itrq_i40e; in i40e_non_lso_chain()
2180 i40e_txq_stat_t *txs = &itrq->itrq_txstat; in i40e_non_lso_chain()
2202 if ((tcb = i40e_tcb_alloc(itrq)) == NULL) { in i40e_non_lso_chain()
2217 tcb = i40e_tx_bind_fragment(itrq, nmp, 0, B_FALSE); in i40e_non_lso_chain()
2234 i40e_tcb_free(itrq, tcb); in i40e_non_lso_chain()
2271 i40e_tcb_free(itrq, tcb); in i40e_non_lso_chain()
2359 i40e_lso_chain(i40e_trqpair_t *itrq, const mblk_t *mp, in i40e_lso_chain() argument
2386 i40e_t *i40e = itrq->itrq_i40e; in i40e_lso_chain()
2387 i40e_txq_stat_t *txs = &itrq->itrq_txstat; in i40e_lso_chain()
2393 if ((tcb = i40e_tcb_alloc(itrq)) == NULL) { in i40e_lso_chain()
2465 if ((tcb = i40e_tcb_alloc(itrq)) == NULL) { in i40e_lso_chain()
2532 tcb = i40e_tx_bind_fragment(itrq, mp, cpoff, B_TRUE); in i40e_lso_chain()
2562 i40e_tcb_free(itrq, tcb); in i40e_lso_chain()
2615 i40e_tcb_free(itrq, tcb); in i40e_lso_chain()
2629 i40e_ring_tx_enter(i40e_trqpair_t *itrq) in i40e_ring_tx_enter() argument
2633 mutex_enter(&itrq->itrq_tx_lock); in i40e_ring_tx_enter()
2634 allow = !itrq->itrq_tx_quiesce; in i40e_ring_tx_enter()
2636 itrq->itrq_tx_active++; in i40e_ring_tx_enter()
2637 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx_enter()
2643 i40e_ring_tx_exit_nolock(i40e_trqpair_t *itrq) in i40e_ring_tx_exit_nolock() argument
2645 ASSERT(MUTEX_HELD(&itrq->itrq_tx_lock)); in i40e_ring_tx_exit_nolock()
2647 itrq->itrq_tx_active--; in i40e_ring_tx_exit_nolock()
2648 if (itrq->itrq_tx_quiesce) in i40e_ring_tx_exit_nolock()
2649 cv_signal(&itrq->itrq_tx_cv); in i40e_ring_tx_exit_nolock()
2653 i40e_ring_tx_exit(i40e_trqpair_t *itrq) in i40e_ring_tx_exit() argument
2655 mutex_enter(&itrq->itrq_tx_lock); in i40e_ring_tx_exit()
2656 i40e_ring_tx_exit_nolock(itrq); in i40e_ring_tx_exit()
2657 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx_exit()
2668 i40e_ring_tx_quiesce(i40e_trqpair_t *itrq) in i40e_ring_tx_quiesce() argument
2670 mutex_enter(&itrq->itrq_tx_lock); in i40e_ring_tx_quiesce()
2671 if (itrq->itrq_tx_quiesce) { in i40e_ring_tx_quiesce()
2676 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx_quiesce()
2684 itrq->itrq_tx_quiesce = B_TRUE; in i40e_ring_tx_quiesce()
2685 while (itrq->itrq_tx_active > 0) in i40e_ring_tx_quiesce()
2686 cv_wait(&itrq->itrq_tx_cv, &itrq->itrq_tx_lock); in i40e_ring_tx_quiesce()
2688 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx_quiesce()
2721 i40e_trqpair_t *itrq = arg; in i40e_ring_tx() local
2722 i40e_t *i40e = itrq->itrq_i40e; in i40e_ring_tx()
2724 i40e_txq_stat_t *txs = &itrq->itrq_txstat; in i40e_ring_tx()
2733 !i40e_ring_tx_enter(itrq)) { in i40e_ring_tx()
2740 itrq->itrq_txstat.itxs_hck_meoifail.value.ui64++; in i40e_ring_tx()
2741 i40e_ring_tx_exit(itrq); in i40e_ring_tx()
2750 if (i40e_tx_context(i40e, itrq, mp, &meo, &tctx) < 0) { in i40e_ring_tx()
2752 itrq->itrq_txstat.itxs_err_context.value.ui64++; in i40e_ring_tx()
2753 i40e_ring_tx_exit(itrq); in i40e_ring_tx()
2777 if ((tcb_ctx = i40e_tcb_alloc(itrq)) == NULL) { in i40e_ring_tx()
2786 tcbhead = i40e_non_lso_chain(itrq, mp, &needed_desc); in i40e_ring_tx()
2788 tcbhead = i40e_lso_chain(itrq, mp, &meo, &tctx, &needed_desc); in i40e_ring_tx()
2804 mutex_enter(&itrq->itrq_tx_lock); in i40e_ring_tx()
2805 if (itrq->itrq_desc_free < i40e->i40e_tx_block_thresh || in i40e_ring_tx()
2806 (itrq->itrq_desc_free - 1) < needed_desc) { in i40e_ring_tx()
2808 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx()
2819 uint_t tail = itrq->itrq_desc_tail; in i40e_ring_tx()
2820 itrq->itrq_desc_free--; in i40e_ring_tx()
2821 ctxdesc = (i40e_tx_context_desc_t *)&itrq->itrq_desc_ring[tail]; in i40e_ring_tx()
2822 itrq->itrq_tcb_work_list[tail] = tcb_ctx; in i40e_ring_tx()
2823 itrq->itrq_desc_tail = i40e_next_desc(tail, 1, in i40e_ring_tx()
2824 itrq->itrq_tx_ring_size); in i40e_ring_tx()
2847 itrq->itrq_tcb_work_list[itrq->itrq_desc_tail] = tcb; in i40e_ring_tx()
2851 i40e_tx_set_data_desc(itrq, &tctx, in i40e_ring_tx()
2862 i40e_tx_set_data_desc(itrq, &tctx, in i40e_ring_tx()
2875 I40E_DMA_SYNC(&itrq->itrq_desc_area, DDI_DMA_SYNC_FORDEV); in i40e_ring_tx()
2877 I40E_WRITE_REG(hw, I40E_QTX_TAIL(itrq->itrq_index), in i40e_ring_tx()
2878 itrq->itrq_desc_tail); in i40e_ring_tx()
2895 i40e_ring_tx_exit_nolock(itrq); in i40e_ring_tx()
2897 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx()
2911 i40e_tcb_free(itrq, tcb_ctx); in i40e_ring_tx()
2923 i40e_tcb_free(itrq, tcb); in i40e_ring_tx()
2927 mutex_enter(&itrq->itrq_tx_lock); in i40e_ring_tx()
2928 i40e_ring_tx_exit_nolock(itrq); in i40e_ring_tx()
2929 itrq->itrq_tx_blocked = B_TRUE; in i40e_ring_tx()
2930 mutex_exit(&itrq->itrq_tx_lock); in i40e_ring_tx()