Lines Matching refs:cst
6564 send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, in send_etid_flowc_wr() argument
6571 mtx_assert(&cst->lock, MA_OWNED); in send_etid_flowc_wr()
6572 MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) == in send_etid_flowc_wr()
6575 flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie); in send_etid_flowc_wr()
6583 V_FW_WR_FLOWID(cst->etid)); in send_etid_flowc_wr()
6591 flowc->mnemval[3].val = htobe32(cst->iqid); in send_etid_flowc_wr()
6595 flowc->mnemval[5].val = htobe32(cst->schedcl); in send_etid_flowc_wr()
6597 commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie); in send_etid_flowc_wr()
6599 cst->flags &= ~EO_FLOWC_PENDING; in send_etid_flowc_wr()
6600 cst->flags |= EO_FLOWC_RPL_PENDING; in send_etid_flowc_wr()
6601 MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */ in send_etid_flowc_wr()
6602 cst->tx_credits -= ETID_FLOWC_LEN16; in send_etid_flowc_wr()
6611 send_etid_flush_wr(struct cxgbe_rate_tag *cst) in send_etid_flush_wr() argument
6616 mtx_assert(&cst->lock, MA_OWNED); in send_etid_flush_wr()
6618 flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie); in send_etid_flush_wr()
6626 V_FW_WR_FLOWID(cst->etid)); in send_etid_flush_wr()
6628 commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie); in send_etid_flush_wr()
6630 cst->flags |= EO_FLUSH_RPL_PENDING; in send_etid_flush_wr()
6631 MPASS(cst->tx_credits >= ETID_FLUSH_LEN16); in send_etid_flush_wr()
6632 cst->tx_credits -= ETID_FLUSH_LEN16; in send_etid_flush_wr()
6633 cst->ncompl++; in send_etid_flush_wr()
6637 write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, in write_ethofld_wr() argument
6649 mtx_assert(&cst->lock, MA_OWNED); in write_ethofld_wr()
6667 V_FW_WR_FLOWID(cst->etid)); in write_ethofld_wr()
6719 ctrl1 = csum_to_ctrl(cst->adapter, m0); in write_ethofld_wr()
6728 cpl->ctrl0 = cst->ctrl0; in write_ethofld_wr()
6787 ethofld_tx(struct cxgbe_rate_tag *cst) in ethofld_tx() argument
6794 mtx_assert(&cst->lock, MA_OWNED); in ethofld_tx()
6796 while ((m = mbufq_first(&cst->pending_tx)) != NULL) { in ethofld_tx()
6802 if (next_credits > cst->tx_credits) { in ethofld_tx()
6808 MPASS(cst->ncompl > 0); in ethofld_tx()
6811 wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie); in ethofld_tx()
6814 MPASS(cst->ncompl > 0); in ethofld_tx()
6817 cst->tx_credits -= next_credits; in ethofld_tx()
6818 cst->tx_nocompl += next_credits; in ethofld_tx()
6819 compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2; in ethofld_tx()
6820 ETHER_BPF_MTAP(cst->com.ifp, m); in ethofld_tx()
6821 write_ethofld_wr(cst, wr, m, compl); in ethofld_tx()
6822 commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie); in ethofld_tx()
6824 cst->ncompl++; in ethofld_tx()
6825 cst->tx_nocompl = 0; in ethofld_tx()
6827 (void) mbufq_dequeue(&cst->pending_tx); in ethofld_tx()
6841 m_snd_tag_rele(&cst->com); in ethofld_tx()
6843 mbufq_enqueue(&cst->pending_fwack, m); in ethofld_tx()
6851 struct cxgbe_rate_tag *cst; in ethofld_transmit() local
6857 cst = mst_to_crt(m0->m_pkthdr.snd_tag); in ethofld_transmit()
6859 mtx_lock(&cst->lock); in ethofld_transmit()
6860 MPASS(cst->flags & EO_SND_TAG_REF); in ethofld_transmit()
6862 if (__predict_false(cst->flags & EO_FLOWC_PENDING)) { in ethofld_transmit()
6869 cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; in ethofld_transmit()
6875 cst->iqid = vi->rss[rss_hash & rss_mask]; in ethofld_transmit()
6876 cst->eo_txq += rss_hash % vi->nofldtxq; in ethofld_transmit()
6877 rc = send_etid_flowc_wr(cst, pi, vi); in ethofld_transmit()
6882 if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { in ethofld_transmit()
6887 mbufq_enqueue(&cst->pending_tx, m0); in ethofld_transmit()
6888 cst->plen += m0->m_pkthdr.len; in ethofld_transmit()
6896 m_snd_tag_ref(&cst->com); in ethofld_transmit()
6897 ethofld_tx(cst); in ethofld_transmit()
6898 mtx_unlock(&cst->lock); in ethofld_transmit()
6899 m_snd_tag_rele(&cst->com); in ethofld_transmit()
6903 mtx_unlock(&cst->lock); in ethofld_transmit()
6915 struct cxgbe_rate_tag *cst; in ethofld_fw4_ack() local
6918 cst = lookup_etid(sc, etid); in ethofld_fw4_ack()
6919 mtx_lock(&cst->lock); in ethofld_fw4_ack()
6920 if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) { in ethofld_fw4_ack()
6923 cst->flags &= ~EO_FLOWC_RPL_PENDING; in ethofld_fw4_ack()
6926 KASSERT(cst->ncompl > 0, in ethofld_fw4_ack()
6928 __func__, etid, cst)); in ethofld_fw4_ack()
6929 cst->ncompl--; in ethofld_fw4_ack()
6932 m = mbufq_dequeue(&cst->pending_fwack); in ethofld_fw4_ack()
6938 MPASS((cst->flags & in ethofld_fw4_ack()
6942 MPASS(cst->tx_credits + cpl->credits == cst->tx_total); in ethofld_fw4_ack()
6943 MPASS(cst->ncompl == 0); in ethofld_fw4_ack()
6945 cst->flags &= ~EO_FLUSH_RPL_PENDING; in ethofld_fw4_ack()
6946 cst->tx_credits += cpl->credits; in ethofld_fw4_ack()
6947 cxgbe_rate_tag_free_locked(cst); in ethofld_fw4_ack()
6957 cst->plen -= m->m_pkthdr.len; in ethofld_fw4_ack()
6961 cst->tx_credits += cpl->credits; in ethofld_fw4_ack()
6962 MPASS(cst->tx_credits <= cst->tx_total); in ethofld_fw4_ack()
6964 if (cst->flags & EO_SND_TAG_REF) { in ethofld_fw4_ack()
6969 m_snd_tag_ref(&cst->com); in ethofld_fw4_ack()
6970 m = mbufq_first(&cst->pending_tx); in ethofld_fw4_ack()
6971 if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m)) in ethofld_fw4_ack()
6972 ethofld_tx(cst); in ethofld_fw4_ack()
6973 mtx_unlock(&cst->lock); in ethofld_fw4_ack()
6974 m_snd_tag_rele(&cst->com); in ethofld_fw4_ack()
6981 MPASS(mbufq_first(&cst->pending_tx) == NULL); in ethofld_fw4_ack()
6982 mtx_unlock(&cst->lock); in ethofld_fw4_ack()