213 alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx)  in alloc_nm_txq()  argument
225 	rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map,  in alloc_nm_txq()
226 	    &nm_txq->ba, (void **)&nm_txq->desc);  in alloc_nm_txq()
230 	nm_txq->pidx = nm_txq->cidx = 0;  in alloc_nm_txq()
231 	nm_txq->sidx = na->num_tx_desc;  in alloc_nm_txq()
232 	nm_txq->nid = idx;  in alloc_nm_txq()
233 	nm_txq->iqidx = iqidx;  in alloc_nm_txq()
234 	nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |  in alloc_nm_txq()
238 		nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR));  in alloc_nm_txq()
240 		nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));  in alloc_nm_txq()
241 	nm_txq->cntxt_id = INVALID_NM_TXQ_CNTXT_ID;  in alloc_nm_txq()
249 	    &nm_txq->cntxt_id, 0, "SGE context id of the queue");  in alloc_nm_txq()
251 	    &nm_txq->cidx, 0, "consumer index");  in alloc_nm_txq()
253 	    &nm_txq->pidx, 0, "producer index");  in alloc_nm_txq()
259 free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq)  in free_nm_txq()  argument
266 	if (nm_txq->cntxt_id != INVALID_NM_TXQ_CNTXT_ID)  in free_nm_txq()
267 		free_nm_txq_hwq(vi, nm_txq);  in free_nm_txq()
268 	MPASS(nm_txq->cntxt_id == INVALID_NM_TXQ_CNTXT_ID);  in free_nm_txq()
270 	free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba,  in free_nm_txq()
271 	    nm_txq->desc);  in free_nm_txq()
405 alloc_nm_txq_hwq(struct vi_info *vi, struct sge_nm_txq *nm_txq)  in alloc_nm_txq_hwq()  argument
414 	MPASS(nm_txq->desc != NULL);  in alloc_nm_txq_hwq()
417 	bzero(nm_txq->desc, len);  in alloc_nm_txq_hwq()
424 	if (nm_txq->cntxt_id == INVALID_NM_TXQ_CNTXT_ID)  in alloc_nm_txq_hwq()
427 		c.eqid_pkd = htobe32(V_FW_EQ_ETH_CMD_EQID(nm_txq->cntxt_id));  in alloc_nm_txq_hwq()
433 		V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));  in alloc_nm_txq_hwq()
439 	c.eqaddr = htobe64(nm_txq->ba);  in alloc_nm_txq_hwq()
448 	nm_txq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));  in alloc_nm_txq_hwq()
449 	cntxt_id = nm_txq->cntxt_id - sc->sge.eq_start;  in alloc_nm_txq_hwq()
453 	sc->sge.eqmap[cntxt_id] = (void *)nm_txq;  in alloc_nm_txq_hwq()
455 	nm_txq->pidx = nm_txq->cidx = 0;  in alloc_nm_txq_hwq()
456 	MPASS(nm_txq->sidx == na->num_tx_desc);  in alloc_nm_txq_hwq()
457 	nm_txq->equiqidx = nm_txq->equeqidx = nm_txq->dbidx = 0;  in alloc_nm_txq_hwq()
459 	nm_txq->doorbells = sc->doorbells;  in alloc_nm_txq_hwq()
460 	if (isset(&nm_txq->doorbells, DOORBELL_UDB) ||  in alloc_nm_txq_hwq()
461 	    isset(&nm_txq->doorbells, DOORBELL_UDBWC) ||  in alloc_nm_txq_hwq()
462 	    isset(&nm_txq->doorbells, DOORBELL_WCWR)) {  in alloc_nm_txq_hwq()
468 		udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;  in alloc_nm_txq_hwq()
469 		nm_txq->udb_qid = nm_txq->cntxt_id & mask;  in alloc_nm_txq_hwq()
470 		if (nm_txq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)  in alloc_nm_txq_hwq()
471 			clrbit(&nm_txq->doorbells, DOORBELL_WCWR);  in alloc_nm_txq_hwq()
473 			udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;  in alloc_nm_txq_hwq()
474 			nm_txq->udb_qid = 0;  in alloc_nm_txq_hwq()
476 		nm_txq->udb = (volatile void *)udb;  in alloc_nm_txq_hwq()
484 		    V_FW_PARAMS_PARAM_YZ(nm_txq->cntxt_id);  in alloc_nm_txq_hwq()
490 			    nm_txq->cntxt_id, rc);  in alloc_nm_txq_hwq()
499 free_nm_txq_hwq(struct vi_info *vi, struct sge_nm_txq *nm_txq)  in free_nm_txq_hwq()  argument
504 	rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, nm_txq->cntxt_id);  in free_nm_txq_hwq()
507 		    nm_txq->cntxt_id, rc);  in free_nm_txq_hwq()
508 	nm_txq->cntxt_id = INVALID_NM_TXQ_CNTXT_ID;  in free_nm_txq_hwq()
693 	struct sge_nm_txq *nm_txq;  in cxgbe_netmap_on()  local
758 	for_each_nm_txq(vi, i, nm_txq) {  in cxgbe_netmap_on()
759 		kring = na->tx_rings[nm_txq->nid];  in cxgbe_netmap_on()
763 		alloc_nm_txq_hwq(vi, nm_txq);  in cxgbe_netmap_on()
782 	struct sge_nm_txq *nm_txq;  in cxgbe_netmap_off()  local
800 	for_each_nm_txq(vi, i, nm_txq) {  in cxgbe_netmap_off()
801 		kring = na->tx_rings[nm_txq->nid];  in cxgbe_netmap_off()
804 		MPASS(nm_txq->cntxt_id != INVALID_NM_TXQ_CNTXT_ID);  in cxgbe_netmap_off()
806 		rc = -t4_eth_eq_stop(sc, sc->mbox, sc->pf, 0, nm_txq->cntxt_id);  in cxgbe_netmap_off()
910 ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)  in ring_nm_txq_db()  argument
913 	u_int db = nm_txq->doorbells;  in ring_nm_txq_db()
915 	MPASS(nm_txq->pidx != nm_txq->dbidx);  in ring_nm_txq_db()
917 	n = NMIDXDIFF(nm_txq, dbidx);  in ring_nm_txq_db()
924 		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));  in ring_nm_txq_db()
935 		KASSERT(nm_txq->udb_qid == 0 && n == 1,  in ring_nm_txq_db()
937 		    __func__, nm_txq->doorbells, n, nm_txq->pidx, nm_txq));  in ring_nm_txq_db()
939 		dst = (volatile void *)((uintptr_t)nm_txq->udb +  in ring_nm_txq_db()
941 		src = (void *)&nm_txq->desc[nm_txq->dbidx];  in ring_nm_txq_db()
942 		while (src != (void *)&nm_txq->desc[nm_txq->dbidx + 1])  in ring_nm_txq_db()
949 		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));  in ring_nm_txq_db()
955 		    V_QID(nm_txq->cntxt_id) | V_PIDX(n));  in ring_nm_txq_db()
958 	nm_txq->dbidx = nm_txq->pidx;  in ring_nm_txq_db()
966 cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,  in cxgbe_nm_tx()  argument
972 	struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];  in cxgbe_nm_tx()
983 		wr = (void *)&nm_txq->desc[nm_txq->pidx];  in cxgbe_nm_tx()
984 		wr->op_pkd = nm_txq->op_pkd;  in cxgbe_nm_tx()
996 			cpl->ctrl0 = nm_txq->cpl_ctrl0;  in cxgbe_nm_tx()
1017 		nm_txq->pidx += npkt_to_ndesc(n);  in cxgbe_nm_tx()
1018 		MPASS(nm_txq->pidx <= nm_txq->sidx);  in cxgbe_nm_tx()
1019 		if (__predict_false(nm_txq->pidx == nm_txq->sidx)) {  in cxgbe_nm_tx()
1025 			nm_txq->pidx = 0;  in cxgbe_nm_tx()
1031 			    NMIDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {  in cxgbe_nm_tx()
1034 				nm_txq->equeqidx = nm_txq->pidx;  in cxgbe_nm_tx()
1035 				nm_txq->equiqidx = nm_txq->pidx;  in cxgbe_nm_tx()
1036 			} else if (NMIDXDIFF(nm_txq, equeqidx) >= 64) {  in cxgbe_nm_tx()
1038 				nm_txq->equeqidx = nm_txq->pidx;  in cxgbe_nm_tx()
1040 			ring_nm_txq_db(sc, nm_txq);  in cxgbe_nm_tx()
1043 		if (NMIDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC) {  in cxgbe_nm_tx()
1044 			if (NMIDXDIFF(nm_txq, equeqidx) >= 64) {  in cxgbe_nm_tx()
1046 				nm_txq->equeqidx = nm_txq->pidx;  in cxgbe_nm_tx()
1048 			ring_nm_txq_db(sc, nm_txq);  in cxgbe_nm_tx()
1058 contiguous_ndesc_available(struct sge_nm_txq *nm_txq)  in contiguous_ndesc_available()  argument
1061 	if (nm_txq->cidx > nm_txq->pidx)  in contiguous_ndesc_available()
1062 		return (nm_txq->cidx - nm_txq->pidx - 1);  in contiguous_ndesc_available()
1063 	else if (nm_txq->cidx > 0)  in contiguous_ndesc_available()
1064 		return (nm_txq->sidx - nm_txq->pidx);  in contiguous_ndesc_available()
1066 		return (nm_txq->sidx - nm_txq->pidx - 1);  in contiguous_ndesc_available()
1070 reclaim_nm_tx_desc(struct sge_nm_txq *nm_txq)  in reclaim_nm_tx_desc()  argument
1072 	struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];  in reclaim_nm_tx_desc()
1079 	while (nm_txq->cidx != hw_cidx) {  in reclaim_nm_tx_desc()
1080 		wr = (void *)&nm_txq->desc[nm_txq->cidx];  in reclaim_nm_tx_desc()
1088 		nm_txq->cidx += npkt_to_ndesc(wr->npkt);  in reclaim_nm_tx_desc()
1095 		MPASS(nm_txq->cidx <= nm_txq->sidx);  in reclaim_nm_tx_desc()
1096 		if (__predict_false(nm_txq->cidx == nm_txq->sidx))  in reclaim_nm_tx_desc()
1097 			nm_txq->cidx = 0;  in reclaim_nm_tx_desc()
1110 	struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[vi->first_nm_txq + kring->ring_id];  in cxgbe_netmap_txsync()  local
1125 		reclaimed += reclaim_nm_tx_desc(nm_txq);  in cxgbe_netmap_txsync()
1126 		ndesc_remaining = contiguous_ndesc_available(nm_txq);  in cxgbe_netmap_txsync()
1147 		cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);  in cxgbe_netmap_txsync()
1151 	MPASS(nm_txq->dbidx == nm_txq->pidx);  in cxgbe_netmap_txsync()
1157 		reclaimed += reclaim_nm_tx_desc(nm_txq);  in cxgbe_netmap_txsync()
1317 	struct sge_nm_txq *nm_txq;  in handle_nm_sge_egr_update()  local
1321 	nm_txq = (void *)sc->sge.eqmap[G_EGR_QID(oq) - sc->sge.eq_start];  in handle_nm_sge_egr_update()
1323 	netmap_tx_irq(ifp, nm_txq->nid);  in handle_nm_sge_egr_update()