1114 t4_setup_vi_queues(struct vi_info *vi)  in t4_setup_vi_queues()  argument
1130 	struct adapter *sc = vi->adapter;  in t4_setup_vi_queues()
1131 	if_t ifp = vi->ifp;  in t4_setup_vi_queues()
1135 	intr_idx = vi->first_intr;  in t4_setup_vi_queues()
1143 		MPASS(vi->first_intr >= 0);  in t4_setup_vi_queues()
1150 		for_each_nm_rxq(vi, i, nm_rxq) {  in t4_setup_vi_queues()
1151 			rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i);  in t4_setup_vi_queues()
1157 		for_each_nm_txq(vi, i, nm_txq) {  in t4_setup_vi_queues()
1158 			iqidx = vi->first_nm_rxq + (i % vi->nnmrxq);  in t4_setup_vi_queues()
1159 			rc = alloc_nm_txq(vi, nm_txq, iqidx, i);  in t4_setup_vi_queues()
1174 	for_each_rxq(vi, i, rxq) {  in t4_setup_vi_queues()
1175 		rc = alloc_rxq(vi, rxq, i, intr_idx, maxp);  in t4_setup_vi_queues()
1183 		intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq);  in t4_setup_vi_queues()
1187 	for_each_ofld_rxq(vi, i, ofld_rxq) {  in t4_setup_vi_queues()
1188 		rc = alloc_ofld_rxq(vi, ofld_rxq, i, intr_idx, maxp);  in t4_setup_vi_queues()
1199 	for_each_txq(vi, i, txq) {  in t4_setup_vi_queues()
1200 		rc = alloc_txq(vi, txq, i);  in t4_setup_vi_queues()
1205 	for_each_ofld_txq(vi, i, ofld_txq) {  in t4_setup_vi_queues()
1206 		rc = alloc_ofld_txq(vi, ofld_txq, i);  in t4_setup_vi_queues()
1213 		t4_teardown_vi_queues(vi);  in t4_setup_vi_queues()
1222 t4_teardown_vi_queues(struct vi_info *vi)  in t4_teardown_vi_queues()  argument
1239 	if (if_getcapabilities(vi->ifp) & IFCAP_NETMAP) {  in t4_teardown_vi_queues()
1240 		for_each_nm_txq(vi, i, nm_txq) {  in t4_teardown_vi_queues()
1241 			free_nm_txq(vi, nm_txq);  in t4_teardown_vi_queues()
1244 		for_each_nm_rxq(vi, i, nm_rxq) {  in t4_teardown_vi_queues()
1245 			free_nm_rxq(vi, nm_rxq);  in t4_teardown_vi_queues()
1255 	for_each_txq(vi, i, txq) {  in t4_teardown_vi_queues()
1256 		free_txq(vi, txq);  in t4_teardown_vi_queues()
1259 	for_each_ofld_txq(vi, i, ofld_txq) {  in t4_teardown_vi_queues()
1260 		free_ofld_txq(vi, ofld_txq);  in t4_teardown_vi_queues()
1268 	for_each_rxq(vi, i, rxq) {  in t4_teardown_vi_queues()
1269 		free_rxq(vi, rxq);  in t4_teardown_vi_queues()
1272 	for_each_ofld_rxq(vi, i, ofld_rxq) {  in t4_teardown_vi_queues()
1273 		free_ofld_rxq(vi, ofld_rxq);  in t4_teardown_vi_queues()
1947 	struct vi_info *vi = if_getsoftc(ifp);  in eth_rx()  local
1989 	if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) &&  in eth_rx()
1999 		rc = pfil_mem_in(vi->pfil, frame, slen, ifp, &m0);  in eth_rx()
2261 	struct vi_info *vi = if_getsoftc(ifp);  in t4_update_fl_bufsize()  local
2262 	struct adapter *sc = vi->adapter;  in t4_update_fl_bufsize()
2271 	for_each_rxq(vi, i, rxq) {  in t4_update_fl_bufsize()
2281 	for_each_ofld_rxq(vi, i, ofld_rxq) {  in t4_update_fl_bufsize()
3147 	struct vi_info *vi = if_getsoftc(ifp);  in eth_tx()  local
3148 	struct adapter *sc = vi->adapter;  in eth_tx()
3210 			if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3224 				if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3231 				if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3280 			if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3324 			if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3331 			if (vi->flags & TX_USES_VM_WR)  in eth_tx()
3476 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,  in alloc_iq_fl()  argument
3481 	struct adapter *sc = vi->adapter;  in alloc_iq_fl()
3543 alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)  in alloc_iq_fl_hwq()  argument
3547 	struct adapter *sc = vi->adapter;  in alloc_iq_fl_hwq()
3548 	struct port_info *pi = vi->pi;  in alloc_iq_fl_hwq()
3578 	    V_FW_IQ_CMD_VIID(vi->viid) |  in alloc_iq_fl_hwq()
3781 	struct vi_info *vi = &sc->port[0]->vi[0];  in alloc_fwq()  local
3791 		rc = alloc_iq_fl(vi, fwq, NULL, &sc->ctx, sc->fwq_oid);  in alloc_fwq()
3802 		rc = alloc_iq_fl_hwq(vi, fwq, NULL);  in alloc_fwq()
3963 alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int idx, int intr_idx,  in alloc_rxq()  argument
3967 	struct adapter *sc = vi->adapter;  in alloc_rxq()
3968 	if_t ifp = vi->ifp;  in alloc_rxq()
3983 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->rxq_oid),  in alloc_rxq()
3987 		init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq,  in alloc_rxq()
3996 		    device_get_nameunit(vi->dev), idx);  in alloc_rxq()
3997 		init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);  in alloc_rxq()
3998 		rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid);  in alloc_rxq()
4000 			CH_ERR(vi, "failed to allocate rxq%d: %d\n", idx, rc);  in alloc_rxq()
4009 		add_rxq_sysctls(&vi->ctx, oid, rxq);  in alloc_rxq()
4014 		rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl);  in alloc_rxq()
4016 			CH_ERR(vi, "failed to create hw rxq%d: %d\n", idx, rc);  in alloc_rxq()
4045 free_rxq(struct vi_info *vi, struct sge_rxq *rxq)  in free_rxq()  argument
4049 		free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl);  in free_rxq()
4058 		free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl);  in free_rxq()
4094 alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,  in alloc_ofld_rxq()  argument
4098 	struct adapter *sc = vi->adapter;  in alloc_ofld_rxq()
4106 		oid = SYSCTL_ADD_NODE(&vi->ctx,  in alloc_ofld_rxq()
4107 		    SYSCTL_CHILDREN(vi->ofld_rxq_oid), OID_AUTO, name,  in alloc_ofld_rxq()
4110 		init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx,  in alloc_ofld_rxq()
4111 		    vi->qsize_rxq, intr_idx, ofld_cong_drop, IQ_OFLD);  in alloc_ofld_rxq()
4113 		    device_get_nameunit(vi->dev), idx);  in alloc_ofld_rxq()
4114 		init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);  in alloc_ofld_rxq()
4115 		rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx,  in alloc_ofld_rxq()
4118 			CH_ERR(vi, "failed to allocate ofld_rxq%d: %d\n", idx,  in alloc_ofld_rxq()
4130 		add_ofld_rxq_sysctls(&vi->ctx, oid, ofld_rxq);  in alloc_ofld_rxq()
4135 		rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl);  in alloc_ofld_rxq()
4137 			CH_ERR(vi, "failed to create hw ofld_rxq%d: %d\n", idx,  in alloc_ofld_rxq()
4150 free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)  in free_ofld_rxq()  argument
4154 		free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);  in free_ofld_rxq()
4160 		free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);  in free_ofld_rxq()
4302 eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)  in eth_eq_alloc()  argument
4316 	    F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));  in eth_eq_alloc()
4330 		device_printf(vi->dev,  in eth_eq_alloc()
4348 ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)  in ofld_eq_alloc()  argument
4375 		device_printf(vi->dev,  in ofld_eq_alloc()
4452 alloc_eq_hwq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)  in alloc_eq_hwq()  argument
4471 		rc = eth_eq_alloc(sc, vi, eq);  in alloc_eq_hwq()
4476 		rc = ofld_eq_alloc(sc, vi, eq);  in alloc_eq_hwq()
4513 free_eq_hwq(struct adapter *sc, struct vi_info *vi __unused, struct sge_eq *eq)  in free_eq_hwq()
4544 alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq,  in alloc_wrq()  argument
4602 alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx)  in alloc_txq()  argument
4605 	struct port_info *pi = vi->pi;  in alloc_txq()
4606 	struct adapter *sc = vi->adapter;  in alloc_txq()
4616 		oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->txq_oid),  in alloc_txq()
4620 		iqidx = vi->first_rxq + (idx % vi->nrxq);  in alloc_txq()
4622 		    device_get_nameunit(vi->dev), idx);  in alloc_txq()
4623 		init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->port_id,  in alloc_txq()
4629 			CH_ERR(vi, "failed to allocate mp_ring for txq%d: %d\n",  in alloc_txq()
4636 		rc = alloc_eq(sc, eq, &vi->ctx, oid);  in alloc_txq()
4638 			CH_ERR(vi, "failed to allocate txq%d: %d\n", idx, rc);  in alloc_txq()
4646 		txq->ifp = vi->ifp;  in alloc_txq()
4651 		add_txq_sysctls(vi, &vi->ctx, oid, txq);  in alloc_txq()
4656 		rc = alloc_eq_hwq(sc, vi, eq);  in alloc_txq()
4658 			CH_ERR(vi, "failed to create hw txq%d: %d\n", idx, rc);  in alloc_txq()
4676 		if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF))  in alloc_txq()
4679 		if (vi->flags & TX_USES_VM_WR)  in alloc_txq()
4685 			    V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));  in alloc_txq()
4697 free_txq(struct vi_info *vi, struct sge_txq *txq)  in free_txq()  argument
4699 	struct adapter *sc = vi->adapter;  in free_txq()
4720 add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx,  in add_txq_sysctls()  argument
4729 	sc = vi->adapter;  in add_txq_sysctls()
4735 	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, txq - sc->sge.txq,  in add_txq_sysctls()
4818 alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)  in alloc_ofld_txq()  argument
4821 	struct port_info *pi = vi->pi;  in alloc_ofld_txq()
4822 	struct adapter *sc = vi->adapter;  in alloc_ofld_txq()
4828 	MPASS(idx < vi->nofldtxq);  in alloc_ofld_txq()
4832 		oid = SYSCTL_ADD_NODE(&vi->ctx,  in alloc_ofld_txq()
4833 		    SYSCTL_CHILDREN(vi->ofld_txq_oid), OID_AUTO, name,  in alloc_ofld_txq()
4837 		    device_get_nameunit(vi->dev), idx);  in alloc_ofld_txq()
4838 		if (vi->nofldrxq > 0) {  in alloc_ofld_txq()
4839 			iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq);  in alloc_ofld_txq()
4840 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->port_id,  in alloc_ofld_txq()
4843 			iqidx = vi->first_rxq + (idx % vi->nrxq);  in alloc_ofld_txq()
4844 			init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->port_id,  in alloc_ofld_txq()
4848 		rc = alloc_wrq(sc, vi, &ofld_txq->wrq, &vi->ctx, oid);  in alloc_ofld_txq()
4850 			CH_ERR(vi, "failed to allocate ofld_txq%d: %d\n", idx,  in alloc_ofld_txq()
4865 		add_ofld_txq_sysctls(&vi->ctx, oid, ofld_txq);  in alloc_ofld_txq()
4872 		rc = alloc_eq_hwq(sc, vi, eq);  in alloc_ofld_txq()
4874 			CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx,  in alloc_ofld_txq()
4888 free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)  in free_ofld_txq()  argument
4890 	struct adapter *sc = vi->adapter;  in free_ofld_txq()
6565     struct vi_info *vi)  in send_etid_flowc_wr()  argument
6863 		struct vi_info *vi = if_getsoftc(ifp);  in ethofld_transmit()  local
6864 		struct port_info *pi = vi->pi;  in ethofld_transmit()
6866 		const uint32_t rss_mask = vi->rss_size - 1;  in ethofld_transmit()
6869 		cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];  in ethofld_transmit()
6875 		cst->iqid = vi->rss[rss_hash & rss_mask];  in ethofld_transmit()
6876 		cst->eo_txq += rss_hash % vi->nofldtxq;  in ethofld_transmit()
6877 		rc = send_etid_flowc_wr(cst, pi, vi);  in ethofld_transmit()