125 alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx,  in alloc_nm_rxq()  argument
140 	rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,  in alloc_nm_rxq()
141 	    &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);  in alloc_nm_rxq()
146 	rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,  in alloc_nm_rxq()
147 	    &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);  in alloc_nm_rxq()
151 	nm_rxq->vi = vi;  in alloc_nm_rxq()
152 	nm_rxq->nid = idx;  in alloc_nm_rxq()
153 	nm_rxq->iq_cidx = 0;  in alloc_nm_rxq()
154 	nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE;  in alloc_nm_rxq()
155 	nm_rxq->iq_gen = F_RSPD_GEN;  in alloc_nm_rxq()
156 	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;  in alloc_nm_rxq()
157 	nm_rxq->fl_sidx = na->num_rx_desc;  in alloc_nm_rxq()
158 	nm_rxq->fl_sidx2 = nm_rxq->fl_sidx;	/* copy for rxsync cacheline */  in alloc_nm_rxq()
159 	nm_rxq->intr_idx = intr_idx;  in alloc_nm_rxq()
160 	nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID;  in alloc_nm_rxq()
171 	    &nm_rxq->iq_abs_id, 0, "absolute id of the queue");  in alloc_nm_rxq()
173 	    &nm_rxq->iq_cntxt_id, 0, "SGE context id of the queue");  in alloc_nm_rxq()
175 	    &nm_rxq->iq_cidx, 0, "consumer index");  in alloc_nm_rxq()
183 	    &nm_rxq->fl_cntxt_id, 0, "SGE context id of the freelist");  in alloc_nm_rxq()
185 	    &nm_rxq->fl_cidx, 0, "consumer index");  in alloc_nm_rxq()
187 	    &nm_rxq->fl_pidx, 0, "producer index");  in alloc_nm_rxq()
193 free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)  in free_nm_rxq()  argument
200 	if (nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID)  in free_nm_rxq()
201 		free_nm_rxq_hwq(vi, nm_rxq);  in free_nm_rxq()
202 	MPASS(nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID);  in free_nm_rxq()
204 	free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,  in free_nm_rxq()
205 	    nm_rxq->iq_desc);  in free_nm_rxq()
206 	free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,  in free_nm_rxq()
207 	    nm_rxq->fl_desc);  in free_nm_rxq()
277 alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)  in alloc_nm_rxq_hwq()  argument
290 	MPASS(nm_rxq->iq_desc != NULL);  in alloc_nm_rxq_hwq()
291 	MPASS(nm_rxq->fl_desc != NULL);  in alloc_nm_rxq_hwq()
293 	bzero(nm_rxq->iq_desc, vi->qsize_rxq * IQ_ESIZE);  in alloc_nm_rxq_hwq()
294 	bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + sp->spg_len);  in alloc_nm_rxq_hwq()
301 	if (nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID)  in alloc_nm_rxq_hwq()
304 		c.iqid = htobe16(nm_rxq->iq_cntxt_id);  in alloc_nm_rxq_hwq()
305 		c.fl0id = htobe16(nm_rxq->fl_cntxt_id);  in alloc_nm_rxq_hwq()
307 		c.physiqid = htobe16(nm_rxq->iq_abs_id);  in alloc_nm_rxq_hwq()
310 	KASSERT(nm_rxq->intr_idx < sc->intr_count,  in alloc_nm_rxq_hwq()
311 	    ("%s: invalid direct intr_idx %d", __func__, nm_rxq->intr_idx));  in alloc_nm_rxq_hwq()
312 	v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);  in alloc_nm_rxq_hwq()
322 	c.iqaddr = htobe64(nm_rxq->iq_ba);  in alloc_nm_rxq_hwq()
340 	c.fl0addr = htobe64(nm_rxq->fl_ba);  in alloc_nm_rxq_hwq()
349 	nm_rxq->iq_cidx = 0;  in alloc_nm_rxq_hwq()
350 	MPASS(nm_rxq->iq_sidx == vi->qsize_rxq - sp->spg_len / IQ_ESIZE);  in alloc_nm_rxq_hwq()
351 	nm_rxq->iq_gen = F_RSPD_GEN;  in alloc_nm_rxq_hwq()
352 	nm_rxq->iq_cntxt_id = be16toh(c.iqid);  in alloc_nm_rxq_hwq()
353 	nm_rxq->iq_abs_id = be16toh(c.physiqid);  in alloc_nm_rxq_hwq()
354 	cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;  in alloc_nm_rxq_hwq()
359 	sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;  in alloc_nm_rxq_hwq()
361 	nm_rxq->fl_cntxt_id = be16toh(c.fl0id);  in alloc_nm_rxq_hwq()
362 	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;  in alloc_nm_rxq_hwq()
363 	nm_rxq->fl_db_saved = 0;  in alloc_nm_rxq_hwq()
365 	nm_rxq->fl_db_threshold = chip_id(sc) <= CHELSIO_T5 ? 8 : 4;  in alloc_nm_rxq_hwq()
366 	MPASS(nm_rxq->fl_sidx == na->num_rx_desc);  in alloc_nm_rxq_hwq()
367 	cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;  in alloc_nm_rxq_hwq()
372 	sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;  in alloc_nm_rxq_hwq()
374 	nm_rxq->fl_db_val = V_QID(nm_rxq->fl_cntxt_id) |  in alloc_nm_rxq_hwq()
378 		t4_sge_set_conm_context(sc, nm_rxq->iq_cntxt_id, cong_drop,  in alloc_nm_rxq_hwq()
383 	    V_INGRESSQID(nm_rxq->iq_cntxt_id) |  in alloc_nm_rxq_hwq()
390 free_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq)  in free_nm_rxq_hwq()  argument
396 	    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);  in free_nm_rxq_hwq()
399 		    __func__, nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, rc);  in free_nm_rxq_hwq()
400 	nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID;  in free_nm_rxq_hwq()
433 		V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));  in alloc_nm_txq_hwq()
517 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_simple_rss()  local
526 	for_each_nm_rxq(vi, j, nm_rxq) {  in cxgbe_netmap_simple_rss()
527 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_simple_rss()
528 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
531 			MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_simple_rss()
533 				defq = nm_rxq->iq_abs_id;  in cxgbe_netmap_simple_rss()
545 			for_each_nm_rxq(vi, j, nm_rxq) {  in cxgbe_netmap_simple_rss()
546 				nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_simple_rss()
547 				kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
552 					MPASS(nm_rxq->iq_cntxt_id !=  in cxgbe_netmap_simple_rss()
554 					vi->nm_rss[i++] = nm_rxq->iq_abs_id;  in cxgbe_netmap_simple_rss()
585 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_split_rss()  local
594 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_split_rss()
596 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_split_rss()
597 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_split_rss()
600 			MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
603 				dq[j] = nm_rxq->iq_abs_id;  in cxgbe_netmap_split_rss()
625 	nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq];  in cxgbe_netmap_split_rss()
628 			nm_state = atomic_load_int(&nm_rxq[j].nm_state);  in cxgbe_netmap_split_rss()
629 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
636 			MPASS(nm_rxq[j].iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
637 			if (dq_norss && defq == nm_rxq[j].iq_abs_id)  in cxgbe_netmap_split_rss()
639 			vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;  in cxgbe_netmap_split_rss()
646 			nm_state = atomic_load_int(&nm_rxq[j].nm_state);  in cxgbe_netmap_split_rss()
647 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
654 			MPASS(nm_rxq[j].iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_split_rss()
655 			if (dq_norss && defq == nm_rxq[j].iq_abs_id)  in cxgbe_netmap_split_rss()
657 			vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;  in cxgbe_netmap_split_rss()
692 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_on()  local
728 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_on()
729 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_on()
733 		alloc_nm_rxq_hwq(vi, nm_rxq);  in cxgbe_netmap_on()
734 		nm_rxq->fl_hwidx = hwidx;  in cxgbe_netmap_on()
740 		MPASS(na->num_rx_desc == nm_rxq->fl_sidx);  in cxgbe_netmap_on()
741 		for (j = 0; j < nm_rxq->fl_sidx; j++) {  in cxgbe_netmap_on()
746 			nm_rxq->fl_desc[j] = htobe64(ba | hwidx);  in cxgbe_netmap_on()
748 		j = nm_rxq->fl_pidx = nm_rxq->fl_sidx - 8;  in cxgbe_netmap_on()
753 		    nm_rxq->fl_db_val | V_PIDX(j));  in cxgbe_netmap_on()
755 		(void) atomic_cmpset_int(&nm_rxq->nm_state, NM_OFF, NM_ON);  in cxgbe_netmap_on()
783 	struct sge_nm_rxq *nm_rxq;  in cxgbe_netmap_off()  local
818 	for_each_nm_rxq(vi, i, nm_rxq) {  in cxgbe_netmap_off()
819 		nm_state = atomic_load_int(&nm_rxq->nm_state);  in cxgbe_netmap_off()
820 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_off()
826 		MPASS(nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID);  in cxgbe_netmap_off()
829 		    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);  in cxgbe_netmap_off()
836 		while (!atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_OFF))  in cxgbe_netmap_off()
1174 	struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq + kring->ring_id];  in cxgbe_netmap_rxsync()  local
1183 		kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);  in cxgbe_netmap_rxsync()
1187 	if (nm_rxq->fl_db_saved > 0 && starve_fl == 0) {  in cxgbe_netmap_rxsync()
1190 		    nm_rxq->fl_db_val | V_PIDX(nm_rxq->fl_db_saved));  in cxgbe_netmap_rxsync()
1191 		nm_rxq->fl_db_saved = 0;  in cxgbe_netmap_rxsync()
1199 		u_int fl_pidx = nm_rxq->fl_pidx;  in cxgbe_netmap_rxsync()
1202 		int i, dbinc = 0, hwidx = nm_rxq->fl_hwidx;  in cxgbe_netmap_rxsync()
1213 		IDXINCR(nm_rxq->fl_pidx, n, nm_rxq->fl_sidx2);  in cxgbe_netmap_rxsync()
1219 				nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);  in cxgbe_netmap_rxsync()
1221 				MPASS(fl_pidx <= nm_rxq->fl_sidx2);  in cxgbe_netmap_rxsync()
1224 			if (fl_pidx == nm_rxq->fl_sidx2) {  in cxgbe_netmap_rxsync()
1228 			if (++dbinc == nm_rxq->fl_db_threshold) {  in cxgbe_netmap_rxsync()
1231 					nm_rxq->fl_db_saved += dbinc;  in cxgbe_netmap_rxsync()
1234 					    nm_rxq->fl_db_val | V_PIDX(dbinc));  in cxgbe_netmap_rxsync()
1239 		MPASS(nm_rxq->fl_pidx == fl_pidx);  in cxgbe_netmap_rxsync()
1244 				nm_rxq->fl_db_saved += dbinc;  in cxgbe_netmap_rxsync()
1247 				    nm_rxq->fl_db_val | V_PIDX(dbinc));  in cxgbe_netmap_rxsync()
1327 service_nm_rxq(struct sge_nm_rxq *nm_rxq)  in service_nm_rxq()  argument
1329 	struct vi_info *vi = nm_rxq->vi;  in service_nm_rxq()
1333 	struct netmap_kring *kring = na->rx_rings[nm_rxq->nid];  in service_nm_rxq()
1335 	struct iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];  in service_nm_rxq()
1340 	uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);  in service_nm_rxq()
1345 	while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {  in service_nm_rxq()
1387 				if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))  in service_nm_rxq()
1392 				    __func__, opcode, nm_rxq);  in service_nm_rxq()
1403 			    __func__, G_RSPD_TYPE(d->rsp.u.type_gen), nm_rxq);  in service_nm_rxq()
1407 		if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {  in service_nm_rxq()
1408 			nm_rxq->iq_cidx = 0;  in service_nm_rxq()
1409 			d = &nm_rxq->iq_desc[0];  in service_nm_rxq()
1410 			nm_rxq->iq_gen ^= F_RSPD_GEN;  in service_nm_rxq()
1414 			atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);  in service_nm_rxq()
1415 			netmap_rx_irq(ifp, nm_rxq->nid, &work);  in service_nm_rxq()
1422 				IDXINCR(nm_rxq->fl_pidx, fl_credits * 8,  in service_nm_rxq()
1423 				    nm_rxq->fl_sidx);  in service_nm_rxq()
1425 				    nm_rxq->fl_db_val | V_PIDX(fl_credits));  in service_nm_rxq()
1430 			    V_INGRESSQID(nm_rxq->iq_cntxt_id) |  in service_nm_rxq()
1436 	atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);  in service_nm_rxq()
1439 		IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx);  in service_nm_rxq()
1441 		    nm_rxq->fl_db_val | V_PIDX(fl_credits));  in service_nm_rxq()
1443 		netmap_rx_irq(ifp, nm_rxq->nid, &work);  in service_nm_rxq()
1446 	    V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) |  in service_nm_rxq()