516 	struct netmap_kring *kring;  in cxgbe_netmap_simple_rss()  local
528 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
529 		if ((nm_state != NM_OFF && !nm_kring_pending_off(kring)) ||  in cxgbe_netmap_simple_rss()
530 		    (nm_state == NM_OFF && nm_kring_pending_on(kring))) {  in cxgbe_netmap_simple_rss()
547 				kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_simple_rss()
549 				    !nm_kring_pending_off(kring)) ||  in cxgbe_netmap_simple_rss()
551 				    nm_kring_pending_on(kring))) {  in cxgbe_netmap_simple_rss()
584 	struct netmap_kring *kring;  in cxgbe_netmap_split_rss()  local
597 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_split_rss()
598 		if ((nm_state != NM_OFF && !nm_kring_pending_off(kring)) ||  in cxgbe_netmap_split_rss()
599 		    (nm_state == NM_OFF && nm_kring_pending_on(kring))) {  in cxgbe_netmap_split_rss()
629 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
631 			    !nm_kring_pending_on(kring)) ||  in cxgbe_netmap_split_rss()
633 			    nm_kring_pending_off(kring))) {  in cxgbe_netmap_split_rss()
647 			kring = na->rx_rings[nm_rxq[j].nid];  in cxgbe_netmap_split_rss()
649 			    !nm_kring_pending_on(kring)) ||  in cxgbe_netmap_split_rss()
651 			    nm_kring_pending_off(kring))) {  in cxgbe_netmap_split_rss()
691 	struct netmap_kring *kring;  in cxgbe_netmap_on()  local
729 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_on()
730 		if (!nm_kring_pending_on(kring))  in cxgbe_netmap_on()
759 		kring = na->tx_rings[nm_txq->nid];  in cxgbe_netmap_on()
760 		if (!nm_kring_pending_on(kring))  in cxgbe_netmap_on()
780 	struct netmap_kring *kring;  in cxgbe_netmap_off()  local
801 		kring = na->tx_rings[nm_txq->nid];  in cxgbe_netmap_off()
802 		if (!nm_kring_pending_off(kring))  in cxgbe_netmap_off()
814 		kring->rhead = kring->rcur = kring->nr_hwcur = 0;  in cxgbe_netmap_off()
815 		kring->rtail = kring->nr_hwtail = kring->nkr_num_slots - 1;  in cxgbe_netmap_off()
820 		kring = na->rx_rings[nm_rxq->nid];  in cxgbe_netmap_off()
821 		if (nm_state != NM_OFF && !nm_kring_pending_off(kring))  in cxgbe_netmap_off()
823 		if (!nm_kring_pending_off(kring))  in cxgbe_netmap_off()
840 		kring->rhead = kring->rcur = kring->nr_hwcur = 0;  in cxgbe_netmap_off()
841 		kring->rtail = kring->nr_hwtail = 0;  in cxgbe_netmap_off()
967     struct netmap_kring *kring, int npkt, int npkt_remaining)  in cxgbe_nm_tx()  argument
969 	struct netmap_ring *ring = kring->ring;  in cxgbe_nm_tx()
971 	const u_int lim = kring->nkr_num_slots - 1;  in cxgbe_nm_tx()
992 			slot = &ring->slot[kring->nr_hwcur];  in cxgbe_nm_tx()
993 			PNMB(kring->na, slot, &ba);  in cxgbe_nm_tx()
1006 			usgl->addr0 = htobe64(ba + nm_get_offset(kring, slot));  in cxgbe_nm_tx()
1012 			kring->nr_hwcur = nm_next(kring->nr_hwcur, lim);  in cxgbe_nm_tx()
1104 cxgbe_netmap_txsync(struct netmap_kring *kring, int flags)  in cxgbe_netmap_txsync()  argument
1106 	struct netmap_adapter *na = kring->na;  in cxgbe_netmap_txsync()
1110 	struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[vi->first_nm_txq + kring->ring_id];  in cxgbe_netmap_txsync()
1111 	const u_int head = kring->rhead;  in cxgbe_netmap_txsync()
1122 	npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :  in cxgbe_netmap_txsync()
1123 	    kring->nkr_num_slots - kring->nr_hwcur + head;  in cxgbe_netmap_txsync()
1147 		cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);  in cxgbe_netmap_txsync()
1150 	MPASS(kring->nr_hwcur == head);  in cxgbe_netmap_txsync()
1156 	if (reclaimed || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {  in cxgbe_netmap_txsync()
1158 		kring->nr_hwtail += reclaimed;  in cxgbe_netmap_txsync()
1159 		if (kring->nr_hwtail >= kring->nkr_num_slots)  in cxgbe_netmap_txsync()
1160 			kring->nr_hwtail -= kring->nkr_num_slots;  in cxgbe_netmap_txsync()
1167 cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags)  in cxgbe_netmap_rxsync()  argument
1169 	struct netmap_adapter *na = kring->na;  in cxgbe_netmap_rxsync()
1170 	struct netmap_ring *ring = kring->ring;  in cxgbe_netmap_rxsync()
1174 	struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq + kring->ring_id];  in cxgbe_netmap_rxsync()
1175 	u_int const head = kring->rhead;  in cxgbe_netmap_rxsync()
1177 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;  in cxgbe_netmap_rxsync()
1183 		kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);  in cxgbe_netmap_rxsync()
1184 		kring->nr_kflags &= ~NKR_PENDINTR;  in cxgbe_netmap_rxsync()
1195 	n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :  in cxgbe_netmap_rxsync()
1196 	    kring->nkr_num_slots - kring->nr_hwcur + head;  in cxgbe_netmap_rxsync()
1212 		IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);  in cxgbe_netmap_rxsync()
1333 	struct netmap_kring *kring = na->rx_rings[nm_rxq->nid];  in service_nm_rxq()  local
1334 	struct netmap_ring *ring = kring->ring;  in service_nm_rxq()
1376 				nm_write_offset(kring, &ring->slot[fl_cidx],  in service_nm_rxq()