539 	struct mgb_ring_data *rdata;  in mgb_tx_queues_alloc()  local
544 	rdata = &sc->tx_ring_data;  in mgb_tx_queues_alloc()
548 		rdata->ring = (struct mgb_ring_desc *) vaddrs[q * ntxqs + 0];  in mgb_tx_queues_alloc()
549 		rdata->ring_bus_addr = paddrs[q * ntxqs + 0];  in mgb_tx_queues_alloc()
552 		rdata->head_wb = (uint32_t *) vaddrs[q * ntxqs + 1];  in mgb_tx_queues_alloc()
553 		rdata->head_wb_bus_addr = paddrs[q * ntxqs + 1];  in mgb_tx_queues_alloc()
563 	struct mgb_ring_data *rdata;  in mgb_rx_queues_alloc()  local
568 	rdata = &sc->rx_ring_data;  in mgb_rx_queues_alloc()
572 		rdata->ring = (struct mgb_ring_desc *) vaddrs[q * nrxqs + 0];  in mgb_rx_queues_alloc()
573 		rdata->ring_bus_addr = paddrs[q * nrxqs + 0];  in mgb_rx_queues_alloc()
576 		rdata->head_wb = (uint32_t *) vaddrs[q * nrxqs + 1];  in mgb_rx_queues_alloc()
577 		rdata->head_wb_bus_addr = paddrs[q * nrxqs + 1];  in mgb_rx_queues_alloc()
962 	struct mgb_ring_data *rdata;  in mgb_isc_txd_encap()  local
971 	rdata = &sc->tx_ring_data;  in mgb_isc_txd_encap()
980 		txd = &rdata->ring[pidx];  in mgb_isc_txd_encap()
1006 	struct mgb_ring_data *rdata;  in mgb_isc_txd_flush()  local
1010 	rdata = &sc->tx_ring_data;  in mgb_isc_txd_flush()
1012 	if (rdata->last_tail != pidx) {  in mgb_isc_txd_flush()
1013 		rdata->last_tail = pidx;  in mgb_isc_txd_flush()
1014 		CSR_WRITE_REG(sc, MGB_DMA_TX_TAIL(txqid), rdata->last_tail);  in mgb_isc_txd_flush()
1023 	struct mgb_ring_data *rdata;  in mgb_isc_txd_credits_update()  local
1036 	rdata = &sc->tx_ring_data;  in mgb_isc_txd_credits_update()
1038 	while (*(rdata->head_wb) != rdata->last_head) {  in mgb_isc_txd_credits_update()
1042 		txd = &rdata->ring[rdata->last_head];  in mgb_isc_txd_credits_update()
1044 		rdata->last_head = MGB_NEXT_RING_IDX(rdata->last_head);  in mgb_isc_txd_credits_update()
1055 	struct mgb_ring_data *rdata;  in mgb_isc_rxd_available()  local
1062 	rdata = &sc->rx_ring_data;  in mgb_isc_rxd_available()
1063 	for (; idx != *(rdata->head_wb); idx = MGB_NEXT_RING_IDX(idx)) {  in mgb_isc_rxd_available()
1076 	struct mgb_ring_data *rdata;  in mgb_isc_rxd_pkt_get()  local
1084 	rdata = &sc->rx_ring_data;  in mgb_isc_rxd_pkt_get()
1086 	while (*(rdata->head_wb) != rdata->last_head) {  in mgb_isc_rxd_pkt_get()
1088 		rxd = rdata->ring[rdata->last_head];  in mgb_isc_rxd_pkt_get()
1115 		ri->iri_frags[0].irf_idx = rdata->last_head;  in mgb_isc_rxd_pkt_get()
1119 		rdata->last_head = MGB_NEXT_RING_IDX(rdata->last_head);  in mgb_isc_rxd_pkt_get()
1132 	struct mgb_ring_data *rdata;  in mgb_isc_rxd_refill()  local
1147 	rdata = &sc->rx_ring_data;  in mgb_isc_rxd_refill()
1151 		rxd = &rdata->ring[idx];  in mgb_isc_rxd_refill()
1249 	struct mgb_ring_data *rdata;  in mgb_dma_rx_ring_init()  local
1252 	rdata = &sc->rx_ring_data;  in mgb_dma_rx_ring_init()
1258 	if (rdata->ring_bus_addr == 0) {  in mgb_dma_rx_ring_init()
1264 	    CSR_TRANSLATE_ADDR_HIGH32(rdata->ring_bus_addr));  in mgb_dma_rx_ring_init()
1266 	    CSR_TRANSLATE_ADDR_LOW32(rdata->ring_bus_addr));  in mgb_dma_rx_ring_init()
1269 	if (rdata->head_wb_bus_addr == 0) {  in mgb_dma_rx_ring_init()
1274 	    CSR_TRANSLATE_ADDR_HIGH32(rdata->head_wb_bus_addr));  in mgb_dma_rx_ring_init()
1276 	    CSR_TRANSLATE_ADDR_LOW32(rdata->head_wb_bus_addr));  in mgb_dma_rx_ring_init()
1291 	rdata->last_head = CSR_READ_REG(sc, MGB_DMA_RX_HEAD(channel));  in mgb_dma_rx_ring_init()
1313 	struct mgb_ring_data *rdata;  in mgb_dma_tx_ring_init()  local
1316 	rdata = &sc->tx_ring_data;  in mgb_dma_tx_ring_init()
1335 	if (rdata->ring_bus_addr == 0) {  in mgb_dma_tx_ring_init()
1340 	    CSR_TRANSLATE_ADDR_HIGH32(rdata->ring_bus_addr));  in mgb_dma_tx_ring_init()
1342 	    CSR_TRANSLATE_ADDR_LOW32(rdata->ring_bus_addr));  in mgb_dma_tx_ring_init()
1355 	if (rdata->head_wb_bus_addr == 0) {  in mgb_dma_tx_ring_init()
1360 	    CSR_TRANSLATE_ADDR_HIGH32(rdata->head_wb_bus_addr));  in mgb_dma_tx_ring_init()
1362 	    CSR_TRANSLATE_ADDR_LOW32(rdata->head_wb_bus_addr));  in mgb_dma_tx_ring_init()
1364 	rdata->last_head = CSR_READ_REG(sc, MGB_DMA_TX_HEAD(channel));  in mgb_dma_tx_ring_init()
1365 	KASSERT(rdata->last_head == 0, ("MGB_DMA_TX_HEAD was not reset.\n"));  in mgb_dma_tx_ring_init()
1366 	rdata->last_tail = 0;  in mgb_dma_tx_ring_init()
1367 	CSR_WRITE_REG(sc, MGB_DMA_TX_TAIL(channel), rdata->last_tail);  in mgb_dma_tx_ring_init()