36 gve_rx_free_ring_gqi(struct gve_priv *priv, int i)  in gve_rx_free_ring_gqi()  argument
38 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_rx_free_ring_gqi()
57 		gve_free_qpl(priv, com->qpl);  in gve_rx_free_ring_gqi()
63 gve_rx_free_ring(struct gve_priv *priv, int i)  in gve_rx_free_ring()  argument
65 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_rx_free_ring()
71 	if (gve_is_gqi(priv))  in gve_rx_free_ring()
72 		gve_rx_free_ring_gqi(priv, i);  in gve_rx_free_ring()
74 		gve_rx_free_ring_dqo(priv, i);  in gve_rx_free_ring()
89 	for (i = 0; i < com->priv->rx_desc_cnt; i++) {  in gve_prefill_rx_slots()
104 gve_rx_alloc_ring_gqi(struct gve_priv *priv, int i)  in gve_rx_alloc_ring_gqi()  argument
106 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_rx_alloc_ring_gqi()
110 	err = gve_dma_alloc_coherent(priv,  in gve_rx_alloc_ring_gqi()
111 	    sizeof(struct gve_rx_desc) * priv->rx_desc_cnt,  in gve_rx_alloc_ring_gqi()
114 		device_printf(priv->dev,  in gve_rx_alloc_ring_gqi()
119 	rx->mask = priv->rx_pages_per_qpl - 1;  in gve_rx_alloc_ring_gqi()
122 	com->qpl = gve_alloc_qpl(priv, i + priv->tx_cfg.max_queues,  in gve_rx_alloc_ring_gqi()
123 	    priv->rx_desc_cnt, /*single_kva=*/false);  in gve_rx_alloc_ring_gqi()
125 		device_printf(priv->dev,  in gve_rx_alloc_ring_gqi()
131 	rx->page_info = malloc(priv->rx_desc_cnt * sizeof(*rx->page_info),  in gve_rx_alloc_ring_gqi()
134 	err = gve_dma_alloc_coherent(priv,  in gve_rx_alloc_ring_gqi()
135 	    sizeof(union gve_rx_data_slot) * priv->rx_desc_cnt,  in gve_rx_alloc_ring_gqi()
138 		device_printf(priv->dev,  in gve_rx_alloc_ring_gqi()
148 	gve_rx_free_ring_gqi(priv, i);  in gve_rx_alloc_ring_gqi()
153 gve_rx_alloc_ring(struct gve_priv *priv, int i)  in gve_rx_alloc_ring()  argument
155 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_rx_alloc_ring()
159 	com->priv = priv;  in gve_rx_alloc_ring()
164 	err = gve_dma_alloc_coherent(priv, sizeof(struct gve_queue_resources),  in gve_rx_alloc_ring()
167 		device_printf(priv->dev,  in gve_rx_alloc_ring()
173 	if (gve_is_gqi(priv))  in gve_rx_alloc_ring()
174 		err = gve_rx_alloc_ring_gqi(priv, i);  in gve_rx_alloc_ring()
176 		err = gve_rx_alloc_ring_dqo(priv, i);  in gve_rx_alloc_ring()
183 	gve_rx_free_ring(priv, i);  in gve_rx_alloc_ring()
188 gve_alloc_rx_rings(struct gve_priv *priv, uint16_t start_idx, uint16_t stop_idx)  in gve_alloc_rx_rings()  argument
193 	KASSERT(priv->rx != NULL, ("priv->rx is NULL!"));  in gve_alloc_rx_rings()
196 		err = gve_rx_alloc_ring(priv, i);  in gve_alloc_rx_rings()
203 	gve_free_rx_rings(priv, start_idx, i);  in gve_alloc_rx_rings()
208 gve_free_rx_rings(struct gve_priv *priv, uint16_t start_idx, uint16_t stop_idx)  in gve_free_rx_rings()  argument
213 		gve_rx_free_ring(priv, i);  in gve_free_rx_rings()
219 	struct gve_priv *priv = rx->com.priv;  in gve_rx_clear_data_ring()  local
231 	for (i = 0; i < priv->rx_desc_cnt; i++) {  in gve_rx_clear_data_ring()
244 	struct gve_priv *priv = rx->com.priv;  in gve_rx_clear_desc_ring()  local
247 	for (i = 0; i < priv->rx_desc_cnt; i++)  in gve_rx_clear_desc_ring()
255 gve_clear_rx_ring(struct gve_priv *priv, int i)  in gve_clear_rx_ring()  argument
257 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_clear_rx_ring()
259 	if (!gve_is_gqi(priv)) {  in gve_clear_rx_ring()
260 		gve_clear_rx_ring_dqo(priv, i);  in gve_clear_rx_ring()
267 	rx->mask = priv->rx_desc_cnt - 1;  in gve_clear_rx_ring()
274 gve_start_rx_ring(struct gve_priv *priv, int i)  in gve_start_rx_ring()  argument
276 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_start_rx_ring()
279 	if ((if_getcapenable(priv->ifp) & IFCAP_LRO) != 0) {  in gve_start_rx_ring()
281 			device_printf(priv->dev, "Failed to init lro for rx ring %d", i);  in gve_start_rx_ring()
282 		rx->lro.ifp = priv->ifp;  in gve_start_rx_ring()
285 	if (gve_is_gqi(priv))  in gve_start_rx_ring()
293 	    "%s rxq %d", device_get_nameunit(priv->dev), i);  in gve_start_rx_ring()
295 	if (gve_is_gqi(priv)) {  in gve_start_rx_ring()
297 		gve_db_bar_write_4(priv, com->db_offset, rx->fill_cnt);  in gve_start_rx_ring()
303 gve_create_rx_rings(struct gve_priv *priv)  in gve_create_rx_rings()  argument
310 	if (gve_get_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK))  in gve_create_rx_rings()
313 	for (i = 0; i < priv->rx_cfg.num_queues; i++)  in gve_create_rx_rings()
314 		gve_clear_rx_ring(priv, i);  in gve_create_rx_rings()
316 	err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);  in gve_create_rx_rings()
320 	bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,  in gve_create_rx_rings()
323 	for (i = 0; i < priv->rx_cfg.num_queues; i++) {  in gve_create_rx_rings()
324 		rx = &priv->rx[i];  in gve_create_rx_rings()
327 		com->irq_db_offset = 4 * be32toh(priv->irq_db_indices[com->ntfy_id].index);  in gve_create_rx_rings()
334 		gve_start_rx_ring(priv, i);  in gve_create_rx_rings()
337 	gve_set_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);  in gve_create_rx_rings()
342 gve_stop_rx_ring(struct gve_priv *priv, int i)  in gve_stop_rx_ring()  argument
344 	struct gve_rx_ring *rx = &priv->rx[i];  in gve_stop_rx_ring()
358 gve_destroy_rx_rings(struct gve_priv *priv)  in gve_destroy_rx_rings()  argument
363 	for (i = 0; i < priv->rx_cfg.num_queues; i++)  in gve_destroy_rx_rings()
364 		gve_stop_rx_ring(priv, i);  in gve_destroy_rx_rings()
366 	if (gve_get_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK)) {  in gve_destroy_rx_rings()
367 		err = gve_adminq_destroy_rx_queues(priv, priv->rx_cfg.num_queues);  in gve_destroy_rx_rings()
370 		gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);  in gve_destroy_rx_rings()
380 	struct gve_priv *priv = rx->com.priv;  in gve_rx_intr()  local
383 	if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))  in gve_rx_intr()
386 	gve_db_bar_write_4(priv, com->irq_db_offset, GVE_IRQ_MASK);  in gve_rx_intr()
424 gve_rx_create_mbuf(struct gve_priv *priv, struct gve_rx_ring *rx,  in gve_rx_create_mbuf()  argument
436 	if (len <= priv->rx_copybreak && is_only_frag) {  in gve_rx_create_mbuf()
529 gve_rx(struct gve_priv *priv, struct gve_rx_ring *rx, struct gve_rx_desc *desc,  in gve_rx()  argument
537 	if_t ifp = priv->ifp;  in gve_rx()
568 	mbuf = gve_rx_create_mbuf(priv, rx, page_info, len, data_slot,  in gve_rx()
581 		mbuf->m_pkthdr.rcvif = priv->ifp;  in gve_rx()
603 		if (((if_getcapenable(priv->ifp) & IFCAP_LRO) != 0) &&      /* LRO is enabled */  in gve_rx()
647 gve_rx_cleanup(struct gve_priv *priv, struct gve_rx_ring *rx, int budget)  in gve_rx_cleanup()  argument
663 		gve_rx(priv, rx, desc, idx);  in gve_rx_cleanup()
676 		device_printf(priv->dev,  in gve_rx_cleanup()
679 		gve_schedule_reset(priv);  in gve_rx_cleanup()
690 	gve_db_bar_write_4(priv, rx->com.db_offset, rx->fill_cnt);  in gve_rx_cleanup()
697 	struct gve_priv *priv = rx->com.priv;  in gve_rx_cleanup_tq()  local
699 	if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))  in gve_rx_cleanup_tq()
702 	gve_rx_cleanup(priv, rx, /*budget=*/128);  in gve_rx_cleanup_tq()
704 	gve_db_bar_write_4(priv, rx->com.irq_db_offset,  in gve_rx_cleanup_tq()
715 		gve_db_bar_write_4(priv, rx->com.irq_db_offset, GVE_IRQ_MASK);  in gve_rx_cleanup_tq()