26 	struct gve_tx_ring *tx = &priv->tx[tx_qid];  in gve_xdp_tx_flush()  local
28 	gve_tx_put_doorbell(priv, tx->q_resources, tx->req);  in gve_xdp_tx_flush()
157 static int gve_clean_xdp_done(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_clean_xdp_done()  argument
168 		idx = tx->done & tx->mask;  in gve_clean_xdp_done()
169 		info = &tx->info[idx];  in gve_clean_xdp_done()
170 		tx->done++;  in gve_clean_xdp_done()
187 	gve_tx_free_fifo(&tx->tx_fifo, space_freed);  in gve_clean_xdp_done()
188 	if (xsk_complete > 0 && tx->xsk_pool)  in gve_clean_xdp_done()
189 		xsk_tx_completed(tx->xsk_pool, xsk_complete);  in gve_clean_xdp_done()
190 	u64_stats_update_begin(&tx->statss);  in gve_clean_xdp_done()
191 	tx->bytes_done += bytes;  in gve_clean_xdp_done()
192 	tx->pkt_done += pkts;  in gve_clean_xdp_done()
193 	u64_stats_update_end(&tx->statss);  in gve_clean_xdp_done()
197 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
203 	struct gve_tx_ring *tx = &priv->tx[idx];  in gve_tx_stop_ring_gqi()  local
209 	if (tx->q_num < priv->tx_cfg.num_queues)  in gve_tx_stop_ring_gqi()
210 		gve_clean_tx_done(priv, tx, priv->tx_desc_cnt, false);  in gve_tx_stop_ring_gqi()
212 		gve_clean_xdp_done(priv, tx, priv->tx_desc_cnt);  in gve_tx_stop_ring_gqi()
213 	netdev_tx_reset_queue(tx->netdev_txq);  in gve_tx_stop_ring_gqi()
217 static void gve_tx_free_ring_gqi(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_tx_free_ring_gqi()  argument
221 	int idx = tx->q_num;  in gve_tx_free_ring_gqi()
226 	slots = tx->mask + 1;  in gve_tx_free_ring_gqi()
227 	dma_free_coherent(hdev, sizeof(*tx->q_resources),  in gve_tx_free_ring_gqi()
228 			  tx->q_resources, tx->q_resources_bus);  in gve_tx_free_ring_gqi()
229 	tx->q_resources = NULL;  in gve_tx_free_ring_gqi()
231 	if (tx->tx_fifo.qpl) {  in gve_tx_free_ring_gqi()
232 		if (tx->tx_fifo.base)  in gve_tx_free_ring_gqi()
233 			gve_tx_fifo_release(priv, &tx->tx_fifo);  in gve_tx_free_ring_gqi()
235 		qpl_id = gve_tx_qpl_id(priv, tx->q_num);  in gve_tx_free_ring_gqi()
236 		gve_free_queue_page_list(priv, tx->tx_fifo.qpl, qpl_id);  in gve_tx_free_ring_gqi()
237 		tx->tx_fifo.qpl = NULL;  in gve_tx_free_ring_gqi()
240 	bytes = sizeof(*tx->desc) * slots;  in gve_tx_free_ring_gqi()
241 	dma_free_coherent(hdev, bytes, tx->desc, tx->bus);  in gve_tx_free_ring_gqi()
242 	tx->desc = NULL;  in gve_tx_free_ring_gqi()
244 	vfree(tx->info);  in gve_tx_free_ring_gqi()
245 	tx->info = NULL;  in gve_tx_free_ring_gqi()
253 	struct gve_tx_ring *tx = &priv->tx[idx];  in gve_tx_start_ring_gqi()  local
257 	tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);  in gve_tx_start_ring_gqi()
263 				 struct gve_tx_ring *tx,  in gve_tx_alloc_ring_gqi()  argument
272 	memset(tx, 0, sizeof(*tx));  in gve_tx_alloc_ring_gqi()
273 	spin_lock_init(&tx->clean_lock);  in gve_tx_alloc_ring_gqi()
274 	spin_lock_init(&tx->xdp_lock);  in gve_tx_alloc_ring_gqi()
275 	tx->q_num = idx;  in gve_tx_alloc_ring_gqi()
277 	tx->mask = cfg->ring_size - 1;  in gve_tx_alloc_ring_gqi()
280 	tx->info = vcalloc(cfg->ring_size, sizeof(*tx->info));  in gve_tx_alloc_ring_gqi()
281 	if (!tx->info)  in gve_tx_alloc_ring_gqi()
285 	bytes = sizeof(*tx->desc) * cfg->ring_size;  in gve_tx_alloc_ring_gqi()
286 	tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);  in gve_tx_alloc_ring_gqi()
287 	if (!tx->desc)  in gve_tx_alloc_ring_gqi()
290 	tx->raw_addressing = cfg->raw_addressing;  in gve_tx_alloc_ring_gqi()
291 	tx->dev = hdev;  in gve_tx_alloc_ring_gqi()
292 	if (!tx->raw_addressing) {  in gve_tx_alloc_ring_gqi()
293 		qpl_id = gve_tx_qpl_id(priv, tx->q_num);  in gve_tx_alloc_ring_gqi()
296 		tx->tx_fifo.qpl = gve_alloc_queue_page_list(priv, qpl_id,  in gve_tx_alloc_ring_gqi()
298 		if (!tx->tx_fifo.qpl)  in gve_tx_alloc_ring_gqi()
302 		if (gve_tx_fifo_init(priv, &tx->tx_fifo))  in gve_tx_alloc_ring_gqi()
306 	tx->q_resources =  in gve_tx_alloc_ring_gqi()
308 				   sizeof(*tx->q_resources),  in gve_tx_alloc_ring_gqi()
309 				   &tx->q_resources_bus,  in gve_tx_alloc_ring_gqi()
311 	if (!tx->q_resources)  in gve_tx_alloc_ring_gqi()
317 	if (!tx->raw_addressing)  in gve_tx_alloc_ring_gqi()
318 		gve_tx_fifo_release(priv, &tx->tx_fifo);  in gve_tx_alloc_ring_gqi()
320 	if (!tx->raw_addressing) {  in gve_tx_alloc_ring_gqi()
321 		gve_free_queue_page_list(priv, tx->tx_fifo.qpl, qpl_id);  in gve_tx_alloc_ring_gqi()
322 		tx->tx_fifo.qpl = NULL;  in gve_tx_alloc_ring_gqi()
325 	dma_free_coherent(hdev, bytes, tx->desc, tx->bus);  in gve_tx_alloc_ring_gqi()
326 	tx->desc = NULL;  in gve_tx_alloc_ring_gqi()
328 	vfree(tx->info);  in gve_tx_alloc_ring_gqi()
329 	tx->info = NULL;  in gve_tx_alloc_ring_gqi()
336 	struct gve_tx_ring *tx = cfg->tx;  in gve_tx_alloc_rings_gqi()  local
347 		tx = kvcalloc(cfg->qcfg->max_queues, sizeof(struct gve_tx_ring),  in gve_tx_alloc_rings_gqi()
349 		if (!tx)  in gve_tx_alloc_rings_gqi()
351 	} else if (!tx) {  in gve_tx_alloc_rings_gqi()
358 		err = gve_tx_alloc_ring_gqi(priv, cfg, &tx[i], i);  in gve_tx_alloc_rings_gqi()
367 	cfg->tx = tx;  in gve_tx_alloc_rings_gqi()
372 		gve_tx_free_ring_gqi(priv, &tx[j], cfg);  in gve_tx_alloc_rings_gqi()
374 		kvfree(tx);  in gve_tx_alloc_rings_gqi()
381 	struct gve_tx_ring *tx = cfg->tx;  in gve_tx_free_rings_gqi()  local
384 	if (!tx)  in gve_tx_free_rings_gqi()
388 		gve_tx_free_ring_gqi(priv, &tx[i], cfg);  in gve_tx_free_rings_gqi()
391 		kvfree(tx);  in gve_tx_free_rings_gqi()
392 		cfg->tx = NULL;  in gve_tx_free_rings_gqi()
403 static inline u32 gve_tx_avail(struct gve_tx_ring *tx)  in gve_tx_avail()  argument
405 	return tx->mask + 1 - (tx->req - tx->done);  in gve_tx_avail()
408 static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,  in gve_skb_fifo_bytes_required()  argument
418 	pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo,  in gve_skb_fifo_bytes_required()
453 static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required)  in gve_can_tx()  argument
457 	if (!tx->raw_addressing)  in gve_can_tx()
458 		can_alloc = gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required);  in gve_can_tx()
460 	return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED && can_alloc);  in gve_can_tx()
466 static int gve_maybe_stop_tx(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_maybe_stop_tx()  argument
474 	if (!tx->raw_addressing)  in gve_maybe_stop_tx()
475 		bytes_required = gve_skb_fifo_bytes_required(tx, skb);  in gve_maybe_stop_tx()
477 	if (likely(gve_can_tx(tx, bytes_required)))  in gve_maybe_stop_tx()
481 	spin_lock(&tx->clean_lock);  in gve_maybe_stop_tx()
482 	nic_done = gve_tx_load_event_counter(priv, tx);  in gve_maybe_stop_tx()
483 	to_do = nic_done - tx->done;  in gve_maybe_stop_tx()
486 	if (to_do + gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED) {  in gve_maybe_stop_tx()
489 			gve_clean_tx_done(priv, tx, to_do, false);  in gve_maybe_stop_tx()
491 		if (likely(gve_can_tx(tx, bytes_required)))  in gve_maybe_stop_tx()
496 		tx->stop_queue++;  in gve_maybe_stop_tx()
497 		netif_tx_stop_queue(tx->netdev_txq);  in gve_maybe_stop_tx()
499 	spin_unlock(&tx->clean_lock);  in gve_maybe_stop_tx()
569 static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, struct sk_buff *skb)  in gve_tx_add_skb_copy()  argument
576 	u32 idx = tx->req & tx->mask;  in gve_tx_add_skb_copy()
582 	info = &tx->info[idx];  in gve_tx_add_skb_copy()
583 	pkt_desc = &tx->desc[idx];  in gve_tx_add_skb_copy()
596 	pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen);  in gve_tx_add_skb_copy()
597 	hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes,  in gve_tx_add_skb_copy()
600 	payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,  in gve_tx_add_skb_copy()
609 		      tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,  in gve_tx_add_skb_copy()
611 	gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,  in gve_tx_add_skb_copy()
617 		next_idx = (tx->req + 1) & tx->mask;  in gve_tx_add_skb_copy()
618 		gve_tx_fill_mtd_desc(&tx->desc[next_idx], skb);  in gve_tx_add_skb_copy()
622 		next_idx = (tx->req + 1 + mtd_desc_nr + i - payload_iov) & tx->mask;  in gve_tx_add_skb_copy()
623 		seg_desc = &tx->desc[next_idx];  in gve_tx_add_skb_copy()
632 			      tx->tx_fifo.base + info->iov[i].iov_offset,  in gve_tx_add_skb_copy()
634 		gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses,  in gve_tx_add_skb_copy()
643 static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_tx_add_skb_no_copy()  argument
652 	u32 idx = tx->req & tx->mask;  in gve_tx_add_skb_no_copy()
657 	info = &tx->info[idx];  in gve_tx_add_skb_no_copy()
658 	pkt_desc = &tx->desc[idx];  in gve_tx_add_skb_no_copy()
671 	addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);  in gve_tx_add_skb_no_copy()
672 	if (unlikely(dma_mapping_error(tx->dev, addr))) {  in gve_tx_add_skb_no_copy()
673 		tx->dma_mapping_error++;  in gve_tx_add_skb_no_copy()
690 		idx = (idx + 1) & tx->mask;  in gve_tx_add_skb_no_copy()
691 		mtd_desc = &tx->desc[idx];  in gve_tx_add_skb_no_copy()
701 		idx = (idx + 1) & tx->mask;  in gve_tx_add_skb_no_copy()
702 		seg_desc = &tx->desc[idx];  in gve_tx_add_skb_no_copy()
711 		idx = (idx + 1) & tx->mask;  in gve_tx_add_skb_no_copy()
712 		seg_desc = &tx->desc[idx];  in gve_tx_add_skb_no_copy()
714 		addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE);  in gve_tx_add_skb_no_copy()
715 		if (unlikely(dma_mapping_error(tx->dev, addr))) {  in gve_tx_add_skb_no_copy()
716 			tx->dma_mapping_error++;  in gve_tx_add_skb_no_copy()
719 		tx->info[idx].skb = NULL;  in gve_tx_add_skb_no_copy()
720 		dma_unmap_len_set(&tx->info[idx], len, len);  in gve_tx_add_skb_no_copy()
721 		dma_unmap_addr_set(&tx->info[idx], dma, addr);  in gve_tx_add_skb_no_copy()
737 		gve_tx_unmap_buf(tx->dev, &tx->info[idx & tx->mask]);  in gve_tx_add_skb_no_copy()
740 	tx->dropped_pkt++;  in gve_tx_add_skb_no_copy()
747 	struct gve_tx_ring *tx;  in gve_tx()  local
752 	tx = &priv->tx[skb_get_queue_mapping(skb)];  in gve_tx()
753 	if (unlikely(gve_maybe_stop_tx(priv, tx, skb))) {  in gve_tx()
759 		gve_tx_put_doorbell(priv, tx->q_resources, tx->req);  in gve_tx()
762 	if (tx->raw_addressing)  in gve_tx()
763 		nsegs = gve_tx_add_skb_no_copy(priv, tx, skb);  in gve_tx()
765 		nsegs = gve_tx_add_skb_copy(priv, tx, skb);  in gve_tx()
769 		netdev_tx_sent_queue(tx->netdev_txq, skb->len);  in gve_tx()
771 		tx->req += nsegs;  in gve_tx()
776 	if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())  in gve_tx()
782 	gve_tx_put_doorbell(priv, tx->q_resources, tx->req);  in gve_tx()
786 static int gve_tx_fill_xdp(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_tx_fill_xdp()  argument
791 	u32 reqi = tx->req;  in gve_tx_fill_xdp()
793 	pad = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, len);  in gve_tx_fill_xdp()
796 	info = &tx->info[reqi & tx->mask];  in gve_tx_fill_xdp()
801 	nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, pad + len,  in gve_tx_fill_xdp()
809 			gve_tx_fill_pkt_desc(&tx->desc[reqi & tx->mask], 0,  in gve_tx_fill_xdp()
814 			gve_tx_fill_seg_desc(&tx->desc[reqi & tx->mask],  in gve_tx_fill_xdp()
819 		memcpy(tx->tx_fifo.base + info->iov[iovi].iov_offset,  in gve_tx_fill_xdp()
822 					tx->tx_fifo.qpl->page_buses,  in gve_tx_fill_xdp()
837 	struct gve_tx_ring *tx;  in gve_xdp_xmit()  local
849 	tx = &priv->tx[qid];  in gve_xdp_xmit()
851 	spin_lock(&tx->xdp_lock);  in gve_xdp_xmit()
853 		err = gve_xdp_xmit_one(priv, tx, frames[i]->data,  in gve_xdp_xmit()
860 		gve_tx_put_doorbell(priv, tx->q_resources, tx->req);  in gve_xdp_xmit()
862 	spin_unlock(&tx->xdp_lock);  in gve_xdp_xmit()
864 	u64_stats_update_begin(&tx->statss);  in gve_xdp_xmit()
865 	tx->xdp_xmit += n;  in gve_xdp_xmit()
866 	tx->xdp_xmit_errors += n - i;  in gve_xdp_xmit()
867 	u64_stats_update_end(&tx->statss);  in gve_xdp_xmit()
872 int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_xdp_xmit_one()  argument
877 	if (!gve_can_tx(tx, len + GVE_GQ_TX_MIN_PKT_DESC_BYTES - 1))  in gve_xdp_xmit_one()
880 	nsegs = gve_tx_fill_xdp(priv, tx, data, len, frame_p, false);  in gve_xdp_xmit_one()
881 	tx->req += nsegs;  in gve_xdp_xmit_one()
888 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_clean_tx_done()  argument
899 		idx = tx->done & tx->mask;  in gve_clean_tx_done()
902 			   tx->q_num, __func__, idx, tx->req, tx->done);  in gve_clean_tx_done()
903 		info = &tx->info[idx];  in gve_clean_tx_done()
907 		if (tx->raw_addressing)  in gve_clean_tx_done()
908 			gve_tx_unmap_buf(tx->dev, info);  in gve_clean_tx_done()
909 		tx->done++;  in gve_clean_tx_done()
916 			if (tx->raw_addressing)  in gve_clean_tx_done()
922 	if (!tx->raw_addressing)  in gve_clean_tx_done()
923 		gve_tx_free_fifo(&tx->tx_fifo, space_freed);  in gve_clean_tx_done()
924 	u64_stats_update_begin(&tx->statss);  in gve_clean_tx_done()
925 	tx->bytes_done += bytes;  in gve_clean_tx_done()
926 	tx->pkt_done += pkts;  in gve_clean_tx_done()
927 	u64_stats_update_end(&tx->statss);  in gve_clean_tx_done()
928 	netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes);  in gve_clean_tx_done()
935 	if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) &&  in gve_clean_tx_done()
936 	    likely(gve_can_tx(tx, GVE_TX_START_THRESH))) {  in gve_clean_tx_done()
937 		tx->wake_queue++;  in gve_clean_tx_done()
938 		netif_tx_wake_queue(tx->netdev_txq);  in gve_clean_tx_done()
945 			      struct gve_tx_ring *tx)  in gve_tx_load_event_counter()  argument
947 	u32 counter_index = be32_to_cpu(tx->q_resources->counter_index);  in gve_tx_load_event_counter()
953 static int gve_xsk_tx(struct gve_priv *priv, struct gve_tx_ring *tx,  in gve_xsk_tx()  argument
960 	spin_lock(&tx->xdp_lock);  in gve_xsk_tx()
962 		if (!gve_can_tx(tx, GVE_TX_START_THRESH))  in gve_xsk_tx()
965 		if (!xsk_tx_peek_desc(tx->xsk_pool, &desc)) {  in gve_xsk_tx()
966 			tx->xdp_xsk_done = tx->xdp_xsk_wakeup;  in gve_xsk_tx()
970 		data = xsk_buff_raw_get_data(tx->xsk_pool, desc.addr);  in gve_xsk_tx()
971 		nsegs = gve_tx_fill_xdp(priv, tx, data, desc.len, NULL, true);  in gve_xsk_tx()
972 		tx->req += nsegs;  in gve_xsk_tx()
977 		gve_tx_put_doorbell(priv, tx->q_resources, tx->req);  in gve_xsk_tx()
978 		xsk_tx_release(tx->xsk_pool);  in gve_xsk_tx()
980 	spin_unlock(&tx->xdp_lock);  in gve_xsk_tx()
988 	struct gve_tx_ring *tx;  in gve_xsk_tx_poll()  local
991 	tx = &priv->tx[gve_xdp_tx_queue_id(priv, rx->q_num)];  in gve_xsk_tx_poll()
992 	if (tx->xsk_pool) {  in gve_xsk_tx_poll()
993 		sent = gve_xsk_tx(priv, tx, budget);  in gve_xsk_tx_poll()
995 		u64_stats_update_begin(&tx->statss);  in gve_xsk_tx_poll()
996 		tx->xdp_xsk_sent += sent;  in gve_xsk_tx_poll()
997 		u64_stats_update_end(&tx->statss);  in gve_xsk_tx_poll()
998 		if (xsk_uses_need_wakeup(tx->xsk_pool))  in gve_xsk_tx_poll()
999 			xsk_set_tx_need_wakeup(tx->xsk_pool);  in gve_xsk_tx_poll()
1008 	struct gve_tx_ring *tx = block->tx;  in gve_xdp_poll()  local
1013 	nic_done = gve_tx_load_event_counter(priv, tx);  in gve_xdp_poll()
1014 	to_do = min_t(u32, (nic_done - tx->done), budget);  in gve_xdp_poll()
1015 	gve_clean_xdp_done(priv, tx, to_do);  in gve_xdp_poll()
1018 	return nic_done != tx->done;  in gve_xdp_poll()
1024 	struct gve_tx_ring *tx = block->tx;  in gve_tx_poll()  local
1036 	spin_lock(&tx->clean_lock);  in gve_tx_poll()
1038 	nic_done = gve_tx_load_event_counter(priv, tx);  in gve_tx_poll()
1039 	to_do = min_t(u32, (nic_done - tx->done), budget);  in gve_tx_poll()
1040 	gve_clean_tx_done(priv, tx, to_do, true);  in gve_tx_poll()
1041 	spin_unlock(&tx->clean_lock);  in gve_tx_poll()
1043 	return nic_done != tx->done;  in gve_tx_poll()
1046 bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx)  in gve_tx_clean_pending()  argument
1048 	u32 nic_done = gve_tx_load_event_counter(priv, tx);  in gve_tx_clean_pending()
1050 	return nic_done != tx->done;  in gve_tx_clean_pending()