116 				     struct otx2_snd_queue *sq,  in otx2_xdp_snd_pkt_handler()  argument
124 	sg = &sq->sg[snd_comp->sqe_id];  in otx2_xdp_snd_pkt_handler()
135 				 struct otx2_snd_queue *sq,  in otx2_snd_pkt_handler()  argument
151 	sg = &sq->sg[snd_comp->sqe_id];  in otx2_snd_pkt_handler()
157 		timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id];  in otx2_snd_pkt_handler()
456 	struct otx2_snd_queue *sq;  in otx2_tx_napi_handler()  local
469 	sq = &pfvf->qset.sq[qidx];  in otx2_tx_napi_handler()
482 			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);  in otx2_tx_napi_handler()
484 			otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],  in otx2_tx_napi_handler()
491 		sq->cons_head++;  in otx2_tx_napi_handler()
492 		sq->cons_head &= (sq->sqe_cnt - 1);  in otx2_tx_napi_handler()
611 void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq,  in otx2_sqe_flush()  argument
620 		memcpy(sq->lmt_addr, sq->sqe_base, size);  in otx2_sqe_flush()
621 		status = otx2_lmt_flush(sq->io_addr);  in otx2_sqe_flush()
624 	sq->head++;  in otx2_sqe_flush()
625 	sq->head &= (sq->sqe_cnt - 1);  in otx2_sqe_flush()
630 static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,  in otx2_sqe_add_sg()  argument
638 	sq->sg[sq->head].num_segs = 0;  in otx2_sqe_add_sg()
642 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_sqe_add_sg()
665 		sq->sg[sq->head].dma_addr[seg] = dma_addr;  in otx2_sqe_add_sg()
666 		sq->sg[sq->head].size[seg] = len;  in otx2_sqe_add_sg()
667 		sq->sg[sq->head].num_segs++;  in otx2_sqe_add_sg()
670 	sq->sg[sq->head].skb = (u64)skb;  in otx2_sqe_add_sg()
675 static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,  in otx2_sqe_add_ext()  argument
680 	ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset);  in otx2_sqe_add_ext()
743 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,  in otx2_sqe_add_mem()  argument
749 	mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset);  in otx2_sqe_add_mem()
766 static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,  in otx2_sqe_add_hdr()  argument
778 		sqe_hdr->aura = sq->aura_id;  in otx2_sqe_add_hdr()
781 		sqe_hdr->sq = (qidx >=  pfvf->hw.tx_queues) ?  in otx2_sqe_add_hdr()
786 	sqe_hdr->sqe_id = sq->head;  in otx2_sqe_add_hdr()
815 				struct otx2_snd_queue *sq,  in otx2_dma_map_tso_skb()  argument
819 	struct sg_list *sg = &sq->sg[sqe];  in otx2_dma_map_tso_skb()
847 static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq,  in otx2_tso_frag_dma_addr()  argument
851 	struct sg_list *sg = &sq->sg[sqe];  in otx2_tso_frag_dma_addr()
865 static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq,  in otx2_sqe_tso_add_sg()  argument
876 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_sqe_tso_add_sg()
896 static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,  in otx2_sq_append_tso()  argument
902 	int first_sqe = sq->head;  in otx2_sq_append_tso()
911 	if (otx2_dma_map_tso_skb(pfvf, sq, skb, first_sqe, hdr_len)) {  in otx2_sq_append_tso()
926 		memset(sq->sqe_base, 0, sq->sqe_size);  in otx2_sq_append_tso()
927 		sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);  in otx2_sq_append_tso()
928 		otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);  in otx2_sq_append_tso()
932 		hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE);  in otx2_sq_append_tso()
935 			sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE);  in otx2_sq_append_tso()
948 				otx2_tso_frag_dma_addr(sq, skb,  in otx2_sq_append_tso()
958 		otx2_sqe_tso_add_sg(sq, &list, &offset);  in otx2_sq_append_tso()
969 			sq->sg[first_sqe].skb = (u64)skb;  in otx2_sq_append_tso()
977 		pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);  in otx2_sq_append_tso()
1095 			      struct otx2_snd_queue *sq, int *offset)  in otx2_set_txtstamp()  argument
1149 		iova = sq->timestamps->iova + (sq->head * sizeof(u64));  in otx2_set_txtstamp()
1150 		otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,  in otx2_set_txtstamp()
1158 			struct otx2_snd_queue *sq,  in otx2_sq_append_skb()  argument
1168 	free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1);  in otx2_sq_append_skb()
1169 	if (free_desc < sq->sqe_thresh)  in otx2_sq_append_skb()
1195 		otx2_sq_append_tso(pfvf, sq, skb, qidx);  in otx2_sq_append_skb()
1202 	memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);  in otx2_sq_append_skb()
1203 	sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);  in otx2_sq_append_skb()
1204 	otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);  in otx2_sq_append_skb()
1208 	otx2_sqe_add_ext(pfvf, sq, skb, &offset);  in otx2_sq_append_skb()
1211 	if (!otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset)) {  in otx2_sq_append_skb()
1212 		otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);  in otx2_sq_append_skb()
1216 	otx2_set_txtstamp(pfvf, skb, sq, &offset);  in otx2_sq_append_skb()
1223 	pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);  in otx2_sq_append_skb()
1271 	struct otx2_snd_queue *sq;  in otx2_cleanup_tx_cqes()  local
1279 	sq = &pfvf->qset.sq[qidx];  in otx2_cleanup_tx_cqes()
1291 		sg = &sq->sg[cqe->comp.sqe_id];  in otx2_cleanup_tx_cqes()
1338 	struct otx2_snd_queue *sq;  in otx2_free_pending_sqe()  local
1344 		sq = &pfvf->qset.sq[sq_idx];  in otx2_free_pending_sqe()
1345 		for (sqe = 0; sqe < sq->sqe_cnt; sqe++) {  in otx2_free_pending_sqe()
1346 			sg = &sq->sg[sqe];  in otx2_free_pending_sqe()
1366 static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,  in otx2_xdp_sqe_add_sg()  argument
1372 	sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_xdp_sqe_add_sg()
1381 	sq->sg[sq->head].dma_addr[0] = dma_addr;  in otx2_xdp_sqe_add_sg()
1382 	sq->sg[sq->head].size[0] = len;  in otx2_xdp_sqe_add_sg()
1383 	sq->sg[sq->head].num_segs = 1;  in otx2_xdp_sqe_add_sg()
1389 	struct otx2_snd_queue *sq;  in otx2_xdp_sq_append_pkt()  local
1392 	sq = &pfvf->qset.sq[qidx];  in otx2_xdp_sq_append_pkt()
1393 	free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb;  in otx2_xdp_sq_append_pkt()
1394 	if (free_sqe < sq->sqe_thresh)  in otx2_xdp_sq_append_pkt()
1397 	memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);  in otx2_xdp_sq_append_pkt()
1399 	sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);  in otx2_xdp_sq_append_pkt()
1402 		sqe_hdr->aura = sq->aura_id;  in otx2_xdp_sq_append_pkt()
1404 		sqe_hdr->sq = qidx;  in otx2_xdp_sq_append_pkt()
1408 	sqe_hdr->sqe_id = sq->head;  in otx2_xdp_sq_append_pkt()
1412 	otx2_xdp_sqe_add_sg(sq, iova, len, &offset);  in otx2_xdp_sq_append_pkt()
1414 	pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);  in otx2_xdp_sq_append_pkt()