20 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
505 				struct snd_queue *sq, int q_len, int qidx)  in nicvf_init_snd_queue()  argument
509 	err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,  in nicvf_init_snd_queue()
514 	sq->desc = sq->dmem.base;  in nicvf_init_snd_queue()
515 	sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL);  in nicvf_init_snd_queue()
516 	if (!sq->skbuff)  in nicvf_init_snd_queue()
519 	sq->head = 0;  in nicvf_init_snd_queue()
520 	sq->tail = 0;  in nicvf_init_snd_queue()
521 	sq->thresh = SND_QUEUE_THRESH;  in nicvf_init_snd_queue()
528 		sq->xdp_page = kcalloc(q_len, sizeof(u64), GFP_KERNEL);  in nicvf_init_snd_queue()
529 		if (!sq->xdp_page)  in nicvf_init_snd_queue()
531 		sq->xdp_desc_cnt = 0;  in nicvf_init_snd_queue()
532 		sq->xdp_free_cnt = q_len - 1;  in nicvf_init_snd_queue()
533 		sq->is_xdp = true;  in nicvf_init_snd_queue()
535 		sq->xdp_page = NULL;  in nicvf_init_snd_queue()
536 		sq->xdp_desc_cnt = 0;  in nicvf_init_snd_queue()
537 		sq->xdp_free_cnt = 0;  in nicvf_init_snd_queue()
538 		sq->is_xdp = false;  in nicvf_init_snd_queue()
540 		atomic_set(&sq->free_cnt, q_len - 1);  in nicvf_init_snd_queue()
543 		sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev,  in nicvf_init_snd_queue()
545 						  &sq->tso_hdrs_phys,  in nicvf_init_snd_queue()
547 		if (!sq->tso_hdrs)  in nicvf_init_snd_queue()
554 void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq,  in nicvf_unmap_sndq_buffers()  argument
563 		hdr_sqe &= (sq->dmem.q_len - 1);  in nicvf_unmap_sndq_buffers()
564 		gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, hdr_sqe);  in nicvf_unmap_sndq_buffers()
572 static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)  in nicvf_free_snd_queue()  argument
579 	if (!sq)  in nicvf_free_snd_queue()
581 	if (!sq->dmem.base)  in nicvf_free_snd_queue()
584 	if (sq->tso_hdrs) {  in nicvf_free_snd_queue()
586 				  sq->dmem.q_len * TSO_HEADER_SIZE,  in nicvf_free_snd_queue()
587 				  sq->tso_hdrs, sq->tso_hdrs_phys);  in nicvf_free_snd_queue()
588 		sq->tso_hdrs = NULL;  in nicvf_free_snd_queue()
593 	while (sq->head != sq->tail) {  in nicvf_free_snd_queue()
594 		skb = (struct sk_buff *)sq->skbuff[sq->head];  in nicvf_free_snd_queue()
595 		if (!skb || !sq->xdp_page)  in nicvf_free_snd_queue()
598 		page = (struct page *)sq->xdp_page[sq->head];  in nicvf_free_snd_queue()
604 		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);  in nicvf_free_snd_queue()
609 			 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);  in nicvf_free_snd_queue()
610 			nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,  in nicvf_free_snd_queue()
613 			nicvf_unmap_sndq_buffers(nic, sq, sq->head,  in nicvf_free_snd_queue()
619 		sq->head++;  in nicvf_free_snd_queue()
620 		sq->head &= (sq->dmem.q_len - 1);  in nicvf_free_snd_queue()
622 	kfree(sq->skbuff);  in nicvf_free_snd_queue()
623 	kfree(sq->xdp_page);  in nicvf_free_snd_queue()
624 	nicvf_free_q_desc_mem(nic, &sq->dmem);  in nicvf_free_snd_queue()
863 	struct snd_queue *sq;  in nicvf_snd_queue_config()  local
866 	sq = &qs->sq[qidx];  in nicvf_snd_queue_config()
867 	sq->enable = enable;  in nicvf_snd_queue_config()
869 	if (!sq->enable) {  in nicvf_snd_queue_config()
877 	sq->cq_qs = qs->vnic_id;  in nicvf_snd_queue_config()
878 	sq->cq_idx = qidx;  in nicvf_snd_queue_config()
881 	mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG;  in nicvf_snd_queue_config()
882 	mbx.sq.qs_num = qs->vnic_id;  in nicvf_snd_queue_config()
883 	mbx.sq.sq_num = qidx;  in nicvf_snd_queue_config()
884 	mbx.sq.sqs_mode = nic->sqs_mode;  in nicvf_snd_queue_config()
885 	mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx;  in nicvf_snd_queue_config()
890 			      qidx, (u64)(sq->dmem.phys_base));  in nicvf_snd_queue_config()
906 	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);  in nicvf_snd_queue_config()
910 		cpumask_set_cpu(qidx, &sq->affinity_mask);  in nicvf_snd_queue_config()
912 				    &sq->affinity_mask, qidx);  in nicvf_snd_queue_config()
1004 		nicvf_free_snd_queue(nic, &qs->sq[qidx]);  in nicvf_free_resources()
1021 		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))  in nicvf_alloc_resources()
1119 static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)  in nicvf_get_sq_desc()  argument
1123 	qentry = sq->tail;  in nicvf_get_sq_desc()
1124 	if (!sq->is_xdp)  in nicvf_get_sq_desc()
1125 		atomic_sub(desc_cnt, &sq->free_cnt);  in nicvf_get_sq_desc()
1127 		sq->xdp_free_cnt -= desc_cnt;  in nicvf_get_sq_desc()
1128 	sq->tail += desc_cnt;  in nicvf_get_sq_desc()
1129 	sq->tail &= (sq->dmem.q_len - 1);  in nicvf_get_sq_desc()
1135 static inline void nicvf_rollback_sq_desc(struct snd_queue *sq,  in nicvf_rollback_sq_desc()  argument
1138 	sq->tail = qentry;  in nicvf_rollback_sq_desc()
1139 	atomic_add(desc_cnt, &sq->free_cnt);  in nicvf_rollback_sq_desc()
1143 void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)  in nicvf_put_sq_desc()  argument
1145 	if (!sq->is_xdp)  in nicvf_put_sq_desc()
1146 		atomic_add(desc_cnt, &sq->free_cnt);  in nicvf_put_sq_desc()
1148 		sq->xdp_free_cnt += desc_cnt;  in nicvf_put_sq_desc()
1149 	sq->head += desc_cnt;  in nicvf_put_sq_desc()
1150 	sq->head &= (sq->dmem.q_len - 1);  in nicvf_put_sq_desc()
1153 static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)  in nicvf_get_nxt_sqentry()  argument
1156 	qentry &= (sq->dmem.q_len - 1);  in nicvf_get_nxt_sqentry()
1160 void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)  in nicvf_sq_enable()  argument
1180 void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,  in nicvf_sq_free_used_descs()  argument
1189 	while (sq->head != head) {  in nicvf_sq_free_used_descs()
1190 		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);  in nicvf_sq_free_used_descs()
1192 			nicvf_put_sq_desc(sq, 1);  in nicvf_sq_free_used_descs()
1195 		skb = (struct sk_buff *)sq->skbuff[sq->head];  in nicvf_sq_free_used_descs()
1201 		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);  in nicvf_sq_free_used_descs()
1207 			   struct snd_queue *sq, int sq_num)  in nicvf_xdp_sq_doorbell()  argument
1209 	if (!sq->xdp_desc_cnt)  in nicvf_xdp_sq_doorbell()
1217 			      sq_num, sq->xdp_desc_cnt);  in nicvf_xdp_sq_doorbell()
1218 	sq->xdp_desc_cnt = 0;  in nicvf_xdp_sq_doorbell()
1222 nicvf_xdp_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,  in nicvf_xdp_sq_add_hdr_subdesc()  argument
1227 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_xdp_sq_add_hdr_subdesc()
1233 	sq->xdp_page[qentry] = (u64)virt_to_page((void *)data);  in nicvf_xdp_sq_add_hdr_subdesc()
1236 int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq,  in nicvf_xdp_sq_append_pkt()  argument
1242 	if (subdesc_cnt > sq->xdp_free_cnt)  in nicvf_xdp_sq_append_pkt()
1245 	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);  in nicvf_xdp_sq_append_pkt()
1247 	nicvf_xdp_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, bufaddr, len);  in nicvf_xdp_sq_append_pkt()
1249 	qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_xdp_sq_append_pkt()
1250 	nicvf_sq_add_gather_subdesc(sq, qentry, len, dma_addr);  in nicvf_xdp_sq_append_pkt()
1252 	sq->xdp_desc_cnt += subdesc_cnt;  in nicvf_xdp_sq_append_pkt()
1330 nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry,  in nicvf_sq_add_hdr_subdesc()  argument
1342 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_hdr_subdesc()
1352 		sq->skbuff[qentry] = (u64)skb;  in nicvf_sq_add_hdr_subdesc()
1419 static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,  in nicvf_sq_add_gather_subdesc()  argument
1424 	qentry &= (sq->dmem.q_len - 1);  in nicvf_sq_add_gather_subdesc()
1425 	gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_gather_subdesc()
1438 static inline void nicvf_sq_add_cqe_subdesc(struct snd_queue *sq, int qentry,  in nicvf_sq_add_cqe_subdesc()  argument
1444 	sq->skbuff[qentry] = (u64)skb;  in nicvf_sq_add_cqe_subdesc()
1446 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_cqe_subdesc()
1458 	qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_add_cqe_subdesc()
1459 	imm = (struct sq_imm_subdesc *)GET_SQ_DESC(sq, qentry);  in nicvf_sq_add_cqe_subdesc()
1486 static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq,  in nicvf_sq_append_tso()  argument
1508 		qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_tso()
1509 		hdr = sq->tso_hdrs + qentry * TSO_HEADER_SIZE;  in nicvf_sq_append_tso()
1511 		nicvf_sq_add_gather_subdesc(sq, qentry, hdr_len,  in nicvf_sq_append_tso()
1512 					    sq->tso_hdrs_phys +  in nicvf_sq_append_tso()
1524 			qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_tso()
1525 			nicvf_sq_add_gather_subdesc(sq, qentry, size,  in nicvf_sq_append_tso()
1533 		nicvf_sq_add_hdr_subdesc(nic, sq, hdr_qentry,  in nicvf_sq_append_tso()
1535 		sq->skbuff[hdr_qentry] = (u64)NULL;  in nicvf_sq_append_tso()
1536 		qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_tso()
1541 	sq->skbuff[hdr_qentry] = (u64)skb;  in nicvf_sq_append_tso()
1550 int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq,  in nicvf_sq_append_skb()  argument
1559 	if (subdesc_cnt > atomic_read(&sq->free_cnt))  in nicvf_sq_append_skb()
1562 	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);  in nicvf_sq_append_skb()
1566 		return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb);  in nicvf_sq_append_skb()
1569 	nicvf_sq_add_hdr_subdesc(nic, sq, qentry, subdesc_cnt - 1,  in nicvf_sq_append_skb()
1574 	qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_skb()
1581 		nicvf_rollback_sq_desc(sq, qentry, subdesc_cnt);  in nicvf_sq_append_skb()
1585 	nicvf_sq_add_gather_subdesc(sq, qentry, size, dma_addr);  in nicvf_sq_append_skb()
1594 		qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_skb()
1605 			nicvf_unmap_sndq_buffers(nic, sq, hdr_sqe, i);  in nicvf_sq_append_skb()
1606 			nicvf_rollback_sq_desc(sq, qentry, subdesc_cnt);  in nicvf_sq_append_skb()
1609 		nicvf_sq_add_gather_subdesc(sq, qentry, size, dma_addr);  in nicvf_sq_append_skb()
1614 		qentry = nicvf_get_nxt_sqentry(sq, qentry);  in nicvf_sq_append_skb()
1615 		nicvf_sq_add_cqe_subdesc(sq, qentry, hdr_sqe, skb);  in nicvf_sq_append_skb()
1827 	struct snd_queue *sq;  in nicvf_update_sq_stats()  local
1833 	sq = &nic->qs->sq[sq_idx];  in nicvf_update_sq_stats()
1834 	sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS);  in nicvf_update_sq_stats()
1835 	sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS);  in nicvf_update_sq_stats()