73 irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)  in irdma_puda_get_bufpool()  argument
76 	struct list_head *list = &rsrc->bufpool;  in irdma_puda_get_bufpool()
79 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);  in irdma_puda_get_bufpool()
82 		rsrc->avail_buf_count--;  in irdma_puda_get_bufpool()
83 		buf->vsi = rsrc->vsi;  in irdma_puda_get_bufpool()
85 		rsrc->stats_buf_alloc_fail++;  in irdma_puda_get_bufpool()
87 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_get_bufpool()
98 irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,  in irdma_puda_ret_bufpool()  argument
104 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);  in irdma_puda_ret_bufpool()
105 	list_add(&buf->list, &rsrc->bufpool);  in irdma_puda_ret_bufpool()
106 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_ret_bufpool()
107 	rsrc->avail_buf_count++;  in irdma_puda_ret_bufpool()
118 irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,  in irdma_puda_post_recvbuf()  argument
122 	struct irdma_sc_qp *qp = &rsrc->qp;  in irdma_puda_post_recvbuf()
126 …dma_sync_single_for_device(hw_to_dev(rsrc->dev->hw), buf->mem.pa, buf->mem.size, DMA_BIDIRECTIONAL…  in irdma_puda_post_recvbuf()
155 irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)  in irdma_puda_replenish_rq()  argument
158 	u32 invalid_cnt = rsrc->rxq_invalid_cnt;  in irdma_puda_replenish_rq()
162 		buf = irdma_puda_get_bufpool(rsrc);  in irdma_puda_replenish_rq()
165 		irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);  in irdma_puda_replenish_rq()
166 		rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);  in irdma_puda_replenish_rq()
167 		rsrc->rxq_invalid_cnt--;  in irdma_puda_replenish_rq()
371 	struct irdma_puda_rsrc *rsrc;  in irdma_puda_poll_cmpl()  local
376 		rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :  in irdma_puda_poll_cmpl()
391 	if (!qp || !rsrc) {  in irdma_puda_poll_cmpl()
396 	if (qp->qp_id != rsrc->qp_id) {  in irdma_puda_poll_cmpl()
410 			rsrc->stats_rcvd_pkt_err++;  in irdma_puda_poll_cmpl()
412 				irdma_ilq_putback_rcvbuf(&rsrc->qp, buf,  in irdma_puda_poll_cmpl()
415 				irdma_puda_ret_bufpool(rsrc, buf);  in irdma_puda_poll_cmpl()
416 				irdma_puda_replenish_rq(rsrc, false);  in irdma_puda_poll_cmpl()
421 		rsrc->stats_pkt_rcvd++;  in irdma_puda_poll_cmpl()
422 		rsrc->compl_rxwqe_idx = info.wqe_idx;  in irdma_puda_poll_cmpl()
424 		rsrc->receive(rsrc->vsi, buf);  in irdma_puda_poll_cmpl()
426 			irdma_ilq_putback_rcvbuf(&rsrc->qp, buf, info.wqe_idx);  in irdma_puda_poll_cmpl()
428 			irdma_puda_replenish_rq(rsrc, false);  in irdma_puda_poll_cmpl()
438 		rsrc->xmit_complete(rsrc->vsi, buf);  in irdma_puda_poll_cmpl()
439 		spin_lock_irqsave(&rsrc->bufpool_lock, flags);  in irdma_puda_poll_cmpl()
440 		rsrc->tx_wqe_avail_cnt++;  in irdma_puda_poll_cmpl()
441 		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_poll_cmpl()
442 		if (!list_empty(&rsrc->txpend))  in irdma_puda_poll_cmpl()
443 			irdma_puda_send_buf(rsrc, NULL);  in irdma_puda_poll_cmpl()
540 irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,  in irdma_puda_send_buf()  argument
547 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);  in irdma_puda_send_buf()
551 	if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {  in irdma_puda_send_buf()
552 		list_add_tail(&buf->list, &rsrc->txpend);  in irdma_puda_send_buf()
553 		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_send_buf()
554 		rsrc->stats_sent_pkt_q++;  in irdma_puda_send_buf()
555 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)  in irdma_puda_send_buf()
556 			irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,  in irdma_puda_send_buf()
560 	rsrc->tx_wqe_avail_cnt--;  in irdma_puda_send_buf()
565 		buf = irdma_puda_get_listbuf(&rsrc->txpend);  in irdma_puda_send_buf()
576 	if (rsrc->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {  in irdma_puda_send_buf()
584 	dma_sync_single_for_cpu(hw_to_dev(rsrc->dev->hw), buf->mem.pa, buf->mem.size, DMA_BIDIRECTIONAL);  in irdma_puda_send_buf()
585 	ret = irdma_puda_send(&rsrc->qp, &info);  in irdma_puda_send_buf()
587 		rsrc->tx_wqe_avail_cnt++;  in irdma_puda_send_buf()
588 		rsrc->stats_sent_pkt_q++;  in irdma_puda_send_buf()
589 		list_add(&buf->list, &rsrc->txpend);  in irdma_puda_send_buf()
590 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)  in irdma_puda_send_buf()
591 			irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,  in irdma_puda_send_buf()
594 		rsrc->stats_pkt_sent++;  in irdma_puda_send_buf()
597 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_send_buf()
605 irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)  in irdma_puda_qp_setctx()  argument
607 	struct irdma_sc_qp *qp = &rsrc->qp;  in irdma_puda_qp_setctx()
616 		      FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));  in irdma_puda_qp_setctx()
621 		      FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |  in irdma_puda_qp_setctx()
622 		      FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));  in irdma_puda_qp_setctx()
624 		      FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));  in irdma_puda_qp_setctx()
627 		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));  in irdma_puda_qp_setctx()
635 	irdma_debug_buf(rsrc->dev, IRDMA_DEBUG_PUDA, "PUDA QP CONTEXT", qp_ctx,  in irdma_puda_qp_setctx()
684 irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)  in irdma_puda_qp_create()  argument
686 	struct irdma_sc_qp *qp = &rsrc->qp;  in irdma_puda_qp_create()
692 	sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;  in irdma_puda_qp_create()
693 	rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;  in irdma_puda_qp_create()
694 	rsrc->qpmem.size = (sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) +  in irdma_puda_qp_create()
696 	rsrc->qpmem.va = irdma_allocate_dma_mem(rsrc->dev->hw, &rsrc->qpmem,  in irdma_puda_qp_create()
697 						rsrc->qpmem.size, IRDMA_HW_PAGE_SIZE);  in irdma_puda_qp_create()
698 	if (!rsrc->qpmem.va)  in irdma_puda_qp_create()
701 	mem = &rsrc->qpmem;  in irdma_puda_qp_create()
702 	memset(mem->va, 0, rsrc->qpmem.size);  in irdma_puda_qp_create()
703 	qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);  in irdma_puda_qp_create()
704 	qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);  in irdma_puda_qp_create()
705 	qp->pd = &rsrc->sc_pd;  in irdma_puda_qp_create()
707 	qp->dev = rsrc->dev;  in irdma_puda_qp_create()
708 	qp->qp_uk.back_qp = rsrc;  in irdma_puda_qp_create()
711 	qp->vsi = rsrc->vsi;  in irdma_puda_qp_create()
713 	ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];  in irdma_puda_qp_create()
714 	ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;  in irdma_puda_qp_create()
720 	ukqp->qp_id = rsrc->qp_id;  in irdma_puda_qp_create()
721 	ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;  in irdma_puda_qp_create()
722 	ukqp->rq_wrid_array = rsrc->rq_wrid_array;  in irdma_puda_qp_create()
723 	ukqp->sq_size = rsrc->sq_size;  in irdma_puda_qp_create()
724 	ukqp->rq_size = rsrc->rq_size;  in irdma_puda_qp_create()
731 	ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);  in irdma_puda_qp_create()
733 		irdma_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem);  in irdma_puda_qp_create()
738 	irdma_puda_qp_setctx(rsrc);  in irdma_puda_qp_create()
740 	if (rsrc->dev->ceq_valid)  in irdma_puda_qp_create()
741 		ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);  in irdma_puda_qp_create()
743 		ret = irdma_puda_qp_wqe(rsrc->dev, qp);  in irdma_puda_qp_create()
746 		rsrc->dev->ws_remove(qp->vsi, qp->user_pri);  in irdma_puda_qp_create()
747 		irdma_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem);  in irdma_puda_qp_create()
812 irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)  in irdma_puda_cq_create()  argument
814 	struct irdma_sc_dev *dev = rsrc->dev;  in irdma_puda_cq_create()
815 	struct irdma_sc_cq *cq = &rsrc->cq;  in irdma_puda_cq_create()
822 	cq->vsi = rsrc->vsi;  in irdma_puda_cq_create()
823 	cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));  in irdma_puda_cq_create()
824 	rsrc->cqmem.size = cqsize + sizeof(struct irdma_cq_shadow_area);  in irdma_puda_cq_create()
825 	rsrc->cqmem.va = irdma_allocate_dma_mem(dev->hw, &rsrc->cqmem,  in irdma_puda_cq_create()
826 						rsrc->cqmem.size,  in irdma_puda_cq_create()
828 	if (!rsrc->cqmem.va)  in irdma_puda_cq_create()
831 	mem = &rsrc->cqmem;  in irdma_puda_cq_create()
833 	info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?  in irdma_puda_cq_create()
835 	info.shadow_read_threshold = rsrc->cq_size >> 2;  in irdma_puda_cq_create()
840 	init_info->cq_size = rsrc->cq_size;  in irdma_puda_cq_create()
841 	init_info->cq_id = rsrc->cq_id;  in irdma_puda_cq_create()
844 	info.vsi = rsrc->vsi;  in irdma_puda_cq_create()
850 	if (rsrc->dev->ceq_valid)  in irdma_puda_cq_create()
856 		irdma_free_dma_mem(dev->hw, &rsrc->cqmem);  in irdma_puda_cq_create()
866 irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)  in irdma_puda_free_qp()  argument
870 	struct irdma_sc_dev *dev = rsrc->dev;  in irdma_puda_free_qp()
872 	if (rsrc->dev->ceq_valid) {  in irdma_puda_free_qp()
873 		irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);  in irdma_puda_free_qp()
874 		rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);  in irdma_puda_free_qp()
878 	ret = irdma_sc_qp_destroy(&rsrc->qp, 0, false, true, true);  in irdma_puda_free_qp()
890 	rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);  in irdma_puda_free_qp()
898 irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)  in irdma_puda_free_cq()  argument
902 	struct irdma_sc_dev *dev = rsrc->dev;  in irdma_puda_free_cq()
904 	if (rsrc->dev->ceq_valid) {  in irdma_puda_free_cq()
905 		irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);  in irdma_puda_free_cq()
909 	ret = irdma_sc_cq_destroy(&rsrc->cq, 0, true);  in irdma_puda_free_cq()
932 	struct irdma_puda_rsrc *rsrc;  in irdma_puda_dele_rsrc()  local
942 		rsrc = vsi->ilq;  in irdma_puda_dele_rsrc()
946 			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);  in irdma_puda_dele_rsrc()
949 		rsrc = vsi->ieq;  in irdma_puda_dele_rsrc()
953 			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);  in irdma_puda_dele_rsrc()
961 	spin_lock_destroy(&rsrc->bufpool_lock);  in irdma_puda_dele_rsrc()
962 	switch (rsrc->cmpl) {  in irdma_puda_dele_rsrc()
964 		irdma_free_hash_desc(rsrc->hash_desc);  in irdma_puda_dele_rsrc()
967 		irdma_qp_rem_qos(&rsrc->qp);  in irdma_puda_dele_rsrc()
970 			irdma_puda_free_qp(rsrc);  in irdma_puda_dele_rsrc()
972 		irdma_free_dma_mem(dev->hw, &rsrc->qpmem);  in irdma_puda_dele_rsrc()
976 			irdma_puda_free_cq(rsrc);  in irdma_puda_dele_rsrc()
978 		irdma_free_dma_mem(dev->hw, &rsrc->cqmem);  in irdma_puda_dele_rsrc()
981 		irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,  in irdma_puda_dele_rsrc()
986 	buf = rsrc->alloclist;  in irdma_puda_dele_rsrc()
991 		rsrc->alloc_buf_count--;  in irdma_puda_dele_rsrc()
1003 irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)  in irdma_puda_allocbufs()  argument
1016 		irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,  in irdma_puda_allocbufs()
1018 		rsrc->stats_buf_alloc_fail++;  in irdma_puda_allocbufs()
1026 	buf->mem.va = irdma_allocate_dma_mem(rsrc->dev->hw, &buf->mem,  in irdma_puda_allocbufs()
1027 					     rsrc->buf_size * count, 1);  in irdma_puda_allocbufs()
1029 		irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,  in irdma_puda_allocbufs()
1032 		rsrc->stats_buf_alloc_fail++;  in irdma_puda_allocbufs()
1041 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);  in irdma_puda_allocbufs()
1045 		buf->mem.va = (char *)dma_mem->va + (i * rsrc->buf_size);  in irdma_puda_allocbufs()
1046 		buf->mem.pa = dma_mem->pa + (i * rsrc->buf_size);  in irdma_puda_allocbufs()
1047 		buf->mem.size = rsrc->buf_size;  in irdma_puda_allocbufs()
1054 		list_add(&buf->list, &rsrc->bufpool);  in irdma_puda_allocbufs()
1055 		rsrc->alloc_buf_count++;  in irdma_puda_allocbufs()
1056 		if (!rsrc->alloclist) {  in irdma_puda_allocbufs()
1057 			rsrc->alloclist = buf;  in irdma_puda_allocbufs()
1059 			nextbuf = rsrc->alloclist;  in irdma_puda_allocbufs()
1060 			rsrc->alloclist = buf;  in irdma_puda_allocbufs()
1064 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);  in irdma_puda_allocbufs()
1066 	rsrc->avail_buf_count = rsrc->alloc_buf_count;  in irdma_puda_allocbufs()
1070 		buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);  in irdma_puda_allocbufs()
1072 			rsrc->stats_buf_alloc_fail++;  in irdma_puda_allocbufs()
1075 		irdma_puda_ret_bufpool(rsrc, buf);  in irdma_puda_allocbufs()
1076 		rsrc->alloc_buf_count++;  in irdma_puda_allocbufs()
1077 		if (!rsrc->alloclist) {  in irdma_puda_allocbufs()
1078 			rsrc->alloclist = buf;  in irdma_puda_allocbufs()
1080 			nextbuf = rsrc->alloclist;  in irdma_puda_allocbufs()
1081 			rsrc->alloclist = buf;  in irdma_puda_allocbufs()
1086 	rsrc->avail_buf_count = rsrc->alloc_buf_count;  in irdma_puda_allocbufs()
1102 	struct irdma_puda_rsrc *rsrc;  in irdma_puda_create_rsrc()  local
1108 	pudasize = sizeof(*rsrc);  in irdma_puda_create_rsrc()
1126 	rsrc = vmem->va;  in irdma_puda_create_rsrc()
1127 	spin_lock_init(&rsrc->bufpool_lock);  in irdma_puda_create_rsrc()
1132 		rsrc->receive = info->receive;  in irdma_puda_create_rsrc()
1133 		rsrc->xmit_complete = info->xmit_complete;  in irdma_puda_create_rsrc()
1138 		rsrc->receive = irdma_ieq_receive;  in irdma_puda_create_rsrc()
1139 		rsrc->xmit_complete = irdma_ieq_tx_compl;  in irdma_puda_create_rsrc()
1145 	rsrc->type = info->type;  in irdma_puda_create_rsrc()
1146 	rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)  in irdma_puda_create_rsrc()
1148 	rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);  in irdma_puda_create_rsrc()
1150 	INIT_LIST_HEAD(&rsrc->bufpool);  in irdma_puda_create_rsrc()
1151 	INIT_LIST_HEAD(&rsrc->txpend);  in irdma_puda_create_rsrc()
1153 	rsrc->tx_wqe_avail_cnt = info->sq_size - 1;  in irdma_puda_create_rsrc()
1154 	irdma_sc_pd_init(dev, &rsrc->sc_pd, info->pd_id, info->abi_ver);  in irdma_puda_create_rsrc()
1155 	rsrc->qp_id = info->qp_id;  in irdma_puda_create_rsrc()
1156 	rsrc->cq_id = info->cq_id;  in irdma_puda_create_rsrc()
1157 	rsrc->sq_size = info->sq_size;  in irdma_puda_create_rsrc()
1158 	rsrc->rq_size = info->rq_size;  in irdma_puda_create_rsrc()
1159 	rsrc->cq_size = info->rq_size + info->sq_size;  in irdma_puda_create_rsrc()
1161 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)  in irdma_puda_create_rsrc()
1162 			rsrc->cq_size += info->rq_size;  in irdma_puda_create_rsrc()
1164 	rsrc->buf_size = info->buf_size;  in irdma_puda_create_rsrc()
1165 	rsrc->dev = dev;  in irdma_puda_create_rsrc()
1166 	rsrc->vsi = vsi;  in irdma_puda_create_rsrc()
1167 	rsrc->stats_idx = info->stats_idx;  in irdma_puda_create_rsrc()
1168 	rsrc->stats_idx_valid = info->stats_idx_valid;  in irdma_puda_create_rsrc()
1170 	ret = irdma_puda_cq_create(rsrc);  in irdma_puda_create_rsrc()
1172 		rsrc->cmpl = PUDA_CQ_CREATED;  in irdma_puda_create_rsrc()
1173 		ret = irdma_puda_qp_create(rsrc);  in irdma_puda_create_rsrc()
1177 			    "error qp_create type=%d, status=%d\n", rsrc->type,  in irdma_puda_create_rsrc()
1181 	rsrc->cmpl = PUDA_QP_CREATED;  in irdma_puda_create_rsrc()
1183 	ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);  in irdma_puda_create_rsrc()
1189 	rsrc->rxq_invalid_cnt = info->rq_size;  in irdma_puda_create_rsrc()
1190 	ret = irdma_puda_replenish_rq(rsrc, true);  in irdma_puda_create_rsrc()
1195 		if (!irdma_init_hash_desc(&rsrc->hash_desc)) {  in irdma_puda_create_rsrc()
1196 			rsrc->check_crc = true;  in irdma_puda_create_rsrc()
1197 			rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;  in irdma_puda_create_rsrc()
1202 	irdma_sc_ccq_arm(&rsrc->cq);  in irdma_puda_create_rsrc()