44 	struct t4_cqe cqe;  in insert_recv_cqe()  local
48 	memset(&cqe, 0, sizeof(cqe));  in insert_recv_cqe()
49 	cqe.header = htobe32(V_CQE_STATUS(T4_ERR_SWFLUSH) |  in insert_recv_cqe()
54 	cqe.bits_type_ts = htobe64(V_CQE_GENBIT((u64)cq->gen));  in insert_recv_cqe()
55 	cq->sw_queue[cq->sw_pidx] = cqe;  in insert_recv_cqe()
77 	struct t4_cqe cqe;  in insert_sq_cqe()  local
81 	memset(&cqe, 0, sizeof(cqe));  in insert_sq_cqe()
82 	cqe.header = htobe32(V_CQE_STATUS(T4_ERR_SWFLUSH) |  in insert_sq_cqe()
87 	CQE_WRID_SQ_IDX(&cqe) = swcqe->idx;  in insert_sq_cqe()
88 	cqe.bits_type_ts = htobe64(V_CQE_GENBIT((u64)cq->gen));  in insert_sq_cqe()
89 	cq->sw_queue[cq->sw_pidx] = cqe;  in insert_sq_cqe()
151 			swsqe->cqe.header |= htobe32(V_CQE_SWCQE(1));  in flush_completed_wrs()
152 			cq->sw_queue[cq->sw_pidx] = swsqe->cqe;  in flush_completed_wrs()
269 			swsqe->cqe = *hw_cqe;  in c4iw_flush_hw_cq()
284 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)  in cqe_completes_wr()  argument
286 	if (CQE_OPCODE(cqe) == FW_RI_TERMINATE)  in cqe_completes_wr()
289 	if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe))  in cqe_completes_wr()
292 	if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe))  in cqe_completes_wr()
295 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))  in cqe_completes_wr()
302 	struct t4_cqe *cqe;  in c4iw_count_rcqes()  local
309 		cqe = &cq->sw_queue[ptr];  in c4iw_count_rcqes()
310 		if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) &&  in c4iw_count_rcqes()
311 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))  in c4iw_count_rcqes()
345 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,  in poll_cq()  argument
488 		swsqe->cqe = *hw_cqe;  in poll_cq()
495 	*cqe = *hw_cqe;  in poll_cq()
563 	struct t4_cqe cqe, *rd_cqe;  in c4iw_poll_cq_one()  local
598 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);  in c4iw_poll_cq_one()
602 	INC_STAT(cqe);  in c4iw_poll_cq_one()
605 	wc->vendor_err = CQE_STATUS(&cqe);  in c4iw_poll_cq_one()
610 	     CQE_QPID(&cqe), CQE_TYPE(&cqe),  in c4iw_poll_cq_one()
611 	     CQE_OPCODE(&cqe), CQE_STATUS(&cqe), CQE_WRID_HI(&cqe),  in c4iw_poll_cq_one()
612 	     CQE_WRID_LOW(&cqe), (unsigned long long)cookie);  in c4iw_poll_cq_one()
614 	if (CQE_TYPE(&cqe) == 0) {  in c4iw_poll_cq_one()
615 		if (!CQE_STATUS(&cqe))  in c4iw_poll_cq_one()
616 			wc->byte_len = CQE_LEN(&cqe);  in c4iw_poll_cq_one()
621 		switch (CQE_OPCODE(&cqe)) {  in c4iw_poll_cq_one()
627 			wc->byte_len = CQE_LEN(&cqe);  in c4iw_poll_cq_one()
641 			     CQE_OPCODE(&cqe), CQE_QPID(&cqe));  in c4iw_poll_cq_one()
651 		switch (CQE_STATUS(&cqe)) {  in c4iw_poll_cq_one()
696 			     CQE_STATUS(&cqe), CQE_QPID(&cqe));  in c4iw_poll_cq_one()
703 			chp->cq.cqid, CQE_QPID(&cqe), CQE_TYPE(&cqe),  in c4iw_poll_cq_one()
704 			CQE_OPCODE(&cqe), CQE_STATUS(&cqe));  in c4iw_poll_cq_one()