42 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)  in insert_recv_cqe()  argument
47 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_recv_cqe()
53 				 V_CQE_QPID(wq->sq.qid));  in insert_recv_cqe()
59 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)  in c4iw_flush_rq()  argument
62 	int in_use = wq->rq.in_use - count;  in c4iw_flush_rq()
66 	     wq, cq, wq->rq.in_use, count);  in c4iw_flush_rq()
68 		insert_recv_cqe(wq, cq);  in c4iw_flush_rq()
74 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,  in insert_sq_cqe()  argument
80 	     wq, cq, cq->sw_cidx, cq->sw_pidx);  in insert_sq_cqe()
86 			         V_CQE_QPID(wq->sq.qid));  in insert_sq_cqe()
93 static void advance_oldest_read(struct t4_wq *wq);
98 	struct t4_wq *wq = &qhp->wq;  in c4iw_flush_sq()  local
104 	if (wq->sq.flush_cidx == -1)  in c4iw_flush_sq()
105 		wq->sq.flush_cidx = wq->sq.cidx;  in c4iw_flush_sq()
106 	idx = wq->sq.flush_cidx;  in c4iw_flush_sq()
107 	BUG_ON(idx >= wq->sq.size);  in c4iw_flush_sq()
108 	while (idx != wq->sq.pidx) {  in c4iw_flush_sq()
109 		swsqe = &wq->sq.sw_sq[idx];  in c4iw_flush_sq()
112 		insert_sq_cqe(wq, cq, swsqe);  in c4iw_flush_sq()
113 		if (wq->sq.oldest_read == swsqe) {  in c4iw_flush_sq()
115 			advance_oldest_read(wq);  in c4iw_flush_sq()
118 		if (++idx == wq->sq.size)  in c4iw_flush_sq()
121 	wq->sq.flush_cidx += flushed;  in c4iw_flush_sq()
122 	if (wq->sq.flush_cidx >= wq->sq.size)  in c4iw_flush_sq()
123 		wq->sq.flush_cidx -= wq->sq.size;  in c4iw_flush_sq()
126 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)  in flush_completed_wrs()  argument
131 	if (wq->sq.flush_cidx == -1)  in flush_completed_wrs()
132 		wq->sq.flush_cidx = wq->sq.cidx;  in flush_completed_wrs()
133 	cidx = wq->sq.flush_cidx;  in flush_completed_wrs()
134 	BUG_ON(cidx >= wq->sq.size);  in flush_completed_wrs()
136 	while (cidx != wq->sq.pidx) {  in flush_completed_wrs()
137 		swsqe = &wq->sq.sw_sq[cidx];  in flush_completed_wrs()
139 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
155 			if (++cidx == wq->sq.size)  in flush_completed_wrs()
157 			wq->sq.flush_cidx = cidx;  in flush_completed_wrs()
163 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,  in create_read_req_cqe()  argument
166 	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;  in create_read_req_cqe()
167 	read_cqe->len = be32toh(wq->sq.oldest_read->read_len);  in create_read_req_cqe()
175 static void advance_oldest_read(struct t4_wq *wq)  in advance_oldest_read()  argument
178 	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;  in advance_oldest_read()
180 	if (rptr == wq->sq.size)  in advance_oldest_read()
182 	while (rptr != wq->sq.pidx) {  in advance_oldest_read()
183 		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];  in advance_oldest_read()
185 		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)  in advance_oldest_read()
187 		if (++rptr == wq->sq.size)  in advance_oldest_read()
190 	wq->sq.oldest_read = NULL;  in advance_oldest_read()
247 			if (!qhp->wq.sq.oldest_read->signaled) {  in c4iw_flush_hw_cq()
248 				advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
256 			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);  in c4iw_flush_hw_cq()
258 			advance_oldest_read(&qhp->wq);  in c4iw_flush_hw_cq()
267 			BUG_ON(idx >= qhp->wq.sq.size);  in c4iw_flush_hw_cq()
268 			swsqe = &qhp->wq.sq.sw_sq[idx];  in c4iw_flush_hw_cq()
271 			flush_completed_wrs(&qhp->wq, &chp->cq);  in c4iw_flush_hw_cq()
284 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)  in cqe_completes_wr()  argument
295 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))  in cqe_completes_wr()
300 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)  in c4iw_count_rcqes()  argument
311 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))  in c4iw_count_rcqes()
345 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,  in poll_cq()  argument
368 	if (wq == NULL) {  in poll_cq()
392 				t4_set_wq_in_error(wq);  in poll_cq()
405 				t4_set_wq_in_error(wq);  in poll_cq()
413 		if (!wq->sq.oldest_read->signaled) {  in poll_cq()
414 			advance_oldest_read(wq);  in poll_cq()
423 		create_read_req_cqe(wq, hw_cqe, &read_cqe);  in poll_cq()
425 		advance_oldest_read(wq);  in poll_cq()
433 	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {  in poll_cq()
435 		wq->error = 1;  in poll_cq()
456 		if (t4_rq_empty(wq)) {  in poll_cq()
457 			t4_set_wq_in_error(wq);  in poll_cq()
461 		if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) {  in poll_cq()
462 			t4_set_wq_in_error(wq);  in poll_cq()
480 	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {  in poll_cq()
486 		BUG_ON(idx >= wq->sq.size);  in poll_cq()
487 		swsqe = &wq->sq.sw_sq[idx];  in poll_cq()
503 		BUG_ON(idx >= wq->sq.size);  in poll_cq()
513 		if (idx < wq->sq.cidx)  in poll_cq()
514 			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;  in poll_cq()
516 			wq->sq.in_use -= idx - wq->sq.cidx;  in poll_cq()
517 		BUG_ON(wq->sq.in_use <= 0 || wq->sq.in_use >= wq->sq.size);  in poll_cq()
519 		wq->sq.cidx = (u16)idx;  in poll_cq()
520 		PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx);  in poll_cq()
521 		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;  in poll_cq()
522 		t4_sq_consume(wq);  in poll_cq()
524 		PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx);  in poll_cq()
525 		BUG_ON(wq->rq.cidx >= wq->rq.size);  in poll_cq()
526 		*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;  in poll_cq()
527 		BUG_ON(t4_rq_empty(wq));  in poll_cq()
528 		t4_rq_consume(wq);  in poll_cq()
536 	flush_completed_wrs(wq, cq);  in poll_cq()
564 	struct t4_wq *wq;  in c4iw_poll_cq_one()  local
593 		wq = NULL;  in c4iw_poll_cq_one()
596 		wq = &(qhp->wq);  in c4iw_poll_cq_one()
598 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);  in c4iw_poll_cq_one()
604 	wc->qp_num = qhp->wq.sq.qid;  in c4iw_poll_cq_one()
706 	if (wq)  in c4iw_poll_cq_one()