52 static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,  in destroy_cq()  argument
76 	res->u.cq.restype = FW_RI_RES_TYPE_CQ;  in destroy_cq()
77 	res->u.cq.op = FW_RI_RES_OP_RESET;  in destroy_cq()
78 	res->u.cq.iqid = cpu_to_be32(cq->cqid);  in destroy_cq()
86 	kfree(cq->sw_queue);  in destroy_cq()
88 			  cq->memsize, cq->queue,  in destroy_cq()
89 			  dma_unmap_addr(cq, mapping));  in destroy_cq()
90 	c4iw_put_cqid(rdev, cq->cqid, uctx);  in destroy_cq()
95 create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,  in create_cq()  argument
111 	cq->cqid = c4iw_get_cqid(rdev, uctx);  in create_cq()
112 	if (!cq->cqid) {  in create_cq()
118 		cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL);  in create_cq()
119 		if (!cq->sw_queue) {  in create_cq()
124 	cq->queue = dma_alloc_coherent(rhp->ibdev.dma_device, cq->memsize,  in create_cq()
125 				       &cq->dma_addr, GFP_KERNEL);  in create_cq()
126 	if (!cq->queue) {  in create_cq()
130 	dma_unmap_addr_set(cq, mapping, cq->dma_addr);  in create_cq()
131 	memset(cq->queue, 0, cq->memsize);  in create_cq()
149 	res->u.cq.restype = FW_RI_RES_TYPE_CQ;  in create_cq()
150 	res->u.cq.op = FW_RI_RES_OP_WRITE;  in create_cq()
151 	res->u.cq.iqid = cpu_to_be32(cq->cqid);  in create_cq()
153 	res->u.cq.iqandst_to_iqandstindex = cpu_to_be32(  in create_cq()
158 	res->u.cq.iqdroprss_to_iqesize = cpu_to_be16(  in create_cq()
164 	res->u.cq.iqsize = cpu_to_be16(cq->size);  in create_cq()
165 	res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr);  in create_cq()
176 	cq->gen = 1;  in create_cq()
177 	cq->rdev = rdev;  in create_cq()
180 	t4_bar2_sge_qregs(rdev->adap, cq->cqid, T4_BAR2_QTYPE_INGRESS, user,  in create_cq()
181 			&cq_bar2_qoffset, &cq->bar2_qid);  in create_cq()
187 		cq->bar2_pa = (rdev->bar2_pa + cq_bar2_qoffset) & PAGE_MASK;  in create_cq()
189 		cq->bar2_va = (void __iomem *)((u64)rdev->bar2_kva +  in create_cq()
194 	dma_free_coherent(rhp->ibdev.dma_device, cq->memsize, cq->queue,  in create_cq()
195 			  dma_unmap_addr(cq, mapping));  in create_cq()
197 	kfree(cq->sw_queue);  in create_cq()
199 	c4iw_put_cqid(rdev, cq->cqid, uctx);  in create_cq()
204 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)  in insert_recv_cqe()  argument
209 	    cq, cq->sw_cidx, cq->sw_pidx);  in insert_recv_cqe()
216 	cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen));  in insert_recv_cqe()
217 	cq->sw_queue[cq->sw_pidx] = cqe;  in insert_recv_cqe()
218 	t4_swcq_produce(cq);  in insert_recv_cqe()
221 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)  in c4iw_flush_rq()  argument
228 	    __func__, wq, cq, wq->rq.in_use, count);  in c4iw_flush_rq()
230 		insert_recv_cqe(wq, cq);  in c4iw_flush_rq()
236 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,  in insert_sq_cqe()  argument
242 	    cq, cq->sw_cidx, cq->sw_pidx);  in insert_sq_cqe()
250 	cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen));  in insert_sq_cqe()
251 	cq->sw_queue[cq->sw_pidx] = cqe;  in insert_sq_cqe()
252 	t4_swcq_produce(cq);  in insert_sq_cqe()
262 	struct t4_cq *cq = &chp->cq;  in c4iw_flush_sq()  local
274 		insert_sq_cqe(wq, cq, swsqe);  in c4iw_flush_sq()
289 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)  in flush_completed_wrs()  argument
313 				__func__, cidx, cq->sw_pidx);  in flush_completed_wrs()
315 			cq->sw_queue[cq->sw_pidx] = swsqe->cqe;  in flush_completed_wrs()
316 			t4_swcq_produce(cq);  in flush_completed_wrs()
368 	CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, &chp->cq,  in c4iw_flush_hw_cq()
369 			chp->cq.cqid);  in c4iw_flush_hw_cq()
370 	ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);  in c4iw_flush_hw_cq()
427 			flush_completed_wrs(&qhp->wq, &chp->cq);  in c4iw_flush_hw_cq()
429 			swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];  in c4iw_flush_hw_cq()
432 			t4_swcq_produce(&chp->cq);  in c4iw_flush_hw_cq()
435 		t4_hwcq_consume(&chp->cq);  in c4iw_flush_hw_cq()
436 		ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);  in c4iw_flush_hw_cq()
456 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)  in c4iw_count_rcqes()  argument
463 	ptr = cq->sw_cidx;  in c4iw_count_rcqes()
464 	while (ptr != cq->sw_pidx) {  in c4iw_count_rcqes()
465 		cqe = &cq->sw_queue[ptr];  in c4iw_count_rcqes()
469 		if (++ptr == cq->size)  in c4iw_count_rcqes()
472 	CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count);  in c4iw_count_rcqes()
491 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,  in poll_cq()  argument
499 	ret = t4_next_cqe(cq, &hw_cqe);  in poll_cq()
691 	flush_completed_wrs(wq, cq);  in poll_cq()
696 		     __func__, cq, cq->cqid, cq->sw_cidx);  in poll_cq()
697 		t4_swcq_consume(cq);  in poll_cq()
700 		     __func__, cq, cq->cqid, cq->cidx);  in poll_cq()
701 		t4_hwcq_consume(cq);  in poll_cq()
725 	ret = t4_next_cqe(&chp->cq, &rd_cqe);  in c4iw_poll_cq_one()
737 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);  in c4iw_poll_cq_one()
894 	remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);  in c4iw_destroy_cq()
900 	destroy_cq(&chp->rhp->rdev, &chp->cq,  in c4iw_destroy_cq()
901 		   ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx);  in c4iw_destroy_cq()
950 	memsize = hwentries * sizeof *chp->cq.queue;  in c4iw_create_cq()
957 	chp->cq.size = hwentries;  in c4iw_create_cq()
958 	chp->cq.memsize = memsize;  in c4iw_create_cq()
959 	chp->cq.vector = vector;  in c4iw_create_cq()
961 	ret = create_cq(&rhp->rdev, &chp->cq,  in c4iw_create_cq()
967 	chp->cq.size--;				/* status page */  in c4iw_create_cq()
973 	ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid);  in c4iw_create_cq()
988 		uresp.cqid = chp->cq.cqid;  in c4iw_create_cq()
989 		uresp.size = chp->cq.size;  in c4iw_create_cq()
990 		uresp.memsize = chp->cq.memsize;  in c4iw_create_cq()
1003 		mm->addr = vtophys(chp->cq.queue);  in c4iw_create_cq()
1004 		mm->len = chp->cq.memsize;  in c4iw_create_cq()
1008 		mm2->addr = chp->cq.bar2_pa;  in c4iw_create_cq()
1014 	    __func__, chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize,  in c4iw_create_cq()
1015 	    (unsigned long long) chp->cq.dma_addr);  in c4iw_create_cq()
1022 	remove_handle(rhp, &rhp->cqidr, chp->cq.cqid);  in c4iw_create_cq()
1024 	destroy_cq(&chp->rhp->rdev, &chp->cq,  in c4iw_create_cq()
1030 int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)  in c4iw_resize_cq()  argument
1042 	if (__predict_false(c4iw_stopped(chp->cq.rdev)))  in c4iw_arm_cq()
1045 	t4_arm_cq(&chp->cq,  in c4iw_arm_cq()
1048 		ret = t4_cq_notempty(&chp->cq);  in c4iw_arm_cq()