169 static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry)  in get_cqe()  argument
171 	return get_cqe_from_buf(&cq->buf, entry);  in get_cqe()
179 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)  in next_cqe_sw()  argument
181 	return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe));  in next_cqe_sw()
204 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,  in update_cons_index()  argument
208 		*cq->set_ci_db = cpu_to_be32(cq->cons_index);  in update_cons_index()
211 		mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1,  in update_cons_index()
224 	struct mthca_cq *cq;  in mthca_cq_completion()  local
226 	cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));  in mthca_cq_completion()
228 	if (!cq) {  in mthca_cq_completion()
233 	++cq->arm_sn;  in mthca_cq_completion()
235 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);  in mthca_cq_completion()
241 	struct mthca_cq *cq;  in mthca_cq_event()  local
246 	cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));  in mthca_cq_event()
247 	if (cq)  in mthca_cq_event()
248 		++cq->refcount;  in mthca_cq_event()
252 	if (!cq) {  in mthca_cq_event()
259 	event.element.cq  = &cq->ibcq;  in mthca_cq_event()
260 	if (cq->ibcq.event_handler)  in mthca_cq_event()
261 		cq->ibcq.event_handler(&event, cq->ibcq.cq_context);  in mthca_cq_event()
264 	if (!--cq->refcount)  in mthca_cq_event()
265 		wake_up(&cq->wait);  in mthca_cq_event()
278 void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn,  in mthca_cq_clean()  argument
285 	spin_lock_irq(&cq->lock);  in mthca_cq_clean()
294 	for (prod_index = cq->cons_index;  in mthca_cq_clean()
295 	     cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe));  in mthca_cq_clean()
297 		if (prod_index == cq->cons_index + cq->ibcq.cqe)  in mthca_cq_clean()
302 			  qpn, cq->cqn, cq->cons_index, prod_index);  in mthca_cq_clean()
308 	while ((int) --prod_index - (int) cq->cons_index >= 0) {  in mthca_cq_clean()
309 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);  in mthca_cq_clean()
315 			memcpy(get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe),  in mthca_cq_clean()
321 			set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe));  in mthca_cq_clean()
323 		cq->cons_index += nfreed;  in mthca_cq_clean()
324 		update_cons_index(dev, cq, nfreed);  in mthca_cq_clean()
327 	spin_unlock_irq(&cq->lock);  in mthca_cq_clean()
330 void mthca_cq_resize_copy_cqes(struct mthca_cq *cq)  in mthca_cq_resize_copy_cqes()  argument
340 	if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) &&  in mthca_cq_resize_copy_cqes()
341 	    cq->ibcq.cqe < cq->resize_buf->cqe) {  in mthca_cq_resize_copy_cqes()
342 		cq->cons_index &= cq->ibcq.cqe;  in mthca_cq_resize_copy_cqes()
343 		if (cqe_sw(get_cqe(cq, cq->ibcq.cqe)))  in mthca_cq_resize_copy_cqes()
344 			cq->cons_index -= cq->ibcq.cqe + 1;  in mthca_cq_resize_copy_cqes()
347 	for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i)  in mthca_cq_resize_copy_cqes()
348 		memcpy(get_cqe_from_buf(&cq->resize_buf->buf,  in mthca_cq_resize_copy_cqes()
349 					i & cq->resize_buf->cqe),  in mthca_cq_resize_copy_cqes()
350 		       get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE);  in mthca_cq_resize_copy_cqes()
377 static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,  in handle_error_cqe()  argument
389 			  cq->cqn, cq->cons_index);  in handle_error_cqe()
484 				 struct mthca_cq *cq,  in mthca_poll_one()  argument
498 	cqe = next_cqe_sw(cq);  in mthca_poll_one()
510 			  cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn),  in mthca_poll_one()
576 		handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,  in mthca_poll_one()
653 		++cq->cons_index;  in mthca_poll_one()
663 	struct mthca_cq *cq = to_mcq(ibcq);  in mthca_poll_cq()  local
670 	spin_lock_irqsave(&cq->lock, flags);  in mthca_poll_cq()
675 		err = mthca_poll_one(dev, cq, &qp,  in mthca_poll_cq()
684 		update_cons_index(dev, cq, freed);  in mthca_poll_cq()
693 	if (unlikely(err == -EAGAIN && cq->resize_buf &&  in mthca_poll_cq()
694 		     cq->resize_buf->state == CQ_RESIZE_READY)) {  in mthca_poll_cq()
703 			cq->cons_index &= cq->ibcq.cqe;  in mthca_poll_cq()
705 		if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf,  in mthca_poll_cq()
706 					    cq->cons_index & cq->resize_buf->cqe))) {  in mthca_poll_cq()
710 			tbuf         = cq->buf;  in mthca_poll_cq()
711 			tcqe         = cq->ibcq.cqe;  in mthca_poll_cq()
712 			cq->buf      = cq->resize_buf->buf;  in mthca_poll_cq()
713 			cq->ibcq.cqe = cq->resize_buf->cqe;  in mthca_poll_cq()
715 			cq->resize_buf->buf   = tbuf;  in mthca_poll_cq()
716 			cq->resize_buf->cqe   = tcqe;  in mthca_poll_cq()
717 			cq->resize_buf->state = CQ_RESIZE_SWAPPED;  in mthca_poll_cq()
723 	spin_unlock_irqrestore(&cq->lock, flags);  in mthca_poll_cq()
728 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags)  in mthca_tavor_arm_cq()  argument
733 		to_mcq(cq)->cqn;  in mthca_tavor_arm_cq()
735 	mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,  in mthca_tavor_arm_cq()
736 		      MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock));  in mthca_tavor_arm_cq()
743 	struct mthca_cq *cq = to_mcq(ibcq);  in mthca_arbel_arm_cq()  local
746 	u32 sn = cq->arm_sn & 3;  in mthca_arbel_arm_cq()
748 	db_rec[0] = cpu_to_be32(cq->cons_index);  in mthca_arbel_arm_cq()
749 	db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) |  in mthca_arbel_arm_cq()
753 	mthca_write_db_rec(db_rec, cq->arm_db);  in mthca_arbel_arm_cq()
764 		 MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn;  in mthca_arbel_arm_cq()
766 	mthca_write64(dbhi, cq->cons_index,  in mthca_arbel_arm_cq()
775 		  struct mthca_cq *cq)  in mthca_init_cq()  argument
781 	cq->ibcq.cqe  = nent - 1;  in mthca_init_cq()
782 	cq->is_kernel = !ctx;  in mthca_init_cq()
784 	cq->cqn = mthca_alloc(&dev->cq_table.alloc);  in mthca_init_cq()
785 	if (cq->cqn == -1)  in mthca_init_cq()
789 		err = mthca_table_get(dev, dev->cq_table.table, cq->cqn);  in mthca_init_cq()
793 		if (cq->is_kernel) {  in mthca_init_cq()
794 			cq->arm_sn = 1;  in mthca_init_cq()
798 			cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,  in mthca_init_cq()
799 							     cq->cqn, &cq->set_ci_db);  in mthca_init_cq()
800 			if (cq->set_ci_db_index < 0)  in mthca_init_cq()
803 			cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,  in mthca_init_cq()
804 							  cq->cqn, &cq->arm_db);  in mthca_init_cq()
805 			if (cq->arm_db_index < 0)  in mthca_init_cq()
816 	if (cq->is_kernel) {  in mthca_init_cq()
817 		err = mthca_alloc_cq_buf(dev, &cq->buf, nent);  in mthca_init_cq()
822 	spin_lock_init(&cq->lock);  in mthca_init_cq()
823 	cq->refcount = 1;  in mthca_init_cq()
824 	init_waitqueue_head(&cq->wait);  in mthca_init_cq()
825 	mutex_init(&cq->mutex);  in mthca_init_cq()
839 	cq_context->lkey            = cpu_to_be32(cq->buf.mr.ibmr.lkey);  in mthca_init_cq()
840 	cq_context->cqn             = cpu_to_be32(cq->cqn);  in mthca_init_cq()
843 		cq_context->ci_db    = cpu_to_be32(cq->set_ci_db_index);  in mthca_init_cq()
844 		cq_context->state_db = cpu_to_be32(cq->arm_db_index);  in mthca_init_cq()
847 	err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn);  in mthca_init_cq()
854 	if (mthca_array_set(&dev->cq_table.cq,  in mthca_init_cq()
855 			    cq->cqn & (dev->limits.num_cqs - 1),  in mthca_init_cq()
856 			    cq)) {  in mthca_init_cq()
862 	cq->cons_index = 0;  in mthca_init_cq()
869 	if (cq->is_kernel)  in mthca_init_cq()
870 		mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);  in mthca_init_cq()
876 	if (cq->is_kernel && mthca_is_memfree(dev))  in mthca_init_cq()
877 		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);  in mthca_init_cq()
880 	if (cq->is_kernel && mthca_is_memfree(dev))  in mthca_init_cq()
881 		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);  in mthca_init_cq()
884 	mthca_table_put(dev, dev->cq_table.table, cq->cqn);  in mthca_init_cq()
887 	mthca_free(&dev->cq_table.alloc, cq->cqn);  in mthca_init_cq()
892 static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq)  in get_cq_refcount()  argument
897 	c = cq->refcount;  in get_cq_refcount()
904 		   struct mthca_cq *cq)  in mthca_free_cq()  argument
915 	err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn);  in mthca_free_cq()
924 		       cq->cqn, cq->cons_index,  in mthca_free_cq()
925 		       cq->is_kernel ? !!next_cqe_sw(cq) : 0);  in mthca_free_cq()
931 	mthca_array_clear(&dev->cq_table.cq,  in mthca_free_cq()
932 			  cq->cqn & (dev->limits.num_cqs - 1));  in mthca_free_cq()
933 	--cq->refcount;  in mthca_free_cq()
941 	wait_event(cq->wait, !get_cq_refcount(dev, cq));  in mthca_free_cq()
943 	if (cq->is_kernel) {  in mthca_free_cq()
944 		mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);  in mthca_free_cq()
946 			mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);  in mthca_free_cq()
947 			mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);  in mthca_free_cq()
951 	mthca_table_put(dev, dev->cq_table.table, cq->cqn);  in mthca_free_cq()
952 	mthca_free(&dev->cq_table.alloc, cq->cqn);  in mthca_free_cq()
969 	err = mthca_array_init(&dev->cq_table.cq,  in mthca_init_cq_table()
979 	mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs);  in mthca_cleanup_cq_table()