44 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)  in mlx4_ib_cq_comp()  argument
46 	struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;  in mlx4_ib_cq_comp()
50 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)  in mlx4_ib_cq_event()  argument
57 		       "on CQ %06x\n", type, cq->cqn);  in mlx4_ib_cq_event()
61 	ibcq = &to_mibcq(cq)->ibcq;  in mlx4_ib_cq_event()
65 		event.element.cq = ibcq;  in mlx4_ib_cq_event()
75 static void *get_cqe(struct mlx4_ib_cq *cq, int n)  in get_cqe()  argument
77 	return get_cqe_from_buf(&cq->buf, n);  in get_cqe()
80 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)  in get_sw_cqe()  argument
82 	struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);  in get_sw_cqe()
83 	struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe);  in get_sw_cqe()
86 		!!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;  in get_sw_cqe()
89 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)  in next_cqe_sw()  argument
91 	return get_sw_cqe(cq, cq->mcq.cons_index);  in next_cqe_sw()
94 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)  in mlx4_ib_modify_cq()  argument
96 	struct mlx4_ib_cq *mcq = to_mcq(cq);  in mlx4_ib_modify_cq()
97 	struct mlx4_ib_dev *dev = to_mdev(cq->device);  in mlx4_ib_modify_cq()
181 	struct mlx4_ib_cq *cq = to_mcq(ibcq);  in mlx4_ib_create_cq()  local
194 	cq->ibcq.cqe = entries - 1;  in mlx4_ib_create_cq()
195 	mutex_init(&cq->resize_mutex);  in mlx4_ib_create_cq()
196 	spin_lock_init(&cq->lock);  in mlx4_ib_create_cq()
197 	cq->resize_buf = NULL;  in mlx4_ib_create_cq()
198 	cq->resize_umem = NULL;  in mlx4_ib_create_cq()
199 	cq->create_flags = attr->flags;  in mlx4_ib_create_cq()
200 	INIT_LIST_HEAD(&cq->send_qp_list);  in mlx4_ib_create_cq()
201 	INIT_LIST_HEAD(&cq->recv_qp_list);  in mlx4_ib_create_cq()
211 		err = mlx4_ib_get_cq_umem(dev, udata, &cq->buf, &cq->umem,  in mlx4_ib_create_cq()
216 		err = mlx4_ib_db_map_user(context, ucmd.db_addr, &cq->db);  in mlx4_ib_create_cq()
222 		err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL);  in mlx4_ib_create_cq()
226 		cq->mcq.set_ci_db  = cq->db.db;  in mlx4_ib_create_cq()
227 		cq->mcq.arm_db     = cq->db.db + 1;  in mlx4_ib_create_cq()
228 		*cq->mcq.set_ci_db = 0;  in mlx4_ib_create_cq()
229 		*cq->mcq.arm_db    = 0;  in mlx4_ib_create_cq()
231 		err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);  in mlx4_ib_create_cq()
241 	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,  in mlx4_ib_create_cq()
242 			    cq->db.dma, &cq->mcq, vector, 0,  in mlx4_ib_create_cq()
243 			    !!(cq->create_flags & IB_CQ_FLAGS_TIMESTAMP_COMPLETION));  in mlx4_ib_create_cq()
247 	cq->mcq.comp = mlx4_ib_cq_comp;  in mlx4_ib_create_cq()
248 	cq->mcq.event = mlx4_ib_cq_event;  in mlx4_ib_create_cq()
251 		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {  in mlx4_ib_create_cq()
259 	mlx4_cq_free(dev->dev, &cq->mcq);  in mlx4_ib_create_cq()
263 		mlx4_ib_db_unmap_user(context, &cq->db);  in mlx4_ib_create_cq()
266 	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);  in mlx4_ib_create_cq()
268 	ib_umem_release(cq->umem);  in mlx4_ib_create_cq()
270 		mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);  in mlx4_ib_create_cq()
274 		mlx4_db_free(dev->dev, &cq->db);  in mlx4_ib_create_cq()
279 static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,  in mlx4_alloc_resize_buf()  argument
284 	if (cq->resize_buf)  in mlx4_alloc_resize_buf()
287 	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);  in mlx4_alloc_resize_buf()
288 	if (!cq->resize_buf)  in mlx4_alloc_resize_buf()
291 	err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);  in mlx4_alloc_resize_buf()
293 		kfree(cq->resize_buf);  in mlx4_alloc_resize_buf()
294 		cq->resize_buf = NULL;  in mlx4_alloc_resize_buf()
298 	cq->resize_buf->cqe = entries - 1;  in mlx4_alloc_resize_buf()
303 static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,  in mlx4_alloc_resize_umem()  argument
309 	if (cq->resize_umem)  in mlx4_alloc_resize_umem()
315 	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);  in mlx4_alloc_resize_umem()
316 	if (!cq->resize_buf)  in mlx4_alloc_resize_umem()
319 	err = mlx4_ib_get_cq_umem(dev, udata, &cq->resize_buf->buf,  in mlx4_alloc_resize_umem()
320 				  &cq->resize_umem, ucmd.buf_addr, entries);  in mlx4_alloc_resize_umem()
322 		kfree(cq->resize_buf);  in mlx4_alloc_resize_umem()
323 		cq->resize_buf = NULL;  in mlx4_alloc_resize_umem()
327 	cq->resize_buf->cqe = entries - 1;  in mlx4_alloc_resize_umem()
332 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)  in mlx4_ib_get_outstanding_cqes()  argument
336 	i = cq->mcq.cons_index;  in mlx4_ib_get_outstanding_cqes()
337 	while (get_sw_cqe(cq, i))  in mlx4_ib_get_outstanding_cqes()
340 	return i - cq->mcq.cons_index;  in mlx4_ib_get_outstanding_cqes()
343 static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)  in mlx4_ib_cq_resize_copy_cqes()  argument
347 	int cqe_size = cq->buf.entry_size;  in mlx4_ib_cq_resize_copy_cqes()
350 	i = cq->mcq.cons_index;  in mlx4_ib_cq_resize_copy_cqes()
351 	cqe = get_cqe(cq, i & cq->ibcq.cqe);  in mlx4_ib_cq_resize_copy_cqes()
355 		new_cqe = get_cqe_from_buf(&cq->resize_buf->buf,  in mlx4_ib_cq_resize_copy_cqes()
356 					   (i + 1) & cq->resize_buf->cqe);  in mlx4_ib_cq_resize_copy_cqes()
357 		memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size);  in mlx4_ib_cq_resize_copy_cqes()
361 			(((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);  in mlx4_ib_cq_resize_copy_cqes()
362 		cqe = get_cqe(cq, ++i & cq->ibcq.cqe);  in mlx4_ib_cq_resize_copy_cqes()
365 	++cq->mcq.cons_index;  in mlx4_ib_cq_resize_copy_cqes()
371 	struct mlx4_ib_cq *cq = to_mcq(ibcq);  in mlx4_ib_resize_cq()  local
376 	mutex_lock(&cq->resize_mutex);  in mlx4_ib_resize_cq()
394 		err = mlx4_alloc_resize_umem(dev, cq, entries, udata);  in mlx4_ib_resize_cq()
399 		outst_cqe = mlx4_ib_get_outstanding_cqes(cq);  in mlx4_ib_resize_cq()
405 		err = mlx4_alloc_resize_buf(dev, cq, entries);  in mlx4_ib_resize_cq()
410 	mtt = cq->buf.mtt;  in mlx4_ib_resize_cq()
412 	err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);  in mlx4_ib_resize_cq()
418 		cq->buf      = cq->resize_buf->buf;  in mlx4_ib_resize_cq()
419 		cq->ibcq.cqe = cq->resize_buf->cqe;  in mlx4_ib_resize_cq()
420 		ib_umem_release(cq->umem);  in mlx4_ib_resize_cq()
421 		cq->umem     = cq->resize_umem;  in mlx4_ib_resize_cq()
423 		kfree(cq->resize_buf);  in mlx4_ib_resize_cq()
424 		cq->resize_buf = NULL;  in mlx4_ib_resize_cq()
425 		cq->resize_umem = NULL;  in mlx4_ib_resize_cq()
430 		spin_lock_irq(&cq->lock);  in mlx4_ib_resize_cq()
431 		if (cq->resize_buf) {  in mlx4_ib_resize_cq()
432 			mlx4_ib_cq_resize_copy_cqes(cq);  in mlx4_ib_resize_cq()
433 			tmp_buf = cq->buf;  in mlx4_ib_resize_cq()
434 			tmp_cqe = cq->ibcq.cqe;  in mlx4_ib_resize_cq()
435 			cq->buf      = cq->resize_buf->buf;  in mlx4_ib_resize_cq()
436 			cq->ibcq.cqe = cq->resize_buf->cqe;  in mlx4_ib_resize_cq()
438 			kfree(cq->resize_buf);  in mlx4_ib_resize_cq()
439 			cq->resize_buf = NULL;  in mlx4_ib_resize_cq()
441 		spin_unlock_irq(&cq->lock);  in mlx4_ib_resize_cq()
450 	mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt);  in mlx4_ib_resize_cq()
452 		mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,  in mlx4_ib_resize_cq()
453 				    cq->resize_buf->cqe);  in mlx4_ib_resize_cq()
455 	kfree(cq->resize_buf);  in mlx4_ib_resize_cq()
456 	cq->resize_buf = NULL;  in mlx4_ib_resize_cq()
458 	ib_umem_release(cq->resize_umem);  in mlx4_ib_resize_cq()
459 	cq->resize_umem = NULL;  in mlx4_ib_resize_cq()
461 	mutex_unlock(&cq->resize_mutex);  in mlx4_ib_resize_cq()
466 void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)  in mlx4_ib_destroy_cq()  argument
468 	struct mlx4_ib_dev *dev = to_mdev(cq->device);  in mlx4_ib_destroy_cq()
469 	struct mlx4_ib_cq *mcq = to_mcq(cq);  in mlx4_ib_destroy_cq()
482 		mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);  in mlx4_ib_destroy_cq()
573 static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,  in use_tunnel_data()  argument
624 static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries,  in mlx4_ib_poll_sw_comp()  argument
633 	list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) {  in mlx4_ib_poll_sw_comp()
639 	list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) {  in mlx4_ib_poll_sw_comp()
649 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,  in mlx4_ib_poll_one()  argument
666 	cqe = next_cqe_sw(cq);  in mlx4_ib_poll_one()
670 	if (cq->buf.entry_size == 64)  in mlx4_ib_poll_one()
673 	++cq->mcq.cons_index;  in mlx4_ib_poll_one()
687 		if (cq->resize_buf) {  in mlx4_ib_poll_one()
688 			struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);  in mlx4_ib_poll_one()
690 			mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);  in mlx4_ib_poll_one()
691 			cq->buf      = cq->resize_buf->buf;  in mlx4_ib_poll_one()
692 			cq->ibcq.cqe = cq->resize_buf->cqe;  in mlx4_ib_poll_one()
694 			kfree(cq->resize_buf);  in mlx4_ib_poll_one()
695 			cq->resize_buf = NULL;  in mlx4_ib_poll_one()
708 		mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,  in mlx4_ib_poll_one()
720 		msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev,  in mlx4_ib_poll_one()
828 		if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) {  in mlx4_ib_poll_one()
832 				use_tunnel_data(*cur_qp, cq, wc, tail, cqe,  in mlx4_ib_poll_one()
869 	struct mlx4_ib_cq *cq = to_mcq(ibcq);  in mlx4_ib_poll_cq()  local
873 	struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);  in mlx4_ib_poll_cq()
875 	spin_lock_irqsave(&cq->lock, flags);  in mlx4_ib_poll_cq()
878 		mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled);  in mlx4_ib_poll_cq()
883 		if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled))  in mlx4_ib_poll_cq()
887 	mlx4_cq_set_ci(&cq->mcq);  in mlx4_ib_poll_cq()
890 	spin_unlock_irqrestore(&cq->lock, flags);  in mlx4_ib_poll_cq()
897 	struct mlx4_ib_cq *cq = to_mcq(ibcq);  in mlx4_ib_arm_cq()  local
898 	struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);  in mlx4_ib_arm_cq()
904 	mlx4_cq_arm(&cq->mcq,  in mlx4_ib_arm_cq()
913 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)  in __mlx4_ib_cq_clean()  argument
919 	int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0;  in __mlx4_ib_cq_clean()
928 	for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)  in __mlx4_ib_cq_clean()
929 		if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)  in __mlx4_ib_cq_clean()
936 	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {  in __mlx4_ib_cq_clean()
937 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);  in __mlx4_ib_cq_clean()
945 			dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);  in __mlx4_ib_cq_clean()
956 		cq->mcq.cons_index += nfreed;  in __mlx4_ib_cq_clean()
962 		mlx4_cq_set_ci(&cq->mcq);  in __mlx4_ib_cq_clean()
966 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)  in mlx4_ib_cq_clean()  argument
968 	spin_lock_irq(&cq->lock);  in mlx4_ib_cq_clean()
969 	__mlx4_ib_cq_clean(cq, qpn, srq);  in mlx4_ib_cq_clean()
970 	spin_unlock_irq(&cq->lock);  in mlx4_ib_cq_clean()