xref: /freebsd/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 /*-
2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include <linux/kref.h>
29 #include <rdma/ib_umem.h>
30 #include <rdma/ib_user_verbs.h>
31 #include <rdma/ib_cache.h>
32 #include "mlx5_ib.h"
33 
34 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq, struct mlx5_eqe *eqe __unused)
35 {
36 	struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
37 
38 	ibcq->comp_handler(ibcq, ibcq->cq_context);
39 }
40 
41 static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, int type)
42 {
43 	struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq);
44 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
45 	struct ib_cq *ibcq = &cq->ibcq;
46 	struct ib_event event;
47 
48 	if (type != MLX5_EVENT_TYPE_CQ_ERROR) {
49 		mlx5_ib_warn(dev, "Unexpected event type %d on CQ %06x\n",
50 			     type, mcq->cqn);
51 		return;
52 	}
53 
54 	if (ibcq->event_handler) {
55 		event.device     = &dev->ib_dev;
56 		event.event      = IB_EVENT_CQ_ERR;
57 		event.element.cq = ibcq;
58 		ibcq->event_handler(&event, ibcq->cq_context);
59 	}
60 }
61 
62 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size)
63 {
64 	return mlx5_buf_offset(&buf->buf, n * size);
65 }
66 
67 static void *get_cqe(struct mlx5_ib_cq *cq, int n)
68 {
69 	return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz);
70 }
71 
72 static u8 sw_ownership_bit(int n, int nent)
73 {
74 	return (n & nent) ? 1 : 0;
75 }
76 
77 static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n)
78 {
79 	void *cqe = get_cqe(cq, n & cq->ibcq.cqe);
80 	struct mlx5_cqe64 *cqe64;
81 
82 	cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
83 
84 	if (likely((cqe64->op_own) >> 4 != MLX5_CQE_INVALID) &&
85 	    !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1)))) {
86 		return cqe;
87 	} else {
88 		return NULL;
89 	}
90 }
91 
92 static void *next_cqe_sw(struct mlx5_ib_cq *cq)
93 {
94 	return get_sw_cqe(cq, cq->mcq.cons_index);
95 }
96 
97 static enum ib_wc_opcode get_umr_comp(struct mlx5_ib_wq *wq, int idx)
98 {
99 	switch (wq->wr_data[idx]) {
100 	case MLX5_IB_WR_UMR:
101 		return 0;
102 
103 	case IB_WR_LOCAL_INV:
104 		return IB_WC_LOCAL_INV;
105 
106 	case IB_WR_REG_MR:
107 		return IB_WC_REG_MR;
108 
109 	default:
110 		pr_warn("unknown completion status\n");
111 		return 0;
112 	}
113 }
114 
115 static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
116 			    struct mlx5_ib_wq *wq, int idx)
117 {
118 	wc->wc_flags = 0;
119 	switch (be32_to_cpu(cqe->sop_drop_qpn) >> 24) {
120 	case MLX5_OPCODE_RDMA_WRITE_IMM:
121 		wc->wc_flags |= IB_WC_WITH_IMM;
122 	case MLX5_OPCODE_RDMA_WRITE:
123 		wc->opcode    = IB_WC_RDMA_WRITE;
124 		break;
125 	case MLX5_OPCODE_SEND_IMM:
126 		wc->wc_flags |= IB_WC_WITH_IMM;
127 	case MLX5_OPCODE_SEND:
128 	case MLX5_OPCODE_SEND_INVAL:
129 		wc->opcode    = IB_WC_SEND;
130 		break;
131 	case MLX5_OPCODE_RDMA_READ:
132 		wc->opcode    = IB_WC_RDMA_READ;
133 		wc->byte_len  = be32_to_cpu(cqe->byte_cnt);
134 		break;
135 	case MLX5_OPCODE_ATOMIC_CS:
136 		wc->opcode    = IB_WC_COMP_SWAP;
137 		wc->byte_len  = 8;
138 		break;
139 	case MLX5_OPCODE_ATOMIC_FA:
140 		wc->opcode    = IB_WC_FETCH_ADD;
141 		wc->byte_len  = 8;
142 		break;
143 	case MLX5_OPCODE_ATOMIC_MASKED_CS:
144 		wc->opcode    = IB_WC_MASKED_COMP_SWAP;
145 		wc->byte_len  = 8;
146 		break;
147 	case MLX5_OPCODE_ATOMIC_MASKED_FA:
148 		wc->opcode    = IB_WC_MASKED_FETCH_ADD;
149 		wc->byte_len  = 8;
150 		break;
151 	case MLX5_OPCODE_UMR:
152 		wc->opcode = get_umr_comp(wq, idx);
153 		break;
154 	}
155 }
156 
157 enum {
158 	MLX5_GRH_IN_BUFFER = 1,
159 	MLX5_GRH_IN_CQE	   = 2,
160 };
161 
162 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
163 			     struct mlx5_ib_qp *qp)
164 {
165 	enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1);
166 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
167 	struct mlx5_ib_srq *srq;
168 	struct mlx5_ib_wq *wq;
169 	u16 wqe_ctr;
170 	u8  roce_packet_type;
171 	bool vlan_present;
172 	u8 g;
173 
174 	if (qp->ibqp.srq || qp->ibqp.xrcd) {
175 		struct mlx5_core_srq *msrq = NULL;
176 
177 		if (qp->ibqp.xrcd) {
178 			msrq = mlx5_core_get_srq(dev->mdev,
179 						 be32_to_cpu(cqe->srqn));
180 			srq = to_mibsrq(msrq);
181 		} else {
182 			srq = to_msrq(qp->ibqp.srq);
183 		}
184 		if (srq) {
185 			wqe_ctr = be16_to_cpu(cqe->wqe_counter);
186 			wc->wr_id = srq->wrid[wqe_ctr];
187 			mlx5_ib_free_srq_wqe(srq, wqe_ctr);
188 			if (msrq && atomic_dec_and_test(&msrq->refcount))
189 				complete(&msrq->free);
190 		}
191 	} else {
192 		wq	  = &qp->rq;
193 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
194 		++wq->tail;
195 	}
196 	wc->byte_len = be32_to_cpu(cqe->byte_cnt);
197 
198 	switch (cqe->op_own >> 4) {
199 	case MLX5_CQE_RESP_WR_IMM:
200 		wc->opcode	= IB_WC_RECV_RDMA_WITH_IMM;
201 		wc->wc_flags	= IB_WC_WITH_IMM;
202 		wc->ex.imm_data = cqe->imm_inval_pkey;
203 		break;
204 	case MLX5_CQE_RESP_SEND:
205 		wc->opcode   = IB_WC_RECV;
206 		wc->wc_flags = IB_WC_IP_CSUM_OK;
207 		if (unlikely(!((cqe->hds_ip_ext & CQE_L3_OK) &&
208 			       (cqe->hds_ip_ext & CQE_L4_OK))))
209 			wc->wc_flags = 0;
210 		break;
211 	case MLX5_CQE_RESP_SEND_IMM:
212 		wc->opcode	= IB_WC_RECV;
213 		wc->wc_flags	= IB_WC_WITH_IMM;
214 		wc->ex.imm_data = cqe->imm_inval_pkey;
215 		break;
216 	case MLX5_CQE_RESP_SEND_INV:
217 		wc->opcode	= IB_WC_RECV;
218 		wc->wc_flags	= IB_WC_WITH_INVALIDATE;
219 		wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
220 		break;
221 	}
222 	wc->slid	   = be16_to_cpu(cqe->slid);
223 	wc->src_qp	   = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
224 	wc->dlid_path_bits = cqe->ml_path;
225 	g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
226 	wc->wc_flags |= g ? IB_WC_GRH : 0;
227 	if (unlikely(is_qp1(qp->ibqp.qp_type))) {
228 		u16 pkey = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
229 
230 		ib_find_cached_pkey(&dev->ib_dev, qp->port, pkey,
231 				    &wc->pkey_index);
232 	} else {
233 		wc->pkey_index = 0;
234 	}
235 
236 	if (ll != IB_LINK_LAYER_ETHERNET) {
237 		wc->sl = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf;
238 		return;
239 	}
240 
241 	vlan_present = cqe_has_vlan(cqe);
242 	roce_packet_type   = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0x3;
243 	if (vlan_present) {
244 		wc->vlan_id = (be16_to_cpu(cqe->vlan_info)) & 0xfff;
245 		wc->sl = (be16_to_cpu(cqe->vlan_info) >> 13) & 0x7;
246 		wc->wc_flags |= IB_WC_WITH_VLAN;
247 	} else {
248 		wc->sl = 0;
249 	}
250 
251 	switch (roce_packet_type) {
252 	case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH:
253 		wc->network_hdr_type = RDMA_NETWORK_IB;
254 		break;
255 	case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6:
256 		wc->network_hdr_type = RDMA_NETWORK_IPV6;
257 		break;
258 	case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV4:
259 		wc->network_hdr_type = RDMA_NETWORK_IPV4;
260 		break;
261 	}
262 	wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
263 }
264 
265 static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)
266 {
267 	__be32 *p = (__be32 *)cqe;
268 	int i;
269 
270 	mlx5_ib_warn(dev, "dump error cqe\n");
271 	for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4)
272 		pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p[0]),
273 			be32_to_cpu(p[1]), be32_to_cpu(p[2]),
274 			be32_to_cpu(p[3]));
275 }
276 
277 static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,
278 				  struct mlx5_err_cqe *cqe,
279 				  struct ib_wc *wc)
280 {
281 	int dump = 1;
282 
283 	switch (cqe->syndrome) {
284 	case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:
285 		wc->status = IB_WC_LOC_LEN_ERR;
286 		break;
287 	case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR:
288 		wc->status = IB_WC_LOC_QP_OP_ERR;
289 		break;
290 	case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:
291 		wc->status = IB_WC_LOC_PROT_ERR;
292 		break;
293 	case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:
294 		dump = 0;
295 		wc->status = IB_WC_WR_FLUSH_ERR;
296 		break;
297 	case MLX5_CQE_SYNDROME_MW_BIND_ERR:
298 		wc->status = IB_WC_MW_BIND_ERR;
299 		break;
300 	case MLX5_CQE_SYNDROME_BAD_RESP_ERR:
301 		wc->status = IB_WC_BAD_RESP_ERR;
302 		break;
303 	case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR:
304 		wc->status = IB_WC_LOC_ACCESS_ERR;
305 		break;
306 	case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
307 		wc->status = IB_WC_REM_INV_REQ_ERR;
308 		break;
309 	case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:
310 		wc->status = IB_WC_REM_ACCESS_ERR;
311 		break;
312 	case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:
313 		wc->status = IB_WC_REM_OP_ERR;
314 		break;
315 	case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
316 		wc->status = IB_WC_RETRY_EXC_ERR;
317 		dump = 0;
318 		break;
319 	case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
320 		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
321 		dump = 0;
322 		break;
323 	case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:
324 		wc->status = IB_WC_REM_ABORT_ERR;
325 		break;
326 	default:
327 		wc->status = IB_WC_GENERAL_ERR;
328 		break;
329 	}
330 
331 	wc->vendor_err = cqe->vendor_err_synd;
332 	if (dump)
333 		dump_cqe(dev, cqe);
334 }
335 
336 static int is_atomic_response(struct mlx5_ib_qp *qp, uint16_t idx)
337 {
338 	/* TBD: waiting decision
339 	*/
340 	return 0;
341 }
342 
343 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp *qp, uint16_t idx)
344 {
345 	struct mlx5_wqe_data_seg *dpseg;
346 	void *addr;
347 
348 	dpseg = mlx5_get_send_wqe(qp, idx) + sizeof(struct mlx5_wqe_ctrl_seg) +
349 		sizeof(struct mlx5_wqe_raddr_seg) +
350 		sizeof(struct mlx5_wqe_atomic_seg);
351 	addr = (void *)(unsigned long)be64_to_cpu(dpseg->addr);
352 	return addr;
353 }
354 
355 static void handle_atomic(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
356 			  uint16_t idx)
357 {
358 	void *addr;
359 	int byte_count;
360 	int i;
361 
362 	if (!is_atomic_response(qp, idx))
363 		return;
364 
365 	byte_count = be32_to_cpu(cqe64->byte_cnt);
366 	addr = mlx5_get_atomic_laddr(qp, idx);
367 
368 	if (byte_count == 4) {
369 		*(uint32_t *)addr = be32_to_cpu(*((__be32 *)addr));
370 	} else {
371 		for (i = 0; i < byte_count; i += 8) {
372 			*(uint64_t *)addr = be64_to_cpu(*((__be64 *)addr));
373 			addr += 8;
374 		}
375 	}
376 
377 	return;
378 }
379 
380 static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
381 			   u16 tail, u16 head)
382 {
383 	u16 idx;
384 
385 	do {
386 		idx = tail & (qp->sq.wqe_cnt - 1);
387 		handle_atomic(qp, cqe64, idx);
388 		if (idx == head)
389 			break;
390 
391 		tail = qp->sq.w_list[idx].next;
392 	} while (1);
393 	tail = qp->sq.w_list[idx].next;
394 	qp->sq.last_poll = tail;
395 }
396 
397 static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)
398 {
399 	mlx5_buf_free(dev->mdev, &buf->buf);
400 }
401 
402 static void get_sig_err_item(struct mlx5_sig_err_cqe *cqe,
403 			     struct ib_sig_err *item)
404 {
405 	u16 syndrome = be16_to_cpu(cqe->syndrome);
406 
407 #define GUARD_ERR   (1 << 13)
408 #define APPTAG_ERR  (1 << 12)
409 #define REFTAG_ERR  (1 << 11)
410 
411 	if (syndrome & GUARD_ERR) {
412 		item->err_type = IB_SIG_BAD_GUARD;
413 		item->expected = be32_to_cpu(cqe->expected_trans_sig) >> 16;
414 		item->actual = be32_to_cpu(cqe->actual_trans_sig) >> 16;
415 	} else
416 	if (syndrome & REFTAG_ERR) {
417 		item->err_type = IB_SIG_BAD_REFTAG;
418 		item->expected = be32_to_cpu(cqe->expected_reftag);
419 		item->actual = be32_to_cpu(cqe->actual_reftag);
420 	} else
421 	if (syndrome & APPTAG_ERR) {
422 		item->err_type = IB_SIG_BAD_APPTAG;
423 		item->expected = be32_to_cpu(cqe->expected_trans_sig) & 0xffff;
424 		item->actual = be32_to_cpu(cqe->actual_trans_sig) & 0xffff;
425 	} else {
426 		pr_err("Got signature completion error with bad syndrome %04x\n",
427 		       syndrome);
428 	}
429 
430 	item->sig_err_offset = be64_to_cpu(cqe->err_offset);
431 	item->key = be32_to_cpu(cqe->mkey);
432 }
433 
434 static void sw_send_comp(struct mlx5_ib_qp *qp, int num_entries,
435 			 struct ib_wc *wc, int *npolled)
436 {
437 	struct mlx5_ib_wq *wq;
438 	unsigned int cur;
439 	unsigned int idx;
440 	int np;
441 	int i;
442 
443 	wq = &qp->sq;
444 	cur = wq->head - wq->tail;
445 	np = *npolled;
446 
447 	if (cur == 0)
448 		return;
449 
450 	for (i = 0;  i < cur && np < num_entries; i++) {
451 		idx = wq->last_poll & (wq->wqe_cnt - 1);
452 		wc->wr_id = wq->wrid[idx];
453 		wc->status = IB_WC_WR_FLUSH_ERR;
454 		wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
455 		wq->tail++;
456 		np++;
457 		wc->qp = &qp->ibqp;
458 		wc++;
459 		wq->last_poll = wq->w_list[idx].next;
460 	}
461 	*npolled = np;
462 }
463 
464 static void sw_recv_comp(struct mlx5_ib_qp *qp, int num_entries,
465 			 struct ib_wc *wc, int *npolled)
466 {
467 	struct mlx5_ib_wq *wq;
468 	unsigned int cur;
469 	int np;
470 	int i;
471 
472 	wq = &qp->rq;
473 	cur = wq->head - wq->tail;
474 	np = *npolled;
475 
476 	if (cur == 0)
477 		return;
478 
479 	for (i = 0;  i < cur && np < num_entries; i++) {
480 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
481 		wc->status = IB_WC_WR_FLUSH_ERR;
482 		wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
483 		wq->tail++;
484 		np++;
485 		wc->qp = &qp->ibqp;
486 		wc++;
487 	}
488 	*npolled = np;
489 }
490 
491 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries,
492 				 struct ib_wc *wc, int *npolled)
493 {
494 	struct mlx5_ib_qp *qp;
495 
496 	*npolled = 0;
497 	/* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
498 	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
499 		sw_send_comp(qp, num_entries, wc + *npolled, npolled);
500 		if (*npolled >= num_entries)
501 			return;
502 	}
503 
504 	list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
505 		sw_recv_comp(qp, num_entries, wc + *npolled, npolled);
506 		if (*npolled >= num_entries)
507 			return;
508 	}
509 }
510 
511 static int mlx5_poll_one(struct mlx5_ib_cq *cq,
512 			 struct mlx5_ib_qp **cur_qp,
513 			 struct ib_wc *wc)
514 {
515 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
516 	struct mlx5_err_cqe *err_cqe;
517 	struct mlx5_cqe64 *cqe64;
518 	struct mlx5_core_qp *mqp;
519 	struct mlx5_ib_wq *wq;
520 	struct mlx5_sig_err_cqe *sig_err_cqe;
521 	struct mlx5_core_mr *mmkey;
522 	struct mlx5_ib_mr *mr;
523 	unsigned long flags;
524 	uint8_t opcode;
525 	uint32_t qpn;
526 	u16 wqe_ctr;
527 	void *cqe;
528 	int idx;
529 
530 repoll:
531 	cqe = next_cqe_sw(cq);
532 	if (!cqe)
533 		return -EAGAIN;
534 
535 	cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
536 
537 	++cq->mcq.cons_index;
538 
539 	/* Make sure we read CQ entry contents after we've checked the
540 	 * ownership bit.
541 	 */
542 	rmb();
543 
544 	opcode = cqe64->op_own >> 4;
545 	if (unlikely(opcode == MLX5_CQE_RESIZE_CQ)) {
546 		if (likely(cq->resize_buf)) {
547 			free_cq_buf(dev, &cq->buf);
548 			cq->buf = *cq->resize_buf;
549 			kfree(cq->resize_buf);
550 			cq->resize_buf = NULL;
551 			goto repoll;
552 		} else {
553 			mlx5_ib_warn(dev, "unexpected resize cqe\n");
554 		}
555 	}
556 
557 	qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff;
558 	if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) {
559 		/* We do not have to take the QP table lock here,
560 		 * because CQs will be locked while QPs are removed
561 		 * from the table.
562 		 */
563 		mqp = __mlx5_qp_lookup(dev->mdev, qpn);
564 		*cur_qp = to_mibqp(mqp);
565 	}
566 
567 	wc->qp  = &(*cur_qp)->ibqp;
568 	switch (opcode) {
569 	case MLX5_CQE_REQ:
570 		wq = &(*cur_qp)->sq;
571 		wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
572 		idx = wqe_ctr & (wq->wqe_cnt - 1);
573 		handle_good_req(wc, cqe64, wq, idx);
574 		handle_atomics(*cur_qp, cqe64, wq->last_poll, idx);
575 		wc->wr_id = wq->wrid[idx];
576 		wq->tail = wq->wqe_head[idx] + 1;
577 		wc->status = IB_WC_SUCCESS;
578 		break;
579 	case MLX5_CQE_RESP_WR_IMM:
580 	case MLX5_CQE_RESP_SEND:
581 	case MLX5_CQE_RESP_SEND_IMM:
582 	case MLX5_CQE_RESP_SEND_INV:
583 		handle_responder(wc, cqe64, *cur_qp);
584 		wc->status = IB_WC_SUCCESS;
585 		break;
586 	case MLX5_CQE_RESIZE_CQ:
587 		break;
588 	case MLX5_CQE_REQ_ERR:
589 	case MLX5_CQE_RESP_ERR:
590 		err_cqe = (struct mlx5_err_cqe *)cqe64;
591 		mlx5_handle_error_cqe(dev, err_cqe, wc);
592 		mlx5_ib_dbg(dev, "%s error cqe on cqn 0x%x:\n",
593 			    opcode == MLX5_CQE_REQ_ERR ?
594 			    "Requestor" : "Responder", cq->mcq.cqn);
595 		mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n",
596 			    err_cqe->syndrome, err_cqe->vendor_err_synd);
597 		if (opcode == MLX5_CQE_REQ_ERR) {
598 			wq = &(*cur_qp)->sq;
599 			wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
600 			idx = wqe_ctr & (wq->wqe_cnt - 1);
601 			wc->wr_id = wq->wrid[idx];
602 			wq->tail = wq->wqe_head[idx] + 1;
603 		} else {
604 			struct mlx5_ib_srq *srq;
605 
606 			if ((*cur_qp)->ibqp.srq) {
607 				srq = to_msrq((*cur_qp)->ibqp.srq);
608 				wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
609 				wc->wr_id = srq->wrid[wqe_ctr];
610 				mlx5_ib_free_srq_wqe(srq, wqe_ctr);
611 			} else {
612 				wq = &(*cur_qp)->rq;
613 				wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
614 				++wq->tail;
615 			}
616 		}
617 		break;
618 	case MLX5_CQE_SIG_ERR:
619 		sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
620 
621 		spin_lock_irqsave(&dev->mdev->priv.mr_table.lock, flags);
622 		mmkey = __mlx5_mr_lookup(dev->mdev,
623 					 mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
624 		mr = to_mibmr(mmkey);
625 		get_sig_err_item(sig_err_cqe, &mr->sig->err_item);
626 		mr->sig->sig_err_exists = true;
627 		mr->sig->sigerr_count++;
628 
629 		mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
630 			     cq->mcq.cqn, mr->sig->err_item.key,
631 			     mr->sig->err_item.err_type,
632 			     (long long)mr->sig->err_item.sig_err_offset,
633 			     mr->sig->err_item.expected,
634 			     mr->sig->err_item.actual);
635 
636 		spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags);
637 		goto repoll;
638 	}
639 
640 	return 0;
641 }
642 
643 static int poll_soft_wc(struct mlx5_ib_cq *cq, int num_entries,
644 			struct ib_wc *wc)
645 {
646 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
647 	struct mlx5_ib_wc *soft_wc, *next;
648 	int npolled = 0;
649 
650 	list_for_each_entry_safe(soft_wc, next, &cq->wc_list, list) {
651 		if (npolled >= num_entries)
652 			break;
653 
654 		mlx5_ib_dbg(dev, "polled software generated completion on CQ 0x%x\n",
655 			    cq->mcq.cqn);
656 
657 		wc[npolled++] = soft_wc->wc;
658 		list_del(&soft_wc->list);
659 		kfree(soft_wc);
660 	}
661 
662 	return npolled;
663 }
664 
665 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
666 {
667 	struct mlx5_ib_cq *cq = to_mcq(ibcq);
668 	struct mlx5_ib_qp *cur_qp = NULL;
669 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
670 	struct mlx5_core_dev *mdev = dev->mdev;
671 	unsigned long flags;
672 	int soft_polled = 0;
673 	int npolled;
674 
675 	spin_lock_irqsave(&cq->lock, flags);
676 	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) {
677 		mlx5_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
678 		goto out;
679 	}
680 
681 	if (unlikely(!list_empty(&cq->wc_list)))
682 		soft_polled = poll_soft_wc(cq, num_entries, wc);
683 
684 	for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
685 		if (mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled))
686 			break;
687 	}
688 
689 	if (npolled)
690 		mlx5_cq_set_ci(&cq->mcq);
691 out:
692 	spin_unlock_irqrestore(&cq->lock, flags);
693 
694 	return soft_polled + npolled;
695 }
696 
697 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
698 {
699 	struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
700 	struct mlx5_ib_cq *cq = to_mcq(ibcq);
701 	void __iomem *uar_page = mdev->priv.uar->map;
702 	unsigned long irq_flags;
703 	int ret = 0;
704 
705 	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR))
706 		return -1;
707 
708 	spin_lock_irqsave(&cq->lock, irq_flags);
709 	if (cq->notify_flags != IB_CQ_NEXT_COMP)
710 		cq->notify_flags = flags & IB_CQ_SOLICITED_MASK;
711 
712 	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !list_empty(&cq->wc_list))
713 		ret = 1;
714 	spin_unlock_irqrestore(&cq->lock, irq_flags);
715 
716 	mlx5_cq_arm(&cq->mcq,
717 		    (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
718 		    MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
719 		    uar_page,
720 		    MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
721 		    cq->mcq.cons_index);
722 
723 	return ret;
724 }
725 
726 static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
727 			int nent, int cqe_size)
728 {
729 	int err;
730 
731 	err = mlx5_buf_alloc(dev->mdev, nent * cqe_size,
732 	    2 * PAGE_SIZE, &buf->buf);
733 	if (err)
734 		return err;
735 
736 	buf->cqe_size = cqe_size;
737 	buf->nent = nent;
738 
739 	return 0;
740 }
741 
742 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
743 			  struct ib_ucontext *context, struct mlx5_ib_cq *cq,
744 			  int entries, u32 **cqb,
745 			  int *cqe_size, int *index, int *inlen)
746 {
747 	struct mlx5_ib_create_cq ucmd;
748 	size_t ucmdlen;
749 	int page_shift;
750 	__be64 *pas;
751 	int npages;
752 	int ncont;
753 	void *cqc;
754 	int err;
755 
756 	ucmdlen = min(udata->inlen, sizeof(ucmd));
757 	if (ucmdlen < offsetof(struct mlx5_ib_create_cq, flags))
758 		return -EINVAL;
759 
760 	if (ib_copy_from_udata(&ucmd, udata, ucmdlen))
761 		return -EFAULT;
762 
763 	if ((ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX)))
764 		return -EINVAL;
765 
766 	if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128)
767 		return -EINVAL;
768 
769 	*cqe_size = ucmd.cqe_size;
770 
771 	cq->buf.umem = ib_umem_get(context, ucmd.buf_addr,
772 				   entries * ucmd.cqe_size,
773 				   IB_ACCESS_LOCAL_WRITE, 1);
774 	if (IS_ERR(cq->buf.umem)) {
775 		err = PTR_ERR(cq->buf.umem);
776 		return err;
777 	}
778 
779 	err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
780 				  &cq->db);
781 	if (err)
782 		goto err_umem;
783 
784 	mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift,
785 			   &ncont, NULL);
786 	mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
787 		    (long long)ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);
788 
789 	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
790 		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
791 	*cqb = mlx5_vzalloc(*inlen);
792 	if (!*cqb) {
793 		err = -ENOMEM;
794 		goto err_db;
795 	}
796 
797 	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
798 	mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0);
799 
800 	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
801 	MLX5_SET(cqc, cqc, log_page_size,
802 		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
803 
804 	if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {
805 		*index = ucmd.uar_page_index;
806 	} else if (to_mucontext(context)->bfregi.lib_uar_dyn) {
807 		err = -EINVAL;
808 		goto err_cqb;
809 	} else {
810 		*index = to_mucontext(context)->bfregi.sys_pages[0];
811 	}
812 
813 	return 0;
814 
815 err_cqb:
816 	kvfree(*cqb);
817 
818 err_db:
819 	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
820 
821 err_umem:
822 	ib_umem_release(cq->buf.umem);
823 	return err;
824 }
825 
826 static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context)
827 {
828 	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
829 	ib_umem_release(cq->buf.umem);
830 }
831 
832 static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf)
833 {
834 	int i;
835 	void *cqe;
836 	struct mlx5_cqe64 *cqe64;
837 
838 	for (i = 0; i < buf->nent; i++) {
839 		cqe = get_cqe_from_buf(buf, i, buf->cqe_size);
840 		cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64;
841 		cqe64->op_own = MLX5_CQE_INVALID << 4;
842 	}
843 }
844 
845 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
846 			    int entries, int cqe_size,
847 			    u32 **cqb, int *index, int *inlen)
848 {
849 	__be64 *pas;
850 	void *cqc;
851 	int err;
852 
853 	err = mlx5_db_alloc(dev->mdev, &cq->db);
854 	if (err)
855 		return err;
856 
857 	cq->mcq.set_ci_db  = cq->db.db;
858 	cq->mcq.arm_db     = cq->db.db + 1;
859 	cq->mcq.cqe_sz = cqe_size;
860 
861 	err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size);
862 	if (err)
863 		goto err_db;
864 
865 	init_cq_buf(cq, &cq->buf);
866 
867 	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
868 		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * cq->buf.buf.npages;
869 	*cqb = mlx5_vzalloc(*inlen);
870 	if (!*cqb) {
871 		err = -ENOMEM;
872 		goto err_buf;
873 	}
874 
875 	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
876 	mlx5_fill_page_array(&cq->buf.buf, pas);
877 
878 	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
879 	MLX5_SET(cqc, cqc, log_page_size,
880 		 cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
881 
882 	*index = dev->mdev->priv.uar->index;
883 
884 	return 0;
885 
886 err_buf:
887 	free_cq_buf(dev, &cq->buf);
888 
889 err_db:
890 	mlx5_db_free(dev->mdev, &cq->db);
891 	return err;
892 }
893 
894 static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
895 {
896 	free_cq_buf(dev, &cq->buf);
897 	mlx5_db_free(dev->mdev, &cq->db);
898 }
899 
900 static void notify_soft_wc_handler(struct work_struct *work)
901 {
902 	struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,
903 					     notify_work);
904 
905 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
906 }
907 
908 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
909 				const struct ib_cq_init_attr *attr,
910 				struct ib_ucontext *context,
911 				struct ib_udata *udata)
912 {
913 	int entries = attr->cqe;
914 	int vector = attr->comp_vector;
915 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
916 	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
917 	struct mlx5_ib_cq *cq;
918 	int uninitialized_var(index);
919 	int uninitialized_var(inlen);
920 	u32 *cqb = NULL;
921 	void *cqc;
922 	int cqe_size;
923 	unsigned int irqn;
924 	int eqn;
925 	int err;
926 
927 	if (entries < 0 ||
928 	    (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
929 		return ERR_PTR(-EINVAL);
930 
931 	if (check_cq_create_flags(attr->flags))
932 		return ERR_PTR(-EOPNOTSUPP);
933 
934 	entries = roundup_pow_of_two(entries + 1);
935 	if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
936 		return ERR_PTR(-EINVAL);
937 
938 	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
939 	if (!cq)
940 		return ERR_PTR(-ENOMEM);
941 
942 	cq->ibcq.cqe = entries - 1;
943 	mutex_init(&cq->resize_mutex);
944 	spin_lock_init(&cq->lock);
945 	cq->resize_buf = NULL;
946 	cq->resize_umem = NULL;
947 	cq->create_flags = attr->flags;
948 	INIT_LIST_HEAD(&cq->list_send_qp);
949 	INIT_LIST_HEAD(&cq->list_recv_qp);
950 
951 	if (context) {
952 		err = create_cq_user(dev, udata, context, cq, entries,
953 				     &cqb, &cqe_size, &index, &inlen);
954 		if (err)
955 			goto err_create;
956 	} else {
957 		cqe_size = cache_line_size() == 128 ? 128 : 64;
958 		err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
959 				       &index, &inlen);
960 		if (err)
961 			goto err_create;
962 
963 		INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
964 	}
965 
966 	err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
967 	if (err)
968 		goto err_cqb;
969 
970 	cq->cqe_size = cqe_size;
971 
972 	cqc = MLX5_ADDR_OF(create_cq_in, cqb, cq_context);
973 	MLX5_SET(cqc, cqc, cqe_sz, cqe_sz_to_mlx_sz(cqe_size));
974 	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
975 	MLX5_SET(cqc, cqc, uar_page, index);
976 	MLX5_SET(cqc, cqc, c_eqn, eqn);
977 	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
978 	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)
979 		MLX5_SET(cqc, cqc, oi, 1);
980 
981 	err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen, out, sizeof(out));
982 	if (err)
983 		goto err_cqb;
984 
985 	mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
986 	cq->mcq.irqn = irqn;
987 	cq->mcq.comp  = mlx5_ib_cq_comp;
988 	cq->mcq.event = mlx5_ib_cq_event;
989 
990 	INIT_LIST_HEAD(&cq->wc_list);
991 
992 	if (context)
993 		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
994 			err = -EFAULT;
995 			goto err_cmd;
996 		}
997 
998 
999 	kvfree(cqb);
1000 	return &cq->ibcq;
1001 
1002 err_cmd:
1003 	mlx5_core_destroy_cq(dev->mdev, &cq->mcq);
1004 
1005 err_cqb:
1006 	kvfree(cqb);
1007 	if (context)
1008 		destroy_cq_user(cq, context);
1009 	else
1010 		destroy_cq_kernel(dev, cq);
1011 
1012 err_create:
1013 	kfree(cq);
1014 
1015 	return ERR_PTR(err);
1016 }
1017 
1018 
1019 int mlx5_ib_destroy_cq(struct ib_cq *cq)
1020 {
1021 	struct mlx5_ib_dev *dev = to_mdev(cq->device);
1022 	struct mlx5_ib_cq *mcq = to_mcq(cq);
1023 	struct ib_ucontext *context = NULL;
1024 
1025 	if (cq->uobject)
1026 		context = cq->uobject->context;
1027 
1028 	mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
1029 	if (context)
1030 		destroy_cq_user(mcq, context);
1031 	else
1032 		destroy_cq_kernel(dev, mcq);
1033 
1034 	kfree(mcq);
1035 
1036 	return 0;
1037 }
1038 
1039 static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)
1040 {
1041 	return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);
1042 }
1043 
1044 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
1045 {
1046 	struct mlx5_cqe64 *cqe64, *dest64;
1047 	void *cqe, *dest;
1048 	u32 prod_index;
1049 	int nfreed = 0;
1050 	u8 owner_bit;
1051 
1052 	if (!cq)
1053 		return;
1054 
1055 	/* First we need to find the current producer index, so we
1056 	 * know where to start cleaning from.  It doesn't matter if HW
1057 	 * adds new entries after this loop -- the QP we're worried
1058 	 * about is already in RESET, so the new entries won't come
1059 	 * from our QP and therefore don't need to be checked.
1060 	 */
1061 	for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++)
1062 		if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
1063 			break;
1064 
1065 	/* Now sweep backwards through the CQ, removing CQ entries
1066 	 * that match our QP by copying older entries on top of them.
1067 	 */
1068 	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
1069 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
1070 		cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
1071 		if (is_equal_rsn(cqe64, rsn)) {
1072 			if (srq && (ntohl(cqe64->srqn) & 0xffffff))
1073 				mlx5_ib_free_srq_wqe(srq, be16_to_cpu(cqe64->wqe_counter));
1074 			++nfreed;
1075 		} else if (nfreed) {
1076 			dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
1077 			dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64;
1078 			owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK;
1079 			memcpy(dest, cqe, cq->mcq.cqe_sz);
1080 			dest64->op_own = owner_bit |
1081 				(dest64->op_own & ~MLX5_CQE_OWNER_MASK);
1082 		}
1083 	}
1084 
1085 	if (nfreed) {
1086 		cq->mcq.cons_index += nfreed;
1087 		/* Make sure update of buffer contents is done before
1088 		 * updating consumer index.
1089 		 */
1090 		wmb();
1091 		mlx5_cq_set_ci(&cq->mcq);
1092 	}
1093 }
1094 
1095 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq)
1096 {
1097 	if (!cq)
1098 		return;
1099 
1100 	spin_lock_irq(&cq->lock);
1101 	__mlx5_ib_cq_clean(cq, qpn, srq);
1102 	spin_unlock_irq(&cq->lock);
1103 }
1104 
1105 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1106 {
1107 	struct mlx5_ib_dev *dev = to_mdev(cq->device);
1108 	struct mlx5_ib_cq *mcq = to_mcq(cq);
1109 	int err;
1110 
1111 	if (!MLX5_CAP_GEN(dev->mdev, cq_moderation))
1112 		return -ENOSYS;
1113 
1114 	err = mlx5_core_modify_cq_moderation(dev->mdev, &mcq->mcq,
1115 					     cq_period, cq_count);
1116 	if (err)
1117 		mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn);
1118 
1119 	return err;
1120 }
1121 
1122 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1123 		       int entries, struct ib_udata *udata, int *npas,
1124 		       int *page_shift, int *cqe_size)
1125 {
1126 	struct mlx5_ib_resize_cq ucmd;
1127 	struct ib_umem *umem;
1128 	int err;
1129 	int npages;
1130 	struct ib_ucontext *context = cq->buf.umem->context;
1131 
1132 	err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
1133 	if (err)
1134 		return err;
1135 
1136 	if (ucmd.reserved0 || ucmd.reserved1)
1137 		return -EINVAL;
1138 
1139 	/* check multiplication overflow */
1140 	if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
1141 		return -EINVAL;
1142 
1143 	umem = ib_umem_get(context, ucmd.buf_addr,
1144 			   (size_t)ucmd.cqe_size * entries,
1145 			   IB_ACCESS_LOCAL_WRITE, 1);
1146 	if (IS_ERR(umem)) {
1147 		err = PTR_ERR(umem);
1148 		return err;
1149 	}
1150 
1151 	mlx5_ib_cont_pages(umem, ucmd.buf_addr, &npages, page_shift,
1152 			   npas, NULL);
1153 
1154 	cq->resize_umem = umem;
1155 	*cqe_size = ucmd.cqe_size;
1156 
1157 	return 0;
1158 }
1159 
1160 static void un_resize_user(struct mlx5_ib_cq *cq)
1161 {
1162 	ib_umem_release(cq->resize_umem);
1163 }
1164 
1165 static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1166 			 int entries, int cqe_size)
1167 {
1168 	int err;
1169 
1170 	cq->resize_buf = kzalloc(sizeof(*cq->resize_buf), GFP_KERNEL);
1171 	if (!cq->resize_buf)
1172 		return -ENOMEM;
1173 
1174 	err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size);
1175 	if (err)
1176 		goto ex;
1177 
1178 	init_cq_buf(cq, cq->resize_buf);
1179 
1180 	return 0;
1181 
1182 ex:
1183 	kfree(cq->resize_buf);
1184 	return err;
1185 }
1186 
1187 static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
1188 {
1189 	free_cq_buf(dev, cq->resize_buf);
1190 	cq->resize_buf = NULL;
1191 }
1192 
1193 static int copy_resize_cqes(struct mlx5_ib_cq *cq)
1194 {
1195 	struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
1196 	struct mlx5_cqe64 *scqe64;
1197 	struct mlx5_cqe64 *dcqe64;
1198 	void *start_cqe;
1199 	void *scqe;
1200 	void *dcqe;
1201 	int ssize;
1202 	int dsize;
1203 	int i;
1204 	u8 sw_own;
1205 
1206 	ssize = cq->buf.cqe_size;
1207 	dsize = cq->resize_buf->cqe_size;
1208 	if (ssize != dsize) {
1209 		mlx5_ib_warn(dev, "resize from different cqe size is not supported\n");
1210 		return -EINVAL;
1211 	}
1212 
1213 	i = cq->mcq.cons_index;
1214 	scqe = get_sw_cqe(cq, i);
1215 	scqe64 = ssize == 64 ? scqe : scqe + 64;
1216 	start_cqe = scqe;
1217 	if (!scqe) {
1218 		mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1219 		return -EINVAL;
1220 	}
1221 
1222 	while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) {
1223 		dcqe = get_cqe_from_buf(cq->resize_buf,
1224 					(i + 1) & (cq->resize_buf->nent),
1225 					dsize);
1226 		dcqe64 = dsize == 64 ? dcqe : dcqe + 64;
1227 		sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent);
1228 		memcpy(dcqe, scqe, dsize);
1229 		dcqe64->op_own = (dcqe64->op_own & ~MLX5_CQE_OWNER_MASK) | sw_own;
1230 
1231 		++i;
1232 		scqe = get_sw_cqe(cq, i);
1233 		scqe64 = ssize == 64 ? scqe : scqe + 64;
1234 		if (!scqe) {
1235 			mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1236 			return -EINVAL;
1237 		}
1238 
1239 		if (scqe == start_cqe) {
1240 			pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n",
1241 				cq->mcq.cqn);
1242 			return -ENOMEM;
1243 		}
1244 	}
1245 	++cq->mcq.cons_index;
1246 	return 0;
1247 }
1248 
1249 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
1250 {
1251 	struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
1252 	struct mlx5_ib_cq *cq = to_mcq(ibcq);
1253 	void *cqc;
1254 	u32 *in;
1255 	int err;
1256 	int npas;
1257 	__be64 *pas;
1258 	int page_shift;
1259 	int inlen;
1260 	int uninitialized_var(cqe_size);
1261 	unsigned long flags;
1262 
1263 	if (!MLX5_CAP_GEN(dev->mdev, cq_resize)) {
1264 		pr_info("Firmware does not support resize CQ\n");
1265 		return -ENOSYS;
1266 	}
1267 
1268 	if (entries < 1 ||
1269 	    entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) {
1270 		mlx5_ib_warn(dev, "wrong entries number %d, max %d\n",
1271 			     entries,
1272 			     1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz));
1273 		return -EINVAL;
1274 	}
1275 
1276 	entries = roundup_pow_of_two(entries + 1);
1277 	if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)) + 1)
1278 		return -EINVAL;
1279 
1280 	if (entries == ibcq->cqe + 1)
1281 		return 0;
1282 
1283 	mutex_lock(&cq->resize_mutex);
1284 	if (udata) {
1285 		err = resize_user(dev, cq, entries, udata, &npas, &page_shift,
1286 				  &cqe_size);
1287 	} else {
1288 		cqe_size = 64;
1289 		err = resize_kernel(dev, cq, entries, cqe_size);
1290 		if (!err) {
1291 			npas = cq->resize_buf->buf.npages;
1292 			page_shift = cq->resize_buf->buf.page_shift;
1293 		}
1294 	}
1295 
1296 	if (err)
1297 		goto ex;
1298 
1299 	inlen = MLX5_ST_SZ_BYTES(modify_cq_in) +
1300 		MLX5_FLD_SZ_BYTES(modify_cq_in, pas[0]) * npas;
1301 
1302 	in = mlx5_vzalloc(inlen);
1303 	if (!in) {
1304 		err = -ENOMEM;
1305 		goto ex_resize;
1306 	}
1307 
1308 	pas = (__be64 *)MLX5_ADDR_OF(modify_cq_in, in, pas);
1309 	if (udata)
1310 		mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,
1311 				     pas, 0);
1312 	else
1313 		mlx5_fill_page_array(&cq->resize_buf->buf, pas);
1314 
1315 	MLX5_SET(modify_cq_in, in,
1316 		 modify_field_select_resize_field_select.resize_field_select.resize_field_select,
1317 		 MLX5_MODIFY_CQ_MASK_LOG_SIZE  |
1318 		 MLX5_MODIFY_CQ_MASK_PG_OFFSET |
1319 		 MLX5_MODIFY_CQ_MASK_PG_SIZE);
1320 
1321 	cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context);
1322 
1323 	MLX5_SET(cqc, cqc, log_page_size,
1324 		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1325 	MLX5_SET(cqc, cqc, cqe_sz, cqe_sz_to_mlx_sz(cqe_size));
1326 	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
1327 
1328 	MLX5_SET(modify_cq_in, in, op_mod, MLX5_CQ_OPMOD_RESIZE);
1329 	MLX5_SET(modify_cq_in, in, cqn, cq->mcq.cqn);
1330 
1331 	err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen);
1332 	if (err)
1333 		goto ex_alloc;
1334 
1335 	if (udata) {
1336 		cq->ibcq.cqe = entries - 1;
1337 		ib_umem_release(cq->buf.umem);
1338 		cq->buf.umem = cq->resize_umem;
1339 		cq->resize_umem = NULL;
1340 	} else {
1341 		struct mlx5_ib_cq_buf tbuf;
1342 		int resized = 0;
1343 
1344 		spin_lock_irqsave(&cq->lock, flags);
1345 		if (cq->resize_buf) {
1346 			err = copy_resize_cqes(cq);
1347 			if (!err) {
1348 				tbuf = cq->buf;
1349 				cq->buf = *cq->resize_buf;
1350 				kfree(cq->resize_buf);
1351 				cq->resize_buf = NULL;
1352 				resized = 1;
1353 			}
1354 		}
1355 		cq->ibcq.cqe = entries - 1;
1356 		spin_unlock_irqrestore(&cq->lock, flags);
1357 		if (resized)
1358 			free_cq_buf(dev, &tbuf);
1359 	}
1360 	mutex_unlock(&cq->resize_mutex);
1361 
1362 	kvfree(in);
1363 	return 0;
1364 
1365 ex_alloc:
1366 	kvfree(in);
1367 
1368 ex_resize:
1369 	if (udata)
1370 		un_resize_user(cq);
1371 	else
1372 		un_resize_kernel(dev, cq);
1373 ex:
1374 	mutex_unlock(&cq->resize_mutex);
1375 	return err;
1376 }
1377 
1378 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
1379 {
1380 	struct mlx5_ib_cq *cq;
1381 
1382 	if (!ibcq)
1383 		return 128;
1384 
1385 	cq = to_mcq(ibcq);
1386 	return cq->cqe_size;
1387 }
1388 
1389 /* Called from atomic context */
1390 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc)
1391 {
1392 	struct mlx5_ib_wc *soft_wc;
1393 	struct mlx5_ib_cq *cq = to_mcq(ibcq);
1394 	unsigned long flags;
1395 
1396 	soft_wc = kmalloc(sizeof(*soft_wc), GFP_ATOMIC);
1397 	if (!soft_wc)
1398 		return -ENOMEM;
1399 
1400 	soft_wc->wc = *wc;
1401 	spin_lock_irqsave(&cq->lock, flags);
1402 	list_add_tail(&soft_wc->list, &cq->wc_list);
1403 	if (cq->notify_flags == IB_CQ_NEXT_COMP ||
1404 	    wc->status != IB_WC_SUCCESS) {
1405 		cq->notify_flags = 0;
1406 		schedule_work(&cq->notify_work);
1407 	}
1408 	spin_unlock_irqrestore(&cq->lock, flags);
1409 
1410 	return 0;
1411 }
1412