1 /*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kref.h> 29 #include <rdma/ib_umem.h> 30 #include <rdma/ib_user_verbs.h> 31 #include "mlx5_ib.h" 32 #include "user.h" 33 34 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq) 35 { 36 struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; 37 38 ibcq->comp_handler(ibcq, ibcq->cq_context); 39 } 40 41 static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, int type) 42 { 43 struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq); 44 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); 45 struct ib_cq *ibcq = &cq->ibcq; 46 struct ib_event event; 47 48 if (type != MLX5_EVENT_TYPE_CQ_ERROR) { 49 mlx5_ib_warn(dev, "Unexpected event type %d on CQ %06x\n", 50 type, mcq->cqn); 51 return; 52 } 53 54 if (ibcq->event_handler) { 55 event.device = &dev->ib_dev; 56 event.event = IB_EVENT_CQ_ERR; 57 event.element.cq = ibcq; 58 ibcq->event_handler(&event, ibcq->cq_context); 59 } 60 } 61 62 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size) 63 { 64 return mlx5_buf_offset(&buf->buf, n * size); 65 } 66 67 static void *get_cqe(struct mlx5_ib_cq *cq, int n) 68 { 69 return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz); 70 } 71 72 static u8 sw_ownership_bit(int n, int nent) 73 { 74 return (n & nent) ? 1 : 0; 75 } 76 77 static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n) 78 { 79 void *cqe = get_cqe(cq, n & cq->ibcq.cqe); 80 struct mlx5_cqe64 *cqe64; 81 82 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; 83 84 if (likely((cqe64->op_own) >> 4 != MLX5_CQE_INVALID) && 85 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1)))) { 86 return cqe; 87 } else { 88 return NULL; 89 } 90 } 91 92 static void *next_cqe_sw(struct mlx5_ib_cq *cq) 93 { 94 return get_sw_cqe(cq, cq->mcq.cons_index); 95 } 96 97 static enum ib_wc_opcode get_umr_comp(struct mlx5_ib_wq *wq, int idx) 98 { 99 switch (wq->swr_ctx[idx].wr_data) { 100 case IB_WR_LOCAL_INV: 101 return IB_WC_LOCAL_INV; 102 103 case IB_WR_FAST_REG_MR: 104 return IB_WC_FAST_REG_MR; 105 106 default: 107 printf("mlx5_ib: WARN: ""unknown completion status\n"); 108 return 0; 109 } 110 } 111 112 static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe, 113 struct mlx5_ib_wq *wq, int idx) 114 { 115 wc->wc_flags = 0; 116 switch (be32_to_cpu(cqe->sop_drop_qpn) >> 24) { 117 case MLX5_OPCODE_RDMA_WRITE_IMM: 118 wc->wc_flags |= IB_WC_WITH_IMM; 119 case MLX5_OPCODE_RDMA_WRITE: 120 wc->opcode = IB_WC_RDMA_WRITE; 121 break; 122 case MLX5_OPCODE_SEND_IMM: 123 wc->wc_flags |= IB_WC_WITH_IMM; 124 case MLX5_OPCODE_NOP: 125 case MLX5_OPCODE_SEND: 126 case MLX5_OPCODE_SEND_INVAL: 127 wc->opcode = IB_WC_SEND; 128 break; 129 case MLX5_OPCODE_RDMA_READ: 130 wc->opcode = IB_WC_RDMA_READ; 131 wc->byte_len = be32_to_cpu(cqe->byte_cnt); 132 break; 133 case MLX5_OPCODE_ATOMIC_CS: 134 wc->opcode = IB_WC_COMP_SWAP; 135 wc->byte_len = 8; 136 break; 137 case MLX5_OPCODE_ATOMIC_FA: 138 wc->opcode = IB_WC_FETCH_ADD; 139 wc->byte_len = 8; 140 break; 141 case MLX5_OPCODE_ATOMIC_MASKED_CS: 142 wc->opcode = IB_WC_MASKED_COMP_SWAP; 143 wc->byte_len = 8; 144 break; 145 case MLX5_OPCODE_ATOMIC_MASKED_FA: 146 wc->opcode = IB_WC_MASKED_FETCH_ADD; 147 wc->byte_len = 8; 148 break; 149 case MLX5_OPCODE_BIND_MW: 150 wc->opcode = IB_WC_BIND_MW; 151 break; 152 case MLX5_OPCODE_UMR: 153 wc->opcode = get_umr_comp(wq, idx); 154 break; 155 } 156 } 157 158 enum { 159 MLX5_GRH_IN_BUFFER = 1, 160 MLX5_GRH_IN_CQE = 2, 161 }; 162 163 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, 164 struct mlx5_ib_qp *qp) 165 { 166 struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); 167 struct mlx5_ib_srq *srq; 168 struct mlx5_ib_wq *wq; 169 u16 wqe_ctr; 170 u8 g; 171 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS) 172 u8 udp_header_valid; 173 #endif 174 175 if (qp->ibqp.srq || qp->ibqp.xrcd) { 176 struct mlx5_core_srq *msrq = NULL; 177 178 if (qp->ibqp.xrcd) { 179 msrq = mlx5_core_get_srq(dev->mdev, 180 be32_to_cpu(cqe->srqn)); 181 srq = to_mibsrq(msrq); 182 } else { 183 srq = to_msrq(qp->ibqp.srq); 184 } 185 if (srq) { 186 wqe_ctr = be16_to_cpu(cqe->wqe_counter); 187 wc->wr_id = srq->wrid[wqe_ctr]; 188 mlx5_ib_free_srq_wqe(srq, wqe_ctr); 189 if (msrq && atomic_dec_and_test(&msrq->refcount)) 190 complete(&msrq->free); 191 } 192 } else { 193 wq = &qp->rq; 194 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid; 195 ++wq->tail; 196 } 197 wc->byte_len = be32_to_cpu(cqe->byte_cnt); 198 199 switch (cqe->op_own >> 4) { 200 case MLX5_CQE_RESP_WR_IMM: 201 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 202 wc->wc_flags = IB_WC_WITH_IMM; 203 wc->ex.imm_data = cqe->imm_inval_pkey; 204 break; 205 case MLX5_CQE_RESP_SEND: 206 wc->opcode = IB_WC_RECV; 207 wc->wc_flags = 0; 208 break; 209 case MLX5_CQE_RESP_SEND_IMM: 210 wc->opcode = IB_WC_RECV; 211 wc->wc_flags = IB_WC_WITH_IMM; 212 wc->ex.imm_data = cqe->imm_inval_pkey; 213 break; 214 case MLX5_CQE_RESP_SEND_INV: 215 wc->opcode = IB_WC_RECV; 216 wc->wc_flags = IB_WC_WITH_INVALIDATE; 217 wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey); 218 break; 219 } 220 wc->slid = be16_to_cpu(cqe->slid); 221 wc->sl = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf; 222 wc->src_qp = be32_to_cpu(cqe->flags_rqpn) & 0xffffff; 223 wc->dlid_path_bits = cqe->ml_path; 224 g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3; 225 wc->wc_flags |= g ? IB_WC_GRH : 0; 226 wc->pkey_index = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff; 227 228 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS) 229 udp_header_valid = wc->sl & 0x8; 230 if (udp_header_valid) 231 wc->wc_flags |= IB_WC_WITH_UDP_HDR; 232 233 #endif 234 } 235 236 static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe) 237 { 238 __be32 *p = (__be32 *)cqe; 239 int i; 240 241 mlx5_ib_warn(dev, "dump error cqe\n"); 242 for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4) 243 printf("mlx5_ib: INFO: ""%08x %08x %08x %08x\n", be32_to_cpu(p[0]), be32_to_cpu(p[1]), be32_to_cpu(p[2]), be32_to_cpu(p[3])); 244 } 245 246 static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev, 247 struct mlx5_err_cqe *cqe, 248 struct ib_wc *wc) 249 { 250 int dump = 1; 251 252 switch (cqe->syndrome) { 253 case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR: 254 wc->status = IB_WC_LOC_LEN_ERR; 255 break; 256 case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR: 257 wc->status = IB_WC_LOC_QP_OP_ERR; 258 break; 259 case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR: 260 wc->status = IB_WC_LOC_PROT_ERR; 261 break; 262 case MLX5_CQE_SYNDROME_WR_FLUSH_ERR: 263 dump = 0; 264 wc->status = IB_WC_WR_FLUSH_ERR; 265 break; 266 case MLX5_CQE_SYNDROME_MW_BIND_ERR: 267 wc->status = IB_WC_MW_BIND_ERR; 268 break; 269 case MLX5_CQE_SYNDROME_BAD_RESP_ERR: 270 wc->status = IB_WC_BAD_RESP_ERR; 271 break; 272 case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR: 273 wc->status = IB_WC_LOC_ACCESS_ERR; 274 break; 275 case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: 276 wc->status = IB_WC_REM_INV_REQ_ERR; 277 break; 278 case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR: 279 wc->status = IB_WC_REM_ACCESS_ERR; 280 break; 281 case MLX5_CQE_SYNDROME_REMOTE_OP_ERR: 282 wc->status = IB_WC_REM_OP_ERR; 283 break; 284 case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: 285 wc->status = IB_WC_RETRY_EXC_ERR; 286 dump = 0; 287 break; 288 case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR: 289 wc->status = IB_WC_RNR_RETRY_EXC_ERR; 290 dump = 0; 291 break; 292 case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR: 293 wc->status = IB_WC_REM_ABORT_ERR; 294 break; 295 default: 296 wc->status = IB_WC_GENERAL_ERR; 297 break; 298 } 299 300 wc->vendor_err = cqe->vendor_err_synd; 301 if (dump) 302 dump_cqe(dev, cqe); 303 } 304 305 static int is_atomic_response(struct mlx5_ib_qp *qp, u16 idx) 306 { 307 /* TBD: waiting decision 308 */ 309 return 0; 310 } 311 312 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp *qp, u16 idx) 313 { 314 struct mlx5_wqe_data_seg *dpseg; 315 void *addr; 316 317 dpseg = mlx5_get_send_wqe(qp, idx) + sizeof(struct mlx5_wqe_ctrl_seg) + 318 sizeof(struct mlx5_wqe_raddr_seg) + 319 sizeof(struct mlx5_wqe_atomic_seg); 320 addr = (void *)(uintptr_t)be64_to_cpu(dpseg->addr); 321 return addr; 322 } 323 324 static void handle_atomic(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, 325 u16 idx) 326 { 327 void *addr; 328 int byte_count; 329 int i; 330 331 if (!is_atomic_response(qp, idx)) 332 return; 333 334 byte_count = be32_to_cpu(cqe64->byte_cnt); 335 addr = mlx5_get_atomic_laddr(qp, idx); 336 337 if (byte_count == 4) { 338 *(u32 *)addr = be32_to_cpu(*((__be32 *)addr)); 339 } else { 340 for (i = 0; i < byte_count; i += 8) { 341 *(u64 *)addr = be64_to_cpu(*((__be64 *)addr)); 342 addr += 8; 343 } 344 } 345 346 return; 347 } 348 349 static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, 350 u16 tail, u16 head) 351 { 352 u16 idx; 353 354 do { 355 idx = tail & (qp->sq.wqe_cnt - 1); 356 handle_atomic(qp, cqe64, idx); 357 if (idx == head) 358 break; 359 360 tail = qp->sq.swr_ctx[idx].w_list.next; 361 } while (1); 362 tail = qp->sq.swr_ctx[idx].w_list.next; 363 qp->sq.last_poll = tail; 364 } 365 366 static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf) 367 { 368 mlx5_buf_free(dev->mdev, &buf->buf); 369 } 370 371 static void sw_send_comp(struct mlx5_ib_qp *qp, int num_entries, 372 struct ib_wc *wc, int *npolled) 373 { 374 struct mlx5_ib_wq *wq; 375 unsigned cur; 376 unsigned idx; 377 int np; 378 int i; 379 380 wq = &qp->sq; 381 cur = wq->head - wq->tail; 382 np = *npolled; 383 384 if (cur == 0) 385 return; 386 387 for (i = 0; i < cur && np < num_entries; i++) { 388 idx = wq->last_poll & (wq->wqe_cnt - 1); 389 wc->wr_id = wq->swr_ctx[idx].wrid; 390 wc->status = IB_WC_WR_FLUSH_ERR; 391 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR; 392 wq->tail++; 393 np++; 394 wc->qp = &qp->ibqp; 395 wc++; 396 wq->last_poll = wq->swr_ctx[idx].w_list.next; 397 } 398 *npolled = np; 399 } 400 401 static void sw_recv_comp(struct mlx5_ib_qp *qp, int num_entries, 402 struct ib_wc *wc, int *npolled) 403 { 404 struct mlx5_ib_wq *wq; 405 unsigned cur; 406 int np; 407 int i; 408 409 wq = &qp->rq; 410 cur = wq->head - wq->tail; 411 np = *npolled; 412 413 if (cur == 0) 414 return; 415 416 for (i = 0; i < cur && np < num_entries; i++) { 417 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid; 418 wc->status = IB_WC_WR_FLUSH_ERR; 419 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR; 420 wq->tail++; 421 np++; 422 wc->qp = &qp->ibqp; 423 wc++; 424 } 425 *npolled = np; 426 } 427 428 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries, 429 struct ib_wc *wc, int *npolled) 430 { 431 struct mlx5_ib_qp *qp; 432 433 *npolled = 0; 434 /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */ 435 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 436 sw_send_comp(qp, num_entries, wc + *npolled, npolled); 437 if (*npolled >= num_entries) 438 return; 439 } 440 441 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 442 sw_recv_comp(qp, num_entries, wc + *npolled, npolled); 443 if (*npolled >= num_entries) 444 return; 445 } 446 } 447 448 static inline u32 mlx5_ib_base_mkey(const u32 key) 449 { 450 return key & 0xffffff00u; 451 } 452 453 static int mlx5_poll_one(struct mlx5_ib_cq *cq, 454 struct mlx5_ib_qp **cur_qp, 455 struct ib_wc *wc) 456 { 457 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); 458 struct mlx5_err_cqe *err_cqe; 459 struct mlx5_cqe64 *cqe64; 460 struct mlx5_core_qp *mqp; 461 struct mlx5_ib_wq *wq; 462 struct mlx5_sig_err_cqe *sig_err_cqe; 463 struct mlx5_core_mr *mmr; 464 struct mlx5_ib_mr *mr; 465 unsigned long flags; 466 u8 opcode; 467 u32 qpn; 468 u16 wqe_ctr; 469 void *cqe; 470 int idx; 471 472 repoll: 473 cqe = next_cqe_sw(cq); 474 if (!cqe) 475 return -EAGAIN; 476 477 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; 478 479 ++cq->mcq.cons_index; 480 481 /* Make sure we read CQ entry contents after we've checked the 482 * ownership bit. 483 */ 484 rmb(); 485 486 opcode = cqe64->op_own >> 4; 487 if (unlikely(opcode == MLX5_CQE_RESIZE_CQ)) { 488 if (likely(cq->resize_buf)) { 489 free_cq_buf(dev, &cq->buf); 490 cq->buf = *cq->resize_buf; 491 kfree(cq->resize_buf); 492 cq->resize_buf = NULL; 493 goto repoll; 494 } else { 495 mlx5_ib_warn(dev, "unexpected resize cqe\n"); 496 } 497 } 498 499 qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff; 500 if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) { 501 /* We do not have to take the QP table lock here, 502 * because CQs will be locked while QPs are removed 503 * from the table. 504 */ 505 mqp = __mlx5_qp_lookup(dev->mdev, qpn); 506 if (unlikely(!mqp)) { 507 mlx5_ib_warn(dev, "CQE@CQ %06x for unknown QPN %6x\n", 508 cq->mcq.cqn, qpn); 509 return -EINVAL; 510 } 511 512 *cur_qp = to_mibqp(mqp); 513 } 514 515 wc->qp = &(*cur_qp)->ibqp; 516 switch (opcode) { 517 case MLX5_CQE_REQ: 518 wq = &(*cur_qp)->sq; 519 wqe_ctr = be16_to_cpu(cqe64->wqe_counter); 520 idx = wqe_ctr & (wq->wqe_cnt - 1); 521 handle_good_req(wc, cqe64, wq, idx); 522 handle_atomics(*cur_qp, cqe64, wq->last_poll, idx); 523 wc->wr_id = wq->swr_ctx[idx].wrid; 524 wq->tail = wq->swr_ctx[idx].wqe_head + 1; 525 if (unlikely(wq->swr_ctx[idx].w_list.opcode & 526 MLX5_OPCODE_SIGNATURE_CANCELED)) 527 wc->status = IB_WC_GENERAL_ERR; 528 else 529 wc->status = IB_WC_SUCCESS; 530 break; 531 case MLX5_CQE_RESP_WR_IMM: 532 case MLX5_CQE_RESP_SEND: 533 case MLX5_CQE_RESP_SEND_IMM: 534 case MLX5_CQE_RESP_SEND_INV: 535 handle_responder(wc, cqe64, *cur_qp); 536 wc->status = IB_WC_SUCCESS; 537 break; 538 case MLX5_CQE_RESIZE_CQ: 539 break; 540 case MLX5_CQE_REQ_ERR: 541 case MLX5_CQE_RESP_ERR: 542 err_cqe = (struct mlx5_err_cqe *)cqe64; 543 mlx5_handle_error_cqe(dev, err_cqe, wc); 544 mlx5_ib_dbg(dev, "%s error cqe on cqn 0x%x:\n", 545 opcode == MLX5_CQE_REQ_ERR ? 546 "Requestor" : "Responder", cq->mcq.cqn); 547 mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n", 548 err_cqe->syndrome, err_cqe->vendor_err_synd); 549 if (opcode == MLX5_CQE_REQ_ERR) { 550 wq = &(*cur_qp)->sq; 551 wqe_ctr = be16_to_cpu(cqe64->wqe_counter); 552 idx = wqe_ctr & (wq->wqe_cnt - 1); 553 wc->wr_id = wq->swr_ctx[idx].wrid; 554 wq->tail = wq->swr_ctx[idx].wqe_head + 1; 555 } else { 556 struct mlx5_ib_srq *srq; 557 558 if ((*cur_qp)->ibqp.srq) { 559 srq = to_msrq((*cur_qp)->ibqp.srq); 560 wqe_ctr = be16_to_cpu(cqe64->wqe_counter); 561 wc->wr_id = srq->wrid[wqe_ctr]; 562 mlx5_ib_free_srq_wqe(srq, wqe_ctr); 563 } else { 564 wq = &(*cur_qp)->rq; 565 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid; 566 ++wq->tail; 567 } 568 } 569 break; 570 case MLX5_CQE_SIG_ERR: 571 sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64; 572 573 spin_lock_irqsave(&dev->mdev->priv.mr_table.lock, flags); 574 mmr = __mlx5_mr_lookup(dev->mdev, 575 mlx5_ib_base_mkey(be32_to_cpu(sig_err_cqe->mkey))); 576 if (unlikely(!mmr)) { 577 spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags); 578 mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n", 579 cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey)); 580 return -EINVAL; 581 } 582 583 mr = to_mibmr(mmr); 584 mr->sig->sig_err_exists = true; 585 mr->sig->sigerr_count++; 586 587 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR\n", cq->mcq.cqn); 588 589 spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags); 590 goto repoll; 591 } 592 593 return 0; 594 } 595 596 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 597 { 598 struct mlx5_ib_cq *cq = to_mcq(ibcq); 599 struct mlx5_ib_qp *cur_qp = NULL; 600 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); 601 struct mlx5_core_dev *mdev = dev->mdev; 602 unsigned long flags; 603 int npolled; 604 int err = 0; 605 606 spin_lock_irqsave(&cq->lock, flags); 607 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 608 mlx5_ib_poll_sw_comp(cq, num_entries, wc, &npolled); 609 goto out; 610 } 611 612 for (npolled = 0; npolled < num_entries; npolled++) { 613 err = mlx5_poll_one(cq, &cur_qp, wc + npolled); 614 if (err) 615 break; 616 } 617 618 if (npolled) 619 mlx5_cq_set_ci(&cq->mcq); 620 out: 621 spin_unlock_irqrestore(&cq->lock, flags); 622 623 if (err == 0 || err == -EAGAIN) 624 return npolled; 625 else 626 return err; 627 } 628 629 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 630 { 631 struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev; 632 void __iomem *uar_page = mdev->priv.uuari.uars[0].map; 633 634 635 mlx5_cq_arm(&to_mcq(ibcq)->mcq, 636 (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 637 MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT, 638 uar_page, 639 MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock), 640 to_mcq(ibcq)->mcq.cons_index); 641 642 return 0; 643 } 644 645 static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf, 646 int nent, int cqe_size) 647 { 648 int err; 649 650 err = mlx5_buf_alloc(dev->mdev, nent * cqe_size, 651 PAGE_SIZE * 2, &buf->buf); 652 if (err) { 653 mlx5_ib_err(dev, "alloc failed\n"); 654 return err; 655 } 656 657 buf->cqe_size = cqe_size; 658 buf->nent = nent; 659 660 return 0; 661 } 662 663 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, 664 struct ib_ucontext *context, struct mlx5_ib_cq *cq, 665 int entries, struct mlx5_create_cq_mbox_in **cqb, 666 int *cqe_size, int *index, int *inlen) 667 { 668 struct mlx5_exp_ib_create_cq ucmd; 669 size_t ucmdlen; 670 int page_shift; 671 int npages; 672 int ncont; 673 int err; 674 675 memset(&ucmd, 0, sizeof(ucmd)); 676 677 ucmdlen = 678 (udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) < 679 sizeof(struct mlx5_ib_create_cq)) ? 680 (sizeof(struct mlx5_ib_create_cq) - sizeof(ucmd.reserved)) : 681 sizeof(struct mlx5_ib_create_cq); 682 683 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 684 mlx5_ib_err(dev, "copy failed\n"); 685 return -EFAULT; 686 } 687 688 if (ucmdlen == sizeof(ucmd) && ucmd.reserved != 0) { 689 mlx5_ib_err(dev, "command corrupted\n"); 690 return -EINVAL; 691 } 692 693 if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128) { 694 mlx5_ib_warn(dev, "wrong CQE size %d\n", ucmd.cqe_size); 695 return -EINVAL; 696 } 697 698 *cqe_size = ucmd.cqe_size; 699 700 cq->buf.umem = ib_umem_get(context, ucmd.buf_addr, 701 entries * ucmd.cqe_size, 702 IB_ACCESS_LOCAL_WRITE, 1); 703 if (IS_ERR(cq->buf.umem)) { 704 err = PTR_ERR(cq->buf.umem); 705 return err; 706 } 707 708 err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr, 709 &cq->db); 710 if (err) { 711 mlx5_ib_warn(dev, "map failed\n"); 712 goto err_umem; 713 } 714 715 mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift, 716 &ncont, NULL); 717 mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n", 718 (unsigned long long)ucmd.buf_addr, entries * ucmd.cqe_size, 719 npages, page_shift, ncont); 720 721 *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * ncont; 722 *cqb = mlx5_vzalloc(*inlen); 723 if (!*cqb) { 724 err = -ENOMEM; 725 goto err_db; 726 } 727 728 mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, 0); 729 (*cqb)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 730 731 *index = to_mucontext(context)->uuari.uars[0].index; 732 733 if (*cqe_size == 64 && MLX5_CAP_GEN(dev->mdev, cqe_compression)) { 734 if (ucmd.exp_data.cqe_comp_en == 1 && 735 (ucmd.exp_data.comp_mask & MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_EN)) { 736 MLX5_SET(cqc, &(*cqb)->ctx, cqe_compression_en, 1); 737 if (ucmd.exp_data.cqe_comp_recv_type == 738 MLX5_IB_CQE_FORMAT_CSUM && 739 (ucmd.exp_data.comp_mask & 740 MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_RECV_TYPE)) 741 MLX5_SET(cqc, &(*cqb)->ctx, mini_cqe_res_format, 742 MLX5_IB_CQE_FORMAT_CSUM); 743 } 744 } 745 746 return 0; 747 748 err_db: 749 mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db); 750 751 err_umem: 752 ib_umem_release(cq->buf.umem); 753 return err; 754 } 755 756 static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context) 757 { 758 759 mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db); 760 ib_umem_release(cq->buf.umem); 761 } 762 763 static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf) 764 { 765 int i; 766 void *cqe; 767 struct mlx5_cqe64 *cqe64; 768 769 for (i = 0; i < buf->nent; i++) { 770 cqe = get_cqe_from_buf(buf, i, buf->cqe_size); 771 cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64; 772 cqe64->op_own = MLX5_CQE_INVALID << 4; 773 } 774 } 775 776 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 777 int entries, int cqe_size, 778 struct mlx5_create_cq_mbox_in **cqb, 779 int *index, int *inlen) 780 { 781 int err; 782 783 err = mlx5_db_alloc(dev->mdev, &cq->db); 784 if (err) 785 return err; 786 787 cq->mcq.set_ci_db = cq->db.db; 788 cq->mcq.arm_db = cq->db.db + 1; 789 cq->mcq.cqe_sz = cqe_size; 790 791 err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size); 792 if (err) 793 goto err_db; 794 795 init_cq_buf(cq, &cq->buf); 796 797 *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * cq->buf.buf.npages; 798 *cqb = mlx5_vzalloc(*inlen); 799 if (!*cqb) { 800 err = -ENOMEM; 801 goto err_buf; 802 } 803 mlx5_fill_page_array(&cq->buf.buf, (*cqb)->pas); 804 805 (*cqb)->ctx.log_pg_sz = cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 806 *index = dev->mdev->priv.uuari.uars[0].index; 807 808 return 0; 809 810 err_buf: 811 free_cq_buf(dev, &cq->buf); 812 813 err_db: 814 mlx5_db_free(dev->mdev, &cq->db); 815 return err; 816 } 817 818 static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) 819 { 820 free_cq_buf(dev, &cq->buf); 821 mlx5_db_free(dev->mdev, &cq->db); 822 } 823 824 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, 825 struct ib_cq_init_attr *attr, 826 struct ib_ucontext *context, 827 struct ib_udata *udata) 828 { 829 struct mlx5_create_cq_mbox_in *cqb = NULL; 830 struct mlx5_ib_dev *dev = to_mdev(ibdev); 831 struct mlx5_ib_cq *cq; 832 int entries = attr->cqe; 833 int vector = attr->comp_vector; 834 int uninitialized_var(index); 835 int uninitialized_var(inlen); 836 int cqe_size; 837 int irqn; 838 int eqn; 839 int err; 840 841 if (entries < 0 || roundup_pow_of_two(entries + 1) > 842 (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) { 843 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n", 844 entries, roundup_pow_of_two(entries + 1), 845 1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)); 846 return ERR_PTR(-EINVAL); 847 } 848 849 entries = roundup_pow_of_two(entries + 1); 850 851 cq = kzalloc(sizeof(*cq), GFP_KERNEL); 852 if (!cq) 853 return ERR_PTR(-ENOMEM); 854 855 cq->ibcq.cqe = entries - 1; 856 mutex_init(&cq->resize_mutex); 857 spin_lock_init(&cq->lock); 858 cq->resize_buf = NULL; 859 cq->resize_umem = NULL; 860 861 INIT_LIST_HEAD(&cq->list_send_qp); 862 INIT_LIST_HEAD(&cq->list_recv_qp); 863 864 if (context) { 865 err = create_cq_user(dev, udata, context, cq, entries, 866 &cqb, &cqe_size, &index, &inlen); 867 if (err) 868 goto err_create; 869 } else { 870 cqe_size = (cache_line_size() >= 128 ? 128 : 64); 871 err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, 872 &index, &inlen); 873 if (err) 874 goto err_create; 875 } 876 877 cq->cqe_size = cqe_size; 878 cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5; 879 cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index); 880 err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn); 881 if (err) 882 goto err_cqb; 883 884 cqb->ctx.c_eqn = cpu_to_be16(eqn); 885 cqb->ctx.db_record_addr = cpu_to_be64(cq->db.dma); 886 887 err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen); 888 if (err) 889 goto err_cqb; 890 891 mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn); 892 cq->mcq.irqn = irqn; 893 cq->mcq.comp = mlx5_ib_cq_comp; 894 cq->mcq.event = mlx5_ib_cq_event; 895 896 if (context) 897 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) { 898 err = -EFAULT; 899 goto err_cmd; 900 } 901 902 903 kvfree(cqb); 904 return &cq->ibcq; 905 906 err_cmd: 907 mlx5_core_destroy_cq(dev->mdev, &cq->mcq); 908 909 err_cqb: 910 kvfree(cqb); 911 if (context) 912 destroy_cq_user(cq, context); 913 else 914 destroy_cq_kernel(dev, cq); 915 916 err_create: 917 kfree(cq); 918 919 return ERR_PTR(err); 920 } 921 922 923 int mlx5_ib_destroy_cq(struct ib_cq *cq) 924 { 925 struct mlx5_ib_dev *dev = to_mdev(cq->device); 926 struct mlx5_ib_cq *mcq = to_mcq(cq); 927 struct ib_ucontext *context = NULL; 928 929 if (cq->uobject) 930 context = cq->uobject->context; 931 932 mlx5_core_destroy_cq(dev->mdev, &mcq->mcq); 933 if (context) 934 destroy_cq_user(mcq, context); 935 else 936 destroy_cq_kernel(dev, mcq); 937 938 kfree(mcq); 939 940 return 0; 941 } 942 943 static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) 944 { 945 return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff); 946 } 947 948 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq) 949 { 950 struct mlx5_cqe64 *cqe64, *dest64; 951 void *cqe, *dest; 952 u32 prod_index; 953 int nfreed = 0; 954 u8 owner_bit; 955 956 if (!cq) 957 return; 958 959 /* First we need to find the current producer index, so we 960 * know where to start cleaning from. It doesn't matter if HW 961 * adds new entries after this loop -- the QP we're worried 962 * about is already in RESET, so the new entries won't come 963 * from our QP and therefore don't need to be checked. 964 */ 965 for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++) 966 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) 967 break; 968 969 /* Now sweep backwards through the CQ, removing CQ entries 970 * that match our QP by copying older entries on top of them. 971 */ 972 while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { 973 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); 974 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; 975 if (is_equal_rsn(cqe64, rsn)) { 976 if (srq && (ntohl(cqe64->srqn) & 0xffffff)) 977 mlx5_ib_free_srq_wqe(srq, be16_to_cpu(cqe64->wqe_counter)); 978 ++nfreed; 979 } else if (nfreed) { 980 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); 981 dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64; 982 owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK; 983 memcpy(dest, cqe, cq->mcq.cqe_sz); 984 dest64->op_own = owner_bit | 985 (dest64->op_own & ~MLX5_CQE_OWNER_MASK); 986 } 987 } 988 989 if (nfreed) { 990 cq->mcq.cons_index += nfreed; 991 /* Make sure update of buffer contents is done before 992 * updating consumer index. 993 */ 994 wmb(); 995 mlx5_cq_set_ci(&cq->mcq); 996 } 997 } 998 999 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq) 1000 { 1001 if (!cq) 1002 return; 1003 1004 spin_lock_irq(&cq->lock); 1005 __mlx5_ib_cq_clean(cq, qpn, srq); 1006 spin_unlock_irq(&cq->lock); 1007 } 1008 1009 int mlx5_ib_modify_cq(struct ib_cq *cq, struct ib_cq_attr *attr, int cq_attr_mask) 1010 { 1011 struct mlx5_modify_cq_mbox_in *in; 1012 struct mlx5_ib_dev *dev = to_mdev(cq->device); 1013 struct mlx5_ib_cq *mcq = to_mcq(cq); 1014 u16 cq_count = attr->moderation.cq_count; 1015 u16 cq_period = attr->moderation.cq_period; 1016 1017 int err; 1018 u32 fsel = 0; 1019 1020 in = kzalloc(sizeof(*in), GFP_KERNEL); 1021 if (!in) 1022 return -ENOMEM; 1023 1024 in->cqn = cpu_to_be32(mcq->mcq.cqn); 1025 if (cq_attr_mask & IB_CQ_MODERATION) { 1026 if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) { 1027 fsel |= (MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT); 1028 if (cq_period & 0xf000) { 1029 /* A value higher than 0xfff is required, better 1030 * use the largest value possible. */ 1031 cq_period = 0xfff; 1032 printf("mlx5_ib: INFO: ""period supported is limited to 12 bits\n"); 1033 } 1034 1035 in->ctx.cq_period = cpu_to_be16(cq_period); 1036 in->ctx.cq_max_count = cpu_to_be16(cq_count); 1037 } else { 1038 err = -ENOSYS; 1039 goto out; 1040 } 1041 } 1042 in->field_select = cpu_to_be32(fsel); 1043 err = mlx5_core_modify_cq(dev->mdev, &mcq->mcq, in, sizeof(*in)); 1044 1045 out: 1046 kfree(in); 1047 if (err) 1048 mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn); 1049 1050 return err; 1051 } 1052 1053 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 1054 int entries, struct ib_udata *udata, int *npas, 1055 int *page_shift, int *cqe_size) 1056 { 1057 struct mlx5_ib_resize_cq ucmd; 1058 struct ib_umem *umem; 1059 int err; 1060 int npages; 1061 struct ib_ucontext *context = cq->buf.umem->context; 1062 1063 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); 1064 if (err) 1065 return err; 1066 1067 if (ucmd.reserved0 || ucmd.reserved1) 1068 return -EINVAL; 1069 1070 umem = ib_umem_get(context, ucmd.buf_addr, entries * ucmd.cqe_size, 1071 IB_ACCESS_LOCAL_WRITE, 1); 1072 if (IS_ERR(umem)) { 1073 err = PTR_ERR(umem); 1074 return err; 1075 } 1076 1077 mlx5_ib_cont_pages(umem, ucmd.buf_addr, &npages, page_shift, 1078 npas, NULL); 1079 1080 cq->resize_umem = umem; 1081 *cqe_size = ucmd.cqe_size; 1082 1083 return 0; 1084 } 1085 1086 static void un_resize_user(struct mlx5_ib_cq *cq) 1087 { 1088 ib_umem_release(cq->resize_umem); 1089 } 1090 1091 static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 1092 int entries, int cqe_size) 1093 { 1094 int err; 1095 1096 cq->resize_buf = kzalloc(sizeof(*cq->resize_buf), GFP_KERNEL); 1097 if (!cq->resize_buf) 1098 return -ENOMEM; 1099 1100 err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size); 1101 if (err) 1102 goto ex; 1103 1104 init_cq_buf(cq, cq->resize_buf); 1105 1106 return 0; 1107 1108 ex: 1109 kfree(cq->resize_buf); 1110 return err; 1111 } 1112 1113 static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) 1114 { 1115 free_cq_buf(dev, cq->resize_buf); 1116 cq->resize_buf = NULL; 1117 } 1118 1119 static int copy_resize_cqes(struct mlx5_ib_cq *cq) 1120 { 1121 struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); 1122 struct mlx5_cqe64 *scqe64; 1123 struct mlx5_cqe64 *dcqe64; 1124 void *start_cqe; 1125 void *scqe; 1126 void *dcqe; 1127 int ssize; 1128 int dsize; 1129 int i; 1130 u8 sw_own; 1131 1132 ssize = cq->buf.cqe_size; 1133 dsize = cq->resize_buf->cqe_size; 1134 if (ssize != dsize) { 1135 mlx5_ib_warn(dev, "resize from different cqe size is not supported\n"); 1136 return -EINVAL; 1137 } 1138 1139 i = cq->mcq.cons_index; 1140 scqe = get_sw_cqe(cq, i); 1141 scqe64 = ssize == 64 ? scqe : scqe + 64; 1142 start_cqe = scqe; 1143 if (!scqe) { 1144 mlx5_ib_warn(dev, "expected cqe in sw ownership\n"); 1145 return -EINVAL; 1146 } 1147 1148 while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) { 1149 dcqe = get_cqe_from_buf(cq->resize_buf, 1150 (i + 1) & (cq->resize_buf->nent), 1151 dsize); 1152 dcqe64 = dsize == 64 ? dcqe : dcqe + 64; 1153 sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent); 1154 memcpy(dcqe, scqe, dsize); 1155 dcqe64->op_own = (dcqe64->op_own & ~MLX5_CQE_OWNER_MASK) | sw_own; 1156 1157 ++i; 1158 scqe = get_sw_cqe(cq, i); 1159 scqe64 = ssize == 64 ? scqe : scqe + 64; 1160 if (!scqe) { 1161 mlx5_ib_warn(dev, "expected cqe in sw ownership\n"); 1162 return -EINVAL; 1163 } 1164 1165 if (scqe == start_cqe) { 1166 printf("mlx5_ib: WARN: ""resize CQ failed to get resize CQE, CQN 0x%x\n", cq->mcq.cqn); 1167 return -ENOMEM; 1168 } 1169 } 1170 ++cq->mcq.cons_index; 1171 return 0; 1172 } 1173 1174 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) 1175 { 1176 struct mlx5_ib_dev *dev = to_mdev(ibcq->device); 1177 struct mlx5_ib_cq *cq = to_mcq(ibcq); 1178 struct mlx5_modify_cq_mbox_in *in; 1179 int err; 1180 int npas; 1181 int page_shift; 1182 int inlen; 1183 int uninitialized_var(cqe_size); 1184 unsigned long flags; 1185 1186 if (!MLX5_CAP_GEN(dev->mdev, cq_resize)) { 1187 mlx5_ib_warn(dev, "Firmware does not support resize CQ\n"); 1188 return -ENOSYS; 1189 } 1190 1191 if (entries < 1 || roundup_pow_of_two(entries + 1) > 1192 (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) { 1193 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n", 1194 entries, roundup_pow_of_two(entries + 1), 1195 1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)); 1196 return -EINVAL; 1197 } 1198 1199 entries = roundup_pow_of_two(entries + 1); 1200 1201 if (entries == ibcq->cqe + 1) 1202 return 0; 1203 1204 mutex_lock(&cq->resize_mutex); 1205 if (udata) { 1206 err = resize_user(dev, cq, entries, udata, &npas, &page_shift, 1207 &cqe_size); 1208 } else { 1209 cqe_size = 64; 1210 err = resize_kernel(dev, cq, entries, cqe_size); 1211 if (!err) { 1212 npas = cq->resize_buf->buf.npages; 1213 page_shift = cq->resize_buf->buf.page_shift; 1214 } 1215 } 1216 1217 if (err) { 1218 mlx5_ib_warn(dev, "resize failed: %d\n", err); 1219 goto ex; 1220 } 1221 1222 inlen = sizeof(*in) + npas * sizeof(in->pas[0]); 1223 in = mlx5_vzalloc(inlen); 1224 if (!in) { 1225 err = -ENOMEM; 1226 goto ex_resize; 1227 } 1228 1229 if (udata) 1230 mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift, 1231 in->pas, 0); 1232 else 1233 mlx5_fill_page_array(&cq->resize_buf->buf, in->pas); 1234 1235 in->field_select = cpu_to_be32(MLX5_MODIFY_CQ_MASK_LOG_SIZE | 1236 MLX5_MODIFY_CQ_MASK_PG_OFFSET | 1237 MLX5_MODIFY_CQ_MASK_PG_SIZE); 1238 in->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 1239 in->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5; 1240 in->ctx.page_offset = 0; 1241 in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(entries) << 24); 1242 in->hdr.opmod = cpu_to_be16(MLX5_CQ_OPMOD_RESIZE); 1243 in->cqn = cpu_to_be32(cq->mcq.cqn); 1244 1245 err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen); 1246 if (err) { 1247 mlx5_ib_warn(dev, "modify cq failed: %d\n", err); 1248 goto ex_alloc; 1249 } 1250 1251 if (udata) { 1252 cq->ibcq.cqe = entries - 1; 1253 ib_umem_release(cq->buf.umem); 1254 cq->buf.umem = cq->resize_umem; 1255 cq->resize_umem = NULL; 1256 } else { 1257 struct mlx5_ib_cq_buf tbuf; 1258 int resized = 0; 1259 1260 spin_lock_irqsave(&cq->lock, flags); 1261 if (cq->resize_buf) { 1262 err = copy_resize_cqes(cq); 1263 if (!err) { 1264 tbuf = cq->buf; 1265 cq->buf = *cq->resize_buf; 1266 kfree(cq->resize_buf); 1267 cq->resize_buf = NULL; 1268 resized = 1; 1269 } 1270 } 1271 cq->ibcq.cqe = entries - 1; 1272 spin_unlock_irqrestore(&cq->lock, flags); 1273 if (resized) 1274 free_cq_buf(dev, &tbuf); 1275 } 1276 mutex_unlock(&cq->resize_mutex); 1277 1278 kvfree(in); 1279 return 0; 1280 1281 ex_alloc: 1282 kvfree(in); 1283 1284 ex_resize: 1285 if (udata) 1286 un_resize_user(cq); 1287 else 1288 un_resize_kernel(dev, cq); 1289 ex: 1290 mutex_unlock(&cq->resize_mutex); 1291 return err; 1292 } 1293 1294 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq) 1295 { 1296 struct mlx5_ib_cq *cq; 1297 1298 if (!ibcq) 1299 return 128; 1300 1301 cq = to_mcq(ibcq); 1302 return cq->cqe_size; 1303 } 1304