1 /* 2 * Copyright (c) 2018-2019 Cavium, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * File: qlnxr_verbs.c 30 */ 31 #include <sys/cdefs.h> 32 #include "qlnxr_def.h" 33 #include "rdma_common.h" 34 #include "qlnxr_roce.h" 35 #include "qlnxr_cm.h" 36 37 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 38 39 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ 40 do { \ 41 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ 42 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ 43 } while (0) 44 45 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 46 do { \ 47 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 48 (sge)->length = cpu_to_le32(vlength); \ 49 (sge)->flags = cpu_to_le32(vflags); \ 50 } while (0) 51 52 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 53 do { \ 54 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ 55 (hdr)->num_sges = num_sge; \ 56 } while (0) 57 58 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 59 do { \ 60 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 61 (sge)->length = cpu_to_le32(vlength); \ 62 (sge)->l_key = cpu_to_le32(vlkey); \ 63 } while (0) 64 65 #define NIPQUAD(addr) \ 66 ((unsigned char *)&addr)[0], \ 67 ((unsigned char *)&addr)[1], \ 68 ((unsigned char *)&addr)[2], \ 69 ((unsigned char *)&addr)[3] 70 71 static int 72 qlnxr_check_srq_params(struct qlnxr_dev *dev, 73 struct ib_srq_init_attr *attrs); 74 75 static int 76 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 77 struct qlnxr_srq *srq, 78 struct qlnxr_create_srq_ureq *ureq, 79 int access, int dmasync); 80 81 static int 82 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 83 struct qlnxr_dev *dev, 84 struct ib_srq_init_attr *init_attr); 85 86 static int 87 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 88 struct qlnxr_srq *srq, 89 struct ib_udata *udata); 90 91 static void 92 qlnxr_free_srq_user_params(struct qlnxr_srq *srq); 93 94 static void 95 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); 96 97 static u32 98 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); 99 100 int 101 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, 102 union ib_gid *sgid) 103 { 104 struct qlnxr_dev *dev; 105 qlnx_host_t *ha; 106 107 dev = get_qlnxr_dev(ibdev); 108 ha = dev->ha; 109 110 QL_DPRINT12(ha, "enter\n"); 111 112 memset(sgid->raw, 0, sizeof(sgid->raw)); 113 114 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); 115 116 QL_DPRINT12(ha, "exit\n"); 117 118 return 0; 119 } 120 121 int 122 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, 123 union ib_gid *sgid) 124 { 125 struct qlnxr_dev *dev; 126 qlnx_host_t *ha; 127 128 dev = get_qlnxr_dev(ibdev); 129 ha = dev->ha; 130 QL_DPRINT12(ha, "enter index: %d\n", index); 131 #if 0 132 int ret = 0; 133 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ 134 //if (!rdma_cap_roce_gid_table(ibdev, port)) { 135 if (!(rdma_protocol_roce(ibdev, port) && 136 ibdev->add_gid && ibdev->del_gid)) { 137 QL_DPRINT11(ha, "acquire gid failed\n"); 138 return -ENODEV; 139 } 140 141 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 142 if (ret == -EAGAIN) { 143 memcpy(sgid, &zgid, sizeof(*sgid)); 144 return 0; 145 } 146 #endif 147 if ((index >= QLNXR_MAX_SGID) || (index < 0)) { 148 QL_DPRINT12(ha, "invalid gid index %d\n", index); 149 memset(sgid, 0, sizeof(*sgid)); 150 return -EINVAL; 151 } 152 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); 153 154 QL_DPRINT12(ha, "exit : %p\n", sgid); 155 156 return 0; 157 } 158 159 int 160 qlnxr_create_srq(struct ib_srq *ibsrq, 161 struct ib_srq_init_attr *init_attr, 162 struct ib_udata *udata) 163 { 164 struct qlnxr_dev *dev; 165 qlnx_host_t *ha; 166 struct ecore_rdma_destroy_srq_in_params destroy_in_params; 167 struct ecore_rdma_create_srq_out_params out_params; 168 struct ecore_rdma_create_srq_in_params in_params; 169 u64 pbl_base_addr, phy_prod_pair_addr; 170 struct qlnxr_srq_hwq_info *hw_srq; 171 struct qlnxr_ucontext *ctx; 172 struct qlnxr_create_srq_ureq ureq; 173 u32 page_cnt, page_size; 174 struct qlnxr_srq *srq = get_qlnxr_srq(ibsrq); 175 int ret = 0; 176 177 dev = get_qlnxr_dev(ibsrq->device); 178 ha = dev->ha; 179 180 QL_DPRINT12(ha, "enter\n"); 181 182 ret = qlnxr_check_srq_params(dev, init_attr); 183 184 srq->dev = dev; 185 hw_srq = &srq->hw_srq; 186 spin_lock_init(&srq->lock); 187 memset(&in_params, 0, sizeof(in_params)); 188 189 if (udata) { 190 ctx = rdma_udata_to_drv_context( 191 udata, struct qlnxr_ucontext, ibucontext); 192 193 memset(&ureq, 0, sizeof(ureq)); 194 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 195 udata->inlen))) { 196 QL_DPRINT11(ha, "problem" 197 " copying data from user space\n"); 198 goto err0; 199 } 200 201 ret = qlnxr_init_srq_user_params(&ctx->ibucontext, srq, &ureq, 0, 0); 202 if (ret) 203 goto err0; 204 205 page_cnt = srq->usrq.pbl_info.num_pbes; 206 pbl_base_addr = srq->usrq.pbl_tbl->pa; 207 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 208 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT 209 // page_size = BIT(srq->usrq.umem->page_shift); 210 // else 211 page_size = srq->usrq.umem->page_size; 212 } else { 213 struct ecore_chain *pbl; 214 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); 215 if (ret) 216 goto err0; 217 pbl = &hw_srq->pbl; 218 219 page_cnt = ecore_chain_get_page_cnt(pbl); 220 pbl_base_addr = ecore_chain_get_pbl_phys(pbl); 221 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 222 page_size = pbl->elem_per_page << 4; 223 } 224 225 in_params.pd_id = get_qlnxr_pd(ibsrq->pd)->pd_id; 226 in_params.pbl_base_addr = pbl_base_addr; 227 in_params.prod_pair_addr = phy_prod_pair_addr; 228 in_params.num_pages = page_cnt; 229 in_params.page_size = page_size; 230 231 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 232 if (ret) 233 goto err1; 234 235 srq->srq_id = out_params.srq_id; 236 237 if (udata) { 238 ret = qlnxr_copy_srq_uresp(dev, srq, udata); 239 if (ret) 240 goto err2; 241 } 242 243 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); 244 return (0); 245 err2: 246 memset(&in_params, 0, sizeof(in_params)); 247 destroy_in_params.srq_id = srq->srq_id; 248 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 249 250 err1: 251 if (udata) 252 qlnxr_free_srq_user_params(srq); 253 else 254 qlnxr_free_srq_kernel_params(srq); 255 256 err0: 257 return (-EFAULT); 258 } 259 260 void 261 qlnxr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 262 { 263 struct qlnxr_dev *dev; 264 struct qlnxr_srq *srq; 265 qlnx_host_t *ha; 266 struct ecore_rdma_destroy_srq_in_params in_params; 267 268 srq = get_qlnxr_srq(ibsrq); 269 dev = srq->dev; 270 ha = dev->ha; 271 272 memset(&in_params, 0, sizeof(in_params)); 273 in_params.srq_id = srq->srq_id; 274 275 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); 276 277 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) 278 qlnxr_free_srq_user_params(srq); 279 else 280 qlnxr_free_srq_kernel_params(srq); 281 282 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); 283 } 284 285 int 286 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 287 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 288 { 289 struct qlnxr_dev *dev; 290 struct qlnxr_srq *srq; 291 qlnx_host_t *ha; 292 struct ecore_rdma_modify_srq_in_params in_params; 293 int ret = 0; 294 295 srq = get_qlnxr_srq(ibsrq); 296 dev = srq->dev; 297 ha = dev->ha; 298 299 QL_DPRINT12(ha, "enter\n"); 300 if (attr_mask & IB_SRQ_MAX_WR) { 301 QL_DPRINT12(ha, "invalid attribute mask=0x%x" 302 " specified for %p\n", attr_mask, srq); 303 return -EINVAL; 304 } 305 306 if (attr_mask & IB_SRQ_LIMIT) { 307 if (attr->srq_limit >= srq->hw_srq.max_wr) { 308 QL_DPRINT12(ha, "invalid srq_limit=0x%x" 309 " (max_srq_limit = 0x%x)\n", 310 attr->srq_limit, srq->hw_srq.max_wr); 311 return -EINVAL; 312 } 313 memset(&in_params, 0, sizeof(in_params)); 314 in_params.srq_id = srq->srq_id; 315 in_params.wqe_limit = attr->srq_limit; 316 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); 317 if (ret) 318 return ret; 319 } 320 321 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); 322 return 0; 323 } 324 325 int 326 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 327 { 328 struct qlnxr_dev *dev; 329 struct qlnxr_srq *srq; 330 qlnx_host_t *ha; 331 struct ecore_rdma_device *qattr; 332 srq = get_qlnxr_srq(ibsrq); 333 dev = srq->dev; 334 ha = dev->ha; 335 //qattr = &dev->attr; 336 qattr = ecore_rdma_query_device(dev->rdma_ctx); 337 QL_DPRINT12(ha, "enter\n"); 338 339 if (!dev->rdma_ctx) { 340 QL_DPRINT12(ha, "called with invalid params" 341 " rdma_ctx is NULL\n"); 342 return -EINVAL; 343 } 344 345 srq_attr->srq_limit = qattr->max_srq; 346 srq_attr->max_wr = qattr->max_srq_wr; 347 srq_attr->max_sge = qattr->max_sge; 348 349 QL_DPRINT12(ha, "exit\n"); 350 return 0; 351 } 352 353 /* Increment srq wr producer by one */ 354 static 355 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) 356 { 357 info->wr_prod_cnt++; 358 } 359 360 /* Increment srq wr consumer by one */ 361 static 362 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) 363 { 364 info->wr_cons_cnt++; 365 } 366 367 /* get_port_immutable verb is not available in FreeBSD */ 368 #if 0 369 int 370 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 371 struct ib_port_immutable *immutable) 372 { 373 struct qlnxr_dev *dev; 374 qlnx_host_t *ha; 375 dev = get_qlnxr_dev(ibdev); 376 ha = dev->ha; 377 378 QL_DPRINT12(ha, "entered but not implemented!!!\n"); 379 } 380 #endif 381 382 int 383 qlnxr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 384 const struct ib_recv_wr **bad_wr) 385 { 386 struct qlnxr_dev *dev; 387 struct qlnxr_srq *srq; 388 qlnx_host_t *ha; 389 struct qlnxr_srq_hwq_info *hw_srq; 390 struct ecore_chain *pbl; 391 unsigned long flags; 392 int status = 0; 393 u32 num_sge, offset; 394 395 srq = get_qlnxr_srq(ibsrq); 396 dev = srq->dev; 397 ha = dev->ha; 398 hw_srq = &srq->hw_srq; 399 400 QL_DPRINT12(ha, "enter\n"); 401 spin_lock_irqsave(&srq->lock, flags); 402 403 pbl = &srq->hw_srq.pbl; 404 while (wr) { 405 struct rdma_srq_wqe_header *hdr; 406 int i; 407 408 if (!qlnxr_srq_elem_left(hw_srq) || 409 wr->num_sge > srq->hw_srq.max_sges) { 410 QL_DPRINT11(ha, "WR cannot be posted" 411 " (%d, %d) || (%d > %d)\n", 412 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 413 wr->num_sge, srq->hw_srq.max_sges); 414 status = -ENOMEM; 415 *bad_wr = wr; 416 break; 417 } 418 419 hdr = ecore_chain_produce(pbl); 420 num_sge = wr->num_sge; 421 /* Set number of sge and WR id in header */ 422 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 423 424 /* PBL is maintained in case of WR granularity. 425 * So increment WR producer in case we post a WR. 426 */ 427 qlnxr_inc_srq_wr_prod(hw_srq); 428 hw_srq->wqe_prod++; 429 hw_srq->sge_prod++; 430 431 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", 432 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 433 434 for (i = 0; i < wr->num_sge; i++) { 435 struct rdma_srq_sge *srq_sge = 436 ecore_chain_produce(pbl); 437 /* Set SGE length, lkey and address */ 438 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 439 wr->sg_list[i].length, wr->sg_list[i].lkey); 440 441 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", 442 i, srq_sge->length, srq_sge->l_key, 443 srq_sge->addr.hi, srq_sge->addr.lo); 444 hw_srq->sge_prod++; 445 } 446 wmb(); 447 /* 448 * SRQ prod is 8 bytes. Need to update SGE prod in index 449 * in first 4 bytes and need to update WQE prod in next 450 * 4 bytes. 451 */ 452 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; 453 offset = offsetof(struct rdma_srq_producers, wqe_prod); 454 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 455 hw_srq->wqe_prod; 456 /* Flush prod after updating it */ 457 wmb(); 458 wr = wr->next; 459 } 460 461 QL_DPRINT12(ha, "Elements in SRQ: %d\n", 462 ecore_chain_get_elem_left(pbl)); 463 464 spin_unlock_irqrestore(&srq->lock, flags); 465 QL_DPRINT12(ha, "exit\n"); 466 return status; 467 } 468 469 int 470 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 471 struct ib_udata *udata) 472 { 473 struct qlnxr_dev *dev; 474 struct ecore_rdma_device *qattr; 475 qlnx_host_t *ha; 476 477 dev = get_qlnxr_dev(ibdev); 478 ha = dev->ha; 479 480 QL_DPRINT12(ha, "enter\n"); 481 482 if (udata->inlen || udata->outlen) 483 return -EINVAL; 484 485 if (dev->rdma_ctx == NULL) { 486 return -EINVAL; 487 } 488 489 qattr = ecore_rdma_query_device(dev->rdma_ctx); 490 491 memset(attr, 0, sizeof *attr); 492 493 attr->fw_ver = qattr->fw_ver; 494 attr->sys_image_guid = qattr->sys_image_guid; 495 attr->max_mr_size = qattr->max_mr_size; 496 attr->page_size_cap = qattr->page_size_caps; 497 attr->vendor_id = qattr->vendor_id; 498 attr->vendor_part_id = qattr->vendor_part_id; 499 attr->hw_ver = qattr->hw_ver; 500 attr->max_qp = qattr->max_qp; 501 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 502 IB_DEVICE_RC_RNR_NAK_GEN | 503 IB_DEVICE_LOCAL_DMA_LKEY | 504 IB_DEVICE_MEM_MGT_EXTENSIONS; 505 506 attr->max_sge = qattr->max_sge; 507 attr->max_sge_rd = qattr->max_sge; 508 attr->max_cq = qattr->max_cq; 509 attr->max_cqe = qattr->max_cqe; 510 attr->max_mr = qattr->max_mr; 511 attr->max_mw = qattr->max_mw; 512 attr->max_pd = qattr->max_pd; 513 attr->atomic_cap = dev->atomic_cap; 514 attr->max_fmr = qattr->max_fmr; 515 attr->max_map_per_fmr = 16; /* TBD: FMR */ 516 517 /* There is an implicit assumption in some of the ib_xxx apps that the 518 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in 519 * communication the qp_rd_atom is passed to the other side and used as 520 * init_rd_atom without check device capabilities for init_rd_atom. 521 * for this reason, we set the qp_rd_atom to be the minimum between the 522 * two...There is an additional assumption in mlx4 driver that the 523 * values are power of two, fls is performed on the value - 1, which 524 * in fact gives a larger power of two for values which are not a power 525 * of two. This should be fixed in mlx4 driver, but until then -> 526 * we provide a value that is a power of two in our code. 527 */ 528 attr->max_qp_init_rd_atom = 529 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 530 attr->max_qp_rd_atom = 531 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 532 attr->max_qp_init_rd_atom); 533 534 attr->max_srq = qattr->max_srq; 535 attr->max_srq_sge = qattr->max_srq_sge; 536 attr->max_srq_wr = qattr->max_srq_wr; 537 538 /* TODO: R&D to more properly configure the following */ 539 attr->local_ca_ack_delay = qattr->dev_ack_delay; 540 attr->max_fast_reg_page_list_len = qattr->max_mr/8; 541 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; 542 attr->max_ah = qattr->max_ah; 543 544 QL_DPRINT12(ha, "exit\n"); 545 return 0; 546 } 547 548 static inline void 549 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) 550 { 551 switch (speed) { 552 case 1000: 553 *ib_speed = IB_SPEED_SDR; 554 *ib_width = IB_WIDTH_1X; 555 break; 556 case 10000: 557 *ib_speed = IB_SPEED_QDR; 558 *ib_width = IB_WIDTH_1X; 559 break; 560 561 case 20000: 562 *ib_speed = IB_SPEED_DDR; 563 *ib_width = IB_WIDTH_4X; 564 break; 565 566 case 25000: 567 *ib_speed = IB_SPEED_EDR; 568 *ib_width = IB_WIDTH_1X; 569 break; 570 571 case 40000: 572 *ib_speed = IB_SPEED_QDR; 573 *ib_width = IB_WIDTH_4X; 574 break; 575 576 case 50000: 577 *ib_speed = IB_SPEED_QDR; 578 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... 579 break; 580 581 case 100000: 582 *ib_speed = IB_SPEED_EDR; 583 *ib_width = IB_WIDTH_4X; 584 break; 585 586 default: 587 /* Unsupported */ 588 *ib_speed = IB_SPEED_SDR; 589 *ib_width = IB_WIDTH_1X; 590 } 591 return; 592 } 593 594 int 595 qlnxr_query_port(struct ib_device *ibdev, uint8_t port, 596 struct ib_port_attr *attr) 597 { 598 struct qlnxr_dev *dev; 599 struct ecore_rdma_port *rdma_port; 600 qlnx_host_t *ha; 601 602 dev = get_qlnxr_dev(ibdev); 603 ha = dev->ha; 604 605 QL_DPRINT12(ha, "enter\n"); 606 607 if (port > 1) { 608 QL_DPRINT12(ha, "port [%d] > 1 \n", port); 609 return -EINVAL; 610 } 611 612 if (dev->rdma_ctx == NULL) { 613 QL_DPRINT12(ha, "rdma_ctx == NULL\n"); 614 return -EINVAL; 615 } 616 617 rdma_port = ecore_rdma_query_port(dev->rdma_ctx); 618 memset(attr, 0, sizeof *attr); 619 620 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { 621 attr->state = IB_PORT_ACTIVE; 622 attr->phys_state = 5; 623 } else { 624 attr->state = IB_PORT_DOWN; 625 attr->phys_state = 3; 626 } 627 628 attr->max_mtu = IB_MTU_4096; 629 attr->active_mtu = iboe_get_mtu(if_getmtu(dev->ha->ifp)); 630 attr->lid = 0; 631 attr->lmc = 0; 632 attr->sm_lid = 0; 633 attr->sm_sl = 0; 634 attr->port_cap_flags = 0; 635 636 if (QLNX_IS_IWARP(dev)) { 637 attr->gid_tbl_len = 1; 638 attr->pkey_tbl_len = 1; 639 } else { 640 attr->gid_tbl_len = QLNXR_MAX_SGID; 641 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; 642 } 643 644 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 645 attr->qkey_viol_cntr = 0; 646 647 get_link_speed_and_width(rdma_port->link_speed, 648 &attr->active_speed, &attr->active_width); 649 650 attr->max_msg_sz = rdma_port->max_msg_size; 651 attr->max_vl_num = 4; /* TODO -> figure this one out... */ 652 653 QL_DPRINT12(ha, "state = %d phys_state = %d " 654 " link_speed = %d active_speed = %d active_width = %d" 655 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" 656 " max_msg_sz = 0x%x max_vl_num = 0x%x \n", 657 attr->state, attr->phys_state, 658 rdma_port->link_speed, attr->active_speed, 659 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, 660 attr->max_msg_sz, attr->max_vl_num); 661 662 QL_DPRINT12(ha, "exit\n"); 663 return 0; 664 } 665 666 int 667 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, 668 struct ib_port_modify *props) 669 { 670 struct qlnxr_dev *dev; 671 qlnx_host_t *ha; 672 673 dev = get_qlnxr_dev(ibdev); 674 ha = dev->ha; 675 676 QL_DPRINT12(ha, "enter\n"); 677 678 if (port > 1) { 679 QL_DPRINT12(ha, "port (%d) > 1\n", port); 680 return -EINVAL; 681 } 682 683 QL_DPRINT12(ha, "exit\n"); 684 return 0; 685 } 686 687 enum rdma_link_layer 688 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) 689 { 690 struct qlnxr_dev *dev; 691 qlnx_host_t *ha; 692 693 dev = get_qlnxr_dev(ibdev); 694 ha = dev->ha; 695 696 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); 697 698 return IB_LINK_LAYER_ETHERNET; 699 } 700 701 int 702 qlnxr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 703 { 704 struct ib_device *ibdev = ibpd->device; 705 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 706 u16 pd_id; 707 int rc; 708 struct qlnxr_dev *dev; 709 qlnx_host_t *ha; 710 711 dev = get_qlnxr_dev(ibdev); 712 ha = dev->ha; 713 714 QL_DPRINT12(ha, "ibdev = %p udata = %p enter\n", ibdev, udata); 715 716 if (dev->rdma_ctx == NULL) { 717 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); 718 rc = -1; 719 goto err; 720 } 721 722 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); 723 if (rc) { 724 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); 725 goto err; 726 } 727 728 pd->pd_id = pd_id; 729 730 if (udata) { 731 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); 732 if (rc) { 733 QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); 734 ecore_rdma_free_pd(dev->rdma_ctx, pd_id); 735 goto err; 736 } 737 738 pd->uctx = rdma_udata_to_drv_context( 739 udata, struct qlnxr_ucontext, ibucontext); 740 pd->uctx->pd = pd; 741 } 742 743 atomic_add_rel_32(&dev->pd_count, 1); 744 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 745 pd, pd_id, dev->pd_count); 746 747 return (0); 748 749 err: 750 QL_DPRINT12(ha, "exit -1\n"); 751 return (rc); 752 } 753 754 void 755 qlnxr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 756 { 757 struct qlnxr_pd *pd; 758 struct qlnxr_dev *dev; 759 qlnx_host_t *ha; 760 761 pd = get_qlnxr_pd(ibpd); 762 dev = get_qlnxr_dev((ibpd->device)); 763 ha = dev->ha; 764 765 QL_DPRINT12(ha, "enter\n"); 766 767 if (pd == NULL) { 768 QL_DPRINT11(ha, "pd = NULL\n"); 769 } else { 770 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); 771 atomic_subtract_rel_32(&dev->pd_count, 1); 772 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 773 pd, pd->pd_id, dev->pd_count); 774 } 775 776 QL_DPRINT12(ha, "exit\n"); 777 } 778 779 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) 780 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ 781 /* Should be part of HSI */ 782 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ 783 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 784 785 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); 786 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); 787 788 int 789 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 790 { 791 struct qlnxr_dev *dev; 792 qlnx_host_t *ha; 793 794 dev = get_qlnxr_dev(ibdev); 795 ha = dev->ha; 796 797 QL_DPRINT12(ha, "enter index = 0x%x\n", index); 798 799 if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 800 return -EINVAL; 801 802 *pkey = QLNXR_ROCE_PKEY_DEFAULT; 803 804 QL_DPRINT12(ha, "exit\n"); 805 return 0; 806 } 807 808 static inline bool 809 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, 810 u16 *vlan_id) 811 { 812 bool ret = false; 813 814 u16 tmp_vlan_id; 815 816 union ib_gid *dgid; 817 818 QL_DPRINT12(ha, "enter \n"); 819 820 *vlan_id = 0; 821 822 dgid = &attr->ah_attr.grh.dgid; 823 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; 824 825 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { 826 *vlan_id = tmp_vlan_id; 827 ret = true; 828 } 829 830 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); 831 832 return (ret); 833 } 834 835 static inline void 836 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, 837 int attr_mask, 838 struct qlnxr_dev *dev, 839 struct qlnxr_qp *qp, 840 struct ecore_rdma_modify_qp_in_params *qp_params) 841 { 842 int i; 843 qlnx_host_t *ha; 844 845 ha = dev->ha; 846 847 QL_DPRINT12(ha, "enter\n"); 848 849 memcpy(&qp_params->sgid.bytes[0], 850 &dev->sgid_tbl[qp->sgid_idx].raw[0], 851 sizeof(qp_params->sgid.bytes)); 852 memcpy(&qp_params->dgid.bytes[0], 853 &attr->ah_attr.grh.dgid.raw[0], 854 sizeof(qp_params->dgid)); 855 856 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); 857 858 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { 859 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 860 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 861 } 862 863 QL_DPRINT12(ha, "exit\n"); 864 return; 865 } 866 867 static int 868 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 869 { 870 struct qlnxr_mm *mm; 871 qlnx_host_t *ha; 872 873 ha = uctx->dev->ha; 874 875 QL_DPRINT12(ha, "enter\n"); 876 877 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 878 if (mm == NULL) { 879 QL_DPRINT11(ha, "mm = NULL\n"); 880 return -ENOMEM; 881 } 882 883 mm->key.phy_addr = phy_addr; 884 885 /* This function might be called with a length which is not a multiple 886 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 887 * forces this granularity by increasing the requested size if needed. 888 * When qedr_mmap is called, it will search the list with the updated 889 * length as a key. To prevent search failures, the length is rounded up 890 * in advance to PAGE_SIZE. 891 */ 892 mm->key.len = roundup(len, PAGE_SIZE); 893 INIT_LIST_HEAD(&mm->entry); 894 895 mutex_lock(&uctx->mm_list_lock); 896 list_add(&mm->entry, &uctx->mm_head); 897 mutex_unlock(&uctx->mm_list_lock); 898 899 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 900 (unsigned long long)mm->key.phy_addr, 901 (unsigned long)mm->key.len, uctx); 902 903 return 0; 904 } 905 906 static bool 907 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 908 { 909 bool found = false; 910 struct qlnxr_mm *mm; 911 qlnx_host_t *ha; 912 913 ha = uctx->dev->ha; 914 915 QL_DPRINT12(ha, "enter\n"); 916 917 mutex_lock(&uctx->mm_list_lock); 918 list_for_each_entry(mm, &uctx->mm_head, entry) { 919 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 920 continue; 921 922 found = true; 923 break; 924 } 925 mutex_unlock(&uctx->mm_list_lock); 926 927 QL_DPRINT12(ha, 928 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", 929 mm->key.phy_addr, mm->key.len, uctx, found); 930 931 return found; 932 } 933 934 int 935 qlnxr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 936 { 937 int rc; 938 struct qlnxr_ucontext *ctx = get_qlnxr_ucontext(uctx); 939 struct qlnxr_alloc_ucontext_resp uresp; 940 struct qlnxr_dev *dev = get_qlnxr_dev(uctx->device); 941 qlnx_host_t *ha = dev->ha; 942 struct ecore_rdma_add_user_out_params oparams; 943 944 if (!udata) 945 return -EFAULT; 946 947 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); 948 if (rc) { 949 QL_DPRINT12(ha, 950 "Failed to allocate a DPI for a new RoCE application " 951 ",rc = %d. To overcome this, consider to increase " 952 "the number of DPIs, increase the doorbell BAR size " 953 "or just close unnecessary RoCE applications. In " 954 "order to increase the number of DPIs consult the " 955 "README\n", rc); 956 goto err; 957 } 958 959 ctx->dpi = oparams.dpi; 960 ctx->dpi_addr = oparams.dpi_addr; 961 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 962 ctx->dpi_size = oparams.dpi_size; 963 INIT_LIST_HEAD(&ctx->mm_head); 964 mutex_init(&ctx->mm_list_lock); 965 966 memset(&uresp, 0, sizeof(uresp)); 967 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) 968 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out 969 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) 970 < udata->outlen ? 1 : 0; //TODO: figure this out 971 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) 972 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 973 uresp.db_pa = ctx->dpi_phys_addr; 974 uresp.db_size = ctx->dpi_size; 975 uresp.max_send_wr = dev->attr.max_sqe; 976 uresp.max_recv_wr = dev->attr.max_rqe; 977 uresp.max_srq_wr = dev->attr.max_srq_wr; 978 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 979 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 980 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 981 uresp.max_cqes = QLNXR_MAX_CQES; 982 983 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 984 if (rc) 985 goto err; 986 987 ctx->dev = dev; 988 989 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 990 if (rc) 991 goto err; 992 QL_DPRINT12(ha, "Allocated user context %p\n", 993 &ctx->ibucontext); 994 995 return (0); 996 err: 997 return (rc); 998 } 999 1000 void 1001 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) 1002 { 1003 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); 1004 struct qlnxr_dev *dev = uctx->dev; 1005 qlnx_host_t *ha = dev->ha; 1006 struct qlnxr_mm *mm, *tmp; 1007 1008 QL_DPRINT12(ha, "Deallocating user context %p\n", 1009 uctx); 1010 1011 if (dev) { 1012 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 1013 } 1014 1015 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 1016 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" 1017 " ctx=%p\n", 1018 mm->key.phy_addr, mm->key.len, uctx); 1019 list_del(&mm->entry); 1020 kfree(mm); 1021 } 1022 } 1023 1024 int 1025 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 1026 { 1027 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); 1028 struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); 1029 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 1030 u64 unmapped_db; 1031 unsigned long len = (vma->vm_end - vma->vm_start); 1032 int rc = 0; 1033 bool found; 1034 qlnx_host_t *ha; 1035 1036 ha = dev->ha; 1037 1038 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 1039 1040 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" 1041 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 1042 vm_page, vma->vm_pgoff, unmapped_db, 1043 dev->db_size, len); 1044 1045 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 1046 QL_DPRINT11(ha, "Vma_start not page aligned " 1047 "vm_start = %ld vma_end = %ld\n", vma->vm_start, 1048 vma->vm_end); 1049 return -EINVAL; 1050 } 1051 1052 found = qlnxr_search_mmap(ucontext, vm_page, len); 1053 if (!found) { 1054 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", 1055 vma->vm_pgoff); 1056 return -EINVAL; 1057 } 1058 1059 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1060 1061 if ((vm_page < unmapped_db) || 1062 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { 1063 QL_DPRINT11(ha, "failed pages are outside of dpi;" 1064 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", 1065 vm_page, unmapped_db, ucontext->dpi_size); 1066 return -EINVAL; 1067 } 1068 1069 if (vma->vm_flags & VM_READ) { 1070 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); 1071 return -EINVAL; 1072 } 1073 1074 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1075 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 1076 vma->vm_page_prot); 1077 1078 QL_DPRINT12(ha, "exit [%d]\n", rc); 1079 return rc; 1080 } 1081 1082 struct ib_mr * 1083 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) 1084 { 1085 struct qlnxr_mr *mr; 1086 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1087 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 1088 int rc; 1089 qlnx_host_t *ha; 1090 1091 ha = dev->ha; 1092 1093 QL_DPRINT12(ha, "enter\n"); 1094 1095 if (acc & IB_ACCESS_MW_BIND) { 1096 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); 1097 } 1098 1099 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1100 if (!mr) { 1101 rc = -ENOMEM; 1102 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); 1103 goto err0; 1104 } 1105 1106 mr->type = QLNXR_MR_DMA; 1107 1108 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1109 if (rc) { 1110 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); 1111 goto err1; 1112 } 1113 1114 /* index only, 18 bit long, lkey = itid << 8 | key */ 1115 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1116 mr->hw_mr.pd = pd->pd_id; 1117 mr->hw_mr.local_read = 1; 1118 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1119 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1120 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1121 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1122 mr->hw_mr.dma_mr = true; 1123 1124 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1125 if (rc) { 1126 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); 1127 goto err2; 1128 } 1129 1130 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1131 1132 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1133 mr->hw_mr.remote_atomic) { 1134 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1135 } 1136 1137 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); 1138 1139 return &mr->ibmr; 1140 1141 err2: 1142 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1143 err1: 1144 kfree(mr); 1145 err0: 1146 QL_DPRINT12(ha, "exit [%d]\n", rc); 1147 1148 return ERR_PTR(rc); 1149 } 1150 1151 static void 1152 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, 1153 struct qlnxr_pbl *pbl) 1154 { 1155 int i; 1156 qlnx_host_t *ha; 1157 1158 ha = dev->ha; 1159 1160 QL_DPRINT12(ha, "enter\n"); 1161 1162 for (i = 0; i < pbl_info->num_pbls; i++) { 1163 if (!pbl[i].va) 1164 continue; 1165 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, 1166 pbl_info->pbl_size); 1167 } 1168 kfree(pbl); 1169 1170 QL_DPRINT12(ha, "exit\n"); 1171 return; 1172 } 1173 1174 #define MIN_FW_PBL_PAGE_SIZE (4*1024) 1175 #define MAX_FW_PBL_PAGE_SIZE (64*1024) 1176 1177 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 1178 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 1179 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) 1180 1181 static struct qlnxr_pbl * 1182 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, 1183 struct qlnxr_pbl_info *pbl_info, gfp_t flags) 1184 { 1185 void *va; 1186 dma_addr_t pa; 1187 dma_addr_t *pbl_main_tbl; 1188 struct qlnxr_pbl *pbl_table; 1189 int i; 1190 qlnx_host_t *ha; 1191 1192 ha = dev->ha; 1193 1194 QL_DPRINT12(ha, "enter\n"); 1195 1196 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); 1197 1198 if (!pbl_table) { 1199 QL_DPRINT12(ha, "pbl_table = NULL\n"); 1200 return NULL; 1201 } 1202 1203 for (i = 0; i < pbl_info->num_pbls; i++) { 1204 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); 1205 if (!va) { 1206 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); 1207 goto err; 1208 } 1209 memset(va, 0, pbl_info->pbl_size); 1210 pbl_table[i].va = va; 1211 pbl_table[i].pa = pa; 1212 } 1213 1214 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 1215 * the first one with physical pointers to all of the rest 1216 */ 1217 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 1218 for (i = 0; i < pbl_info->num_pbls - 1; i++) 1219 pbl_main_tbl[i] = pbl_table[i + 1].pa; 1220 1221 QL_DPRINT12(ha, "exit\n"); 1222 return pbl_table; 1223 1224 err: 1225 qlnxr_free_pbl(dev, pbl_info, pbl_table); 1226 1227 QL_DPRINT12(ha, "exit with error\n"); 1228 return NULL; 1229 } 1230 1231 static int 1232 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, 1233 struct qlnxr_pbl_info *pbl_info, 1234 u32 num_pbes, 1235 int two_layer_capable) 1236 { 1237 u32 pbl_capacity; 1238 u32 pbl_size; 1239 u32 num_pbls; 1240 qlnx_host_t *ha; 1241 1242 ha = dev->ha; 1243 1244 QL_DPRINT12(ha, "enter\n"); 1245 1246 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 1247 if (num_pbes > MAX_PBES_TWO_LAYER) { 1248 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", 1249 num_pbes); 1250 return -EINVAL; 1251 } 1252 1253 /* calculate required pbl page size */ 1254 pbl_size = MIN_FW_PBL_PAGE_SIZE; 1255 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 1256 NUM_PBES_ON_PAGE(pbl_size); 1257 1258 while (pbl_capacity < num_pbes) { 1259 pbl_size *= 2; 1260 pbl_capacity = pbl_size / sizeof(u64); 1261 pbl_capacity = pbl_capacity * pbl_capacity; 1262 } 1263 1264 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 1265 num_pbls++; /* One for the layer0 ( points to the pbls) */ 1266 pbl_info->two_layered = true; 1267 } else { 1268 /* One layered PBL */ 1269 num_pbls = 1; 1270 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ 1271 roundup_pow_of_two((num_pbes * sizeof(u64)))); 1272 pbl_info->two_layered = false; 1273 } 1274 1275 pbl_info->num_pbls = num_pbls; 1276 pbl_info->pbl_size = pbl_size; 1277 pbl_info->num_pbes = num_pbes; 1278 1279 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", 1280 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 1281 1282 return 0; 1283 } 1284 1285 static void 1286 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, 1287 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 1288 { 1289 struct regpair *pbe; 1290 struct qlnxr_pbl *pbl_tbl; 1291 struct scatterlist *sg; 1292 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 1293 qlnx_host_t *ha; 1294 int entry; 1295 1296 ha = dev->ha; 1297 1298 QL_DPRINT12(ha, "enter\n"); 1299 1300 if (!pbl_info) { 1301 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 1302 return; 1303 } 1304 1305 if (!pbl_info->num_pbes) { 1306 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 1307 return; 1308 } 1309 1310 /* If we have a two layered pbl, the first pbl points to the rest 1311 * of the pbls and the first entry lays on the second pbl in the table 1312 */ 1313 if (pbl_info->two_layered) 1314 pbl_tbl = &pbl[1]; 1315 else 1316 pbl_tbl = pbl; 1317 1318 pbe = (struct regpair *)pbl_tbl->va; 1319 if (!pbe) { 1320 QL_DPRINT12(ha, "pbe is NULL\n"); 1321 return; 1322 } 1323 1324 pbe_cnt = 0; 1325 1326 shift = ilog2(umem->page_size); 1327 1328 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 1329 pages = sg_dma_len(sg) >> shift; 1330 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 1331 /* store the page address in pbe */ 1332 pbe->lo = 1333 cpu_to_le32(sg_dma_address(sg) + 1334 (umem->page_size * pg_cnt)); 1335 pbe->hi = 1336 cpu_to_le32(upper_32_bits 1337 ((sg_dma_address(sg) + 1338 umem->page_size * pg_cnt))); 1339 1340 QL_DPRINT12(ha, 1341 "Populate pbl table:" 1342 " pbe->addr=0x%x:0x%x " 1343 " pbe_cnt = %d total_num_pbes=%d" 1344 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 1345 total_num_pbes, pbe); 1346 1347 pbe_cnt ++; 1348 total_num_pbes ++; 1349 pbe++; 1350 1351 if (total_num_pbes == pbl_info->num_pbes) 1352 return; 1353 1354 /* if the given pbl is full storing the pbes, 1355 * move to next pbl. 1356 */ 1357 if (pbe_cnt == 1358 (pbl_info->pbl_size / sizeof(u64))) { 1359 pbl_tbl++; 1360 pbe = (struct regpair *)pbl_tbl->va; 1361 pbe_cnt = 0; 1362 } 1363 } 1364 } 1365 QL_DPRINT12(ha, "exit\n"); 1366 return; 1367 } 1368 1369 static void 1370 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) 1371 { 1372 struct qlnxr_pbl *pbl, *tmp; 1373 qlnx_host_t *ha; 1374 1375 ha = dev->ha; 1376 1377 QL_DPRINT12(ha, "enter\n"); 1378 1379 if (info->pbl_table) 1380 list_add_tail(&info->pbl_table->list_entry, 1381 &info->free_pbl_list); 1382 1383 if (!list_empty(&info->inuse_pbl_list)) 1384 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 1385 1386 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 1387 list_del(&pbl->list_entry); 1388 qlnxr_free_pbl(dev, &info->pbl_info, pbl); 1389 } 1390 QL_DPRINT12(ha, "exit\n"); 1391 1392 return; 1393 } 1394 1395 static int 1396 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, 1397 size_t page_list_len, bool two_layered) 1398 { 1399 int rc; 1400 struct qlnxr_pbl *tmp; 1401 qlnx_host_t *ha; 1402 1403 ha = dev->ha; 1404 1405 QL_DPRINT12(ha, "enter\n"); 1406 1407 INIT_LIST_HEAD(&info->free_pbl_list); 1408 INIT_LIST_HEAD(&info->inuse_pbl_list); 1409 1410 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, 1411 page_list_len, two_layered); 1412 if (rc) { 1413 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); 1414 goto done; 1415 } 1416 1417 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1418 1419 if (!info->pbl_table) { 1420 rc = -ENOMEM; 1421 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); 1422 goto done; 1423 } 1424 1425 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); 1426 1427 /* in usual case we use 2 PBLs, so we add one to free 1428 * list and allocating another one 1429 */ 1430 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1431 1432 if (!tmp) { 1433 QL_DPRINT11(ha, "Extra PBL is not allocated\n"); 1434 goto done; /* it's OK if second allocation fails, so rc = 0*/ 1435 } 1436 1437 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 1438 1439 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); 1440 1441 done: 1442 if (rc) 1443 free_mr_info(dev, info); 1444 1445 QL_DPRINT12(ha, "exit [%d]\n", rc); 1446 1447 return rc; 1448 } 1449 1450 struct ib_mr * 1451 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1452 u64 usr_addr, int acc, struct ib_udata *udata) 1453 { 1454 int rc = -ENOMEM; 1455 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1456 struct qlnxr_mr *mr; 1457 struct qlnxr_pd *pd; 1458 qlnx_host_t *ha; 1459 1460 ha = dev->ha; 1461 1462 QL_DPRINT12(ha, "enter\n"); 1463 1464 pd = get_qlnxr_pd(ibpd); 1465 1466 QL_DPRINT12(ha, "qedr_register user mr pd = %d" 1467 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 1468 pd->pd_id, start, len, usr_addr, acc); 1469 1470 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 1471 QL_DPRINT11(ha, 1472 "(acc & IB_ACCESS_REMOTE_WRITE &&" 1473 " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 1474 return ERR_PTR(-EINVAL); 1475 } 1476 1477 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1478 if (!mr) { 1479 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 1480 return ERR_PTR(rc); 1481 } 1482 1483 mr->type = QLNXR_MR_USER; 1484 1485 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 1486 if (IS_ERR(mr->umem)) { 1487 rc = -EFAULT; 1488 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); 1489 goto err0; 1490 } 1491 1492 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 1493 if (rc) { 1494 QL_DPRINT11(ha, 1495 "qlnxr_init_mr_info failed [%d]\n", rc); 1496 goto err1; 1497 } 1498 1499 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 1500 &mr->info.pbl_info); 1501 1502 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1503 1504 if (rc) { 1505 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 1506 goto err1; 1507 } 1508 1509 /* index only, 18 bit long, lkey = itid << 8 | key */ 1510 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1511 mr->hw_mr.key = 0; 1512 mr->hw_mr.pd = pd->pd_id; 1513 mr->hw_mr.local_read = 1; 1514 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1515 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1516 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1517 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1518 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 1519 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 1520 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 1521 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 1522 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ 1523 1524 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 1525 mr->hw_mr.length = len; 1526 mr->hw_mr.vaddr = usr_addr; 1527 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 1528 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 1529 mr->hw_mr.dma_mr = false; 1530 1531 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1532 if (rc) { 1533 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 1534 goto err2; 1535 } 1536 1537 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1538 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1539 mr->hw_mr.remote_atomic) 1540 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1541 1542 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); 1543 1544 return (&mr->ibmr); 1545 1546 err2: 1547 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1548 err1: 1549 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1550 err0: 1551 kfree(mr); 1552 1553 QL_DPRINT12(ha, "exit [%d]\n", rc); 1554 return (ERR_PTR(rc)); 1555 } 1556 1557 int 1558 qlnxr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 1559 { 1560 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); 1561 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); 1562 int rc = 0; 1563 qlnx_host_t *ha; 1564 1565 ha = dev->ha; 1566 1567 QL_DPRINT12(ha, "enter\n"); 1568 1569 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) 1570 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1571 1572 /* it could be user registered memory. */ 1573 if (mr->umem) 1574 ib_umem_release(mr->umem); 1575 1576 kfree(mr->pages); 1577 1578 kfree(mr); 1579 1580 QL_DPRINT12(ha, "exit\n"); 1581 return rc; 1582 } 1583 1584 static int 1585 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, 1586 struct qlnxr_cq *cq, struct ib_udata *udata) 1587 { 1588 struct qlnxr_create_cq_uresp uresp; 1589 int rc; 1590 qlnx_host_t *ha; 1591 1592 ha = dev->ha; 1593 1594 QL_DPRINT12(ha, "enter\n"); 1595 1596 memset(&uresp, 0, sizeof(uresp)); 1597 1598 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1599 uresp.icid = cq->icid; 1600 1601 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1602 1603 if (rc) { 1604 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", 1605 cq->icid, rc); 1606 } 1607 1608 QL_DPRINT12(ha, "exit [%d]\n", rc); 1609 return rc; 1610 } 1611 1612 static void 1613 consume_cqe(struct qlnxr_cq *cq) 1614 { 1615 1616 if (cq->latest_cqe == cq->toggle_cqe) 1617 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1618 1619 cq->latest_cqe = ecore_chain_consume(&cq->pbl); 1620 } 1621 1622 static inline int 1623 qlnxr_align_cq_entries(int entries) 1624 { 1625 u64 size, aligned_size; 1626 1627 /* We allocate an extra entry that we don't report to the FW. 1628 * Why? 1629 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes 1630 * (for performance purposes). Allocating an extra entry and telling 1631 * the FW we have less prevents overwriting the first entry in case of 1632 * a wrap i.e. when the FW writes the last entry and the application 1633 * hasn't read the first one. 1634 */ 1635 size = (entries + 1) * QLNXR_CQE_SIZE; 1636 1637 /* We align to PAGE_SIZE. 1638 * Why? 1639 * Since the CQ is going to be mapped and the mapping is anyhow in whole 1640 * kernel pages we benefit from the possibly extra CQEs. 1641 */ 1642 aligned_size = ALIGN(size, PAGE_SIZE); 1643 1644 /* note: for CQs created in user space the result of this function 1645 * should match the size mapped in user space 1646 */ 1647 return (aligned_size / QLNXR_CQE_SIZE); 1648 } 1649 1650 static inline int 1651 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, 1652 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, 1653 int access, int dmasync, int alloc_and_init) 1654 { 1655 int page_cnt; 1656 int rc; 1657 qlnx_host_t *ha; 1658 1659 ha = dev->ha; 1660 1661 QL_DPRINT12(ha, "enter\n"); 1662 1663 q->buf_addr = buf_addr; 1664 q->buf_len = buf_len; 1665 1666 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" 1667 " dmasync : %x\n", q->buf_addr, q->buf_len, 1668 access, dmasync); 1669 1670 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 1671 1672 if (IS_ERR(q->umem)) { 1673 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); 1674 return PTR_ERR(q->umem); 1675 } 1676 1677 page_cnt = ib_umem_page_count(q->umem); 1678 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 1679 0 /* SQ and RQ don't support dual layer pbl. 1680 * CQ may, but this is yet uncoded. 1681 */); 1682 if (rc) { 1683 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); 1684 goto err; 1685 } 1686 1687 if (alloc_and_init) { 1688 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 1689 1690 if (!q->pbl_tbl) { 1691 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1692 rc = -ENOMEM; 1693 goto err; 1694 } 1695 1696 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); 1697 } else { 1698 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 1699 1700 if (!q->pbl_tbl) { 1701 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1702 rc = -ENOMEM; 1703 goto err; 1704 } 1705 } 1706 1707 QL_DPRINT12(ha, "exit\n"); 1708 return 0; 1709 1710 err: 1711 ib_umem_release(q->umem); 1712 q->umem = NULL; 1713 1714 QL_DPRINT12(ha, "exit [%d]\n", rc); 1715 return rc; 1716 } 1717 1718 int 1719 qlnxr_create_cq(struct ib_cq *ibcq, 1720 const struct ib_cq_init_attr *attr, 1721 struct ib_udata *udata) 1722 { 1723 struct qlnxr_ucontext *ctx; 1724 struct ecore_rdma_destroy_cq_out_params destroy_oparams; 1725 struct ecore_rdma_destroy_cq_in_params destroy_iparams; 1726 struct qlnxr_dev *dev; 1727 struct ecore_rdma_create_cq_in_params params; 1728 struct qlnxr_create_cq_ureq ureq; 1729 1730 int vector = attr->comp_vector; 1731 int entries = attr->cqe; 1732 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 1733 int chain_entries, rc, page_cnt; 1734 u64 pbl_ptr; 1735 u16 icid; 1736 qlnx_host_t *ha; 1737 1738 dev = get_qlnxr_dev(ibcq->device); 1739 ha = dev->ha; 1740 1741 QL_DPRINT12(ha, "called from %s. entries = %d, " 1742 "vector = %d\n", 1743 (udata ? "User Lib" : "Kernel"), entries, vector); 1744 1745 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); 1746 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); 1747 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); 1748 1749 if (entries > QLNXR_MAX_CQES) { 1750 QL_DPRINT11(ha, 1751 "the number of entries %d is too high. " 1752 "Must be equal or below %d.\n", 1753 entries, QLNXR_MAX_CQES); 1754 return -EINVAL; 1755 } 1756 chain_entries = qlnxr_align_cq_entries(entries); 1757 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); 1758 1759 if (udata) { 1760 ctx = rdma_udata_to_drv_context( 1761 udata, struct qlnxr_ucontext, ibucontext); 1762 1763 memset(&ureq, 0, sizeof(ureq)); 1764 1765 if (ib_copy_from_udata(&ureq, udata, 1766 min(sizeof(ureq), udata->inlen))) { 1767 QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); 1768 goto err0; 1769 } 1770 1771 if (!ureq.len) { 1772 QL_DPRINT11(ha, "ureq.len == 0\n"); 1773 goto err0; 1774 } 1775 1776 cq->cq_type = QLNXR_CQ_TYPE_USER; 1777 1778 qlnxr_init_user_queue(&ctx->ibucontext, dev, &cq->q, ureq.addr, ureq.len, 1779 IB_ACCESS_LOCAL_WRITE, 1, 1); 1780 1781 pbl_ptr = cq->q.pbl_tbl->pa; 1782 page_cnt = cq->q.pbl_info.num_pbes; 1783 cq->ibcq.cqe = chain_entries; 1784 } else { 1785 ctx = NULL; 1786 1787 cq->cq_type = QLNXR_CQ_TYPE_KERNEL; 1788 1789 rc = ecore_chain_alloc(&dev->ha->cdev, 1790 ECORE_CHAIN_USE_TO_CONSUME, 1791 ECORE_CHAIN_MODE_PBL, 1792 ECORE_CHAIN_CNT_TYPE_U32, 1793 chain_entries, 1794 sizeof(union roce_cqe), 1795 &cq->pbl, NULL); 1796 1797 if (rc) 1798 goto err1; 1799 1800 page_cnt = ecore_chain_get_page_cnt(&cq->pbl); 1801 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); 1802 cq->ibcq.cqe = cq->pbl.capacity; 1803 } 1804 1805 params.cq_handle_hi = upper_32_bits((uintptr_t)cq); 1806 params.cq_handle_lo = lower_32_bits((uintptr_t)cq); 1807 params.cnq_id = vector; 1808 params.cq_size = chain_entries - 1; 1809 params.pbl_num_pages = page_cnt; 1810 params.pbl_ptr = pbl_ptr; 1811 params.pbl_two_level = 0; 1812 1813 if (udata) { 1814 params.dpi = ctx->dpi; 1815 } else { 1816 params.dpi = dev->dpi; 1817 } 1818 1819 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 1820 if (rc) 1821 goto err2; 1822 1823 cq->icid = icid; 1824 cq->sig = QLNXR_CQ_MAGIC_NUMBER; 1825 spin_lock_init(&cq->cq_lock); 1826 1827 if (udata) { 1828 rc = qlnxr_copy_cq_uresp(dev, cq, udata); 1829 if (rc) 1830 goto err3; 1831 } else { 1832 /* Generate doorbell address. 1833 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. 1834 * TODO: consider moving to device scope as it is a function of 1835 * the device. 1836 * TODO: add ifdef if plan to support 16 bit. 1837 */ 1838 cq->db_addr = dev->db_addr + 1839 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1840 cq->db.data.icid = cq->icid; 1841 cq->db.data.params = DB_AGG_CMD_SET << 1842 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1843 1844 /* point to the very last element, passing it we will toggle */ 1845 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); 1846 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1847 1848 /* must be different from pbl_toggle */ 1849 cq->latest_cqe = NULL; 1850 consume_cqe(cq); 1851 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 1852 } 1853 1854 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," 1855 " number of entries = 0x%x\n", 1856 cq->icid, cq, params.cq_size); 1857 QL_DPRINT12(ha,"cq_addr = %p\n", cq); 1858 return (0); 1859 1860 err3: 1861 destroy_iparams.icid = cq->icid; 1862 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); 1863 err2: 1864 if (udata) 1865 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1866 else 1867 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 1868 err1: 1869 if (udata) 1870 ib_umem_release(cq->q.umem); 1871 err0: 1872 QL_DPRINT12(ha, "exit error\n"); 1873 1874 return (-EINVAL); 1875 } 1876 1877 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1878 { 1879 int status = 0; 1880 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 1881 qlnx_host_t *ha; 1882 1883 ha = dev->ha; 1884 1885 QL_DPRINT12(ha, "enter/exit\n"); 1886 1887 return status; 1888 } 1889 1890 void 1891 qlnxr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1892 { 1893 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 1894 struct ecore_rdma_destroy_cq_out_params oparams; 1895 struct ecore_rdma_destroy_cq_in_params iparams; 1896 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 1897 int rc = 0; 1898 qlnx_host_t *ha; 1899 1900 ha = dev->ha; 1901 1902 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); 1903 1904 cq->destroyed = 1; 1905 1906 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation 1907 * that all completions with notification are dealt with. The rest 1908 * of the completions are not interesting 1909 */ 1910 1911 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1912 1913 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { 1914 iparams.icid = cq->icid; 1915 1916 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1917 1918 if (rc) { 1919 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", 1920 cq->icid); 1921 return; 1922 } 1923 1924 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); 1925 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 1926 } 1927 1928 if (udata) { 1929 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1930 ib_umem_release(cq->q.umem); 1931 } 1932 1933 cq->sig = ~cq->sig; 1934 1935 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); 1936 } 1937 1938 static int 1939 qlnxr_check_qp_attrs(struct ib_pd *ibpd, 1940 struct qlnxr_dev *dev, 1941 struct ib_qp_init_attr *attrs, 1942 struct ib_udata *udata) 1943 { 1944 struct ecore_rdma_device *qattr; 1945 qlnx_host_t *ha; 1946 1947 qattr = ecore_rdma_query_device(dev->rdma_ctx); 1948 ha = dev->ha; 1949 1950 QL_DPRINT12(ha, "enter\n"); 1951 1952 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); 1953 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); 1954 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); 1955 1956 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); 1957 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); 1958 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); 1959 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); 1960 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); 1961 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", 1962 attrs->cap.max_inline_data); 1963 1964 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); 1965 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); 1966 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); 1967 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); 1968 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); 1969 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", 1970 (void *)qattr->sys_image_guid); 1971 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); 1972 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); 1973 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); 1974 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); 1975 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); 1976 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); 1977 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", 1978 qattr->max_qp_resp_rd_atomic_resc); 1979 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", 1980 qattr->max_qp_req_rd_atomic_resc); 1981 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", 1982 qattr->max_dev_resp_rd_atomic_resc); 1983 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); 1984 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); 1985 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); 1986 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); 1987 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); 1988 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); 1989 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); 1990 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); 1991 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", 1992 qattr->max_mr_mw_fmr_pbl); 1993 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", 1994 (void *)qattr->max_mr_mw_fmr_size); 1995 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); 1996 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); 1997 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); 1998 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); 1999 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", 2000 qattr->max_stats_queues); 2001 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); 2002 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", 2003 (void *)qattr->page_size_caps); 2004 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); 2005 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); 2006 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", 2007 qattr->bad_pkey_counter); 2008 2009 if ((attrs->qp_type == IB_QPT_GSI) && udata) { 2010 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); 2011 return -EINVAL; 2012 } 2013 2014 if (udata && !(ibpd->uobject && ibpd->uobject->context)) { 2015 QL_DPRINT12(ha, "called from user without context\n"); 2016 return -EINVAL; 2017 } 2018 2019 /* QP0... attrs->qp_type == IB_QPT_GSI */ 2020 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 2021 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 2022 attrs->qp_type); 2023 return -EINVAL; 2024 } 2025 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { 2026 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); 2027 return -EINVAL; 2028 } 2029 /* Skip the check for QP1 to support CM size of 128 */ 2030 if (attrs->cap.max_send_wr > qattr->max_wqe) { 2031 QL_DPRINT12(ha, "cannot create a SQ with %d elements " 2032 " (max_send_wr=0x%x)\n", 2033 attrs->cap.max_send_wr, qattr->max_wqe); 2034 return -EINVAL; 2035 } 2036 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { 2037 QL_DPRINT12(ha, "cannot create a RQ with %d elements" 2038 " (max_recv_wr=0x%x)\n", 2039 attrs->cap.max_recv_wr, qattr->max_wqe); 2040 return -EINVAL; 2041 } 2042 if (attrs->cap.max_inline_data > qattr->max_inline) { 2043 QL_DPRINT12(ha, 2044 "unsupported inline data size=0x%x " 2045 "requested (max_inline=0x%x)\n", 2046 attrs->cap.max_inline_data, qattr->max_inline); 2047 return -EINVAL; 2048 } 2049 if (attrs->cap.max_send_sge > qattr->max_sge) { 2050 QL_DPRINT12(ha, 2051 "unsupported send_sge=0x%x " 2052 "requested (max_send_sge=0x%x)\n", 2053 attrs->cap.max_send_sge, qattr->max_sge); 2054 return -EINVAL; 2055 } 2056 if (attrs->cap.max_recv_sge > qattr->max_sge) { 2057 QL_DPRINT12(ha, 2058 "unsupported recv_sge=0x%x requested " 2059 " (max_recv_sge=0x%x)\n", 2060 attrs->cap.max_recv_sge, qattr->max_sge); 2061 return -EINVAL; 2062 } 2063 /* unprivileged user space cannot create special QP */ 2064 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 2065 QL_DPRINT12(ha, 2066 "userspace can't create special QPs of type=0x%x\n", 2067 attrs->qp_type); 2068 return -EINVAL; 2069 } 2070 /* allow creating only one GSI type of QP */ 2071 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 2072 QL_DPRINT12(ha, 2073 "create qp: GSI special QPs already created.\n"); 2074 return -EINVAL; 2075 } 2076 2077 /* verify consumer QPs are not trying to use GSI QP's CQ */ 2078 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 2079 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); 2080 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); 2081 2082 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || 2083 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { 2084 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); 2085 return -EINVAL; 2086 } 2087 } 2088 QL_DPRINT12(ha, "exit\n"); 2089 return 0; 2090 } 2091 2092 static int 2093 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 2094 struct qlnxr_srq *srq, 2095 struct ib_udata *udata) 2096 { 2097 struct qlnxr_create_srq_uresp uresp; 2098 qlnx_host_t *ha; 2099 int rc; 2100 2101 ha = dev->ha; 2102 2103 QL_DPRINT12(ha, "enter\n"); 2104 2105 memset(&uresp, 0, sizeof(uresp)); 2106 2107 uresp.srq_id = srq->srq_id; 2108 2109 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2110 2111 QL_DPRINT12(ha, "exit [%d]\n", rc); 2112 return rc; 2113 } 2114 2115 static void 2116 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, 2117 struct qlnxr_create_qp_uresp *uresp, 2118 struct qlnxr_qp *qp) 2119 { 2120 qlnx_host_t *ha; 2121 2122 ha = dev->ha; 2123 2124 /* Return if QP is associated with SRQ instead of RQ */ 2125 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); 2126 2127 if (qp->srq) 2128 return; 2129 2130 /* iWARP requires two doorbells per RQ. */ 2131 if (QLNX_IS_IWARP(dev)) { 2132 uresp->rq_db_offset = 2133 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2134 uresp->rq_db2_offset = 2135 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2136 2137 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " 2138 "uresp->rq_db2_offset = 0x%x\n", 2139 uresp->rq_db_offset, uresp->rq_db2_offset); 2140 } else { 2141 uresp->rq_db_offset = 2142 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2143 } 2144 uresp->rq_icid = qp->icid; 2145 2146 QL_DPRINT12(ha, "exit\n"); 2147 return; 2148 } 2149 2150 static void 2151 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, 2152 struct qlnxr_create_qp_uresp *uresp, 2153 struct qlnxr_qp *qp) 2154 { 2155 qlnx_host_t *ha; 2156 2157 ha = dev->ha; 2158 2159 QL_DPRINT12(ha, "enter\n"); 2160 2161 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2162 2163 /* iWARP uses the same cid for rq and sq*/ 2164 if (QLNX_IS_IWARP(dev)) { 2165 uresp->sq_icid = qp->icid; 2166 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); 2167 } else 2168 uresp->sq_icid = qp->icid + 1; 2169 2170 QL_DPRINT12(ha, "exit\n"); 2171 return; 2172 } 2173 2174 static int 2175 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, 2176 struct qlnxr_qp *qp, 2177 struct ib_udata *udata) 2178 { 2179 int rc; 2180 struct qlnxr_create_qp_uresp uresp; 2181 qlnx_host_t *ha; 2182 2183 ha = dev->ha; 2184 2185 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); 2186 2187 memset(&uresp, 0, sizeof(uresp)); 2188 qlnxr_copy_sq_uresp(dev, &uresp, qp); 2189 qlnxr_copy_rq_uresp(dev, &uresp, qp); 2190 2191 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 2192 uresp.qp_id = qp->qp_id; 2193 2194 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2195 2196 QL_DPRINT12(ha, "exit [%d]\n", rc); 2197 return rc; 2198 } 2199 2200 static void 2201 qlnxr_set_common_qp_params(struct qlnxr_dev *dev, 2202 struct qlnxr_qp *qp, 2203 struct qlnxr_pd *pd, 2204 struct ib_qp_init_attr *attrs) 2205 { 2206 qlnx_host_t *ha; 2207 2208 ha = dev->ha; 2209 2210 QL_DPRINT12(ha, "enter\n"); 2211 2212 spin_lock_init(&qp->q_lock); 2213 2214 atomic_set(&qp->refcnt, 1); 2215 qp->pd = pd; 2216 qp->sig = QLNXR_QP_MAGIC_NUMBER; 2217 qp->qp_type = attrs->qp_type; 2218 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2219 qp->sq.max_sges = attrs->cap.max_send_sge; 2220 qp->state = ECORE_ROCE_QP_STATE_RESET; 2221 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 2222 qp->sq_cq = get_qlnxr_cq(attrs->send_cq); 2223 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); 2224 qp->dev = dev; 2225 2226 if (!attrs->srq) { 2227 /* QP is associated with RQ instead of SRQ */ 2228 qp->rq.max_sges = attrs->cap.max_recv_sge; 2229 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 2230 qp->rq.max_sges, qp->rq_cq->icid); 2231 } else { 2232 qp->srq = get_qlnxr_srq(attrs->srq); 2233 } 2234 2235 QL_DPRINT12(ha, 2236 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," 2237 " state = %d, signaled = %d, use_srq=%d\n", 2238 pd->pd_id, qp->qp_type, qp->max_inline_data, 2239 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); 2240 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 2241 qp->sq.max_sges, qp->sq_cq->icid); 2242 return; 2243 } 2244 2245 static int 2246 qlnxr_check_srq_params(struct qlnxr_dev *dev, 2247 struct ib_srq_init_attr *attrs) 2248 { 2249 struct ecore_rdma_device *qattr; 2250 qlnx_host_t *ha; 2251 2252 ha = dev->ha; 2253 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2254 2255 QL_DPRINT12(ha, "enter\n"); 2256 2257 if (attrs->attr.max_wr > qattr->max_srq_wqe) { 2258 QL_DPRINT12(ha, "unsupported srq_wr=0x%x" 2259 " requested (max_srq_wr=0x%x)\n", 2260 attrs->attr.max_wr, qattr->max_srq_wr); 2261 return -EINVAL; 2262 } 2263 2264 if (attrs->attr.max_sge > qattr->max_sge) { 2265 QL_DPRINT12(ha, 2266 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 2267 attrs->attr.max_sge, qattr->max_sge); 2268 return -EINVAL; 2269 } 2270 2271 if (attrs->attr.srq_limit > attrs->attr.max_wr) { 2272 QL_DPRINT12(ha, 2273 "unsupported srq_limit=0x%x requested" 2274 " (max_srq_limit=0x%x)\n", 2275 attrs->attr.srq_limit, attrs->attr.srq_limit); 2276 return -EINVAL; 2277 } 2278 2279 QL_DPRINT12(ha, "exit\n"); 2280 return 0; 2281 } 2282 2283 static void 2284 qlnxr_free_srq_user_params(struct qlnxr_srq *srq) 2285 { 2286 struct qlnxr_dev *dev = srq->dev; 2287 qlnx_host_t *ha; 2288 2289 ha = dev->ha; 2290 2291 QL_DPRINT12(ha, "enter\n"); 2292 2293 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2294 ib_umem_release(srq->usrq.umem); 2295 ib_umem_release(srq->prod_umem); 2296 2297 QL_DPRINT12(ha, "exit\n"); 2298 return; 2299 } 2300 2301 static void 2302 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) 2303 { 2304 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2305 struct qlnxr_dev *dev = srq->dev; 2306 qlnx_host_t *ha; 2307 2308 ha = dev->ha; 2309 2310 QL_DPRINT12(ha, "enter\n"); 2311 2312 ecore_chain_free(dev->cdev, &hw_srq->pbl); 2313 2314 qlnx_dma_free_coherent(&dev->cdev, 2315 hw_srq->virt_prod_pair_addr, 2316 hw_srq->phy_prod_pair_addr, 2317 sizeof(struct rdma_srq_producers)); 2318 2319 QL_DPRINT12(ha, "exit\n"); 2320 2321 return; 2322 } 2323 2324 static int 2325 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 2326 struct qlnxr_srq *srq, 2327 struct qlnxr_create_srq_ureq *ureq, 2328 int access, int dmasync) 2329 { 2330 struct scatterlist *sg; 2331 int rc; 2332 struct qlnxr_dev *dev = srq->dev; 2333 qlnx_host_t *ha; 2334 2335 ha = dev->ha; 2336 2337 QL_DPRINT12(ha, "enter\n"); 2338 2339 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 2340 ureq->srq_len, access, dmasync, 1); 2341 if (rc) 2342 return rc; 2343 2344 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 2345 sizeof(struct rdma_srq_producers), 2346 access, dmasync); 2347 if (IS_ERR(srq->prod_umem)) { 2348 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2349 ib_umem_release(srq->usrq.umem); 2350 2351 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", 2352 PTR_ERR(srq->prod_umem)); 2353 2354 return PTR_ERR(srq->prod_umem); 2355 } 2356 2357 sg = srq->prod_umem->sg_head.sgl; 2358 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 2359 2360 QL_DPRINT12(ha, "exit\n"); 2361 return 0; 2362 } 2363 2364 static int 2365 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 2366 struct qlnxr_dev *dev, 2367 struct ib_srq_init_attr *init_attr) 2368 { 2369 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2370 dma_addr_t phy_prod_pair_addr; 2371 u32 num_elems, max_wr; 2372 void *va; 2373 int rc; 2374 qlnx_host_t *ha; 2375 2376 ha = dev->ha; 2377 2378 QL_DPRINT12(ha, "enter\n"); 2379 2380 va = qlnx_dma_alloc_coherent(&dev->cdev, 2381 &phy_prod_pair_addr, 2382 sizeof(struct rdma_srq_producers)); 2383 if (!va) { 2384 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); 2385 return -ENOMEM; 2386 } 2387 2388 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 2389 hw_srq->virt_prod_pair_addr = va; 2390 2391 max_wr = init_attr->attr.max_wr; 2392 2393 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; 2394 2395 rc = ecore_chain_alloc(dev->cdev, 2396 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2397 ECORE_CHAIN_MODE_PBL, 2398 ECORE_CHAIN_CNT_TYPE_U32, 2399 num_elems, 2400 ECORE_RDMA_SRQ_WQE_ELEM_SIZE, 2401 &hw_srq->pbl, NULL); 2402 2403 if (rc) { 2404 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); 2405 goto err0; 2406 } 2407 2408 hw_srq->max_wr = max_wr; 2409 hw_srq->num_elems = num_elems; 2410 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; 2411 2412 QL_DPRINT12(ha, "exit\n"); 2413 return 0; 2414 2415 err0: 2416 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, 2417 sizeof(struct rdma_srq_producers)); 2418 2419 QL_DPRINT12(ha, "exit [%d]\n", rc); 2420 return rc; 2421 } 2422 2423 static inline void 2424 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, 2425 struct qlnxr_pd *pd, 2426 struct qlnxr_qp *qp, 2427 struct ib_qp_init_attr *attrs, 2428 bool fmr_and_reserved_lkey, 2429 struct ecore_rdma_create_qp_in_params *params) 2430 { 2431 qlnx_host_t *ha; 2432 2433 ha = dev->ha; 2434 2435 QL_DPRINT12(ha, "enter\n"); 2436 2437 /* QP handle to be written in an async event */ 2438 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 2439 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 2440 2441 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 2442 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 2443 params->pd = pd->pd_id; 2444 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 2445 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 2446 params->stats_queue = 0; 2447 2448 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 2449 2450 if (qp->srq) { 2451 /* QP is associated with SRQ instead of RQ */ 2452 params->srq_id = qp->srq->srq_id; 2453 params->use_srq = true; 2454 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 2455 params->srq_id, params->use_srq); 2456 return; 2457 } 2458 2459 params->srq_id = 0; 2460 params->use_srq = false; 2461 2462 QL_DPRINT12(ha, "exit\n"); 2463 return; 2464 } 2465 2466 static inline void 2467 qlnxr_qp_user_print( struct qlnxr_dev *dev, 2468 struct qlnxr_qp *qp) 2469 { 2470 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " 2471 "rq_addr=0x%llx, rq_len=%zd\n", 2472 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, 2473 qp->urq.buf_len); 2474 return; 2475 } 2476 2477 static int 2478 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) 2479 { 2480 u32 newid; 2481 int rc; 2482 qlnx_host_t *ha; 2483 2484 ha = dev->ha; 2485 2486 QL_DPRINT12(ha, "enter\n"); 2487 2488 if (!QLNX_IS_IWARP(dev)) 2489 return 0; 2490 2491 do { 2492 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { 2493 QL_DPRINT11(ha, "idr_pre_get failed\n"); 2494 return -ENOMEM; 2495 } 2496 2497 mtx_lock(&dev->idr_lock); 2498 2499 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); 2500 2501 mtx_unlock(&dev->idr_lock); 2502 2503 } while (rc == -EAGAIN); 2504 2505 QL_DPRINT12(ha, "exit [%d]\n", rc); 2506 2507 return rc; 2508 } 2509 2510 static void 2511 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) 2512 { 2513 qlnx_host_t *ha; 2514 2515 ha = dev->ha; 2516 2517 QL_DPRINT12(ha, "enter\n"); 2518 2519 if (!QLNX_IS_IWARP(dev)) 2520 return; 2521 2522 mtx_lock(&dev->idr_lock); 2523 idr_remove(&dev->qpidr, id); 2524 mtx_unlock(&dev->idr_lock); 2525 2526 QL_DPRINT12(ha, "exit \n"); 2527 2528 return; 2529 } 2530 2531 static inline void 2532 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, 2533 struct qlnxr_qp *qp, 2534 struct ecore_rdma_create_qp_out_params *out_params) 2535 { 2536 qlnx_host_t *ha; 2537 2538 ha = dev->ha; 2539 2540 QL_DPRINT12(ha, "enter\n"); 2541 2542 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 2543 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 2544 2545 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 2546 &qp->usq.pbl_info); 2547 2548 if (qp->srq) { 2549 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); 2550 return; 2551 } 2552 2553 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 2554 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 2555 2556 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 2557 &qp->urq.pbl_info); 2558 2559 QL_DPRINT12(ha, "exit\n"); 2560 return; 2561 } 2562 2563 static int 2564 qlnxr_create_user_qp(struct qlnxr_dev *dev, 2565 struct qlnxr_qp *qp, 2566 struct ib_pd *ibpd, 2567 struct ib_udata *udata, 2568 struct ib_qp_init_attr *attrs) 2569 { 2570 struct ecore_rdma_destroy_qp_out_params d_out_params; 2571 struct ecore_rdma_create_qp_in_params in_params; 2572 struct ecore_rdma_create_qp_out_params out_params; 2573 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2574 struct ib_ucontext *ib_ctx = NULL; 2575 struct qlnxr_create_qp_ureq ureq; 2576 int alloc_and_init = QLNX_IS_ROCE(dev); 2577 int rc = -EINVAL; 2578 qlnx_host_t *ha; 2579 2580 ha = dev->ha; 2581 2582 QL_DPRINT12(ha, "enter\n"); 2583 2584 ib_ctx = ibpd->uobject->context; 2585 2586 memset(&ureq, 0, sizeof(ureq)); 2587 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 2588 2589 if (rc) { 2590 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); 2591 return rc; 2592 } 2593 2594 /* SQ - read access only (0), dma sync not required (0) */ 2595 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 2596 ureq.sq_len, 0, 0, 2597 alloc_and_init); 2598 if (rc) { 2599 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2600 return rc; 2601 } 2602 2603 if (!qp->srq) { 2604 /* RQ - read access only (0), dma sync not required (0) */ 2605 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 2606 ureq.rq_len, 0, 0, 2607 alloc_and_init); 2608 2609 if (rc) { 2610 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2611 return rc; 2612 } 2613 } 2614 2615 memset(&in_params, 0, sizeof(in_params)); 2616 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 2617 in_params.qp_handle_lo = ureq.qp_handle_lo; 2618 in_params.qp_handle_hi = ureq.qp_handle_hi; 2619 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 2620 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 2621 2622 if (!qp->srq) { 2623 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 2624 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 2625 } 2626 2627 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); 2628 2629 if (!qp->ecore_qp) { 2630 rc = -ENOMEM; 2631 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2632 goto err1; 2633 } 2634 2635 if (QLNX_IS_IWARP(dev)) 2636 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); 2637 2638 qp->qp_id = out_params.qp_id; 2639 qp->icid = out_params.icid; 2640 2641 rc = qlnxr_copy_qp_uresp(dev, qp, udata); 2642 2643 if (rc) { 2644 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); 2645 goto err; 2646 } 2647 2648 qlnxr_qp_user_print(dev, qp); 2649 2650 QL_DPRINT12(ha, "exit\n"); 2651 return 0; 2652 err: 2653 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2654 2655 if (rc) 2656 QL_DPRINT12(ha, "fatal fault\n"); 2657 2658 err1: 2659 qlnxr_cleanup_user(dev, qp); 2660 2661 QL_DPRINT12(ha, "exit[%d]\n", rc); 2662 return rc; 2663 } 2664 2665 static void 2666 qlnxr_set_roce_db_info(struct qlnxr_dev *dev, 2667 struct qlnxr_qp *qp) 2668 { 2669 qlnx_host_t *ha; 2670 2671 ha = dev->ha; 2672 2673 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2674 2675 qp->sq.db = dev->db_addr + 2676 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2677 qp->sq.db_data.data.icid = qp->icid + 1; 2678 2679 if (!qp->srq) { 2680 qp->rq.db = dev->db_addr + 2681 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2682 qp->rq.db_data.data.icid = qp->icid; 2683 } 2684 2685 QL_DPRINT12(ha, "exit\n"); 2686 return; 2687 } 2688 2689 static void 2690 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, 2691 struct qlnxr_qp *qp) 2692 2693 { 2694 qlnx_host_t *ha; 2695 2696 ha = dev->ha; 2697 2698 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2699 2700 qp->sq.db = dev->db_addr + 2701 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2702 qp->sq.db_data.data.icid = qp->icid; 2703 2704 if (!qp->srq) { 2705 qp->rq.db = dev->db_addr + 2706 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2707 qp->rq.db_data.data.icid = qp->icid; 2708 2709 qp->rq.iwarp_db2 = dev->db_addr + 2710 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2711 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2712 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2713 } 2714 2715 QL_DPRINT12(ha, 2716 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" 2717 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" 2718 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" 2719 " qp->rq.iwarp_db2.data.prod_val =0x%x\n", 2720 qp->sq.db, qp->sq.db_data.data.icid, 2721 qp->rq.db, qp->rq.db_data.data.icid, 2722 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, 2723 qp->rq.iwarp_db2_data.data.value); 2724 2725 QL_DPRINT12(ha, "exit\n"); 2726 return; 2727 } 2728 2729 static int 2730 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, 2731 struct qlnxr_qp *qp, 2732 struct ecore_rdma_create_qp_in_params *in_params, 2733 u32 n_sq_elems, 2734 u32 n_rq_elems) 2735 { 2736 struct ecore_rdma_create_qp_out_params out_params; 2737 int rc; 2738 qlnx_host_t *ha; 2739 2740 ha = dev->ha; 2741 2742 QL_DPRINT12(ha, "enter\n"); 2743 2744 rc = ecore_chain_alloc( 2745 dev->cdev, 2746 ECORE_CHAIN_USE_TO_PRODUCE, 2747 ECORE_CHAIN_MODE_PBL, 2748 ECORE_CHAIN_CNT_TYPE_U32, 2749 n_sq_elems, 2750 QLNXR_SQE_ELEMENT_SIZE, 2751 &qp->sq.pbl, 2752 NULL); 2753 2754 if (rc) { 2755 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); 2756 return rc; 2757 } 2758 2759 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); 2760 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); 2761 2762 if (!qp->srq) { 2763 rc = ecore_chain_alloc( 2764 dev->cdev, 2765 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2766 ECORE_CHAIN_MODE_PBL, 2767 ECORE_CHAIN_CNT_TYPE_U32, 2768 n_rq_elems, 2769 QLNXR_RQE_ELEMENT_SIZE, 2770 &qp->rq.pbl, 2771 NULL); 2772 2773 if (rc) { 2774 QL_DPRINT11(ha, 2775 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); 2776 return rc; 2777 } 2778 2779 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); 2780 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); 2781 } 2782 2783 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2784 2785 if (!qp->ecore_qp) { 2786 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); 2787 return -EINVAL; 2788 } 2789 2790 qp->qp_id = out_params.qp_id; 2791 qp->icid = out_params.icid; 2792 2793 qlnxr_set_roce_db_info(dev, qp); 2794 2795 QL_DPRINT12(ha, "exit\n"); 2796 return 0; 2797 } 2798 2799 static int 2800 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, 2801 struct qlnxr_qp *qp, 2802 struct ecore_rdma_create_qp_in_params *in_params, 2803 u32 n_sq_elems, 2804 u32 n_rq_elems) 2805 { 2806 struct ecore_rdma_destroy_qp_out_params d_out_params; 2807 struct ecore_rdma_create_qp_out_params out_params; 2808 struct ecore_chain_ext_pbl ext_pbl; 2809 int rc; 2810 qlnx_host_t *ha; 2811 2812 ha = dev->ha; 2813 2814 QL_DPRINT12(ha, "enter\n"); 2815 2816 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, 2817 QLNXR_SQE_ELEMENT_SIZE, 2818 ECORE_CHAIN_MODE_PBL); 2819 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, 2820 QLNXR_RQE_ELEMENT_SIZE, 2821 ECORE_CHAIN_MODE_PBL); 2822 2823 QL_DPRINT12(ha, "n_sq_elems = 0x%x" 2824 " n_rq_elems = 0x%x in_params\n" 2825 "\t\t\tqp_handle_lo\t\t= 0x%08x\n" 2826 "\t\t\tqp_handle_hi\t\t= 0x%08x\n" 2827 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" 2828 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" 2829 "\t\t\tuse_srq\t\t\t= 0x%x\n" 2830 "\t\t\tsignal_all\t\t= 0x%x\n" 2831 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" 2832 "\t\t\tpd\t\t\t= 0x%x\n" 2833 "\t\t\tdpi\t\t\t= 0x%x\n" 2834 "\t\t\tsq_cq_id\t\t\t= 0x%x\n" 2835 "\t\t\tsq_num_pages\t\t= 0x%x\n" 2836 "\t\t\tsq_pbl_ptr\t\t= %p\n" 2837 "\t\t\tmax_sq_sges\t\t= 0x%x\n" 2838 "\t\t\trq_cq_id\t\t\t= 0x%x\n" 2839 "\t\t\trq_num_pages\t\t= 0x%x\n" 2840 "\t\t\trq_pbl_ptr\t\t= %p\n" 2841 "\t\t\tsrq_id\t\t\t= 0x%x\n" 2842 "\t\t\tstats_queue\t\t= 0x%x\n", 2843 n_sq_elems, n_rq_elems, 2844 in_params->qp_handle_lo, 2845 in_params->qp_handle_hi, 2846 in_params->qp_handle_async_lo, 2847 in_params->qp_handle_async_hi, 2848 in_params->use_srq, 2849 in_params->signal_all, 2850 in_params->fmr_and_reserved_lkey, 2851 in_params->pd, 2852 in_params->dpi, 2853 in_params->sq_cq_id, 2854 in_params->sq_num_pages, 2855 (void *)in_params->sq_pbl_ptr, 2856 in_params->max_sq_sges, 2857 in_params->rq_cq_id, 2858 in_params->rq_num_pages, 2859 (void *)in_params->rq_pbl_ptr, 2860 in_params->srq_id, 2861 in_params->stats_queue ); 2862 2863 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); 2864 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); 2865 2866 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2867 2868 if (!qp->ecore_qp) { 2869 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2870 return -EINVAL; 2871 } 2872 2873 /* Now we allocate the chain */ 2874 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 2875 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 2876 2877 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 2878 "ext_pbl.p_pbl_phys = %p\n", 2879 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 2880 2881 rc = ecore_chain_alloc( 2882 dev->cdev, 2883 ECORE_CHAIN_USE_TO_PRODUCE, 2884 ECORE_CHAIN_MODE_PBL, 2885 ECORE_CHAIN_CNT_TYPE_U32, 2886 n_sq_elems, 2887 QLNXR_SQE_ELEMENT_SIZE, 2888 &qp->sq.pbl, 2889 &ext_pbl); 2890 2891 if (rc) { 2892 QL_DPRINT11(ha, 2893 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); 2894 goto err; 2895 } 2896 2897 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 2898 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 2899 2900 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 2901 "ext_pbl.p_pbl_phys = %p\n", 2902 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 2903 2904 if (!qp->srq) { 2905 rc = ecore_chain_alloc( 2906 dev->cdev, 2907 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2908 ECORE_CHAIN_MODE_PBL, 2909 ECORE_CHAIN_CNT_TYPE_U32, 2910 n_rq_elems, 2911 QLNXR_RQE_ELEMENT_SIZE, 2912 &qp->rq.pbl, 2913 &ext_pbl); 2914 2915 if (rc) { 2916 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" 2917 " failed rc = %d\n", rc); 2918 goto err; 2919 } 2920 } 2921 2922 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", 2923 out_params.qp_id, out_params.icid); 2924 2925 qp->qp_id = out_params.qp_id; 2926 qp->icid = out_params.icid; 2927 2928 qlnxr_set_iwarp_db_info(dev, qp); 2929 2930 QL_DPRINT12(ha, "exit\n"); 2931 return 0; 2932 2933 err: 2934 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2935 2936 QL_DPRINT12(ha, "exit rc = %d\n", rc); 2937 return rc; 2938 } 2939 2940 static int 2941 qlnxr_create_kernel_qp(struct qlnxr_dev *dev, 2942 struct qlnxr_qp *qp, 2943 struct ib_pd *ibpd, 2944 struct ib_qp_init_attr *attrs) 2945 { 2946 struct ecore_rdma_create_qp_in_params in_params; 2947 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2948 int rc = -EINVAL; 2949 u32 n_rq_elems; 2950 u32 n_sq_elems; 2951 u32 n_sq_entries; 2952 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 2953 qlnx_host_t *ha; 2954 2955 ha = dev->ha; 2956 2957 QL_DPRINT12(ha, "enter\n"); 2958 2959 memset(&in_params, 0, sizeof(in_params)); 2960 2961 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in 2962 * the ring. The ring should allow at least a single WR, even if the 2963 * user requested none, due to allocation issues. 2964 * We should add an extra WR since the prod and cons indices of 2965 * wqe_wr_id are managed in such a way that the WQ is considered full 2966 * when (prod+1)%max_wr==cons. We currently don't do that because we 2967 * double the number of entries due an iSER issue that pushes far more 2968 * WRs than indicated. If we decline its ib_post_send() then we get 2969 * error prints in the dmesg we'd like to avoid. 2970 */ 2971 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2972 qattr->max_wqe); 2973 2974 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 2975 GFP_KERNEL); 2976 if (!qp->wqe_wr_id) { 2977 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); 2978 return -ENOMEM; 2979 } 2980 2981 /* QP handle to be written in CQE */ 2982 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); 2983 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); 2984 2985 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in 2986 * the ring. There ring should allow at least a single WR, even if the 2987 * user requested none, due to allocation issues. 2988 */ 2989 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); 2990 2991 /* Allocate driver internal RQ array */ 2992 if (!qp->srq) { 2993 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 2994 GFP_KERNEL); 2995 if (!qp->rqe_wr_id) { 2996 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); 2997 kfree(qp->wqe_wr_id); 2998 return -ENOMEM; 2999 } 3000 } 3001 3002 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 3003 3004 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 3005 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 3006 3007 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 3008 in_params.fmr_and_reserved_lkey = true; 3009 in_params.pd = pd->pd_id; 3010 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 3011 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 3012 in_params.stats_queue = 0; 3013 3014 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 3015 3016 if (qp->srq) { 3017 /* QP is associated with SRQ instead of RQ */ 3018 in_params.srq_id = qp->srq->srq_id; 3019 in_params.use_srq = true; 3020 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 3021 in_params.srq_id, in_params.use_srq); 3022 } else { 3023 in_params.srq_id = 0; 3024 in_params.use_srq = false; 3025 } 3026 3027 n_sq_entries = attrs->cap.max_send_wr; 3028 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); 3029 n_sq_entries = max_t(u32, n_sq_entries, 1); 3030 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 3031 3032 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 3033 3034 if (QLNX_IS_ROCE(dev)) { 3035 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, 3036 n_sq_elems, n_rq_elems); 3037 } else { 3038 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, 3039 n_sq_elems, n_rq_elems); 3040 } 3041 3042 if (rc) 3043 qlnxr_cleanup_kernel(dev, qp); 3044 3045 QL_DPRINT12(ha, "exit [%d]\n", rc); 3046 return rc; 3047 } 3048 3049 struct ib_qp * 3050 qlnxr_create_qp(struct ib_pd *ibpd, 3051 struct ib_qp_init_attr *attrs, 3052 struct ib_udata *udata) 3053 { 3054 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); 3055 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3056 struct qlnxr_qp *qp; 3057 int rc = 0; 3058 qlnx_host_t *ha; 3059 3060 ha = dev->ha; 3061 3062 QL_DPRINT12(ha, "enter\n"); 3063 3064 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); 3065 if (rc) { 3066 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); 3067 return ERR_PTR(rc); 3068 } 3069 3070 QL_DPRINT12(ha, "called from %s, event_handle=%p," 3071 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 3072 (udata ? "user library" : "kernel"), 3073 attrs->event_handler, pd, 3074 get_qlnxr_cq(attrs->send_cq), 3075 get_qlnxr_cq(attrs->send_cq)->icid, 3076 get_qlnxr_cq(attrs->recv_cq), 3077 get_qlnxr_cq(attrs->recv_cq)->icid); 3078 3079 qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); 3080 3081 if (!qp) { 3082 QL_DPRINT11(ha, "kzalloc(qp) failed\n"); 3083 return ERR_PTR(-ENOMEM); 3084 } 3085 3086 qlnxr_set_common_qp_params(dev, qp, pd, attrs); 3087 3088 if (attrs->qp_type == IB_QPT_GSI) { 3089 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); 3090 return qlnxr_create_gsi_qp(dev, attrs, qp); 3091 } 3092 3093 if (udata) { 3094 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); 3095 3096 if (rc) { 3097 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); 3098 goto err; 3099 } 3100 } else { 3101 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); 3102 3103 if (rc) { 3104 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); 3105 goto err; 3106 } 3107 } 3108 3109 qp->ibqp.qp_num = qp->qp_id; 3110 3111 rc = qlnxr_idr_add(dev, qp, qp->qp_id); 3112 3113 if (rc) { 3114 QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); 3115 goto err; 3116 } 3117 3118 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); 3119 3120 return &qp->ibqp; 3121 err: 3122 QL_DPRINT12(ha, "failed exit\n"); 3123 return ERR_PTR(-EFAULT); 3124 } 3125 3126 static enum ib_qp_state 3127 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) 3128 { 3129 enum ib_qp_state state = IB_QPS_ERR; 3130 3131 switch (qp_state) { 3132 case ECORE_ROCE_QP_STATE_RESET: 3133 state = IB_QPS_RESET; 3134 break; 3135 3136 case ECORE_ROCE_QP_STATE_INIT: 3137 state = IB_QPS_INIT; 3138 break; 3139 3140 case ECORE_ROCE_QP_STATE_RTR: 3141 state = IB_QPS_RTR; 3142 break; 3143 3144 case ECORE_ROCE_QP_STATE_RTS: 3145 state = IB_QPS_RTS; 3146 break; 3147 3148 case ECORE_ROCE_QP_STATE_SQD: 3149 state = IB_QPS_SQD; 3150 break; 3151 3152 case ECORE_ROCE_QP_STATE_ERR: 3153 state = IB_QPS_ERR; 3154 break; 3155 3156 case ECORE_ROCE_QP_STATE_SQE: 3157 state = IB_QPS_SQE; 3158 break; 3159 } 3160 return state; 3161 } 3162 3163 static enum ecore_roce_qp_state 3164 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) 3165 { 3166 enum ecore_roce_qp_state ecore_qp_state; 3167 3168 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3169 3170 switch (qp_state) { 3171 case IB_QPS_RESET: 3172 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; 3173 break; 3174 3175 case IB_QPS_INIT: 3176 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; 3177 break; 3178 3179 case IB_QPS_RTR: 3180 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; 3181 break; 3182 3183 case IB_QPS_RTS: 3184 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; 3185 break; 3186 3187 case IB_QPS_SQD: 3188 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; 3189 break; 3190 3191 case IB_QPS_ERR: 3192 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3193 break; 3194 3195 default: 3196 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3197 break; 3198 } 3199 3200 return (ecore_qp_state); 3201 } 3202 3203 static void 3204 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) 3205 { 3206 ecore_chain_reset(&qph->pbl); 3207 qph->prod = qph->cons = 0; 3208 qph->wqe_cons = 0; 3209 qph->db_data.data.value = cpu_to_le16(0); 3210 3211 return; 3212 } 3213 3214 static int 3215 qlnxr_update_qp_state(struct qlnxr_dev *dev, 3216 struct qlnxr_qp *qp, 3217 enum ecore_roce_qp_state new_state) 3218 { 3219 int status = 0; 3220 uint32_t reg_addr; 3221 struct ecore_dev *cdev; 3222 qlnx_host_t *ha; 3223 3224 ha = dev->ha; 3225 cdev = &ha->cdev; 3226 3227 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", 3228 qp, new_state, qp->state); 3229 3230 if (new_state == qp->state) { 3231 return 0; 3232 } 3233 3234 switch (qp->state) { 3235 case ECORE_ROCE_QP_STATE_RESET: 3236 switch (new_state) { 3237 case ECORE_ROCE_QP_STATE_INIT: 3238 qp->prev_wqe_size = 0; 3239 qlnxr_reset_qp_hwq_info(&qp->sq); 3240 if (!(qp->srq)) 3241 qlnxr_reset_qp_hwq_info(&qp->rq); 3242 break; 3243 default: 3244 status = -EINVAL; 3245 break; 3246 }; 3247 break; 3248 case ECORE_ROCE_QP_STATE_INIT: 3249 /* INIT->XXX */ 3250 switch (new_state) { 3251 case ECORE_ROCE_QP_STATE_RTR: 3252 /* Update doorbell (in case post_recv was done before move to RTR) */ 3253 if (qp->srq) 3254 break; 3255 wmb(); 3256 //writel(qp->rq.db_data.raw, qp->rq.db); 3257 //if (QLNX_IS_IWARP(dev)) 3258 // writel(qp->rq.iwarp_db2_data.raw, 3259 // qp->rq.iwarp_db2); 3260 3261 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 3262 (uint8_t *)cdev->doorbells); 3263 3264 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 3265 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 3266 3267 if (QLNX_IS_IWARP(dev)) { 3268 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 3269 (uint8_t *)cdev->doorbells); 3270 bus_write_4(ha->pci_dbells, reg_addr,\ 3271 qp->rq.iwarp_db2_data.raw); 3272 bus_barrier(ha->pci_dbells, 0, 0,\ 3273 BUS_SPACE_BARRIER_READ); 3274 } 3275 3276 3277 mmiowb(); 3278 break; 3279 case ECORE_ROCE_QP_STATE_ERR: 3280 /* TBD:flush qps... */ 3281 break; 3282 default: 3283 /* invalid state change. */ 3284 status = -EINVAL; 3285 break; 3286 }; 3287 break; 3288 case ECORE_ROCE_QP_STATE_RTR: 3289 /* RTR->XXX */ 3290 switch (new_state) { 3291 case ECORE_ROCE_QP_STATE_RTS: 3292 break; 3293 case ECORE_ROCE_QP_STATE_ERR: 3294 break; 3295 default: 3296 /* invalid state change. */ 3297 status = -EINVAL; 3298 break; 3299 }; 3300 break; 3301 case ECORE_ROCE_QP_STATE_RTS: 3302 /* RTS->XXX */ 3303 switch (new_state) { 3304 case ECORE_ROCE_QP_STATE_SQD: 3305 break; 3306 case ECORE_ROCE_QP_STATE_ERR: 3307 break; 3308 default: 3309 /* invalid state change. */ 3310 status = -EINVAL; 3311 break; 3312 }; 3313 break; 3314 case ECORE_ROCE_QP_STATE_SQD: 3315 /* SQD->XXX */ 3316 switch (new_state) { 3317 case ECORE_ROCE_QP_STATE_RTS: 3318 case ECORE_ROCE_QP_STATE_ERR: 3319 break; 3320 default: 3321 /* invalid state change. */ 3322 status = -EINVAL; 3323 break; 3324 }; 3325 break; 3326 case ECORE_ROCE_QP_STATE_ERR: 3327 /* ERR->XXX */ 3328 switch (new_state) { 3329 case ECORE_ROCE_QP_STATE_RESET: 3330 if ((qp->rq.prod != qp->rq.cons) || 3331 (qp->sq.prod != qp->sq.cons)) { 3332 QL_DPRINT11(ha, 3333 "Error->Reset with rq/sq " 3334 "not empty rq.prod=0x%x rq.cons=0x%x" 3335 " sq.prod=0x%x sq.cons=0x%x\n", 3336 qp->rq.prod, qp->rq.cons, 3337 qp->sq.prod, qp->sq.cons); 3338 status = -EINVAL; 3339 } 3340 break; 3341 default: 3342 status = -EINVAL; 3343 break; 3344 }; 3345 break; 3346 default: 3347 status = -EINVAL; 3348 break; 3349 }; 3350 3351 QL_DPRINT12(ha, "exit\n"); 3352 return status; 3353 } 3354 3355 int 3356 qlnxr_modify_qp(struct ib_qp *ibqp, 3357 struct ib_qp_attr *attr, 3358 int attr_mask, 3359 struct ib_udata *udata) 3360 { 3361 int rc = 0; 3362 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3363 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); 3364 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 3365 enum ib_qp_state old_qp_state, new_qp_state; 3366 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3367 qlnx_host_t *ha; 3368 3369 ha = dev->ha; 3370 3371 QL_DPRINT12(ha, 3372 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", 3373 qp, attr_mask, attr->qp_state, udata); 3374 3375 old_qp_state = qlnxr_get_ibqp_state(qp->state); 3376 if (attr_mask & IB_QP_STATE) 3377 new_qp_state = attr->qp_state; 3378 else 3379 new_qp_state = old_qp_state; 3380 3381 if (QLNX_IS_ROCE(dev)) { 3382 if (!ib_modify_qp_is_ok(old_qp_state, 3383 new_qp_state, 3384 ibqp->qp_type, 3385 attr_mask )) { 3386 QL_DPRINT12(ha, 3387 "invalid attribute mask=0x%x" 3388 " specified for qpn=0x%x of type=0x%x \n" 3389 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3390 attr_mask, qp->qp_id, ibqp->qp_type, 3391 old_qp_state, new_qp_state); 3392 rc = -EINVAL; 3393 goto err; 3394 } 3395 } 3396 /* translate the masks... */ 3397 if (attr_mask & IB_QP_STATE) { 3398 SET_FIELD(qp_params.modify_flags, 3399 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 3400 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); 3401 } 3402 3403 // TBD consider changing ecore to be a flag as well... 3404 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 3405 qp_params.sqd_async = true; 3406 3407 if (attr_mask & IB_QP_PKEY_INDEX) { 3408 SET_FIELD(qp_params.modify_flags, 3409 ECORE_ROCE_MODIFY_QP_VALID_PKEY, 3410 1); 3411 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { 3412 rc = -EINVAL; 3413 goto err; 3414 } 3415 3416 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; 3417 } 3418 3419 if (attr_mask & IB_QP_QKEY) { 3420 qp->qkey = attr->qkey; 3421 } 3422 3423 /* tbd consider splitting in ecore.. */ 3424 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3425 SET_FIELD(qp_params.modify_flags, 3426 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 3427 qp_params.incoming_rdma_read_en = 3428 attr->qp_access_flags & IB_ACCESS_REMOTE_READ; 3429 qp_params.incoming_rdma_write_en = 3430 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; 3431 qp_params.incoming_atomic_en = 3432 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; 3433 } 3434 3435 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 3436 if (attr_mask & IB_QP_PATH_MTU) { 3437 if (attr->path_mtu < IB_MTU_256 || 3438 attr->path_mtu > IB_MTU_4096) { 3439 QL_DPRINT12(ha, 3440 "Only MTU sizes of 256, 512, 1024," 3441 " 2048 and 4096 are supported " 3442 " attr->path_mtu = [%d]\n", 3443 attr->path_mtu); 3444 3445 rc = -EINVAL; 3446 goto err; 3447 } 3448 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 3449 ib_mtu_enum_to_int( 3450 iboe_get_mtu(if_getmtu(dev->ha->ifp)))); 3451 } 3452 3453 if (qp->mtu == 0) { 3454 qp->mtu = ib_mtu_enum_to_int( 3455 iboe_get_mtu(if_getmtu(dev->ha->ifp))); 3456 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", 3457 qp->mtu); 3458 } 3459 3460 SET_FIELD(qp_params.modify_flags, 3461 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 3462 1); 3463 3464 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; 3465 qp_params.flow_label = attr->ah_attr.grh.flow_label; 3466 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; 3467 3468 qp->sgid_idx = attr->ah_attr.grh.sgid_index; 3469 3470 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); 3471 3472 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); 3473 if (rc) 3474 return rc; 3475 3476 qp_params.use_local_mac = true; 3477 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 3478 3479 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", 3480 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 3481 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 3482 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", 3483 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 3484 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 3485 QL_DPRINT12(ha, 3486 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", 3487 qp_params.remote_mac_addr[0], 3488 qp_params.remote_mac_addr[1], 3489 qp_params.remote_mac_addr[2], 3490 qp_params.remote_mac_addr[3], 3491 qp_params.remote_mac_addr[4], 3492 qp_params.remote_mac_addr[5]); 3493 3494 qp_params.mtu = qp->mtu; 3495 } 3496 3497 if (qp_params.mtu == 0) { 3498 /* stay with current MTU */ 3499 if (qp->mtu) { 3500 qp_params.mtu = qp->mtu; 3501 } else { 3502 qp_params.mtu = ib_mtu_enum_to_int( 3503 iboe_get_mtu(if_getmtu(dev->ha->ifp))); 3504 } 3505 } 3506 3507 if (attr_mask & IB_QP_TIMEOUT) { 3508 SET_FIELD(qp_params.modify_flags, \ 3509 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 3510 3511 qp_params.ack_timeout = attr->timeout; 3512 if (attr->timeout) { 3513 u32 temp; 3514 3515 /* 12.7.34 LOCAL ACK TIMEOUT 3516 * Value representing the transport (ACK) timeout for 3517 * use by the remote, expressed as (4.096 μS*2Local ACK 3518 * Timeout) 3519 */ 3520 /* We use 1UL since the temporal value may be overflow 3521 * 32 bits 3522 */ 3523 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 3524 qp_params.ack_timeout = temp; /* FW requires [msec] */ 3525 } 3526 else 3527 qp_params.ack_timeout = 0; /* infinite */ 3528 } 3529 if (attr_mask & IB_QP_RETRY_CNT) { 3530 SET_FIELD(qp_params.modify_flags,\ 3531 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 3532 qp_params.retry_cnt = attr->retry_cnt; 3533 } 3534 3535 if (attr_mask & IB_QP_RNR_RETRY) { 3536 SET_FIELD(qp_params.modify_flags, 3537 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 3538 1); 3539 qp_params.rnr_retry_cnt = attr->rnr_retry; 3540 } 3541 3542 if (attr_mask & IB_QP_RQ_PSN) { 3543 SET_FIELD(qp_params.modify_flags, 3544 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 3545 1); 3546 qp_params.rq_psn = attr->rq_psn; 3547 qp->rq_psn = attr->rq_psn; 3548 } 3549 3550 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 3551 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { 3552 rc = -EINVAL; 3553 QL_DPRINT12(ha, 3554 "unsupported max_rd_atomic=%d, supported=%d\n", 3555 attr->max_rd_atomic, 3556 qattr->max_qp_req_rd_atomic_resc); 3557 goto err; 3558 } 3559 3560 SET_FIELD(qp_params.modify_flags, 3561 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 3562 1); 3563 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 3564 } 3565 3566 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 3567 SET_FIELD(qp_params.modify_flags, 3568 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 3569 1); 3570 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 3571 } 3572 3573 if (attr_mask & IB_QP_SQ_PSN) { 3574 SET_FIELD(qp_params.modify_flags, 3575 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 3576 1); 3577 qp_params.sq_psn = attr->sq_psn; 3578 qp->sq_psn = attr->sq_psn; 3579 } 3580 3581 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 3582 if (attr->max_dest_rd_atomic > 3583 qattr->max_qp_resp_rd_atomic_resc) { 3584 QL_DPRINT12(ha, 3585 "unsupported max_dest_rd_atomic=%d, " 3586 "supported=%d\n", 3587 attr->max_dest_rd_atomic, 3588 qattr->max_qp_resp_rd_atomic_resc); 3589 3590 rc = -EINVAL; 3591 goto err; 3592 } 3593 3594 SET_FIELD(qp_params.modify_flags, 3595 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 3596 1); 3597 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 3598 } 3599 3600 if (attr_mask & IB_QP_DEST_QPN) { 3601 SET_FIELD(qp_params.modify_flags, 3602 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 3603 1); 3604 3605 qp_params.dest_qp = attr->dest_qp_num; 3606 qp->dest_qp_num = attr->dest_qp_num; 3607 } 3608 3609 /* 3610 * Update the QP state before the actual ramrod to prevent a race with 3611 * fast path. Modifying the QP state to error will cause the device to 3612 * flush the CQEs and while polling the flushed CQEs will considered as 3613 * a potential issue if the QP isn't in error state. 3614 */ 3615 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && 3616 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) 3617 qp->state = ECORE_ROCE_QP_STATE_ERR; 3618 3619 if (qp->qp_type != IB_QPT_GSI) 3620 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 3621 3622 if (attr_mask & IB_QP_STATE) { 3623 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 3624 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); 3625 qp->state = qp_params.new_state; 3626 } 3627 3628 err: 3629 QL_DPRINT12(ha, "exit\n"); 3630 return rc; 3631 } 3632 3633 static int 3634 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) 3635 { 3636 int ib_qp_acc_flags = 0; 3637 3638 if (params->incoming_rdma_write_en) 3639 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 3640 if (params->incoming_rdma_read_en) 3641 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 3642 if (params->incoming_atomic_en) 3643 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 3644 if (true) /* FIXME -> local write ?? */ 3645 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 3646 3647 return ib_qp_acc_flags; 3648 } 3649 3650 static enum ib_mtu 3651 qlnxr_mtu_int_to_enum(u16 mtu) 3652 { 3653 enum ib_mtu ib_mtu_size; 3654 3655 switch (mtu) { 3656 case 256: 3657 ib_mtu_size = IB_MTU_256; 3658 break; 3659 3660 case 512: 3661 ib_mtu_size = IB_MTU_512; 3662 break; 3663 3664 case 1024: 3665 ib_mtu_size = IB_MTU_1024; 3666 break; 3667 3668 case 2048: 3669 ib_mtu_size = IB_MTU_2048; 3670 break; 3671 3672 case 4096: 3673 ib_mtu_size = IB_MTU_4096; 3674 break; 3675 3676 default: 3677 ib_mtu_size = IB_MTU_1024; 3678 break; 3679 } 3680 return (ib_mtu_size); 3681 } 3682 3683 int 3684 qlnxr_query_qp(struct ib_qp *ibqp, 3685 struct ib_qp_attr *qp_attr, 3686 int attr_mask, 3687 struct ib_qp_init_attr *qp_init_attr) 3688 { 3689 int rc = 0; 3690 struct ecore_rdma_query_qp_out_params params; 3691 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3692 struct qlnxr_dev *dev = qp->dev; 3693 qlnx_host_t *ha; 3694 3695 ha = dev->ha; 3696 3697 QL_DPRINT12(ha, "enter\n"); 3698 3699 memset(¶ms, 0, sizeof(params)); 3700 3701 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); 3702 if (rc) 3703 goto err; 3704 3705 memset(qp_attr, 0, sizeof(*qp_attr)); 3706 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 3707 3708 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); 3709 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); 3710 3711 /* In some cases in iWARP qelr will ask for the state only */ 3712 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { 3713 QL_DPRINT11(ha, "only state requested\n"); 3714 return 0; 3715 } 3716 3717 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); 3718 qp_attr->path_mig_state = IB_MIG_MIGRATED; 3719 qp_attr->rq_psn = params.rq_psn; 3720 qp_attr->sq_psn = params.sq_psn; 3721 qp_attr->dest_qp_num = params.dest_qp; 3722 3723 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); 3724 3725 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " 3726 "path_mtu = %d qp_access_flags = 0x%x\n", 3727 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, 3728 qp_attr->qp_access_flags); 3729 3730 qp_attr->cap.max_send_wr = qp->sq.max_wr; 3731 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 3732 qp_attr->cap.max_send_sge = qp->sq.max_sges; 3733 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 3734 qp_attr->cap.max_inline_data = qp->max_inline_data; 3735 qp_init_attr->cap = qp_attr->cap; 3736 3737 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], 3738 sizeof(qp_attr->ah_attr.grh.dgid.raw)); 3739 3740 qp_attr->ah_attr.grh.flow_label = params.flow_label; 3741 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; 3742 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; 3743 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; 3744 3745 qp_attr->ah_attr.ah_flags = IB_AH_GRH; 3746 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ 3747 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ 3748 qp_attr->timeout = params.timeout; 3749 qp_attr->rnr_retry = params.rnr_retry; 3750 qp_attr->retry_cnt = params.retry_cnt; 3751 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 3752 qp_attr->pkey_index = params.pkey_index; 3753 qp_attr->port_num = 1; /* FIXME -> check this */ 3754 qp_attr->ah_attr.src_path_bits = 0; 3755 qp_attr->ah_attr.static_rate = 0; 3756 qp_attr->alt_pkey_index = 0; 3757 qp_attr->alt_port_num = 0; 3758 qp_attr->alt_timeout = 0; 3759 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 3760 3761 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; 3762 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 3763 qp_attr->max_rd_atomic = params.max_rd_atomic; 3764 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; 3765 3766 QL_DPRINT12(ha, "max_inline_data=%d\n", 3767 qp_attr->cap.max_inline_data); 3768 3769 err: 3770 QL_DPRINT12(ha, "exit\n"); 3771 return rc; 3772 } 3773 3774 static void 3775 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3776 { 3777 qlnx_host_t *ha; 3778 3779 ha = dev->ha; 3780 3781 QL_DPRINT12(ha, "enter\n"); 3782 3783 if (qp->usq.umem) 3784 ib_umem_release(qp->usq.umem); 3785 3786 qp->usq.umem = NULL; 3787 3788 if (qp->urq.umem) 3789 ib_umem_release(qp->urq.umem); 3790 3791 qp->urq.umem = NULL; 3792 3793 QL_DPRINT12(ha, "exit\n"); 3794 return; 3795 } 3796 3797 static void 3798 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3799 { 3800 qlnx_host_t *ha; 3801 3802 ha = dev->ha; 3803 3804 QL_DPRINT12(ha, "enter\n"); 3805 3806 if (qlnxr_qp_has_sq(qp)) { 3807 QL_DPRINT12(ha, "freeing SQ\n"); 3808 ha->qlnxr_debug = 1; 3809 // ecore_chain_free(dev->cdev, &qp->sq.pbl); 3810 ha->qlnxr_debug = 0; 3811 kfree(qp->wqe_wr_id); 3812 } 3813 3814 if (qlnxr_qp_has_rq(qp)) { 3815 QL_DPRINT12(ha, "freeing RQ\n"); 3816 ha->qlnxr_debug = 1; 3817 // ecore_chain_free(dev->cdev, &qp->rq.pbl); 3818 ha->qlnxr_debug = 0; 3819 kfree(qp->rqe_wr_id); 3820 } 3821 3822 QL_DPRINT12(ha, "exit\n"); 3823 return; 3824 } 3825 3826 static int 3827 qlnxr_free_qp_resources(struct qlnxr_dev *dev, 3828 struct qlnxr_qp *qp, struct ib_udata *udata) 3829 { 3830 int rc = 0; 3831 qlnx_host_t *ha; 3832 struct ecore_rdma_destroy_qp_out_params d_out_params; 3833 3834 ha = dev->ha; 3835 3836 QL_DPRINT12(ha, "enter\n"); 3837 3838 #if 0 3839 if (qp->qp_type != IB_QPT_GSI) { 3840 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 3841 &d_out_params); 3842 if (rc) 3843 return rc; 3844 } 3845 3846 if (udata) 3847 qlnxr_cleanup_user(dev, qp); 3848 else 3849 qlnxr_cleanup_kernel(dev, qp); 3850 #endif 3851 3852 if (udata) 3853 qlnxr_cleanup_user(dev, qp); 3854 else 3855 qlnxr_cleanup_kernel(dev, qp); 3856 3857 if (qp->qp_type != IB_QPT_GSI) { 3858 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 3859 &d_out_params); 3860 if (rc) 3861 return rc; 3862 } 3863 3864 QL_DPRINT12(ha, "exit\n"); 3865 return 0; 3866 } 3867 3868 int 3869 qlnxr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 3870 { 3871 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3872 struct qlnxr_dev *dev = qp->dev; 3873 int rc = 0; 3874 struct ib_qp_attr attr; 3875 int attr_mask = 0; 3876 qlnx_host_t *ha; 3877 3878 ha = dev->ha; 3879 3880 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); 3881 3882 qp->destroyed = 1; 3883 3884 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | 3885 ECORE_ROCE_QP_STATE_ERR | 3886 ECORE_ROCE_QP_STATE_INIT))) { 3887 attr.qp_state = IB_QPS_ERR; 3888 attr_mask |= IB_QP_STATE; 3889 3890 /* change the QP state to ERROR */ 3891 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); 3892 } 3893 3894 if (qp->qp_type == IB_QPT_GSI) 3895 qlnxr_destroy_gsi_qp(dev); 3896 3897 qp->sig = ~qp->sig; 3898 3899 qlnxr_free_qp_resources(dev, qp, udata); 3900 3901 if (atomic_dec_and_test(&qp->refcnt)) { 3902 /* TODO: only for iWARP? */ 3903 qlnxr_idr_remove(dev, qp->qp_id); 3904 } 3905 3906 QL_DPRINT12(ha, "exit\n"); 3907 return rc; 3908 } 3909 3910 static inline int 3911 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) 3912 { 3913 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3914 } 3915 3916 static int 3917 sge_data_len(struct ib_sge *sg_list, int num_sge) 3918 { 3919 int i, len = 0; 3920 for (i = 0; i < num_sge; i++) 3921 len += sg_list[i].length; 3922 return len; 3923 } 3924 3925 static void 3926 swap_wqe_data64(u64 *p) 3927 { 3928 int i; 3929 3930 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3931 *p = cpu_to_be64(cpu_to_le64(*p)); 3932 } 3933 3934 static u32 3935 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, 3936 struct qlnxr_qp *qp, 3937 u8 *wqe_size, 3938 const struct ib_send_wr *wr, 3939 const struct ib_send_wr **bad_wr, 3940 u8 *bits, 3941 u8 bit) 3942 { 3943 int i, seg_siz; 3944 char *seg_prt, *wqe; 3945 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3946 qlnx_host_t *ha; 3947 3948 ha = dev->ha; 3949 3950 QL_DPRINT12(ha, "enter[%d]\n", data_size); 3951 3952 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3953 QL_DPRINT12(ha, 3954 "Too much inline data in WR:[%d, %d]\n", 3955 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); 3956 *bad_wr = wr; 3957 return 0; 3958 } 3959 3960 if (!data_size) 3961 return data_size; 3962 3963 /* set the bit */ 3964 *bits |= bit; 3965 3966 seg_prt = wqe = NULL; 3967 seg_siz = 0; 3968 3969 /* copy data inline */ 3970 for (i = 0; i < wr->num_sge; i++) { 3971 u32 len = wr->sg_list[i].length; 3972 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3973 3974 while (len > 0) { 3975 u32 cur; 3976 3977 /* new segment required */ 3978 if (!seg_siz) { 3979 wqe = (char *)ecore_chain_produce(&qp->sq.pbl); 3980 seg_prt = wqe; 3981 seg_siz = sizeof(struct rdma_sq_common_wqe); 3982 (*wqe_size)++; 3983 } 3984 3985 /* calculate currently allowed length */ 3986 cur = MIN(len, seg_siz); 3987 3988 memcpy(seg_prt, src, cur); 3989 3990 /* update segment variables */ 3991 seg_prt += cur; 3992 seg_siz -= cur; 3993 /* update sge variables */ 3994 src += cur; 3995 len -= cur; 3996 3997 /* swap fully-completed segments */ 3998 if (!seg_siz) 3999 swap_wqe_data64((u64 *)wqe); 4000 } 4001 } 4002 4003 /* swap last not completed segment */ 4004 if (seg_siz) 4005 swap_wqe_data64((u64 *)wqe); 4006 4007 QL_DPRINT12(ha, "exit\n"); 4008 return data_size; 4009 } 4010 4011 static u32 4012 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, 4013 u8 *wqe_size, const struct ib_send_wr *wr) 4014 { 4015 int i; 4016 u32 data_size = 0; 4017 qlnx_host_t *ha; 4018 4019 ha = dev->ha; 4020 4021 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); 4022 4023 for (i = 0; i < wr->num_sge; i++) { 4024 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); 4025 4026 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); 4027 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 4028 sge->length = cpu_to_le32(wr->sg_list[i].length); 4029 data_size += wr->sg_list[i].length; 4030 } 4031 4032 if (wqe_size) 4033 *wqe_size += wr->num_sge; 4034 4035 QL_DPRINT12(ha, "exit data_size = %d\n", data_size); 4036 return data_size; 4037 } 4038 4039 static u32 4040 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, 4041 struct qlnxr_qp *qp, 4042 struct rdma_sq_rdma_wqe_1st *rwqe, 4043 struct rdma_sq_rdma_wqe_2nd *rwqe2, 4044 const struct ib_send_wr *wr, 4045 const struct ib_send_wr **bad_wr) 4046 { 4047 qlnx_host_t *ha; 4048 u32 ret = 0; 4049 4050 ha = dev->ha; 4051 4052 QL_DPRINT12(ha, "enter\n"); 4053 4054 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 4055 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); 4056 4057 if (wr->send_flags & IB_SEND_INLINE) { 4058 u8 flags = 0; 4059 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 4060 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, 4061 wr, bad_wr, &rwqe->flags, flags); 4062 } 4063 4064 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); 4065 4066 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4067 4068 return (ret); 4069 } 4070 4071 static u32 4072 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, 4073 struct qlnxr_qp *qp, 4074 struct rdma_sq_send_wqe *swqe, 4075 struct rdma_sq_send_wqe *swqe2, 4076 const struct ib_send_wr *wr, 4077 const struct ib_send_wr **bad_wr) 4078 { 4079 qlnx_host_t *ha; 4080 u32 ret = 0; 4081 4082 ha = dev->ha; 4083 4084 QL_DPRINT12(ha, "enter\n"); 4085 4086 memset(swqe2, 0, sizeof(*swqe2)); 4087 4088 if (wr->send_flags & IB_SEND_INLINE) { 4089 u8 flags = 0; 4090 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 4091 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, 4092 wr, bad_wr, &swqe->flags, flags); 4093 } 4094 4095 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); 4096 4097 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4098 4099 return (ret); 4100 } 4101 4102 static void 4103 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) 4104 { 4105 qlnx_host_t *ha; 4106 4107 ha = dev->ha; 4108 4109 int work = info->completed - info->completed_handled - 1; 4110 4111 QL_DPRINT12(ha, "enter [%d]\n", work); 4112 4113 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 4114 struct qlnxr_pbl *pbl; 4115 4116 /* Free all the page list that are possible to be freed 4117 * (all the ones that were invalidated), under the assumption 4118 * that if an FMR was completed successfully that means that 4119 * if there was an invalidate operation before it also ended 4120 */ 4121 pbl = list_first_entry(&info->inuse_pbl_list, 4122 struct qlnxr_pbl, 4123 list_entry); 4124 list_del(&pbl->list_entry); 4125 list_add_tail(&pbl->list_entry, &info->free_pbl_list); 4126 info->completed_handled++; 4127 } 4128 4129 QL_DPRINT12(ha, "exit\n"); 4130 return; 4131 } 4132 4133 static int qlnxr_prepare_reg(struct qlnxr_qp *qp, 4134 struct rdma_sq_fmr_wqe_1st *fwqe1, 4135 const struct ib_reg_wr *wr) 4136 { 4137 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); 4138 struct rdma_sq_fmr_wqe_2nd *fwqe2; 4139 4140 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); 4141 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 4142 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 4143 fwqe1->l_key = wr->key; 4144 4145 fwqe2->access_ctrl = 0; 4146 4147 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 4148 !!(wr->access & IB_ACCESS_REMOTE_READ)); 4149 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 4150 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 4151 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 4152 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 4153 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 4154 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 4155 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 4156 fwqe2->fmr_ctrl = 0; 4157 4158 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4159 ilog2(mr->ibmr.page_size) - 12); 4160 4161 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ 4162 fwqe2->length_lo = mr->ibmr.length; 4163 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 4164 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 4165 4166 qp->wqe_wr_id[qp->sq.prod].mr = mr; 4167 4168 return 0; 4169 } 4170 4171 static enum ib_wc_opcode 4172 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 4173 { 4174 switch (opcode) { 4175 case IB_WR_RDMA_WRITE: 4176 case IB_WR_RDMA_WRITE_WITH_IMM: 4177 return IB_WC_RDMA_WRITE; 4178 case IB_WR_SEND_WITH_IMM: 4179 case IB_WR_SEND: 4180 case IB_WR_SEND_WITH_INV: 4181 return IB_WC_SEND; 4182 case IB_WR_RDMA_READ: 4183 return IB_WC_RDMA_READ; 4184 case IB_WR_ATOMIC_CMP_AND_SWP: 4185 return IB_WC_COMP_SWAP; 4186 case IB_WR_ATOMIC_FETCH_AND_ADD: 4187 return IB_WC_FETCH_ADD; 4188 case IB_WR_REG_MR: 4189 return IB_WC_REG_MR; 4190 case IB_WR_LOCAL_INV: 4191 return IB_WC_LOCAL_INV; 4192 default: 4193 return IB_WC_SEND; 4194 } 4195 } 4196 static inline bool 4197 qlnxr_can_post_send(struct qlnxr_qp *qp, const struct ib_send_wr *wr) 4198 { 4199 int wq_is_full, err_wr, pbl_is_full; 4200 struct qlnxr_dev *dev = qp->dev; 4201 qlnx_host_t *ha; 4202 4203 ha = dev->ha; 4204 4205 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); 4206 4207 /* prevent SQ overflow and/or processing of a bad WR */ 4208 err_wr = wr->num_sge > qp->sq.max_sges; 4209 wq_is_full = qlnxr_wq_is_full(&qp->sq); 4210 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < 4211 QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 4212 if (wq_is_full || err_wr || pbl_is_full) { 4213 if (wq_is_full && 4214 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { 4215 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; 4216 4217 QL_DPRINT12(ha, 4218 "error: WQ is full. Post send on QP failed" 4219 " (this error appears only once) " 4220 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4221 qp, wr, qp->err_bitmap); 4222 } 4223 4224 if (err_wr && 4225 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { 4226 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; 4227 4228 QL_DPRINT12(ha, 4229 "error: WQ is bad. Post send on QP failed" 4230 " (this error appears only once) " 4231 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4232 qp, wr, qp->err_bitmap); 4233 } 4234 4235 if (pbl_is_full && 4236 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { 4237 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; 4238 4239 QL_DPRINT12(ha, 4240 "error: WQ PBL is full. Post send on QP failed" 4241 " (this error appears only once) " 4242 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4243 qp, wr, qp->err_bitmap); 4244 } 4245 return false; 4246 } 4247 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); 4248 return true; 4249 } 4250 4251 int 4252 qlnxr_post_send(struct ib_qp *ibqp, 4253 const struct ib_send_wr *wr, 4254 const struct ib_send_wr **bad_wr) 4255 { 4256 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); 4257 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4258 unsigned long flags; 4259 int status = 0, rc = 0; 4260 bool comp; 4261 qlnx_host_t *ha; 4262 uint32_t reg_addr; 4263 4264 *bad_wr = NULL; 4265 ha = dev->ha; 4266 4267 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4268 ibqp, wr, bad_wr); 4269 4270 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 4271 return -EINVAL; 4272 4273 if (qp->qp_type == IB_QPT_GSI) 4274 return qlnxr_gsi_post_send(ibqp, wr, bad_wr); 4275 4276 spin_lock_irqsave(&qp->q_lock, flags); 4277 4278 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && 4279 (qp->state != ECORE_ROCE_QP_STATE_ERR) && 4280 (qp->state != ECORE_ROCE_QP_STATE_SQD)) { 4281 spin_unlock_irqrestore(&qp->q_lock, flags); 4282 *bad_wr = wr; 4283 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", 4284 qp->icid, qp->state); 4285 return -EINVAL; 4286 } 4287 4288 if (!wr) { 4289 QL_DPRINT11(ha, "Got an empty post send???\n"); 4290 } 4291 4292 while (wr) { 4293 struct rdma_sq_common_wqe *wqe; 4294 struct rdma_sq_send_wqe *swqe; 4295 struct rdma_sq_send_wqe *swqe2; 4296 struct rdma_sq_rdma_wqe_1st *rwqe; 4297 struct rdma_sq_rdma_wqe_2nd *rwqe2; 4298 struct rdma_sq_local_inv_wqe *iwqe; 4299 struct rdma_sq_atomic_wqe *awqe1; 4300 struct rdma_sq_atomic_wqe *awqe2; 4301 struct rdma_sq_atomic_wqe *awqe3; 4302 struct rdma_sq_fmr_wqe_1st *fwqe1; 4303 4304 if (!qlnxr_can_post_send(qp, wr)) { 4305 status = -ENOMEM; 4306 *bad_wr = wr; 4307 break; 4308 } 4309 4310 wqe = ecore_chain_produce(&qp->sq.pbl); 4311 4312 qp->wqe_wr_id[qp->sq.prod].signaled = 4313 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 4314 4315 /* common fields */ 4316 wqe->flags = 0; 4317 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << 4318 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); 4319 4320 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ 4321 !!(wr->send_flags & IB_SEND_SOLICITED)); 4322 4323 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || 4324 (qp->signaled); 4325 4326 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 4327 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ 4328 !!(wr->send_flags & IB_SEND_FENCE)); 4329 4330 wqe->prev_wqe_size = qp->prev_wqe_size; 4331 4332 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); 4333 4334 switch (wr->opcode) { 4335 case IB_WR_SEND_WITH_IMM: 4336 4337 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 4338 swqe = (struct rdma_sq_send_wqe *)wqe; 4339 swqe->wqe_size = 2; 4340 swqe2 = (struct rdma_sq_send_wqe *) 4341 ecore_chain_produce(&qp->sq.pbl); 4342 swqe->inv_key_or_imm_data = 4343 cpu_to_le32(wr->ex.imm_data); 4344 swqe->length = cpu_to_le32( 4345 qlnxr_prepare_sq_send_data(dev, 4346 qp, swqe, swqe2, wr, 4347 bad_wr)); 4348 4349 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4350 qp->prev_wqe_size = swqe->wqe_size; 4351 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4352 4353 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", 4354 swqe->length, wr->ex.imm_data); 4355 4356 break; 4357 4358 case IB_WR_SEND: 4359 4360 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 4361 swqe = (struct rdma_sq_send_wqe *)wqe; 4362 4363 swqe->wqe_size = 2; 4364 swqe2 = (struct rdma_sq_send_wqe *) 4365 ecore_chain_produce(&qp->sq.pbl); 4366 swqe->length = cpu_to_le32( 4367 qlnxr_prepare_sq_send_data(dev, 4368 qp, swqe, swqe2, wr, 4369 bad_wr)); 4370 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4371 qp->prev_wqe_size = swqe->wqe_size; 4372 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4373 4374 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", 4375 swqe->length); 4376 4377 break; 4378 4379 case IB_WR_SEND_WITH_INV: 4380 4381 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 4382 swqe = (struct rdma_sq_send_wqe *)wqe; 4383 swqe2 = (struct rdma_sq_send_wqe *) 4384 ecore_chain_produce(&qp->sq.pbl); 4385 swqe->wqe_size = 2; 4386 swqe->inv_key_or_imm_data = 4387 cpu_to_le32(wr->ex.invalidate_rkey); 4388 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, 4389 qp, swqe, swqe2, wr, bad_wr)); 4390 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4391 qp->prev_wqe_size = swqe->wqe_size; 4392 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4393 4394 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", 4395 swqe->length); 4396 break; 4397 4398 case IB_WR_RDMA_WRITE_WITH_IMM: 4399 4400 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 4401 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4402 4403 rwqe->wqe_size = 2; 4404 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 4405 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4406 ecore_chain_produce(&qp->sq.pbl); 4407 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4408 qp, rwqe, rwqe2, wr, bad_wr)); 4409 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4410 qp->prev_wqe_size = rwqe->wqe_size; 4411 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4412 4413 QL_DPRINT12(ha, 4414 "RDMA WRITE w/ IMM length = %d imm data=%x\n", 4415 rwqe->length, rwqe->imm_data); 4416 4417 break; 4418 4419 case IB_WR_RDMA_WRITE: 4420 4421 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 4422 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4423 4424 rwqe->wqe_size = 2; 4425 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4426 ecore_chain_produce(&qp->sq.pbl); 4427 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4428 qp, rwqe, rwqe2, wr, bad_wr)); 4429 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4430 qp->prev_wqe_size = rwqe->wqe_size; 4431 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4432 4433 QL_DPRINT12(ha, 4434 "RDMA WRITE w/o IMM length = %d\n", 4435 rwqe->length); 4436 4437 break; 4438 4439 case IB_WR_RDMA_READ_WITH_INV: 4440 4441 QL_DPRINT12(ha, 4442 "RDMA READ WITH INVALIDATE not supported\n"); 4443 4444 *bad_wr = wr; 4445 rc = -EINVAL; 4446 4447 break; 4448 4449 case IB_WR_RDMA_READ: 4450 4451 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 4452 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4453 4454 rwqe->wqe_size = 2; 4455 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4456 ecore_chain_produce(&qp->sq.pbl); 4457 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4458 qp, rwqe, rwqe2, wr, bad_wr)); 4459 4460 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4461 qp->prev_wqe_size = rwqe->wqe_size; 4462 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4463 4464 QL_DPRINT12(ha, "RDMA READ length = %d\n", 4465 rwqe->length); 4466 4467 break; 4468 4469 case IB_WR_ATOMIC_CMP_AND_SWP: 4470 case IB_WR_ATOMIC_FETCH_AND_ADD: 4471 4472 QL_DPRINT12(ha, 4473 "ATOMIC operation = %s\n", 4474 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? 4475 "IB_WR_ATOMIC_CMP_AND_SWP" : 4476 "IB_WR_ATOMIC_FETCH_AND_ADD")); 4477 4478 awqe1 = (struct rdma_sq_atomic_wqe *)wqe; 4479 awqe1->prev_wqe_size = 4; 4480 4481 awqe2 = (struct rdma_sq_atomic_wqe *) 4482 ecore_chain_produce(&qp->sq.pbl); 4483 4484 TYPEPTR_ADDR_SET(awqe2, remote_va, \ 4485 atomic_wr(wr)->remote_addr); 4486 4487 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 4488 4489 awqe3 = (struct rdma_sq_atomic_wqe *) 4490 ecore_chain_produce(&qp->sq.pbl); 4491 4492 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 4493 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 4494 TYPEPTR_ADDR_SET(awqe3, swap_data, 4495 atomic_wr(wr)->compare_add); 4496 } else { 4497 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 4498 TYPEPTR_ADDR_SET(awqe3, swap_data, 4499 atomic_wr(wr)->swap); 4500 TYPEPTR_ADDR_SET(awqe3, cmp_data, 4501 atomic_wr(wr)->compare_add); 4502 } 4503 4504 qlnxr_prepare_sq_sges(dev, qp, NULL, wr); 4505 4506 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; 4507 qp->prev_wqe_size = awqe1->prev_wqe_size; 4508 4509 break; 4510 4511 case IB_WR_LOCAL_INV: 4512 4513 QL_DPRINT12(ha, 4514 "INVALIDATE length (IB_WR_LOCAL_INV)\n"); 4515 4516 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 4517 iwqe->prev_wqe_size = 1; 4518 4519 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 4520 iwqe->inv_l_key = wr->ex.invalidate_rkey; 4521 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; 4522 qp->prev_wqe_size = iwqe->prev_wqe_size; 4523 4524 break; 4525 4526 case IB_WR_REG_MR: 4527 4528 QL_DPRINT12(ha, "IB_WR_REG_MR\n"); 4529 4530 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4531 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4532 fwqe1->wqe_size = 2; 4533 4534 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); 4535 if (rc) { 4536 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); 4537 *bad_wr = wr; 4538 break; 4539 } 4540 4541 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 4542 qp->prev_wqe_size = fwqe1->wqe_size; 4543 4544 break; 4545 4546 default: 4547 4548 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); 4549 4550 rc = -EINVAL; 4551 *bad_wr = wr; 4552 break; 4553 } 4554 4555 if (*bad_wr) { 4556 /* 4557 * restore prod to its position before this WR was processed 4558 */ 4559 ecore_chain_set_prod(&qp->sq.pbl, 4560 le16_to_cpu(qp->sq.db_data.data.value), 4561 wqe); 4562 /* restore prev_wqe_size */ 4563 qp->prev_wqe_size = wqe->prev_wqe_size; 4564 status = rc; 4565 4566 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); 4567 break; /* out of the loop */ 4568 } 4569 4570 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 4571 4572 qlnxr_inc_sw_prod(&qp->sq); 4573 4574 qp->sq.db_data.data.value++; 4575 4576 wr = wr->next; 4577 } 4578 4579 /* Trigger doorbell 4580 * If there was a failure in the first WR then it will be triggered in 4581 * vane. However this is not harmful (as long as the producer value is 4582 * unchanged). For performance reasons we avoid checking for this 4583 * redundant doorbell. 4584 */ 4585 wmb(); 4586 //writel(qp->sq.db_data.raw, qp->sq.db); 4587 4588 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); 4589 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); 4590 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4591 4592 mmiowb(); 4593 4594 spin_unlock_irqrestore(&qp->q_lock, flags); 4595 4596 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4597 ibqp, wr, bad_wr); 4598 4599 return status; 4600 } 4601 4602 static u32 4603 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) 4604 { 4605 u32 used; 4606 4607 /* Calculate number of elements used based on producer 4608 * count and consumer count and subtract it from max 4609 * work request supported so that we get elements left. 4610 */ 4611 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 4612 4613 return hw_srq->max_wr - used; 4614 } 4615 4616 int 4617 qlnxr_post_recv(struct ib_qp *ibqp, 4618 const struct ib_recv_wr *wr, 4619 const struct ib_recv_wr **bad_wr) 4620 { 4621 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4622 struct qlnxr_dev *dev = qp->dev; 4623 unsigned long flags; 4624 int status = 0; 4625 qlnx_host_t *ha; 4626 uint32_t reg_addr; 4627 4628 ha = dev->ha; 4629 4630 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 4631 return -EINVAL; 4632 4633 QL_DPRINT12(ha, "enter\n"); 4634 4635 if (qp->qp_type == IB_QPT_GSI) { 4636 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); 4637 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); 4638 } 4639 4640 if (qp->srq) { 4641 QL_DPRINT11(ha, "qp->srq [%p]" 4642 " QP is associated with SRQ, cannot post RQ buffers\n", 4643 qp->srq); 4644 return -EINVAL; 4645 } 4646 4647 spin_lock_irqsave(&qp->q_lock, flags); 4648 4649 if (qp->state == ECORE_ROCE_QP_STATE_RESET) { 4650 spin_unlock_irqrestore(&qp->q_lock, flags); 4651 *bad_wr = wr; 4652 4653 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); 4654 4655 return -EINVAL; 4656 } 4657 4658 while (wr) { 4659 int i; 4660 4661 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < 4662 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || 4663 (wr->num_sge > qp->rq.max_sges)) { 4664 status = -ENOMEM; 4665 *bad_wr = wr; 4666 break; 4667 } 4668 for (i = 0; i < wr->num_sge; i++) { 4669 u32 flags = 0; 4670 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 4671 4672 /* first one must include the number of SGE in the list */ 4673 if (!i) 4674 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); 4675 4676 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); 4677 4678 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ 4679 wr->sg_list[i].length, flags); 4680 } 4681 /* Special case of no sges. FW requires between 1-4 sges... 4682 * in this case we need to post 1 sge with length zero. this is 4683 * because rdma write with immediate consumes an RQ. */ 4684 if (!wr->num_sge) { 4685 u32 flags = 0; 4686 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 4687 4688 /* first one must include the number of SGE in the list */ 4689 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 4690 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 4691 4692 //RQ_SGE_SET(rqe, 0, 0, flags); 4693 rqe->addr.hi = 0; 4694 rqe->addr.lo = 0; 4695 4696 rqe->length = 0; 4697 rqe->flags = cpu_to_le32(flags); 4698 4699 i = 1; 4700 } 4701 4702 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 4703 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 4704 4705 qlnxr_inc_sw_prod(&qp->rq); 4706 4707 wmb(); 4708 4709 qp->rq.db_data.data.value++; 4710 4711 // writel(qp->rq.db_data.raw, qp->rq.db); 4712 mmiowb(); 4713 // if (QLNX_IS_IWARP(dev)) { 4714 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 4715 // mmiowb(); /* for second doorbell */ 4716 // } 4717 4718 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 4719 (uint8_t *)ha->cdev.doorbells); 4720 4721 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 4722 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4723 4724 if (QLNX_IS_IWARP(dev)) { 4725 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 4726 (uint8_t *)ha->cdev.doorbells); 4727 bus_write_4(ha->pci_dbells, reg_addr, \ 4728 qp->rq.iwarp_db2_data.raw); 4729 bus_barrier(ha->pci_dbells, 0, 0, \ 4730 BUS_SPACE_BARRIER_READ); 4731 } 4732 4733 wr = wr->next; 4734 } 4735 4736 spin_unlock_irqrestore(&qp->q_lock, flags); 4737 4738 QL_DPRINT12(ha, "exit status = 0x%x\n", status); 4739 4740 return status; 4741 } 4742 4743 /* In fmr we need to increase the number of fmr completed counter for the fmr 4744 * algorithm determining whether we can free a pbl or not. 4745 * we need to perform this whether the work request was signaled or not. for 4746 * this purpose we call this function from the condition that checks if a wr 4747 * should be skipped, to make sure we don't miss it ( possibly this fmr 4748 * operation was not signalted) 4749 */ 4750 static inline void 4751 qlnxr_chk_if_fmr(struct qlnxr_qp *qp) 4752 { 4753 4754 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4755 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4756 } 4757 4758 static int 4759 process_req(struct qlnxr_dev *dev, 4760 struct qlnxr_qp *qp, 4761 struct qlnxr_cq *cq, 4762 int num_entries, 4763 struct ib_wc *wc, 4764 u16 hw_cons, 4765 enum ib_wc_status status, 4766 int force) 4767 { 4768 u16 cnt = 0; 4769 qlnx_host_t *ha = dev->ha; 4770 4771 QL_DPRINT12(ha, "enter\n"); 4772 4773 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4774 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4775 qlnxr_chk_if_fmr(qp); 4776 /* skip WC */ 4777 goto next_cqe; 4778 } 4779 4780 /* fill WC */ 4781 wc->status = status; 4782 wc->vendor_err = 0; 4783 wc->wc_flags = 0; 4784 wc->src_qp = qp->id; 4785 wc->qp = &qp->ibqp; 4786 4787 // common section 4788 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4789 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4790 4791 switch (wc->opcode) { 4792 case IB_WC_RDMA_WRITE: 4793 4794 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4795 4796 QL_DPRINT12(ha, 4797 "opcode = IB_WC_RDMA_WRITE bytes = %d\n", 4798 qp->wqe_wr_id[qp->sq.cons].bytes_len); 4799 break; 4800 4801 case IB_WC_COMP_SWAP: 4802 case IB_WC_FETCH_ADD: 4803 wc->byte_len = 8; 4804 break; 4805 4806 case IB_WC_REG_MR: 4807 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4808 break; 4809 4810 case IB_WC_RDMA_READ: 4811 case IB_WC_SEND: 4812 4813 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); 4814 break; 4815 default: 4816 ;//DP_ERR("TBD ERROR"); 4817 } 4818 4819 num_entries--; 4820 wc++; 4821 cnt++; 4822 next_cqe: 4823 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4824 ecore_chain_consume(&qp->sq.pbl); 4825 qlnxr_inc_sw_cons(&qp->sq); 4826 } 4827 4828 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 4829 return cnt; 4830 } 4831 4832 static int 4833 qlnxr_poll_cq_req(struct qlnxr_dev *dev, 4834 struct qlnxr_qp *qp, 4835 struct qlnxr_cq *cq, 4836 int num_entries, 4837 struct ib_wc *wc, 4838 struct rdma_cqe_requester *req) 4839 { 4840 int cnt = 0; 4841 qlnx_host_t *ha = dev->ha; 4842 4843 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); 4844 4845 switch (req->status) { 4846 case RDMA_CQE_REQ_STS_OK: 4847 4848 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4849 IB_WC_SUCCESS, 0); 4850 break; 4851 4852 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4853 4854 if (qp->state != ECORE_ROCE_QP_STATE_ERR) 4855 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4856 IB_WC_WR_FLUSH_ERR, 1); 4857 break; 4858 4859 default: /* other errors case */ 4860 4861 /* process all WQE before the cosumer */ 4862 qp->state = ECORE_ROCE_QP_STATE_ERR; 4863 cnt = process_req(dev, qp, cq, num_entries, wc, 4864 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4865 wc += cnt; 4866 /* if we have extra WC fill it with actual error info */ 4867 4868 if (cnt < num_entries) { 4869 enum ib_wc_status wc_status; 4870 4871 switch (req->status) { 4872 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4873 wc_status = IB_WC_BAD_RESP_ERR; 4874 break; 4875 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4876 wc_status = IB_WC_LOC_LEN_ERR; 4877 break; 4878 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4879 wc_status = IB_WC_LOC_QP_OP_ERR; 4880 break; 4881 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4882 wc_status = IB_WC_LOC_PROT_ERR; 4883 break; 4884 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4885 wc_status = IB_WC_MW_BIND_ERR; 4886 break; 4887 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4888 wc_status = IB_WC_REM_INV_REQ_ERR; 4889 break; 4890 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4891 wc_status = IB_WC_REM_ACCESS_ERR; 4892 break; 4893 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4894 wc_status = IB_WC_REM_OP_ERR; 4895 break; 4896 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4897 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4898 break; 4899 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4900 wc_status = IB_WC_RETRY_EXC_ERR; 4901 break; 4902 default: 4903 wc_status = IB_WC_GENERAL_ERR; 4904 } 4905 4906 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4907 wc_status, 1 /* force use of WC */); 4908 } 4909 } 4910 4911 QL_DPRINT12(ha, "exit cnt = %d\n", cnt); 4912 return cnt; 4913 } 4914 4915 static void 4916 __process_resp_one(struct qlnxr_dev *dev, 4917 struct qlnxr_qp *qp, 4918 struct qlnxr_cq *cq, 4919 struct ib_wc *wc, 4920 struct rdma_cqe_responder *resp, 4921 u64 wr_id) 4922 { 4923 enum ib_wc_status wc_status = IB_WC_SUCCESS; 4924 qlnx_host_t *ha = dev->ha; 4925 4926 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", 4927 qp, resp->status); 4928 4929 wc->opcode = IB_WC_RECV; 4930 wc->wc_flags = 0; 4931 4932 switch (resp->status) { 4933 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4934 wc_status = IB_WC_LOC_ACCESS_ERR; 4935 break; 4936 4937 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4938 wc_status = IB_WC_LOC_LEN_ERR; 4939 break; 4940 4941 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4942 wc_status = IB_WC_LOC_QP_OP_ERR; 4943 break; 4944 4945 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4946 wc_status = IB_WC_LOC_PROT_ERR; 4947 break; 4948 4949 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4950 wc_status = IB_WC_MW_BIND_ERR; 4951 break; 4952 4953 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4954 wc_status = IB_WC_REM_INV_RD_REQ_ERR; 4955 break; 4956 4957 case RDMA_CQE_RESP_STS_OK: 4958 if (resp->flags & QLNXR_RESP_IMM) { 4959 wc->ex.imm_data = 4960 le32_to_cpu(resp->imm_data_or_inv_r_Key); 4961 wc->wc_flags |= IB_WC_WITH_IMM; 4962 4963 if (resp->flags & QLNXR_RESP_RDMA) 4964 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4965 4966 if (resp->flags & QLNXR_RESP_INV) { 4967 QL_DPRINT11(ha, 4968 "Invalid flags QLNXR_RESP_INV [0x%x]" 4969 "qp = %p qp->id = 0x%x cq = %p" 4970 " cq->icid = 0x%x\n", 4971 resp->flags, qp, qp->id, cq, cq->icid ); 4972 } 4973 } else if (resp->flags & QLNXR_RESP_INV) { 4974 wc->ex.imm_data = 4975 le32_to_cpu(resp->imm_data_or_inv_r_Key); 4976 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4977 4978 if (resp->flags & QLNXR_RESP_RDMA) { 4979 QL_DPRINT11(ha, 4980 "Invalid flags QLNXR_RESP_RDMA [0x%x]" 4981 "qp = %p qp->id = 0x%x cq = %p" 4982 " cq->icid = 0x%x\n", 4983 resp->flags, qp, qp->id, cq, cq->icid ); 4984 } 4985 } else if (resp->flags & QLNXR_RESP_RDMA) { 4986 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 4987 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 4988 resp->flags, qp, qp->id, cq, cq->icid ); 4989 } 4990 break; 4991 4992 default: 4993 wc_status = IB_WC_GENERAL_ERR; 4994 } 4995 4996 /* fill WC */ 4997 wc->status = wc_status; 4998 wc->vendor_err = 0; 4999 wc->src_qp = qp->id; 5000 wc->qp = &qp->ibqp; 5001 wc->wr_id = wr_id; 5002 5003 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); 5004 5005 return; 5006 } 5007 5008 static int 5009 process_resp_one_srq(struct qlnxr_dev *dev, 5010 struct qlnxr_qp *qp, 5011 struct qlnxr_cq *cq, 5012 struct ib_wc *wc, 5013 struct rdma_cqe_responder *resp) 5014 { 5015 struct qlnxr_srq *srq = qp->srq; 5016 u64 wr_id; 5017 qlnx_host_t *ha = dev->ha; 5018 5019 QL_DPRINT12(ha, "enter\n"); 5020 5021 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); 5022 5023 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5024 wc->status = IB_WC_WR_FLUSH_ERR; 5025 wc->vendor_err = 0; 5026 wc->wr_id = wr_id; 5027 wc->byte_len = 0; 5028 wc->src_qp = qp->id; 5029 wc->qp = &qp->ibqp; 5030 wc->wr_id = wr_id; 5031 } else { 5032 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5033 } 5034 5035 /* PBL is maintained in case of WR granularity. 5036 * So increment WR consumer after consuming WR 5037 */ 5038 srq->hw_srq.wr_cons_cnt++; 5039 5040 QL_DPRINT12(ha, "exit\n"); 5041 return 1; 5042 } 5043 5044 static int 5045 process_resp_one(struct qlnxr_dev *dev, 5046 struct qlnxr_qp *qp, 5047 struct qlnxr_cq *cq, 5048 struct ib_wc *wc, 5049 struct rdma_cqe_responder *resp) 5050 { 5051 qlnx_host_t *ha = dev->ha; 5052 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5053 5054 QL_DPRINT12(ha, "enter\n"); 5055 5056 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5057 5058 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5059 ecore_chain_consume(&qp->rq.pbl); 5060 qlnxr_inc_sw_cons(&qp->rq); 5061 5062 QL_DPRINT12(ha, "exit\n"); 5063 return 1; 5064 } 5065 5066 static int 5067 process_resp_flush(struct qlnxr_qp *qp, 5068 int num_entries, 5069 struct ib_wc *wc, 5070 u16 hw_cons) 5071 { 5072 u16 cnt = 0; 5073 qlnx_host_t *ha = qp->dev->ha; 5074 5075 QL_DPRINT12(ha, "enter\n"); 5076 5077 while (num_entries && qp->rq.wqe_cons != hw_cons) { 5078 /* fill WC */ 5079 wc->status = IB_WC_WR_FLUSH_ERR; 5080 wc->vendor_err = 0; 5081 wc->wc_flags = 0; 5082 wc->src_qp = qp->id; 5083 wc->byte_len = 0; 5084 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5085 wc->qp = &qp->ibqp; 5086 num_entries--; 5087 wc++; 5088 cnt++; 5089 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5090 ecore_chain_consume(&qp->rq.pbl); 5091 qlnxr_inc_sw_cons(&qp->rq); 5092 } 5093 5094 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5095 return cnt; 5096 } 5097 5098 static void 5099 try_consume_resp_cqe(struct qlnxr_cq *cq, 5100 struct qlnxr_qp *qp, 5101 struct rdma_cqe_responder *resp, 5102 int *update) 5103 { 5104 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 5105 consume_cqe(cq); 5106 *update |= 1; 5107 } 5108 } 5109 5110 static int 5111 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, 5112 struct qlnxr_qp *qp, 5113 struct qlnxr_cq *cq, 5114 int num_entries, 5115 struct ib_wc *wc, 5116 struct rdma_cqe_responder *resp, 5117 int *update) 5118 { 5119 int cnt; 5120 qlnx_host_t *ha = dev->ha; 5121 5122 QL_DPRINT12(ha, "enter\n"); 5123 5124 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 5125 consume_cqe(cq); 5126 *update |= 1; 5127 5128 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5129 return cnt; 5130 } 5131 5132 static int 5133 qlnxr_poll_cq_resp(struct qlnxr_dev *dev, 5134 struct qlnxr_qp *qp, 5135 struct qlnxr_cq *cq, 5136 int num_entries, 5137 struct ib_wc *wc, 5138 struct rdma_cqe_responder *resp, 5139 int *update) 5140 { 5141 int cnt; 5142 qlnx_host_t *ha = dev->ha; 5143 5144 QL_DPRINT12(ha, "enter\n"); 5145 5146 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5147 cnt = process_resp_flush(qp, num_entries, wc, 5148 resp->rq_cons); 5149 try_consume_resp_cqe(cq, qp, resp, update); 5150 } else { 5151 cnt = process_resp_one(dev, qp, cq, wc, resp); 5152 consume_cqe(cq); 5153 *update |= 1; 5154 } 5155 5156 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5157 return cnt; 5158 } 5159 5160 static void 5161 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, 5162 struct rdma_cqe_requester *req, int *update) 5163 { 5164 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 5165 consume_cqe(cq); 5166 *update |= 1; 5167 } 5168 } 5169 5170 static void 5171 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) 5172 { 5173 uint64_t reg_addr; 5174 qlnx_host_t *ha = dev->ha; 5175 5176 QL_DPRINT12(ha, "enter\n"); 5177 5178 wmb(); 5179 cq->db.data.agg_flags = flags; 5180 cq->db.data.value = cpu_to_le32(cons); 5181 5182 reg_addr = (uint64_t)((uint8_t *)cq->db_addr - 5183 (uint8_t *)(ha->cdev.doorbells)); 5184 5185 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); 5186 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5187 5188 QL_DPRINT12(ha, "exit\n"); 5189 return; 5190 5191 //#ifdef __LP64__ 5192 // writeq(cq->db.raw, cq->db_addr); 5193 //#else 5194 /* Note that since the FW allows 64 bit write only, in 32bit systems 5195 * the value of db_addr must be low enough. This is currently not 5196 * enforced. 5197 */ 5198 // writel(cq->db.raw & 0xffffffff, cq->db_addr); 5199 // mmiowb(); 5200 //#endif 5201 } 5202 5203 static int 5204 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) 5205 { 5206 struct rdma_cqe_requester *resp_cqe = &cqe->req; 5207 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == 5208 cq->pbl_toggle; 5209 } 5210 5211 int 5212 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 5213 { 5214 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5215 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 5216 int done = 0; 5217 union rdma_cqe *cqe = cq->latest_cqe; 5218 int update = 0; 5219 u32 old_cons, new_cons; 5220 unsigned long flags; 5221 qlnx_host_t *ha = dev->ha; 5222 5223 QL_DPRINT12(ha, "enter\n"); 5224 5225 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 5226 return -EINVAL; 5227 5228 if (cq->destroyed) { 5229 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", 5230 cq, cq->icid); 5231 return 0; 5232 } 5233 5234 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) 5235 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); 5236 5237 spin_lock_irqsave(&cq->cq_lock, flags); 5238 5239 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5240 5241 while (num_entries && is_valid_cqe(cq, cqe)) { 5242 int cnt = 0; 5243 struct qlnxr_qp *qp; 5244 struct rdma_cqe_requester *resp_cqe; 5245 enum rdma_cqe_type cqe_type; 5246 5247 /* prevent speculative reads of any field of CQE */ 5248 rmb(); 5249 5250 resp_cqe = &cqe->req; 5251 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, 5252 resp_cqe->qp_handle.lo); 5253 5254 if (!qp) { 5255 QL_DPRINT11(ha, "qp = NULL\n"); 5256 break; 5257 } 5258 5259 wc->qp = &qp->ibqp; 5260 5261 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 5262 5263 switch (cqe_type) { 5264 case RDMA_CQE_TYPE_REQUESTER: 5265 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, 5266 wc, &cqe->req); 5267 try_consume_req_cqe(cq, qp, &cqe->req, &update); 5268 break; 5269 case RDMA_CQE_TYPE_RESPONDER_RQ: 5270 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, 5271 wc, &cqe->resp, &update); 5272 break; 5273 case RDMA_CQE_TYPE_RESPONDER_SRQ: 5274 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, 5275 wc, &cqe->resp, &update); 5276 break; 5277 case RDMA_CQE_TYPE_INVALID: 5278 default: 5279 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); 5280 break; 5281 } 5282 num_entries -= cnt; 5283 wc += cnt; 5284 done += cnt; 5285 5286 cqe = cq->latest_cqe; 5287 } 5288 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5289 5290 cq->cq_cons += new_cons - old_cons; 5291 5292 if (update) { 5293 /* doorbell notifies abount latest VALID entry, 5294 * but chain already point to the next INVALID one 5295 */ 5296 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); 5297 QL_DPRINT12(ha, "cq = %p cons = 0x%x " 5298 "arm_flags = 0x%x db.icid = 0x%x\n", cq, 5299 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); 5300 } 5301 5302 spin_unlock_irqrestore(&cq->cq_lock, flags); 5303 5304 QL_DPRINT12(ha, "exit\n"); 5305 5306 return done; 5307 } 5308 5309 int 5310 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 5311 { 5312 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5313 unsigned long sflags; 5314 struct qlnxr_dev *dev; 5315 qlnx_host_t *ha; 5316 5317 dev = get_qlnxr_dev((ibcq->device)); 5318 ha = dev->ha; 5319 5320 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " 5321 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, 5322 flags, cq, cq->cq_cons, cq->cq_type); 5323 5324 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 5325 return -EINVAL; 5326 5327 if (cq->destroyed) { 5328 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", 5329 cq, cq->icid); 5330 return -EINVAL; 5331 } 5332 5333 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { 5334 return 0; 5335 } 5336 5337 spin_lock_irqsave(&cq->cq_lock, sflags); 5338 5339 cq->arm_flags = 0; 5340 5341 if (flags & IB_CQ_SOLICITED) { 5342 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 5343 } 5344 if (flags & IB_CQ_NEXT_COMP) { 5345 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 5346 } 5347 5348 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); 5349 5350 spin_unlock_irqrestore(&cq->cq_lock, sflags); 5351 5352 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); 5353 return 0; 5354 } 5355 5356 static struct qlnxr_mr * 5357 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) 5358 { 5359 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 5360 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 5361 struct qlnxr_mr *mr; 5362 int rc = -ENOMEM; 5363 qlnx_host_t *ha; 5364 5365 ha = dev->ha; 5366 5367 QL_DPRINT12(ha, "enter ibpd = %p pd = %p " 5368 " pd_id = %d max_page_list_len = %d\n", 5369 ibpd, pd, pd->pd_id, max_page_list_len); 5370 5371 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5372 if (!mr) { 5373 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 5374 return ERR_PTR(rc); 5375 } 5376 5377 mr->dev = dev; 5378 mr->type = QLNXR_MR_FRMR; 5379 5380 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 5381 1 /* allow dual layer pbl */); 5382 if (rc) { 5383 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); 5384 goto err0; 5385 } 5386 5387 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 5388 if (rc) { 5389 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); 5390 goto err0; 5391 } 5392 5393 /* index only, 18 bit long, lkey = itid << 8 | key */ 5394 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; 5395 mr->hw_mr.key = 0; 5396 mr->hw_mr.pd = pd->pd_id; 5397 mr->hw_mr.local_read = 1; 5398 mr->hw_mr.local_write = 0; 5399 mr->hw_mr.remote_read = 0; 5400 mr->hw_mr.remote_write = 0; 5401 mr->hw_mr.remote_atomic = 0; 5402 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 5403 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ 5404 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 5405 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 5406 mr->hw_mr.fbo = 0; 5407 mr->hw_mr.length = 0; 5408 mr->hw_mr.vaddr = 0; 5409 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 5410 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ 5411 mr->hw_mr.dma_mr = false; 5412 5413 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 5414 if (rc) { 5415 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); 5416 goto err1; 5417 } 5418 5419 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 5420 mr->ibmr.rkey = mr->ibmr.lkey; 5421 5422 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", 5423 mr, mr->ibmr.lkey); 5424 5425 return mr; 5426 5427 err1: 5428 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 5429 err0: 5430 kfree(mr); 5431 5432 QL_DPRINT12(ha, "exit\n"); 5433 5434 return ERR_PTR(rc); 5435 } 5436 5437 struct ib_mr * 5438 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 5439 u32 max_num_sg, struct ib_udata *udata) 5440 { 5441 struct qlnxr_dev *dev; 5442 struct qlnxr_mr *mr; 5443 qlnx_host_t *ha; 5444 5445 dev = get_qlnxr_dev(ibpd->device); 5446 ha = dev->ha; 5447 5448 QL_DPRINT12(ha, "enter\n"); 5449 5450 if (mr_type != IB_MR_TYPE_MEM_REG) 5451 return ERR_PTR(-EINVAL); 5452 5453 mr = __qlnxr_alloc_mr(ibpd, max_num_sg); 5454 5455 if (IS_ERR(mr)) 5456 return ERR_PTR(-EINVAL); 5457 5458 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); 5459 5460 return &mr->ibmr; 5461 } 5462 5463 static int 5464 qlnxr_set_page(struct ib_mr *ibmr, u64 addr) 5465 { 5466 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5467 struct qlnxr_pbl *pbl_table; 5468 struct regpair *pbe; 5469 struct qlnxr_dev *dev; 5470 qlnx_host_t *ha; 5471 u32 pbes_in_page; 5472 5473 dev = mr->dev; 5474 ha = dev->ha; 5475 5476 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 5477 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); 5478 return -ENOMEM; 5479 } 5480 5481 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, 5482 ((void *)addr)); 5483 5484 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 5485 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 5486 pbe = (struct regpair *)pbl_table->va; 5487 pbe += mr->npages % pbes_in_page; 5488 pbe->lo = cpu_to_le32((u32)addr); 5489 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 5490 5491 mr->npages++; 5492 5493 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, 5494 ((void *)addr)); 5495 return 0; 5496 } 5497 5498 int 5499 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 5500 int sg_nents, unsigned int *sg_offset) 5501 { 5502 int ret; 5503 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5504 qlnx_host_t *ha; 5505 5506 if (mr == NULL) 5507 return (-1); 5508 5509 if (mr->dev == NULL) 5510 return (-1); 5511 5512 ha = mr->dev->ha; 5513 5514 QL_DPRINT12(ha, "enter\n"); 5515 5516 mr->npages = 0; 5517 qlnx_handle_completed_mrs(mr->dev, &mr->info); 5518 5519 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); 5520 5521 QL_DPRINT12(ha, "exit ret = %d\n", ret); 5522 5523 return (ret); 5524 } 5525 5526 int 5527 qlnxr_create_ah(struct ib_ah *ibah, 5528 struct ib_ah_attr *attr, u32 flags, 5529 struct ib_udata *udata) 5530 { 5531 struct qlnxr_dev *dev; 5532 qlnx_host_t *ha; 5533 struct qlnxr_ah *ah = get_qlnxr_ah(ibah); 5534 5535 dev = get_qlnxr_dev(ibah->device); 5536 ha = dev->ha; 5537 5538 QL_DPRINT12(ha, "in create_ah\n"); 5539 5540 ah->attr = *attr; 5541 5542 return (0); 5543 } 5544 5545 void 5546 qlnxr_destroy_ah(struct ib_ah *ibah, u32 flags) 5547 { 5548 } 5549 5550 int 5551 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 5552 { 5553 struct qlnxr_dev *dev; 5554 qlnx_host_t *ha; 5555 5556 dev = get_qlnxr_dev((ibah->device)); 5557 ha = dev->ha; 5558 QL_DPRINT12(ha, "Query AH not supported\n"); 5559 return -EINVAL; 5560 } 5561 5562 int 5563 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 5564 { 5565 struct qlnxr_dev *dev; 5566 qlnx_host_t *ha; 5567 5568 dev = get_qlnxr_dev((ibah->device)); 5569 ha = dev->ha; 5570 QL_DPRINT12(ha, "Modify AH not supported\n"); 5571 return -ENOSYS; 5572 } 5573 5574 int 5575 qlnxr_process_mad(struct ib_device *ibdev, 5576 int process_mad_flags, 5577 u8 port_num, 5578 const struct ib_wc *in_wc, 5579 const struct ib_grh *in_grh, 5580 const struct ib_mad_hdr *mad_hdr, 5581 size_t in_mad_size, 5582 struct ib_mad_hdr *out_mad, 5583 size_t *out_mad_size, 5584 u16 *out_mad_pkey_index) 5585 { 5586 struct qlnxr_dev *dev; 5587 qlnx_host_t *ha; 5588 5589 dev = get_qlnxr_dev(ibdev); 5590 ha = dev->ha; 5591 QL_DPRINT12(ha, "process mad not supported\n"); 5592 5593 return -ENOSYS; 5594 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", 5595 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, 5596 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, 5597 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, 5598 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); 5599 5600 // return IB_MAD_RESULT_SUCCESS; 5601 } 5602 5603 int 5604 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, 5605 struct ib_port_immutable *immutable) 5606 { 5607 struct qlnxr_dev *dev; 5608 qlnx_host_t *ha; 5609 struct ib_port_attr attr; 5610 int err; 5611 5612 dev = get_qlnxr_dev(ibdev); 5613 ha = dev->ha; 5614 5615 QL_DPRINT12(ha, "enter\n"); 5616 5617 err = qlnxr_query_port(ibdev, port_num, &attr); 5618 if (err) 5619 return err; 5620 5621 if (QLNX_IS_IWARP(dev)) { 5622 immutable->pkey_tbl_len = 1; 5623 immutable->gid_tbl_len = 1; 5624 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 5625 immutable->max_mad_size = 0; 5626 } else { 5627 immutable->pkey_tbl_len = attr.pkey_tbl_len; 5628 immutable->gid_tbl_len = attr.gid_tbl_len; 5629 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 5630 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 5631 } 5632 5633 QL_DPRINT12(ha, "exit\n"); 5634 return 0; 5635 } 5636 5637 /***** iWARP related functions *************/ 5638 5639 static void 5640 qlnxr_iw_mpa_request(void *context, 5641 struct ecore_iwarp_cm_event_params *params) 5642 { 5643 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; 5644 struct qlnxr_dev *dev = listener->dev; 5645 struct qlnxr_iw_ep *ep; 5646 struct iw_cm_event event; 5647 struct sockaddr_in *laddr; 5648 struct sockaddr_in *raddr; 5649 qlnx_host_t *ha; 5650 5651 ha = dev->ha; 5652 5653 QL_DPRINT12(ha, "enter\n"); 5654 5655 if (params->cm_info->ip_version != ECORE_TCP_IPV4) { 5656 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", 5657 params->cm_info->ip_version); 5658 return; 5659 } 5660 5661 ep = kzalloc(sizeof(*ep), GFP_ATOMIC); 5662 5663 if (!ep) { 5664 QL_DPRINT11(ha, "kzalloc{ep) failed\n"); 5665 return; 5666 } 5667 5668 ep->dev = dev; 5669 ep->ecore_context = params->ep_context; 5670 5671 memset(&event, 0, sizeof(event)); 5672 5673 event.event = IW_CM_EVENT_CONNECT_REQUEST; 5674 event.status = params->status; 5675 5676 laddr = (struct sockaddr_in *)&event.local_addr; 5677 raddr = (struct sockaddr_in *)&event.remote_addr; 5678 5679 laddr->sin_family = AF_INET; 5680 raddr->sin_family = AF_INET; 5681 5682 laddr->sin_port = htons(params->cm_info->local_port); 5683 raddr->sin_port = htons(params->cm_info->remote_port); 5684 5685 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); 5686 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); 5687 5688 event.provider_data = (void *)ep; 5689 event.private_data = (void *)params->cm_info->private_data; 5690 event.private_data_len = (u8)params->cm_info->private_data_len; 5691 5692 event.ord = params->cm_info->ord; 5693 event.ird = params->cm_info->ird; 5694 5695 listener->cm_id->event_handler(listener->cm_id, &event); 5696 5697 QL_DPRINT12(ha, "exit\n"); 5698 5699 return; 5700 } 5701 5702 static void 5703 qlnxr_iw_issue_event(void *context, 5704 struct ecore_iwarp_cm_event_params *params, 5705 enum iw_cm_event_type event_type, 5706 char *str) 5707 { 5708 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5709 struct qlnxr_dev *dev = ep->dev; 5710 struct iw_cm_event event; 5711 qlnx_host_t *ha; 5712 5713 ha = dev->ha; 5714 5715 QL_DPRINT12(ha, "enter\n"); 5716 5717 memset(&event, 0, sizeof(event)); 5718 event.status = params->status; 5719 event.event = event_type; 5720 5721 if (params->cm_info != NULL) { 5722 event.ird = params->cm_info->ird; 5723 event.ord = params->cm_info->ord; 5724 QL_DPRINT12(ha, "ord=[%d] \n", event.ord); 5725 QL_DPRINT12(ha, "ird=[%d] \n", event.ird); 5726 5727 event.private_data_len = params->cm_info->private_data_len; 5728 event.private_data = (void *)params->cm_info->private_data; 5729 QL_DPRINT12(ha, "private_data_len=[%d] \n", 5730 event.private_data_len); 5731 } 5732 5733 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); 5734 QL_DPRINT12(ha, "status=[%d] \n", event.status); 5735 5736 if (ep) { 5737 if (ep->cm_id) 5738 ep->cm_id->event_handler(ep->cm_id, &event); 5739 else 5740 QL_DPRINT11(ha, "ep->cm_id == NULL \n"); 5741 } else { 5742 QL_DPRINT11(ha, "ep == NULL \n"); 5743 } 5744 5745 QL_DPRINT12(ha, "exit\n"); 5746 5747 return; 5748 } 5749 5750 static void 5751 qlnxr_iw_close_event(void *context, 5752 struct ecore_iwarp_cm_event_params *params) 5753 { 5754 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5755 struct qlnxr_dev *dev = ep->dev; 5756 qlnx_host_t *ha; 5757 5758 ha = dev->ha; 5759 5760 QL_DPRINT12(ha, "enter\n"); 5761 5762 if (ep->cm_id) { 5763 qlnxr_iw_issue_event(context, 5764 params, 5765 IW_CM_EVENT_CLOSE, 5766 "IW_CM_EVENT_EVENT_CLOSE"); 5767 ep->cm_id->rem_ref(ep->cm_id); 5768 ep->cm_id = NULL; 5769 } 5770 5771 QL_DPRINT12(ha, "exit\n"); 5772 5773 return; 5774 } 5775 5776 static void 5777 qlnxr_iw_passive_complete(void *context, 5778 struct ecore_iwarp_cm_event_params *params) 5779 { 5780 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5781 struct qlnxr_dev *dev = ep->dev; 5782 qlnx_host_t *ha; 5783 5784 ha = dev->ha; 5785 5786 /* We will only reach the following state if MPA_REJECT was called on 5787 * passive. In this case there will be no associated QP. 5788 */ 5789 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { 5790 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); 5791 kfree(ep); 5792 return; 5793 } 5794 5795 /* We always issue an established event, however, ofed does not look 5796 * at event code for established. So if there was a failure, we follow 5797 * with close... 5798 */ 5799 qlnxr_iw_issue_event(context, 5800 params, 5801 IW_CM_EVENT_ESTABLISHED, 5802 "IW_CM_EVENT_ESTABLISHED"); 5803 5804 if (params->status < 0) { 5805 qlnxr_iw_close_event(context, params); 5806 } 5807 5808 return; 5809 } 5810 5811 struct qlnxr_discon_work { 5812 struct work_struct work; 5813 struct qlnxr_iw_ep *ep; 5814 enum ecore_iwarp_event_type event; 5815 int status; 5816 }; 5817 5818 static void 5819 qlnxr_iw_disconnect_worker(struct work_struct *work) 5820 { 5821 struct qlnxr_discon_work *dwork = 5822 container_of(work, struct qlnxr_discon_work, work); 5823 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 5824 struct qlnxr_iw_ep *ep = dwork->ep; 5825 struct qlnxr_dev *dev = ep->dev; 5826 struct qlnxr_qp *qp = ep->qp; 5827 struct iw_cm_event event; 5828 5829 if (qp->destroyed) { 5830 kfree(dwork); 5831 qlnxr_iw_qp_rem_ref(&qp->ibqp); 5832 return; 5833 } 5834 5835 memset(&event, 0, sizeof(event)); 5836 event.status = dwork->status; 5837 event.event = IW_CM_EVENT_DISCONNECT; 5838 5839 /* Success means graceful disconnect was requested. modifying 5840 * to SQD is translated to graceful disconnect. O/w reset is sent 5841 */ 5842 if (dwork->status) 5843 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; 5844 else 5845 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; 5846 5847 kfree(dwork); 5848 5849 if (ep->cm_id) 5850 ep->cm_id->event_handler(ep->cm_id, &event); 5851 5852 SET_FIELD(qp_params.modify_flags, 5853 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 5854 5855 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 5856 5857 qlnxr_iw_qp_rem_ref(&qp->ibqp); 5858 5859 return; 5860 } 5861 5862 void 5863 qlnxr_iw_disconnect_event(void *context, 5864 struct ecore_iwarp_cm_event_params *params) 5865 { 5866 struct qlnxr_discon_work *work; 5867 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5868 struct qlnxr_dev *dev = ep->dev; 5869 struct qlnxr_qp *qp = ep->qp; 5870 5871 work = kzalloc(sizeof(*work), GFP_ATOMIC); 5872 if (!work) 5873 return; 5874 5875 qlnxr_iw_qp_add_ref(&qp->ibqp); 5876 work->ep = ep; 5877 work->event = params->event; 5878 work->status = params->status; 5879 5880 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); 5881 queue_work(dev->iwarp_wq, &work->work); 5882 5883 return; 5884 } 5885 5886 static int 5887 qlnxr_iw_mpa_reply(void *context, 5888 struct ecore_iwarp_cm_event_params *params) 5889 { 5890 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5891 struct qlnxr_dev *dev = ep->dev; 5892 struct ecore_iwarp_send_rtr_in rtr_in; 5893 int rc; 5894 qlnx_host_t *ha; 5895 5896 ha = dev->ha; 5897 5898 QL_DPRINT12(ha, "enter\n"); 5899 5900 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 5901 return -EINVAL; 5902 5903 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); 5904 rtr_in.ep_context = params->ep_context; 5905 5906 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); 5907 5908 QL_DPRINT12(ha, "exit rc = %d\n", rc); 5909 return rc; 5910 } 5911 5912 void 5913 qlnxr_iw_qp_event(void *context, 5914 struct ecore_iwarp_cm_event_params *params, 5915 enum ib_event_type ib_event, 5916 char *str) 5917 { 5918 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5919 struct qlnxr_dev *dev = ep->dev; 5920 struct ib_qp *ibqp = &(ep->qp->ibqp); 5921 struct ib_event event; 5922 qlnx_host_t *ha; 5923 5924 ha = dev->ha; 5925 5926 QL_DPRINT12(ha, 5927 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", 5928 context, params->event, str, ibqp->event_handler); 5929 5930 if (ibqp->event_handler) { 5931 event.event = ib_event; 5932 event.device = ibqp->device; 5933 event.element.qp = ibqp; 5934 ibqp->event_handler(&event, ibqp->qp_context); 5935 } 5936 5937 return; 5938 } 5939 5940 int 5941 qlnxr_iw_event_handler(void *context, 5942 struct ecore_iwarp_cm_event_params *params) 5943 { 5944 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5945 struct qlnxr_dev *dev = ep->dev; 5946 qlnx_host_t *ha; 5947 5948 ha = dev->ha; 5949 5950 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 5951 "enter\n", context, params->event); 5952 5953 switch (params->event) { 5954 /* Passive side request received */ 5955 case ECORE_IWARP_EVENT_MPA_REQUEST: 5956 qlnxr_iw_mpa_request(context, params); 5957 break; 5958 5959 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: 5960 qlnxr_iw_mpa_reply(context, params); 5961 break; 5962 5963 /* Passive side established ( ack on mpa response ) */ 5964 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: 5965 ep->during_connect = 0; 5966 qlnxr_iw_passive_complete(context, params); 5967 break; 5968 5969 /* Active side reply received */ 5970 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: 5971 ep->during_connect = 0; 5972 qlnxr_iw_issue_event(context, 5973 params, 5974 IW_CM_EVENT_CONNECT_REPLY, 5975 "IW_CM_EVENT_CONNECT_REPLY"); 5976 if (params->status < 0) { 5977 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5978 5979 ep->cm_id->rem_ref(ep->cm_id); 5980 ep->cm_id = NULL; 5981 } 5982 break; 5983 5984 case ECORE_IWARP_EVENT_DISCONNECT: 5985 qlnxr_iw_disconnect_event(context, params); 5986 break; 5987 5988 case ECORE_IWARP_EVENT_CLOSE: 5989 ep->during_connect = 0; 5990 qlnxr_iw_close_event(context, params); 5991 break; 5992 5993 case ECORE_IWARP_EVENT_RQ_EMPTY: 5994 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 5995 "IWARP_EVENT_RQ_EMPTY"); 5996 break; 5997 5998 case ECORE_IWARP_EVENT_IRQ_FULL: 5999 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6000 "IWARP_EVENT_IRQ_FULL"); 6001 break; 6002 6003 case ECORE_IWARP_EVENT_LLP_TIMEOUT: 6004 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6005 "IWARP_EVENT_LLP_TIMEOUT"); 6006 break; 6007 6008 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: 6009 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6010 "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); 6011 break; 6012 6013 case ECORE_IWARP_EVENT_CQ_OVERFLOW: 6014 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6015 "QED_IWARP_EVENT_CQ_OVERFLOW"); 6016 break; 6017 6018 case ECORE_IWARP_EVENT_QP_CATASTROPHIC: 6019 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6020 "QED_IWARP_EVENT_QP_CATASTROPHIC"); 6021 break; 6022 6023 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: 6024 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6025 "IWARP_EVENT_LOCAL_ACCESS_ERROR"); 6026 break; 6027 6028 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: 6029 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6030 "IWARP_EVENT_REMOTE_OPERATION_ERROR"); 6031 break; 6032 6033 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: 6034 QL_DPRINT12(ha, "Got terminate message" 6035 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); 6036 break; 6037 6038 default: 6039 QL_DPRINT12(ha, 6040 "Unknown event [0x%x] received \n", params->event); 6041 break; 6042 }; 6043 6044 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6045 "exit\n", context, params->event); 6046 return 0; 6047 } 6048 6049 static int 6050 qlnxr_addr4_resolve(struct qlnxr_dev *dev, 6051 struct sockaddr_in *src_in, 6052 struct sockaddr_in *dst_in, 6053 u8 *dst_mac) 6054 { 6055 int rc; 6056 6057 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, 6058 dst_mac, NULL, NULL); 6059 6060 QL_DPRINT12(dev->ha, "rc = %d " 6061 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " 6062 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, 6063 dst_in->sin_len, dst_in->sin_family, 6064 NIPQUAD((dst_in->sin_addr.s_addr)), 6065 dst_mac[0], dst_mac[1], dst_mac[2], 6066 dst_mac[3], dst_mac[4], dst_mac[5]); 6067 6068 return rc; 6069 } 6070 6071 int 6072 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 6073 { 6074 struct qlnxr_dev *dev; 6075 struct ecore_iwarp_connect_out out_params; 6076 struct ecore_iwarp_connect_in in_params; 6077 struct qlnxr_iw_ep *ep; 6078 struct qlnxr_qp *qp; 6079 struct sockaddr_in *laddr; 6080 struct sockaddr_in *raddr; 6081 int rc = 0; 6082 qlnx_host_t *ha; 6083 6084 dev = get_qlnxr_dev((cm_id->device)); 6085 ha = dev->ha; 6086 6087 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " 6088 "enter \n", cm_id, conn_param); 6089 6090 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 6091 return -EINVAL; 6092 6093 qp = idr_find(&dev->qpidr, conn_param->qpn); 6094 6095 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6096 raddr = (struct sockaddr_in *)&cm_id->remote_addr; 6097 6098 QL_DPRINT12(ha, 6099 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", 6100 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, 6101 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); 6102 6103 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 6104 if (!ep) { 6105 QL_DPRINT11(ha, "struct qlnxr_iw_ep " 6106 "alloc memory failed\n"); 6107 return -ENOMEM; 6108 } 6109 6110 ep->dev = dev; 6111 ep->qp = qp; 6112 cm_id->add_ref(cm_id); 6113 ep->cm_id = cm_id; 6114 6115 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); 6116 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); 6117 6118 in_params.event_cb = qlnxr_iw_event_handler; 6119 in_params.cb_context = ep; 6120 6121 in_params.cm_info.ip_version = ECORE_TCP_IPV4; 6122 6123 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); 6124 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); 6125 in_params.cm_info.remote_port = ntohs(raddr->sin_port); 6126 in_params.cm_info.local_port = ntohs(laddr->sin_port); 6127 in_params.cm_info.vlan = 0; 6128 in_params.mss = if_getmtu(dev->ha->ifp) - 40; 6129 6130 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " 6131 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " 6132 "vlan = %d\n", 6133 NIPQUAD((in_params.cm_info.remote_ip[0])), 6134 NIPQUAD((in_params.cm_info.local_ip[0])), 6135 in_params.cm_info.remote_port, in_params.cm_info.local_port, 6136 in_params.cm_info.vlan); 6137 6138 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); 6139 6140 if (rc) { 6141 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); 6142 goto err; 6143 } 6144 6145 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" 6146 " private_data_len=%d rq_psn=%d\n", 6147 conn_param->ord, conn_param->ird, conn_param->private_data, 6148 conn_param->private_data_len, qp->rq_psn); 6149 6150 in_params.cm_info.ord = conn_param->ord; 6151 in_params.cm_info.ird = conn_param->ird; 6152 in_params.cm_info.private_data = conn_param->private_data; 6153 in_params.cm_info.private_data_len = conn_param->private_data_len; 6154 in_params.qp = qp->ecore_qp; 6155 6156 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 6157 6158 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); 6159 6160 if (rc) { 6161 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); 6162 goto err; 6163 } 6164 6165 QL_DPRINT12(ha, "exit\n"); 6166 6167 return rc; 6168 6169 err: 6170 cm_id->rem_ref(cm_id); 6171 kfree(ep); 6172 6173 QL_DPRINT12(ha, "exit [%d]\n", rc); 6174 return rc; 6175 } 6176 6177 int 6178 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) 6179 { 6180 struct qlnxr_dev *dev; 6181 struct qlnxr_iw_listener *listener; 6182 struct ecore_iwarp_listen_in iparams; 6183 struct ecore_iwarp_listen_out oparams; 6184 struct sockaddr_in *laddr; 6185 qlnx_host_t *ha; 6186 int rc; 6187 6188 dev = get_qlnxr_dev((cm_id->device)); 6189 ha = dev->ha; 6190 6191 QL_DPRINT12(ha, "enter\n"); 6192 6193 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 6194 return -EINVAL; 6195 6196 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6197 6198 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 6199 6200 if (listener == NULL) { 6201 QL_DPRINT11(ha, "listener memory alloc failed\n"); 6202 return -ENOMEM; 6203 } 6204 6205 listener->dev = dev; 6206 cm_id->add_ref(cm_id); 6207 listener->cm_id = cm_id; 6208 listener->backlog = backlog; 6209 6210 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); 6211 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); 6212 6213 iparams.cb_context = listener; 6214 iparams.event_cb = qlnxr_iw_event_handler; 6215 iparams.max_backlog = backlog; 6216 6217 iparams.ip_version = ECORE_TCP_IPV4; 6218 6219 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); 6220 iparams.port = ntohs(laddr->sin_port); 6221 iparams.vlan = 0; 6222 6223 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", 6224 NIPQUAD((laddr->sin_addr.s_addr)), 6225 laddr->sin_port, iparams.port); 6226 6227 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); 6228 if (rc) { 6229 QL_DPRINT11(ha, 6230 "ecore_iwarp_create_listen failed rc = %d\n", rc); 6231 goto err; 6232 } 6233 6234 listener->ecore_handle = oparams.handle; 6235 cm_id->provider_data = listener; 6236 6237 QL_DPRINT12(ha, "exit\n"); 6238 return rc; 6239 6240 err: 6241 cm_id->rem_ref(cm_id); 6242 kfree(listener); 6243 6244 QL_DPRINT12(ha, "exit [%d]\n", rc); 6245 return rc; 6246 } 6247 6248 void 6249 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) 6250 { 6251 struct qlnxr_iw_listener *listener = cm_id->provider_data; 6252 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); 6253 int rc = 0; 6254 qlnx_host_t *ha; 6255 6256 ha = dev->ha; 6257 6258 QL_DPRINT12(ha, "enter\n"); 6259 6260 if (listener->ecore_handle) 6261 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, 6262 listener->ecore_handle); 6263 6264 cm_id->rem_ref(cm_id); 6265 6266 QL_DPRINT12(ha, "exit [%d]\n", rc); 6267 return; 6268 } 6269 6270 int 6271 qlnxr_iw_accept(struct iw_cm_id *cm_id, 6272 struct iw_cm_conn_param *conn_param) 6273 { 6274 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 6275 struct qlnxr_dev *dev = ep->dev; 6276 struct qlnxr_qp *qp; 6277 struct ecore_iwarp_accept_in params; 6278 int rc; 6279 qlnx_host_t *ha; 6280 6281 ha = dev->ha; 6282 6283 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); 6284 6285 if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) 6286 return -EINVAL; 6287 6288 qp = idr_find(&dev->qpidr, conn_param->qpn); 6289 if (!qp) { 6290 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", 6291 conn_param->qpn); 6292 return -EINVAL; 6293 } 6294 ep->qp = qp; 6295 qp->ep = ep; 6296 cm_id->add_ref(cm_id); 6297 ep->cm_id = cm_id; 6298 6299 params.ep_context = ep->ecore_context; 6300 params.cb_context = ep; 6301 params.qp = ep->qp->ecore_qp; 6302 params.private_data = conn_param->private_data; 6303 params.private_data_len = conn_param->private_data_len; 6304 params.ird = conn_param->ird; 6305 params.ord = conn_param->ord; 6306 6307 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); 6308 if (rc) { 6309 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); 6310 goto err; 6311 } 6312 6313 QL_DPRINT12(ha, "exit\n"); 6314 return 0; 6315 err: 6316 cm_id->rem_ref(cm_id); 6317 QL_DPRINT12(ha, "exit rc = %d\n", rc); 6318 return rc; 6319 } 6320 6321 int 6322 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 6323 { 6324 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 6325 struct qlnxr_dev *dev = ep->dev; 6326 struct ecore_iwarp_reject_in params; 6327 int rc; 6328 6329 params.ep_context = ep->ecore_context; 6330 params.cb_context = ep; 6331 params.private_data = pdata; 6332 params.private_data_len = pdata_len; 6333 ep->qp = NULL; 6334 6335 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); 6336 6337 return rc; 6338 } 6339 6340 void 6341 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) 6342 { 6343 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 6344 qlnx_host_t *ha; 6345 6346 ha = qp->dev->ha; 6347 6348 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); 6349 6350 atomic_inc(&qp->refcnt); 6351 6352 QL_DPRINT12(ha, "exit \n"); 6353 return; 6354 } 6355 6356 void 6357 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) 6358 { 6359 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 6360 qlnx_host_t *ha; 6361 6362 ha = qp->dev->ha; 6363 6364 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); 6365 6366 if (atomic_dec_and_test(&qp->refcnt)) { 6367 qlnxr_idr_remove(qp->dev, qp->qp_id); 6368 } 6369 6370 QL_DPRINT12(ha, "exit \n"); 6371 return; 6372 } 6373 6374 struct ib_qp * 6375 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) 6376 { 6377 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 6378 struct ib_qp *qp; 6379 qlnx_host_t *ha; 6380 6381 ha = dev->ha; 6382 6383 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); 6384 6385 qp = idr_find(&dev->qpidr, qpn); 6386 6387 QL_DPRINT12(ha, "exit qp = %p\n", qp); 6388 6389 return (qp); 6390 } 6391