1 /* 2 * Copyright (c) 2018-2019 Cavium, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * File: qlnxr_verbs.c 30 */ 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "qlnxr_def.h" 35 #include "rdma_common.h" 36 #include "qlnxr_roce.h" 37 #include "qlnxr_cm.h" 38 39 #define upper_32_bits(x) (uint32_t)(x >> 32) 40 #define lower_32_bits(x) (uint32_t)(x) 41 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 42 43 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ 44 do { \ 45 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ 46 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ 47 } while (0) 48 49 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 50 do { \ 51 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 52 (sge)->length = cpu_to_le32(vlength); \ 53 (sge)->flags = cpu_to_le32(vflags); \ 54 } while (0) 55 56 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 57 do { \ 58 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ 59 (hdr)->num_sges = num_sge; \ 60 } while (0) 61 62 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 63 do { \ 64 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 65 (sge)->length = cpu_to_le32(vlength); \ 66 (sge)->l_key = cpu_to_le32(vlkey); \ 67 } while (0) 68 69 #define NIPQUAD(addr) \ 70 ((unsigned char *)&addr)[0], \ 71 ((unsigned char *)&addr)[1], \ 72 ((unsigned char *)&addr)[2], \ 73 ((unsigned char *)&addr)[3] 74 75 static int 76 qlnxr_check_srq_params(struct ib_pd *ibpd, 77 struct qlnxr_dev *dev, 78 struct ib_srq_init_attr *attrs); 79 80 static int 81 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 82 struct qlnxr_srq *srq, 83 struct qlnxr_create_srq_ureq *ureq, 84 int access, int dmasync); 85 86 static int 87 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 88 struct qlnxr_dev *dev, 89 struct ib_srq_init_attr *init_attr); 90 91 static int 92 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 93 struct qlnxr_srq *srq, 94 struct ib_udata *udata); 95 96 static void 97 qlnxr_free_srq_user_params(struct qlnxr_srq *srq); 98 99 static void 100 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); 101 102 static u32 103 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); 104 105 int 106 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, 107 union ib_gid *sgid) 108 { 109 struct qlnxr_dev *dev; 110 qlnx_host_t *ha; 111 112 dev = get_qlnxr_dev(ibdev); 113 ha = dev->ha; 114 115 QL_DPRINT12(ha, "enter\n"); 116 117 memset(sgid->raw, 0, sizeof(sgid->raw)); 118 119 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); 120 121 QL_DPRINT12(ha, "exit\n"); 122 123 return 0; 124 } 125 126 int 127 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, 128 union ib_gid *sgid) 129 { 130 struct qlnxr_dev *dev; 131 qlnx_host_t *ha; 132 133 dev = get_qlnxr_dev(ibdev); 134 ha = dev->ha; 135 QL_DPRINT12(ha, "enter index: %d\n", index); 136 #if 0 137 int ret = 0; 138 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ 139 //if (!rdma_cap_roce_gid_table(ibdev, port)) { 140 if (!(rdma_protocol_roce(ibdev, port) && 141 ibdev->add_gid && ibdev->del_gid)) { 142 QL_DPRINT11(ha, "acquire gid failed\n"); 143 return -ENODEV; 144 } 145 146 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 147 if (ret == -EAGAIN) { 148 memcpy(sgid, &zgid, sizeof(*sgid)); 149 return 0; 150 } 151 #endif 152 if ((index >= QLNXR_MAX_SGID) || (index < 0)) { 153 QL_DPRINT12(ha, "invalid gid index %d\n", index); 154 memset(sgid, 0, sizeof(*sgid)); 155 return -EINVAL; 156 } 157 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); 158 159 QL_DPRINT12(ha, "exit : %p\n", sgid); 160 161 return 0; 162 } 163 164 struct ib_srq * 165 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr, 166 struct ib_udata *udata) 167 { 168 struct qlnxr_dev *dev; 169 qlnx_host_t *ha; 170 struct ecore_rdma_destroy_srq_in_params destroy_in_params; 171 struct ecore_rdma_create_srq_out_params out_params; 172 struct ecore_rdma_create_srq_in_params in_params; 173 u64 pbl_base_addr, phy_prod_pair_addr; 174 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 175 struct ib_ucontext *ib_ctx = NULL; 176 struct qlnxr_srq_hwq_info *hw_srq; 177 struct qlnxr_ucontext *ctx = NULL; 178 struct qlnxr_create_srq_ureq ureq; 179 u32 page_cnt, page_size; 180 struct qlnxr_srq *srq; 181 int ret = 0; 182 183 dev = get_qlnxr_dev((ibpd->device)); 184 ha = dev->ha; 185 186 QL_DPRINT12(ha, "enter\n"); 187 188 ret = qlnxr_check_srq_params(ibpd, dev, init_attr); 189 190 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 191 if (!srq) { 192 QL_DPRINT11(ha, "cannot allocate memory for srq\n"); 193 return NULL; //@@@ : TODO what to return here? 194 } 195 196 srq->dev = dev; 197 hw_srq = &srq->hw_srq; 198 spin_lock_init(&srq->lock); 199 memset(&in_params, 0, sizeof(in_params)); 200 201 if (udata && ibpd->uobject && ibpd->uobject->context) { 202 ib_ctx = ibpd->uobject->context; 203 ctx = get_qlnxr_ucontext(ib_ctx); 204 205 memset(&ureq, 0, sizeof(ureq)); 206 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 207 udata->inlen))) { 208 QL_DPRINT11(ha, "problem" 209 " copying data from user space\n"); 210 goto err0; 211 } 212 213 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0); 214 if (ret) 215 goto err0; 216 217 page_cnt = srq->usrq.pbl_info.num_pbes; 218 pbl_base_addr = srq->usrq.pbl_tbl->pa; 219 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 220 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT 221 // page_size = BIT(srq->usrq.umem->page_shift); 222 // else 223 page_size = srq->usrq.umem->page_size; 224 } else { 225 struct ecore_chain *pbl; 226 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); 227 if (ret) 228 goto err0; 229 pbl = &hw_srq->pbl; 230 231 page_cnt = ecore_chain_get_page_cnt(pbl); 232 pbl_base_addr = ecore_chain_get_pbl_phys(pbl); 233 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 234 page_size = pbl->elem_per_page << 4; 235 } 236 237 in_params.pd_id = pd->pd_id; 238 in_params.pbl_base_addr = pbl_base_addr; 239 in_params.prod_pair_addr = phy_prod_pair_addr; 240 in_params.num_pages = page_cnt; 241 in_params.page_size = page_size; 242 243 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 244 if (ret) 245 goto err1; 246 247 srq->srq_id = out_params.srq_id; 248 249 if (udata) { 250 ret = qlnxr_copy_srq_uresp(dev, srq, udata); 251 if (ret) 252 goto err2; 253 } 254 255 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); 256 return &srq->ibsrq; 257 err2: 258 memset(&in_params, 0, sizeof(in_params)); 259 destroy_in_params.srq_id = srq->srq_id; 260 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 261 262 err1: 263 if (udata) 264 qlnxr_free_srq_user_params(srq); 265 else 266 qlnxr_free_srq_kernel_params(srq); 267 268 err0: 269 kfree(srq); 270 return ERR_PTR(-EFAULT); 271 } 272 273 int 274 qlnxr_destroy_srq(struct ib_srq *ibsrq) 275 { 276 struct qlnxr_dev *dev; 277 struct qlnxr_srq *srq; 278 qlnx_host_t *ha; 279 struct ecore_rdma_destroy_srq_in_params in_params; 280 281 srq = get_qlnxr_srq(ibsrq); 282 dev = srq->dev; 283 ha = dev->ha; 284 285 memset(&in_params, 0, sizeof(in_params)); 286 in_params.srq_id = srq->srq_id; 287 288 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); 289 290 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) 291 qlnxr_free_srq_user_params(srq); 292 else 293 qlnxr_free_srq_kernel_params(srq); 294 295 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); 296 kfree(srq); 297 return 0; 298 } 299 300 int 301 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 302 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 303 { 304 struct qlnxr_dev *dev; 305 struct qlnxr_srq *srq; 306 qlnx_host_t *ha; 307 struct ecore_rdma_modify_srq_in_params in_params; 308 int ret = 0; 309 310 srq = get_qlnxr_srq(ibsrq); 311 dev = srq->dev; 312 ha = dev->ha; 313 314 QL_DPRINT12(ha, "enter\n"); 315 if (attr_mask & IB_SRQ_MAX_WR) { 316 QL_DPRINT12(ha, "invalid attribute mask=0x%x" 317 " specified for %p\n", attr_mask, srq); 318 return -EINVAL; 319 } 320 321 if (attr_mask & IB_SRQ_LIMIT) { 322 if (attr->srq_limit >= srq->hw_srq.max_wr) { 323 QL_DPRINT12(ha, "invalid srq_limit=0x%x" 324 " (max_srq_limit = 0x%x)\n", 325 attr->srq_limit, srq->hw_srq.max_wr); 326 return -EINVAL; 327 } 328 memset(&in_params, 0, sizeof(in_params)); 329 in_params.srq_id = srq->srq_id; 330 in_params.wqe_limit = attr->srq_limit; 331 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); 332 if (ret) 333 return ret; 334 } 335 336 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); 337 return 0; 338 } 339 340 int 341 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 342 { 343 struct qlnxr_dev *dev; 344 struct qlnxr_srq *srq; 345 qlnx_host_t *ha; 346 struct ecore_rdma_device *qattr; 347 srq = get_qlnxr_srq(ibsrq); 348 dev = srq->dev; 349 ha = dev->ha; 350 //qattr = &dev->attr; 351 qattr = ecore_rdma_query_device(dev->rdma_ctx); 352 QL_DPRINT12(ha, "enter\n"); 353 354 if (!dev->rdma_ctx) { 355 QL_DPRINT12(ha, "called with invalid params" 356 " rdma_ctx is NULL\n"); 357 return -EINVAL; 358 } 359 360 srq_attr->srq_limit = qattr->max_srq; 361 srq_attr->max_wr = qattr->max_srq_wr; 362 srq_attr->max_sge = qattr->max_sge; 363 364 QL_DPRINT12(ha, "exit\n"); 365 return 0; 366 } 367 368 /* Increment srq wr producer by one */ 369 static 370 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) 371 { 372 info->wr_prod_cnt++; 373 } 374 375 /* Increment srq wr consumer by one */ 376 static 377 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) 378 { 379 info->wr_cons_cnt++; 380 } 381 382 /* get_port_immutable verb is not available in FreeBSD */ 383 #if 0 384 int 385 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 386 struct ib_port_immutable *immutable) 387 { 388 struct qlnxr_dev *dev; 389 qlnx_host_t *ha; 390 dev = get_qlnxr_dev(ibdev); 391 ha = dev->ha; 392 393 QL_DPRINT12(ha, "entered but not implemented!!!\n"); 394 } 395 #endif 396 397 int 398 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 399 struct ib_recv_wr **bad_wr) 400 { 401 struct qlnxr_dev *dev; 402 struct qlnxr_srq *srq; 403 qlnx_host_t *ha; 404 struct qlnxr_srq_hwq_info *hw_srq; 405 struct ecore_chain *pbl; 406 unsigned long flags; 407 int status = 0; 408 u32 num_sge, offset; 409 410 srq = get_qlnxr_srq(ibsrq); 411 dev = srq->dev; 412 ha = dev->ha; 413 hw_srq = &srq->hw_srq; 414 415 QL_DPRINT12(ha, "enter\n"); 416 spin_lock_irqsave(&srq->lock, flags); 417 418 pbl = &srq->hw_srq.pbl; 419 while (wr) { 420 struct rdma_srq_wqe_header *hdr; 421 int i; 422 423 if (!qlnxr_srq_elem_left(hw_srq) || 424 wr->num_sge > srq->hw_srq.max_sges) { 425 QL_DPRINT11(ha, "WR cannot be posted" 426 " (%d, %d) || (%d > %d)\n", 427 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 428 wr->num_sge, srq->hw_srq.max_sges); 429 status = -ENOMEM; 430 *bad_wr = wr; 431 break; 432 } 433 434 hdr = ecore_chain_produce(pbl); 435 num_sge = wr->num_sge; 436 /* Set number of sge and WR id in header */ 437 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 438 439 /* PBL is maintained in case of WR granularity. 440 * So increment WR producer in case we post a WR. 441 */ 442 qlnxr_inc_srq_wr_prod(hw_srq); 443 hw_srq->wqe_prod++; 444 hw_srq->sge_prod++; 445 446 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", 447 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 448 449 for (i = 0; i < wr->num_sge; i++) { 450 struct rdma_srq_sge *srq_sge = 451 ecore_chain_produce(pbl); 452 /* Set SGE length, lkey and address */ 453 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 454 wr->sg_list[i].length, wr->sg_list[i].lkey); 455 456 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", 457 i, srq_sge->length, srq_sge->l_key, 458 srq_sge->addr.hi, srq_sge->addr.lo); 459 hw_srq->sge_prod++; 460 } 461 wmb(); 462 /* 463 * SRQ prod is 8 bytes. Need to update SGE prod in index 464 * in first 4 bytes and need to update WQE prod in next 465 * 4 bytes. 466 */ 467 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; 468 offset = offsetof(struct rdma_srq_producers, wqe_prod); 469 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 470 hw_srq->wqe_prod; 471 /* Flush prod after updating it */ 472 wmb(); 473 wr = wr->next; 474 } 475 476 QL_DPRINT12(ha, "Elements in SRQ: %d\n", 477 ecore_chain_get_elem_left(pbl)); 478 479 spin_unlock_irqrestore(&srq->lock, flags); 480 QL_DPRINT12(ha, "exit\n"); 481 return status; 482 } 483 484 int 485 #if __FreeBSD_version < 1102000 486 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr) 487 #else 488 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 489 struct ib_udata *udata) 490 #endif /* #if __FreeBSD_version < 1102000 */ 491 492 { 493 struct qlnxr_dev *dev; 494 struct ecore_rdma_device *qattr; 495 qlnx_host_t *ha; 496 497 dev = get_qlnxr_dev(ibdev); 498 ha = dev->ha; 499 500 QL_DPRINT12(ha, "enter\n"); 501 502 #if __FreeBSD_version > 1102000 503 if (udata->inlen || udata->outlen) 504 return -EINVAL; 505 #endif /* #if __FreeBSD_version > 1102000 */ 506 507 if (dev->rdma_ctx == NULL) { 508 return -EINVAL; 509 } 510 511 qattr = ecore_rdma_query_device(dev->rdma_ctx); 512 513 memset(attr, 0, sizeof *attr); 514 515 attr->fw_ver = qattr->fw_ver; 516 attr->sys_image_guid = qattr->sys_image_guid; 517 attr->max_mr_size = qattr->max_mr_size; 518 attr->page_size_cap = qattr->page_size_caps; 519 attr->vendor_id = qattr->vendor_id; 520 attr->vendor_part_id = qattr->vendor_part_id; 521 attr->hw_ver = qattr->hw_ver; 522 attr->max_qp = qattr->max_qp; 523 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 524 IB_DEVICE_RC_RNR_NAK_GEN | 525 IB_DEVICE_LOCAL_DMA_LKEY | 526 IB_DEVICE_MEM_MGT_EXTENSIONS; 527 528 attr->max_sge = qattr->max_sge; 529 attr->max_sge_rd = qattr->max_sge; 530 attr->max_cq = qattr->max_cq; 531 attr->max_cqe = qattr->max_cqe; 532 attr->max_mr = qattr->max_mr; 533 attr->max_mw = qattr->max_mw; 534 attr->max_pd = qattr->max_pd; 535 attr->atomic_cap = dev->atomic_cap; 536 attr->max_fmr = qattr->max_fmr; 537 attr->max_map_per_fmr = 16; /* TBD: FMR */ 538 539 /* There is an implicit assumption in some of the ib_xxx apps that the 540 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in 541 * communication the qp_rd_atom is passed to the other side and used as 542 * init_rd_atom without check device capabilities for init_rd_atom. 543 * for this reason, we set the qp_rd_atom to be the minimum between the 544 * two...There is an additional assumption in mlx4 driver that the 545 * values are power of two, fls is performed on the value - 1, which 546 * in fact gives a larger power of two for values which are not a power 547 * of two. This should be fixed in mlx4 driver, but until then -> 548 * we provide a value that is a power of two in our code. 549 */ 550 attr->max_qp_init_rd_atom = 551 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 552 attr->max_qp_rd_atom = 553 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 554 attr->max_qp_init_rd_atom); 555 556 attr->max_srq = qattr->max_srq; 557 attr->max_srq_sge = qattr->max_srq_sge; 558 attr->max_srq_wr = qattr->max_srq_wr; 559 560 /* TODO: R&D to more properly configure the following */ 561 attr->local_ca_ack_delay = qattr->dev_ack_delay; 562 attr->max_fast_reg_page_list_len = qattr->max_mr/8; 563 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; 564 attr->max_ah = qattr->max_ah; 565 566 QL_DPRINT12(ha, "exit\n"); 567 return 0; 568 } 569 570 static inline void 571 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) 572 { 573 switch (speed) { 574 case 1000: 575 *ib_speed = IB_SPEED_SDR; 576 *ib_width = IB_WIDTH_1X; 577 break; 578 case 10000: 579 *ib_speed = IB_SPEED_QDR; 580 *ib_width = IB_WIDTH_1X; 581 break; 582 583 case 20000: 584 *ib_speed = IB_SPEED_DDR; 585 *ib_width = IB_WIDTH_4X; 586 break; 587 588 case 25000: 589 *ib_speed = IB_SPEED_EDR; 590 *ib_width = IB_WIDTH_1X; 591 break; 592 593 case 40000: 594 *ib_speed = IB_SPEED_QDR; 595 *ib_width = IB_WIDTH_4X; 596 break; 597 598 case 50000: 599 *ib_speed = IB_SPEED_QDR; 600 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... 601 break; 602 603 case 100000: 604 *ib_speed = IB_SPEED_EDR; 605 *ib_width = IB_WIDTH_4X; 606 break; 607 608 default: 609 /* Unsupported */ 610 *ib_speed = IB_SPEED_SDR; 611 *ib_width = IB_WIDTH_1X; 612 } 613 return; 614 } 615 616 int 617 qlnxr_query_port(struct ib_device *ibdev, uint8_t port, 618 struct ib_port_attr *attr) 619 { 620 struct qlnxr_dev *dev; 621 struct ecore_rdma_port *rdma_port; 622 qlnx_host_t *ha; 623 624 dev = get_qlnxr_dev(ibdev); 625 ha = dev->ha; 626 627 QL_DPRINT12(ha, "enter\n"); 628 629 if (port > 1) { 630 QL_DPRINT12(ha, "port [%d] > 1 \n", port); 631 return -EINVAL; 632 } 633 634 if (dev->rdma_ctx == NULL) { 635 QL_DPRINT12(ha, "rdma_ctx == NULL\n"); 636 return -EINVAL; 637 } 638 639 rdma_port = ecore_rdma_query_port(dev->rdma_ctx); 640 memset(attr, 0, sizeof *attr); 641 642 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { 643 attr->state = IB_PORT_ACTIVE; 644 attr->phys_state = 5; 645 } else { 646 attr->state = IB_PORT_DOWN; 647 attr->phys_state = 3; 648 } 649 650 attr->max_mtu = IB_MTU_4096; 651 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); 652 attr->lid = 0; 653 attr->lmc = 0; 654 attr->sm_lid = 0; 655 attr->sm_sl = 0; 656 attr->port_cap_flags = 0; 657 658 if (QLNX_IS_IWARP(dev)) { 659 attr->gid_tbl_len = 1; 660 attr->pkey_tbl_len = 1; 661 } else { 662 attr->gid_tbl_len = QLNXR_MAX_SGID; 663 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; 664 } 665 666 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 667 attr->qkey_viol_cntr = 0; 668 669 get_link_speed_and_width(rdma_port->link_speed, 670 &attr->active_speed, &attr->active_width); 671 672 attr->max_msg_sz = rdma_port->max_msg_size; 673 attr->max_vl_num = 4; /* TODO -> figure this one out... */ 674 675 QL_DPRINT12(ha, "state = %d phys_state = %d " 676 " link_speed = %d active_speed = %d active_width = %d" 677 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" 678 " max_msg_sz = 0x%x max_vl_num = 0x%x \n", 679 attr->state, attr->phys_state, 680 rdma_port->link_speed, attr->active_speed, 681 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, 682 attr->max_msg_sz, attr->max_vl_num); 683 684 QL_DPRINT12(ha, "exit\n"); 685 return 0; 686 } 687 688 int 689 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, 690 struct ib_port_modify *props) 691 { 692 struct qlnxr_dev *dev; 693 qlnx_host_t *ha; 694 695 dev = get_qlnxr_dev(ibdev); 696 ha = dev->ha; 697 698 QL_DPRINT12(ha, "enter\n"); 699 700 if (port > 1) { 701 QL_DPRINT12(ha, "port (%d) > 1\n", port); 702 return -EINVAL; 703 } 704 705 QL_DPRINT12(ha, "exit\n"); 706 return 0; 707 } 708 709 enum rdma_link_layer 710 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) 711 { 712 struct qlnxr_dev *dev; 713 qlnx_host_t *ha; 714 715 dev = get_qlnxr_dev(ibdev); 716 ha = dev->ha; 717 718 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); 719 720 return IB_LINK_LAYER_ETHERNET; 721 } 722 723 struct ib_pd * 724 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, 725 struct ib_udata *udata) 726 { 727 struct qlnxr_pd *pd = NULL; 728 u16 pd_id; 729 int rc; 730 struct qlnxr_dev *dev; 731 qlnx_host_t *ha; 732 733 dev = get_qlnxr_dev(ibdev); 734 ha = dev->ha; 735 736 QL_DPRINT12(ha, "ibdev = %p context = %p" 737 " udata = %p enter\n", ibdev, context, udata); 738 739 if (dev->rdma_ctx == NULL) { 740 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); 741 rc = -1; 742 goto err; 743 } 744 745 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 746 if (!pd) { 747 rc = -ENOMEM; 748 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n"); 749 goto err; 750 } 751 752 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); 753 if (rc) { 754 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); 755 goto err; 756 } 757 758 pd->pd_id = pd_id; 759 760 if (udata && context) { 761 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); 762 if (rc) { 763 QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); 764 ecore_rdma_free_pd(dev->rdma_ctx, pd_id); 765 goto err; 766 } 767 768 pd->uctx = get_qlnxr_ucontext(context); 769 pd->uctx->pd = pd; 770 } 771 772 atomic_add_rel_32(&dev->pd_count, 1); 773 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 774 pd, pd_id, dev->pd_count); 775 776 return &pd->ibpd; 777 778 err: 779 kfree(pd); 780 QL_DPRINT12(ha, "exit -1\n"); 781 return ERR_PTR(rc); 782 } 783 784 int 785 qlnxr_dealloc_pd(struct ib_pd *ibpd) 786 { 787 struct qlnxr_pd *pd; 788 struct qlnxr_dev *dev; 789 qlnx_host_t *ha; 790 791 pd = get_qlnxr_pd(ibpd); 792 dev = get_qlnxr_dev((ibpd->device)); 793 ha = dev->ha; 794 795 QL_DPRINT12(ha, "enter\n"); 796 797 if (pd == NULL) { 798 QL_DPRINT11(ha, "pd = NULL\n"); 799 } else { 800 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); 801 kfree(pd); 802 atomic_subtract_rel_32(&dev->pd_count, 1); 803 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 804 pd, pd->pd_id, dev->pd_count); 805 } 806 807 QL_DPRINT12(ha, "exit\n"); 808 return 0; 809 } 810 811 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) 812 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ 813 /* Should be part of HSI */ 814 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ 815 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 816 817 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); 818 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); 819 820 int 821 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 822 { 823 struct qlnxr_dev *dev; 824 qlnx_host_t *ha; 825 826 dev = get_qlnxr_dev(ibdev); 827 ha = dev->ha; 828 829 QL_DPRINT12(ha, "enter index = 0x%x\n", index); 830 831 if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 832 return -EINVAL; 833 834 *pkey = QLNXR_ROCE_PKEY_DEFAULT; 835 836 QL_DPRINT12(ha, "exit\n"); 837 return 0; 838 } 839 840 static inline bool 841 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, 842 u16 *vlan_id) 843 { 844 bool ret = false; 845 846 QL_DPRINT12(ha, "enter \n"); 847 848 *vlan_id = 0; 849 850 #if __FreeBSD_version >= 1100000 851 u16 tmp_vlan_id; 852 853 #if __FreeBSD_version >= 1102000 854 union ib_gid *dgid; 855 856 dgid = &attr->ah_attr.grh.dgid; 857 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; 858 859 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { 860 *vlan_id = tmp_vlan_id; 861 ret = true; 862 } 863 #else 864 tmp_vlan_id = attr->vlan_id; 865 866 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) { 867 *vlan_id = tmp_vlan_id; 868 ret = true; 869 } 870 871 #endif /* #if __FreeBSD_version > 1102000 */ 872 873 #else 874 ret = true; 875 876 #endif /* #if __FreeBSD_version >= 1100000 */ 877 878 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); 879 880 return (ret); 881 } 882 883 static inline void 884 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, 885 int attr_mask, 886 struct qlnxr_dev *dev, 887 struct qlnxr_qp *qp, 888 struct ecore_rdma_modify_qp_in_params *qp_params) 889 { 890 int i; 891 qlnx_host_t *ha; 892 893 ha = dev->ha; 894 895 QL_DPRINT12(ha, "enter\n"); 896 897 memcpy(&qp_params->sgid.bytes[0], 898 &dev->sgid_tbl[qp->sgid_idx].raw[0], 899 sizeof(qp_params->sgid.bytes)); 900 memcpy(&qp_params->dgid.bytes[0], 901 &attr->ah_attr.grh.dgid.raw[0], 902 sizeof(qp_params->dgid)); 903 904 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); 905 906 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { 907 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 908 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 909 } 910 911 QL_DPRINT12(ha, "exit\n"); 912 return; 913 } 914 915 static int 916 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 917 { 918 struct qlnxr_mm *mm; 919 qlnx_host_t *ha; 920 921 ha = uctx->dev->ha; 922 923 QL_DPRINT12(ha, "enter\n"); 924 925 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 926 if (mm == NULL) { 927 QL_DPRINT11(ha, "mm = NULL\n"); 928 return -ENOMEM; 929 } 930 931 mm->key.phy_addr = phy_addr; 932 933 /* This function might be called with a length which is not a multiple 934 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 935 * forces this granularity by increasing the requested size if needed. 936 * When qedr_mmap is called, it will search the list with the updated 937 * length as a key. To prevent search failures, the length is rounded up 938 * in advance to PAGE_SIZE. 939 */ 940 mm->key.len = roundup(len, PAGE_SIZE); 941 INIT_LIST_HEAD(&mm->entry); 942 943 mutex_lock(&uctx->mm_list_lock); 944 list_add(&mm->entry, &uctx->mm_head); 945 mutex_unlock(&uctx->mm_list_lock); 946 947 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 948 (unsigned long long)mm->key.phy_addr, 949 (unsigned long)mm->key.len, uctx); 950 951 return 0; 952 } 953 954 static bool 955 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 956 { 957 bool found = false; 958 struct qlnxr_mm *mm; 959 qlnx_host_t *ha; 960 961 ha = uctx->dev->ha; 962 963 QL_DPRINT12(ha, "enter\n"); 964 965 mutex_lock(&uctx->mm_list_lock); 966 list_for_each_entry(mm, &uctx->mm_head, entry) { 967 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 968 continue; 969 970 found = true; 971 break; 972 } 973 mutex_unlock(&uctx->mm_list_lock); 974 975 QL_DPRINT12(ha, 976 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", 977 mm->key.phy_addr, mm->key.len, uctx, found); 978 979 return found; 980 } 981 982 struct 983 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev, 984 struct ib_udata *udata) 985 { 986 int rc; 987 struct qlnxr_ucontext *ctx; 988 struct qlnxr_alloc_ucontext_resp uresp; 989 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 990 qlnx_host_t *ha = dev->ha; 991 struct ecore_rdma_add_user_out_params oparams; 992 993 if (!udata) { 994 return ERR_PTR(-EFAULT); 995 } 996 997 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 998 if (!ctx) 999 return ERR_PTR(-ENOMEM); 1000 1001 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); 1002 if (rc) { 1003 QL_DPRINT12(ha, 1004 "Failed to allocate a DPI for a new RoCE application " 1005 ",rc = %d. To overcome this, consider to increase " 1006 "the number of DPIs, increase the doorbell BAR size " 1007 "or just close unnecessary RoCE applications. In " 1008 "order to increase the number of DPIs consult the " 1009 "README\n", rc); 1010 goto err; 1011 } 1012 1013 ctx->dpi = oparams.dpi; 1014 ctx->dpi_addr = oparams.dpi_addr; 1015 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 1016 ctx->dpi_size = oparams.dpi_size; 1017 INIT_LIST_HEAD(&ctx->mm_head); 1018 mutex_init(&ctx->mm_list_lock); 1019 1020 memset(&uresp, 0, sizeof(uresp)); 1021 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) 1022 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out 1023 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) 1024 < udata->outlen ? 1 : 0; //TODO: figure this out 1025 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) 1026 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 1027 uresp.db_pa = ctx->dpi_phys_addr; 1028 uresp.db_size = ctx->dpi_size; 1029 uresp.max_send_wr = dev->attr.max_sqe; 1030 uresp.max_recv_wr = dev->attr.max_rqe; 1031 uresp.max_srq_wr = dev->attr.max_srq_wr; 1032 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 1033 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 1034 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 1035 uresp.max_cqes = QLNXR_MAX_CQES; 1036 1037 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1038 if (rc) 1039 goto err; 1040 1041 ctx->dev = dev; 1042 1043 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 1044 if (rc) 1045 goto err; 1046 QL_DPRINT12(ha, "Allocated user context %p\n", 1047 &ctx->ibucontext); 1048 1049 return &ctx->ibucontext; 1050 err: 1051 kfree(ctx); 1052 return ERR_PTR(rc); 1053 } 1054 1055 int 1056 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) 1057 { 1058 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); 1059 struct qlnxr_dev *dev = uctx->dev; 1060 qlnx_host_t *ha = dev->ha; 1061 struct qlnxr_mm *mm, *tmp; 1062 int status = 0; 1063 1064 QL_DPRINT12(ha, "Deallocating user context %p\n", 1065 uctx); 1066 1067 if (dev) { 1068 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 1069 } 1070 1071 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 1072 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" 1073 " ctx=%p\n", 1074 mm->key.phy_addr, mm->key.len, uctx); 1075 list_del(&mm->entry); 1076 kfree(mm); 1077 } 1078 kfree(uctx); 1079 return status; 1080 } 1081 1082 int 1083 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 1084 { 1085 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); 1086 struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); 1087 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 1088 u64 unmapped_db; 1089 unsigned long len = (vma->vm_end - vma->vm_start); 1090 int rc = 0; 1091 bool found; 1092 qlnx_host_t *ha; 1093 1094 ha = dev->ha; 1095 1096 #if __FreeBSD_version > 1102000 1097 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 1098 #else 1099 unmapped_db = dev->db_phys_addr; 1100 #endif /* #if __FreeBSD_version > 1102000 */ 1101 1102 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" 1103 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 1104 vm_page, vma->vm_pgoff, unmapped_db, 1105 dev->db_size, len); 1106 1107 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 1108 QL_DPRINT11(ha, "Vma_start not page aligned " 1109 "vm_start = %ld vma_end = %ld\n", vma->vm_start, 1110 vma->vm_end); 1111 return -EINVAL; 1112 } 1113 1114 found = qlnxr_search_mmap(ucontext, vm_page, len); 1115 if (!found) { 1116 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", 1117 vma->vm_pgoff); 1118 return -EINVAL; 1119 } 1120 1121 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1122 1123 #if __FreeBSD_version > 1102000 1124 1125 if ((vm_page < unmapped_db) || 1126 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { 1127 QL_DPRINT11(ha, "failed pages are outside of dpi;" 1128 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", 1129 vm_page, unmapped_db, ucontext->dpi_size); 1130 return -EINVAL; 1131 } 1132 1133 if (vma->vm_flags & VM_READ) { 1134 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); 1135 return -EINVAL; 1136 } 1137 1138 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1139 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 1140 vma->vm_page_prot); 1141 1142 #else 1143 1144 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 1145 dev->db_size))) { 1146 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1147 1148 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1149 1150 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1151 PAGE_SIZE, vma->vm_page_prot); 1152 } else { 1153 QL_DPRINT12(ha, "Mapping chains\n"); 1154 rc = io_remap_pfn_range(vma, vma->vm_start, 1155 vma->vm_pgoff, len, vma->vm_page_prot); 1156 } 1157 1158 #endif /* #if __FreeBSD_version > 1102000 */ 1159 1160 QL_DPRINT12(ha, "exit [%d]\n", rc); 1161 return rc; 1162 } 1163 1164 struct ib_mr * 1165 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) 1166 { 1167 struct qlnxr_mr *mr; 1168 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1169 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 1170 int rc; 1171 qlnx_host_t *ha; 1172 1173 ha = dev->ha; 1174 1175 QL_DPRINT12(ha, "enter\n"); 1176 1177 if (acc & IB_ACCESS_MW_BIND) { 1178 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); 1179 } 1180 1181 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1182 if (!mr) { 1183 rc = -ENOMEM; 1184 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); 1185 goto err0; 1186 } 1187 1188 mr->type = QLNXR_MR_DMA; 1189 1190 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1191 if (rc) { 1192 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); 1193 goto err1; 1194 } 1195 1196 /* index only, 18 bit long, lkey = itid << 8 | key */ 1197 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1198 mr->hw_mr.pd = pd->pd_id; 1199 mr->hw_mr.local_read = 1; 1200 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1201 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1202 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1203 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1204 mr->hw_mr.dma_mr = true; 1205 1206 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1207 if (rc) { 1208 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); 1209 goto err2; 1210 } 1211 1212 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1213 1214 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1215 mr->hw_mr.remote_atomic) { 1216 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1217 } 1218 1219 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); 1220 1221 return &mr->ibmr; 1222 1223 err2: 1224 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1225 err1: 1226 kfree(mr); 1227 err0: 1228 QL_DPRINT12(ha, "exit [%d]\n", rc); 1229 1230 return ERR_PTR(rc); 1231 } 1232 1233 static void 1234 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, 1235 struct qlnxr_pbl *pbl) 1236 { 1237 int i; 1238 qlnx_host_t *ha; 1239 1240 ha = dev->ha; 1241 1242 QL_DPRINT12(ha, "enter\n"); 1243 1244 for (i = 0; i < pbl_info->num_pbls; i++) { 1245 if (!pbl[i].va) 1246 continue; 1247 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, 1248 pbl_info->pbl_size); 1249 } 1250 kfree(pbl); 1251 1252 QL_DPRINT12(ha, "exit\n"); 1253 return; 1254 } 1255 1256 #define MIN_FW_PBL_PAGE_SIZE (4*1024) 1257 #define MAX_FW_PBL_PAGE_SIZE (64*1024) 1258 1259 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 1260 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 1261 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) 1262 1263 static struct qlnxr_pbl * 1264 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, 1265 struct qlnxr_pbl_info *pbl_info, gfp_t flags) 1266 { 1267 void *va; 1268 dma_addr_t pa; 1269 dma_addr_t *pbl_main_tbl; 1270 struct qlnxr_pbl *pbl_table; 1271 int i, rc = 0; 1272 qlnx_host_t *ha; 1273 1274 ha = dev->ha; 1275 1276 QL_DPRINT12(ha, "enter\n"); 1277 1278 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); 1279 1280 if (!pbl_table) { 1281 QL_DPRINT12(ha, "pbl_table = NULL\n"); 1282 return NULL; 1283 } 1284 1285 for (i = 0; i < pbl_info->num_pbls; i++) { 1286 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); 1287 if (!va) { 1288 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); 1289 rc = -ENOMEM; 1290 goto err; 1291 } 1292 memset(va, 0, pbl_info->pbl_size); 1293 pbl_table[i].va = va; 1294 pbl_table[i].pa = pa; 1295 } 1296 1297 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 1298 * the first one with physical pointers to all of the rest 1299 */ 1300 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 1301 for (i = 0; i < pbl_info->num_pbls - 1; i++) 1302 pbl_main_tbl[i] = pbl_table[i + 1].pa; 1303 1304 QL_DPRINT12(ha, "exit\n"); 1305 return pbl_table; 1306 1307 err: 1308 qlnxr_free_pbl(dev, pbl_info, pbl_table); 1309 1310 QL_DPRINT12(ha, "exit with error\n"); 1311 return NULL; 1312 } 1313 1314 static int 1315 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, 1316 struct qlnxr_pbl_info *pbl_info, 1317 u32 num_pbes, 1318 int two_layer_capable) 1319 { 1320 u32 pbl_capacity; 1321 u32 pbl_size; 1322 u32 num_pbls; 1323 qlnx_host_t *ha; 1324 1325 ha = dev->ha; 1326 1327 QL_DPRINT12(ha, "enter\n"); 1328 1329 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 1330 if (num_pbes > MAX_PBES_TWO_LAYER) { 1331 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", 1332 num_pbes); 1333 return -EINVAL; 1334 } 1335 1336 /* calculate required pbl page size */ 1337 pbl_size = MIN_FW_PBL_PAGE_SIZE; 1338 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 1339 NUM_PBES_ON_PAGE(pbl_size); 1340 1341 while (pbl_capacity < num_pbes) { 1342 pbl_size *= 2; 1343 pbl_capacity = pbl_size / sizeof(u64); 1344 pbl_capacity = pbl_capacity * pbl_capacity; 1345 } 1346 1347 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 1348 num_pbls++; /* One for the layer0 ( points to the pbls) */ 1349 pbl_info->two_layered = true; 1350 } else { 1351 /* One layered PBL */ 1352 num_pbls = 1; 1353 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ 1354 roundup_pow_of_two((num_pbes * sizeof(u64)))); 1355 pbl_info->two_layered = false; 1356 } 1357 1358 pbl_info->num_pbls = num_pbls; 1359 pbl_info->pbl_size = pbl_size; 1360 pbl_info->num_pbes = num_pbes; 1361 1362 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", 1363 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 1364 1365 return 0; 1366 } 1367 1368 #define upper_32_bits(x) (uint32_t)(x >> 32) 1369 #define lower_32_bits(x) (uint32_t)(x) 1370 1371 static void 1372 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, 1373 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 1374 { 1375 struct regpair *pbe; 1376 struct qlnxr_pbl *pbl_tbl; 1377 struct scatterlist *sg; 1378 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 1379 qlnx_host_t *ha; 1380 1381 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1382 int i; 1383 struct ib_umem_chunk *chunk = NULL; 1384 #else 1385 int entry; 1386 #endif 1387 1388 ha = dev->ha; 1389 1390 QL_DPRINT12(ha, "enter\n"); 1391 1392 if (!pbl_info) { 1393 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 1394 return; 1395 } 1396 1397 if (!pbl_info->num_pbes) { 1398 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 1399 return; 1400 } 1401 1402 /* If we have a two layered pbl, the first pbl points to the rest 1403 * of the pbls and the first entry lays on the second pbl in the table 1404 */ 1405 if (pbl_info->two_layered) 1406 pbl_tbl = &pbl[1]; 1407 else 1408 pbl_tbl = pbl; 1409 1410 pbe = (struct regpair *)pbl_tbl->va; 1411 if (!pbe) { 1412 QL_DPRINT12(ha, "pbe is NULL\n"); 1413 return; 1414 } 1415 1416 pbe_cnt = 0; 1417 1418 shift = ilog2(umem->page_size); 1419 1420 #ifndef DEFINE_IB_UMEM_WITH_CHUNK 1421 1422 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 1423 #else 1424 list_for_each_entry(chunk, &umem->chunk_list, list) { 1425 /* get all the dma regions from the chunk. */ 1426 for (i = 0; i < chunk->nmap; i++) { 1427 sg = &chunk->page_list[i]; 1428 #endif 1429 pages = sg_dma_len(sg) >> shift; 1430 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 1431 /* store the page address in pbe */ 1432 pbe->lo = 1433 cpu_to_le32(sg_dma_address(sg) + 1434 (umem->page_size * pg_cnt)); 1435 pbe->hi = 1436 cpu_to_le32(upper_32_bits 1437 ((sg_dma_address(sg) + 1438 umem->page_size * pg_cnt))); 1439 1440 QL_DPRINT12(ha, 1441 "Populate pbl table:" 1442 " pbe->addr=0x%x:0x%x " 1443 " pbe_cnt = %d total_num_pbes=%d" 1444 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 1445 total_num_pbes, pbe); 1446 1447 pbe_cnt ++; 1448 total_num_pbes ++; 1449 pbe++; 1450 1451 if (total_num_pbes == pbl_info->num_pbes) 1452 return; 1453 1454 /* if the given pbl is full storing the pbes, 1455 * move to next pbl. 1456 */ 1457 if (pbe_cnt == 1458 (pbl_info->pbl_size / sizeof(u64))) { 1459 pbl_tbl++; 1460 pbe = (struct regpair *)pbl_tbl->va; 1461 pbe_cnt = 0; 1462 } 1463 } 1464 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1465 } 1466 #endif 1467 } 1468 QL_DPRINT12(ha, "exit\n"); 1469 return; 1470 } 1471 1472 static void 1473 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) 1474 { 1475 struct qlnxr_pbl *pbl, *tmp; 1476 qlnx_host_t *ha; 1477 1478 ha = dev->ha; 1479 1480 QL_DPRINT12(ha, "enter\n"); 1481 1482 if (info->pbl_table) 1483 list_add_tail(&info->pbl_table->list_entry, 1484 &info->free_pbl_list); 1485 1486 if (!list_empty(&info->inuse_pbl_list)) 1487 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 1488 1489 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 1490 list_del(&pbl->list_entry); 1491 qlnxr_free_pbl(dev, &info->pbl_info, pbl); 1492 } 1493 QL_DPRINT12(ha, "exit\n"); 1494 1495 return; 1496 } 1497 1498 static int 1499 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, 1500 size_t page_list_len, bool two_layered) 1501 { 1502 int rc; 1503 struct qlnxr_pbl *tmp; 1504 qlnx_host_t *ha; 1505 1506 ha = dev->ha; 1507 1508 QL_DPRINT12(ha, "enter\n"); 1509 1510 INIT_LIST_HEAD(&info->free_pbl_list); 1511 INIT_LIST_HEAD(&info->inuse_pbl_list); 1512 1513 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, 1514 page_list_len, two_layered); 1515 if (rc) { 1516 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); 1517 goto done; 1518 } 1519 1520 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1521 1522 if (!info->pbl_table) { 1523 rc = -ENOMEM; 1524 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); 1525 goto done; 1526 } 1527 1528 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); 1529 1530 /* in usual case we use 2 PBLs, so we add one to free 1531 * list and allocating another one 1532 */ 1533 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1534 1535 if (!tmp) { 1536 QL_DPRINT11(ha, "Extra PBL is not allocated\n"); 1537 goto done; /* it's OK if second allocation fails, so rc = 0*/ 1538 } 1539 1540 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 1541 1542 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); 1543 1544 done: 1545 if (rc) 1546 free_mr_info(dev, info); 1547 1548 QL_DPRINT12(ha, "exit [%d]\n", rc); 1549 1550 return rc; 1551 } 1552 1553 struct ib_mr * 1554 #if __FreeBSD_version >= 1102000 1555 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1556 u64 usr_addr, int acc, struct ib_udata *udata) 1557 #else 1558 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1559 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id) 1560 #endif /* #if __FreeBSD_version >= 1102000 */ 1561 { 1562 int rc = -ENOMEM; 1563 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1564 struct qlnxr_mr *mr; 1565 struct qlnxr_pd *pd; 1566 qlnx_host_t *ha; 1567 1568 ha = dev->ha; 1569 1570 QL_DPRINT12(ha, "enter\n"); 1571 1572 pd = get_qlnxr_pd(ibpd); 1573 1574 QL_DPRINT12(ha, "qedr_register user mr pd = %d" 1575 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 1576 pd->pd_id, start, len, usr_addr, acc); 1577 1578 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 1579 QL_DPRINT11(ha, 1580 "(acc & IB_ACCESS_REMOTE_WRITE &&" 1581 " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 1582 return ERR_PTR(-EINVAL); 1583 } 1584 1585 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1586 if (!mr) { 1587 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 1588 return ERR_PTR(rc); 1589 } 1590 1591 mr->type = QLNXR_MR_USER; 1592 1593 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 1594 if (IS_ERR(mr->umem)) { 1595 rc = -EFAULT; 1596 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); 1597 goto err0; 1598 } 1599 1600 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 1601 if (rc) { 1602 QL_DPRINT11(ha, 1603 "qlnxr_init_mr_info failed [%d]\n", rc); 1604 goto err1; 1605 } 1606 1607 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 1608 &mr->info.pbl_info); 1609 1610 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1611 1612 if (rc) { 1613 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 1614 goto err1; 1615 } 1616 1617 /* index only, 18 bit long, lkey = itid << 8 | key */ 1618 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1619 mr->hw_mr.key = 0; 1620 mr->hw_mr.pd = pd->pd_id; 1621 mr->hw_mr.local_read = 1; 1622 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1623 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1624 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1625 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1626 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 1627 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 1628 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 1629 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 1630 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ 1631 1632 #if __FreeBSD_version >= 1102000 1633 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 1634 #else 1635 mr->hw_mr.fbo = mr->umem->offset; 1636 #endif 1637 mr->hw_mr.length = len; 1638 mr->hw_mr.vaddr = usr_addr; 1639 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 1640 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 1641 mr->hw_mr.dma_mr = false; 1642 1643 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1644 if (rc) { 1645 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 1646 goto err2; 1647 } 1648 1649 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1650 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1651 mr->hw_mr.remote_atomic) 1652 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1653 1654 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); 1655 1656 return (&mr->ibmr); 1657 1658 err2: 1659 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1660 err1: 1661 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1662 err0: 1663 kfree(mr); 1664 1665 QL_DPRINT12(ha, "exit [%d]\n", rc); 1666 return (ERR_PTR(rc)); 1667 } 1668 1669 int 1670 qlnxr_dereg_mr(struct ib_mr *ib_mr) 1671 { 1672 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); 1673 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); 1674 int rc = 0; 1675 qlnx_host_t *ha; 1676 1677 ha = dev->ha; 1678 1679 QL_DPRINT12(ha, "enter\n"); 1680 1681 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) 1682 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1683 1684 /* it could be user registered memory. */ 1685 if (mr->umem) 1686 ib_umem_release(mr->umem); 1687 1688 kfree(mr->pages); 1689 1690 kfree(mr); 1691 1692 QL_DPRINT12(ha, "exit\n"); 1693 return rc; 1694 } 1695 1696 static int 1697 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, 1698 struct qlnxr_cq *cq, struct ib_udata *udata) 1699 { 1700 struct qlnxr_create_cq_uresp uresp; 1701 int rc; 1702 qlnx_host_t *ha; 1703 1704 ha = dev->ha; 1705 1706 QL_DPRINT12(ha, "enter\n"); 1707 1708 memset(&uresp, 0, sizeof(uresp)); 1709 1710 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1711 uresp.icid = cq->icid; 1712 1713 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1714 1715 if (rc) { 1716 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", 1717 cq->icid, rc); 1718 } 1719 1720 QL_DPRINT12(ha, "exit [%d]\n", rc); 1721 return rc; 1722 } 1723 1724 static void 1725 consume_cqe(struct qlnxr_cq *cq) 1726 { 1727 1728 if (cq->latest_cqe == cq->toggle_cqe) 1729 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1730 1731 cq->latest_cqe = ecore_chain_consume(&cq->pbl); 1732 } 1733 1734 static inline int 1735 qlnxr_align_cq_entries(int entries) 1736 { 1737 u64 size, aligned_size; 1738 1739 /* We allocate an extra entry that we don't report to the FW. 1740 * Why? 1741 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes 1742 * (for performance purposes). Allocating an extra entry and telling 1743 * the FW we have less prevents overwriting the first entry in case of 1744 * a wrap i.e. when the FW writes the last entry and the application 1745 * hasn't read the first one. 1746 */ 1747 size = (entries + 1) * QLNXR_CQE_SIZE; 1748 1749 /* We align to PAGE_SIZE. 1750 * Why? 1751 * Since the CQ is going to be mapped and the mapping is anyhow in whole 1752 * kernel pages we benefit from the possibly extra CQEs. 1753 */ 1754 aligned_size = ALIGN(size, PAGE_SIZE); 1755 1756 /* note: for CQs created in user space the result of this function 1757 * should match the size mapped in user space 1758 */ 1759 return (aligned_size / QLNXR_CQE_SIZE); 1760 } 1761 1762 static inline int 1763 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, 1764 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, 1765 int access, int dmasync, int alloc_and_init) 1766 { 1767 int page_cnt; 1768 int rc; 1769 qlnx_host_t *ha; 1770 1771 ha = dev->ha; 1772 1773 QL_DPRINT12(ha, "enter\n"); 1774 1775 q->buf_addr = buf_addr; 1776 q->buf_len = buf_len; 1777 1778 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" 1779 " dmasync : %x\n", q->buf_addr, q->buf_len, 1780 access, dmasync); 1781 1782 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 1783 1784 if (IS_ERR(q->umem)) { 1785 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); 1786 return PTR_ERR(q->umem); 1787 } 1788 1789 page_cnt = ib_umem_page_count(q->umem); 1790 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 1791 0 /* SQ and RQ don't support dual layer pbl. 1792 * CQ may, but this is yet uncoded. 1793 */); 1794 if (rc) { 1795 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); 1796 goto err; 1797 } 1798 1799 if (alloc_and_init) { 1800 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 1801 1802 if (!q->pbl_tbl) { 1803 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1804 rc = -ENOMEM; 1805 goto err; 1806 } 1807 1808 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); 1809 } else { 1810 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 1811 1812 if (!q->pbl_tbl) { 1813 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1814 rc = -ENOMEM; 1815 goto err; 1816 } 1817 } 1818 1819 QL_DPRINT12(ha, "exit\n"); 1820 return 0; 1821 1822 err: 1823 ib_umem_release(q->umem); 1824 q->umem = NULL; 1825 1826 QL_DPRINT12(ha, "exit [%d]\n", rc); 1827 return rc; 1828 } 1829 1830 #if __FreeBSD_version >= 1102000 1831 1832 struct ib_cq * 1833 qlnxr_create_cq(struct ib_device *ibdev, 1834 const struct ib_cq_init_attr *attr, 1835 struct ib_ucontext *ib_ctx, 1836 struct ib_udata *udata) 1837 1838 #else 1839 1840 #if __FreeBSD_version >= 1100000 1841 1842 struct ib_cq * 1843 qlnxr_create_cq(struct ib_device *ibdev, 1844 struct ib_cq_init_attr *attr, 1845 struct ib_ucontext *ib_ctx, 1846 struct ib_udata *udata) 1847 1848 #else 1849 1850 struct ib_cq * 1851 qlnxr_create_cq(struct ib_device *ibdev, 1852 int entries, 1853 int vector, 1854 struct ib_ucontext *ib_ctx, 1855 struct ib_udata *udata) 1856 #endif /* #if __FreeBSD_version >= 1100000 */ 1857 1858 #endif /* #if __FreeBSD_version >= 1102000 */ 1859 { 1860 struct qlnxr_ucontext *ctx; 1861 struct ecore_rdma_destroy_cq_out_params destroy_oparams; 1862 struct ecore_rdma_destroy_cq_in_params destroy_iparams; 1863 struct qlnxr_dev *dev; 1864 struct ecore_rdma_create_cq_in_params params; 1865 struct qlnxr_create_cq_ureq ureq; 1866 1867 #if __FreeBSD_version >= 1100000 1868 int vector = attr->comp_vector; 1869 int entries = attr->cqe; 1870 #endif 1871 struct qlnxr_cq *cq; 1872 int chain_entries, rc, page_cnt; 1873 u64 pbl_ptr; 1874 u16 icid; 1875 qlnx_host_t *ha; 1876 1877 dev = get_qlnxr_dev(ibdev); 1878 ha = dev->ha; 1879 1880 QL_DPRINT12(ha, "called from %s. entries = %d, " 1881 "vector = %d\n", 1882 (udata ? "User Lib" : "Kernel"), entries, vector); 1883 1884 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); 1885 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); 1886 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); 1887 1888 if (entries > QLNXR_MAX_CQES) { 1889 QL_DPRINT11(ha, 1890 "the number of entries %d is too high. " 1891 "Must be equal or below %d.\n", 1892 entries, QLNXR_MAX_CQES); 1893 return ERR_PTR(-EINVAL); 1894 } 1895 chain_entries = qlnxr_align_cq_entries(entries); 1896 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); 1897 1898 cq = qlnx_zalloc((sizeof(struct qlnxr_cq))); 1899 1900 if (!cq) 1901 return ERR_PTR(-ENOMEM); 1902 1903 if (udata) { 1904 memset(&ureq, 0, sizeof(ureq)); 1905 1906 if (ib_copy_from_udata(&ureq, udata, 1907 min(sizeof(ureq), udata->inlen))) { 1908 QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); 1909 goto err0; 1910 } 1911 1912 if (!ureq.len) { 1913 QL_DPRINT11(ha, "ureq.len == 0\n"); 1914 goto err0; 1915 } 1916 1917 cq->cq_type = QLNXR_CQ_TYPE_USER; 1918 1919 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len, 1920 IB_ACCESS_LOCAL_WRITE, 1, 1); 1921 1922 pbl_ptr = cq->q.pbl_tbl->pa; 1923 page_cnt = cq->q.pbl_info.num_pbes; 1924 cq->ibcq.cqe = chain_entries; 1925 } else { 1926 cq->cq_type = QLNXR_CQ_TYPE_KERNEL; 1927 1928 rc = ecore_chain_alloc(&dev->ha->cdev, 1929 ECORE_CHAIN_USE_TO_CONSUME, 1930 ECORE_CHAIN_MODE_PBL, 1931 ECORE_CHAIN_CNT_TYPE_U32, 1932 chain_entries, 1933 sizeof(union roce_cqe), 1934 &cq->pbl, NULL); 1935 1936 if (rc) 1937 goto err1; 1938 1939 page_cnt = ecore_chain_get_page_cnt(&cq->pbl); 1940 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); 1941 cq->ibcq.cqe = cq->pbl.capacity; 1942 } 1943 1944 params.cq_handle_hi = upper_32_bits((uintptr_t)cq); 1945 params.cq_handle_lo = lower_32_bits((uintptr_t)cq); 1946 params.cnq_id = vector; 1947 params.cq_size = chain_entries - 1; 1948 params.pbl_num_pages = page_cnt; 1949 params.pbl_ptr = pbl_ptr; 1950 params.pbl_two_level = 0; 1951 1952 if (ib_ctx != NULL) { 1953 ctx = get_qlnxr_ucontext(ib_ctx); 1954 params.dpi = ctx->dpi; 1955 } else { 1956 params.dpi = dev->dpi; 1957 } 1958 1959 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 1960 if (rc) 1961 goto err2; 1962 1963 cq->icid = icid; 1964 cq->sig = QLNXR_CQ_MAGIC_NUMBER; 1965 spin_lock_init(&cq->cq_lock); 1966 1967 if (ib_ctx) { 1968 rc = qlnxr_copy_cq_uresp(dev, cq, udata); 1969 if (rc) 1970 goto err3; 1971 } else { 1972 /* Generate doorbell address. 1973 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. 1974 * TODO: consider moving to device scope as it is a function of 1975 * the device. 1976 * TODO: add ifdef if plan to support 16 bit. 1977 */ 1978 cq->db_addr = dev->db_addr + 1979 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1980 cq->db.data.icid = cq->icid; 1981 cq->db.data.params = DB_AGG_CMD_SET << 1982 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1983 1984 /* point to the very last element, passing it we will toggle */ 1985 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); 1986 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1987 1988 /* must be different from pbl_toggle */ 1989 cq->latest_cqe = NULL; 1990 consume_cqe(cq); 1991 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 1992 } 1993 1994 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," 1995 " number of entries = 0x%x\n", 1996 cq->icid, cq, params.cq_size); 1997 QL_DPRINT12(ha,"cq_addr = %p\n", cq); 1998 return &cq->ibcq; 1999 2000 err3: 2001 destroy_iparams.icid = cq->icid; 2002 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); 2003 err2: 2004 if (udata) 2005 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2006 else 2007 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2008 err1: 2009 if (udata) 2010 ib_umem_release(cq->q.umem); 2011 err0: 2012 kfree(cq); 2013 2014 QL_DPRINT12(ha, "exit error\n"); 2015 2016 return ERR_PTR(-EINVAL); 2017 } 2018 2019 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 2020 { 2021 int status = 0; 2022 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2023 qlnx_host_t *ha; 2024 2025 ha = dev->ha; 2026 2027 QL_DPRINT12(ha, "enter/exit\n"); 2028 2029 return status; 2030 } 2031 2032 int 2033 qlnxr_destroy_cq(struct ib_cq *ibcq) 2034 { 2035 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2036 struct ecore_rdma_destroy_cq_out_params oparams; 2037 struct ecore_rdma_destroy_cq_in_params iparams; 2038 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 2039 int rc = 0; 2040 qlnx_host_t *ha; 2041 2042 ha = dev->ha; 2043 2044 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); 2045 2046 cq->destroyed = 1; 2047 2048 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation 2049 * that all completions with notification are dealt with. The rest 2050 * of the completions are not interesting 2051 */ 2052 2053 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 2054 2055 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { 2056 iparams.icid = cq->icid; 2057 2058 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 2059 2060 if (rc) { 2061 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", 2062 cq->icid); 2063 return rc; 2064 } 2065 2066 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); 2067 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2068 } 2069 2070 if (ibcq->uobject && ibcq->uobject->context) { 2071 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2072 ib_umem_release(cq->q.umem); 2073 } 2074 2075 cq->sig = ~cq->sig; 2076 2077 kfree(cq); 2078 2079 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); 2080 2081 return rc; 2082 } 2083 2084 static int 2085 qlnxr_check_qp_attrs(struct ib_pd *ibpd, 2086 struct qlnxr_dev *dev, 2087 struct ib_qp_init_attr *attrs, 2088 struct ib_udata *udata) 2089 { 2090 struct ecore_rdma_device *qattr; 2091 qlnx_host_t *ha; 2092 2093 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2094 ha = dev->ha; 2095 2096 QL_DPRINT12(ha, "enter\n"); 2097 2098 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); 2099 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); 2100 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); 2101 2102 #if __FreeBSD_version < 1102000 2103 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type); 2104 #endif 2105 2106 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); 2107 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); 2108 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); 2109 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); 2110 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); 2111 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", 2112 attrs->cap.max_inline_data); 2113 2114 #if __FreeBSD_version < 1102000 2115 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n", 2116 attrs->cap.qpg_tss_mask_sz); 2117 #endif 2118 2119 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); 2120 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); 2121 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); 2122 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); 2123 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); 2124 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", 2125 (void *)qattr->sys_image_guid); 2126 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); 2127 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); 2128 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); 2129 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); 2130 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); 2131 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); 2132 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", 2133 qattr->max_qp_resp_rd_atomic_resc); 2134 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", 2135 qattr->max_qp_req_rd_atomic_resc); 2136 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", 2137 qattr->max_dev_resp_rd_atomic_resc); 2138 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); 2139 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); 2140 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); 2141 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); 2142 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); 2143 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); 2144 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); 2145 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); 2146 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", 2147 qattr->max_mr_mw_fmr_pbl); 2148 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", 2149 (void *)qattr->max_mr_mw_fmr_size); 2150 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); 2151 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); 2152 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); 2153 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); 2154 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", 2155 qattr->max_stats_queues); 2156 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); 2157 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", 2158 (void *)qattr->page_size_caps); 2159 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); 2160 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); 2161 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", 2162 qattr->bad_pkey_counter); 2163 2164 if ((attrs->qp_type == IB_QPT_GSI) && udata) { 2165 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); 2166 return -EINVAL; 2167 } 2168 2169 if (udata && !(ibpd->uobject && ibpd->uobject->context)) { 2170 QL_DPRINT12(ha, "called from user without context\n"); 2171 return -EINVAL; 2172 } 2173 2174 /* QP0... attrs->qp_type == IB_QPT_GSI */ 2175 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 2176 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 2177 attrs->qp_type); 2178 return -EINVAL; 2179 } 2180 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { 2181 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); 2182 return -EINVAL; 2183 } 2184 /* Skip the check for QP1 to support CM size of 128 */ 2185 if (attrs->cap.max_send_wr > qattr->max_wqe) { 2186 QL_DPRINT12(ha, "cannot create a SQ with %d elements " 2187 " (max_send_wr=0x%x)\n", 2188 attrs->cap.max_send_wr, qattr->max_wqe); 2189 return -EINVAL; 2190 } 2191 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { 2192 QL_DPRINT12(ha, "cannot create a RQ with %d elements" 2193 " (max_recv_wr=0x%x)\n", 2194 attrs->cap.max_recv_wr, qattr->max_wqe); 2195 return -EINVAL; 2196 } 2197 if (attrs->cap.max_inline_data > qattr->max_inline) { 2198 QL_DPRINT12(ha, 2199 "unsupported inline data size=0x%x " 2200 "requested (max_inline=0x%x)\n", 2201 attrs->cap.max_inline_data, qattr->max_inline); 2202 return -EINVAL; 2203 } 2204 if (attrs->cap.max_send_sge > qattr->max_sge) { 2205 QL_DPRINT12(ha, 2206 "unsupported send_sge=0x%x " 2207 "requested (max_send_sge=0x%x)\n", 2208 attrs->cap.max_send_sge, qattr->max_sge); 2209 return -EINVAL; 2210 } 2211 if (attrs->cap.max_recv_sge > qattr->max_sge) { 2212 QL_DPRINT12(ha, 2213 "unsupported recv_sge=0x%x requested " 2214 " (max_recv_sge=0x%x)\n", 2215 attrs->cap.max_recv_sge, qattr->max_sge); 2216 return -EINVAL; 2217 } 2218 /* unprivileged user space cannot create special QP */ 2219 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 2220 QL_DPRINT12(ha, 2221 "userspace can't create special QPs of type=0x%x\n", 2222 attrs->qp_type); 2223 return -EINVAL; 2224 } 2225 /* allow creating only one GSI type of QP */ 2226 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 2227 QL_DPRINT12(ha, 2228 "create qp: GSI special QPs already created.\n"); 2229 return -EINVAL; 2230 } 2231 2232 /* verify consumer QPs are not trying to use GSI QP's CQ */ 2233 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 2234 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); 2235 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); 2236 2237 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || 2238 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { 2239 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); 2240 return -EINVAL; 2241 } 2242 } 2243 QL_DPRINT12(ha, "exit\n"); 2244 return 0; 2245 } 2246 2247 static int 2248 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 2249 struct qlnxr_srq *srq, 2250 struct ib_udata *udata) 2251 { 2252 struct qlnxr_create_srq_uresp uresp; 2253 qlnx_host_t *ha; 2254 int rc; 2255 2256 ha = dev->ha; 2257 2258 QL_DPRINT12(ha, "enter\n"); 2259 2260 memset(&uresp, 0, sizeof(uresp)); 2261 2262 uresp.srq_id = srq->srq_id; 2263 2264 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2265 2266 QL_DPRINT12(ha, "exit [%d]\n", rc); 2267 return rc; 2268 } 2269 2270 static void 2271 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, 2272 struct qlnxr_create_qp_uresp *uresp, 2273 struct qlnxr_qp *qp) 2274 { 2275 qlnx_host_t *ha; 2276 2277 ha = dev->ha; 2278 2279 /* Return if QP is associated with SRQ instead of RQ */ 2280 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); 2281 2282 if (qp->srq) 2283 return; 2284 2285 /* iWARP requires two doorbells per RQ. */ 2286 if (QLNX_IS_IWARP(dev)) { 2287 uresp->rq_db_offset = 2288 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2289 uresp->rq_db2_offset = 2290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2291 2292 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " 2293 "uresp->rq_db2_offset = 0x%x\n", 2294 uresp->rq_db_offset, uresp->rq_db2_offset); 2295 } else { 2296 uresp->rq_db_offset = 2297 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2298 } 2299 uresp->rq_icid = qp->icid; 2300 2301 QL_DPRINT12(ha, "exit\n"); 2302 return; 2303 } 2304 2305 static void 2306 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, 2307 struct qlnxr_create_qp_uresp *uresp, 2308 struct qlnxr_qp *qp) 2309 { 2310 qlnx_host_t *ha; 2311 2312 ha = dev->ha; 2313 2314 QL_DPRINT12(ha, "enter\n"); 2315 2316 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2317 2318 /* iWARP uses the same cid for rq and sq*/ 2319 if (QLNX_IS_IWARP(dev)) { 2320 uresp->sq_icid = qp->icid; 2321 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); 2322 } else 2323 uresp->sq_icid = qp->icid + 1; 2324 2325 QL_DPRINT12(ha, "exit\n"); 2326 return; 2327 } 2328 2329 static int 2330 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, 2331 struct qlnxr_qp *qp, 2332 struct ib_udata *udata) 2333 { 2334 int rc; 2335 struct qlnxr_create_qp_uresp uresp; 2336 qlnx_host_t *ha; 2337 2338 ha = dev->ha; 2339 2340 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); 2341 2342 memset(&uresp, 0, sizeof(uresp)); 2343 qlnxr_copy_sq_uresp(dev, &uresp, qp); 2344 qlnxr_copy_rq_uresp(dev, &uresp, qp); 2345 2346 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 2347 uresp.qp_id = qp->qp_id; 2348 2349 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2350 2351 QL_DPRINT12(ha, "exit [%d]\n", rc); 2352 return rc; 2353 } 2354 2355 static void 2356 qlnxr_set_common_qp_params(struct qlnxr_dev *dev, 2357 struct qlnxr_qp *qp, 2358 struct qlnxr_pd *pd, 2359 struct ib_qp_init_attr *attrs) 2360 { 2361 qlnx_host_t *ha; 2362 2363 ha = dev->ha; 2364 2365 QL_DPRINT12(ha, "enter\n"); 2366 2367 spin_lock_init(&qp->q_lock); 2368 2369 atomic_set(&qp->refcnt, 1); 2370 qp->pd = pd; 2371 qp->sig = QLNXR_QP_MAGIC_NUMBER; 2372 qp->qp_type = attrs->qp_type; 2373 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2374 qp->sq.max_sges = attrs->cap.max_send_sge; 2375 qp->state = ECORE_ROCE_QP_STATE_RESET; 2376 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 2377 qp->sq_cq = get_qlnxr_cq(attrs->send_cq); 2378 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); 2379 qp->dev = dev; 2380 2381 if (!attrs->srq) { 2382 /* QP is associated with RQ instead of SRQ */ 2383 qp->rq.max_sges = attrs->cap.max_recv_sge; 2384 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 2385 qp->rq.max_sges, qp->rq_cq->icid); 2386 } else { 2387 qp->srq = get_qlnxr_srq(attrs->srq); 2388 } 2389 2390 QL_DPRINT12(ha, 2391 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," 2392 " state = %d, signaled = %d, use_srq=%d\n", 2393 pd->pd_id, qp->qp_type, qp->max_inline_data, 2394 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); 2395 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 2396 qp->sq.max_sges, qp->sq_cq->icid); 2397 return; 2398 } 2399 2400 static int 2401 qlnxr_check_srq_params(struct ib_pd *ibpd, 2402 struct qlnxr_dev *dev, 2403 struct ib_srq_init_attr *attrs) 2404 { 2405 struct ecore_rdma_device *qattr; 2406 qlnx_host_t *ha; 2407 2408 ha = dev->ha; 2409 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2410 2411 QL_DPRINT12(ha, "enter\n"); 2412 2413 if (attrs->attr.max_wr > qattr->max_srq_wqe) { 2414 QL_DPRINT12(ha, "unsupported srq_wr=0x%x" 2415 " requested (max_srq_wr=0x%x)\n", 2416 attrs->attr.max_wr, qattr->max_srq_wr); 2417 return -EINVAL; 2418 } 2419 2420 if (attrs->attr.max_sge > qattr->max_sge) { 2421 QL_DPRINT12(ha, 2422 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 2423 attrs->attr.max_sge, qattr->max_sge); 2424 return -EINVAL; 2425 } 2426 2427 if (attrs->attr.srq_limit > attrs->attr.max_wr) { 2428 QL_DPRINT12(ha, 2429 "unsupported srq_limit=0x%x requested" 2430 " (max_srq_limit=0x%x)\n", 2431 attrs->attr.srq_limit, attrs->attr.srq_limit); 2432 return -EINVAL; 2433 } 2434 2435 QL_DPRINT12(ha, "exit\n"); 2436 return 0; 2437 } 2438 2439 static void 2440 qlnxr_free_srq_user_params(struct qlnxr_srq *srq) 2441 { 2442 struct qlnxr_dev *dev = srq->dev; 2443 qlnx_host_t *ha; 2444 2445 ha = dev->ha; 2446 2447 QL_DPRINT12(ha, "enter\n"); 2448 2449 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2450 ib_umem_release(srq->usrq.umem); 2451 ib_umem_release(srq->prod_umem); 2452 2453 QL_DPRINT12(ha, "exit\n"); 2454 return; 2455 } 2456 2457 static void 2458 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) 2459 { 2460 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2461 struct qlnxr_dev *dev = srq->dev; 2462 qlnx_host_t *ha; 2463 2464 ha = dev->ha; 2465 2466 QL_DPRINT12(ha, "enter\n"); 2467 2468 ecore_chain_free(dev->cdev, &hw_srq->pbl); 2469 2470 qlnx_dma_free_coherent(&dev->cdev, 2471 hw_srq->virt_prod_pair_addr, 2472 hw_srq->phy_prod_pair_addr, 2473 sizeof(struct rdma_srq_producers)); 2474 2475 QL_DPRINT12(ha, "exit\n"); 2476 2477 return; 2478 } 2479 2480 static int 2481 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 2482 struct qlnxr_srq *srq, 2483 struct qlnxr_create_srq_ureq *ureq, 2484 int access, int dmasync) 2485 { 2486 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2487 struct ib_umem_chunk *chunk; 2488 #endif 2489 struct scatterlist *sg; 2490 int rc; 2491 struct qlnxr_dev *dev = srq->dev; 2492 qlnx_host_t *ha; 2493 2494 ha = dev->ha; 2495 2496 QL_DPRINT12(ha, "enter\n"); 2497 2498 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 2499 ureq->srq_len, access, dmasync, 1); 2500 if (rc) 2501 return rc; 2502 2503 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 2504 sizeof(struct rdma_srq_producers), 2505 access, dmasync); 2506 if (IS_ERR(srq->prod_umem)) { 2507 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2508 ib_umem_release(srq->usrq.umem); 2509 2510 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", 2511 PTR_ERR(srq->prod_umem)); 2512 2513 return PTR_ERR(srq->prod_umem); 2514 } 2515 2516 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2517 chunk = container_of((&srq->prod_umem->chunk_list)->next, 2518 typeof(*chunk), list); 2519 sg = &chunk->page_list[0]; 2520 #else 2521 sg = srq->prod_umem->sg_head.sgl; 2522 #endif 2523 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 2524 2525 QL_DPRINT12(ha, "exit\n"); 2526 return 0; 2527 } 2528 2529 static int 2530 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 2531 struct qlnxr_dev *dev, 2532 struct ib_srq_init_attr *init_attr) 2533 { 2534 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2535 dma_addr_t phy_prod_pair_addr; 2536 u32 num_elems, max_wr; 2537 void *va; 2538 int rc; 2539 qlnx_host_t *ha; 2540 2541 ha = dev->ha; 2542 2543 QL_DPRINT12(ha, "enter\n"); 2544 2545 va = qlnx_dma_alloc_coherent(&dev->cdev, 2546 &phy_prod_pair_addr, 2547 sizeof(struct rdma_srq_producers)); 2548 if (!va) { 2549 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); 2550 return -ENOMEM; 2551 } 2552 2553 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 2554 hw_srq->virt_prod_pair_addr = va; 2555 2556 max_wr = init_attr->attr.max_wr; 2557 2558 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; 2559 2560 rc = ecore_chain_alloc(dev->cdev, 2561 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2562 ECORE_CHAIN_MODE_PBL, 2563 ECORE_CHAIN_CNT_TYPE_U32, 2564 num_elems, 2565 ECORE_RDMA_SRQ_WQE_ELEM_SIZE, 2566 &hw_srq->pbl, NULL); 2567 2568 if (rc) { 2569 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); 2570 goto err0; 2571 } 2572 2573 hw_srq->max_wr = max_wr; 2574 hw_srq->num_elems = num_elems; 2575 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; 2576 2577 QL_DPRINT12(ha, "exit\n"); 2578 return 0; 2579 2580 err0: 2581 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, 2582 sizeof(struct rdma_srq_producers)); 2583 2584 QL_DPRINT12(ha, "exit [%d]\n", rc); 2585 return rc; 2586 } 2587 2588 static inline void 2589 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, 2590 struct qlnxr_pd *pd, 2591 struct qlnxr_qp *qp, 2592 struct ib_qp_init_attr *attrs, 2593 bool fmr_and_reserved_lkey, 2594 struct ecore_rdma_create_qp_in_params *params) 2595 { 2596 qlnx_host_t *ha; 2597 2598 ha = dev->ha; 2599 2600 QL_DPRINT12(ha, "enter\n"); 2601 2602 /* QP handle to be written in an async event */ 2603 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 2604 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 2605 2606 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 2607 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 2608 params->pd = pd->pd_id; 2609 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 2610 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 2611 params->stats_queue = 0; 2612 2613 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 2614 2615 if (qp->srq) { 2616 /* QP is associated with SRQ instead of RQ */ 2617 params->srq_id = qp->srq->srq_id; 2618 params->use_srq = true; 2619 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 2620 params->srq_id, params->use_srq); 2621 return; 2622 } 2623 2624 params->srq_id = 0; 2625 params->use_srq = false; 2626 2627 QL_DPRINT12(ha, "exit\n"); 2628 return; 2629 } 2630 2631 static inline void 2632 qlnxr_qp_user_print( struct qlnxr_dev *dev, 2633 struct qlnxr_qp *qp) 2634 { 2635 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " 2636 "rq_addr=0x%llx, rq_len=%zd\n", 2637 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, 2638 qp->urq.buf_len); 2639 return; 2640 } 2641 2642 static int 2643 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) 2644 { 2645 u32 newid; 2646 int rc; 2647 qlnx_host_t *ha; 2648 2649 ha = dev->ha; 2650 2651 QL_DPRINT12(ha, "enter\n"); 2652 2653 if (!QLNX_IS_IWARP(dev)) 2654 return 0; 2655 2656 do { 2657 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { 2658 QL_DPRINT11(ha, "idr_pre_get failed\n"); 2659 return -ENOMEM; 2660 } 2661 2662 mtx_lock(&dev->idr_lock); 2663 2664 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); 2665 2666 mtx_unlock(&dev->idr_lock); 2667 2668 } while (rc == -EAGAIN); 2669 2670 QL_DPRINT12(ha, "exit [%d]\n", rc); 2671 2672 return rc; 2673 } 2674 2675 static void 2676 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) 2677 { 2678 qlnx_host_t *ha; 2679 2680 ha = dev->ha; 2681 2682 QL_DPRINT12(ha, "enter\n"); 2683 2684 if (!QLNX_IS_IWARP(dev)) 2685 return; 2686 2687 mtx_lock(&dev->idr_lock); 2688 idr_remove(&dev->qpidr, id); 2689 mtx_unlock(&dev->idr_lock); 2690 2691 QL_DPRINT12(ha, "exit \n"); 2692 2693 return; 2694 } 2695 2696 static inline void 2697 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, 2698 struct qlnxr_qp *qp, 2699 struct ecore_rdma_create_qp_out_params *out_params) 2700 { 2701 qlnx_host_t *ha; 2702 2703 ha = dev->ha; 2704 2705 QL_DPRINT12(ha, "enter\n"); 2706 2707 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 2708 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 2709 2710 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 2711 &qp->usq.pbl_info); 2712 2713 if (qp->srq) { 2714 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); 2715 return; 2716 } 2717 2718 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 2719 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 2720 2721 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 2722 &qp->urq.pbl_info); 2723 2724 QL_DPRINT12(ha, "exit\n"); 2725 return; 2726 } 2727 2728 static int 2729 qlnxr_create_user_qp(struct qlnxr_dev *dev, 2730 struct qlnxr_qp *qp, 2731 struct ib_pd *ibpd, 2732 struct ib_udata *udata, 2733 struct ib_qp_init_attr *attrs) 2734 { 2735 struct ecore_rdma_destroy_qp_out_params d_out_params; 2736 struct ecore_rdma_create_qp_in_params in_params; 2737 struct ecore_rdma_create_qp_out_params out_params; 2738 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2739 struct ib_ucontext *ib_ctx = NULL; 2740 struct qlnxr_ucontext *ctx = NULL; 2741 struct qlnxr_create_qp_ureq ureq; 2742 int alloc_and_init = QLNX_IS_ROCE(dev); 2743 int rc = -EINVAL; 2744 qlnx_host_t *ha; 2745 2746 ha = dev->ha; 2747 2748 QL_DPRINT12(ha, "enter\n"); 2749 2750 ib_ctx = ibpd->uobject->context; 2751 ctx = get_qlnxr_ucontext(ib_ctx); 2752 2753 memset(&ureq, 0, sizeof(ureq)); 2754 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 2755 2756 if (rc) { 2757 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); 2758 return rc; 2759 } 2760 2761 /* SQ - read access only (0), dma sync not required (0) */ 2762 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 2763 ureq.sq_len, 0, 0, 2764 alloc_and_init); 2765 if (rc) { 2766 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2767 return rc; 2768 } 2769 2770 if (!qp->srq) { 2771 /* RQ - read access only (0), dma sync not required (0) */ 2772 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 2773 ureq.rq_len, 0, 0, 2774 alloc_and_init); 2775 2776 if (rc) { 2777 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2778 return rc; 2779 } 2780 } 2781 2782 memset(&in_params, 0, sizeof(in_params)); 2783 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 2784 in_params.qp_handle_lo = ureq.qp_handle_lo; 2785 in_params.qp_handle_hi = ureq.qp_handle_hi; 2786 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 2787 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 2788 2789 if (!qp->srq) { 2790 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 2791 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 2792 } 2793 2794 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); 2795 2796 if (!qp->ecore_qp) { 2797 rc = -ENOMEM; 2798 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2799 goto err1; 2800 } 2801 2802 if (QLNX_IS_IWARP(dev)) 2803 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); 2804 2805 qp->qp_id = out_params.qp_id; 2806 qp->icid = out_params.icid; 2807 2808 rc = qlnxr_copy_qp_uresp(dev, qp, udata); 2809 2810 if (rc) { 2811 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); 2812 goto err; 2813 } 2814 2815 qlnxr_qp_user_print(dev, qp); 2816 2817 QL_DPRINT12(ha, "exit\n"); 2818 return 0; 2819 err: 2820 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2821 2822 if (rc) 2823 QL_DPRINT12(ha, "fatal fault\n"); 2824 2825 err1: 2826 qlnxr_cleanup_user(dev, qp); 2827 2828 QL_DPRINT12(ha, "exit[%d]\n", rc); 2829 return rc; 2830 } 2831 2832 static void 2833 qlnxr_set_roce_db_info(struct qlnxr_dev *dev, 2834 struct qlnxr_qp *qp) 2835 { 2836 qlnx_host_t *ha; 2837 2838 ha = dev->ha; 2839 2840 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2841 2842 qp->sq.db = dev->db_addr + 2843 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2844 qp->sq.db_data.data.icid = qp->icid + 1; 2845 2846 if (!qp->srq) { 2847 qp->rq.db = dev->db_addr + 2848 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2849 qp->rq.db_data.data.icid = qp->icid; 2850 } 2851 2852 QL_DPRINT12(ha, "exit\n"); 2853 return; 2854 } 2855 2856 static void 2857 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, 2858 struct qlnxr_qp *qp) 2859 2860 { 2861 qlnx_host_t *ha; 2862 2863 ha = dev->ha; 2864 2865 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2866 2867 qp->sq.db = dev->db_addr + 2868 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2869 qp->sq.db_data.data.icid = qp->icid; 2870 2871 if (!qp->srq) { 2872 qp->rq.db = dev->db_addr + 2873 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2874 qp->rq.db_data.data.icid = qp->icid; 2875 2876 qp->rq.iwarp_db2 = dev->db_addr + 2877 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2878 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2879 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2880 } 2881 2882 QL_DPRINT12(ha, 2883 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" 2884 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" 2885 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" 2886 " qp->rq.iwarp_db2.data.prod_val =0x%x\n", 2887 qp->sq.db, qp->sq.db_data.data.icid, 2888 qp->rq.db, qp->rq.db_data.data.icid, 2889 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, 2890 qp->rq.iwarp_db2_data.data.value); 2891 2892 QL_DPRINT12(ha, "exit\n"); 2893 return; 2894 } 2895 2896 static int 2897 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, 2898 struct qlnxr_qp *qp, 2899 struct ecore_rdma_create_qp_in_params *in_params, 2900 u32 n_sq_elems, 2901 u32 n_rq_elems) 2902 { 2903 struct ecore_rdma_create_qp_out_params out_params; 2904 int rc; 2905 qlnx_host_t *ha; 2906 2907 ha = dev->ha; 2908 2909 QL_DPRINT12(ha, "enter\n"); 2910 2911 rc = ecore_chain_alloc( 2912 dev->cdev, 2913 ECORE_CHAIN_USE_TO_PRODUCE, 2914 ECORE_CHAIN_MODE_PBL, 2915 ECORE_CHAIN_CNT_TYPE_U32, 2916 n_sq_elems, 2917 QLNXR_SQE_ELEMENT_SIZE, 2918 &qp->sq.pbl, 2919 NULL); 2920 2921 if (rc) { 2922 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); 2923 return rc; 2924 } 2925 2926 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); 2927 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); 2928 2929 if (!qp->srq) { 2930 rc = ecore_chain_alloc( 2931 dev->cdev, 2932 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2933 ECORE_CHAIN_MODE_PBL, 2934 ECORE_CHAIN_CNT_TYPE_U32, 2935 n_rq_elems, 2936 QLNXR_RQE_ELEMENT_SIZE, 2937 &qp->rq.pbl, 2938 NULL); 2939 2940 if (rc) { 2941 QL_DPRINT11(ha, 2942 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); 2943 return rc; 2944 } 2945 2946 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); 2947 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); 2948 } 2949 2950 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2951 2952 if (!qp->ecore_qp) { 2953 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); 2954 return -EINVAL; 2955 } 2956 2957 qp->qp_id = out_params.qp_id; 2958 qp->icid = out_params.icid; 2959 2960 qlnxr_set_roce_db_info(dev, qp); 2961 2962 QL_DPRINT12(ha, "exit\n"); 2963 return 0; 2964 } 2965 2966 static int 2967 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, 2968 struct qlnxr_qp *qp, 2969 struct ecore_rdma_create_qp_in_params *in_params, 2970 u32 n_sq_elems, 2971 u32 n_rq_elems) 2972 { 2973 struct ecore_rdma_destroy_qp_out_params d_out_params; 2974 struct ecore_rdma_create_qp_out_params out_params; 2975 struct ecore_chain_ext_pbl ext_pbl; 2976 int rc; 2977 qlnx_host_t *ha; 2978 2979 ha = dev->ha; 2980 2981 QL_DPRINT12(ha, "enter\n"); 2982 2983 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, 2984 QLNXR_SQE_ELEMENT_SIZE, 2985 ECORE_CHAIN_MODE_PBL); 2986 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, 2987 QLNXR_RQE_ELEMENT_SIZE, 2988 ECORE_CHAIN_MODE_PBL); 2989 2990 QL_DPRINT12(ha, "n_sq_elems = 0x%x" 2991 " n_rq_elems = 0x%x in_params\n" 2992 "\t\t\tqp_handle_lo\t\t= 0x%08x\n" 2993 "\t\t\tqp_handle_hi\t\t= 0x%08x\n" 2994 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" 2995 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" 2996 "\t\t\tuse_srq\t\t\t= 0x%x\n" 2997 "\t\t\tsignal_all\t\t= 0x%x\n" 2998 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" 2999 "\t\t\tpd\t\t\t= 0x%x\n" 3000 "\t\t\tdpi\t\t\t= 0x%x\n" 3001 "\t\t\tsq_cq_id\t\t\t= 0x%x\n" 3002 "\t\t\tsq_num_pages\t\t= 0x%x\n" 3003 "\t\t\tsq_pbl_ptr\t\t= %p\n" 3004 "\t\t\tmax_sq_sges\t\t= 0x%x\n" 3005 "\t\t\trq_cq_id\t\t\t= 0x%x\n" 3006 "\t\t\trq_num_pages\t\t= 0x%x\n" 3007 "\t\t\trq_pbl_ptr\t\t= %p\n" 3008 "\t\t\tsrq_id\t\t\t= 0x%x\n" 3009 "\t\t\tstats_queue\t\t= 0x%x\n", 3010 n_sq_elems, n_rq_elems, 3011 in_params->qp_handle_lo, 3012 in_params->qp_handle_hi, 3013 in_params->qp_handle_async_lo, 3014 in_params->qp_handle_async_hi, 3015 in_params->use_srq, 3016 in_params->signal_all, 3017 in_params->fmr_and_reserved_lkey, 3018 in_params->pd, 3019 in_params->dpi, 3020 in_params->sq_cq_id, 3021 in_params->sq_num_pages, 3022 (void *)in_params->sq_pbl_ptr, 3023 in_params->max_sq_sges, 3024 in_params->rq_cq_id, 3025 in_params->rq_num_pages, 3026 (void *)in_params->rq_pbl_ptr, 3027 in_params->srq_id, 3028 in_params->stats_queue ); 3029 3030 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); 3031 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); 3032 3033 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 3034 3035 if (!qp->ecore_qp) { 3036 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 3037 return -EINVAL; 3038 } 3039 3040 /* Now we allocate the chain */ 3041 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 3042 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 3043 3044 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3045 "ext_pbl.p_pbl_phys = %p\n", 3046 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3047 3048 rc = ecore_chain_alloc( 3049 dev->cdev, 3050 ECORE_CHAIN_USE_TO_PRODUCE, 3051 ECORE_CHAIN_MODE_PBL, 3052 ECORE_CHAIN_CNT_TYPE_U32, 3053 n_sq_elems, 3054 QLNXR_SQE_ELEMENT_SIZE, 3055 &qp->sq.pbl, 3056 &ext_pbl); 3057 3058 if (rc) { 3059 QL_DPRINT11(ha, 3060 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); 3061 goto err; 3062 } 3063 3064 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 3065 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 3066 3067 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3068 "ext_pbl.p_pbl_phys = %p\n", 3069 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3070 3071 if (!qp->srq) { 3072 rc = ecore_chain_alloc( 3073 dev->cdev, 3074 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 3075 ECORE_CHAIN_MODE_PBL, 3076 ECORE_CHAIN_CNT_TYPE_U32, 3077 n_rq_elems, 3078 QLNXR_RQE_ELEMENT_SIZE, 3079 &qp->rq.pbl, 3080 &ext_pbl); 3081 3082 if (rc) { 3083 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" 3084 " failed rc = %d\n", rc); 3085 goto err; 3086 } 3087 } 3088 3089 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", 3090 out_params.qp_id, out_params.icid); 3091 3092 qp->qp_id = out_params.qp_id; 3093 qp->icid = out_params.icid; 3094 3095 qlnxr_set_iwarp_db_info(dev, qp); 3096 3097 QL_DPRINT12(ha, "exit\n"); 3098 return 0; 3099 3100 err: 3101 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 3102 3103 QL_DPRINT12(ha, "exit rc = %d\n", rc); 3104 return rc; 3105 } 3106 3107 static int 3108 qlnxr_create_kernel_qp(struct qlnxr_dev *dev, 3109 struct qlnxr_qp *qp, 3110 struct ib_pd *ibpd, 3111 struct ib_qp_init_attr *attrs) 3112 { 3113 struct ecore_rdma_create_qp_in_params in_params; 3114 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3115 int rc = -EINVAL; 3116 u32 n_rq_elems; 3117 u32 n_sq_elems; 3118 u32 n_sq_entries; 3119 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3120 qlnx_host_t *ha; 3121 3122 ha = dev->ha; 3123 3124 QL_DPRINT12(ha, "enter\n"); 3125 3126 memset(&in_params, 0, sizeof(in_params)); 3127 3128 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in 3129 * the ring. The ring should allow at least a single WR, even if the 3130 * user requested none, due to allocation issues. 3131 * We should add an extra WR since the prod and cons indices of 3132 * wqe_wr_id are managed in such a way that the WQ is considered full 3133 * when (prod+1)%max_wr==cons. We currently don't do that because we 3134 * double the number of entries due an iSER issue that pushes far more 3135 * WRs than indicated. If we decline its ib_post_send() then we get 3136 * error prints in the dmesg we'd like to avoid. 3137 */ 3138 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 3139 qattr->max_wqe); 3140 3141 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 3142 GFP_KERNEL); 3143 if (!qp->wqe_wr_id) { 3144 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); 3145 return -ENOMEM; 3146 } 3147 3148 /* QP handle to be written in CQE */ 3149 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); 3150 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); 3151 3152 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in 3153 * the ring. There ring should allow at least a single WR, even if the 3154 * user requested none, due to allocation issues. 3155 */ 3156 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); 3157 3158 /* Allocate driver internal RQ array */ 3159 if (!qp->srq) { 3160 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 3161 GFP_KERNEL); 3162 if (!qp->rqe_wr_id) { 3163 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); 3164 kfree(qp->wqe_wr_id); 3165 return -ENOMEM; 3166 } 3167 } 3168 3169 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 3170 3171 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 3172 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 3173 3174 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 3175 in_params.fmr_and_reserved_lkey = true; 3176 in_params.pd = pd->pd_id; 3177 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 3178 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 3179 in_params.stats_queue = 0; 3180 3181 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 3182 3183 if (qp->srq) { 3184 /* QP is associated with SRQ instead of RQ */ 3185 in_params.srq_id = qp->srq->srq_id; 3186 in_params.use_srq = true; 3187 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 3188 in_params.srq_id, in_params.use_srq); 3189 } else { 3190 in_params.srq_id = 0; 3191 in_params.use_srq = false; 3192 } 3193 3194 n_sq_entries = attrs->cap.max_send_wr; 3195 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); 3196 n_sq_entries = max_t(u32, n_sq_entries, 1); 3197 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 3198 3199 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 3200 3201 if (QLNX_IS_ROCE(dev)) { 3202 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, 3203 n_sq_elems, n_rq_elems); 3204 } else { 3205 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, 3206 n_sq_elems, n_rq_elems); 3207 } 3208 3209 if (rc) 3210 qlnxr_cleanup_kernel(dev, qp); 3211 3212 QL_DPRINT12(ha, "exit [%d]\n", rc); 3213 return rc; 3214 } 3215 3216 struct ib_qp * 3217 qlnxr_create_qp(struct ib_pd *ibpd, 3218 struct ib_qp_init_attr *attrs, 3219 struct ib_udata *udata) 3220 { 3221 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); 3222 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3223 struct qlnxr_qp *qp; 3224 int rc = 0; 3225 qlnx_host_t *ha; 3226 3227 ha = dev->ha; 3228 3229 QL_DPRINT12(ha, "enter\n"); 3230 3231 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); 3232 if (rc) { 3233 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); 3234 return ERR_PTR(rc); 3235 } 3236 3237 QL_DPRINT12(ha, "called from %s, event_handle=%p," 3238 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 3239 (udata ? "user library" : "kernel"), 3240 attrs->event_handler, pd, 3241 get_qlnxr_cq(attrs->send_cq), 3242 get_qlnxr_cq(attrs->send_cq)->icid, 3243 get_qlnxr_cq(attrs->recv_cq), 3244 get_qlnxr_cq(attrs->recv_cq)->icid); 3245 3246 qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); 3247 3248 if (!qp) { 3249 QL_DPRINT11(ha, "kzalloc(qp) failed\n"); 3250 return ERR_PTR(-ENOMEM); 3251 } 3252 3253 qlnxr_set_common_qp_params(dev, qp, pd, attrs); 3254 3255 if (attrs->qp_type == IB_QPT_GSI) { 3256 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); 3257 return qlnxr_create_gsi_qp(dev, attrs, qp); 3258 } 3259 3260 if (udata) { 3261 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); 3262 3263 if (rc) { 3264 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); 3265 goto err; 3266 } 3267 } else { 3268 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); 3269 3270 if (rc) { 3271 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); 3272 goto err; 3273 } 3274 } 3275 3276 qp->ibqp.qp_num = qp->qp_id; 3277 3278 rc = qlnxr_idr_add(dev, qp, qp->qp_id); 3279 3280 if (rc) { 3281 QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); 3282 goto err; 3283 } 3284 3285 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); 3286 3287 return &qp->ibqp; 3288 err: 3289 kfree(qp); 3290 3291 QL_DPRINT12(ha, "failed exit\n"); 3292 return ERR_PTR(-EFAULT); 3293 } 3294 3295 static enum ib_qp_state 3296 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) 3297 { 3298 enum ib_qp_state state = IB_QPS_ERR; 3299 3300 switch (qp_state) { 3301 case ECORE_ROCE_QP_STATE_RESET: 3302 state = IB_QPS_RESET; 3303 break; 3304 3305 case ECORE_ROCE_QP_STATE_INIT: 3306 state = IB_QPS_INIT; 3307 break; 3308 3309 case ECORE_ROCE_QP_STATE_RTR: 3310 state = IB_QPS_RTR; 3311 break; 3312 3313 case ECORE_ROCE_QP_STATE_RTS: 3314 state = IB_QPS_RTS; 3315 break; 3316 3317 case ECORE_ROCE_QP_STATE_SQD: 3318 state = IB_QPS_SQD; 3319 break; 3320 3321 case ECORE_ROCE_QP_STATE_ERR: 3322 state = IB_QPS_ERR; 3323 break; 3324 3325 case ECORE_ROCE_QP_STATE_SQE: 3326 state = IB_QPS_SQE; 3327 break; 3328 } 3329 return state; 3330 } 3331 3332 static enum ecore_roce_qp_state 3333 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) 3334 { 3335 enum ecore_roce_qp_state ecore_qp_state; 3336 3337 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3338 3339 switch (qp_state) { 3340 case IB_QPS_RESET: 3341 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; 3342 break; 3343 3344 case IB_QPS_INIT: 3345 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; 3346 break; 3347 3348 case IB_QPS_RTR: 3349 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; 3350 break; 3351 3352 case IB_QPS_RTS: 3353 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; 3354 break; 3355 3356 case IB_QPS_SQD: 3357 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; 3358 break; 3359 3360 case IB_QPS_ERR: 3361 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3362 break; 3363 3364 default: 3365 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3366 break; 3367 } 3368 3369 return (ecore_qp_state); 3370 } 3371 3372 static void 3373 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) 3374 { 3375 ecore_chain_reset(&qph->pbl); 3376 qph->prod = qph->cons = 0; 3377 qph->wqe_cons = 0; 3378 qph->db_data.data.value = cpu_to_le16(0); 3379 3380 return; 3381 } 3382 3383 static int 3384 qlnxr_update_qp_state(struct qlnxr_dev *dev, 3385 struct qlnxr_qp *qp, 3386 enum ecore_roce_qp_state new_state) 3387 { 3388 int status = 0; 3389 uint32_t reg_addr; 3390 struct ecore_dev *cdev; 3391 qlnx_host_t *ha; 3392 3393 ha = dev->ha; 3394 cdev = &ha->cdev; 3395 3396 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", 3397 qp, new_state, qp->state); 3398 3399 if (new_state == qp->state) { 3400 return 0; 3401 } 3402 3403 switch (qp->state) { 3404 case ECORE_ROCE_QP_STATE_RESET: 3405 switch (new_state) { 3406 case ECORE_ROCE_QP_STATE_INIT: 3407 qp->prev_wqe_size = 0; 3408 qlnxr_reset_qp_hwq_info(&qp->sq); 3409 if (!(qp->srq)) 3410 qlnxr_reset_qp_hwq_info(&qp->rq); 3411 break; 3412 default: 3413 status = -EINVAL; 3414 break; 3415 }; 3416 break; 3417 case ECORE_ROCE_QP_STATE_INIT: 3418 /* INIT->XXX */ 3419 switch (new_state) { 3420 case ECORE_ROCE_QP_STATE_RTR: 3421 /* Update doorbell (in case post_recv was done before move to RTR) */ 3422 if (qp->srq) 3423 break; 3424 wmb(); 3425 //writel(qp->rq.db_data.raw, qp->rq.db); 3426 //if (QLNX_IS_IWARP(dev)) 3427 // writel(qp->rq.iwarp_db2_data.raw, 3428 // qp->rq.iwarp_db2); 3429 3430 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 3431 (uint8_t *)cdev->doorbells); 3432 3433 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 3434 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 3435 3436 if (QLNX_IS_IWARP(dev)) { 3437 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 3438 (uint8_t *)cdev->doorbells); 3439 bus_write_4(ha->pci_dbells, reg_addr,\ 3440 qp->rq.iwarp_db2_data.raw); 3441 bus_barrier(ha->pci_dbells, 0, 0,\ 3442 BUS_SPACE_BARRIER_READ); 3443 } 3444 3445 3446 mmiowb(); 3447 break; 3448 case ECORE_ROCE_QP_STATE_ERR: 3449 /* TBD:flush qps... */ 3450 break; 3451 default: 3452 /* invalid state change. */ 3453 status = -EINVAL; 3454 break; 3455 }; 3456 break; 3457 case ECORE_ROCE_QP_STATE_RTR: 3458 /* RTR->XXX */ 3459 switch (new_state) { 3460 case ECORE_ROCE_QP_STATE_RTS: 3461 break; 3462 case ECORE_ROCE_QP_STATE_ERR: 3463 break; 3464 default: 3465 /* invalid state change. */ 3466 status = -EINVAL; 3467 break; 3468 }; 3469 break; 3470 case ECORE_ROCE_QP_STATE_RTS: 3471 /* RTS->XXX */ 3472 switch (new_state) { 3473 case ECORE_ROCE_QP_STATE_SQD: 3474 break; 3475 case ECORE_ROCE_QP_STATE_ERR: 3476 break; 3477 default: 3478 /* invalid state change. */ 3479 status = -EINVAL; 3480 break; 3481 }; 3482 break; 3483 case ECORE_ROCE_QP_STATE_SQD: 3484 /* SQD->XXX */ 3485 switch (new_state) { 3486 case ECORE_ROCE_QP_STATE_RTS: 3487 case ECORE_ROCE_QP_STATE_ERR: 3488 break; 3489 default: 3490 /* invalid state change. */ 3491 status = -EINVAL; 3492 break; 3493 }; 3494 break; 3495 case ECORE_ROCE_QP_STATE_ERR: 3496 /* ERR->XXX */ 3497 switch (new_state) { 3498 case ECORE_ROCE_QP_STATE_RESET: 3499 if ((qp->rq.prod != qp->rq.cons) || 3500 (qp->sq.prod != qp->sq.cons)) { 3501 QL_DPRINT11(ha, 3502 "Error->Reset with rq/sq " 3503 "not empty rq.prod=0x%x rq.cons=0x%x" 3504 " sq.prod=0x%x sq.cons=0x%x\n", 3505 qp->rq.prod, qp->rq.cons, 3506 qp->sq.prod, qp->sq.cons); 3507 status = -EINVAL; 3508 } 3509 break; 3510 default: 3511 status = -EINVAL; 3512 break; 3513 }; 3514 break; 3515 default: 3516 status = -EINVAL; 3517 break; 3518 }; 3519 3520 QL_DPRINT12(ha, "exit\n"); 3521 return status; 3522 } 3523 3524 int 3525 qlnxr_modify_qp(struct ib_qp *ibqp, 3526 struct ib_qp_attr *attr, 3527 int attr_mask, 3528 struct ib_udata *udata) 3529 { 3530 int rc = 0; 3531 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3532 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); 3533 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 3534 enum ib_qp_state old_qp_state, new_qp_state; 3535 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3536 qlnx_host_t *ha; 3537 3538 ha = dev->ha; 3539 3540 QL_DPRINT12(ha, 3541 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", 3542 qp, attr_mask, attr->qp_state, udata); 3543 3544 old_qp_state = qlnxr_get_ibqp_state(qp->state); 3545 if (attr_mask & IB_QP_STATE) 3546 new_qp_state = attr->qp_state; 3547 else 3548 new_qp_state = old_qp_state; 3549 3550 if (QLNX_IS_ROCE(dev)) { 3551 #if __FreeBSD_version >= 1100000 3552 if (!ib_modify_qp_is_ok(old_qp_state, 3553 new_qp_state, 3554 ibqp->qp_type, 3555 attr_mask, 3556 IB_LINK_LAYER_ETHERNET)) { 3557 QL_DPRINT12(ha, 3558 "invalid attribute mask=0x%x" 3559 " specified for qpn=0x%x of type=0x%x \n" 3560 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3561 attr_mask, qp->qp_id, ibqp->qp_type, 3562 old_qp_state, new_qp_state); 3563 rc = -EINVAL; 3564 goto err; 3565 } 3566 #else 3567 if (!ib_modify_qp_is_ok(old_qp_state, 3568 new_qp_state, 3569 ibqp->qp_type, 3570 attr_mask )) { 3571 QL_DPRINT12(ha, 3572 "invalid attribute mask=0x%x" 3573 " specified for qpn=0x%x of type=0x%x \n" 3574 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3575 attr_mask, qp->qp_id, ibqp->qp_type, 3576 old_qp_state, new_qp_state); 3577 rc = -EINVAL; 3578 goto err; 3579 } 3580 3581 #endif /* #if __FreeBSD_version >= 1100000 */ 3582 } 3583 /* translate the masks... */ 3584 if (attr_mask & IB_QP_STATE) { 3585 SET_FIELD(qp_params.modify_flags, 3586 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 3587 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); 3588 } 3589 3590 // TBD consider changing ecore to be a flag as well... 3591 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 3592 qp_params.sqd_async = true; 3593 3594 if (attr_mask & IB_QP_PKEY_INDEX) { 3595 SET_FIELD(qp_params.modify_flags, 3596 ECORE_ROCE_MODIFY_QP_VALID_PKEY, 3597 1); 3598 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { 3599 rc = -EINVAL; 3600 goto err; 3601 } 3602 3603 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; 3604 } 3605 3606 if (attr_mask & IB_QP_QKEY) { 3607 qp->qkey = attr->qkey; 3608 } 3609 3610 /* tbd consider splitting in ecore.. */ 3611 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3612 SET_FIELD(qp_params.modify_flags, 3613 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 3614 qp_params.incoming_rdma_read_en = 3615 attr->qp_access_flags & IB_ACCESS_REMOTE_READ; 3616 qp_params.incoming_rdma_write_en = 3617 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; 3618 qp_params.incoming_atomic_en = 3619 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; 3620 } 3621 3622 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 3623 if (attr_mask & IB_QP_PATH_MTU) { 3624 if (attr->path_mtu < IB_MTU_256 || 3625 attr->path_mtu > IB_MTU_4096) { 3626 QL_DPRINT12(ha, 3627 "Only MTU sizes of 256, 512, 1024," 3628 " 2048 and 4096 are supported " 3629 " attr->path_mtu = [%d]\n", 3630 attr->path_mtu); 3631 3632 rc = -EINVAL; 3633 goto err; 3634 } 3635 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 3636 ib_mtu_enum_to_int( 3637 iboe_get_mtu(dev->ha->ifp->if_mtu))); 3638 } 3639 3640 if (qp->mtu == 0) { 3641 qp->mtu = ib_mtu_enum_to_int( 3642 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3643 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", 3644 qp->mtu); 3645 } 3646 3647 SET_FIELD(qp_params.modify_flags, 3648 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 3649 1); 3650 3651 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; 3652 qp_params.flow_label = attr->ah_attr.grh.flow_label; 3653 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; 3654 3655 qp->sgid_idx = attr->ah_attr.grh.sgid_index; 3656 3657 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); 3658 3659 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); 3660 if (rc) 3661 return rc; 3662 3663 qp_params.use_local_mac = true; 3664 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 3665 3666 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", 3667 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 3668 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 3669 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", 3670 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 3671 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 3672 QL_DPRINT12(ha, 3673 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", 3674 qp_params.remote_mac_addr[0], 3675 qp_params.remote_mac_addr[1], 3676 qp_params.remote_mac_addr[2], 3677 qp_params.remote_mac_addr[3], 3678 qp_params.remote_mac_addr[4], 3679 qp_params.remote_mac_addr[5]); 3680 3681 qp_params.mtu = qp->mtu; 3682 } 3683 3684 if (qp_params.mtu == 0) { 3685 /* stay with current MTU */ 3686 if (qp->mtu) { 3687 qp_params.mtu = qp->mtu; 3688 } else { 3689 qp_params.mtu = ib_mtu_enum_to_int( 3690 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3691 } 3692 } 3693 3694 if (attr_mask & IB_QP_TIMEOUT) { 3695 SET_FIELD(qp_params.modify_flags, \ 3696 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 3697 3698 qp_params.ack_timeout = attr->timeout; 3699 if (attr->timeout) { 3700 u32 temp; 3701 3702 /* 12.7.34 LOCAL ACK TIMEOUT 3703 * Value representing the transport (ACK) timeout for 3704 * use by the remote, expressed as (4.096 μS*2Local ACK 3705 * Timeout) 3706 */ 3707 /* We use 1UL since the temporal value may be overflow 3708 * 32 bits 3709 */ 3710 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 3711 qp_params.ack_timeout = temp; /* FW requires [msec] */ 3712 } 3713 else 3714 qp_params.ack_timeout = 0; /* infinite */ 3715 } 3716 if (attr_mask & IB_QP_RETRY_CNT) { 3717 SET_FIELD(qp_params.modify_flags,\ 3718 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 3719 qp_params.retry_cnt = attr->retry_cnt; 3720 } 3721 3722 if (attr_mask & IB_QP_RNR_RETRY) { 3723 SET_FIELD(qp_params.modify_flags, 3724 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 3725 1); 3726 qp_params.rnr_retry_cnt = attr->rnr_retry; 3727 } 3728 3729 if (attr_mask & IB_QP_RQ_PSN) { 3730 SET_FIELD(qp_params.modify_flags, 3731 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 3732 1); 3733 qp_params.rq_psn = attr->rq_psn; 3734 qp->rq_psn = attr->rq_psn; 3735 } 3736 3737 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 3738 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { 3739 rc = -EINVAL; 3740 QL_DPRINT12(ha, 3741 "unsupported max_rd_atomic=%d, supported=%d\n", 3742 attr->max_rd_atomic, 3743 qattr->max_qp_req_rd_atomic_resc); 3744 goto err; 3745 } 3746 3747 SET_FIELD(qp_params.modify_flags, 3748 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 3749 1); 3750 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 3751 } 3752 3753 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 3754 SET_FIELD(qp_params.modify_flags, 3755 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 3756 1); 3757 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 3758 } 3759 3760 if (attr_mask & IB_QP_SQ_PSN) { 3761 SET_FIELD(qp_params.modify_flags, 3762 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 3763 1); 3764 qp_params.sq_psn = attr->sq_psn; 3765 qp->sq_psn = attr->sq_psn; 3766 } 3767 3768 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 3769 if (attr->max_dest_rd_atomic > 3770 qattr->max_qp_resp_rd_atomic_resc) { 3771 QL_DPRINT12(ha, 3772 "unsupported max_dest_rd_atomic=%d, " 3773 "supported=%d\n", 3774 attr->max_dest_rd_atomic, 3775 qattr->max_qp_resp_rd_atomic_resc); 3776 3777 rc = -EINVAL; 3778 goto err; 3779 } 3780 3781 SET_FIELD(qp_params.modify_flags, 3782 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 3783 1); 3784 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 3785 } 3786 3787 if (attr_mask & IB_QP_DEST_QPN) { 3788 SET_FIELD(qp_params.modify_flags, 3789 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 3790 1); 3791 3792 qp_params.dest_qp = attr->dest_qp_num; 3793 qp->dest_qp_num = attr->dest_qp_num; 3794 } 3795 3796 /* 3797 * Update the QP state before the actual ramrod to prevent a race with 3798 * fast path. Modifying the QP state to error will cause the device to 3799 * flush the CQEs and while polling the flushed CQEs will considered as 3800 * a potential issue if the QP isn't in error state. 3801 */ 3802 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && 3803 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) 3804 qp->state = ECORE_ROCE_QP_STATE_ERR; 3805 3806 if (qp->qp_type != IB_QPT_GSI) 3807 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 3808 3809 if (attr_mask & IB_QP_STATE) { 3810 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 3811 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); 3812 qp->state = qp_params.new_state; 3813 } 3814 3815 err: 3816 QL_DPRINT12(ha, "exit\n"); 3817 return rc; 3818 } 3819 3820 static int 3821 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) 3822 { 3823 int ib_qp_acc_flags = 0; 3824 3825 if (params->incoming_rdma_write_en) 3826 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 3827 if (params->incoming_rdma_read_en) 3828 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 3829 if (params->incoming_atomic_en) 3830 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 3831 if (true) /* FIXME -> local write ?? */ 3832 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 3833 3834 return ib_qp_acc_flags; 3835 } 3836 3837 static enum ib_mtu 3838 qlnxr_mtu_int_to_enum(u16 mtu) 3839 { 3840 enum ib_mtu ib_mtu_size; 3841 3842 switch (mtu) { 3843 case 256: 3844 ib_mtu_size = IB_MTU_256; 3845 break; 3846 3847 case 512: 3848 ib_mtu_size = IB_MTU_512; 3849 break; 3850 3851 case 1024: 3852 ib_mtu_size = IB_MTU_1024; 3853 break; 3854 3855 case 2048: 3856 ib_mtu_size = IB_MTU_2048; 3857 break; 3858 3859 case 4096: 3860 ib_mtu_size = IB_MTU_4096; 3861 break; 3862 3863 default: 3864 ib_mtu_size = IB_MTU_1024; 3865 break; 3866 } 3867 return (ib_mtu_size); 3868 } 3869 3870 int 3871 qlnxr_query_qp(struct ib_qp *ibqp, 3872 struct ib_qp_attr *qp_attr, 3873 int attr_mask, 3874 struct ib_qp_init_attr *qp_init_attr) 3875 { 3876 int rc = 0; 3877 struct ecore_rdma_query_qp_out_params params; 3878 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3879 struct qlnxr_dev *dev = qp->dev; 3880 qlnx_host_t *ha; 3881 3882 ha = dev->ha; 3883 3884 QL_DPRINT12(ha, "enter\n"); 3885 3886 memset(¶ms, 0, sizeof(params)); 3887 3888 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); 3889 if (rc) 3890 goto err; 3891 3892 memset(qp_attr, 0, sizeof(*qp_attr)); 3893 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 3894 3895 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); 3896 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); 3897 3898 /* In some cases in iWARP qelr will ask for the state only */ 3899 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { 3900 QL_DPRINT11(ha, "only state requested\n"); 3901 return 0; 3902 } 3903 3904 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); 3905 qp_attr->path_mig_state = IB_MIG_MIGRATED; 3906 qp_attr->rq_psn = params.rq_psn; 3907 qp_attr->sq_psn = params.sq_psn; 3908 qp_attr->dest_qp_num = params.dest_qp; 3909 3910 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); 3911 3912 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " 3913 "path_mtu = %d qp_access_flags = 0x%x\n", 3914 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, 3915 qp_attr->qp_access_flags); 3916 3917 qp_attr->cap.max_send_wr = qp->sq.max_wr; 3918 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 3919 qp_attr->cap.max_send_sge = qp->sq.max_sges; 3920 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 3921 qp_attr->cap.max_inline_data = qp->max_inline_data; 3922 qp_init_attr->cap = qp_attr->cap; 3923 3924 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], 3925 sizeof(qp_attr->ah_attr.grh.dgid.raw)); 3926 3927 qp_attr->ah_attr.grh.flow_label = params.flow_label; 3928 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; 3929 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; 3930 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; 3931 3932 qp_attr->ah_attr.ah_flags = IB_AH_GRH; 3933 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ 3934 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ 3935 qp_attr->timeout = params.timeout; 3936 qp_attr->rnr_retry = params.rnr_retry; 3937 qp_attr->retry_cnt = params.retry_cnt; 3938 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 3939 qp_attr->pkey_index = params.pkey_index; 3940 qp_attr->port_num = 1; /* FIXME -> check this */ 3941 qp_attr->ah_attr.src_path_bits = 0; 3942 qp_attr->ah_attr.static_rate = 0; 3943 qp_attr->alt_pkey_index = 0; 3944 qp_attr->alt_port_num = 0; 3945 qp_attr->alt_timeout = 0; 3946 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 3947 3948 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; 3949 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 3950 qp_attr->max_rd_atomic = params.max_rd_atomic; 3951 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; 3952 3953 QL_DPRINT12(ha, "max_inline_data=%d\n", 3954 qp_attr->cap.max_inline_data); 3955 3956 err: 3957 QL_DPRINT12(ha, "exit\n"); 3958 return rc; 3959 } 3960 3961 static void 3962 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3963 { 3964 qlnx_host_t *ha; 3965 3966 ha = dev->ha; 3967 3968 QL_DPRINT12(ha, "enter\n"); 3969 3970 if (qp->usq.umem) 3971 ib_umem_release(qp->usq.umem); 3972 3973 qp->usq.umem = NULL; 3974 3975 if (qp->urq.umem) 3976 ib_umem_release(qp->urq.umem); 3977 3978 qp->urq.umem = NULL; 3979 3980 QL_DPRINT12(ha, "exit\n"); 3981 return; 3982 } 3983 3984 static void 3985 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3986 { 3987 qlnx_host_t *ha; 3988 3989 ha = dev->ha; 3990 3991 QL_DPRINT12(ha, "enter\n"); 3992 3993 if (qlnxr_qp_has_sq(qp)) { 3994 QL_DPRINT12(ha, "freeing SQ\n"); 3995 ha->qlnxr_debug = 1; 3996 // ecore_chain_free(dev->cdev, &qp->sq.pbl); 3997 ha->qlnxr_debug = 0; 3998 kfree(qp->wqe_wr_id); 3999 } 4000 4001 if (qlnxr_qp_has_rq(qp)) { 4002 QL_DPRINT12(ha, "freeing RQ\n"); 4003 ha->qlnxr_debug = 1; 4004 // ecore_chain_free(dev->cdev, &qp->rq.pbl); 4005 ha->qlnxr_debug = 0; 4006 kfree(qp->rqe_wr_id); 4007 } 4008 4009 QL_DPRINT12(ha, "exit\n"); 4010 return; 4011 } 4012 4013 int 4014 qlnxr_free_qp_resources(struct qlnxr_dev *dev, 4015 struct qlnxr_qp *qp) 4016 { 4017 int rc = 0; 4018 qlnx_host_t *ha; 4019 struct ecore_rdma_destroy_qp_out_params d_out_params; 4020 4021 ha = dev->ha; 4022 4023 QL_DPRINT12(ha, "enter\n"); 4024 4025 #if 0 4026 if (qp->qp_type != IB_QPT_GSI) { 4027 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4028 &d_out_params); 4029 if (rc) 4030 return rc; 4031 } 4032 4033 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4034 qlnxr_cleanup_user(dev, qp); 4035 else 4036 qlnxr_cleanup_kernel(dev, qp); 4037 #endif 4038 4039 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4040 qlnxr_cleanup_user(dev, qp); 4041 else 4042 qlnxr_cleanup_kernel(dev, qp); 4043 4044 if (qp->qp_type != IB_QPT_GSI) { 4045 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4046 &d_out_params); 4047 if (rc) 4048 return rc; 4049 } 4050 4051 QL_DPRINT12(ha, "exit\n"); 4052 return 0; 4053 } 4054 4055 int 4056 qlnxr_destroy_qp(struct ib_qp *ibqp) 4057 { 4058 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4059 struct qlnxr_dev *dev = qp->dev; 4060 int rc = 0; 4061 struct ib_qp_attr attr; 4062 int attr_mask = 0; 4063 qlnx_host_t *ha; 4064 4065 ha = dev->ha; 4066 4067 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); 4068 4069 qp->destroyed = 1; 4070 4071 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | 4072 ECORE_ROCE_QP_STATE_ERR | 4073 ECORE_ROCE_QP_STATE_INIT))) { 4074 attr.qp_state = IB_QPS_ERR; 4075 attr_mask |= IB_QP_STATE; 4076 4077 /* change the QP state to ERROR */ 4078 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); 4079 } 4080 4081 if (qp->qp_type == IB_QPT_GSI) 4082 qlnxr_destroy_gsi_qp(dev); 4083 4084 qp->sig = ~qp->sig; 4085 4086 qlnxr_free_qp_resources(dev, qp); 4087 4088 if (atomic_dec_and_test(&qp->refcnt)) { 4089 /* TODO: only for iWARP? */ 4090 qlnxr_idr_remove(dev, qp->qp_id); 4091 kfree(qp); 4092 } 4093 4094 QL_DPRINT12(ha, "exit\n"); 4095 return rc; 4096 } 4097 4098 static inline int 4099 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) 4100 { 4101 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 4102 } 4103 4104 static int 4105 sge_data_len(struct ib_sge *sg_list, int num_sge) 4106 { 4107 int i, len = 0; 4108 for (i = 0; i < num_sge; i++) 4109 len += sg_list[i].length; 4110 return len; 4111 } 4112 4113 static void 4114 swap_wqe_data64(u64 *p) 4115 { 4116 int i; 4117 4118 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 4119 *p = cpu_to_be64(cpu_to_le64(*p)); 4120 } 4121 4122 static u32 4123 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, 4124 struct qlnxr_qp *qp, 4125 u8 *wqe_size, 4126 struct ib_send_wr *wr, 4127 struct ib_send_wr **bad_wr, 4128 u8 *bits, 4129 u8 bit) 4130 { 4131 int i, seg_siz; 4132 char *seg_prt, *wqe; 4133 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 4134 qlnx_host_t *ha; 4135 4136 ha = dev->ha; 4137 4138 QL_DPRINT12(ha, "enter[%d]\n", data_size); 4139 4140 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 4141 QL_DPRINT12(ha, 4142 "Too much inline data in WR:[%d, %d]\n", 4143 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); 4144 *bad_wr = wr; 4145 return 0; 4146 } 4147 4148 if (!data_size) 4149 return data_size; 4150 4151 /* set the bit */ 4152 *bits |= bit; 4153 4154 seg_prt = wqe = NULL; 4155 seg_siz = 0; 4156 4157 /* copy data inline */ 4158 for (i = 0; i < wr->num_sge; i++) { 4159 u32 len = wr->sg_list[i].length; 4160 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 4161 4162 while (len > 0) { 4163 u32 cur; 4164 4165 /* new segment required */ 4166 if (!seg_siz) { 4167 wqe = (char *)ecore_chain_produce(&qp->sq.pbl); 4168 seg_prt = wqe; 4169 seg_siz = sizeof(struct rdma_sq_common_wqe); 4170 (*wqe_size)++; 4171 } 4172 4173 /* calculate currently allowed length */ 4174 cur = MIN(len, seg_siz); 4175 4176 memcpy(seg_prt, src, cur); 4177 4178 /* update segment variables */ 4179 seg_prt += cur; 4180 seg_siz -= cur; 4181 /* update sge variables */ 4182 src += cur; 4183 len -= cur; 4184 4185 /* swap fully-completed segments */ 4186 if (!seg_siz) 4187 swap_wqe_data64((u64 *)wqe); 4188 } 4189 } 4190 4191 /* swap last not completed segment */ 4192 if (seg_siz) 4193 swap_wqe_data64((u64 *)wqe); 4194 4195 QL_DPRINT12(ha, "exit\n"); 4196 return data_size; 4197 } 4198 4199 static u32 4200 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, 4201 u8 *wqe_size, struct ib_send_wr *wr) 4202 { 4203 int i; 4204 u32 data_size = 0; 4205 qlnx_host_t *ha; 4206 4207 ha = dev->ha; 4208 4209 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); 4210 4211 for (i = 0; i < wr->num_sge; i++) { 4212 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); 4213 4214 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); 4215 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 4216 sge->length = cpu_to_le32(wr->sg_list[i].length); 4217 data_size += wr->sg_list[i].length; 4218 } 4219 4220 if (wqe_size) 4221 *wqe_size += wr->num_sge; 4222 4223 QL_DPRINT12(ha, "exit data_size = %d\n", data_size); 4224 return data_size; 4225 } 4226 4227 static u32 4228 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, 4229 struct qlnxr_qp *qp, 4230 struct rdma_sq_rdma_wqe_1st *rwqe, 4231 struct rdma_sq_rdma_wqe_2nd *rwqe2, 4232 struct ib_send_wr *wr, 4233 struct ib_send_wr **bad_wr) 4234 { 4235 qlnx_host_t *ha; 4236 u32 ret = 0; 4237 4238 ha = dev->ha; 4239 4240 QL_DPRINT12(ha, "enter\n"); 4241 4242 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 4243 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); 4244 4245 if (wr->send_flags & IB_SEND_INLINE) { 4246 u8 flags = 0; 4247 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 4248 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, 4249 wr, bad_wr, &rwqe->flags, flags); 4250 } 4251 4252 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); 4253 4254 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4255 4256 return (ret); 4257 } 4258 4259 static u32 4260 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, 4261 struct qlnxr_qp *qp, 4262 struct rdma_sq_send_wqe *swqe, 4263 struct rdma_sq_send_wqe *swqe2, 4264 struct ib_send_wr *wr, 4265 struct ib_send_wr **bad_wr) 4266 { 4267 qlnx_host_t *ha; 4268 u32 ret = 0; 4269 4270 ha = dev->ha; 4271 4272 QL_DPRINT12(ha, "enter\n"); 4273 4274 memset(swqe2, 0, sizeof(*swqe2)); 4275 4276 if (wr->send_flags & IB_SEND_INLINE) { 4277 u8 flags = 0; 4278 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 4279 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, 4280 wr, bad_wr, &swqe->flags, flags); 4281 } 4282 4283 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); 4284 4285 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4286 4287 return (ret); 4288 } 4289 4290 static void 4291 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) 4292 { 4293 qlnx_host_t *ha; 4294 4295 ha = dev->ha; 4296 4297 int work = info->completed - info->completed_handled - 1; 4298 4299 QL_DPRINT12(ha, "enter [%d]\n", work); 4300 4301 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 4302 struct qlnxr_pbl *pbl; 4303 4304 /* Free all the page list that are possible to be freed 4305 * (all the ones that were invalidated), under the assumption 4306 * that if an FMR was completed successfully that means that 4307 * if there was an invalidate operation before it also ended 4308 */ 4309 pbl = list_first_entry(&info->inuse_pbl_list, 4310 struct qlnxr_pbl, 4311 list_entry); 4312 list_del(&pbl->list_entry); 4313 list_add_tail(&pbl->list_entry, &info->free_pbl_list); 4314 info->completed_handled++; 4315 } 4316 4317 QL_DPRINT12(ha, "exit\n"); 4318 return; 4319 } 4320 4321 #if __FreeBSD_version >= 1102000 4322 4323 static int qlnxr_prepare_reg(struct qlnxr_qp *qp, 4324 struct rdma_sq_fmr_wqe_1st *fwqe1, 4325 struct ib_reg_wr *wr) 4326 { 4327 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); 4328 struct rdma_sq_fmr_wqe_2nd *fwqe2; 4329 4330 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); 4331 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 4332 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 4333 fwqe1->l_key = wr->key; 4334 4335 fwqe2->access_ctrl = 0; 4336 4337 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 4338 !!(wr->access & IB_ACCESS_REMOTE_READ)); 4339 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 4340 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 4341 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 4342 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 4343 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 4344 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 4345 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 4346 fwqe2->fmr_ctrl = 0; 4347 4348 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4349 ilog2(mr->ibmr.page_size) - 12); 4350 4351 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ 4352 fwqe2->length_lo = mr->ibmr.length; 4353 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 4354 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 4355 4356 qp->wqe_wr_id[qp->sq.prod].mr = mr; 4357 4358 return 0; 4359 } 4360 4361 #else 4362 4363 static void 4364 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr, 4365 struct mr_info *info) 4366 { 4367 int i; 4368 u64 buf_addr = 0; 4369 int num_pbes, total_num_pbes = 0; 4370 struct regpair *pbe; 4371 struct qlnxr_pbl *pbl_tbl = info->pbl_table; 4372 struct qlnxr_pbl_info *pbl_info = &info->pbl_info; 4373 qlnx_host_t *ha; 4374 4375 ha = dev->ha; 4376 4377 QL_DPRINT12(ha, "enter\n"); 4378 4379 pbe = (struct regpair *)pbl_tbl->va; 4380 num_pbes = 0; 4381 4382 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { 4383 buf_addr = wr->wr.fast_reg.page_list->page_list[i]; 4384 pbe->lo = cpu_to_le32((u32)buf_addr); 4385 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr)); 4386 4387 num_pbes += 1; 4388 pbe++; 4389 total_num_pbes++; 4390 4391 if (total_num_pbes == pbl_info->num_pbes) 4392 return; 4393 4394 /* if the given pbl is full storing the pbes, 4395 * move to next pbl. 4396 */ 4397 if (num_pbes == 4398 (pbl_info->pbl_size / sizeof(u64))) { 4399 pbl_tbl++; 4400 pbe = (struct regpair *)pbl_tbl->va; 4401 num_pbes = 0; 4402 } 4403 } 4404 QL_DPRINT12(ha, "exit\n"); 4405 4406 return; 4407 } 4408 4409 static int 4410 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info) 4411 { 4412 int rc = 0; 4413 qlnx_host_t *ha; 4414 4415 ha = dev->ha; 4416 4417 QL_DPRINT12(ha, "enter\n"); 4418 4419 if (info->completed == 0) { 4420 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n"); 4421 /* first fmr */ 4422 return 0; 4423 } 4424 4425 qlnx_handle_completed_mrs(dev, info); 4426 4427 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list); 4428 4429 if (list_empty(&info->free_pbl_list)) { 4430 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, 4431 GFP_ATOMIC); 4432 } else { 4433 info->pbl_table = list_first_entry(&info->free_pbl_list, 4434 struct qlnxr_pbl, 4435 list_entry); 4436 list_del(&info->pbl_table->list_entry); 4437 } 4438 4439 if (!info->pbl_table) 4440 rc = -ENOMEM; 4441 4442 QL_DPRINT12(ha, "exit\n"); 4443 return rc; 4444 } 4445 4446 static inline int 4447 qlnxr_prepare_fmr(struct qlnxr_qp *qp, 4448 struct rdma_sq_fmr_wqe_1st *fwqe1, 4449 struct ib_send_wr *wr) 4450 { 4451 struct qlnxr_dev *dev = qp->dev; 4452 u64 fbo; 4453 struct qlnxr_fast_reg_page_list *frmr_list = 4454 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list); 4455 struct rdma_sq_fmr_wqe *fwqe2 = 4456 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl); 4457 int rc = 0; 4458 qlnx_host_t *ha; 4459 4460 ha = dev->ha; 4461 4462 QL_DPRINT12(ha, "enter\n"); 4463 4464 if (wr->wr.fast_reg.page_list_len == 0) 4465 BUG(); 4466 4467 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info); 4468 if (rc) 4469 return rc; 4470 4471 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start); 4472 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start); 4473 fwqe1->l_key = wr->wr.fast_reg.rkey; 4474 4475 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ, 4476 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)); 4477 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE, 4478 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)); 4479 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC, 4480 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC)); 4481 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1); 4482 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE, 4483 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)); 4484 4485 fwqe2->fmr_ctrl = 0; 4486 4487 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4488 ilog2(1 << wr->wr.fast_reg.page_shift) - 12); 4489 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0); 4490 4491 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */ 4492 fwqe2->length_lo = wr->wr.fast_reg.length; 4493 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa); 4494 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa); 4495 4496 /* produce another wqe for fwqe3 */ 4497 ecore_chain_produce(&qp->sq.pbl); 4498 4499 fbo = wr->wr.fast_reg.iova_start - 4500 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK); 4501 4502 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x" 4503 " length = %x pbl_addr %x:%x\n", 4504 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey, 4505 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo, 4506 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo); 4507 4508 build_frmr_pbes(dev, wr, &frmr_list->info); 4509 4510 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list; 4511 4512 QL_DPRINT12(ha, "exit\n"); 4513 return 0; 4514 } 4515 4516 #endif /* #if __FreeBSD_version >= 1102000 */ 4517 4518 static enum ib_wc_opcode 4519 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 4520 { 4521 switch (opcode) { 4522 case IB_WR_RDMA_WRITE: 4523 case IB_WR_RDMA_WRITE_WITH_IMM: 4524 return IB_WC_RDMA_WRITE; 4525 case IB_WR_SEND_WITH_IMM: 4526 case IB_WR_SEND: 4527 case IB_WR_SEND_WITH_INV: 4528 return IB_WC_SEND; 4529 case IB_WR_RDMA_READ: 4530 return IB_WC_RDMA_READ; 4531 case IB_WR_ATOMIC_CMP_AND_SWP: 4532 return IB_WC_COMP_SWAP; 4533 case IB_WR_ATOMIC_FETCH_AND_ADD: 4534 return IB_WC_FETCH_ADD; 4535 4536 #if __FreeBSD_version >= 1102000 4537 case IB_WR_REG_MR: 4538 return IB_WC_REG_MR; 4539 #else 4540 case IB_WR_FAST_REG_MR: 4541 return IB_WC_FAST_REG_MR; 4542 #endif /* #if __FreeBSD_version >= 1102000 */ 4543 4544 case IB_WR_LOCAL_INV: 4545 return IB_WC_LOCAL_INV; 4546 default: 4547 return IB_WC_SEND; 4548 } 4549 } 4550 static inline bool 4551 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr) 4552 { 4553 int wq_is_full, err_wr, pbl_is_full; 4554 struct qlnxr_dev *dev = qp->dev; 4555 qlnx_host_t *ha; 4556 4557 ha = dev->ha; 4558 4559 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); 4560 4561 /* prevent SQ overflow and/or processing of a bad WR */ 4562 err_wr = wr->num_sge > qp->sq.max_sges; 4563 wq_is_full = qlnxr_wq_is_full(&qp->sq); 4564 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < 4565 QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 4566 if (wq_is_full || err_wr || pbl_is_full) { 4567 if (wq_is_full && 4568 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { 4569 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; 4570 4571 QL_DPRINT12(ha, 4572 "error: WQ is full. Post send on QP failed" 4573 " (this error appears only once) " 4574 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4575 qp, wr, qp->err_bitmap); 4576 } 4577 4578 if (err_wr && 4579 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { 4580 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; 4581 4582 QL_DPRINT12(ha, 4583 "error: WQ is bad. Post send on QP failed" 4584 " (this error appears only once) " 4585 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4586 qp, wr, qp->err_bitmap); 4587 } 4588 4589 if (pbl_is_full && 4590 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { 4591 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; 4592 4593 QL_DPRINT12(ha, 4594 "error: WQ PBL is full. Post send on QP failed" 4595 " (this error appears only once) " 4596 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4597 qp, wr, qp->err_bitmap); 4598 } 4599 return false; 4600 } 4601 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); 4602 return true; 4603 } 4604 4605 int 4606 qlnxr_post_send(struct ib_qp *ibqp, 4607 struct ib_send_wr *wr, 4608 struct ib_send_wr **bad_wr) 4609 { 4610 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); 4611 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4612 unsigned long flags; 4613 int status = 0, rc = 0; 4614 bool comp; 4615 qlnx_host_t *ha; 4616 uint32_t reg_addr; 4617 4618 *bad_wr = NULL; 4619 ha = dev->ha; 4620 4621 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4622 ibqp, wr, bad_wr); 4623 4624 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 4625 return -EINVAL; 4626 4627 if (qp->qp_type == IB_QPT_GSI) 4628 return qlnxr_gsi_post_send(ibqp, wr, bad_wr); 4629 4630 spin_lock_irqsave(&qp->q_lock, flags); 4631 4632 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && 4633 (qp->state != ECORE_ROCE_QP_STATE_ERR) && 4634 (qp->state != ECORE_ROCE_QP_STATE_SQD)) { 4635 spin_unlock_irqrestore(&qp->q_lock, flags); 4636 *bad_wr = wr; 4637 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", 4638 qp->icid, qp->state); 4639 return -EINVAL; 4640 } 4641 4642 if (!wr) { 4643 QL_DPRINT11(ha, "Got an empty post send???\n"); 4644 } 4645 4646 while (wr) { 4647 struct rdma_sq_common_wqe *wqe; 4648 struct rdma_sq_send_wqe *swqe; 4649 struct rdma_sq_send_wqe *swqe2; 4650 struct rdma_sq_rdma_wqe_1st *rwqe; 4651 struct rdma_sq_rdma_wqe_2nd *rwqe2; 4652 struct rdma_sq_local_inv_wqe *iwqe; 4653 struct rdma_sq_atomic_wqe *awqe1; 4654 struct rdma_sq_atomic_wqe *awqe2; 4655 struct rdma_sq_atomic_wqe *awqe3; 4656 struct rdma_sq_fmr_wqe_1st *fwqe1; 4657 4658 if (!qlnxr_can_post_send(qp, wr)) { 4659 status = -ENOMEM; 4660 *bad_wr = wr; 4661 break; 4662 } 4663 4664 wqe = ecore_chain_produce(&qp->sq.pbl); 4665 4666 qp->wqe_wr_id[qp->sq.prod].signaled = 4667 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 4668 4669 /* common fields */ 4670 wqe->flags = 0; 4671 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << 4672 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); 4673 4674 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ 4675 !!(wr->send_flags & IB_SEND_SOLICITED)); 4676 4677 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || 4678 (qp->signaled); 4679 4680 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 4681 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ 4682 !!(wr->send_flags & IB_SEND_FENCE)); 4683 4684 wqe->prev_wqe_size = qp->prev_wqe_size; 4685 4686 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); 4687 4688 switch (wr->opcode) { 4689 case IB_WR_SEND_WITH_IMM: 4690 4691 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 4692 swqe = (struct rdma_sq_send_wqe *)wqe; 4693 swqe->wqe_size = 2; 4694 swqe2 = (struct rdma_sq_send_wqe *) 4695 ecore_chain_produce(&qp->sq.pbl); 4696 swqe->inv_key_or_imm_data = 4697 cpu_to_le32(wr->ex.imm_data); 4698 swqe->length = cpu_to_le32( 4699 qlnxr_prepare_sq_send_data(dev, 4700 qp, swqe, swqe2, wr, 4701 bad_wr)); 4702 4703 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4704 qp->prev_wqe_size = swqe->wqe_size; 4705 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4706 4707 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", 4708 swqe->length, wr->ex.imm_data); 4709 4710 break; 4711 4712 case IB_WR_SEND: 4713 4714 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 4715 swqe = (struct rdma_sq_send_wqe *)wqe; 4716 4717 swqe->wqe_size = 2; 4718 swqe2 = (struct rdma_sq_send_wqe *) 4719 ecore_chain_produce(&qp->sq.pbl); 4720 swqe->length = cpu_to_le32( 4721 qlnxr_prepare_sq_send_data(dev, 4722 qp, swqe, swqe2, wr, 4723 bad_wr)); 4724 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4725 qp->prev_wqe_size = swqe->wqe_size; 4726 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4727 4728 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", 4729 swqe->length); 4730 4731 break; 4732 4733 case IB_WR_SEND_WITH_INV: 4734 4735 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 4736 swqe = (struct rdma_sq_send_wqe *)wqe; 4737 swqe2 = (struct rdma_sq_send_wqe *) 4738 ecore_chain_produce(&qp->sq.pbl); 4739 swqe->wqe_size = 2; 4740 swqe->inv_key_or_imm_data = 4741 cpu_to_le32(wr->ex.invalidate_rkey); 4742 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, 4743 qp, swqe, swqe2, wr, bad_wr)); 4744 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4745 qp->prev_wqe_size = swqe->wqe_size; 4746 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4747 4748 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", 4749 swqe->length); 4750 break; 4751 4752 case IB_WR_RDMA_WRITE_WITH_IMM: 4753 4754 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 4755 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4756 4757 rwqe->wqe_size = 2; 4758 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 4759 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4760 ecore_chain_produce(&qp->sq.pbl); 4761 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4762 qp, rwqe, rwqe2, wr, bad_wr)); 4763 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4764 qp->prev_wqe_size = rwqe->wqe_size; 4765 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4766 4767 QL_DPRINT12(ha, 4768 "RDMA WRITE w/ IMM length = %d imm data=%x\n", 4769 rwqe->length, rwqe->imm_data); 4770 4771 break; 4772 4773 case IB_WR_RDMA_WRITE: 4774 4775 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 4776 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4777 4778 rwqe->wqe_size = 2; 4779 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4780 ecore_chain_produce(&qp->sq.pbl); 4781 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4782 qp, rwqe, rwqe2, wr, bad_wr)); 4783 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4784 qp->prev_wqe_size = rwqe->wqe_size; 4785 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4786 4787 QL_DPRINT12(ha, 4788 "RDMA WRITE w/o IMM length = %d\n", 4789 rwqe->length); 4790 4791 break; 4792 4793 case IB_WR_RDMA_READ_WITH_INV: 4794 4795 QL_DPRINT12(ha, 4796 "RDMA READ WITH INVALIDATE not supported\n"); 4797 4798 *bad_wr = wr; 4799 rc = -EINVAL; 4800 4801 break; 4802 4803 case IB_WR_RDMA_READ: 4804 4805 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 4806 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4807 4808 rwqe->wqe_size = 2; 4809 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4810 ecore_chain_produce(&qp->sq.pbl); 4811 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4812 qp, rwqe, rwqe2, wr, bad_wr)); 4813 4814 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4815 qp->prev_wqe_size = rwqe->wqe_size; 4816 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4817 4818 QL_DPRINT12(ha, "RDMA READ length = %d\n", 4819 rwqe->length); 4820 4821 break; 4822 4823 case IB_WR_ATOMIC_CMP_AND_SWP: 4824 case IB_WR_ATOMIC_FETCH_AND_ADD: 4825 4826 QL_DPRINT12(ha, 4827 "ATOMIC operation = %s\n", 4828 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? 4829 "IB_WR_ATOMIC_CMP_AND_SWP" : 4830 "IB_WR_ATOMIC_FETCH_AND_ADD")); 4831 4832 awqe1 = (struct rdma_sq_atomic_wqe *)wqe; 4833 awqe1->prev_wqe_size = 4; 4834 4835 awqe2 = (struct rdma_sq_atomic_wqe *) 4836 ecore_chain_produce(&qp->sq.pbl); 4837 4838 TYPEPTR_ADDR_SET(awqe2, remote_va, \ 4839 atomic_wr(wr)->remote_addr); 4840 4841 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 4842 4843 awqe3 = (struct rdma_sq_atomic_wqe *) 4844 ecore_chain_produce(&qp->sq.pbl); 4845 4846 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 4847 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 4848 TYPEPTR_ADDR_SET(awqe3, swap_data, 4849 atomic_wr(wr)->compare_add); 4850 } else { 4851 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 4852 TYPEPTR_ADDR_SET(awqe3, swap_data, 4853 atomic_wr(wr)->swap); 4854 TYPEPTR_ADDR_SET(awqe3, cmp_data, 4855 atomic_wr(wr)->compare_add); 4856 } 4857 4858 qlnxr_prepare_sq_sges(dev, qp, NULL, wr); 4859 4860 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; 4861 qp->prev_wqe_size = awqe1->prev_wqe_size; 4862 4863 break; 4864 4865 case IB_WR_LOCAL_INV: 4866 4867 QL_DPRINT12(ha, 4868 "INVALIDATE length (IB_WR_LOCAL_INV)\n"); 4869 4870 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 4871 iwqe->prev_wqe_size = 1; 4872 4873 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 4874 iwqe->inv_l_key = wr->ex.invalidate_rkey; 4875 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; 4876 qp->prev_wqe_size = iwqe->prev_wqe_size; 4877 4878 break; 4879 4880 #if __FreeBSD_version >= 1102000 4881 4882 case IB_WR_REG_MR: 4883 4884 QL_DPRINT12(ha, "IB_WR_REG_MR\n"); 4885 4886 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4887 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4888 fwqe1->wqe_size = 2; 4889 4890 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); 4891 if (rc) { 4892 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); 4893 *bad_wr = wr; 4894 break; 4895 } 4896 4897 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 4898 qp->prev_wqe_size = fwqe1->wqe_size; 4899 4900 break; 4901 #else 4902 case IB_WR_FAST_REG_MR: 4903 4904 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n"); 4905 4906 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4907 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4908 fwqe1->prev_wqe_size = 3; 4909 4910 rc = qlnxr_prepare_fmr(qp, fwqe1, wr); 4911 4912 if (rc) { 4913 QL_DPRINT12(ha, 4914 "FAST_MR (IB_WR_FAST_REG_MR) failed" 4915 " rc = %d\n", rc); 4916 *bad_wr = wr; 4917 break; 4918 } 4919 4920 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size; 4921 qp->prev_wqe_size = fwqe1->prev_wqe_size; 4922 4923 break; 4924 #endif /* #if __FreeBSD_version >= 1102000 */ 4925 4926 default: 4927 4928 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); 4929 4930 rc = -EINVAL; 4931 *bad_wr = wr; 4932 break; 4933 } 4934 4935 if (*bad_wr) { 4936 /* 4937 * restore prod to its position before this WR was processed 4938 */ 4939 ecore_chain_set_prod(&qp->sq.pbl, 4940 le16_to_cpu(qp->sq.db_data.data.value), 4941 wqe); 4942 /* restore prev_wqe_size */ 4943 qp->prev_wqe_size = wqe->prev_wqe_size; 4944 status = rc; 4945 4946 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); 4947 break; /* out of the loop */ 4948 } 4949 4950 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 4951 4952 qlnxr_inc_sw_prod(&qp->sq); 4953 4954 qp->sq.db_data.data.value++; 4955 4956 wr = wr->next; 4957 } 4958 4959 /* Trigger doorbell 4960 * If there was a failure in the first WR then it will be triggered in 4961 * vane. However this is not harmful (as long as the producer value is 4962 * unchanged). For performance reasons we avoid checking for this 4963 * redundant doorbell. 4964 */ 4965 wmb(); 4966 //writel(qp->sq.db_data.raw, qp->sq.db); 4967 4968 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); 4969 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); 4970 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4971 4972 mmiowb(); 4973 4974 spin_unlock_irqrestore(&qp->q_lock, flags); 4975 4976 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4977 ibqp, wr, bad_wr); 4978 4979 return status; 4980 } 4981 4982 static u32 4983 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) 4984 { 4985 u32 used; 4986 4987 /* Calculate number of elements used based on producer 4988 * count and consumer count and subtract it from max 4989 * work request supported so that we get elements left. 4990 */ 4991 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 4992 4993 return hw_srq->max_wr - used; 4994 } 4995 4996 int 4997 qlnxr_post_recv(struct ib_qp *ibqp, 4998 struct ib_recv_wr *wr, 4999 struct ib_recv_wr **bad_wr) 5000 { 5001 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 5002 struct qlnxr_dev *dev = qp->dev; 5003 unsigned long flags; 5004 int status = 0; 5005 qlnx_host_t *ha; 5006 uint32_t reg_addr; 5007 5008 ha = dev->ha; 5009 5010 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5011 return -EINVAL; 5012 5013 QL_DPRINT12(ha, "enter\n"); 5014 5015 if (qp->qp_type == IB_QPT_GSI) { 5016 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); 5017 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); 5018 } 5019 5020 if (qp->srq) { 5021 QL_DPRINT11(ha, "qp->srq [%p]" 5022 " QP is associated with SRQ, cannot post RQ buffers\n", 5023 qp->srq); 5024 return -EINVAL; 5025 } 5026 5027 spin_lock_irqsave(&qp->q_lock, flags); 5028 5029 if (qp->state == ECORE_ROCE_QP_STATE_RESET) { 5030 spin_unlock_irqrestore(&qp->q_lock, flags); 5031 *bad_wr = wr; 5032 5033 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); 5034 5035 return -EINVAL; 5036 } 5037 5038 while (wr) { 5039 int i; 5040 5041 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < 5042 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || 5043 (wr->num_sge > qp->rq.max_sges)) { 5044 status = -ENOMEM; 5045 *bad_wr = wr; 5046 break; 5047 } 5048 for (i = 0; i < wr->num_sge; i++) { 5049 u32 flags = 0; 5050 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5051 5052 /* first one must include the number of SGE in the list */ 5053 if (!i) 5054 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); 5055 5056 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); 5057 5058 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ 5059 wr->sg_list[i].length, flags); 5060 } 5061 /* Special case of no sges. FW requires between 1-4 sges... 5062 * in this case we need to post 1 sge with length zero. this is 5063 * because rdma write with immediate consumes an RQ. */ 5064 if (!wr->num_sge) { 5065 u32 flags = 0; 5066 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5067 5068 /* first one must include the number of SGE in the list */ 5069 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 5070 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 5071 5072 //RQ_SGE_SET(rqe, 0, 0, flags); 5073 rqe->addr.hi = 0; 5074 rqe->addr.lo = 0; 5075 5076 rqe->length = 0; 5077 rqe->flags = cpu_to_le32(flags); 5078 5079 i = 1; 5080 } 5081 5082 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 5083 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 5084 5085 qlnxr_inc_sw_prod(&qp->rq); 5086 5087 wmb(); 5088 5089 qp->rq.db_data.data.value++; 5090 5091 // writel(qp->rq.db_data.raw, qp->rq.db); 5092 mmiowb(); 5093 // if (QLNX_IS_IWARP(dev)) { 5094 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 5095 // mmiowb(); /* for second doorbell */ 5096 // } 5097 5098 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 5099 (uint8_t *)ha->cdev.doorbells); 5100 5101 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 5102 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5103 5104 if (QLNX_IS_IWARP(dev)) { 5105 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 5106 (uint8_t *)ha->cdev.doorbells); 5107 bus_write_4(ha->pci_dbells, reg_addr, \ 5108 qp->rq.iwarp_db2_data.raw); 5109 bus_barrier(ha->pci_dbells, 0, 0, \ 5110 BUS_SPACE_BARRIER_READ); 5111 } 5112 5113 wr = wr->next; 5114 } 5115 5116 spin_unlock_irqrestore(&qp->q_lock, flags); 5117 5118 QL_DPRINT12(ha, "exit status = 0x%x\n", status); 5119 5120 return status; 5121 } 5122 5123 /* In fmr we need to increase the number of fmr completed counter for the fmr 5124 * algorithm determining whether we can free a pbl or not. 5125 * we need to perform this whether the work request was signaled or not. for 5126 * this purpose we call this function from the condition that checks if a wr 5127 * should be skipped, to make sure we don't miss it ( possibly this fmr 5128 * operation was not signalted) 5129 */ 5130 static inline void 5131 qlnxr_chk_if_fmr(struct qlnxr_qp *qp) 5132 { 5133 #if __FreeBSD_version >= 1102000 5134 5135 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 5136 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5137 #else 5138 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR) 5139 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5140 5141 #endif /* #if __FreeBSD_version >= 1102000 */ 5142 } 5143 5144 static int 5145 process_req(struct qlnxr_dev *dev, 5146 struct qlnxr_qp *qp, 5147 struct qlnxr_cq *cq, 5148 int num_entries, 5149 struct ib_wc *wc, 5150 u16 hw_cons, 5151 enum ib_wc_status status, 5152 int force) 5153 { 5154 u16 cnt = 0; 5155 qlnx_host_t *ha = dev->ha; 5156 5157 QL_DPRINT12(ha, "enter\n"); 5158 5159 while (num_entries && qp->sq.wqe_cons != hw_cons) { 5160 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 5161 qlnxr_chk_if_fmr(qp); 5162 /* skip WC */ 5163 goto next_cqe; 5164 } 5165 5166 /* fill WC */ 5167 wc->status = status; 5168 wc->vendor_err = 0; 5169 wc->wc_flags = 0; 5170 wc->src_qp = qp->id; 5171 wc->qp = &qp->ibqp; 5172 5173 // common section 5174 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 5175 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 5176 5177 switch (wc->opcode) { 5178 case IB_WC_RDMA_WRITE: 5179 5180 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 5181 5182 QL_DPRINT12(ha, 5183 "opcode = IB_WC_RDMA_WRITE bytes = %d\n", 5184 qp->wqe_wr_id[qp->sq.cons].bytes_len); 5185 break; 5186 5187 case IB_WC_COMP_SWAP: 5188 case IB_WC_FETCH_ADD: 5189 wc->byte_len = 8; 5190 break; 5191 5192 #if __FreeBSD_version >= 1102000 5193 case IB_WC_REG_MR: 5194 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5195 break; 5196 #else 5197 case IB_WC_FAST_REG_MR: 5198 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5199 break; 5200 #endif /* #if __FreeBSD_version >= 1102000 */ 5201 5202 case IB_WC_RDMA_READ: 5203 case IB_WC_SEND: 5204 5205 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); 5206 break; 5207 default: 5208 ;//DP_ERR("TBD ERROR"); 5209 } 5210 5211 num_entries--; 5212 wc++; 5213 cnt++; 5214 next_cqe: 5215 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 5216 ecore_chain_consume(&qp->sq.pbl); 5217 qlnxr_inc_sw_cons(&qp->sq); 5218 } 5219 5220 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5221 return cnt; 5222 } 5223 5224 static int 5225 qlnxr_poll_cq_req(struct qlnxr_dev *dev, 5226 struct qlnxr_qp *qp, 5227 struct qlnxr_cq *cq, 5228 int num_entries, 5229 struct ib_wc *wc, 5230 struct rdma_cqe_requester *req) 5231 { 5232 int cnt = 0; 5233 qlnx_host_t *ha = dev->ha; 5234 5235 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); 5236 5237 switch (req->status) { 5238 case RDMA_CQE_REQ_STS_OK: 5239 5240 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5241 IB_WC_SUCCESS, 0); 5242 break; 5243 5244 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 5245 5246 if (qp->state != ECORE_ROCE_QP_STATE_ERR) 5247 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5248 IB_WC_WR_FLUSH_ERR, 1); 5249 break; 5250 5251 default: /* other errors case */ 5252 5253 /* process all WQE before the cosumer */ 5254 qp->state = ECORE_ROCE_QP_STATE_ERR; 5255 cnt = process_req(dev, qp, cq, num_entries, wc, 5256 req->sq_cons - 1, IB_WC_SUCCESS, 0); 5257 wc += cnt; 5258 /* if we have extra WC fill it with actual error info */ 5259 5260 if (cnt < num_entries) { 5261 enum ib_wc_status wc_status; 5262 5263 switch (req->status) { 5264 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 5265 wc_status = IB_WC_BAD_RESP_ERR; 5266 break; 5267 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 5268 wc_status = IB_WC_LOC_LEN_ERR; 5269 break; 5270 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 5271 wc_status = IB_WC_LOC_QP_OP_ERR; 5272 break; 5273 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 5274 wc_status = IB_WC_LOC_PROT_ERR; 5275 break; 5276 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 5277 wc_status = IB_WC_MW_BIND_ERR; 5278 break; 5279 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 5280 wc_status = IB_WC_REM_INV_REQ_ERR; 5281 break; 5282 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 5283 wc_status = IB_WC_REM_ACCESS_ERR; 5284 break; 5285 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 5286 wc_status = IB_WC_REM_OP_ERR; 5287 break; 5288 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 5289 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 5290 break; 5291 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 5292 wc_status = IB_WC_RETRY_EXC_ERR; 5293 break; 5294 default: 5295 wc_status = IB_WC_GENERAL_ERR; 5296 } 5297 5298 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 5299 wc_status, 1 /* force use of WC */); 5300 } 5301 } 5302 5303 QL_DPRINT12(ha, "exit cnt = %d\n", cnt); 5304 return cnt; 5305 } 5306 5307 static void 5308 __process_resp_one(struct qlnxr_dev *dev, 5309 struct qlnxr_qp *qp, 5310 struct qlnxr_cq *cq, 5311 struct ib_wc *wc, 5312 struct rdma_cqe_responder *resp, 5313 u64 wr_id) 5314 { 5315 enum ib_wc_status wc_status = IB_WC_SUCCESS; 5316 #if __FreeBSD_version < 1102000 5317 u8 flags; 5318 #endif 5319 qlnx_host_t *ha = dev->ha; 5320 5321 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", 5322 qp, resp->status); 5323 5324 wc->opcode = IB_WC_RECV; 5325 wc->wc_flags = 0; 5326 5327 switch (resp->status) { 5328 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 5329 wc_status = IB_WC_LOC_ACCESS_ERR; 5330 break; 5331 5332 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 5333 wc_status = IB_WC_LOC_LEN_ERR; 5334 break; 5335 5336 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 5337 wc_status = IB_WC_LOC_QP_OP_ERR; 5338 break; 5339 5340 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 5341 wc_status = IB_WC_LOC_PROT_ERR; 5342 break; 5343 5344 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 5345 wc_status = IB_WC_MW_BIND_ERR; 5346 break; 5347 5348 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 5349 wc_status = IB_WC_REM_INV_RD_REQ_ERR; 5350 break; 5351 5352 case RDMA_CQE_RESP_STS_OK: 5353 5354 #if __FreeBSD_version >= 1102000 5355 if (resp->flags & QLNXR_RESP_IMM) { 5356 wc->ex.imm_data = 5357 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5358 wc->wc_flags |= IB_WC_WITH_IMM; 5359 5360 if (resp->flags & QLNXR_RESP_RDMA) 5361 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5362 5363 if (resp->flags & QLNXR_RESP_INV) { 5364 QL_DPRINT11(ha, 5365 "Invalid flags QLNXR_RESP_INV [0x%x]" 5366 "qp = %p qp->id = 0x%x cq = %p" 5367 " cq->icid = 0x%x\n", 5368 resp->flags, qp, qp->id, cq, cq->icid ); 5369 } 5370 } else if (resp->flags & QLNXR_RESP_INV) { 5371 wc->ex.imm_data = 5372 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5373 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 5374 5375 if (resp->flags & QLNXR_RESP_RDMA) { 5376 QL_DPRINT11(ha, 5377 "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5378 "qp = %p qp->id = 0x%x cq = %p" 5379 " cq->icid = 0x%x\n", 5380 resp->flags, qp, qp->id, cq, cq->icid ); 5381 } 5382 } else if (resp->flags & QLNXR_RESP_RDMA) { 5383 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5384 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5385 resp->flags, qp, qp->id, cq, cq->icid ); 5386 } 5387 #else 5388 wc_status = IB_WC_SUCCESS; 5389 wc->byte_len = le32_to_cpu(resp->length); 5390 5391 flags = resp->flags & QLNXR_RESP_RDMA_IMM; 5392 5393 switch (flags) { 5394 case QLNXR_RESP_RDMA_IMM: 5395 /* update opcode */ 5396 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5397 /* fall to set imm data */ 5398 case QLNXR_RESP_IMM: 5399 wc->ex.imm_data = 5400 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5401 wc->wc_flags |= IB_WC_WITH_IMM; 5402 break; 5403 case QLNXR_RESP_RDMA: 5404 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5405 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5406 resp->flags, qp, qp->id, cq, cq->icid ); 5407 break; 5408 default: 5409 /* valid configuration, but nothing todo here */ 5410 ; 5411 } 5412 #endif /* #if __FreeBSD_version >= 1102000 */ 5413 5414 break; 5415 default: 5416 wc_status = IB_WC_GENERAL_ERR; 5417 } 5418 5419 /* fill WC */ 5420 wc->status = wc_status; 5421 wc->vendor_err = 0; 5422 wc->src_qp = qp->id; 5423 wc->qp = &qp->ibqp; 5424 wc->wr_id = wr_id; 5425 5426 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); 5427 5428 return; 5429 } 5430 5431 static int 5432 process_resp_one_srq(struct qlnxr_dev *dev, 5433 struct qlnxr_qp *qp, 5434 struct qlnxr_cq *cq, 5435 struct ib_wc *wc, 5436 struct rdma_cqe_responder *resp) 5437 { 5438 struct qlnxr_srq *srq = qp->srq; 5439 u64 wr_id; 5440 qlnx_host_t *ha = dev->ha; 5441 5442 QL_DPRINT12(ha, "enter\n"); 5443 5444 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); 5445 5446 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5447 wc->status = IB_WC_WR_FLUSH_ERR; 5448 wc->vendor_err = 0; 5449 wc->wr_id = wr_id; 5450 wc->byte_len = 0; 5451 wc->src_qp = qp->id; 5452 wc->qp = &qp->ibqp; 5453 wc->wr_id = wr_id; 5454 } else { 5455 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5456 } 5457 5458 /* PBL is maintained in case of WR granularity. 5459 * So increment WR consumer after consuming WR 5460 */ 5461 srq->hw_srq.wr_cons_cnt++; 5462 5463 QL_DPRINT12(ha, "exit\n"); 5464 return 1; 5465 } 5466 5467 static int 5468 process_resp_one(struct qlnxr_dev *dev, 5469 struct qlnxr_qp *qp, 5470 struct qlnxr_cq *cq, 5471 struct ib_wc *wc, 5472 struct rdma_cqe_responder *resp) 5473 { 5474 qlnx_host_t *ha = dev->ha; 5475 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5476 5477 QL_DPRINT12(ha, "enter\n"); 5478 5479 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5480 5481 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5482 ecore_chain_consume(&qp->rq.pbl); 5483 qlnxr_inc_sw_cons(&qp->rq); 5484 5485 QL_DPRINT12(ha, "exit\n"); 5486 return 1; 5487 } 5488 5489 static int 5490 process_resp_flush(struct qlnxr_qp *qp, 5491 int num_entries, 5492 struct ib_wc *wc, 5493 u16 hw_cons) 5494 { 5495 u16 cnt = 0; 5496 qlnx_host_t *ha = qp->dev->ha; 5497 5498 QL_DPRINT12(ha, "enter\n"); 5499 5500 while (num_entries && qp->rq.wqe_cons != hw_cons) { 5501 /* fill WC */ 5502 wc->status = IB_WC_WR_FLUSH_ERR; 5503 wc->vendor_err = 0; 5504 wc->wc_flags = 0; 5505 wc->src_qp = qp->id; 5506 wc->byte_len = 0; 5507 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5508 wc->qp = &qp->ibqp; 5509 num_entries--; 5510 wc++; 5511 cnt++; 5512 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5513 ecore_chain_consume(&qp->rq.pbl); 5514 qlnxr_inc_sw_cons(&qp->rq); 5515 } 5516 5517 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5518 return cnt; 5519 } 5520 5521 static void 5522 try_consume_resp_cqe(struct qlnxr_cq *cq, 5523 struct qlnxr_qp *qp, 5524 struct rdma_cqe_responder *resp, 5525 int *update) 5526 { 5527 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 5528 consume_cqe(cq); 5529 *update |= 1; 5530 } 5531 } 5532 5533 static int 5534 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, 5535 struct qlnxr_qp *qp, 5536 struct qlnxr_cq *cq, 5537 int num_entries, 5538 struct ib_wc *wc, 5539 struct rdma_cqe_responder *resp, 5540 int *update) 5541 { 5542 int cnt; 5543 qlnx_host_t *ha = dev->ha; 5544 5545 QL_DPRINT12(ha, "enter\n"); 5546 5547 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 5548 consume_cqe(cq); 5549 *update |= 1; 5550 5551 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5552 return cnt; 5553 } 5554 5555 static int 5556 qlnxr_poll_cq_resp(struct qlnxr_dev *dev, 5557 struct qlnxr_qp *qp, 5558 struct qlnxr_cq *cq, 5559 int num_entries, 5560 struct ib_wc *wc, 5561 struct rdma_cqe_responder *resp, 5562 int *update) 5563 { 5564 int cnt; 5565 qlnx_host_t *ha = dev->ha; 5566 5567 QL_DPRINT12(ha, "enter\n"); 5568 5569 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5570 cnt = process_resp_flush(qp, num_entries, wc, 5571 resp->rq_cons); 5572 try_consume_resp_cqe(cq, qp, resp, update); 5573 } else { 5574 cnt = process_resp_one(dev, qp, cq, wc, resp); 5575 consume_cqe(cq); 5576 *update |= 1; 5577 } 5578 5579 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5580 return cnt; 5581 } 5582 5583 static void 5584 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, 5585 struct rdma_cqe_requester *req, int *update) 5586 { 5587 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 5588 consume_cqe(cq); 5589 *update |= 1; 5590 } 5591 } 5592 5593 static void 5594 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) 5595 { 5596 uint64_t reg_addr; 5597 qlnx_host_t *ha = dev->ha; 5598 5599 QL_DPRINT12(ha, "enter\n"); 5600 5601 wmb(); 5602 cq->db.data.agg_flags = flags; 5603 cq->db.data.value = cpu_to_le32(cons); 5604 5605 reg_addr = (uint64_t)((uint8_t *)cq->db_addr - 5606 (uint8_t *)(ha->cdev.doorbells)); 5607 5608 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); 5609 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5610 5611 QL_DPRINT12(ha, "exit\n"); 5612 return; 5613 5614 //#ifdef __LP64__ 5615 // writeq(cq->db.raw, cq->db_addr); 5616 //#else 5617 /* Note that since the FW allows 64 bit write only, in 32bit systems 5618 * the value of db_addr must be low enough. This is currently not 5619 * enforced. 5620 */ 5621 // writel(cq->db.raw & 0xffffffff, cq->db_addr); 5622 // mmiowb(); 5623 //#endif 5624 } 5625 5626 static int 5627 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) 5628 { 5629 struct rdma_cqe_requester *resp_cqe = &cqe->req; 5630 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == 5631 cq->pbl_toggle; 5632 } 5633 5634 int 5635 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 5636 { 5637 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5638 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 5639 int done = 0; 5640 union rdma_cqe *cqe = cq->latest_cqe; 5641 int update = 0; 5642 u32 old_cons, new_cons; 5643 unsigned long flags; 5644 qlnx_host_t *ha = dev->ha; 5645 5646 QL_DPRINT12(ha, "enter\n"); 5647 5648 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5649 return -EINVAL; 5650 5651 if (cq->destroyed) { 5652 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", 5653 cq, cq->icid); 5654 return 0; 5655 } 5656 5657 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) 5658 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); 5659 5660 spin_lock_irqsave(&cq->cq_lock, flags); 5661 5662 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5663 5664 while (num_entries && is_valid_cqe(cq, cqe)) { 5665 int cnt = 0; 5666 struct qlnxr_qp *qp; 5667 struct rdma_cqe_requester *resp_cqe; 5668 enum rdma_cqe_type cqe_type; 5669 5670 /* prevent speculative reads of any field of CQE */ 5671 rmb(); 5672 5673 resp_cqe = &cqe->req; 5674 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, 5675 resp_cqe->qp_handle.lo); 5676 5677 if (!qp) { 5678 QL_DPRINT11(ha, "qp = NULL\n"); 5679 break; 5680 } 5681 5682 wc->qp = &qp->ibqp; 5683 5684 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 5685 5686 switch (cqe_type) { 5687 case RDMA_CQE_TYPE_REQUESTER: 5688 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, 5689 wc, &cqe->req); 5690 try_consume_req_cqe(cq, qp, &cqe->req, &update); 5691 break; 5692 case RDMA_CQE_TYPE_RESPONDER_RQ: 5693 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, 5694 wc, &cqe->resp, &update); 5695 break; 5696 case RDMA_CQE_TYPE_RESPONDER_SRQ: 5697 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, 5698 wc, &cqe->resp, &update); 5699 break; 5700 case RDMA_CQE_TYPE_INVALID: 5701 default: 5702 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); 5703 break; 5704 } 5705 num_entries -= cnt; 5706 wc += cnt; 5707 done += cnt; 5708 5709 cqe = cq->latest_cqe; 5710 } 5711 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5712 5713 cq->cq_cons += new_cons - old_cons; 5714 5715 if (update) { 5716 /* doorbell notifies abount latest VALID entry, 5717 * but chain already point to the next INVALID one 5718 */ 5719 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); 5720 QL_DPRINT12(ha, "cq = %p cons = 0x%x " 5721 "arm_flags = 0x%x db.icid = 0x%x\n", cq, 5722 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); 5723 } 5724 5725 spin_unlock_irqrestore(&cq->cq_lock, flags); 5726 5727 QL_DPRINT12(ha, "exit\n"); 5728 5729 return done; 5730 } 5731 5732 int 5733 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 5734 { 5735 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5736 unsigned long sflags; 5737 struct qlnxr_dev *dev; 5738 qlnx_host_t *ha; 5739 5740 dev = get_qlnxr_dev((ibcq->device)); 5741 ha = dev->ha; 5742 5743 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " 5744 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, 5745 flags, cq, cq->cq_cons, cq->cq_type); 5746 5747 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5748 return -EINVAL; 5749 5750 if (cq->destroyed) { 5751 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", 5752 cq, cq->icid); 5753 return -EINVAL; 5754 } 5755 5756 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { 5757 return 0; 5758 } 5759 5760 spin_lock_irqsave(&cq->cq_lock, sflags); 5761 5762 cq->arm_flags = 0; 5763 5764 if (flags & IB_CQ_SOLICITED) { 5765 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 5766 } 5767 if (flags & IB_CQ_NEXT_COMP) { 5768 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 5769 } 5770 5771 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); 5772 5773 spin_unlock_irqrestore(&cq->cq_lock, sflags); 5774 5775 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); 5776 return 0; 5777 } 5778 5779 static struct qlnxr_mr * 5780 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) 5781 { 5782 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 5783 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 5784 struct qlnxr_mr *mr; 5785 int rc = -ENOMEM; 5786 qlnx_host_t *ha; 5787 5788 ha = dev->ha; 5789 5790 QL_DPRINT12(ha, "enter ibpd = %p pd = %p " 5791 " pd_id = %d max_page_list_len = %d\n", 5792 ibpd, pd, pd->pd_id, max_page_list_len); 5793 5794 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5795 if (!mr) { 5796 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 5797 return ERR_PTR(rc); 5798 } 5799 5800 mr->dev = dev; 5801 mr->type = QLNXR_MR_FRMR; 5802 5803 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 5804 1 /* allow dual layer pbl */); 5805 if (rc) { 5806 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); 5807 goto err0; 5808 } 5809 5810 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 5811 if (rc) { 5812 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); 5813 goto err0; 5814 } 5815 5816 /* index only, 18 bit long, lkey = itid << 8 | key */ 5817 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; 5818 mr->hw_mr.key = 0; 5819 mr->hw_mr.pd = pd->pd_id; 5820 mr->hw_mr.local_read = 1; 5821 mr->hw_mr.local_write = 0; 5822 mr->hw_mr.remote_read = 0; 5823 mr->hw_mr.remote_write = 0; 5824 mr->hw_mr.remote_atomic = 0; 5825 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 5826 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ 5827 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 5828 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 5829 mr->hw_mr.fbo = 0; 5830 mr->hw_mr.length = 0; 5831 mr->hw_mr.vaddr = 0; 5832 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 5833 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ 5834 mr->hw_mr.dma_mr = false; 5835 5836 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 5837 if (rc) { 5838 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); 5839 goto err1; 5840 } 5841 5842 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 5843 mr->ibmr.rkey = mr->ibmr.lkey; 5844 5845 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", 5846 mr, mr->ibmr.lkey); 5847 5848 return mr; 5849 5850 err1: 5851 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 5852 err0: 5853 kfree(mr); 5854 5855 QL_DPRINT12(ha, "exit\n"); 5856 5857 return ERR_PTR(rc); 5858 } 5859 5860 #if __FreeBSD_version >= 1102000 5861 5862 struct ib_mr * 5863 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) 5864 { 5865 struct qlnxr_dev *dev; 5866 struct qlnxr_mr *mr; 5867 qlnx_host_t *ha; 5868 5869 dev = get_qlnxr_dev(ibpd->device); 5870 ha = dev->ha; 5871 5872 QL_DPRINT12(ha, "enter\n"); 5873 5874 if (mr_type != IB_MR_TYPE_MEM_REG) 5875 return ERR_PTR(-EINVAL); 5876 5877 mr = __qlnxr_alloc_mr(ibpd, max_num_sg); 5878 5879 if (IS_ERR(mr)) 5880 return ERR_PTR(-EINVAL); 5881 5882 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); 5883 5884 return &mr->ibmr; 5885 } 5886 5887 static int 5888 qlnxr_set_page(struct ib_mr *ibmr, u64 addr) 5889 { 5890 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5891 struct qlnxr_pbl *pbl_table; 5892 struct regpair *pbe; 5893 struct qlnxr_dev *dev; 5894 qlnx_host_t *ha; 5895 u32 pbes_in_page; 5896 5897 dev = mr->dev; 5898 ha = dev->ha; 5899 5900 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 5901 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); 5902 return -ENOMEM; 5903 } 5904 5905 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, 5906 ((void *)addr)); 5907 5908 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 5909 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 5910 pbe = (struct regpair *)pbl_table->va; 5911 pbe += mr->npages % pbes_in_page; 5912 pbe->lo = cpu_to_le32((u32)addr); 5913 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 5914 5915 mr->npages++; 5916 5917 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, 5918 ((void *)addr)); 5919 return 0; 5920 } 5921 5922 int 5923 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 5924 int sg_nents, unsigned int *sg_offset) 5925 { 5926 int ret; 5927 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5928 qlnx_host_t *ha; 5929 5930 if (mr == NULL) 5931 return (-1); 5932 5933 if (mr->dev == NULL) 5934 return (-1); 5935 5936 ha = mr->dev->ha; 5937 5938 QL_DPRINT12(ha, "enter\n"); 5939 5940 mr->npages = 0; 5941 qlnx_handle_completed_mrs(mr->dev, &mr->info); 5942 5943 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); 5944 5945 QL_DPRINT12(ha, "exit ret = %d\n", ret); 5946 5947 return (ret); 5948 } 5949 5950 #else 5951 5952 struct ib_mr * 5953 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len) 5954 { 5955 struct qlnxr_dev *dev; 5956 struct qlnxr_mr *mr; 5957 qlnx_host_t *ha; 5958 struct ib_mr *ibmr = NULL; 5959 5960 dev = get_qlnxr_dev((ibpd->device)); 5961 ha = dev->ha; 5962 5963 QL_DPRINT12(ha, "enter\n"); 5964 5965 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len); 5966 5967 if (IS_ERR(mr)) { 5968 ibmr = ERR_PTR(-EINVAL); 5969 } else { 5970 ibmr = &mr->ibmr; 5971 } 5972 5973 QL_DPRINT12(ha, "exit %p\n", ibmr); 5974 return (ibmr); 5975 } 5976 5977 void 5978 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list) 5979 { 5980 struct qlnxr_fast_reg_page_list *frmr_list; 5981 5982 frmr_list = get_qlnxr_frmr_list(page_list); 5983 5984 free_mr_info(frmr_list->dev, &frmr_list->info); 5985 5986 kfree(frmr_list->ibfrpl.page_list); 5987 kfree(frmr_list); 5988 5989 return; 5990 } 5991 5992 struct ib_fast_reg_page_list * 5993 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len) 5994 { 5995 struct qlnxr_fast_reg_page_list *frmr_list = NULL; 5996 struct qlnxr_dev *dev; 5997 int size = page_list_len * sizeof(u64); 5998 int rc = -ENOMEM; 5999 qlnx_host_t *ha; 6000 6001 dev = get_qlnxr_dev(ibdev); 6002 ha = dev->ha; 6003 6004 QL_DPRINT12(ha, "enter\n"); 6005 6006 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL); 6007 if (!frmr_list) { 6008 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n"); 6009 goto err; 6010 } 6011 6012 frmr_list->dev = dev; 6013 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL); 6014 if (!frmr_list->ibfrpl.page_list) { 6015 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n"); 6016 goto err0; 6017 } 6018 6019 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len, 6020 1 /* allow dual layer pbl */); 6021 if (rc) 6022 goto err1; 6023 6024 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl); 6025 6026 return &frmr_list->ibfrpl; 6027 6028 err1: 6029 kfree(frmr_list->ibfrpl.page_list); 6030 err0: 6031 kfree(frmr_list); 6032 err: 6033 QL_DPRINT12(ha, "exit with error\n"); 6034 6035 return ERR_PTR(rc); 6036 } 6037 6038 static int 6039 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6040 int buf_cnt, uint64_t *total_size) 6041 { 6042 u64 size = 0; 6043 6044 *total_size = 0; 6045 6046 if (!buf_cnt || buf_list == NULL) { 6047 QL_DPRINT11(ha, 6048 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt); 6049 return (-1); 6050 } 6051 6052 size = buf_list->size; 6053 6054 if (!size) { 6055 QL_DPRINT11(ha, 6056 "failed buf_list = %p buf_cnt = %d" 6057 " buf_list->size = 0\n", buf_list, buf_cnt); 6058 return (-1); 6059 } 6060 6061 while (buf_cnt) { 6062 *total_size += buf_list->size; 6063 6064 if (buf_list->size != size) { 6065 QL_DPRINT11(ha, 6066 "failed buf_list = %p buf_cnt = %d" 6067 " all buffers should have same size\n", 6068 buf_list, buf_cnt); 6069 return (-1); 6070 } 6071 6072 buf_list++; 6073 buf_cnt--; 6074 } 6075 return (0); 6076 } 6077 6078 static size_t 6079 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6080 int buf_cnt) 6081 { 6082 int i; 6083 size_t num_pages = 0; 6084 u64 size; 6085 6086 for (i = 0; i < buf_cnt; i++) { 6087 size = 0; 6088 while (size < buf_list->size) { 6089 size += PAGE_SIZE; 6090 num_pages++; 6091 } 6092 buf_list++; 6093 } 6094 return (num_pages); 6095 } 6096 6097 static void 6098 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev, 6099 struct ib_phys_buf *buf_list, int buf_cnt, 6100 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 6101 { 6102 struct regpair *pbe; 6103 struct qlnxr_pbl *pbl_tbl; 6104 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 6105 qlnx_host_t *ha; 6106 int i; 6107 u64 pbe_addr; 6108 6109 ha = dev->ha; 6110 6111 QL_DPRINT12(ha, "enter\n"); 6112 6113 if (!pbl_info) { 6114 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 6115 return; 6116 } 6117 6118 if (!pbl_info->num_pbes) { 6119 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 6120 return; 6121 } 6122 6123 /* If we have a two layered pbl, the first pbl points to the rest 6124 * of the pbls and the first entry lays on the second pbl in the table 6125 */ 6126 if (pbl_info->two_layered) 6127 pbl_tbl = &pbl[1]; 6128 else 6129 pbl_tbl = pbl; 6130 6131 pbe = (struct regpair *)pbl_tbl->va; 6132 if (!pbe) { 6133 QL_DPRINT12(ha, "pbe is NULL\n"); 6134 return; 6135 } 6136 6137 pbe_cnt = 0; 6138 6139 for (i = 0; i < buf_cnt; i++) { 6140 pages = buf_list->size >> PAGE_SHIFT; 6141 6142 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 6143 /* store the page address in pbe */ 6144 6145 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt); 6146 6147 pbe->lo = cpu_to_le32((u32)pbe_addr); 6148 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32))); 6149 6150 QL_DPRINT12(ha, "Populate pbl table:" 6151 " pbe->addr=0x%x:0x%x " 6152 " pbe_cnt = %d total_num_pbes=%d" 6153 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 6154 total_num_pbes, pbe); 6155 6156 pbe_cnt ++; 6157 total_num_pbes ++; 6158 pbe++; 6159 6160 if (total_num_pbes == pbl_info->num_pbes) 6161 return; 6162 6163 /* if the given pbl is full storing the pbes, 6164 * move to next pbl. */ 6165 6166 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 6167 pbl_tbl++; 6168 pbe = (struct regpair *)pbl_tbl->va; 6169 pbe_cnt = 0; 6170 } 6171 } 6172 buf_list++; 6173 } 6174 QL_DPRINT12(ha, "exit\n"); 6175 return; 6176 } 6177 6178 struct ib_mr * 6179 qlnxr_reg_kernel_mr(struct ib_pd *ibpd, 6180 struct ib_phys_buf *buf_list, 6181 int buf_cnt, int acc, u64 *iova_start) 6182 { 6183 int rc = -ENOMEM; 6184 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 6185 struct qlnxr_mr *mr; 6186 struct qlnxr_pd *pd; 6187 qlnx_host_t *ha; 6188 size_t num_pages = 0; 6189 uint64_t length; 6190 6191 ha = dev->ha; 6192 6193 QL_DPRINT12(ha, "enter\n"); 6194 6195 pd = get_qlnxr_pd(ibpd); 6196 6197 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d," 6198 " iova_start = %p, acc = %d\n", 6199 pd->pd_id, buf_list, buf_cnt, iova_start, acc); 6200 6201 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 6202 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&" 6203 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 6204 // return ERR_PTR(-EINVAL); 6205 //} 6206 6207 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 6208 if (!mr) { 6209 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 6210 return ERR_PTR(rc); 6211 } 6212 6213 mr->type = QLNXR_MR_KERNEL; 6214 mr->iova_start = iova_start; 6215 6216 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length); 6217 if (rc) 6218 goto err0; 6219 6220 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt); 6221 if (!num_pages) 6222 goto err0; 6223 6224 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1); 6225 if (rc) { 6226 QL_DPRINT11(ha, 6227 "qlnxr_init_mr_info failed [%d]\n", rc); 6228 goto err1; 6229 } 6230 6231 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table, 6232 &mr->info.pbl_info); 6233 6234 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 6235 6236 if (rc) { 6237 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 6238 goto err1; 6239 } 6240 6241 /* index only, 18 bit long, lkey = itid << 8 | key */ 6242 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 6243 mr->hw_mr.key = 0; 6244 mr->hw_mr.pd = pd->pd_id; 6245 mr->hw_mr.local_read = 1; 6246 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 6247 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 6248 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 6249 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 6250 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 6251 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 6252 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 6253 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 6254 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */ 6255 6256 mr->hw_mr.fbo = 0; 6257 6258 mr->hw_mr.length = length; 6259 mr->hw_mr.vaddr = (uint64_t)iova_start; 6260 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 6261 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 6262 mr->hw_mr.dma_mr = false; 6263 6264 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 6265 if (rc) { 6266 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 6267 goto err2; 6268 } 6269 6270 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6271 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 6272 mr->hw_mr.remote_atomic) 6273 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6274 6275 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey); 6276 6277 return (&mr->ibmr); 6278 6279 err2: 6280 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 6281 err1: 6282 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 6283 err0: 6284 kfree(mr); 6285 6286 QL_DPRINT12(ha, "exit [%d]\n", rc); 6287 return (ERR_PTR(rc)); 6288 } 6289 6290 #endif /* #if __FreeBSD_version >= 1102000 */ 6291 6292 struct ib_ah * 6293 #if __FreeBSD_version >= 1102000 6294 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, 6295 struct ib_udata *udata) 6296 #else 6297 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) 6298 #endif /* #if __FreeBSD_version >= 1102000 */ 6299 { 6300 struct qlnxr_dev *dev; 6301 qlnx_host_t *ha; 6302 struct qlnxr_ah *ah; 6303 6304 dev = get_qlnxr_dev((ibpd->device)); 6305 ha = dev->ha; 6306 6307 QL_DPRINT12(ha, "in create_ah\n"); 6308 6309 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 6310 if (!ah) { 6311 QL_DPRINT12(ha, "no address handle can be allocated\n"); 6312 return ERR_PTR(-ENOMEM); 6313 } 6314 6315 ah->attr = *attr; 6316 6317 return &ah->ibah; 6318 } 6319 6320 int 6321 qlnxr_destroy_ah(struct ib_ah *ibah) 6322 { 6323 struct qlnxr_dev *dev; 6324 qlnx_host_t *ha; 6325 struct qlnxr_ah *ah = get_qlnxr_ah(ibah); 6326 6327 dev = get_qlnxr_dev((ibah->device)); 6328 ha = dev->ha; 6329 6330 QL_DPRINT12(ha, "in destroy_ah\n"); 6331 6332 kfree(ah); 6333 return 0; 6334 } 6335 6336 int 6337 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6338 { 6339 struct qlnxr_dev *dev; 6340 qlnx_host_t *ha; 6341 6342 dev = get_qlnxr_dev((ibah->device)); 6343 ha = dev->ha; 6344 QL_DPRINT12(ha, "Query AH not supported\n"); 6345 return -EINVAL; 6346 } 6347 6348 int 6349 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6350 { 6351 struct qlnxr_dev *dev; 6352 qlnx_host_t *ha; 6353 6354 dev = get_qlnxr_dev((ibah->device)); 6355 ha = dev->ha; 6356 QL_DPRINT12(ha, "Modify AH not supported\n"); 6357 return -ENOSYS; 6358 } 6359 6360 #if __FreeBSD_version >= 1102000 6361 int 6362 qlnxr_process_mad(struct ib_device *ibdev, 6363 int process_mad_flags, 6364 u8 port_num, 6365 const struct ib_wc *in_wc, 6366 const struct ib_grh *in_grh, 6367 const struct ib_mad_hdr *mad_hdr, 6368 size_t in_mad_size, 6369 struct ib_mad_hdr *out_mad, 6370 size_t *out_mad_size, 6371 u16 *out_mad_pkey_index) 6372 6373 #else 6374 6375 int 6376 qlnxr_process_mad(struct ib_device *ibdev, 6377 int process_mad_flags, 6378 u8 port_num, 6379 struct ib_wc *in_wc, 6380 struct ib_grh *in_grh, 6381 struct ib_mad *in_mad, 6382 struct ib_mad *out_mad) 6383 6384 #endif /* #if __FreeBSD_version >= 1102000 */ 6385 { 6386 struct qlnxr_dev *dev; 6387 qlnx_host_t *ha; 6388 6389 dev = get_qlnxr_dev(ibdev); 6390 ha = dev->ha; 6391 QL_DPRINT12(ha, "process mad not supported\n"); 6392 6393 return -ENOSYS; 6394 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", 6395 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, 6396 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, 6397 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, 6398 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); 6399 6400 // return IB_MAD_RESULT_SUCCESS; 6401 } 6402 6403 #if __FreeBSD_version >= 1102000 6404 int 6405 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, 6406 struct ib_port_immutable *immutable) 6407 { 6408 struct qlnxr_dev *dev; 6409 qlnx_host_t *ha; 6410 struct ib_port_attr attr; 6411 int err; 6412 6413 dev = get_qlnxr_dev(ibdev); 6414 ha = dev->ha; 6415 6416 QL_DPRINT12(ha, "enter\n"); 6417 6418 err = qlnxr_query_port(ibdev, port_num, &attr); 6419 if (err) 6420 return err; 6421 6422 if (QLNX_IS_IWARP(dev)) { 6423 immutable->pkey_tbl_len = 1; 6424 immutable->gid_tbl_len = 1; 6425 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 6426 immutable->max_mad_size = 0; 6427 } else { 6428 immutable->pkey_tbl_len = attr.pkey_tbl_len; 6429 immutable->gid_tbl_len = attr.gid_tbl_len; 6430 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 6431 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 6432 } 6433 6434 QL_DPRINT12(ha, "exit\n"); 6435 return 0; 6436 } 6437 #endif /* #if __FreeBSD_version > 1102000 */ 6438 6439 /***** iWARP related functions *************/ 6440 6441 static void 6442 qlnxr_iw_mpa_request(void *context, 6443 struct ecore_iwarp_cm_event_params *params) 6444 { 6445 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; 6446 struct qlnxr_dev *dev = listener->dev; 6447 struct qlnxr_iw_ep *ep; 6448 struct iw_cm_event event; 6449 struct sockaddr_in *laddr; 6450 struct sockaddr_in *raddr; 6451 qlnx_host_t *ha; 6452 6453 ha = dev->ha; 6454 6455 QL_DPRINT12(ha, "enter\n"); 6456 6457 if (params->cm_info->ip_version != ECORE_TCP_IPV4) { 6458 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", 6459 params->cm_info->ip_version); 6460 return; 6461 } 6462 6463 ep = kzalloc(sizeof(*ep), GFP_ATOMIC); 6464 6465 if (!ep) { 6466 QL_DPRINT11(ha, "kzalloc{ep) failed\n"); 6467 return; 6468 } 6469 6470 ep->dev = dev; 6471 ep->ecore_context = params->ep_context; 6472 6473 memset(&event, 0, sizeof(event)); 6474 6475 event.event = IW_CM_EVENT_CONNECT_REQUEST; 6476 event.status = params->status; 6477 6478 laddr = (struct sockaddr_in *)&event.local_addr; 6479 raddr = (struct sockaddr_in *)&event.remote_addr; 6480 6481 laddr->sin_family = AF_INET; 6482 raddr->sin_family = AF_INET; 6483 6484 laddr->sin_port = htons(params->cm_info->local_port); 6485 raddr->sin_port = htons(params->cm_info->remote_port); 6486 6487 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); 6488 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); 6489 6490 event.provider_data = (void *)ep; 6491 event.private_data = (void *)params->cm_info->private_data; 6492 event.private_data_len = (u8)params->cm_info->private_data_len; 6493 6494 #if __FreeBSD_version >= 1100000 6495 event.ord = params->cm_info->ord; 6496 event.ird = params->cm_info->ird; 6497 #endif /* #if __FreeBSD_version >= 1100000 */ 6498 6499 listener->cm_id->event_handler(listener->cm_id, &event); 6500 6501 QL_DPRINT12(ha, "exit\n"); 6502 6503 return; 6504 } 6505 6506 static void 6507 qlnxr_iw_issue_event(void *context, 6508 struct ecore_iwarp_cm_event_params *params, 6509 enum iw_cm_event_type event_type, 6510 char *str) 6511 { 6512 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6513 struct qlnxr_dev *dev = ep->dev; 6514 struct iw_cm_event event; 6515 qlnx_host_t *ha; 6516 6517 ha = dev->ha; 6518 6519 QL_DPRINT12(ha, "enter\n"); 6520 6521 memset(&event, 0, sizeof(event)); 6522 event.status = params->status; 6523 event.event = event_type; 6524 6525 if (params->cm_info != NULL) { 6526 #if __FreeBSD_version >= 1100000 6527 event.ird = params->cm_info->ird; 6528 event.ord = params->cm_info->ord; 6529 QL_DPRINT12(ha, "ord=[%d] \n", event.ord); 6530 QL_DPRINT12(ha, "ird=[%d] \n", event.ird); 6531 #endif /* #if __FreeBSD_version >= 1100000 */ 6532 6533 event.private_data_len = params->cm_info->private_data_len; 6534 event.private_data = (void *)params->cm_info->private_data; 6535 QL_DPRINT12(ha, "private_data_len=[%d] \n", 6536 event.private_data_len); 6537 } 6538 6539 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); 6540 QL_DPRINT12(ha, "status=[%d] \n", event.status); 6541 6542 if (ep) { 6543 if (ep->cm_id) 6544 ep->cm_id->event_handler(ep->cm_id, &event); 6545 else 6546 QL_DPRINT11(ha, "ep->cm_id == NULL \n"); 6547 } else { 6548 QL_DPRINT11(ha, "ep == NULL \n"); 6549 } 6550 6551 QL_DPRINT12(ha, "exit\n"); 6552 6553 return; 6554 } 6555 6556 static void 6557 qlnxr_iw_close_event(void *context, 6558 struct ecore_iwarp_cm_event_params *params) 6559 { 6560 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6561 struct qlnxr_dev *dev = ep->dev; 6562 qlnx_host_t *ha; 6563 6564 ha = dev->ha; 6565 6566 QL_DPRINT12(ha, "enter\n"); 6567 6568 if (ep->cm_id) { 6569 qlnxr_iw_issue_event(context, 6570 params, 6571 IW_CM_EVENT_CLOSE, 6572 "IW_CM_EVENT_EVENT_CLOSE"); 6573 ep->cm_id->rem_ref(ep->cm_id); 6574 ep->cm_id = NULL; 6575 } 6576 6577 QL_DPRINT12(ha, "exit\n"); 6578 6579 return; 6580 } 6581 6582 #if __FreeBSD_version >= 1102000 6583 6584 static void 6585 qlnxr_iw_passive_complete(void *context, 6586 struct ecore_iwarp_cm_event_params *params) 6587 { 6588 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6589 struct qlnxr_dev *dev = ep->dev; 6590 qlnx_host_t *ha; 6591 6592 ha = dev->ha; 6593 6594 /* We will only reach the following state if MPA_REJECT was called on 6595 * passive. In this case there will be no associated QP. 6596 */ 6597 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { 6598 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); 6599 kfree(ep); 6600 return; 6601 } 6602 6603 /* We always issue an established event, however, ofed does not look 6604 * at event code for established. So if there was a failure, we follow 6605 * with close... 6606 */ 6607 qlnxr_iw_issue_event(context, 6608 params, 6609 IW_CM_EVENT_ESTABLISHED, 6610 "IW_CM_EVENT_ESTABLISHED"); 6611 6612 if (params->status < 0) { 6613 qlnxr_iw_close_event(context, params); 6614 } 6615 6616 return; 6617 } 6618 6619 struct qlnxr_discon_work { 6620 struct work_struct work; 6621 struct qlnxr_iw_ep *ep; 6622 enum ecore_iwarp_event_type event; 6623 int status; 6624 }; 6625 6626 static void 6627 qlnxr_iw_disconnect_worker(struct work_struct *work) 6628 { 6629 struct qlnxr_discon_work *dwork = 6630 container_of(work, struct qlnxr_discon_work, work); 6631 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 6632 struct qlnxr_iw_ep *ep = dwork->ep; 6633 struct qlnxr_dev *dev = ep->dev; 6634 struct qlnxr_qp *qp = ep->qp; 6635 struct iw_cm_event event; 6636 6637 if (qp->destroyed) { 6638 kfree(dwork); 6639 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6640 return; 6641 } 6642 6643 memset(&event, 0, sizeof(event)); 6644 event.status = dwork->status; 6645 event.event = IW_CM_EVENT_DISCONNECT; 6646 6647 /* Success means graceful disconnect was requested. modifying 6648 * to SQD is translated to graceful disconnect. O/w reset is sent 6649 */ 6650 if (dwork->status) 6651 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; 6652 else 6653 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; 6654 6655 kfree(dwork); 6656 6657 if (ep->cm_id) 6658 ep->cm_id->event_handler(ep->cm_id, &event); 6659 6660 SET_FIELD(qp_params.modify_flags, 6661 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 6662 6663 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 6664 6665 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6666 6667 return; 6668 } 6669 6670 void 6671 qlnxr_iw_disconnect_event(void *context, 6672 struct ecore_iwarp_cm_event_params *params) 6673 { 6674 struct qlnxr_discon_work *work; 6675 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6676 struct qlnxr_dev *dev = ep->dev; 6677 struct qlnxr_qp *qp = ep->qp; 6678 6679 work = kzalloc(sizeof(*work), GFP_ATOMIC); 6680 if (!work) 6681 return; 6682 6683 qlnxr_iw_qp_add_ref(&qp->ibqp); 6684 work->ep = ep; 6685 work->event = params->event; 6686 work->status = params->status; 6687 6688 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); 6689 queue_work(dev->iwarp_wq, &work->work); 6690 6691 return; 6692 } 6693 6694 #endif /* #if __FreeBSD_version >= 1102000 */ 6695 6696 static int 6697 qlnxr_iw_mpa_reply(void *context, 6698 struct ecore_iwarp_cm_event_params *params) 6699 { 6700 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6701 struct qlnxr_dev *dev = ep->dev; 6702 struct ecore_iwarp_send_rtr_in rtr_in; 6703 int rc; 6704 qlnx_host_t *ha; 6705 6706 ha = dev->ha; 6707 6708 QL_DPRINT12(ha, "enter\n"); 6709 6710 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6711 return -EINVAL; 6712 6713 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); 6714 rtr_in.ep_context = params->ep_context; 6715 6716 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); 6717 6718 QL_DPRINT12(ha, "exit rc = %d\n", rc); 6719 return rc; 6720 } 6721 6722 void 6723 qlnxr_iw_qp_event(void *context, 6724 struct ecore_iwarp_cm_event_params *params, 6725 enum ib_event_type ib_event, 6726 char *str) 6727 { 6728 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6729 struct qlnxr_dev *dev = ep->dev; 6730 struct ib_qp *ibqp = &(ep->qp->ibqp); 6731 struct ib_event event; 6732 qlnx_host_t *ha; 6733 6734 ha = dev->ha; 6735 6736 QL_DPRINT12(ha, 6737 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", 6738 context, params->event, str, ibqp->event_handler); 6739 6740 if (ibqp->event_handler) { 6741 event.event = ib_event; 6742 event.device = ibqp->device; 6743 event.element.qp = ibqp; 6744 ibqp->event_handler(&event, ibqp->qp_context); 6745 } 6746 6747 return; 6748 } 6749 6750 int 6751 qlnxr_iw_event_handler(void *context, 6752 struct ecore_iwarp_cm_event_params *params) 6753 { 6754 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6755 struct qlnxr_dev *dev = ep->dev; 6756 qlnx_host_t *ha; 6757 6758 ha = dev->ha; 6759 6760 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6761 "enter\n", context, params->event); 6762 6763 switch (params->event) { 6764 /* Passive side request received */ 6765 case ECORE_IWARP_EVENT_MPA_REQUEST: 6766 qlnxr_iw_mpa_request(context, params); 6767 break; 6768 6769 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: 6770 qlnxr_iw_mpa_reply(context, params); 6771 break; 6772 6773 /* Passive side established ( ack on mpa response ) */ 6774 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: 6775 6776 #if __FreeBSD_version >= 1102000 6777 6778 ep->during_connect = 0; 6779 qlnxr_iw_passive_complete(context, params); 6780 6781 #else 6782 qlnxr_iw_issue_event(context, 6783 params, 6784 IW_CM_EVENT_ESTABLISHED, 6785 "IW_CM_EVENT_ESTABLISHED"); 6786 #endif /* #if __FreeBSD_version >= 1102000 */ 6787 break; 6788 6789 /* Active side reply received */ 6790 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: 6791 ep->during_connect = 0; 6792 qlnxr_iw_issue_event(context, 6793 params, 6794 IW_CM_EVENT_CONNECT_REPLY, 6795 "IW_CM_EVENT_CONNECT_REPLY"); 6796 if (params->status < 0) { 6797 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6798 6799 ep->cm_id->rem_ref(ep->cm_id); 6800 ep->cm_id = NULL; 6801 } 6802 break; 6803 6804 case ECORE_IWARP_EVENT_DISCONNECT: 6805 6806 #if __FreeBSD_version >= 1102000 6807 qlnxr_iw_disconnect_event(context, params); 6808 #else 6809 qlnxr_iw_issue_event(context, 6810 params, 6811 IW_CM_EVENT_DISCONNECT, 6812 "IW_CM_EVENT_DISCONNECT"); 6813 qlnxr_iw_close_event(context, params); 6814 #endif /* #if __FreeBSD_version >= 1102000 */ 6815 break; 6816 6817 case ECORE_IWARP_EVENT_CLOSE: 6818 ep->during_connect = 0; 6819 qlnxr_iw_close_event(context, params); 6820 break; 6821 6822 case ECORE_IWARP_EVENT_RQ_EMPTY: 6823 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6824 "IWARP_EVENT_RQ_EMPTY"); 6825 break; 6826 6827 case ECORE_IWARP_EVENT_IRQ_FULL: 6828 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6829 "IWARP_EVENT_IRQ_FULL"); 6830 break; 6831 6832 case ECORE_IWARP_EVENT_LLP_TIMEOUT: 6833 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6834 "IWARP_EVENT_LLP_TIMEOUT"); 6835 break; 6836 6837 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: 6838 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6839 "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); 6840 break; 6841 6842 case ECORE_IWARP_EVENT_CQ_OVERFLOW: 6843 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6844 "QED_IWARP_EVENT_CQ_OVERFLOW"); 6845 break; 6846 6847 case ECORE_IWARP_EVENT_QP_CATASTROPHIC: 6848 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6849 "QED_IWARP_EVENT_QP_CATASTROPHIC"); 6850 break; 6851 6852 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: 6853 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6854 "IWARP_EVENT_LOCAL_ACCESS_ERROR"); 6855 break; 6856 6857 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: 6858 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6859 "IWARP_EVENT_REMOTE_OPERATION_ERROR"); 6860 break; 6861 6862 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: 6863 QL_DPRINT12(ha, "Got terminate message" 6864 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); 6865 break; 6866 6867 default: 6868 QL_DPRINT12(ha, 6869 "Unknown event [0x%x] received \n", params->event); 6870 break; 6871 }; 6872 6873 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6874 "exit\n", context, params->event); 6875 return 0; 6876 } 6877 6878 static int 6879 qlnxr_addr4_resolve(struct qlnxr_dev *dev, 6880 struct sockaddr_in *src_in, 6881 struct sockaddr_in *dst_in, 6882 u8 *dst_mac) 6883 { 6884 int rc; 6885 6886 #if __FreeBSD_version >= 1100000 6887 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, 6888 dst_mac, NULL, NULL); 6889 #else 6890 struct llentry *lle; 6891 6892 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in, 6893 dst_mac, &lle); 6894 #endif 6895 6896 QL_DPRINT12(dev->ha, "rc = %d " 6897 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " 6898 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, 6899 dst_in->sin_len, dst_in->sin_family, 6900 NIPQUAD((dst_in->sin_addr.s_addr)), 6901 dst_mac[0], dst_mac[1], dst_mac[2], 6902 dst_mac[3], dst_mac[4], dst_mac[5]); 6903 6904 return rc; 6905 } 6906 6907 int 6908 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 6909 { 6910 struct qlnxr_dev *dev; 6911 struct ecore_iwarp_connect_out out_params; 6912 struct ecore_iwarp_connect_in in_params; 6913 struct qlnxr_iw_ep *ep; 6914 struct qlnxr_qp *qp; 6915 struct sockaddr_in *laddr; 6916 struct sockaddr_in *raddr; 6917 int rc = 0; 6918 qlnx_host_t *ha; 6919 6920 dev = get_qlnxr_dev((cm_id->device)); 6921 ha = dev->ha; 6922 6923 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " 6924 "enter \n", cm_id, conn_param); 6925 6926 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6927 return -EINVAL; 6928 6929 qp = idr_find(&dev->qpidr, conn_param->qpn); 6930 6931 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6932 raddr = (struct sockaddr_in *)&cm_id->remote_addr; 6933 6934 QL_DPRINT12(ha, 6935 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", 6936 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, 6937 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); 6938 6939 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 6940 if (!ep) { 6941 QL_DPRINT11(ha, "struct qlnxr_iw_ep " 6942 "alloc memory failed\n"); 6943 return -ENOMEM; 6944 } 6945 6946 ep->dev = dev; 6947 ep->qp = qp; 6948 cm_id->add_ref(cm_id); 6949 ep->cm_id = cm_id; 6950 6951 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); 6952 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); 6953 6954 in_params.event_cb = qlnxr_iw_event_handler; 6955 in_params.cb_context = ep; 6956 6957 in_params.cm_info.ip_version = ECORE_TCP_IPV4; 6958 6959 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); 6960 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); 6961 in_params.cm_info.remote_port = ntohs(raddr->sin_port); 6962 in_params.cm_info.local_port = ntohs(laddr->sin_port); 6963 in_params.cm_info.vlan = 0; 6964 in_params.mss = dev->ha->ifp->if_mtu - 40; 6965 6966 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " 6967 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " 6968 "vlan = %d\n", 6969 NIPQUAD((in_params.cm_info.remote_ip[0])), 6970 NIPQUAD((in_params.cm_info.local_ip[0])), 6971 in_params.cm_info.remote_port, in_params.cm_info.local_port, 6972 in_params.cm_info.vlan); 6973 6974 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); 6975 6976 if (rc) { 6977 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); 6978 goto err; 6979 } 6980 6981 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" 6982 " private_data_len=%d rq_psn=%d\n", 6983 conn_param->ord, conn_param->ird, conn_param->private_data, 6984 conn_param->private_data_len, qp->rq_psn); 6985 6986 in_params.cm_info.ord = conn_param->ord; 6987 in_params.cm_info.ird = conn_param->ird; 6988 in_params.cm_info.private_data = conn_param->private_data; 6989 in_params.cm_info.private_data_len = conn_param->private_data_len; 6990 in_params.qp = qp->ecore_qp; 6991 6992 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 6993 6994 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); 6995 6996 if (rc) { 6997 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); 6998 goto err; 6999 } 7000 7001 QL_DPRINT12(ha, "exit\n"); 7002 7003 return rc; 7004 7005 err: 7006 cm_id->rem_ref(cm_id); 7007 kfree(ep); 7008 7009 QL_DPRINT12(ha, "exit [%d]\n", rc); 7010 return rc; 7011 } 7012 7013 int 7014 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) 7015 { 7016 struct qlnxr_dev *dev; 7017 struct qlnxr_iw_listener *listener; 7018 struct ecore_iwarp_listen_in iparams; 7019 struct ecore_iwarp_listen_out oparams; 7020 struct sockaddr_in *laddr; 7021 qlnx_host_t *ha; 7022 int rc; 7023 7024 dev = get_qlnxr_dev((cm_id->device)); 7025 ha = dev->ha; 7026 7027 QL_DPRINT12(ha, "enter\n"); 7028 7029 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7030 return -EINVAL; 7031 7032 laddr = (struct sockaddr_in *)&cm_id->local_addr; 7033 7034 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 7035 7036 if (listener == NULL) { 7037 QL_DPRINT11(ha, "listener memory alloc failed\n"); 7038 return -ENOMEM; 7039 } 7040 7041 listener->dev = dev; 7042 cm_id->add_ref(cm_id); 7043 listener->cm_id = cm_id; 7044 listener->backlog = backlog; 7045 7046 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); 7047 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); 7048 7049 iparams.cb_context = listener; 7050 iparams.event_cb = qlnxr_iw_event_handler; 7051 iparams.max_backlog = backlog; 7052 7053 iparams.ip_version = ECORE_TCP_IPV4; 7054 7055 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); 7056 iparams.port = ntohs(laddr->sin_port); 7057 iparams.vlan = 0; 7058 7059 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", 7060 NIPQUAD((laddr->sin_addr.s_addr)), 7061 laddr->sin_port, iparams.port); 7062 7063 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); 7064 if (rc) { 7065 QL_DPRINT11(ha, 7066 "ecore_iwarp_create_listen failed rc = %d\n", rc); 7067 goto err; 7068 } 7069 7070 listener->ecore_handle = oparams.handle; 7071 cm_id->provider_data = listener; 7072 7073 QL_DPRINT12(ha, "exit\n"); 7074 return rc; 7075 7076 err: 7077 cm_id->rem_ref(cm_id); 7078 kfree(listener); 7079 7080 QL_DPRINT12(ha, "exit [%d]\n", rc); 7081 return rc; 7082 } 7083 7084 void 7085 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) 7086 { 7087 struct qlnxr_iw_listener *listener = cm_id->provider_data; 7088 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); 7089 int rc = 0; 7090 qlnx_host_t *ha; 7091 7092 ha = dev->ha; 7093 7094 QL_DPRINT12(ha, "enter\n"); 7095 7096 if (listener->ecore_handle) 7097 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, 7098 listener->ecore_handle); 7099 7100 cm_id->rem_ref(cm_id); 7101 7102 QL_DPRINT12(ha, "exit [%d]\n", rc); 7103 return; 7104 } 7105 7106 int 7107 qlnxr_iw_accept(struct iw_cm_id *cm_id, 7108 struct iw_cm_conn_param *conn_param) 7109 { 7110 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7111 struct qlnxr_dev *dev = ep->dev; 7112 struct qlnxr_qp *qp; 7113 struct ecore_iwarp_accept_in params; 7114 int rc; 7115 qlnx_host_t *ha; 7116 7117 ha = dev->ha; 7118 7119 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); 7120 7121 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7122 return -EINVAL; 7123 7124 qp = idr_find(&dev->qpidr, conn_param->qpn); 7125 if (!qp) { 7126 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", 7127 conn_param->qpn); 7128 return -EINVAL; 7129 } 7130 ep->qp = qp; 7131 qp->ep = ep; 7132 cm_id->add_ref(cm_id); 7133 ep->cm_id = cm_id; 7134 7135 params.ep_context = ep->ecore_context; 7136 params.cb_context = ep; 7137 params.qp = ep->qp->ecore_qp; 7138 params.private_data = conn_param->private_data; 7139 params.private_data_len = conn_param->private_data_len; 7140 params.ird = conn_param->ird; 7141 params.ord = conn_param->ord; 7142 7143 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); 7144 if (rc) { 7145 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); 7146 goto err; 7147 } 7148 7149 QL_DPRINT12(ha, "exit\n"); 7150 return 0; 7151 err: 7152 cm_id->rem_ref(cm_id); 7153 QL_DPRINT12(ha, "exit rc = %d\n", rc); 7154 return rc; 7155 } 7156 7157 int 7158 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 7159 { 7160 #if __FreeBSD_version >= 1102000 7161 7162 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7163 struct qlnxr_dev *dev = ep->dev; 7164 struct ecore_iwarp_reject_in params; 7165 int rc; 7166 7167 params.ep_context = ep->ecore_context; 7168 params.cb_context = ep; 7169 params.private_data = pdata; 7170 params.private_data_len = pdata_len; 7171 ep->qp = NULL; 7172 7173 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); 7174 7175 return rc; 7176 7177 #else 7178 7179 printf("iWARP reject_cr not implemented\n"); 7180 return -EINVAL; 7181 7182 #endif /* #if __FreeBSD_version >= 1102000 */ 7183 } 7184 7185 void 7186 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) 7187 { 7188 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7189 qlnx_host_t *ha; 7190 7191 ha = qp->dev->ha; 7192 7193 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); 7194 7195 atomic_inc(&qp->refcnt); 7196 7197 QL_DPRINT12(ha, "exit \n"); 7198 return; 7199 } 7200 7201 void 7202 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) 7203 { 7204 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7205 qlnx_host_t *ha; 7206 7207 ha = qp->dev->ha; 7208 7209 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); 7210 7211 if (atomic_dec_and_test(&qp->refcnt)) { 7212 qlnxr_idr_remove(qp->dev, qp->qp_id); 7213 kfree(qp); 7214 } 7215 7216 QL_DPRINT12(ha, "exit \n"); 7217 return; 7218 } 7219 7220 struct ib_qp * 7221 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) 7222 { 7223 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 7224 struct ib_qp *qp; 7225 qlnx_host_t *ha; 7226 7227 ha = dev->ha; 7228 7229 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); 7230 7231 qp = idr_find(&dev->qpidr, qpn); 7232 7233 QL_DPRINT12(ha, "exit qp = %p\n", qp); 7234 7235 return (qp); 7236 } 7237