1 /* 2 * Copyright (c) 2018-2019 Cavium, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 29 /* 30 * File: qlnxr_verbs.c 31 */ 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "qlnxr_def.h" 36 #include "rdma_common.h" 37 #include "qlnxr_roce.h" 38 #include "qlnxr_cm.h" 39 40 #define upper_32_bits(x) (uint32_t)(x >> 32) 41 #define lower_32_bits(x) (uint32_t)(x) 42 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 43 44 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ 45 do { \ 46 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ 47 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ 48 } while (0) 49 50 51 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 52 do { \ 53 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 54 (sge)->length = cpu_to_le32(vlength); \ 55 (sge)->flags = cpu_to_le32(vflags); \ 56 } while (0) 57 58 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 59 do { \ 60 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ 61 (hdr)->num_sges = num_sge; \ 62 } while (0) 63 64 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 65 do { \ 66 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 67 (sge)->length = cpu_to_le32(vlength); \ 68 (sge)->l_key = cpu_to_le32(vlkey); \ 69 } while (0) 70 71 #define NIPQUAD(addr) \ 72 ((unsigned char *)&addr)[0], \ 73 ((unsigned char *)&addr)[1], \ 74 ((unsigned char *)&addr)[2], \ 75 ((unsigned char *)&addr)[3] 76 77 static int 78 qlnxr_check_srq_params(struct ib_pd *ibpd, 79 struct qlnxr_dev *dev, 80 struct ib_srq_init_attr *attrs); 81 82 static int 83 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 84 struct qlnxr_srq *srq, 85 struct qlnxr_create_srq_ureq *ureq, 86 int access, int dmasync); 87 88 static int 89 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 90 struct qlnxr_dev *dev, 91 struct ib_srq_init_attr *init_attr); 92 93 94 static int 95 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 96 struct qlnxr_srq *srq, 97 struct ib_udata *udata); 98 99 static void 100 qlnxr_free_srq_user_params(struct qlnxr_srq *srq); 101 102 static void 103 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); 104 105 106 static u32 107 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); 108 109 int 110 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, 111 union ib_gid *sgid) 112 { 113 struct qlnxr_dev *dev; 114 qlnx_host_t *ha; 115 116 dev = get_qlnxr_dev(ibdev); 117 ha = dev->ha; 118 119 QL_DPRINT12(ha, "enter\n"); 120 121 memset(sgid->raw, 0, sizeof(sgid->raw)); 122 123 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); 124 125 QL_DPRINT12(ha, "exit\n"); 126 127 return 0; 128 } 129 130 int 131 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, 132 union ib_gid *sgid) 133 { 134 struct qlnxr_dev *dev; 135 qlnx_host_t *ha; 136 137 dev = get_qlnxr_dev(ibdev); 138 ha = dev->ha; 139 QL_DPRINT12(ha, "enter index: %d\n", index); 140 #if 0 141 int ret = 0; 142 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ 143 //if (!rdma_cap_roce_gid_table(ibdev, port)) { 144 if (!(rdma_protocol_roce(ibdev, port) && 145 ibdev->add_gid && ibdev->del_gid)) { 146 QL_DPRINT11(ha, "acquire gid failed\n"); 147 return -ENODEV; 148 } 149 150 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 151 if (ret == -EAGAIN) { 152 memcpy(sgid, &zgid, sizeof(*sgid)); 153 return 0; 154 } 155 #endif 156 if ((index >= QLNXR_MAX_SGID) || (index < 0)) { 157 QL_DPRINT12(ha, "invalid gid index %d\n", index); 158 memset(sgid, 0, sizeof(*sgid)); 159 return -EINVAL; 160 } 161 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); 162 163 QL_DPRINT12(ha, "exit : %p\n", sgid); 164 165 return 0; 166 } 167 168 struct ib_srq * 169 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr, 170 struct ib_udata *udata) 171 { 172 struct qlnxr_dev *dev; 173 qlnx_host_t *ha; 174 struct ecore_rdma_destroy_srq_in_params destroy_in_params; 175 struct ecore_rdma_create_srq_out_params out_params; 176 struct ecore_rdma_create_srq_in_params in_params; 177 u64 pbl_base_addr, phy_prod_pair_addr; 178 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 179 struct ib_ucontext *ib_ctx = NULL; 180 struct qlnxr_srq_hwq_info *hw_srq; 181 struct qlnxr_ucontext *ctx = NULL; 182 struct qlnxr_create_srq_ureq ureq; 183 u32 page_cnt, page_size; 184 struct qlnxr_srq *srq; 185 int ret = 0; 186 187 dev = get_qlnxr_dev((ibpd->device)); 188 ha = dev->ha; 189 190 QL_DPRINT12(ha, "enter\n"); 191 192 ret = qlnxr_check_srq_params(ibpd, dev, init_attr); 193 194 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 195 if (!srq) { 196 QL_DPRINT11(ha, "cannot allocate memory for srq\n"); 197 return NULL; //@@@ : TODO what to return here? 198 } 199 200 srq->dev = dev; 201 hw_srq = &srq->hw_srq; 202 spin_lock_init(&srq->lock); 203 memset(&in_params, 0, sizeof(in_params)); 204 205 if (udata && ibpd->uobject && ibpd->uobject->context) { 206 ib_ctx = ibpd->uobject->context; 207 ctx = get_qlnxr_ucontext(ib_ctx); 208 209 memset(&ureq, 0, sizeof(ureq)); 210 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 211 udata->inlen))) { 212 QL_DPRINT11(ha, "problem" 213 " copying data from user space\n"); 214 goto err0; 215 } 216 217 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0); 218 if (ret) 219 goto err0; 220 221 page_cnt = srq->usrq.pbl_info.num_pbes; 222 pbl_base_addr = srq->usrq.pbl_tbl->pa; 223 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 224 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT 225 // page_size = BIT(srq->usrq.umem->page_shift); 226 // else 227 page_size = srq->usrq.umem->page_size; 228 } else { 229 struct ecore_chain *pbl; 230 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); 231 if (ret) 232 goto err0; 233 pbl = &hw_srq->pbl; 234 235 page_cnt = ecore_chain_get_page_cnt(pbl); 236 pbl_base_addr = ecore_chain_get_pbl_phys(pbl); 237 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 238 page_size = pbl->elem_per_page << 4; 239 } 240 241 in_params.pd_id = pd->pd_id; 242 in_params.pbl_base_addr = pbl_base_addr; 243 in_params.prod_pair_addr = phy_prod_pair_addr; 244 in_params.num_pages = page_cnt; 245 in_params.page_size = page_size; 246 247 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 248 if (ret) 249 goto err1; 250 251 srq->srq_id = out_params.srq_id; 252 253 if (udata) { 254 ret = qlnxr_copy_srq_uresp(dev, srq, udata); 255 if (ret) 256 goto err2; 257 } 258 259 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); 260 return &srq->ibsrq; 261 err2: 262 memset(&in_params, 0, sizeof(in_params)); 263 destroy_in_params.srq_id = srq->srq_id; 264 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 265 266 err1: 267 if (udata) 268 qlnxr_free_srq_user_params(srq); 269 else 270 qlnxr_free_srq_kernel_params(srq); 271 272 err0: 273 kfree(srq); 274 return ERR_PTR(-EFAULT); 275 } 276 277 int 278 qlnxr_destroy_srq(struct ib_srq *ibsrq) 279 { 280 struct qlnxr_dev *dev; 281 struct qlnxr_srq *srq; 282 qlnx_host_t *ha; 283 struct ecore_rdma_destroy_srq_in_params in_params; 284 285 srq = get_qlnxr_srq(ibsrq); 286 dev = srq->dev; 287 ha = dev->ha; 288 289 memset(&in_params, 0, sizeof(in_params)); 290 in_params.srq_id = srq->srq_id; 291 292 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); 293 294 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) 295 qlnxr_free_srq_user_params(srq); 296 else 297 qlnxr_free_srq_kernel_params(srq); 298 299 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); 300 kfree(srq); 301 return 0; 302 } 303 304 int 305 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 306 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 307 { 308 struct qlnxr_dev *dev; 309 struct qlnxr_srq *srq; 310 qlnx_host_t *ha; 311 struct ecore_rdma_modify_srq_in_params in_params; 312 int ret = 0; 313 314 srq = get_qlnxr_srq(ibsrq); 315 dev = srq->dev; 316 ha = dev->ha; 317 318 QL_DPRINT12(ha, "enter\n"); 319 if (attr_mask & IB_SRQ_MAX_WR) { 320 QL_DPRINT12(ha, "invalid attribute mask=0x%x" 321 " specified for %p\n", attr_mask, srq); 322 return -EINVAL; 323 } 324 325 if (attr_mask & IB_SRQ_LIMIT) { 326 if (attr->srq_limit >= srq->hw_srq.max_wr) { 327 QL_DPRINT12(ha, "invalid srq_limit=0x%x" 328 " (max_srq_limit = 0x%x)\n", 329 attr->srq_limit, srq->hw_srq.max_wr); 330 return -EINVAL; 331 } 332 memset(&in_params, 0, sizeof(in_params)); 333 in_params.srq_id = srq->srq_id; 334 in_params.wqe_limit = attr->srq_limit; 335 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); 336 if (ret) 337 return ret; 338 } 339 340 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); 341 return 0; 342 } 343 344 int 345 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 346 { 347 struct qlnxr_dev *dev; 348 struct qlnxr_srq *srq; 349 qlnx_host_t *ha; 350 struct ecore_rdma_device *qattr; 351 srq = get_qlnxr_srq(ibsrq); 352 dev = srq->dev; 353 ha = dev->ha; 354 //qattr = &dev->attr; 355 qattr = ecore_rdma_query_device(dev->rdma_ctx); 356 QL_DPRINT12(ha, "enter\n"); 357 358 if (!dev->rdma_ctx) { 359 QL_DPRINT12(ha, "called with invalid params" 360 " rdma_ctx is NULL\n"); 361 return -EINVAL; 362 } 363 364 srq_attr->srq_limit = qattr->max_srq; 365 srq_attr->max_wr = qattr->max_srq_wr; 366 srq_attr->max_sge = qattr->max_sge; 367 368 QL_DPRINT12(ha, "exit\n"); 369 return 0; 370 } 371 372 /* Increment srq wr producer by one */ 373 static 374 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) 375 { 376 info->wr_prod_cnt++; 377 } 378 379 /* Increment srq wr consumer by one */ 380 static 381 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) 382 { 383 info->wr_cons_cnt++; 384 } 385 386 /* get_port_immutable verb is not available in FreeBSD */ 387 #if 0 388 int 389 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 390 struct ib_port_immutable *immutable) 391 { 392 struct qlnxr_dev *dev; 393 qlnx_host_t *ha; 394 dev = get_qlnxr_dev(ibdev); 395 ha = dev->ha; 396 397 QL_DPRINT12(ha, "entered but not implemented!!!\n"); 398 } 399 #endif 400 401 int 402 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 403 struct ib_recv_wr **bad_wr) 404 { 405 struct qlnxr_dev *dev; 406 struct qlnxr_srq *srq; 407 qlnx_host_t *ha; 408 struct qlnxr_srq_hwq_info *hw_srq; 409 struct ecore_chain *pbl; 410 unsigned long flags; 411 int status = 0; 412 u32 num_sge, offset; 413 414 srq = get_qlnxr_srq(ibsrq); 415 dev = srq->dev; 416 ha = dev->ha; 417 hw_srq = &srq->hw_srq; 418 419 QL_DPRINT12(ha, "enter\n"); 420 spin_lock_irqsave(&srq->lock, flags); 421 422 pbl = &srq->hw_srq.pbl; 423 while (wr) { 424 struct rdma_srq_wqe_header *hdr; 425 int i; 426 427 if (!qlnxr_srq_elem_left(hw_srq) || 428 wr->num_sge > srq->hw_srq.max_sges) { 429 QL_DPRINT11(ha, "WR cannot be posted" 430 " (%d, %d) || (%d > %d)\n", 431 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 432 wr->num_sge, srq->hw_srq.max_sges); 433 status = -ENOMEM; 434 *bad_wr = wr; 435 break; 436 } 437 438 hdr = ecore_chain_produce(pbl); 439 num_sge = wr->num_sge; 440 /* Set number of sge and WR id in header */ 441 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 442 443 /* PBL is maintained in case of WR granularity. 444 * So increment WR producer in case we post a WR. 445 */ 446 qlnxr_inc_srq_wr_prod(hw_srq); 447 hw_srq->wqe_prod++; 448 hw_srq->sge_prod++; 449 450 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", 451 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 452 453 for (i = 0; i < wr->num_sge; i++) { 454 struct rdma_srq_sge *srq_sge = 455 ecore_chain_produce(pbl); 456 /* Set SGE length, lkey and address */ 457 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 458 wr->sg_list[i].length, wr->sg_list[i].lkey); 459 460 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", 461 i, srq_sge->length, srq_sge->l_key, 462 srq_sge->addr.hi, srq_sge->addr.lo); 463 hw_srq->sge_prod++; 464 } 465 wmb(); 466 /* 467 * SRQ prod is 8 bytes. Need to update SGE prod in index 468 * in first 4 bytes and need to update WQE prod in next 469 * 4 bytes. 470 */ 471 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; 472 offset = offsetof(struct rdma_srq_producers, wqe_prod); 473 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 474 hw_srq->wqe_prod; 475 /* Flush prod after updating it */ 476 wmb(); 477 wr = wr->next; 478 } 479 480 QL_DPRINT12(ha, "Elements in SRQ: %d\n", 481 ecore_chain_get_elem_left(pbl)); 482 483 spin_unlock_irqrestore(&srq->lock, flags); 484 QL_DPRINT12(ha, "exit\n"); 485 return status; 486 } 487 488 int 489 #if __FreeBSD_version < 1102000 490 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr) 491 #else 492 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 493 struct ib_udata *udata) 494 #endif /* #if __FreeBSD_version < 1102000 */ 495 496 { 497 struct qlnxr_dev *dev; 498 struct ecore_rdma_device *qattr; 499 qlnx_host_t *ha; 500 501 dev = get_qlnxr_dev(ibdev); 502 ha = dev->ha; 503 504 QL_DPRINT12(ha, "enter\n"); 505 506 #if __FreeBSD_version > 1102000 507 if (udata->inlen || udata->outlen) 508 return -EINVAL; 509 #endif /* #if __FreeBSD_version > 1102000 */ 510 511 if (dev->rdma_ctx == NULL) { 512 return -EINVAL; 513 } 514 515 qattr = ecore_rdma_query_device(dev->rdma_ctx); 516 517 memset(attr, 0, sizeof *attr); 518 519 attr->fw_ver = qattr->fw_ver; 520 attr->sys_image_guid = qattr->sys_image_guid; 521 attr->max_mr_size = qattr->max_mr_size; 522 attr->page_size_cap = qattr->page_size_caps; 523 attr->vendor_id = qattr->vendor_id; 524 attr->vendor_part_id = qattr->vendor_part_id; 525 attr->hw_ver = qattr->hw_ver; 526 attr->max_qp = qattr->max_qp; 527 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 528 IB_DEVICE_RC_RNR_NAK_GEN | 529 IB_DEVICE_LOCAL_DMA_LKEY | 530 IB_DEVICE_MEM_MGT_EXTENSIONS; 531 532 attr->max_sge = qattr->max_sge; 533 attr->max_sge_rd = qattr->max_sge; 534 attr->max_cq = qattr->max_cq; 535 attr->max_cqe = qattr->max_cqe; 536 attr->max_mr = qattr->max_mr; 537 attr->max_mw = qattr->max_mw; 538 attr->max_pd = qattr->max_pd; 539 attr->atomic_cap = dev->atomic_cap; 540 attr->max_fmr = qattr->max_fmr; 541 attr->max_map_per_fmr = 16; /* TBD: FMR */ 542 543 /* There is an implicit assumption in some of the ib_xxx apps that the 544 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in 545 * communication the qp_rd_atom is passed to the other side and used as 546 * init_rd_atom without check device capabilities for init_rd_atom. 547 * for this reason, we set the qp_rd_atom to be the minimum between the 548 * two...There is an additional assumption in mlx4 driver that the 549 * values are power of two, fls is performed on the value - 1, which 550 * in fact gives a larger power of two for values which are not a power 551 * of two. This should be fixed in mlx4 driver, but until then -> 552 * we provide a value that is a power of two in our code. 553 */ 554 attr->max_qp_init_rd_atom = 555 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 556 attr->max_qp_rd_atom = 557 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 558 attr->max_qp_init_rd_atom); 559 560 attr->max_srq = qattr->max_srq; 561 attr->max_srq_sge = qattr->max_srq_sge; 562 attr->max_srq_wr = qattr->max_srq_wr; 563 564 /* TODO: R&D to more properly configure the following */ 565 attr->local_ca_ack_delay = qattr->dev_ack_delay; 566 attr->max_fast_reg_page_list_len = qattr->max_mr/8; 567 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; 568 attr->max_ah = qattr->max_ah; 569 570 QL_DPRINT12(ha, "exit\n"); 571 return 0; 572 } 573 574 static inline void 575 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) 576 { 577 switch (speed) { 578 case 1000: 579 *ib_speed = IB_SPEED_SDR; 580 *ib_width = IB_WIDTH_1X; 581 break; 582 case 10000: 583 *ib_speed = IB_SPEED_QDR; 584 *ib_width = IB_WIDTH_1X; 585 break; 586 587 case 20000: 588 *ib_speed = IB_SPEED_DDR; 589 *ib_width = IB_WIDTH_4X; 590 break; 591 592 case 25000: 593 *ib_speed = IB_SPEED_EDR; 594 *ib_width = IB_WIDTH_1X; 595 break; 596 597 case 40000: 598 *ib_speed = IB_SPEED_QDR; 599 *ib_width = IB_WIDTH_4X; 600 break; 601 602 case 50000: 603 *ib_speed = IB_SPEED_QDR; 604 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... 605 break; 606 607 case 100000: 608 *ib_speed = IB_SPEED_EDR; 609 *ib_width = IB_WIDTH_4X; 610 break; 611 612 default: 613 /* Unsupported */ 614 *ib_speed = IB_SPEED_SDR; 615 *ib_width = IB_WIDTH_1X; 616 } 617 return; 618 } 619 620 int 621 qlnxr_query_port(struct ib_device *ibdev, uint8_t port, 622 struct ib_port_attr *attr) 623 { 624 struct qlnxr_dev *dev; 625 struct ecore_rdma_port *rdma_port; 626 qlnx_host_t *ha; 627 628 dev = get_qlnxr_dev(ibdev); 629 ha = dev->ha; 630 631 QL_DPRINT12(ha, "enter\n"); 632 633 if (port > 1) { 634 QL_DPRINT12(ha, "port [%d] > 1 \n", port); 635 return -EINVAL; 636 } 637 638 if (dev->rdma_ctx == NULL) { 639 QL_DPRINT12(ha, "rdma_ctx == NULL\n"); 640 return -EINVAL; 641 } 642 643 rdma_port = ecore_rdma_query_port(dev->rdma_ctx); 644 memset(attr, 0, sizeof *attr); 645 646 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { 647 attr->state = IB_PORT_ACTIVE; 648 attr->phys_state = 5; 649 } else { 650 attr->state = IB_PORT_DOWN; 651 attr->phys_state = 3; 652 } 653 654 attr->max_mtu = IB_MTU_4096; 655 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); 656 attr->lid = 0; 657 attr->lmc = 0; 658 attr->sm_lid = 0; 659 attr->sm_sl = 0; 660 attr->port_cap_flags = 0; 661 662 if (QLNX_IS_IWARP(dev)) { 663 attr->gid_tbl_len = 1; 664 attr->pkey_tbl_len = 1; 665 } else { 666 attr->gid_tbl_len = QLNXR_MAX_SGID; 667 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; 668 } 669 670 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 671 attr->qkey_viol_cntr = 0; 672 673 get_link_speed_and_width(rdma_port->link_speed, 674 &attr->active_speed, &attr->active_width); 675 676 attr->max_msg_sz = rdma_port->max_msg_size; 677 attr->max_vl_num = 4; /* TODO -> figure this one out... */ 678 679 QL_DPRINT12(ha, "state = %d phys_state = %d " 680 " link_speed = %d active_speed = %d active_width = %d" 681 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" 682 " max_msg_sz = 0x%x max_vl_num = 0x%x \n", 683 attr->state, attr->phys_state, 684 rdma_port->link_speed, attr->active_speed, 685 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, 686 attr->max_msg_sz, attr->max_vl_num); 687 688 QL_DPRINT12(ha, "exit\n"); 689 return 0; 690 } 691 692 int 693 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, 694 struct ib_port_modify *props) 695 { 696 struct qlnxr_dev *dev; 697 qlnx_host_t *ha; 698 699 dev = get_qlnxr_dev(ibdev); 700 ha = dev->ha; 701 702 QL_DPRINT12(ha, "enter\n"); 703 704 if (port > 1) { 705 QL_DPRINT12(ha, "port (%d) > 1\n", port); 706 return -EINVAL; 707 } 708 709 QL_DPRINT12(ha, "exit\n"); 710 return 0; 711 } 712 713 enum rdma_link_layer 714 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) 715 { 716 struct qlnxr_dev *dev; 717 qlnx_host_t *ha; 718 719 dev = get_qlnxr_dev(ibdev); 720 ha = dev->ha; 721 722 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); 723 724 return IB_LINK_LAYER_ETHERNET; 725 } 726 727 struct ib_pd * 728 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, 729 struct ib_udata *udata) 730 { 731 struct qlnxr_pd *pd = NULL; 732 u16 pd_id; 733 int rc; 734 struct qlnxr_dev *dev; 735 qlnx_host_t *ha; 736 737 dev = get_qlnxr_dev(ibdev); 738 ha = dev->ha; 739 740 QL_DPRINT12(ha, "ibdev = %p context = %p" 741 " udata = %p enter\n", ibdev, context, udata); 742 743 if (dev->rdma_ctx == NULL) { 744 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); 745 rc = -1; 746 goto err; 747 } 748 749 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 750 if (!pd) { 751 rc = -ENOMEM; 752 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n"); 753 goto err; 754 } 755 756 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); 757 if (rc) { 758 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); 759 goto err; 760 } 761 762 pd->pd_id = pd_id; 763 764 if (udata && context) { 765 766 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); 767 if (rc) { 768 QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); 769 ecore_rdma_free_pd(dev->rdma_ctx, pd_id); 770 goto err; 771 } 772 773 pd->uctx = get_qlnxr_ucontext(context); 774 pd->uctx->pd = pd; 775 } 776 777 atomic_add_rel_32(&dev->pd_count, 1); 778 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 779 pd, pd_id, dev->pd_count); 780 781 return &pd->ibpd; 782 783 err: 784 kfree(pd); 785 QL_DPRINT12(ha, "exit -1\n"); 786 return ERR_PTR(rc); 787 } 788 789 int 790 qlnxr_dealloc_pd(struct ib_pd *ibpd) 791 { 792 struct qlnxr_pd *pd; 793 struct qlnxr_dev *dev; 794 qlnx_host_t *ha; 795 796 pd = get_qlnxr_pd(ibpd); 797 dev = get_qlnxr_dev((ibpd->device)); 798 ha = dev->ha; 799 800 QL_DPRINT12(ha, "enter\n"); 801 802 if (pd == NULL) { 803 QL_DPRINT11(ha, "pd = NULL\n"); 804 } else { 805 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); 806 kfree(pd); 807 atomic_subtract_rel_32(&dev->pd_count, 1); 808 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 809 pd, pd->pd_id, dev->pd_count); 810 } 811 812 QL_DPRINT12(ha, "exit\n"); 813 return 0; 814 } 815 816 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) 817 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ 818 /* Should be part of HSI */ 819 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ 820 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 821 822 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); 823 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); 824 825 int 826 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 827 { 828 struct qlnxr_dev *dev; 829 qlnx_host_t *ha; 830 831 dev = get_qlnxr_dev(ibdev); 832 ha = dev->ha; 833 834 QL_DPRINT12(ha, "enter index = 0x%x\n", index); 835 836 if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 837 return -EINVAL; 838 839 *pkey = QLNXR_ROCE_PKEY_DEFAULT; 840 841 QL_DPRINT12(ha, "exit\n"); 842 return 0; 843 } 844 845 846 static inline bool 847 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, 848 u16 *vlan_id) 849 { 850 bool ret = false; 851 852 QL_DPRINT12(ha, "enter \n"); 853 854 *vlan_id = 0; 855 856 #if __FreeBSD_version >= 1100000 857 u16 tmp_vlan_id; 858 859 #if __FreeBSD_version >= 1102000 860 union ib_gid *dgid; 861 862 dgid = &attr->ah_attr.grh.dgid; 863 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; 864 865 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { 866 *vlan_id = tmp_vlan_id; 867 ret = true; 868 } 869 #else 870 tmp_vlan_id = attr->vlan_id; 871 872 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) { 873 *vlan_id = tmp_vlan_id; 874 ret = true; 875 } 876 877 #endif /* #if __FreeBSD_version > 1102000 */ 878 879 #else 880 ret = true; 881 882 #endif /* #if __FreeBSD_version >= 1100000 */ 883 884 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); 885 886 return (ret); 887 } 888 889 static inline void 890 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, 891 int attr_mask, 892 struct qlnxr_dev *dev, 893 struct qlnxr_qp *qp, 894 struct ecore_rdma_modify_qp_in_params *qp_params) 895 { 896 int i; 897 qlnx_host_t *ha; 898 899 ha = dev->ha; 900 901 QL_DPRINT12(ha, "enter\n"); 902 903 memcpy(&qp_params->sgid.bytes[0], 904 &dev->sgid_tbl[qp->sgid_idx].raw[0], 905 sizeof(qp_params->sgid.bytes)); 906 memcpy(&qp_params->dgid.bytes[0], 907 &attr->ah_attr.grh.dgid.raw[0], 908 sizeof(qp_params->dgid)); 909 910 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); 911 912 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { 913 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 914 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 915 } 916 917 QL_DPRINT12(ha, "exit\n"); 918 return; 919 } 920 921 922 923 static int 924 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 925 { 926 struct qlnxr_mm *mm; 927 qlnx_host_t *ha; 928 929 ha = uctx->dev->ha; 930 931 QL_DPRINT12(ha, "enter\n"); 932 933 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 934 if (mm == NULL) { 935 QL_DPRINT11(ha, "mm = NULL\n"); 936 return -ENOMEM; 937 } 938 939 mm->key.phy_addr = phy_addr; 940 941 /* This function might be called with a length which is not a multiple 942 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 943 * forces this granularity by increasing the requested size if needed. 944 * When qedr_mmap is called, it will search the list with the updated 945 * length as a key. To prevent search failures, the length is rounded up 946 * in advance to PAGE_SIZE. 947 */ 948 mm->key.len = roundup(len, PAGE_SIZE); 949 INIT_LIST_HEAD(&mm->entry); 950 951 mutex_lock(&uctx->mm_list_lock); 952 list_add(&mm->entry, &uctx->mm_head); 953 mutex_unlock(&uctx->mm_list_lock); 954 955 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 956 (unsigned long long)mm->key.phy_addr, 957 (unsigned long)mm->key.len, uctx); 958 959 return 0; 960 } 961 962 static bool 963 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 964 { 965 bool found = false; 966 struct qlnxr_mm *mm; 967 qlnx_host_t *ha; 968 969 ha = uctx->dev->ha; 970 971 QL_DPRINT12(ha, "enter\n"); 972 973 mutex_lock(&uctx->mm_list_lock); 974 list_for_each_entry(mm, &uctx->mm_head, entry) { 975 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 976 continue; 977 978 found = true; 979 break; 980 } 981 mutex_unlock(&uctx->mm_list_lock); 982 983 QL_DPRINT12(ha, 984 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", 985 mm->key.phy_addr, mm->key.len, uctx, found); 986 987 return found; 988 } 989 990 struct 991 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev, 992 struct ib_udata *udata) 993 { 994 int rc; 995 struct qlnxr_ucontext *ctx; 996 struct qlnxr_alloc_ucontext_resp uresp; 997 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 998 qlnx_host_t *ha = dev->ha; 999 struct ecore_rdma_add_user_out_params oparams; 1000 1001 if (!udata) { 1002 return ERR_PTR(-EFAULT); 1003 } 1004 1005 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 1006 if (!ctx) 1007 return ERR_PTR(-ENOMEM); 1008 1009 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); 1010 if (rc) { 1011 QL_DPRINT12(ha, 1012 "Failed to allocate a DPI for a new RoCE application " 1013 ",rc = %d. To overcome this, consider to increase " 1014 "the number of DPIs, increase the doorbell BAR size " 1015 "or just close unnecessary RoCE applications. In " 1016 "order to increase the number of DPIs consult the " 1017 "README\n", rc); 1018 goto err; 1019 } 1020 1021 ctx->dpi = oparams.dpi; 1022 ctx->dpi_addr = oparams.dpi_addr; 1023 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 1024 ctx->dpi_size = oparams.dpi_size; 1025 INIT_LIST_HEAD(&ctx->mm_head); 1026 mutex_init(&ctx->mm_list_lock); 1027 1028 memset(&uresp, 0, sizeof(uresp)); 1029 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) 1030 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out 1031 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) 1032 < udata->outlen ? 1 : 0; //TODO: figure this out 1033 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) 1034 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 1035 uresp.db_pa = ctx->dpi_phys_addr; 1036 uresp.db_size = ctx->dpi_size; 1037 uresp.max_send_wr = dev->attr.max_sqe; 1038 uresp.max_recv_wr = dev->attr.max_rqe; 1039 uresp.max_srq_wr = dev->attr.max_srq_wr; 1040 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 1041 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 1042 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 1043 uresp.max_cqes = QLNXR_MAX_CQES; 1044 1045 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1046 if (rc) 1047 goto err; 1048 1049 ctx->dev = dev; 1050 1051 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 1052 if (rc) 1053 goto err; 1054 QL_DPRINT12(ha, "Allocated user context %p\n", 1055 &ctx->ibucontext); 1056 1057 return &ctx->ibucontext; 1058 err: 1059 kfree(ctx); 1060 return ERR_PTR(rc); 1061 } 1062 1063 int 1064 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) 1065 { 1066 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); 1067 struct qlnxr_dev *dev = uctx->dev; 1068 qlnx_host_t *ha = dev->ha; 1069 struct qlnxr_mm *mm, *tmp; 1070 int status = 0; 1071 1072 QL_DPRINT12(ha, "Deallocating user context %p\n", 1073 uctx); 1074 1075 if (dev) { 1076 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 1077 } 1078 1079 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 1080 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" 1081 " ctx=%p\n", 1082 mm->key.phy_addr, mm->key.len, uctx); 1083 list_del(&mm->entry); 1084 kfree(mm); 1085 } 1086 kfree(uctx); 1087 return status; 1088 } 1089 1090 int 1091 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 1092 { 1093 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); 1094 struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); 1095 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 1096 u64 unmapped_db; 1097 unsigned long len = (vma->vm_end - vma->vm_start); 1098 int rc = 0; 1099 bool found; 1100 qlnx_host_t *ha; 1101 1102 ha = dev->ha; 1103 1104 #if __FreeBSD_version > 1102000 1105 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 1106 #else 1107 unmapped_db = dev->db_phys_addr; 1108 #endif /* #if __FreeBSD_version > 1102000 */ 1109 1110 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" 1111 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 1112 vm_page, vma->vm_pgoff, unmapped_db, 1113 dev->db_size, len); 1114 1115 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 1116 QL_DPRINT11(ha, "Vma_start not page aligned " 1117 "vm_start = %ld vma_end = %ld\n", vma->vm_start, 1118 vma->vm_end); 1119 return -EINVAL; 1120 } 1121 1122 found = qlnxr_search_mmap(ucontext, vm_page, len); 1123 if (!found) { 1124 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", 1125 vma->vm_pgoff); 1126 return -EINVAL; 1127 } 1128 1129 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1130 1131 #if __FreeBSD_version > 1102000 1132 1133 if ((vm_page < unmapped_db) || 1134 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { 1135 QL_DPRINT11(ha, "failed pages are outside of dpi;" 1136 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", 1137 vm_page, unmapped_db, ucontext->dpi_size); 1138 return -EINVAL; 1139 } 1140 1141 if (vma->vm_flags & VM_READ) { 1142 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); 1143 return -EINVAL; 1144 } 1145 1146 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1147 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 1148 vma->vm_page_prot); 1149 1150 #else 1151 1152 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 1153 dev->db_size))) { 1154 1155 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1156 1157 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1158 1159 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1160 PAGE_SIZE, vma->vm_page_prot); 1161 } else { 1162 QL_DPRINT12(ha, "Mapping chains\n"); 1163 rc = io_remap_pfn_range(vma, vma->vm_start, 1164 vma->vm_pgoff, len, vma->vm_page_prot); 1165 } 1166 1167 #endif /* #if __FreeBSD_version > 1102000 */ 1168 1169 QL_DPRINT12(ha, "exit [%d]\n", rc); 1170 return rc; 1171 } 1172 1173 struct ib_mr * 1174 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) 1175 { 1176 struct qlnxr_mr *mr; 1177 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1178 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 1179 int rc; 1180 qlnx_host_t *ha; 1181 1182 ha = dev->ha; 1183 1184 QL_DPRINT12(ha, "enter\n"); 1185 1186 if (acc & IB_ACCESS_MW_BIND) { 1187 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); 1188 } 1189 1190 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1191 if (!mr) { 1192 rc = -ENOMEM; 1193 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); 1194 goto err0; 1195 } 1196 1197 mr->type = QLNXR_MR_DMA; 1198 1199 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1200 if (rc) { 1201 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); 1202 goto err1; 1203 } 1204 1205 /* index only, 18 bit long, lkey = itid << 8 | key */ 1206 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1207 mr->hw_mr.pd = pd->pd_id; 1208 mr->hw_mr.local_read = 1; 1209 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1210 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1211 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1212 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1213 mr->hw_mr.dma_mr = true; 1214 1215 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1216 if (rc) { 1217 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); 1218 goto err2; 1219 } 1220 1221 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1222 1223 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1224 mr->hw_mr.remote_atomic) { 1225 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1226 } 1227 1228 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); 1229 1230 return &mr->ibmr; 1231 1232 err2: 1233 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1234 err1: 1235 kfree(mr); 1236 err0: 1237 QL_DPRINT12(ha, "exit [%d]\n", rc); 1238 1239 return ERR_PTR(rc); 1240 } 1241 1242 static void 1243 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, 1244 struct qlnxr_pbl *pbl) 1245 { 1246 int i; 1247 qlnx_host_t *ha; 1248 1249 ha = dev->ha; 1250 1251 QL_DPRINT12(ha, "enter\n"); 1252 1253 for (i = 0; i < pbl_info->num_pbls; i++) { 1254 if (!pbl[i].va) 1255 continue; 1256 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, 1257 pbl_info->pbl_size); 1258 } 1259 kfree(pbl); 1260 1261 QL_DPRINT12(ha, "exit\n"); 1262 return; 1263 } 1264 1265 #define MIN_FW_PBL_PAGE_SIZE (4*1024) 1266 #define MAX_FW_PBL_PAGE_SIZE (64*1024) 1267 1268 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 1269 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 1270 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) 1271 1272 static struct qlnxr_pbl * 1273 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, 1274 struct qlnxr_pbl_info *pbl_info, gfp_t flags) 1275 { 1276 void *va; 1277 dma_addr_t pa; 1278 dma_addr_t *pbl_main_tbl; 1279 struct qlnxr_pbl *pbl_table; 1280 int i, rc = 0; 1281 qlnx_host_t *ha; 1282 1283 ha = dev->ha; 1284 1285 QL_DPRINT12(ha, "enter\n"); 1286 1287 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); 1288 1289 if (!pbl_table) { 1290 QL_DPRINT12(ha, "pbl_table = NULL\n"); 1291 return NULL; 1292 } 1293 1294 for (i = 0; i < pbl_info->num_pbls; i++) { 1295 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); 1296 if (!va) { 1297 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); 1298 rc = -ENOMEM; 1299 goto err; 1300 } 1301 memset(va, 0, pbl_info->pbl_size); 1302 pbl_table[i].va = va; 1303 pbl_table[i].pa = pa; 1304 } 1305 1306 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 1307 * the first one with physical pointers to all of the rest 1308 */ 1309 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 1310 for (i = 0; i < pbl_info->num_pbls - 1; i++) 1311 pbl_main_tbl[i] = pbl_table[i + 1].pa; 1312 1313 QL_DPRINT12(ha, "exit\n"); 1314 return pbl_table; 1315 1316 err: 1317 qlnxr_free_pbl(dev, pbl_info, pbl_table); 1318 1319 QL_DPRINT12(ha, "exit with error\n"); 1320 return NULL; 1321 } 1322 1323 static int 1324 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, 1325 struct qlnxr_pbl_info *pbl_info, 1326 u32 num_pbes, 1327 int two_layer_capable) 1328 { 1329 u32 pbl_capacity; 1330 u32 pbl_size; 1331 u32 num_pbls; 1332 qlnx_host_t *ha; 1333 1334 ha = dev->ha; 1335 1336 QL_DPRINT12(ha, "enter\n"); 1337 1338 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 1339 if (num_pbes > MAX_PBES_TWO_LAYER) { 1340 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", 1341 num_pbes); 1342 return -EINVAL; 1343 } 1344 1345 /* calculate required pbl page size */ 1346 pbl_size = MIN_FW_PBL_PAGE_SIZE; 1347 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 1348 NUM_PBES_ON_PAGE(pbl_size); 1349 1350 while (pbl_capacity < num_pbes) { 1351 pbl_size *= 2; 1352 pbl_capacity = pbl_size / sizeof(u64); 1353 pbl_capacity = pbl_capacity * pbl_capacity; 1354 } 1355 1356 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 1357 num_pbls++; /* One for the layer0 ( points to the pbls) */ 1358 pbl_info->two_layered = true; 1359 } else { 1360 /* One layered PBL */ 1361 num_pbls = 1; 1362 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ 1363 roundup_pow_of_two((num_pbes * sizeof(u64)))); 1364 pbl_info->two_layered = false; 1365 } 1366 1367 pbl_info->num_pbls = num_pbls; 1368 pbl_info->pbl_size = pbl_size; 1369 pbl_info->num_pbes = num_pbes; 1370 1371 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", 1372 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 1373 1374 return 0; 1375 } 1376 1377 #define upper_32_bits(x) (uint32_t)(x >> 32) 1378 #define lower_32_bits(x) (uint32_t)(x) 1379 1380 static void 1381 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, 1382 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 1383 { 1384 struct regpair *pbe; 1385 struct qlnxr_pbl *pbl_tbl; 1386 struct scatterlist *sg; 1387 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 1388 qlnx_host_t *ha; 1389 1390 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1391 int i; 1392 struct ib_umem_chunk *chunk = NULL; 1393 #else 1394 int entry; 1395 #endif 1396 1397 1398 ha = dev->ha; 1399 1400 QL_DPRINT12(ha, "enter\n"); 1401 1402 if (!pbl_info) { 1403 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 1404 return; 1405 } 1406 1407 if (!pbl_info->num_pbes) { 1408 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 1409 return; 1410 } 1411 1412 /* If we have a two layered pbl, the first pbl points to the rest 1413 * of the pbls and the first entry lays on the second pbl in the table 1414 */ 1415 if (pbl_info->two_layered) 1416 pbl_tbl = &pbl[1]; 1417 else 1418 pbl_tbl = pbl; 1419 1420 pbe = (struct regpair *)pbl_tbl->va; 1421 if (!pbe) { 1422 QL_DPRINT12(ha, "pbe is NULL\n"); 1423 return; 1424 } 1425 1426 pbe_cnt = 0; 1427 1428 shift = ilog2(umem->page_size); 1429 1430 #ifndef DEFINE_IB_UMEM_WITH_CHUNK 1431 1432 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 1433 1434 #else 1435 list_for_each_entry(chunk, &umem->chunk_list, list) { 1436 /* get all the dma regions from the chunk. */ 1437 for (i = 0; i < chunk->nmap; i++) { 1438 sg = &chunk->page_list[i]; 1439 #endif 1440 pages = sg_dma_len(sg) >> shift; 1441 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 1442 /* store the page address in pbe */ 1443 pbe->lo = 1444 cpu_to_le32(sg_dma_address(sg) + 1445 (umem->page_size * pg_cnt)); 1446 pbe->hi = 1447 cpu_to_le32(upper_32_bits 1448 ((sg_dma_address(sg) + 1449 umem->page_size * pg_cnt))); 1450 1451 QL_DPRINT12(ha, 1452 "Populate pbl table:" 1453 " pbe->addr=0x%x:0x%x " 1454 " pbe_cnt = %d total_num_pbes=%d" 1455 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 1456 total_num_pbes, pbe); 1457 1458 pbe_cnt ++; 1459 total_num_pbes ++; 1460 pbe++; 1461 1462 if (total_num_pbes == pbl_info->num_pbes) 1463 return; 1464 1465 /* if the given pbl is full storing the pbes, 1466 * move to next pbl. 1467 */ 1468 if (pbe_cnt == 1469 (pbl_info->pbl_size / sizeof(u64))) { 1470 pbl_tbl++; 1471 pbe = (struct regpair *)pbl_tbl->va; 1472 pbe_cnt = 0; 1473 } 1474 } 1475 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1476 } 1477 #endif 1478 } 1479 QL_DPRINT12(ha, "exit\n"); 1480 return; 1481 } 1482 1483 static void 1484 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) 1485 { 1486 struct qlnxr_pbl *pbl, *tmp; 1487 qlnx_host_t *ha; 1488 1489 ha = dev->ha; 1490 1491 QL_DPRINT12(ha, "enter\n"); 1492 1493 if (info->pbl_table) 1494 list_add_tail(&info->pbl_table->list_entry, 1495 &info->free_pbl_list); 1496 1497 if (!list_empty(&info->inuse_pbl_list)) 1498 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 1499 1500 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 1501 list_del(&pbl->list_entry); 1502 qlnxr_free_pbl(dev, &info->pbl_info, pbl); 1503 } 1504 QL_DPRINT12(ha, "exit\n"); 1505 1506 return; 1507 } 1508 1509 static int 1510 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, 1511 size_t page_list_len, bool two_layered) 1512 { 1513 int rc; 1514 struct qlnxr_pbl *tmp; 1515 qlnx_host_t *ha; 1516 1517 ha = dev->ha; 1518 1519 QL_DPRINT12(ha, "enter\n"); 1520 1521 INIT_LIST_HEAD(&info->free_pbl_list); 1522 INIT_LIST_HEAD(&info->inuse_pbl_list); 1523 1524 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, 1525 page_list_len, two_layered); 1526 if (rc) { 1527 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); 1528 goto done; 1529 } 1530 1531 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1532 1533 if (!info->pbl_table) { 1534 rc = -ENOMEM; 1535 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); 1536 goto done; 1537 } 1538 1539 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); 1540 1541 /* in usual case we use 2 PBLs, so we add one to free 1542 * list and allocating another one 1543 */ 1544 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1545 1546 if (!tmp) { 1547 QL_DPRINT11(ha, "Extra PBL is not allocated\n"); 1548 goto done; /* it's OK if second allocation fails, so rc = 0*/ 1549 } 1550 1551 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 1552 1553 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); 1554 1555 done: 1556 if (rc) 1557 free_mr_info(dev, info); 1558 1559 QL_DPRINT12(ha, "exit [%d]\n", rc); 1560 1561 return rc; 1562 } 1563 1564 1565 struct ib_mr * 1566 #if __FreeBSD_version >= 1102000 1567 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1568 u64 usr_addr, int acc, struct ib_udata *udata) 1569 #else 1570 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1571 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id) 1572 #endif /* #if __FreeBSD_version >= 1102000 */ 1573 { 1574 int rc = -ENOMEM; 1575 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1576 struct qlnxr_mr *mr; 1577 struct qlnxr_pd *pd; 1578 qlnx_host_t *ha; 1579 1580 ha = dev->ha; 1581 1582 QL_DPRINT12(ha, "enter\n"); 1583 1584 pd = get_qlnxr_pd(ibpd); 1585 1586 QL_DPRINT12(ha, "qedr_register user mr pd = %d" 1587 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 1588 pd->pd_id, start, len, usr_addr, acc); 1589 1590 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 1591 QL_DPRINT11(ha, 1592 "(acc & IB_ACCESS_REMOTE_WRITE &&" 1593 " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 1594 return ERR_PTR(-EINVAL); 1595 } 1596 1597 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1598 if (!mr) { 1599 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 1600 return ERR_PTR(rc); 1601 } 1602 1603 mr->type = QLNXR_MR_USER; 1604 1605 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 1606 if (IS_ERR(mr->umem)) { 1607 rc = -EFAULT; 1608 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); 1609 goto err0; 1610 } 1611 1612 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 1613 if (rc) { 1614 QL_DPRINT11(ha, 1615 "qlnxr_init_mr_info failed [%d]\n", rc); 1616 goto err1; 1617 } 1618 1619 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 1620 &mr->info.pbl_info); 1621 1622 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1623 1624 if (rc) { 1625 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 1626 goto err1; 1627 } 1628 1629 /* index only, 18 bit long, lkey = itid << 8 | key */ 1630 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1631 mr->hw_mr.key = 0; 1632 mr->hw_mr.pd = pd->pd_id; 1633 mr->hw_mr.local_read = 1; 1634 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1635 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1636 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1637 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1638 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 1639 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 1640 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 1641 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 1642 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ 1643 1644 #if __FreeBSD_version >= 1102000 1645 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 1646 #else 1647 mr->hw_mr.fbo = mr->umem->offset; 1648 #endif 1649 mr->hw_mr.length = len; 1650 mr->hw_mr.vaddr = usr_addr; 1651 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 1652 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 1653 mr->hw_mr.dma_mr = false; 1654 1655 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1656 if (rc) { 1657 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 1658 goto err2; 1659 } 1660 1661 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1662 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1663 mr->hw_mr.remote_atomic) 1664 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1665 1666 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); 1667 1668 return (&mr->ibmr); 1669 1670 err2: 1671 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1672 err1: 1673 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1674 err0: 1675 kfree(mr); 1676 1677 QL_DPRINT12(ha, "exit [%d]\n", rc); 1678 return (ERR_PTR(rc)); 1679 } 1680 1681 int 1682 qlnxr_dereg_mr(struct ib_mr *ib_mr) 1683 { 1684 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); 1685 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); 1686 int rc = 0; 1687 qlnx_host_t *ha; 1688 1689 ha = dev->ha; 1690 1691 QL_DPRINT12(ha, "enter\n"); 1692 1693 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) 1694 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1695 1696 /* it could be user registered memory. */ 1697 if (mr->umem) 1698 ib_umem_release(mr->umem); 1699 1700 kfree(mr->pages); 1701 1702 kfree(mr); 1703 1704 QL_DPRINT12(ha, "exit\n"); 1705 return rc; 1706 } 1707 1708 static int 1709 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, 1710 struct qlnxr_cq *cq, struct ib_udata *udata) 1711 { 1712 struct qlnxr_create_cq_uresp uresp; 1713 int rc; 1714 qlnx_host_t *ha; 1715 1716 ha = dev->ha; 1717 1718 QL_DPRINT12(ha, "enter\n"); 1719 1720 memset(&uresp, 0, sizeof(uresp)); 1721 1722 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1723 uresp.icid = cq->icid; 1724 1725 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1726 1727 if (rc) { 1728 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", 1729 cq->icid, rc); 1730 } 1731 1732 QL_DPRINT12(ha, "exit [%d]\n", rc); 1733 return rc; 1734 } 1735 1736 static void 1737 consume_cqe(struct qlnxr_cq *cq) 1738 { 1739 1740 if (cq->latest_cqe == cq->toggle_cqe) 1741 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1742 1743 cq->latest_cqe = ecore_chain_consume(&cq->pbl); 1744 } 1745 1746 static inline int 1747 qlnxr_align_cq_entries(int entries) 1748 { 1749 u64 size, aligned_size; 1750 1751 /* We allocate an extra entry that we don't report to the FW. 1752 * Why? 1753 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes 1754 * (for performance purposes). Allocating an extra entry and telling 1755 * the FW we have less prevents overwriting the first entry in case of 1756 * a wrap i.e. when the FW writes the last entry and the application 1757 * hasn't read the first one. 1758 */ 1759 size = (entries + 1) * QLNXR_CQE_SIZE; 1760 1761 /* We align to PAGE_SIZE. 1762 * Why? 1763 * Since the CQ is going to be mapped and the mapping is anyhow in whole 1764 * kernel pages we benefit from the possibly extra CQEs. 1765 */ 1766 aligned_size = ALIGN(size, PAGE_SIZE); 1767 1768 /* note: for CQs created in user space the result of this function 1769 * should match the size mapped in user space 1770 */ 1771 return (aligned_size / QLNXR_CQE_SIZE); 1772 } 1773 1774 static inline int 1775 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, 1776 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, 1777 int access, int dmasync, int alloc_and_init) 1778 { 1779 int page_cnt; 1780 int rc; 1781 qlnx_host_t *ha; 1782 1783 ha = dev->ha; 1784 1785 QL_DPRINT12(ha, "enter\n"); 1786 1787 q->buf_addr = buf_addr; 1788 q->buf_len = buf_len; 1789 1790 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" 1791 " dmasync : %x\n", q->buf_addr, q->buf_len, 1792 access, dmasync); 1793 1794 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 1795 1796 if (IS_ERR(q->umem)) { 1797 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); 1798 return PTR_ERR(q->umem); 1799 } 1800 1801 page_cnt = ib_umem_page_count(q->umem); 1802 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 1803 0 /* SQ and RQ don't support dual layer pbl. 1804 * CQ may, but this is yet uncoded. 1805 */); 1806 if (rc) { 1807 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); 1808 goto err; 1809 } 1810 1811 if (alloc_and_init) { 1812 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 1813 1814 if (!q->pbl_tbl) { 1815 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1816 rc = -ENOMEM; 1817 goto err; 1818 } 1819 1820 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); 1821 } else { 1822 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 1823 1824 if (!q->pbl_tbl) { 1825 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1826 rc = -ENOMEM; 1827 goto err; 1828 } 1829 } 1830 1831 QL_DPRINT12(ha, "exit\n"); 1832 return 0; 1833 1834 err: 1835 ib_umem_release(q->umem); 1836 q->umem = NULL; 1837 1838 QL_DPRINT12(ha, "exit [%d]\n", rc); 1839 return rc; 1840 } 1841 1842 #if __FreeBSD_version >= 1102000 1843 1844 struct ib_cq * 1845 qlnxr_create_cq(struct ib_device *ibdev, 1846 const struct ib_cq_init_attr *attr, 1847 struct ib_ucontext *ib_ctx, 1848 struct ib_udata *udata) 1849 1850 #else 1851 1852 #if __FreeBSD_version >= 1100000 1853 1854 struct ib_cq * 1855 qlnxr_create_cq(struct ib_device *ibdev, 1856 struct ib_cq_init_attr *attr, 1857 struct ib_ucontext *ib_ctx, 1858 struct ib_udata *udata) 1859 1860 #else 1861 1862 struct ib_cq * 1863 qlnxr_create_cq(struct ib_device *ibdev, 1864 int entries, 1865 int vector, 1866 struct ib_ucontext *ib_ctx, 1867 struct ib_udata *udata) 1868 #endif /* #if __FreeBSD_version >= 1100000 */ 1869 1870 #endif /* #if __FreeBSD_version >= 1102000 */ 1871 { 1872 struct qlnxr_ucontext *ctx; 1873 struct ecore_rdma_destroy_cq_out_params destroy_oparams; 1874 struct ecore_rdma_destroy_cq_in_params destroy_iparams; 1875 struct qlnxr_dev *dev; 1876 struct ecore_rdma_create_cq_in_params params; 1877 struct qlnxr_create_cq_ureq ureq; 1878 1879 #if __FreeBSD_version >= 1100000 1880 int vector = attr->comp_vector; 1881 int entries = attr->cqe; 1882 #endif 1883 struct qlnxr_cq *cq; 1884 int chain_entries, rc, page_cnt; 1885 u64 pbl_ptr; 1886 u16 icid; 1887 qlnx_host_t *ha; 1888 1889 dev = get_qlnxr_dev(ibdev); 1890 ha = dev->ha; 1891 1892 QL_DPRINT12(ha, "called from %s. entries = %d, " 1893 "vector = %d\n", 1894 (udata ? "User Lib" : "Kernel"), entries, vector); 1895 1896 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); 1897 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); 1898 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); 1899 1900 if (entries > QLNXR_MAX_CQES) { 1901 QL_DPRINT11(ha, 1902 "the number of entries %d is too high. " 1903 "Must be equal or below %d.\n", 1904 entries, QLNXR_MAX_CQES); 1905 return ERR_PTR(-EINVAL); 1906 } 1907 chain_entries = qlnxr_align_cq_entries(entries); 1908 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); 1909 1910 cq = qlnx_zalloc((sizeof(struct qlnxr_cq))); 1911 1912 if (!cq) 1913 return ERR_PTR(-ENOMEM); 1914 1915 if (udata) { 1916 memset(&ureq, 0, sizeof(ureq)); 1917 1918 if (ib_copy_from_udata(&ureq, udata, 1919 min(sizeof(ureq), udata->inlen))) { 1920 QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); 1921 goto err0; 1922 } 1923 1924 if (!ureq.len) { 1925 QL_DPRINT11(ha, "ureq.len == 0\n"); 1926 goto err0; 1927 } 1928 1929 cq->cq_type = QLNXR_CQ_TYPE_USER; 1930 1931 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len, 1932 IB_ACCESS_LOCAL_WRITE, 1, 1); 1933 1934 pbl_ptr = cq->q.pbl_tbl->pa; 1935 page_cnt = cq->q.pbl_info.num_pbes; 1936 cq->ibcq.cqe = chain_entries; 1937 } else { 1938 cq->cq_type = QLNXR_CQ_TYPE_KERNEL; 1939 1940 rc = ecore_chain_alloc(&dev->ha->cdev, 1941 ECORE_CHAIN_USE_TO_CONSUME, 1942 ECORE_CHAIN_MODE_PBL, 1943 ECORE_CHAIN_CNT_TYPE_U32, 1944 chain_entries, 1945 sizeof(union roce_cqe), 1946 &cq->pbl, NULL); 1947 1948 if (rc) 1949 goto err1; 1950 1951 page_cnt = ecore_chain_get_page_cnt(&cq->pbl); 1952 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); 1953 cq->ibcq.cqe = cq->pbl.capacity; 1954 } 1955 1956 params.cq_handle_hi = upper_32_bits((uintptr_t)cq); 1957 params.cq_handle_lo = lower_32_bits((uintptr_t)cq); 1958 params.cnq_id = vector; 1959 params.cq_size = chain_entries - 1; 1960 params.pbl_num_pages = page_cnt; 1961 params.pbl_ptr = pbl_ptr; 1962 params.pbl_two_level = 0; 1963 1964 if (ib_ctx != NULL) { 1965 ctx = get_qlnxr_ucontext(ib_ctx); 1966 params.dpi = ctx->dpi; 1967 } else { 1968 params.dpi = dev->dpi; 1969 } 1970 1971 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 1972 if (rc) 1973 goto err2; 1974 1975 cq->icid = icid; 1976 cq->sig = QLNXR_CQ_MAGIC_NUMBER; 1977 spin_lock_init(&cq->cq_lock); 1978 1979 if (ib_ctx) { 1980 rc = qlnxr_copy_cq_uresp(dev, cq, udata); 1981 if (rc) 1982 goto err3; 1983 } else { 1984 /* Generate doorbell address. 1985 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. 1986 * TODO: consider moving to device scope as it is a function of 1987 * the device. 1988 * TODO: add ifdef if plan to support 16 bit. 1989 */ 1990 cq->db_addr = dev->db_addr + 1991 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1992 cq->db.data.icid = cq->icid; 1993 cq->db.data.params = DB_AGG_CMD_SET << 1994 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1995 1996 /* point to the very last element, passing it we will toggle */ 1997 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); 1998 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1999 2000 /* must be different from pbl_toggle */ 2001 cq->latest_cqe = NULL; 2002 consume_cqe(cq); 2003 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 2004 } 2005 2006 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," 2007 " number of entries = 0x%x\n", 2008 cq->icid, cq, params.cq_size); 2009 QL_DPRINT12(ha,"cq_addr = %p\n", cq); 2010 return &cq->ibcq; 2011 2012 err3: 2013 destroy_iparams.icid = cq->icid; 2014 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); 2015 err2: 2016 if (udata) 2017 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2018 else 2019 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2020 err1: 2021 if (udata) 2022 ib_umem_release(cq->q.umem); 2023 err0: 2024 kfree(cq); 2025 2026 QL_DPRINT12(ha, "exit error\n"); 2027 2028 return ERR_PTR(-EINVAL); 2029 } 2030 2031 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 2032 { 2033 int status = 0; 2034 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2035 qlnx_host_t *ha; 2036 2037 ha = dev->ha; 2038 2039 QL_DPRINT12(ha, "enter/exit\n"); 2040 2041 return status; 2042 } 2043 2044 int 2045 qlnxr_destroy_cq(struct ib_cq *ibcq) 2046 { 2047 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2048 struct ecore_rdma_destroy_cq_out_params oparams; 2049 struct ecore_rdma_destroy_cq_in_params iparams; 2050 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 2051 int rc = 0; 2052 qlnx_host_t *ha; 2053 2054 ha = dev->ha; 2055 2056 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); 2057 2058 cq->destroyed = 1; 2059 2060 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation 2061 * that all completions with notification are dealt with. The rest 2062 * of the completions are not interesting 2063 */ 2064 2065 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 2066 2067 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { 2068 2069 iparams.icid = cq->icid; 2070 2071 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 2072 2073 if (rc) { 2074 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", 2075 cq->icid); 2076 return rc; 2077 } 2078 2079 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); 2080 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2081 } 2082 2083 if (ibcq->uobject && ibcq->uobject->context) { 2084 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2085 ib_umem_release(cq->q.umem); 2086 } 2087 2088 cq->sig = ~cq->sig; 2089 2090 kfree(cq); 2091 2092 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); 2093 2094 return rc; 2095 } 2096 2097 static int 2098 qlnxr_check_qp_attrs(struct ib_pd *ibpd, 2099 struct qlnxr_dev *dev, 2100 struct ib_qp_init_attr *attrs, 2101 struct ib_udata *udata) 2102 { 2103 struct ecore_rdma_device *qattr; 2104 qlnx_host_t *ha; 2105 2106 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2107 ha = dev->ha; 2108 2109 QL_DPRINT12(ha, "enter\n"); 2110 2111 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); 2112 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); 2113 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); 2114 2115 #if __FreeBSD_version < 1102000 2116 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type); 2117 #endif 2118 2119 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); 2120 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); 2121 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); 2122 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); 2123 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); 2124 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", 2125 attrs->cap.max_inline_data); 2126 2127 #if __FreeBSD_version < 1102000 2128 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n", 2129 attrs->cap.qpg_tss_mask_sz); 2130 #endif 2131 2132 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); 2133 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); 2134 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); 2135 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); 2136 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); 2137 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", 2138 (void *)qattr->sys_image_guid); 2139 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); 2140 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); 2141 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); 2142 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); 2143 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); 2144 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); 2145 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", 2146 qattr->max_qp_resp_rd_atomic_resc); 2147 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", 2148 qattr->max_qp_req_rd_atomic_resc); 2149 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", 2150 qattr->max_dev_resp_rd_atomic_resc); 2151 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); 2152 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); 2153 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); 2154 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); 2155 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); 2156 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); 2157 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); 2158 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); 2159 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", 2160 qattr->max_mr_mw_fmr_pbl); 2161 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", 2162 (void *)qattr->max_mr_mw_fmr_size); 2163 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); 2164 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); 2165 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); 2166 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); 2167 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", 2168 qattr->max_stats_queues); 2169 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); 2170 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", 2171 (void *)qattr->page_size_caps); 2172 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); 2173 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); 2174 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", 2175 qattr->bad_pkey_counter); 2176 2177 if ((attrs->qp_type == IB_QPT_GSI) && udata) { 2178 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); 2179 return -EINVAL; 2180 } 2181 2182 if (udata && !(ibpd->uobject && ibpd->uobject->context)) { 2183 QL_DPRINT12(ha, "called from user without context\n"); 2184 return -EINVAL; 2185 } 2186 2187 /* QP0... attrs->qp_type == IB_QPT_GSI */ 2188 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 2189 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 2190 attrs->qp_type); 2191 return -EINVAL; 2192 } 2193 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { 2194 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); 2195 return -EINVAL; 2196 } 2197 /* Skip the check for QP1 to support CM size of 128 */ 2198 if (attrs->cap.max_send_wr > qattr->max_wqe) { 2199 QL_DPRINT12(ha, "cannot create a SQ with %d elements " 2200 " (max_send_wr=0x%x)\n", 2201 attrs->cap.max_send_wr, qattr->max_wqe); 2202 return -EINVAL; 2203 } 2204 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { 2205 QL_DPRINT12(ha, "cannot create a RQ with %d elements" 2206 " (max_recv_wr=0x%x)\n", 2207 attrs->cap.max_recv_wr, qattr->max_wqe); 2208 return -EINVAL; 2209 } 2210 if (attrs->cap.max_inline_data > qattr->max_inline) { 2211 QL_DPRINT12(ha, 2212 "unsupported inline data size=0x%x " 2213 "requested (max_inline=0x%x)\n", 2214 attrs->cap.max_inline_data, qattr->max_inline); 2215 return -EINVAL; 2216 } 2217 if (attrs->cap.max_send_sge > qattr->max_sge) { 2218 QL_DPRINT12(ha, 2219 "unsupported send_sge=0x%x " 2220 "requested (max_send_sge=0x%x)\n", 2221 attrs->cap.max_send_sge, qattr->max_sge); 2222 return -EINVAL; 2223 } 2224 if (attrs->cap.max_recv_sge > qattr->max_sge) { 2225 QL_DPRINT12(ha, 2226 "unsupported recv_sge=0x%x requested " 2227 " (max_recv_sge=0x%x)\n", 2228 attrs->cap.max_recv_sge, qattr->max_sge); 2229 return -EINVAL; 2230 } 2231 /* unprivileged user space cannot create special QP */ 2232 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 2233 QL_DPRINT12(ha, 2234 "userspace can't create special QPs of type=0x%x\n", 2235 attrs->qp_type); 2236 return -EINVAL; 2237 } 2238 /* allow creating only one GSI type of QP */ 2239 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 2240 QL_DPRINT12(ha, 2241 "create qp: GSI special QPs already created.\n"); 2242 return -EINVAL; 2243 } 2244 2245 /* verify consumer QPs are not trying to use GSI QP's CQ */ 2246 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 2247 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); 2248 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); 2249 2250 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || 2251 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { 2252 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); 2253 return -EINVAL; 2254 } 2255 } 2256 QL_DPRINT12(ha, "exit\n"); 2257 return 0; 2258 } 2259 2260 static int 2261 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 2262 struct qlnxr_srq *srq, 2263 struct ib_udata *udata) 2264 { 2265 struct qlnxr_create_srq_uresp uresp; 2266 qlnx_host_t *ha; 2267 int rc; 2268 2269 ha = dev->ha; 2270 2271 QL_DPRINT12(ha, "enter\n"); 2272 2273 memset(&uresp, 0, sizeof(uresp)); 2274 2275 uresp.srq_id = srq->srq_id; 2276 2277 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2278 2279 QL_DPRINT12(ha, "exit [%d]\n", rc); 2280 return rc; 2281 } 2282 2283 static void 2284 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, 2285 struct qlnxr_create_qp_uresp *uresp, 2286 struct qlnxr_qp *qp) 2287 { 2288 qlnx_host_t *ha; 2289 2290 ha = dev->ha; 2291 2292 /* Return if QP is associated with SRQ instead of RQ */ 2293 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); 2294 2295 if (qp->srq) 2296 return; 2297 2298 /* iWARP requires two doorbells per RQ. */ 2299 if (QLNX_IS_IWARP(dev)) { 2300 2301 uresp->rq_db_offset = 2302 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2303 uresp->rq_db2_offset = 2304 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2305 2306 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " 2307 "uresp->rq_db2_offset = 0x%x\n", 2308 uresp->rq_db_offset, uresp->rq_db2_offset); 2309 } else { 2310 uresp->rq_db_offset = 2311 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2312 } 2313 uresp->rq_icid = qp->icid; 2314 2315 QL_DPRINT12(ha, "exit\n"); 2316 return; 2317 } 2318 2319 static void 2320 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, 2321 struct qlnxr_create_qp_uresp *uresp, 2322 struct qlnxr_qp *qp) 2323 { 2324 qlnx_host_t *ha; 2325 2326 ha = dev->ha; 2327 2328 QL_DPRINT12(ha, "enter\n"); 2329 2330 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2331 2332 /* iWARP uses the same cid for rq and sq*/ 2333 if (QLNX_IS_IWARP(dev)) { 2334 uresp->sq_icid = qp->icid; 2335 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); 2336 } else 2337 uresp->sq_icid = qp->icid + 1; 2338 2339 QL_DPRINT12(ha, "exit\n"); 2340 return; 2341 } 2342 2343 static int 2344 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, 2345 struct qlnxr_qp *qp, 2346 struct ib_udata *udata) 2347 { 2348 int rc; 2349 struct qlnxr_create_qp_uresp uresp; 2350 qlnx_host_t *ha; 2351 2352 ha = dev->ha; 2353 2354 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); 2355 2356 memset(&uresp, 0, sizeof(uresp)); 2357 qlnxr_copy_sq_uresp(dev, &uresp, qp); 2358 qlnxr_copy_rq_uresp(dev, &uresp, qp); 2359 2360 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 2361 uresp.qp_id = qp->qp_id; 2362 2363 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2364 2365 QL_DPRINT12(ha, "exit [%d]\n", rc); 2366 return rc; 2367 } 2368 2369 2370 static void 2371 qlnxr_set_common_qp_params(struct qlnxr_dev *dev, 2372 struct qlnxr_qp *qp, 2373 struct qlnxr_pd *pd, 2374 struct ib_qp_init_attr *attrs) 2375 { 2376 qlnx_host_t *ha; 2377 2378 ha = dev->ha; 2379 2380 QL_DPRINT12(ha, "enter\n"); 2381 2382 spin_lock_init(&qp->q_lock); 2383 2384 atomic_set(&qp->refcnt, 1); 2385 qp->pd = pd; 2386 qp->sig = QLNXR_QP_MAGIC_NUMBER; 2387 qp->qp_type = attrs->qp_type; 2388 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2389 qp->sq.max_sges = attrs->cap.max_send_sge; 2390 qp->state = ECORE_ROCE_QP_STATE_RESET; 2391 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 2392 qp->sq_cq = get_qlnxr_cq(attrs->send_cq); 2393 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); 2394 qp->dev = dev; 2395 2396 if (!attrs->srq) { 2397 /* QP is associated with RQ instead of SRQ */ 2398 qp->rq.max_sges = attrs->cap.max_recv_sge; 2399 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 2400 qp->rq.max_sges, qp->rq_cq->icid); 2401 } else { 2402 qp->srq = get_qlnxr_srq(attrs->srq); 2403 } 2404 2405 QL_DPRINT12(ha, 2406 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," 2407 " state = %d, signaled = %d, use_srq=%d\n", 2408 pd->pd_id, qp->qp_type, qp->max_inline_data, 2409 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); 2410 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 2411 qp->sq.max_sges, qp->sq_cq->icid); 2412 return; 2413 } 2414 2415 static int 2416 qlnxr_check_srq_params(struct ib_pd *ibpd, 2417 struct qlnxr_dev *dev, 2418 struct ib_srq_init_attr *attrs) 2419 { 2420 struct ecore_rdma_device *qattr; 2421 qlnx_host_t *ha; 2422 2423 ha = dev->ha; 2424 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2425 2426 QL_DPRINT12(ha, "enter\n"); 2427 2428 if (attrs->attr.max_wr > qattr->max_srq_wqe) { 2429 QL_DPRINT12(ha, "unsupported srq_wr=0x%x" 2430 " requested (max_srq_wr=0x%x)\n", 2431 attrs->attr.max_wr, qattr->max_srq_wr); 2432 return -EINVAL; 2433 } 2434 2435 if (attrs->attr.max_sge > qattr->max_sge) { 2436 QL_DPRINT12(ha, 2437 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 2438 attrs->attr.max_sge, qattr->max_sge); 2439 return -EINVAL; 2440 } 2441 2442 if (attrs->attr.srq_limit > attrs->attr.max_wr) { 2443 QL_DPRINT12(ha, 2444 "unsupported srq_limit=0x%x requested" 2445 " (max_srq_limit=0x%x)\n", 2446 attrs->attr.srq_limit, attrs->attr.srq_limit); 2447 return -EINVAL; 2448 } 2449 2450 QL_DPRINT12(ha, "exit\n"); 2451 return 0; 2452 } 2453 2454 2455 static void 2456 qlnxr_free_srq_user_params(struct qlnxr_srq *srq) 2457 { 2458 struct qlnxr_dev *dev = srq->dev; 2459 qlnx_host_t *ha; 2460 2461 ha = dev->ha; 2462 2463 QL_DPRINT12(ha, "enter\n"); 2464 2465 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2466 ib_umem_release(srq->usrq.umem); 2467 ib_umem_release(srq->prod_umem); 2468 2469 QL_DPRINT12(ha, "exit\n"); 2470 return; 2471 } 2472 2473 static void 2474 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) 2475 { 2476 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2477 struct qlnxr_dev *dev = srq->dev; 2478 qlnx_host_t *ha; 2479 2480 ha = dev->ha; 2481 2482 QL_DPRINT12(ha, "enter\n"); 2483 2484 ecore_chain_free(dev->cdev, &hw_srq->pbl); 2485 2486 qlnx_dma_free_coherent(&dev->cdev, 2487 hw_srq->virt_prod_pair_addr, 2488 hw_srq->phy_prod_pair_addr, 2489 sizeof(struct rdma_srq_producers)); 2490 2491 QL_DPRINT12(ha, "exit\n"); 2492 2493 return; 2494 } 2495 2496 static int 2497 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 2498 struct qlnxr_srq *srq, 2499 struct qlnxr_create_srq_ureq *ureq, 2500 int access, int dmasync) 2501 { 2502 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2503 struct ib_umem_chunk *chunk; 2504 #endif 2505 struct scatterlist *sg; 2506 int rc; 2507 struct qlnxr_dev *dev = srq->dev; 2508 qlnx_host_t *ha; 2509 2510 ha = dev->ha; 2511 2512 QL_DPRINT12(ha, "enter\n"); 2513 2514 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 2515 ureq->srq_len, access, dmasync, 1); 2516 if (rc) 2517 return rc; 2518 2519 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 2520 sizeof(struct rdma_srq_producers), 2521 access, dmasync); 2522 if (IS_ERR(srq->prod_umem)) { 2523 2524 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2525 ib_umem_release(srq->usrq.umem); 2526 2527 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", 2528 PTR_ERR(srq->prod_umem)); 2529 2530 return PTR_ERR(srq->prod_umem); 2531 } 2532 2533 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2534 chunk = container_of((&srq->prod_umem->chunk_list)->next, 2535 typeof(*chunk), list); 2536 sg = &chunk->page_list[0]; 2537 #else 2538 sg = srq->prod_umem->sg_head.sgl; 2539 #endif 2540 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 2541 2542 QL_DPRINT12(ha, "exit\n"); 2543 return 0; 2544 } 2545 2546 2547 static int 2548 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 2549 struct qlnxr_dev *dev, 2550 struct ib_srq_init_attr *init_attr) 2551 { 2552 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2553 dma_addr_t phy_prod_pair_addr; 2554 u32 num_elems, max_wr; 2555 void *va; 2556 int rc; 2557 qlnx_host_t *ha; 2558 2559 ha = dev->ha; 2560 2561 QL_DPRINT12(ha, "enter\n"); 2562 2563 va = qlnx_dma_alloc_coherent(&dev->cdev, 2564 &phy_prod_pair_addr, 2565 sizeof(struct rdma_srq_producers)); 2566 if (!va) { 2567 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); 2568 return -ENOMEM; 2569 } 2570 2571 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 2572 hw_srq->virt_prod_pair_addr = va; 2573 2574 max_wr = init_attr->attr.max_wr; 2575 2576 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; 2577 2578 rc = ecore_chain_alloc(dev->cdev, 2579 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2580 ECORE_CHAIN_MODE_PBL, 2581 ECORE_CHAIN_CNT_TYPE_U32, 2582 num_elems, 2583 ECORE_RDMA_SRQ_WQE_ELEM_SIZE, 2584 &hw_srq->pbl, NULL); 2585 2586 if (rc) { 2587 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); 2588 goto err0; 2589 } 2590 2591 hw_srq->max_wr = max_wr; 2592 hw_srq->num_elems = num_elems; 2593 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; 2594 2595 QL_DPRINT12(ha, "exit\n"); 2596 return 0; 2597 2598 err0: 2599 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, 2600 sizeof(struct rdma_srq_producers)); 2601 2602 QL_DPRINT12(ha, "exit [%d]\n", rc); 2603 return rc; 2604 } 2605 2606 static inline void 2607 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, 2608 struct qlnxr_pd *pd, 2609 struct qlnxr_qp *qp, 2610 struct ib_qp_init_attr *attrs, 2611 bool fmr_and_reserved_lkey, 2612 struct ecore_rdma_create_qp_in_params *params) 2613 { 2614 qlnx_host_t *ha; 2615 2616 ha = dev->ha; 2617 2618 QL_DPRINT12(ha, "enter\n"); 2619 2620 /* QP handle to be written in an async event */ 2621 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 2622 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 2623 2624 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 2625 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 2626 params->pd = pd->pd_id; 2627 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 2628 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 2629 params->stats_queue = 0; 2630 2631 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 2632 2633 if (qp->srq) { 2634 /* QP is associated with SRQ instead of RQ */ 2635 params->srq_id = qp->srq->srq_id; 2636 params->use_srq = true; 2637 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 2638 params->srq_id, params->use_srq); 2639 return; 2640 } 2641 2642 params->srq_id = 0; 2643 params->use_srq = false; 2644 2645 QL_DPRINT12(ha, "exit\n"); 2646 return; 2647 } 2648 2649 2650 static inline void 2651 qlnxr_qp_user_print( struct qlnxr_dev *dev, 2652 struct qlnxr_qp *qp) 2653 { 2654 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " 2655 "rq_addr=0x%llx, rq_len=%zd\n", 2656 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, 2657 qp->urq.buf_len); 2658 return; 2659 } 2660 2661 static int 2662 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) 2663 { 2664 u32 newid; 2665 int rc; 2666 qlnx_host_t *ha; 2667 2668 ha = dev->ha; 2669 2670 QL_DPRINT12(ha, "enter\n"); 2671 2672 if (!QLNX_IS_IWARP(dev)) 2673 return 0; 2674 2675 do { 2676 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { 2677 QL_DPRINT11(ha, "idr_pre_get failed\n"); 2678 return -ENOMEM; 2679 } 2680 2681 mtx_lock(&dev->idr_lock); 2682 2683 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); 2684 2685 mtx_unlock(&dev->idr_lock); 2686 2687 } while (rc == -EAGAIN); 2688 2689 QL_DPRINT12(ha, "exit [%d]\n", rc); 2690 2691 return rc; 2692 } 2693 2694 static void 2695 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) 2696 { 2697 qlnx_host_t *ha; 2698 2699 ha = dev->ha; 2700 2701 QL_DPRINT12(ha, "enter\n"); 2702 2703 if (!QLNX_IS_IWARP(dev)) 2704 return; 2705 2706 mtx_lock(&dev->idr_lock); 2707 idr_remove(&dev->qpidr, id); 2708 mtx_unlock(&dev->idr_lock); 2709 2710 QL_DPRINT12(ha, "exit \n"); 2711 2712 return; 2713 } 2714 2715 static inline void 2716 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, 2717 struct qlnxr_qp *qp, 2718 struct ecore_rdma_create_qp_out_params *out_params) 2719 { 2720 qlnx_host_t *ha; 2721 2722 ha = dev->ha; 2723 2724 QL_DPRINT12(ha, "enter\n"); 2725 2726 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 2727 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 2728 2729 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 2730 &qp->usq.pbl_info); 2731 2732 if (qp->srq) { 2733 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); 2734 return; 2735 } 2736 2737 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 2738 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 2739 2740 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 2741 &qp->urq.pbl_info); 2742 2743 QL_DPRINT12(ha, "exit\n"); 2744 return; 2745 } 2746 2747 static int 2748 qlnxr_create_user_qp(struct qlnxr_dev *dev, 2749 struct qlnxr_qp *qp, 2750 struct ib_pd *ibpd, 2751 struct ib_udata *udata, 2752 struct ib_qp_init_attr *attrs) 2753 { 2754 struct ecore_rdma_destroy_qp_out_params d_out_params; 2755 struct ecore_rdma_create_qp_in_params in_params; 2756 struct ecore_rdma_create_qp_out_params out_params; 2757 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2758 struct ib_ucontext *ib_ctx = NULL; 2759 struct qlnxr_ucontext *ctx = NULL; 2760 struct qlnxr_create_qp_ureq ureq; 2761 int alloc_and_init = QLNX_IS_ROCE(dev); 2762 int rc = -EINVAL; 2763 qlnx_host_t *ha; 2764 2765 ha = dev->ha; 2766 2767 QL_DPRINT12(ha, "enter\n"); 2768 2769 ib_ctx = ibpd->uobject->context; 2770 ctx = get_qlnxr_ucontext(ib_ctx); 2771 2772 memset(&ureq, 0, sizeof(ureq)); 2773 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 2774 2775 if (rc) { 2776 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); 2777 return rc; 2778 } 2779 2780 /* SQ - read access only (0), dma sync not required (0) */ 2781 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 2782 ureq.sq_len, 0, 0, 2783 alloc_and_init); 2784 if (rc) { 2785 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2786 return rc; 2787 } 2788 2789 if (!qp->srq) { 2790 /* RQ - read access only (0), dma sync not required (0) */ 2791 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 2792 ureq.rq_len, 0, 0, 2793 alloc_and_init); 2794 2795 if (rc) { 2796 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2797 return rc; 2798 } 2799 } 2800 2801 memset(&in_params, 0, sizeof(in_params)); 2802 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 2803 in_params.qp_handle_lo = ureq.qp_handle_lo; 2804 in_params.qp_handle_hi = ureq.qp_handle_hi; 2805 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 2806 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 2807 2808 if (!qp->srq) { 2809 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 2810 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 2811 } 2812 2813 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); 2814 2815 if (!qp->ecore_qp) { 2816 rc = -ENOMEM; 2817 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2818 goto err1; 2819 } 2820 2821 if (QLNX_IS_IWARP(dev)) 2822 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); 2823 2824 qp->qp_id = out_params.qp_id; 2825 qp->icid = out_params.icid; 2826 2827 rc = qlnxr_copy_qp_uresp(dev, qp, udata); 2828 2829 if (rc) { 2830 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); 2831 goto err; 2832 } 2833 2834 qlnxr_qp_user_print(dev, qp); 2835 2836 QL_DPRINT12(ha, "exit\n"); 2837 return 0; 2838 err: 2839 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2840 2841 if (rc) 2842 QL_DPRINT12(ha, "fatal fault\n"); 2843 2844 err1: 2845 qlnxr_cleanup_user(dev, qp); 2846 2847 QL_DPRINT12(ha, "exit[%d]\n", rc); 2848 return rc; 2849 } 2850 2851 static void 2852 qlnxr_set_roce_db_info(struct qlnxr_dev *dev, 2853 struct qlnxr_qp *qp) 2854 { 2855 qlnx_host_t *ha; 2856 2857 ha = dev->ha; 2858 2859 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2860 2861 qp->sq.db = dev->db_addr + 2862 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2863 qp->sq.db_data.data.icid = qp->icid + 1; 2864 2865 if (!qp->srq) { 2866 qp->rq.db = dev->db_addr + 2867 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2868 qp->rq.db_data.data.icid = qp->icid; 2869 } 2870 2871 QL_DPRINT12(ha, "exit\n"); 2872 return; 2873 } 2874 2875 static void 2876 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, 2877 struct qlnxr_qp *qp) 2878 2879 { 2880 qlnx_host_t *ha; 2881 2882 ha = dev->ha; 2883 2884 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2885 2886 qp->sq.db = dev->db_addr + 2887 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2888 qp->sq.db_data.data.icid = qp->icid; 2889 2890 if (!qp->srq) { 2891 qp->rq.db = dev->db_addr + 2892 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2893 qp->rq.db_data.data.icid = qp->icid; 2894 2895 qp->rq.iwarp_db2 = dev->db_addr + 2896 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2897 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2898 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2899 } 2900 2901 QL_DPRINT12(ha, 2902 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" 2903 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" 2904 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" 2905 " qp->rq.iwarp_db2.data.prod_val =0x%x\n", 2906 qp->sq.db, qp->sq.db_data.data.icid, 2907 qp->rq.db, qp->rq.db_data.data.icid, 2908 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, 2909 qp->rq.iwarp_db2_data.data.value); 2910 2911 QL_DPRINT12(ha, "exit\n"); 2912 return; 2913 } 2914 2915 static int 2916 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, 2917 struct qlnxr_qp *qp, 2918 struct ecore_rdma_create_qp_in_params *in_params, 2919 u32 n_sq_elems, 2920 u32 n_rq_elems) 2921 { 2922 struct ecore_rdma_create_qp_out_params out_params; 2923 int rc; 2924 qlnx_host_t *ha; 2925 2926 ha = dev->ha; 2927 2928 QL_DPRINT12(ha, "enter\n"); 2929 2930 rc = ecore_chain_alloc( 2931 dev->cdev, 2932 ECORE_CHAIN_USE_TO_PRODUCE, 2933 ECORE_CHAIN_MODE_PBL, 2934 ECORE_CHAIN_CNT_TYPE_U32, 2935 n_sq_elems, 2936 QLNXR_SQE_ELEMENT_SIZE, 2937 &qp->sq.pbl, 2938 NULL); 2939 2940 if (rc) { 2941 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); 2942 return rc; 2943 } 2944 2945 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); 2946 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); 2947 2948 if (!qp->srq) { 2949 2950 rc = ecore_chain_alloc( 2951 dev->cdev, 2952 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2953 ECORE_CHAIN_MODE_PBL, 2954 ECORE_CHAIN_CNT_TYPE_U32, 2955 n_rq_elems, 2956 QLNXR_RQE_ELEMENT_SIZE, 2957 &qp->rq.pbl, 2958 NULL); 2959 2960 if (rc) { 2961 QL_DPRINT11(ha, 2962 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); 2963 return rc; 2964 } 2965 2966 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); 2967 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); 2968 } 2969 2970 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2971 2972 if (!qp->ecore_qp) { 2973 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); 2974 return -EINVAL; 2975 } 2976 2977 qp->qp_id = out_params.qp_id; 2978 qp->icid = out_params.icid; 2979 2980 qlnxr_set_roce_db_info(dev, qp); 2981 2982 QL_DPRINT12(ha, "exit\n"); 2983 return 0; 2984 } 2985 2986 static int 2987 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, 2988 struct qlnxr_qp *qp, 2989 struct ecore_rdma_create_qp_in_params *in_params, 2990 u32 n_sq_elems, 2991 u32 n_rq_elems) 2992 { 2993 struct ecore_rdma_destroy_qp_out_params d_out_params; 2994 struct ecore_rdma_create_qp_out_params out_params; 2995 struct ecore_chain_ext_pbl ext_pbl; 2996 int rc; 2997 qlnx_host_t *ha; 2998 2999 ha = dev->ha; 3000 3001 QL_DPRINT12(ha, "enter\n"); 3002 3003 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, 3004 QLNXR_SQE_ELEMENT_SIZE, 3005 ECORE_CHAIN_MODE_PBL); 3006 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, 3007 QLNXR_RQE_ELEMENT_SIZE, 3008 ECORE_CHAIN_MODE_PBL); 3009 3010 QL_DPRINT12(ha, "n_sq_elems = 0x%x" 3011 " n_rq_elems = 0x%x in_params\n" 3012 "\t\t\tqp_handle_lo\t\t= 0x%08x\n" 3013 "\t\t\tqp_handle_hi\t\t= 0x%08x\n" 3014 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" 3015 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" 3016 "\t\t\tuse_srq\t\t\t= 0x%x\n" 3017 "\t\t\tsignal_all\t\t= 0x%x\n" 3018 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" 3019 "\t\t\tpd\t\t\t= 0x%x\n" 3020 "\t\t\tdpi\t\t\t= 0x%x\n" 3021 "\t\t\tsq_cq_id\t\t\t= 0x%x\n" 3022 "\t\t\tsq_num_pages\t\t= 0x%x\n" 3023 "\t\t\tsq_pbl_ptr\t\t= %p\n" 3024 "\t\t\tmax_sq_sges\t\t= 0x%x\n" 3025 "\t\t\trq_cq_id\t\t\t= 0x%x\n" 3026 "\t\t\trq_num_pages\t\t= 0x%x\n" 3027 "\t\t\trq_pbl_ptr\t\t= %p\n" 3028 "\t\t\tsrq_id\t\t\t= 0x%x\n" 3029 "\t\t\tstats_queue\t\t= 0x%x\n", 3030 n_sq_elems, n_rq_elems, 3031 in_params->qp_handle_lo, 3032 in_params->qp_handle_hi, 3033 in_params->qp_handle_async_lo, 3034 in_params->qp_handle_async_hi, 3035 in_params->use_srq, 3036 in_params->signal_all, 3037 in_params->fmr_and_reserved_lkey, 3038 in_params->pd, 3039 in_params->dpi, 3040 in_params->sq_cq_id, 3041 in_params->sq_num_pages, 3042 (void *)in_params->sq_pbl_ptr, 3043 in_params->max_sq_sges, 3044 in_params->rq_cq_id, 3045 in_params->rq_num_pages, 3046 (void *)in_params->rq_pbl_ptr, 3047 in_params->srq_id, 3048 in_params->stats_queue ); 3049 3050 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); 3051 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); 3052 3053 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 3054 3055 if (!qp->ecore_qp) { 3056 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 3057 return -EINVAL; 3058 } 3059 3060 /* Now we allocate the chain */ 3061 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 3062 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 3063 3064 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3065 "ext_pbl.p_pbl_phys = %p\n", 3066 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3067 3068 rc = ecore_chain_alloc( 3069 dev->cdev, 3070 ECORE_CHAIN_USE_TO_PRODUCE, 3071 ECORE_CHAIN_MODE_PBL, 3072 ECORE_CHAIN_CNT_TYPE_U32, 3073 n_sq_elems, 3074 QLNXR_SQE_ELEMENT_SIZE, 3075 &qp->sq.pbl, 3076 &ext_pbl); 3077 3078 if (rc) { 3079 QL_DPRINT11(ha, 3080 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); 3081 goto err; 3082 } 3083 3084 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 3085 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 3086 3087 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3088 "ext_pbl.p_pbl_phys = %p\n", 3089 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3090 3091 if (!qp->srq) { 3092 3093 rc = ecore_chain_alloc( 3094 dev->cdev, 3095 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 3096 ECORE_CHAIN_MODE_PBL, 3097 ECORE_CHAIN_CNT_TYPE_U32, 3098 n_rq_elems, 3099 QLNXR_RQE_ELEMENT_SIZE, 3100 &qp->rq.pbl, 3101 &ext_pbl); 3102 3103 if (rc) { 3104 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" 3105 " failed rc = %d\n", rc); 3106 goto err; 3107 } 3108 } 3109 3110 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", 3111 out_params.qp_id, out_params.icid); 3112 3113 qp->qp_id = out_params.qp_id; 3114 qp->icid = out_params.icid; 3115 3116 qlnxr_set_iwarp_db_info(dev, qp); 3117 3118 QL_DPRINT12(ha, "exit\n"); 3119 return 0; 3120 3121 err: 3122 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 3123 3124 QL_DPRINT12(ha, "exit rc = %d\n", rc); 3125 return rc; 3126 } 3127 3128 static int 3129 qlnxr_create_kernel_qp(struct qlnxr_dev *dev, 3130 struct qlnxr_qp *qp, 3131 struct ib_pd *ibpd, 3132 struct ib_qp_init_attr *attrs) 3133 { 3134 struct ecore_rdma_create_qp_in_params in_params; 3135 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3136 int rc = -EINVAL; 3137 u32 n_rq_elems; 3138 u32 n_sq_elems; 3139 u32 n_sq_entries; 3140 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3141 qlnx_host_t *ha; 3142 3143 ha = dev->ha; 3144 3145 QL_DPRINT12(ha, "enter\n"); 3146 3147 memset(&in_params, 0, sizeof(in_params)); 3148 3149 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in 3150 * the ring. The ring should allow at least a single WR, even if the 3151 * user requested none, due to allocation issues. 3152 * We should add an extra WR since the prod and cons indices of 3153 * wqe_wr_id are managed in such a way that the WQ is considered full 3154 * when (prod+1)%max_wr==cons. We currently don't do that because we 3155 * double the number of entries due an iSER issue that pushes far more 3156 * WRs than indicated. If we decline its ib_post_send() then we get 3157 * error prints in the dmesg we'd like to avoid. 3158 */ 3159 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 3160 qattr->max_wqe); 3161 3162 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 3163 GFP_KERNEL); 3164 if (!qp->wqe_wr_id) { 3165 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); 3166 return -ENOMEM; 3167 } 3168 3169 /* QP handle to be written in CQE */ 3170 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); 3171 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); 3172 3173 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in 3174 * the ring. There ring should allow at least a single WR, even if the 3175 * user requested none, due to allocation issues. 3176 */ 3177 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); 3178 3179 /* Allocate driver internal RQ array */ 3180 if (!qp->srq) { 3181 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 3182 GFP_KERNEL); 3183 if (!qp->rqe_wr_id) { 3184 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); 3185 kfree(qp->wqe_wr_id); 3186 return -ENOMEM; 3187 } 3188 } 3189 3190 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 3191 3192 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 3193 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 3194 3195 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 3196 in_params.fmr_and_reserved_lkey = true; 3197 in_params.pd = pd->pd_id; 3198 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 3199 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 3200 in_params.stats_queue = 0; 3201 3202 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 3203 3204 if (qp->srq) { 3205 /* QP is associated with SRQ instead of RQ */ 3206 in_params.srq_id = qp->srq->srq_id; 3207 in_params.use_srq = true; 3208 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 3209 in_params.srq_id, in_params.use_srq); 3210 } else { 3211 in_params.srq_id = 0; 3212 in_params.use_srq = false; 3213 } 3214 3215 n_sq_entries = attrs->cap.max_send_wr; 3216 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); 3217 n_sq_entries = max_t(u32, n_sq_entries, 1); 3218 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 3219 3220 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 3221 3222 if (QLNX_IS_ROCE(dev)) { 3223 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, 3224 n_sq_elems, n_rq_elems); 3225 } else { 3226 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, 3227 n_sq_elems, n_rq_elems); 3228 } 3229 3230 if (rc) 3231 qlnxr_cleanup_kernel(dev, qp); 3232 3233 QL_DPRINT12(ha, "exit [%d]\n", rc); 3234 return rc; 3235 } 3236 3237 struct ib_qp * 3238 qlnxr_create_qp(struct ib_pd *ibpd, 3239 struct ib_qp_init_attr *attrs, 3240 struct ib_udata *udata) 3241 { 3242 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); 3243 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3244 struct qlnxr_qp *qp; 3245 int rc = 0; 3246 qlnx_host_t *ha; 3247 3248 ha = dev->ha; 3249 3250 QL_DPRINT12(ha, "enter\n"); 3251 3252 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); 3253 if (rc) { 3254 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); 3255 return ERR_PTR(rc); 3256 } 3257 3258 QL_DPRINT12(ha, "called from %s, event_handle=%p," 3259 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 3260 (udata ? "user library" : "kernel"), 3261 attrs->event_handler, pd, 3262 get_qlnxr_cq(attrs->send_cq), 3263 get_qlnxr_cq(attrs->send_cq)->icid, 3264 get_qlnxr_cq(attrs->recv_cq), 3265 get_qlnxr_cq(attrs->recv_cq)->icid); 3266 3267 qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); 3268 3269 if (!qp) { 3270 QL_DPRINT11(ha, "kzalloc(qp) failed\n"); 3271 return ERR_PTR(-ENOMEM); 3272 } 3273 3274 qlnxr_set_common_qp_params(dev, qp, pd, attrs); 3275 3276 if (attrs->qp_type == IB_QPT_GSI) { 3277 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); 3278 return qlnxr_create_gsi_qp(dev, attrs, qp); 3279 } 3280 3281 if (udata) { 3282 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); 3283 3284 if (rc) { 3285 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); 3286 goto err; 3287 } 3288 } else { 3289 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); 3290 3291 if (rc) { 3292 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); 3293 goto err; 3294 } 3295 } 3296 3297 qp->ibqp.qp_num = qp->qp_id; 3298 3299 rc = qlnxr_idr_add(dev, qp, qp->qp_id); 3300 3301 if (rc) { 3302 QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); 3303 goto err; 3304 } 3305 3306 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); 3307 3308 return &qp->ibqp; 3309 err: 3310 kfree(qp); 3311 3312 QL_DPRINT12(ha, "failed exit\n"); 3313 return ERR_PTR(-EFAULT); 3314 } 3315 3316 3317 static enum ib_qp_state 3318 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) 3319 { 3320 enum ib_qp_state state = IB_QPS_ERR; 3321 3322 switch (qp_state) { 3323 case ECORE_ROCE_QP_STATE_RESET: 3324 state = IB_QPS_RESET; 3325 break; 3326 3327 case ECORE_ROCE_QP_STATE_INIT: 3328 state = IB_QPS_INIT; 3329 break; 3330 3331 case ECORE_ROCE_QP_STATE_RTR: 3332 state = IB_QPS_RTR; 3333 break; 3334 3335 case ECORE_ROCE_QP_STATE_RTS: 3336 state = IB_QPS_RTS; 3337 break; 3338 3339 case ECORE_ROCE_QP_STATE_SQD: 3340 state = IB_QPS_SQD; 3341 break; 3342 3343 case ECORE_ROCE_QP_STATE_ERR: 3344 state = IB_QPS_ERR; 3345 break; 3346 3347 case ECORE_ROCE_QP_STATE_SQE: 3348 state = IB_QPS_SQE; 3349 break; 3350 } 3351 return state; 3352 } 3353 3354 static enum ecore_roce_qp_state 3355 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) 3356 { 3357 enum ecore_roce_qp_state ecore_qp_state; 3358 3359 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3360 3361 switch (qp_state) { 3362 case IB_QPS_RESET: 3363 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; 3364 break; 3365 3366 case IB_QPS_INIT: 3367 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; 3368 break; 3369 3370 case IB_QPS_RTR: 3371 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; 3372 break; 3373 3374 case IB_QPS_RTS: 3375 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; 3376 break; 3377 3378 case IB_QPS_SQD: 3379 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; 3380 break; 3381 3382 case IB_QPS_ERR: 3383 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3384 break; 3385 3386 default: 3387 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3388 break; 3389 } 3390 3391 return (ecore_qp_state); 3392 } 3393 3394 static void 3395 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) 3396 { 3397 ecore_chain_reset(&qph->pbl); 3398 qph->prod = qph->cons = 0; 3399 qph->wqe_cons = 0; 3400 qph->db_data.data.value = cpu_to_le16(0); 3401 3402 return; 3403 } 3404 3405 static int 3406 qlnxr_update_qp_state(struct qlnxr_dev *dev, 3407 struct qlnxr_qp *qp, 3408 enum ecore_roce_qp_state new_state) 3409 { 3410 int status = 0; 3411 uint32_t reg_addr; 3412 struct ecore_dev *cdev; 3413 qlnx_host_t *ha; 3414 3415 ha = dev->ha; 3416 cdev = &ha->cdev; 3417 3418 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", 3419 qp, new_state, qp->state); 3420 3421 if (new_state == qp->state) { 3422 return 0; 3423 } 3424 3425 switch (qp->state) { 3426 case ECORE_ROCE_QP_STATE_RESET: 3427 switch (new_state) { 3428 case ECORE_ROCE_QP_STATE_INIT: 3429 qp->prev_wqe_size = 0; 3430 qlnxr_reset_qp_hwq_info(&qp->sq); 3431 if (!(qp->srq)) 3432 qlnxr_reset_qp_hwq_info(&qp->rq); 3433 break; 3434 default: 3435 status = -EINVAL; 3436 break; 3437 }; 3438 break; 3439 case ECORE_ROCE_QP_STATE_INIT: 3440 /* INIT->XXX */ 3441 switch (new_state) { 3442 case ECORE_ROCE_QP_STATE_RTR: 3443 /* Update doorbell (in case post_recv was done before move to RTR) */ 3444 if (qp->srq) 3445 break; 3446 wmb(); 3447 //writel(qp->rq.db_data.raw, qp->rq.db); 3448 //if (QLNX_IS_IWARP(dev)) 3449 // writel(qp->rq.iwarp_db2_data.raw, 3450 // qp->rq.iwarp_db2); 3451 3452 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 3453 (uint8_t *)cdev->doorbells); 3454 3455 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 3456 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 3457 3458 if (QLNX_IS_IWARP(dev)) { 3459 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 3460 (uint8_t *)cdev->doorbells); 3461 bus_write_4(ha->pci_dbells, reg_addr,\ 3462 qp->rq.iwarp_db2_data.raw); 3463 bus_barrier(ha->pci_dbells, 0, 0,\ 3464 BUS_SPACE_BARRIER_READ); 3465 } 3466 3467 3468 mmiowb(); 3469 break; 3470 case ECORE_ROCE_QP_STATE_ERR: 3471 /* TBD:flush qps... */ 3472 break; 3473 default: 3474 /* invalid state change. */ 3475 status = -EINVAL; 3476 break; 3477 }; 3478 break; 3479 case ECORE_ROCE_QP_STATE_RTR: 3480 /* RTR->XXX */ 3481 switch (new_state) { 3482 case ECORE_ROCE_QP_STATE_RTS: 3483 break; 3484 case ECORE_ROCE_QP_STATE_ERR: 3485 break; 3486 default: 3487 /* invalid state change. */ 3488 status = -EINVAL; 3489 break; 3490 }; 3491 break; 3492 case ECORE_ROCE_QP_STATE_RTS: 3493 /* RTS->XXX */ 3494 switch (new_state) { 3495 case ECORE_ROCE_QP_STATE_SQD: 3496 break; 3497 case ECORE_ROCE_QP_STATE_ERR: 3498 break; 3499 default: 3500 /* invalid state change. */ 3501 status = -EINVAL; 3502 break; 3503 }; 3504 break; 3505 case ECORE_ROCE_QP_STATE_SQD: 3506 /* SQD->XXX */ 3507 switch (new_state) { 3508 case ECORE_ROCE_QP_STATE_RTS: 3509 case ECORE_ROCE_QP_STATE_ERR: 3510 break; 3511 default: 3512 /* invalid state change. */ 3513 status = -EINVAL; 3514 break; 3515 }; 3516 break; 3517 case ECORE_ROCE_QP_STATE_ERR: 3518 /* ERR->XXX */ 3519 switch (new_state) { 3520 case ECORE_ROCE_QP_STATE_RESET: 3521 if ((qp->rq.prod != qp->rq.cons) || 3522 (qp->sq.prod != qp->sq.cons)) { 3523 QL_DPRINT11(ha, 3524 "Error->Reset with rq/sq " 3525 "not empty rq.prod=0x%x rq.cons=0x%x" 3526 " sq.prod=0x%x sq.cons=0x%x\n", 3527 qp->rq.prod, qp->rq.cons, 3528 qp->sq.prod, qp->sq.cons); 3529 status = -EINVAL; 3530 } 3531 break; 3532 default: 3533 status = -EINVAL; 3534 break; 3535 }; 3536 break; 3537 default: 3538 status = -EINVAL; 3539 break; 3540 }; 3541 3542 QL_DPRINT12(ha, "exit\n"); 3543 return status; 3544 } 3545 3546 int 3547 qlnxr_modify_qp(struct ib_qp *ibqp, 3548 struct ib_qp_attr *attr, 3549 int attr_mask, 3550 struct ib_udata *udata) 3551 { 3552 int rc = 0; 3553 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3554 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); 3555 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 3556 enum ib_qp_state old_qp_state, new_qp_state; 3557 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3558 qlnx_host_t *ha; 3559 3560 ha = dev->ha; 3561 3562 QL_DPRINT12(ha, 3563 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", 3564 qp, attr_mask, attr->qp_state, udata); 3565 3566 old_qp_state = qlnxr_get_ibqp_state(qp->state); 3567 if (attr_mask & IB_QP_STATE) 3568 new_qp_state = attr->qp_state; 3569 else 3570 new_qp_state = old_qp_state; 3571 3572 if (QLNX_IS_ROCE(dev)) { 3573 #if __FreeBSD_version >= 1100000 3574 if (!ib_modify_qp_is_ok(old_qp_state, 3575 new_qp_state, 3576 ibqp->qp_type, 3577 attr_mask, 3578 IB_LINK_LAYER_ETHERNET)) { 3579 QL_DPRINT12(ha, 3580 "invalid attribute mask=0x%x" 3581 " specified for qpn=0x%x of type=0x%x \n" 3582 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3583 attr_mask, qp->qp_id, ibqp->qp_type, 3584 old_qp_state, new_qp_state); 3585 rc = -EINVAL; 3586 goto err; 3587 } 3588 #else 3589 if (!ib_modify_qp_is_ok(old_qp_state, 3590 new_qp_state, 3591 ibqp->qp_type, 3592 attr_mask )) { 3593 QL_DPRINT12(ha, 3594 "invalid attribute mask=0x%x" 3595 " specified for qpn=0x%x of type=0x%x \n" 3596 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3597 attr_mask, qp->qp_id, ibqp->qp_type, 3598 old_qp_state, new_qp_state); 3599 rc = -EINVAL; 3600 goto err; 3601 } 3602 3603 #endif /* #if __FreeBSD_version >= 1100000 */ 3604 } 3605 /* translate the masks... */ 3606 if (attr_mask & IB_QP_STATE) { 3607 SET_FIELD(qp_params.modify_flags, 3608 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 3609 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); 3610 } 3611 3612 // TBD consider changing ecore to be a flag as well... 3613 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 3614 qp_params.sqd_async = true; 3615 3616 if (attr_mask & IB_QP_PKEY_INDEX) { 3617 SET_FIELD(qp_params.modify_flags, 3618 ECORE_ROCE_MODIFY_QP_VALID_PKEY, 3619 1); 3620 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { 3621 rc = -EINVAL; 3622 goto err; 3623 } 3624 3625 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; 3626 } 3627 3628 if (attr_mask & IB_QP_QKEY) { 3629 qp->qkey = attr->qkey; 3630 } 3631 3632 /* tbd consider splitting in ecore.. */ 3633 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3634 SET_FIELD(qp_params.modify_flags, 3635 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 3636 qp_params.incoming_rdma_read_en = 3637 attr->qp_access_flags & IB_ACCESS_REMOTE_READ; 3638 qp_params.incoming_rdma_write_en = 3639 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; 3640 qp_params.incoming_atomic_en = 3641 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; 3642 } 3643 3644 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 3645 if (attr_mask & IB_QP_PATH_MTU) { 3646 if (attr->path_mtu < IB_MTU_256 || 3647 attr->path_mtu > IB_MTU_4096) { 3648 3649 QL_DPRINT12(ha, 3650 "Only MTU sizes of 256, 512, 1024," 3651 " 2048 and 4096 are supported " 3652 " attr->path_mtu = [%d]\n", 3653 attr->path_mtu); 3654 3655 rc = -EINVAL; 3656 goto err; 3657 } 3658 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 3659 ib_mtu_enum_to_int( 3660 iboe_get_mtu(dev->ha->ifp->if_mtu))); 3661 } 3662 3663 if (qp->mtu == 0) { 3664 qp->mtu = ib_mtu_enum_to_int( 3665 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3666 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", 3667 qp->mtu); 3668 } 3669 3670 SET_FIELD(qp_params.modify_flags, 3671 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 3672 1); 3673 3674 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; 3675 qp_params.flow_label = attr->ah_attr.grh.flow_label; 3676 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; 3677 3678 qp->sgid_idx = attr->ah_attr.grh.sgid_index; 3679 3680 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); 3681 3682 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); 3683 if (rc) 3684 return rc; 3685 3686 qp_params.use_local_mac = true; 3687 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 3688 3689 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", 3690 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 3691 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 3692 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", 3693 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 3694 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 3695 QL_DPRINT12(ha, 3696 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", 3697 qp_params.remote_mac_addr[0], 3698 qp_params.remote_mac_addr[1], 3699 qp_params.remote_mac_addr[2], 3700 qp_params.remote_mac_addr[3], 3701 qp_params.remote_mac_addr[4], 3702 qp_params.remote_mac_addr[5]); 3703 3704 qp_params.mtu = qp->mtu; 3705 } 3706 3707 if (qp_params.mtu == 0) { 3708 /* stay with current MTU */ 3709 if (qp->mtu) { 3710 qp_params.mtu = qp->mtu; 3711 } else { 3712 qp_params.mtu = ib_mtu_enum_to_int( 3713 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3714 } 3715 } 3716 3717 if (attr_mask & IB_QP_TIMEOUT) { 3718 SET_FIELD(qp_params.modify_flags, \ 3719 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 3720 3721 qp_params.ack_timeout = attr->timeout; 3722 if (attr->timeout) { 3723 u32 temp; 3724 3725 /* 12.7.34 LOCAL ACK TIMEOUT 3726 * Value representing the transport (ACK) timeout for 3727 * use by the remote, expressed as (4.096 μS*2Local ACK 3728 * Timeout) 3729 */ 3730 /* We use 1UL since the temporal value may be overflow 3731 * 32 bits 3732 */ 3733 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 3734 qp_params.ack_timeout = temp; /* FW requires [msec] */ 3735 } 3736 else 3737 qp_params.ack_timeout = 0; /* infinite */ 3738 } 3739 if (attr_mask & IB_QP_RETRY_CNT) { 3740 SET_FIELD(qp_params.modify_flags,\ 3741 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 3742 qp_params.retry_cnt = attr->retry_cnt; 3743 } 3744 3745 if (attr_mask & IB_QP_RNR_RETRY) { 3746 SET_FIELD(qp_params.modify_flags, 3747 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 3748 1); 3749 qp_params.rnr_retry_cnt = attr->rnr_retry; 3750 } 3751 3752 if (attr_mask & IB_QP_RQ_PSN) { 3753 SET_FIELD(qp_params.modify_flags, 3754 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 3755 1); 3756 qp_params.rq_psn = attr->rq_psn; 3757 qp->rq_psn = attr->rq_psn; 3758 } 3759 3760 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 3761 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { 3762 rc = -EINVAL; 3763 QL_DPRINT12(ha, 3764 "unsupported max_rd_atomic=%d, supported=%d\n", 3765 attr->max_rd_atomic, 3766 qattr->max_qp_req_rd_atomic_resc); 3767 goto err; 3768 } 3769 3770 SET_FIELD(qp_params.modify_flags, 3771 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 3772 1); 3773 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 3774 } 3775 3776 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 3777 SET_FIELD(qp_params.modify_flags, 3778 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 3779 1); 3780 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 3781 } 3782 3783 if (attr_mask & IB_QP_SQ_PSN) { 3784 SET_FIELD(qp_params.modify_flags, 3785 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 3786 1); 3787 qp_params.sq_psn = attr->sq_psn; 3788 qp->sq_psn = attr->sq_psn; 3789 } 3790 3791 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 3792 if (attr->max_dest_rd_atomic > 3793 qattr->max_qp_resp_rd_atomic_resc) { 3794 QL_DPRINT12(ha, 3795 "unsupported max_dest_rd_atomic=%d, " 3796 "supported=%d\n", 3797 attr->max_dest_rd_atomic, 3798 qattr->max_qp_resp_rd_atomic_resc); 3799 3800 rc = -EINVAL; 3801 goto err; 3802 } 3803 3804 SET_FIELD(qp_params.modify_flags, 3805 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 3806 1); 3807 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 3808 } 3809 3810 if (attr_mask & IB_QP_DEST_QPN) { 3811 SET_FIELD(qp_params.modify_flags, 3812 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 3813 1); 3814 3815 qp_params.dest_qp = attr->dest_qp_num; 3816 qp->dest_qp_num = attr->dest_qp_num; 3817 } 3818 3819 /* 3820 * Update the QP state before the actual ramrod to prevent a race with 3821 * fast path. Modifying the QP state to error will cause the device to 3822 * flush the CQEs and while polling the flushed CQEs will considered as 3823 * a potential issue if the QP isn't in error state. 3824 */ 3825 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && 3826 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) 3827 qp->state = ECORE_ROCE_QP_STATE_ERR; 3828 3829 if (qp->qp_type != IB_QPT_GSI) 3830 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 3831 3832 if (attr_mask & IB_QP_STATE) { 3833 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 3834 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); 3835 qp->state = qp_params.new_state; 3836 } 3837 3838 err: 3839 QL_DPRINT12(ha, "exit\n"); 3840 return rc; 3841 } 3842 3843 static int 3844 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) 3845 { 3846 int ib_qp_acc_flags = 0; 3847 3848 if (params->incoming_rdma_write_en) 3849 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 3850 if (params->incoming_rdma_read_en) 3851 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 3852 if (params->incoming_atomic_en) 3853 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 3854 if (true) /* FIXME -> local write ?? */ 3855 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 3856 3857 return ib_qp_acc_flags; 3858 } 3859 3860 static enum ib_mtu 3861 qlnxr_mtu_int_to_enum(u16 mtu) 3862 { 3863 enum ib_mtu ib_mtu_size; 3864 3865 switch (mtu) { 3866 case 256: 3867 ib_mtu_size = IB_MTU_256; 3868 break; 3869 3870 case 512: 3871 ib_mtu_size = IB_MTU_512; 3872 break; 3873 3874 case 1024: 3875 ib_mtu_size = IB_MTU_1024; 3876 break; 3877 3878 case 2048: 3879 ib_mtu_size = IB_MTU_2048; 3880 break; 3881 3882 case 4096: 3883 ib_mtu_size = IB_MTU_4096; 3884 break; 3885 3886 default: 3887 ib_mtu_size = IB_MTU_1024; 3888 break; 3889 } 3890 return (ib_mtu_size); 3891 } 3892 3893 int 3894 qlnxr_query_qp(struct ib_qp *ibqp, 3895 struct ib_qp_attr *qp_attr, 3896 int attr_mask, 3897 struct ib_qp_init_attr *qp_init_attr) 3898 { 3899 int rc = 0; 3900 struct ecore_rdma_query_qp_out_params params; 3901 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3902 struct qlnxr_dev *dev = qp->dev; 3903 qlnx_host_t *ha; 3904 3905 ha = dev->ha; 3906 3907 QL_DPRINT12(ha, "enter\n"); 3908 3909 memset(¶ms, 0, sizeof(params)); 3910 3911 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); 3912 if (rc) 3913 goto err; 3914 3915 memset(qp_attr, 0, sizeof(*qp_attr)); 3916 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 3917 3918 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); 3919 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); 3920 3921 /* In some cases in iWARP qelr will ask for the state only */ 3922 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { 3923 QL_DPRINT11(ha, "only state requested\n"); 3924 return 0; 3925 } 3926 3927 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); 3928 qp_attr->path_mig_state = IB_MIG_MIGRATED; 3929 qp_attr->rq_psn = params.rq_psn; 3930 qp_attr->sq_psn = params.sq_psn; 3931 qp_attr->dest_qp_num = params.dest_qp; 3932 3933 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); 3934 3935 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " 3936 "path_mtu = %d qp_access_flags = 0x%x\n", 3937 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, 3938 qp_attr->qp_access_flags); 3939 3940 qp_attr->cap.max_send_wr = qp->sq.max_wr; 3941 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 3942 qp_attr->cap.max_send_sge = qp->sq.max_sges; 3943 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 3944 qp_attr->cap.max_inline_data = qp->max_inline_data; 3945 qp_init_attr->cap = qp_attr->cap; 3946 3947 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], 3948 sizeof(qp_attr->ah_attr.grh.dgid.raw)); 3949 3950 qp_attr->ah_attr.grh.flow_label = params.flow_label; 3951 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; 3952 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; 3953 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; 3954 3955 qp_attr->ah_attr.ah_flags = IB_AH_GRH; 3956 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ 3957 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ 3958 qp_attr->timeout = params.timeout; 3959 qp_attr->rnr_retry = params.rnr_retry; 3960 qp_attr->retry_cnt = params.retry_cnt; 3961 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 3962 qp_attr->pkey_index = params.pkey_index; 3963 qp_attr->port_num = 1; /* FIXME -> check this */ 3964 qp_attr->ah_attr.src_path_bits = 0; 3965 qp_attr->ah_attr.static_rate = 0; 3966 qp_attr->alt_pkey_index = 0; 3967 qp_attr->alt_port_num = 0; 3968 qp_attr->alt_timeout = 0; 3969 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 3970 3971 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; 3972 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 3973 qp_attr->max_rd_atomic = params.max_rd_atomic; 3974 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; 3975 3976 QL_DPRINT12(ha, "max_inline_data=%d\n", 3977 qp_attr->cap.max_inline_data); 3978 3979 err: 3980 QL_DPRINT12(ha, "exit\n"); 3981 return rc; 3982 } 3983 3984 3985 static void 3986 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3987 { 3988 qlnx_host_t *ha; 3989 3990 ha = dev->ha; 3991 3992 QL_DPRINT12(ha, "enter\n"); 3993 3994 if (qp->usq.umem) 3995 ib_umem_release(qp->usq.umem); 3996 3997 qp->usq.umem = NULL; 3998 3999 if (qp->urq.umem) 4000 ib_umem_release(qp->urq.umem); 4001 4002 qp->urq.umem = NULL; 4003 4004 QL_DPRINT12(ha, "exit\n"); 4005 return; 4006 } 4007 4008 static void 4009 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 4010 { 4011 qlnx_host_t *ha; 4012 4013 ha = dev->ha; 4014 4015 QL_DPRINT12(ha, "enter\n"); 4016 4017 if (qlnxr_qp_has_sq(qp)) { 4018 QL_DPRINT12(ha, "freeing SQ\n"); 4019 ha->qlnxr_debug = 1; 4020 // ecore_chain_free(dev->cdev, &qp->sq.pbl); 4021 ha->qlnxr_debug = 0; 4022 kfree(qp->wqe_wr_id); 4023 } 4024 4025 if (qlnxr_qp_has_rq(qp)) { 4026 QL_DPRINT12(ha, "freeing RQ\n"); 4027 ha->qlnxr_debug = 1; 4028 // ecore_chain_free(dev->cdev, &qp->rq.pbl); 4029 ha->qlnxr_debug = 0; 4030 kfree(qp->rqe_wr_id); 4031 } 4032 4033 QL_DPRINT12(ha, "exit\n"); 4034 return; 4035 } 4036 4037 int 4038 qlnxr_free_qp_resources(struct qlnxr_dev *dev, 4039 struct qlnxr_qp *qp) 4040 { 4041 int rc = 0; 4042 qlnx_host_t *ha; 4043 struct ecore_rdma_destroy_qp_out_params d_out_params; 4044 4045 ha = dev->ha; 4046 4047 QL_DPRINT12(ha, "enter\n"); 4048 4049 #if 0 4050 if (qp->qp_type != IB_QPT_GSI) { 4051 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4052 &d_out_params); 4053 if (rc) 4054 return rc; 4055 } 4056 4057 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4058 qlnxr_cleanup_user(dev, qp); 4059 else 4060 qlnxr_cleanup_kernel(dev, qp); 4061 #endif 4062 4063 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4064 qlnxr_cleanup_user(dev, qp); 4065 else 4066 qlnxr_cleanup_kernel(dev, qp); 4067 4068 if (qp->qp_type != IB_QPT_GSI) { 4069 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4070 &d_out_params); 4071 if (rc) 4072 return rc; 4073 } 4074 4075 QL_DPRINT12(ha, "exit\n"); 4076 return 0; 4077 } 4078 4079 int 4080 qlnxr_destroy_qp(struct ib_qp *ibqp) 4081 { 4082 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4083 struct qlnxr_dev *dev = qp->dev; 4084 int rc = 0; 4085 struct ib_qp_attr attr; 4086 int attr_mask = 0; 4087 qlnx_host_t *ha; 4088 4089 ha = dev->ha; 4090 4091 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); 4092 4093 qp->destroyed = 1; 4094 4095 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | 4096 ECORE_ROCE_QP_STATE_ERR | 4097 ECORE_ROCE_QP_STATE_INIT))) { 4098 4099 attr.qp_state = IB_QPS_ERR; 4100 attr_mask |= IB_QP_STATE; 4101 4102 /* change the QP state to ERROR */ 4103 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); 4104 } 4105 4106 if (qp->qp_type == IB_QPT_GSI) 4107 qlnxr_destroy_gsi_qp(dev); 4108 4109 qp->sig = ~qp->sig; 4110 4111 qlnxr_free_qp_resources(dev, qp); 4112 4113 if (atomic_dec_and_test(&qp->refcnt)) { 4114 /* TODO: only for iWARP? */ 4115 qlnxr_idr_remove(dev, qp->qp_id); 4116 kfree(qp); 4117 } 4118 4119 QL_DPRINT12(ha, "exit\n"); 4120 return rc; 4121 } 4122 4123 static inline int 4124 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) 4125 { 4126 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 4127 } 4128 4129 static int 4130 sge_data_len(struct ib_sge *sg_list, int num_sge) 4131 { 4132 int i, len = 0; 4133 for (i = 0; i < num_sge; i++) 4134 len += sg_list[i].length; 4135 return len; 4136 } 4137 4138 static void 4139 swap_wqe_data64(u64 *p) 4140 { 4141 int i; 4142 4143 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 4144 *p = cpu_to_be64(cpu_to_le64(*p)); 4145 } 4146 4147 4148 static u32 4149 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, 4150 struct qlnxr_qp *qp, 4151 u8 *wqe_size, 4152 struct ib_send_wr *wr, 4153 struct ib_send_wr **bad_wr, 4154 u8 *bits, 4155 u8 bit) 4156 { 4157 int i, seg_siz; 4158 char *seg_prt, *wqe; 4159 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 4160 qlnx_host_t *ha; 4161 4162 ha = dev->ha; 4163 4164 QL_DPRINT12(ha, "enter[%d]\n", data_size); 4165 4166 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 4167 QL_DPRINT12(ha, 4168 "Too much inline data in WR:[%d, %d]\n", 4169 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); 4170 *bad_wr = wr; 4171 return 0; 4172 } 4173 4174 if (!data_size) 4175 return data_size; 4176 4177 /* set the bit */ 4178 *bits |= bit; 4179 4180 seg_prt = wqe = NULL; 4181 seg_siz = 0; 4182 4183 /* copy data inline */ 4184 for (i = 0; i < wr->num_sge; i++) { 4185 u32 len = wr->sg_list[i].length; 4186 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 4187 4188 while (len > 0) { 4189 u32 cur; 4190 4191 /* new segment required */ 4192 if (!seg_siz) { 4193 wqe = (char *)ecore_chain_produce(&qp->sq.pbl); 4194 seg_prt = wqe; 4195 seg_siz = sizeof(struct rdma_sq_common_wqe); 4196 (*wqe_size)++; 4197 } 4198 4199 /* calculate currently allowed length */ 4200 cur = MIN(len, seg_siz); 4201 4202 memcpy(seg_prt, src, cur); 4203 4204 /* update segment variables */ 4205 seg_prt += cur; 4206 seg_siz -= cur; 4207 /* update sge variables */ 4208 src += cur; 4209 len -= cur; 4210 4211 /* swap fully-completed segments */ 4212 if (!seg_siz) 4213 swap_wqe_data64((u64 *)wqe); 4214 } 4215 } 4216 4217 /* swap last not completed segment */ 4218 if (seg_siz) 4219 swap_wqe_data64((u64 *)wqe); 4220 4221 QL_DPRINT12(ha, "exit\n"); 4222 return data_size; 4223 } 4224 4225 static u32 4226 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, 4227 u8 *wqe_size, struct ib_send_wr *wr) 4228 { 4229 int i; 4230 u32 data_size = 0; 4231 qlnx_host_t *ha; 4232 4233 ha = dev->ha; 4234 4235 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); 4236 4237 for (i = 0; i < wr->num_sge; i++) { 4238 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); 4239 4240 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); 4241 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 4242 sge->length = cpu_to_le32(wr->sg_list[i].length); 4243 data_size += wr->sg_list[i].length; 4244 } 4245 4246 if (wqe_size) 4247 *wqe_size += wr->num_sge; 4248 4249 QL_DPRINT12(ha, "exit data_size = %d\n", data_size); 4250 return data_size; 4251 } 4252 4253 static u32 4254 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, 4255 struct qlnxr_qp *qp, 4256 struct rdma_sq_rdma_wqe_1st *rwqe, 4257 struct rdma_sq_rdma_wqe_2nd *rwqe2, 4258 struct ib_send_wr *wr, 4259 struct ib_send_wr **bad_wr) 4260 { 4261 qlnx_host_t *ha; 4262 u32 ret = 0; 4263 4264 ha = dev->ha; 4265 4266 QL_DPRINT12(ha, "enter\n"); 4267 4268 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 4269 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); 4270 4271 if (wr->send_flags & IB_SEND_INLINE) { 4272 u8 flags = 0; 4273 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 4274 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, 4275 wr, bad_wr, &rwqe->flags, flags); 4276 } 4277 4278 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); 4279 4280 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4281 4282 return (ret); 4283 } 4284 4285 static u32 4286 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, 4287 struct qlnxr_qp *qp, 4288 struct rdma_sq_send_wqe *swqe, 4289 struct rdma_sq_send_wqe *swqe2, 4290 struct ib_send_wr *wr, 4291 struct ib_send_wr **bad_wr) 4292 { 4293 qlnx_host_t *ha; 4294 u32 ret = 0; 4295 4296 ha = dev->ha; 4297 4298 QL_DPRINT12(ha, "enter\n"); 4299 4300 memset(swqe2, 0, sizeof(*swqe2)); 4301 4302 if (wr->send_flags & IB_SEND_INLINE) { 4303 u8 flags = 0; 4304 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 4305 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, 4306 wr, bad_wr, &swqe->flags, flags); 4307 } 4308 4309 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); 4310 4311 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4312 4313 return (ret); 4314 } 4315 4316 static void 4317 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) 4318 { 4319 qlnx_host_t *ha; 4320 4321 ha = dev->ha; 4322 4323 int work = info->completed - info->completed_handled - 1; 4324 4325 QL_DPRINT12(ha, "enter [%d]\n", work); 4326 4327 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 4328 struct qlnxr_pbl *pbl; 4329 4330 /* Free all the page list that are possible to be freed 4331 * (all the ones that were invalidated), under the assumption 4332 * that if an FMR was completed successfully that means that 4333 * if there was an invalidate operation before it also ended 4334 */ 4335 pbl = list_first_entry(&info->inuse_pbl_list, 4336 struct qlnxr_pbl, 4337 list_entry); 4338 list_del(&pbl->list_entry); 4339 list_add_tail(&pbl->list_entry, &info->free_pbl_list); 4340 info->completed_handled++; 4341 } 4342 4343 QL_DPRINT12(ha, "exit\n"); 4344 return; 4345 } 4346 4347 #if __FreeBSD_version >= 1102000 4348 4349 static int qlnxr_prepare_reg(struct qlnxr_qp *qp, 4350 struct rdma_sq_fmr_wqe_1st *fwqe1, 4351 struct ib_reg_wr *wr) 4352 { 4353 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); 4354 struct rdma_sq_fmr_wqe_2nd *fwqe2; 4355 4356 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); 4357 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 4358 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 4359 fwqe1->l_key = wr->key; 4360 4361 fwqe2->access_ctrl = 0; 4362 4363 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 4364 !!(wr->access & IB_ACCESS_REMOTE_READ)); 4365 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 4366 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 4367 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 4368 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 4369 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 4370 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 4371 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 4372 fwqe2->fmr_ctrl = 0; 4373 4374 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4375 ilog2(mr->ibmr.page_size) - 12); 4376 4377 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ 4378 fwqe2->length_lo = mr->ibmr.length; 4379 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 4380 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 4381 4382 qp->wqe_wr_id[qp->sq.prod].mr = mr; 4383 4384 return 0; 4385 } 4386 4387 #else 4388 4389 static void 4390 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr, 4391 struct mr_info *info) 4392 { 4393 int i; 4394 u64 buf_addr = 0; 4395 int num_pbes, total_num_pbes = 0; 4396 struct regpair *pbe; 4397 struct qlnxr_pbl *pbl_tbl = info->pbl_table; 4398 struct qlnxr_pbl_info *pbl_info = &info->pbl_info; 4399 qlnx_host_t *ha; 4400 4401 ha = dev->ha; 4402 4403 QL_DPRINT12(ha, "enter\n"); 4404 4405 pbe = (struct regpair *)pbl_tbl->va; 4406 num_pbes = 0; 4407 4408 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { 4409 buf_addr = wr->wr.fast_reg.page_list->page_list[i]; 4410 pbe->lo = cpu_to_le32((u32)buf_addr); 4411 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr)); 4412 4413 num_pbes += 1; 4414 pbe++; 4415 total_num_pbes++; 4416 4417 if (total_num_pbes == pbl_info->num_pbes) 4418 return; 4419 4420 /* if the given pbl is full storing the pbes, 4421 * move to next pbl. 4422 */ 4423 if (num_pbes == 4424 (pbl_info->pbl_size / sizeof(u64))) { 4425 pbl_tbl++; 4426 pbe = (struct regpair *)pbl_tbl->va; 4427 num_pbes = 0; 4428 } 4429 } 4430 QL_DPRINT12(ha, "exit\n"); 4431 4432 return; 4433 } 4434 4435 static int 4436 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info) 4437 { 4438 int rc = 0; 4439 qlnx_host_t *ha; 4440 4441 ha = dev->ha; 4442 4443 QL_DPRINT12(ha, "enter\n"); 4444 4445 if (info->completed == 0) { 4446 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n"); 4447 /* first fmr */ 4448 return 0; 4449 } 4450 4451 qlnx_handle_completed_mrs(dev, info); 4452 4453 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list); 4454 4455 if (list_empty(&info->free_pbl_list)) { 4456 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, 4457 GFP_ATOMIC); 4458 } else { 4459 info->pbl_table = list_first_entry(&info->free_pbl_list, 4460 struct qlnxr_pbl, 4461 list_entry); 4462 list_del(&info->pbl_table->list_entry); 4463 } 4464 4465 if (!info->pbl_table) 4466 rc = -ENOMEM; 4467 4468 QL_DPRINT12(ha, "exit\n"); 4469 return rc; 4470 } 4471 4472 static inline int 4473 qlnxr_prepare_fmr(struct qlnxr_qp *qp, 4474 struct rdma_sq_fmr_wqe_1st *fwqe1, 4475 struct ib_send_wr *wr) 4476 { 4477 struct qlnxr_dev *dev = qp->dev; 4478 u64 fbo; 4479 struct qlnxr_fast_reg_page_list *frmr_list = 4480 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list); 4481 struct rdma_sq_fmr_wqe *fwqe2 = 4482 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl); 4483 int rc = 0; 4484 qlnx_host_t *ha; 4485 4486 ha = dev->ha; 4487 4488 QL_DPRINT12(ha, "enter\n"); 4489 4490 if (wr->wr.fast_reg.page_list_len == 0) 4491 BUG(); 4492 4493 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info); 4494 if (rc) 4495 return rc; 4496 4497 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start); 4498 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start); 4499 fwqe1->l_key = wr->wr.fast_reg.rkey; 4500 4501 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ, 4502 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)); 4503 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE, 4504 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)); 4505 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC, 4506 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC)); 4507 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1); 4508 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE, 4509 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)); 4510 4511 fwqe2->fmr_ctrl = 0; 4512 4513 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4514 ilog2(1 << wr->wr.fast_reg.page_shift) - 12); 4515 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0); 4516 4517 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */ 4518 fwqe2->length_lo = wr->wr.fast_reg.length; 4519 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa); 4520 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa); 4521 4522 /* produce another wqe for fwqe3 */ 4523 ecore_chain_produce(&qp->sq.pbl); 4524 4525 fbo = wr->wr.fast_reg.iova_start - 4526 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK); 4527 4528 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x" 4529 " length = %x pbl_addr %x:%x\n", 4530 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey, 4531 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo, 4532 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo); 4533 4534 build_frmr_pbes(dev, wr, &frmr_list->info); 4535 4536 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list; 4537 4538 QL_DPRINT12(ha, "exit\n"); 4539 return 0; 4540 } 4541 4542 #endif /* #if __FreeBSD_version >= 1102000 */ 4543 4544 static enum ib_wc_opcode 4545 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 4546 { 4547 switch (opcode) { 4548 case IB_WR_RDMA_WRITE: 4549 case IB_WR_RDMA_WRITE_WITH_IMM: 4550 return IB_WC_RDMA_WRITE; 4551 case IB_WR_SEND_WITH_IMM: 4552 case IB_WR_SEND: 4553 case IB_WR_SEND_WITH_INV: 4554 return IB_WC_SEND; 4555 case IB_WR_RDMA_READ: 4556 return IB_WC_RDMA_READ; 4557 case IB_WR_ATOMIC_CMP_AND_SWP: 4558 return IB_WC_COMP_SWAP; 4559 case IB_WR_ATOMIC_FETCH_AND_ADD: 4560 return IB_WC_FETCH_ADD; 4561 4562 #if __FreeBSD_version >= 1102000 4563 case IB_WR_REG_MR: 4564 return IB_WC_REG_MR; 4565 #else 4566 case IB_WR_FAST_REG_MR: 4567 return IB_WC_FAST_REG_MR; 4568 #endif /* #if __FreeBSD_version >= 1102000 */ 4569 4570 case IB_WR_LOCAL_INV: 4571 return IB_WC_LOCAL_INV; 4572 default: 4573 return IB_WC_SEND; 4574 } 4575 } 4576 static inline bool 4577 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr) 4578 { 4579 int wq_is_full, err_wr, pbl_is_full; 4580 struct qlnxr_dev *dev = qp->dev; 4581 qlnx_host_t *ha; 4582 4583 ha = dev->ha; 4584 4585 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); 4586 4587 /* prevent SQ overflow and/or processing of a bad WR */ 4588 err_wr = wr->num_sge > qp->sq.max_sges; 4589 wq_is_full = qlnxr_wq_is_full(&qp->sq); 4590 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < 4591 QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 4592 if (wq_is_full || err_wr || pbl_is_full) { 4593 if (wq_is_full && 4594 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { 4595 4596 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; 4597 4598 QL_DPRINT12(ha, 4599 "error: WQ is full. Post send on QP failed" 4600 " (this error appears only once) " 4601 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4602 qp, wr, qp->err_bitmap); 4603 } 4604 4605 if (err_wr && 4606 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { 4607 4608 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; 4609 4610 QL_DPRINT12(ha, 4611 "error: WQ is bad. Post send on QP failed" 4612 " (this error appears only once) " 4613 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4614 qp, wr, qp->err_bitmap); 4615 } 4616 4617 if (pbl_is_full && 4618 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { 4619 4620 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; 4621 4622 QL_DPRINT12(ha, 4623 "error: WQ PBL is full. Post send on QP failed" 4624 " (this error appears only once) " 4625 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4626 qp, wr, qp->err_bitmap); 4627 } 4628 return false; 4629 } 4630 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); 4631 return true; 4632 } 4633 4634 int 4635 qlnxr_post_send(struct ib_qp *ibqp, 4636 struct ib_send_wr *wr, 4637 struct ib_send_wr **bad_wr) 4638 { 4639 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); 4640 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4641 unsigned long flags; 4642 int status = 0, rc = 0; 4643 bool comp; 4644 qlnx_host_t *ha; 4645 uint32_t reg_addr; 4646 4647 *bad_wr = NULL; 4648 ha = dev->ha; 4649 4650 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4651 ibqp, wr, bad_wr); 4652 4653 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 4654 return -EINVAL; 4655 4656 if (qp->qp_type == IB_QPT_GSI) 4657 return qlnxr_gsi_post_send(ibqp, wr, bad_wr); 4658 4659 spin_lock_irqsave(&qp->q_lock, flags); 4660 4661 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && 4662 (qp->state != ECORE_ROCE_QP_STATE_ERR) && 4663 (qp->state != ECORE_ROCE_QP_STATE_SQD)) { 4664 spin_unlock_irqrestore(&qp->q_lock, flags); 4665 *bad_wr = wr; 4666 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", 4667 qp->icid, qp->state); 4668 return -EINVAL; 4669 } 4670 4671 if (!wr) { 4672 QL_DPRINT11(ha, "Got an empty post send???\n"); 4673 } 4674 4675 while (wr) { 4676 struct rdma_sq_common_wqe *wqe; 4677 struct rdma_sq_send_wqe *swqe; 4678 struct rdma_sq_send_wqe *swqe2; 4679 struct rdma_sq_rdma_wqe_1st *rwqe; 4680 struct rdma_sq_rdma_wqe_2nd *rwqe2; 4681 struct rdma_sq_local_inv_wqe *iwqe; 4682 struct rdma_sq_atomic_wqe *awqe1; 4683 struct rdma_sq_atomic_wqe *awqe2; 4684 struct rdma_sq_atomic_wqe *awqe3; 4685 struct rdma_sq_fmr_wqe_1st *fwqe1; 4686 4687 if (!qlnxr_can_post_send(qp, wr)) { 4688 status = -ENOMEM; 4689 *bad_wr = wr; 4690 break; 4691 } 4692 4693 wqe = ecore_chain_produce(&qp->sq.pbl); 4694 4695 qp->wqe_wr_id[qp->sq.prod].signaled = 4696 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 4697 4698 /* common fields */ 4699 wqe->flags = 0; 4700 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << 4701 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); 4702 4703 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ 4704 !!(wr->send_flags & IB_SEND_SOLICITED)); 4705 4706 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || 4707 (qp->signaled); 4708 4709 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 4710 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ 4711 !!(wr->send_flags & IB_SEND_FENCE)); 4712 4713 wqe->prev_wqe_size = qp->prev_wqe_size; 4714 4715 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); 4716 4717 4718 switch (wr->opcode) { 4719 4720 case IB_WR_SEND_WITH_IMM: 4721 4722 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 4723 swqe = (struct rdma_sq_send_wqe *)wqe; 4724 swqe->wqe_size = 2; 4725 swqe2 = (struct rdma_sq_send_wqe *) 4726 ecore_chain_produce(&qp->sq.pbl); 4727 swqe->inv_key_or_imm_data = 4728 cpu_to_le32(wr->ex.imm_data); 4729 swqe->length = cpu_to_le32( 4730 qlnxr_prepare_sq_send_data(dev, 4731 qp, swqe, swqe2, wr, 4732 bad_wr)); 4733 4734 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4735 qp->prev_wqe_size = swqe->wqe_size; 4736 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4737 4738 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", 4739 swqe->length, wr->ex.imm_data); 4740 4741 break; 4742 4743 case IB_WR_SEND: 4744 4745 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 4746 swqe = (struct rdma_sq_send_wqe *)wqe; 4747 4748 swqe->wqe_size = 2; 4749 swqe2 = (struct rdma_sq_send_wqe *) 4750 ecore_chain_produce(&qp->sq.pbl); 4751 swqe->length = cpu_to_le32( 4752 qlnxr_prepare_sq_send_data(dev, 4753 qp, swqe, swqe2, wr, 4754 bad_wr)); 4755 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4756 qp->prev_wqe_size = swqe->wqe_size; 4757 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4758 4759 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", 4760 swqe->length); 4761 4762 break; 4763 4764 case IB_WR_SEND_WITH_INV: 4765 4766 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 4767 swqe = (struct rdma_sq_send_wqe *)wqe; 4768 swqe2 = (struct rdma_sq_send_wqe *) 4769 ecore_chain_produce(&qp->sq.pbl); 4770 swqe->wqe_size = 2; 4771 swqe->inv_key_or_imm_data = 4772 cpu_to_le32(wr->ex.invalidate_rkey); 4773 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, 4774 qp, swqe, swqe2, wr, bad_wr)); 4775 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4776 qp->prev_wqe_size = swqe->wqe_size; 4777 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4778 4779 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", 4780 swqe->length); 4781 break; 4782 4783 case IB_WR_RDMA_WRITE_WITH_IMM: 4784 4785 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 4786 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4787 4788 rwqe->wqe_size = 2; 4789 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 4790 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4791 ecore_chain_produce(&qp->sq.pbl); 4792 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4793 qp, rwqe, rwqe2, wr, bad_wr)); 4794 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4795 qp->prev_wqe_size = rwqe->wqe_size; 4796 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4797 4798 QL_DPRINT12(ha, 4799 "RDMA WRITE w/ IMM length = %d imm data=%x\n", 4800 rwqe->length, rwqe->imm_data); 4801 4802 break; 4803 4804 case IB_WR_RDMA_WRITE: 4805 4806 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 4807 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4808 4809 rwqe->wqe_size = 2; 4810 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4811 ecore_chain_produce(&qp->sq.pbl); 4812 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4813 qp, rwqe, rwqe2, wr, bad_wr)); 4814 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4815 qp->prev_wqe_size = rwqe->wqe_size; 4816 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4817 4818 QL_DPRINT12(ha, 4819 "RDMA WRITE w/o IMM length = %d\n", 4820 rwqe->length); 4821 4822 break; 4823 4824 case IB_WR_RDMA_READ_WITH_INV: 4825 4826 QL_DPRINT12(ha, 4827 "RDMA READ WITH INVALIDATE not supported\n"); 4828 4829 *bad_wr = wr; 4830 rc = -EINVAL; 4831 4832 break; 4833 4834 case IB_WR_RDMA_READ: 4835 4836 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 4837 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4838 4839 rwqe->wqe_size = 2; 4840 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4841 ecore_chain_produce(&qp->sq.pbl); 4842 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4843 qp, rwqe, rwqe2, wr, bad_wr)); 4844 4845 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4846 qp->prev_wqe_size = rwqe->wqe_size; 4847 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4848 4849 QL_DPRINT12(ha, "RDMA READ length = %d\n", 4850 rwqe->length); 4851 4852 break; 4853 4854 case IB_WR_ATOMIC_CMP_AND_SWP: 4855 case IB_WR_ATOMIC_FETCH_AND_ADD: 4856 4857 QL_DPRINT12(ha, 4858 "ATOMIC operation = %s\n", 4859 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? 4860 "IB_WR_ATOMIC_CMP_AND_SWP" : 4861 "IB_WR_ATOMIC_FETCH_AND_ADD")); 4862 4863 awqe1 = (struct rdma_sq_atomic_wqe *)wqe; 4864 awqe1->prev_wqe_size = 4; 4865 4866 awqe2 = (struct rdma_sq_atomic_wqe *) 4867 ecore_chain_produce(&qp->sq.pbl); 4868 4869 TYPEPTR_ADDR_SET(awqe2, remote_va, \ 4870 atomic_wr(wr)->remote_addr); 4871 4872 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 4873 4874 awqe3 = (struct rdma_sq_atomic_wqe *) 4875 ecore_chain_produce(&qp->sq.pbl); 4876 4877 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 4878 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 4879 TYPEPTR_ADDR_SET(awqe3, swap_data, 4880 atomic_wr(wr)->compare_add); 4881 } else { 4882 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 4883 TYPEPTR_ADDR_SET(awqe3, swap_data, 4884 atomic_wr(wr)->swap); 4885 TYPEPTR_ADDR_SET(awqe3, cmp_data, 4886 atomic_wr(wr)->compare_add); 4887 } 4888 4889 qlnxr_prepare_sq_sges(dev, qp, NULL, wr); 4890 4891 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; 4892 qp->prev_wqe_size = awqe1->prev_wqe_size; 4893 4894 break; 4895 4896 case IB_WR_LOCAL_INV: 4897 4898 QL_DPRINT12(ha, 4899 "INVALIDATE length (IB_WR_LOCAL_INV)\n"); 4900 4901 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 4902 iwqe->prev_wqe_size = 1; 4903 4904 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 4905 iwqe->inv_l_key = wr->ex.invalidate_rkey; 4906 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; 4907 qp->prev_wqe_size = iwqe->prev_wqe_size; 4908 4909 break; 4910 4911 #if __FreeBSD_version >= 1102000 4912 4913 case IB_WR_REG_MR: 4914 4915 QL_DPRINT12(ha, "IB_WR_REG_MR\n"); 4916 4917 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4918 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4919 fwqe1->wqe_size = 2; 4920 4921 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); 4922 if (rc) { 4923 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); 4924 *bad_wr = wr; 4925 break; 4926 } 4927 4928 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 4929 qp->prev_wqe_size = fwqe1->wqe_size; 4930 4931 break; 4932 #else 4933 case IB_WR_FAST_REG_MR: 4934 4935 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n"); 4936 4937 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4938 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4939 fwqe1->prev_wqe_size = 3; 4940 4941 rc = qlnxr_prepare_fmr(qp, fwqe1, wr); 4942 4943 if (rc) { 4944 QL_DPRINT12(ha, 4945 "FAST_MR (IB_WR_FAST_REG_MR) failed" 4946 " rc = %d\n", rc); 4947 *bad_wr = wr; 4948 break; 4949 } 4950 4951 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size; 4952 qp->prev_wqe_size = fwqe1->prev_wqe_size; 4953 4954 break; 4955 #endif /* #if __FreeBSD_version >= 1102000 */ 4956 4957 default: 4958 4959 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); 4960 4961 rc = -EINVAL; 4962 *bad_wr = wr; 4963 break; 4964 } 4965 4966 if (*bad_wr) { 4967 /* 4968 * restore prod to its position before this WR was processed 4969 */ 4970 ecore_chain_set_prod(&qp->sq.pbl, 4971 le16_to_cpu(qp->sq.db_data.data.value), 4972 wqe); 4973 /* restore prev_wqe_size */ 4974 qp->prev_wqe_size = wqe->prev_wqe_size; 4975 status = rc; 4976 4977 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); 4978 break; /* out of the loop */ 4979 } 4980 4981 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 4982 4983 qlnxr_inc_sw_prod(&qp->sq); 4984 4985 qp->sq.db_data.data.value++; 4986 4987 wr = wr->next; 4988 } 4989 4990 /* Trigger doorbell 4991 * If there was a failure in the first WR then it will be triggered in 4992 * vane. However this is not harmful (as long as the producer value is 4993 * unchanged). For performance reasons we avoid checking for this 4994 * redundant doorbell. 4995 */ 4996 wmb(); 4997 //writel(qp->sq.db_data.raw, qp->sq.db); 4998 4999 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); 5000 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); 5001 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5002 5003 mmiowb(); 5004 5005 spin_unlock_irqrestore(&qp->q_lock, flags); 5006 5007 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 5008 ibqp, wr, bad_wr); 5009 5010 return status; 5011 } 5012 5013 static u32 5014 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) 5015 { 5016 u32 used; 5017 5018 /* Calculate number of elements used based on producer 5019 * count and consumer count and subtract it from max 5020 * work request supported so that we get elements left. 5021 */ 5022 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 5023 5024 return hw_srq->max_wr - used; 5025 } 5026 5027 5028 int 5029 qlnxr_post_recv(struct ib_qp *ibqp, 5030 struct ib_recv_wr *wr, 5031 struct ib_recv_wr **bad_wr) 5032 { 5033 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 5034 struct qlnxr_dev *dev = qp->dev; 5035 unsigned long flags; 5036 int status = 0; 5037 qlnx_host_t *ha; 5038 uint32_t reg_addr; 5039 5040 ha = dev->ha; 5041 5042 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5043 return -EINVAL; 5044 5045 QL_DPRINT12(ha, "enter\n"); 5046 5047 if (qp->qp_type == IB_QPT_GSI) { 5048 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); 5049 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); 5050 } 5051 5052 if (qp->srq) { 5053 QL_DPRINT11(ha, "qp->srq [%p]" 5054 " QP is associated with SRQ, cannot post RQ buffers\n", 5055 qp->srq); 5056 return -EINVAL; 5057 } 5058 5059 spin_lock_irqsave(&qp->q_lock, flags); 5060 5061 if (qp->state == ECORE_ROCE_QP_STATE_RESET) { 5062 spin_unlock_irqrestore(&qp->q_lock, flags); 5063 *bad_wr = wr; 5064 5065 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); 5066 5067 return -EINVAL; 5068 } 5069 5070 while (wr) { 5071 int i; 5072 5073 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < 5074 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || 5075 (wr->num_sge > qp->rq.max_sges)) { 5076 status = -ENOMEM; 5077 *bad_wr = wr; 5078 break; 5079 } 5080 for (i = 0; i < wr->num_sge; i++) { 5081 u32 flags = 0; 5082 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5083 5084 /* first one must include the number of SGE in the list */ 5085 if (!i) 5086 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); 5087 5088 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); 5089 5090 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ 5091 wr->sg_list[i].length, flags); 5092 } 5093 /* Special case of no sges. FW requires between 1-4 sges... 5094 * in this case we need to post 1 sge with length zero. this is 5095 * because rdma write with immediate consumes an RQ. */ 5096 if (!wr->num_sge) { 5097 u32 flags = 0; 5098 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5099 5100 /* first one must include the number of SGE in the list */ 5101 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 5102 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 5103 5104 //RQ_SGE_SET(rqe, 0, 0, flags); 5105 rqe->addr.hi = 0; 5106 rqe->addr.lo = 0; 5107 5108 rqe->length = 0; 5109 rqe->flags = cpu_to_le32(flags); 5110 5111 i = 1; 5112 } 5113 5114 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 5115 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 5116 5117 qlnxr_inc_sw_prod(&qp->rq); 5118 5119 wmb(); 5120 5121 qp->rq.db_data.data.value++; 5122 5123 // writel(qp->rq.db_data.raw, qp->rq.db); 5124 mmiowb(); 5125 // if (QLNX_IS_IWARP(dev)) { 5126 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 5127 // mmiowb(); /* for second doorbell */ 5128 // } 5129 5130 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 5131 (uint8_t *)ha->cdev.doorbells); 5132 5133 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 5134 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5135 5136 if (QLNX_IS_IWARP(dev)) { 5137 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 5138 (uint8_t *)ha->cdev.doorbells); 5139 bus_write_4(ha->pci_dbells, reg_addr, \ 5140 qp->rq.iwarp_db2_data.raw); 5141 bus_barrier(ha->pci_dbells, 0, 0, \ 5142 BUS_SPACE_BARRIER_READ); 5143 } 5144 5145 wr = wr->next; 5146 } 5147 5148 spin_unlock_irqrestore(&qp->q_lock, flags); 5149 5150 QL_DPRINT12(ha, "exit status = 0x%x\n", status); 5151 5152 return status; 5153 } 5154 5155 /* In fmr we need to increase the number of fmr completed counter for the fmr 5156 * algorithm determining whether we can free a pbl or not. 5157 * we need to perform this whether the work request was signaled or not. for 5158 * this purpose we call this function from the condition that checks if a wr 5159 * should be skipped, to make sure we don't miss it ( possibly this fmr 5160 * operation was not signalted) 5161 */ 5162 static inline void 5163 qlnxr_chk_if_fmr(struct qlnxr_qp *qp) 5164 { 5165 #if __FreeBSD_version >= 1102000 5166 5167 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 5168 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5169 #else 5170 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR) 5171 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5172 5173 #endif /* #if __FreeBSD_version >= 1102000 */ 5174 } 5175 5176 static int 5177 process_req(struct qlnxr_dev *dev, 5178 struct qlnxr_qp *qp, 5179 struct qlnxr_cq *cq, 5180 int num_entries, 5181 struct ib_wc *wc, 5182 u16 hw_cons, 5183 enum ib_wc_status status, 5184 int force) 5185 { 5186 u16 cnt = 0; 5187 qlnx_host_t *ha = dev->ha; 5188 5189 QL_DPRINT12(ha, "enter\n"); 5190 5191 while (num_entries && qp->sq.wqe_cons != hw_cons) { 5192 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 5193 qlnxr_chk_if_fmr(qp); 5194 /* skip WC */ 5195 goto next_cqe; 5196 } 5197 5198 /* fill WC */ 5199 wc->status = status; 5200 wc->vendor_err = 0; 5201 wc->wc_flags = 0; 5202 wc->src_qp = qp->id; 5203 wc->qp = &qp->ibqp; 5204 5205 // common section 5206 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 5207 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 5208 5209 switch (wc->opcode) { 5210 5211 case IB_WC_RDMA_WRITE: 5212 5213 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 5214 5215 QL_DPRINT12(ha, 5216 "opcode = IB_WC_RDMA_WRITE bytes = %d\n", 5217 qp->wqe_wr_id[qp->sq.cons].bytes_len); 5218 break; 5219 5220 case IB_WC_COMP_SWAP: 5221 case IB_WC_FETCH_ADD: 5222 wc->byte_len = 8; 5223 break; 5224 5225 #if __FreeBSD_version >= 1102000 5226 case IB_WC_REG_MR: 5227 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5228 break; 5229 #else 5230 case IB_WC_FAST_REG_MR: 5231 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5232 break; 5233 #endif /* #if __FreeBSD_version >= 1102000 */ 5234 5235 case IB_WC_RDMA_READ: 5236 case IB_WC_SEND: 5237 5238 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); 5239 break; 5240 default: 5241 ;//DP_ERR("TBD ERROR"); 5242 } 5243 5244 num_entries--; 5245 wc++; 5246 cnt++; 5247 next_cqe: 5248 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 5249 ecore_chain_consume(&qp->sq.pbl); 5250 qlnxr_inc_sw_cons(&qp->sq); 5251 } 5252 5253 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5254 return cnt; 5255 } 5256 5257 static int 5258 qlnxr_poll_cq_req(struct qlnxr_dev *dev, 5259 struct qlnxr_qp *qp, 5260 struct qlnxr_cq *cq, 5261 int num_entries, 5262 struct ib_wc *wc, 5263 struct rdma_cqe_requester *req) 5264 { 5265 int cnt = 0; 5266 qlnx_host_t *ha = dev->ha; 5267 5268 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); 5269 5270 switch (req->status) { 5271 5272 case RDMA_CQE_REQ_STS_OK: 5273 5274 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5275 IB_WC_SUCCESS, 0); 5276 break; 5277 5278 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 5279 5280 if (qp->state != ECORE_ROCE_QP_STATE_ERR) 5281 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5282 IB_WC_WR_FLUSH_ERR, 1); 5283 break; 5284 5285 default: /* other errors case */ 5286 5287 /* process all WQE before the cosumer */ 5288 qp->state = ECORE_ROCE_QP_STATE_ERR; 5289 cnt = process_req(dev, qp, cq, num_entries, wc, 5290 req->sq_cons - 1, IB_WC_SUCCESS, 0); 5291 wc += cnt; 5292 /* if we have extra WC fill it with actual error info */ 5293 5294 if (cnt < num_entries) { 5295 enum ib_wc_status wc_status; 5296 5297 switch (req->status) { 5298 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 5299 wc_status = IB_WC_BAD_RESP_ERR; 5300 break; 5301 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 5302 wc_status = IB_WC_LOC_LEN_ERR; 5303 break; 5304 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 5305 wc_status = IB_WC_LOC_QP_OP_ERR; 5306 break; 5307 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 5308 wc_status = IB_WC_LOC_PROT_ERR; 5309 break; 5310 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 5311 wc_status = IB_WC_MW_BIND_ERR; 5312 break; 5313 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 5314 wc_status = IB_WC_REM_INV_REQ_ERR; 5315 break; 5316 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 5317 wc_status = IB_WC_REM_ACCESS_ERR; 5318 break; 5319 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 5320 wc_status = IB_WC_REM_OP_ERR; 5321 break; 5322 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 5323 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 5324 break; 5325 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 5326 wc_status = IB_WC_RETRY_EXC_ERR; 5327 break; 5328 default: 5329 wc_status = IB_WC_GENERAL_ERR; 5330 } 5331 5332 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 5333 wc_status, 1 /* force use of WC */); 5334 } 5335 } 5336 5337 QL_DPRINT12(ha, "exit cnt = %d\n", cnt); 5338 return cnt; 5339 } 5340 5341 static void 5342 __process_resp_one(struct qlnxr_dev *dev, 5343 struct qlnxr_qp *qp, 5344 struct qlnxr_cq *cq, 5345 struct ib_wc *wc, 5346 struct rdma_cqe_responder *resp, 5347 u64 wr_id) 5348 { 5349 enum ib_wc_status wc_status = IB_WC_SUCCESS; 5350 #if __FreeBSD_version < 1102000 5351 u8 flags; 5352 #endif 5353 qlnx_host_t *ha = dev->ha; 5354 5355 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", 5356 qp, resp->status); 5357 5358 wc->opcode = IB_WC_RECV; 5359 wc->wc_flags = 0; 5360 5361 switch (resp->status) { 5362 5363 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 5364 wc_status = IB_WC_LOC_ACCESS_ERR; 5365 break; 5366 5367 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 5368 wc_status = IB_WC_LOC_LEN_ERR; 5369 break; 5370 5371 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 5372 wc_status = IB_WC_LOC_QP_OP_ERR; 5373 break; 5374 5375 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 5376 wc_status = IB_WC_LOC_PROT_ERR; 5377 break; 5378 5379 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 5380 wc_status = IB_WC_MW_BIND_ERR; 5381 break; 5382 5383 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 5384 wc_status = IB_WC_REM_INV_RD_REQ_ERR; 5385 break; 5386 5387 case RDMA_CQE_RESP_STS_OK: 5388 5389 #if __FreeBSD_version >= 1102000 5390 if (resp->flags & QLNXR_RESP_IMM) { 5391 wc->ex.imm_data = 5392 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5393 wc->wc_flags |= IB_WC_WITH_IMM; 5394 5395 if (resp->flags & QLNXR_RESP_RDMA) 5396 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5397 5398 if (resp->flags & QLNXR_RESP_INV) { 5399 QL_DPRINT11(ha, 5400 "Invalid flags QLNXR_RESP_INV [0x%x]" 5401 "qp = %p qp->id = 0x%x cq = %p" 5402 " cq->icid = 0x%x\n", 5403 resp->flags, qp, qp->id, cq, cq->icid ); 5404 } 5405 } else if (resp->flags & QLNXR_RESP_INV) { 5406 wc->ex.imm_data = 5407 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5408 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 5409 5410 if (resp->flags & QLNXR_RESP_RDMA) { 5411 QL_DPRINT11(ha, 5412 "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5413 "qp = %p qp->id = 0x%x cq = %p" 5414 " cq->icid = 0x%x\n", 5415 resp->flags, qp, qp->id, cq, cq->icid ); 5416 } 5417 } else if (resp->flags & QLNXR_RESP_RDMA) { 5418 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5419 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5420 resp->flags, qp, qp->id, cq, cq->icid ); 5421 } 5422 #else 5423 wc_status = IB_WC_SUCCESS; 5424 wc->byte_len = le32_to_cpu(resp->length); 5425 5426 flags = resp->flags & QLNXR_RESP_RDMA_IMM; 5427 5428 switch (flags) { 5429 5430 case QLNXR_RESP_RDMA_IMM: 5431 /* update opcode */ 5432 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5433 /* fall to set imm data */ 5434 case QLNXR_RESP_IMM: 5435 wc->ex.imm_data = 5436 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5437 wc->wc_flags |= IB_WC_WITH_IMM; 5438 break; 5439 case QLNXR_RESP_RDMA: 5440 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5441 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5442 resp->flags, qp, qp->id, cq, cq->icid ); 5443 break; 5444 default: 5445 /* valid configuration, but nothing todo here */ 5446 ; 5447 } 5448 #endif /* #if __FreeBSD_version >= 1102000 */ 5449 5450 break; 5451 default: 5452 wc_status = IB_WC_GENERAL_ERR; 5453 } 5454 5455 /* fill WC */ 5456 wc->status = wc_status; 5457 wc->vendor_err = 0; 5458 wc->src_qp = qp->id; 5459 wc->qp = &qp->ibqp; 5460 wc->wr_id = wr_id; 5461 5462 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); 5463 5464 return; 5465 } 5466 5467 static int 5468 process_resp_one_srq(struct qlnxr_dev *dev, 5469 struct qlnxr_qp *qp, 5470 struct qlnxr_cq *cq, 5471 struct ib_wc *wc, 5472 struct rdma_cqe_responder *resp) 5473 { 5474 struct qlnxr_srq *srq = qp->srq; 5475 u64 wr_id; 5476 qlnx_host_t *ha = dev->ha; 5477 5478 QL_DPRINT12(ha, "enter\n"); 5479 5480 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); 5481 5482 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5483 wc->status = IB_WC_WR_FLUSH_ERR; 5484 wc->vendor_err = 0; 5485 wc->wr_id = wr_id; 5486 wc->byte_len = 0; 5487 wc->src_qp = qp->id; 5488 wc->qp = &qp->ibqp; 5489 wc->wr_id = wr_id; 5490 } else { 5491 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5492 } 5493 5494 /* PBL is maintained in case of WR granularity. 5495 * So increment WR consumer after consuming WR 5496 */ 5497 srq->hw_srq.wr_cons_cnt++; 5498 5499 QL_DPRINT12(ha, "exit\n"); 5500 return 1; 5501 } 5502 5503 static int 5504 process_resp_one(struct qlnxr_dev *dev, 5505 struct qlnxr_qp *qp, 5506 struct qlnxr_cq *cq, 5507 struct ib_wc *wc, 5508 struct rdma_cqe_responder *resp) 5509 { 5510 qlnx_host_t *ha = dev->ha; 5511 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5512 5513 QL_DPRINT12(ha, "enter\n"); 5514 5515 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5516 5517 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5518 ecore_chain_consume(&qp->rq.pbl); 5519 qlnxr_inc_sw_cons(&qp->rq); 5520 5521 QL_DPRINT12(ha, "exit\n"); 5522 return 1; 5523 } 5524 5525 static int 5526 process_resp_flush(struct qlnxr_qp *qp, 5527 int num_entries, 5528 struct ib_wc *wc, 5529 u16 hw_cons) 5530 { 5531 u16 cnt = 0; 5532 qlnx_host_t *ha = qp->dev->ha; 5533 5534 QL_DPRINT12(ha, "enter\n"); 5535 5536 while (num_entries && qp->rq.wqe_cons != hw_cons) { 5537 /* fill WC */ 5538 wc->status = IB_WC_WR_FLUSH_ERR; 5539 wc->vendor_err = 0; 5540 wc->wc_flags = 0; 5541 wc->src_qp = qp->id; 5542 wc->byte_len = 0; 5543 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5544 wc->qp = &qp->ibqp; 5545 num_entries--; 5546 wc++; 5547 cnt++; 5548 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5549 ecore_chain_consume(&qp->rq.pbl); 5550 qlnxr_inc_sw_cons(&qp->rq); 5551 } 5552 5553 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5554 return cnt; 5555 } 5556 5557 static void 5558 try_consume_resp_cqe(struct qlnxr_cq *cq, 5559 struct qlnxr_qp *qp, 5560 struct rdma_cqe_responder *resp, 5561 int *update) 5562 { 5563 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 5564 consume_cqe(cq); 5565 *update |= 1; 5566 } 5567 } 5568 5569 static int 5570 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, 5571 struct qlnxr_qp *qp, 5572 struct qlnxr_cq *cq, 5573 int num_entries, 5574 struct ib_wc *wc, 5575 struct rdma_cqe_responder *resp, 5576 int *update) 5577 { 5578 int cnt; 5579 qlnx_host_t *ha = dev->ha; 5580 5581 QL_DPRINT12(ha, "enter\n"); 5582 5583 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 5584 consume_cqe(cq); 5585 *update |= 1; 5586 5587 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5588 return cnt; 5589 } 5590 5591 static int 5592 qlnxr_poll_cq_resp(struct qlnxr_dev *dev, 5593 struct qlnxr_qp *qp, 5594 struct qlnxr_cq *cq, 5595 int num_entries, 5596 struct ib_wc *wc, 5597 struct rdma_cqe_responder *resp, 5598 int *update) 5599 { 5600 int cnt; 5601 qlnx_host_t *ha = dev->ha; 5602 5603 QL_DPRINT12(ha, "enter\n"); 5604 5605 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5606 cnt = process_resp_flush(qp, num_entries, wc, 5607 resp->rq_cons); 5608 try_consume_resp_cqe(cq, qp, resp, update); 5609 } else { 5610 cnt = process_resp_one(dev, qp, cq, wc, resp); 5611 consume_cqe(cq); 5612 *update |= 1; 5613 } 5614 5615 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5616 return cnt; 5617 } 5618 5619 static void 5620 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, 5621 struct rdma_cqe_requester *req, int *update) 5622 { 5623 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 5624 consume_cqe(cq); 5625 *update |= 1; 5626 } 5627 } 5628 5629 static void 5630 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) 5631 { 5632 uint64_t reg_addr; 5633 qlnx_host_t *ha = dev->ha; 5634 5635 QL_DPRINT12(ha, "enter\n"); 5636 5637 wmb(); 5638 cq->db.data.agg_flags = flags; 5639 cq->db.data.value = cpu_to_le32(cons); 5640 5641 reg_addr = (uint64_t)((uint8_t *)cq->db_addr - 5642 (uint8_t *)(ha->cdev.doorbells)); 5643 5644 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); 5645 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5646 5647 QL_DPRINT12(ha, "exit\n"); 5648 return; 5649 5650 //#ifdef __LP64__ 5651 // writeq(cq->db.raw, cq->db_addr); 5652 //#else 5653 /* Note that since the FW allows 64 bit write only, in 32bit systems 5654 * the value of db_addr must be low enough. This is currently not 5655 * enforced. 5656 */ 5657 // writel(cq->db.raw & 0xffffffff, cq->db_addr); 5658 // mmiowb(); 5659 //#endif 5660 } 5661 5662 5663 static int 5664 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) 5665 { 5666 struct rdma_cqe_requester *resp_cqe = &cqe->req; 5667 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == 5668 cq->pbl_toggle; 5669 } 5670 5671 int 5672 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 5673 { 5674 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5675 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 5676 int done = 0; 5677 union rdma_cqe *cqe = cq->latest_cqe; 5678 int update = 0; 5679 u32 old_cons, new_cons; 5680 unsigned long flags; 5681 qlnx_host_t *ha = dev->ha; 5682 5683 QL_DPRINT12(ha, "enter\n"); 5684 5685 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5686 return -EINVAL; 5687 5688 if (cq->destroyed) { 5689 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", 5690 cq, cq->icid); 5691 return 0; 5692 } 5693 5694 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) 5695 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); 5696 5697 spin_lock_irqsave(&cq->cq_lock, flags); 5698 5699 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5700 5701 while (num_entries && is_valid_cqe(cq, cqe)) { 5702 int cnt = 0; 5703 struct qlnxr_qp *qp; 5704 struct rdma_cqe_requester *resp_cqe; 5705 enum rdma_cqe_type cqe_type; 5706 5707 /* prevent speculative reads of any field of CQE */ 5708 rmb(); 5709 5710 resp_cqe = &cqe->req; 5711 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, 5712 resp_cqe->qp_handle.lo); 5713 5714 if (!qp) { 5715 QL_DPRINT11(ha, "qp = NULL\n"); 5716 break; 5717 } 5718 5719 wc->qp = &qp->ibqp; 5720 5721 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 5722 5723 switch (cqe_type) { 5724 case RDMA_CQE_TYPE_REQUESTER: 5725 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, 5726 wc, &cqe->req); 5727 try_consume_req_cqe(cq, qp, &cqe->req, &update); 5728 break; 5729 case RDMA_CQE_TYPE_RESPONDER_RQ: 5730 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, 5731 wc, &cqe->resp, &update); 5732 break; 5733 case RDMA_CQE_TYPE_RESPONDER_SRQ: 5734 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, 5735 wc, &cqe->resp, &update); 5736 break; 5737 case RDMA_CQE_TYPE_INVALID: 5738 default: 5739 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); 5740 break; 5741 } 5742 num_entries -= cnt; 5743 wc += cnt; 5744 done += cnt; 5745 5746 cqe = cq->latest_cqe; 5747 } 5748 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5749 5750 cq->cq_cons += new_cons - old_cons; 5751 5752 if (update) { 5753 /* doorbell notifies abount latest VALID entry, 5754 * but chain already point to the next INVALID one 5755 */ 5756 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); 5757 QL_DPRINT12(ha, "cq = %p cons = 0x%x " 5758 "arm_flags = 0x%x db.icid = 0x%x\n", cq, 5759 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); 5760 } 5761 5762 spin_unlock_irqrestore(&cq->cq_lock, flags); 5763 5764 QL_DPRINT12(ha, "exit\n"); 5765 5766 return done; 5767 } 5768 5769 5770 int 5771 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 5772 { 5773 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5774 unsigned long sflags; 5775 struct qlnxr_dev *dev; 5776 qlnx_host_t *ha; 5777 5778 dev = get_qlnxr_dev((ibcq->device)); 5779 ha = dev->ha; 5780 5781 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " 5782 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, 5783 flags, cq, cq->cq_cons, cq->cq_type); 5784 5785 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5786 return -EINVAL; 5787 5788 if (cq->destroyed) { 5789 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", 5790 cq, cq->icid); 5791 return -EINVAL; 5792 } 5793 5794 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { 5795 return 0; 5796 } 5797 5798 spin_lock_irqsave(&cq->cq_lock, sflags); 5799 5800 cq->arm_flags = 0; 5801 5802 if (flags & IB_CQ_SOLICITED) { 5803 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 5804 } 5805 if (flags & IB_CQ_NEXT_COMP) { 5806 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 5807 } 5808 5809 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); 5810 5811 spin_unlock_irqrestore(&cq->cq_lock, sflags); 5812 5813 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); 5814 return 0; 5815 } 5816 5817 5818 static struct qlnxr_mr * 5819 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) 5820 { 5821 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 5822 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 5823 struct qlnxr_mr *mr; 5824 int rc = -ENOMEM; 5825 qlnx_host_t *ha; 5826 5827 ha = dev->ha; 5828 5829 QL_DPRINT12(ha, "enter ibpd = %p pd = %p " 5830 " pd_id = %d max_page_list_len = %d\n", 5831 ibpd, pd, pd->pd_id, max_page_list_len); 5832 5833 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5834 if (!mr) { 5835 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 5836 return ERR_PTR(rc); 5837 } 5838 5839 mr->dev = dev; 5840 mr->type = QLNXR_MR_FRMR; 5841 5842 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 5843 1 /* allow dual layer pbl */); 5844 if (rc) { 5845 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); 5846 goto err0; 5847 } 5848 5849 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 5850 if (rc) { 5851 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); 5852 goto err0; 5853 } 5854 5855 /* index only, 18 bit long, lkey = itid << 8 | key */ 5856 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; 5857 mr->hw_mr.key = 0; 5858 mr->hw_mr.pd = pd->pd_id; 5859 mr->hw_mr.local_read = 1; 5860 mr->hw_mr.local_write = 0; 5861 mr->hw_mr.remote_read = 0; 5862 mr->hw_mr.remote_write = 0; 5863 mr->hw_mr.remote_atomic = 0; 5864 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 5865 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ 5866 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 5867 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 5868 mr->hw_mr.fbo = 0; 5869 mr->hw_mr.length = 0; 5870 mr->hw_mr.vaddr = 0; 5871 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 5872 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ 5873 mr->hw_mr.dma_mr = false; 5874 5875 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 5876 if (rc) { 5877 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); 5878 goto err1; 5879 } 5880 5881 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 5882 mr->ibmr.rkey = mr->ibmr.lkey; 5883 5884 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", 5885 mr, mr->ibmr.lkey); 5886 5887 return mr; 5888 5889 err1: 5890 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 5891 err0: 5892 kfree(mr); 5893 5894 QL_DPRINT12(ha, "exit\n"); 5895 5896 return ERR_PTR(rc); 5897 } 5898 5899 #if __FreeBSD_version >= 1102000 5900 5901 struct ib_mr * 5902 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) 5903 { 5904 struct qlnxr_dev *dev; 5905 struct qlnxr_mr *mr; 5906 qlnx_host_t *ha; 5907 5908 dev = get_qlnxr_dev(ibpd->device); 5909 ha = dev->ha; 5910 5911 QL_DPRINT12(ha, "enter\n"); 5912 5913 if (mr_type != IB_MR_TYPE_MEM_REG) 5914 return ERR_PTR(-EINVAL); 5915 5916 mr = __qlnxr_alloc_mr(ibpd, max_num_sg); 5917 5918 if (IS_ERR(mr)) 5919 return ERR_PTR(-EINVAL); 5920 5921 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); 5922 5923 return &mr->ibmr; 5924 } 5925 5926 static int 5927 qlnxr_set_page(struct ib_mr *ibmr, u64 addr) 5928 { 5929 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5930 struct qlnxr_pbl *pbl_table; 5931 struct regpair *pbe; 5932 struct qlnxr_dev *dev; 5933 qlnx_host_t *ha; 5934 u32 pbes_in_page; 5935 5936 dev = mr->dev; 5937 ha = dev->ha; 5938 5939 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 5940 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); 5941 return -ENOMEM; 5942 } 5943 5944 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, 5945 ((void *)addr)); 5946 5947 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 5948 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 5949 pbe = (struct regpair *)pbl_table->va; 5950 pbe += mr->npages % pbes_in_page; 5951 pbe->lo = cpu_to_le32((u32)addr); 5952 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 5953 5954 mr->npages++; 5955 5956 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, 5957 ((void *)addr)); 5958 return 0; 5959 } 5960 5961 int 5962 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 5963 int sg_nents, unsigned int *sg_offset) 5964 { 5965 int ret; 5966 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5967 qlnx_host_t *ha; 5968 5969 if (mr == NULL) 5970 return (-1); 5971 5972 if (mr->dev == NULL) 5973 return (-1); 5974 5975 ha = mr->dev->ha; 5976 5977 QL_DPRINT12(ha, "enter\n"); 5978 5979 mr->npages = 0; 5980 qlnx_handle_completed_mrs(mr->dev, &mr->info); 5981 5982 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); 5983 5984 QL_DPRINT12(ha, "exit ret = %d\n", ret); 5985 5986 return (ret); 5987 } 5988 5989 #else 5990 5991 struct ib_mr * 5992 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len) 5993 { 5994 struct qlnxr_dev *dev; 5995 struct qlnxr_mr *mr; 5996 qlnx_host_t *ha; 5997 struct ib_mr *ibmr = NULL; 5998 5999 dev = get_qlnxr_dev((ibpd->device)); 6000 ha = dev->ha; 6001 6002 QL_DPRINT12(ha, "enter\n"); 6003 6004 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len); 6005 6006 if (IS_ERR(mr)) { 6007 ibmr = ERR_PTR(-EINVAL); 6008 } else { 6009 ibmr = &mr->ibmr; 6010 } 6011 6012 QL_DPRINT12(ha, "exit %p\n", ibmr); 6013 return (ibmr); 6014 } 6015 6016 void 6017 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list) 6018 { 6019 struct qlnxr_fast_reg_page_list *frmr_list; 6020 6021 frmr_list = get_qlnxr_frmr_list(page_list); 6022 6023 free_mr_info(frmr_list->dev, &frmr_list->info); 6024 6025 kfree(frmr_list->ibfrpl.page_list); 6026 kfree(frmr_list); 6027 6028 return; 6029 } 6030 6031 struct ib_fast_reg_page_list * 6032 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len) 6033 { 6034 struct qlnxr_fast_reg_page_list *frmr_list = NULL; 6035 struct qlnxr_dev *dev; 6036 int size = page_list_len * sizeof(u64); 6037 int rc = -ENOMEM; 6038 qlnx_host_t *ha; 6039 6040 dev = get_qlnxr_dev(ibdev); 6041 ha = dev->ha; 6042 6043 QL_DPRINT12(ha, "enter\n"); 6044 6045 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL); 6046 if (!frmr_list) { 6047 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n"); 6048 goto err; 6049 } 6050 6051 frmr_list->dev = dev; 6052 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL); 6053 if (!frmr_list->ibfrpl.page_list) { 6054 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n"); 6055 goto err0; 6056 } 6057 6058 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len, 6059 1 /* allow dual layer pbl */); 6060 if (rc) 6061 goto err1; 6062 6063 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl); 6064 6065 return &frmr_list->ibfrpl; 6066 6067 err1: 6068 kfree(frmr_list->ibfrpl.page_list); 6069 err0: 6070 kfree(frmr_list); 6071 err: 6072 QL_DPRINT12(ha, "exit with error\n"); 6073 6074 return ERR_PTR(rc); 6075 } 6076 6077 static int 6078 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6079 int buf_cnt, uint64_t *total_size) 6080 { 6081 u64 size = 0; 6082 6083 *total_size = 0; 6084 6085 if (!buf_cnt || buf_list == NULL) { 6086 QL_DPRINT11(ha, 6087 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt); 6088 return (-1); 6089 } 6090 6091 size = buf_list->size; 6092 6093 if (!size) { 6094 QL_DPRINT11(ha, 6095 "failed buf_list = %p buf_cnt = %d" 6096 " buf_list->size = 0\n", buf_list, buf_cnt); 6097 return (-1); 6098 } 6099 6100 while (buf_cnt) { 6101 6102 *total_size += buf_list->size; 6103 6104 if (buf_list->size != size) { 6105 QL_DPRINT11(ha, 6106 "failed buf_list = %p buf_cnt = %d" 6107 " all buffers should have same size\n", 6108 buf_list, buf_cnt); 6109 return (-1); 6110 } 6111 6112 buf_list++; 6113 buf_cnt--; 6114 } 6115 return (0); 6116 } 6117 6118 static size_t 6119 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6120 int buf_cnt) 6121 { 6122 int i; 6123 size_t num_pages = 0; 6124 u64 size; 6125 6126 for (i = 0; i < buf_cnt; i++) { 6127 6128 size = 0; 6129 while (size < buf_list->size) { 6130 size += PAGE_SIZE; 6131 num_pages++; 6132 } 6133 buf_list++; 6134 } 6135 return (num_pages); 6136 } 6137 6138 static void 6139 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev, 6140 struct ib_phys_buf *buf_list, int buf_cnt, 6141 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 6142 { 6143 struct regpair *pbe; 6144 struct qlnxr_pbl *pbl_tbl; 6145 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 6146 qlnx_host_t *ha; 6147 int i; 6148 u64 pbe_addr; 6149 6150 ha = dev->ha; 6151 6152 QL_DPRINT12(ha, "enter\n"); 6153 6154 if (!pbl_info) { 6155 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 6156 return; 6157 } 6158 6159 if (!pbl_info->num_pbes) { 6160 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 6161 return; 6162 } 6163 6164 /* If we have a two layered pbl, the first pbl points to the rest 6165 * of the pbls and the first entry lays on the second pbl in the table 6166 */ 6167 if (pbl_info->two_layered) 6168 pbl_tbl = &pbl[1]; 6169 else 6170 pbl_tbl = pbl; 6171 6172 pbe = (struct regpair *)pbl_tbl->va; 6173 if (!pbe) { 6174 QL_DPRINT12(ha, "pbe is NULL\n"); 6175 return; 6176 } 6177 6178 pbe_cnt = 0; 6179 6180 for (i = 0; i < buf_cnt; i++) { 6181 6182 pages = buf_list->size >> PAGE_SHIFT; 6183 6184 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 6185 /* store the page address in pbe */ 6186 6187 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt); 6188 6189 pbe->lo = cpu_to_le32((u32)pbe_addr); 6190 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32))); 6191 6192 QL_DPRINT12(ha, "Populate pbl table:" 6193 " pbe->addr=0x%x:0x%x " 6194 " pbe_cnt = %d total_num_pbes=%d" 6195 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 6196 total_num_pbes, pbe); 6197 6198 pbe_cnt ++; 6199 total_num_pbes ++; 6200 pbe++; 6201 6202 if (total_num_pbes == pbl_info->num_pbes) 6203 return; 6204 6205 /* if the given pbl is full storing the pbes, 6206 * move to next pbl. */ 6207 6208 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 6209 pbl_tbl++; 6210 pbe = (struct regpair *)pbl_tbl->va; 6211 pbe_cnt = 0; 6212 } 6213 } 6214 buf_list++; 6215 } 6216 QL_DPRINT12(ha, "exit\n"); 6217 return; 6218 } 6219 6220 struct ib_mr * 6221 qlnxr_reg_kernel_mr(struct ib_pd *ibpd, 6222 struct ib_phys_buf *buf_list, 6223 int buf_cnt, int acc, u64 *iova_start) 6224 { 6225 int rc = -ENOMEM; 6226 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 6227 struct qlnxr_mr *mr; 6228 struct qlnxr_pd *pd; 6229 qlnx_host_t *ha; 6230 size_t num_pages = 0; 6231 uint64_t length; 6232 6233 ha = dev->ha; 6234 6235 QL_DPRINT12(ha, "enter\n"); 6236 6237 pd = get_qlnxr_pd(ibpd); 6238 6239 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d," 6240 " iova_start = %p, acc = %d\n", 6241 pd->pd_id, buf_list, buf_cnt, iova_start, acc); 6242 6243 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 6244 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&" 6245 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 6246 // return ERR_PTR(-EINVAL); 6247 //} 6248 6249 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 6250 if (!mr) { 6251 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 6252 return ERR_PTR(rc); 6253 } 6254 6255 mr->type = QLNXR_MR_KERNEL; 6256 mr->iova_start = iova_start; 6257 6258 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length); 6259 if (rc) 6260 goto err0; 6261 6262 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt); 6263 if (!num_pages) 6264 goto err0; 6265 6266 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1); 6267 if (rc) { 6268 QL_DPRINT11(ha, 6269 "qlnxr_init_mr_info failed [%d]\n", rc); 6270 goto err1; 6271 } 6272 6273 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table, 6274 &mr->info.pbl_info); 6275 6276 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 6277 6278 if (rc) { 6279 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 6280 goto err1; 6281 } 6282 6283 /* index only, 18 bit long, lkey = itid << 8 | key */ 6284 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 6285 mr->hw_mr.key = 0; 6286 mr->hw_mr.pd = pd->pd_id; 6287 mr->hw_mr.local_read = 1; 6288 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 6289 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 6290 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 6291 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 6292 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 6293 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 6294 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 6295 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 6296 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */ 6297 6298 mr->hw_mr.fbo = 0; 6299 6300 mr->hw_mr.length = length; 6301 mr->hw_mr.vaddr = (uint64_t)iova_start; 6302 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 6303 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 6304 mr->hw_mr.dma_mr = false; 6305 6306 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 6307 if (rc) { 6308 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 6309 goto err2; 6310 } 6311 6312 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6313 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 6314 mr->hw_mr.remote_atomic) 6315 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6316 6317 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey); 6318 6319 return (&mr->ibmr); 6320 6321 err2: 6322 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 6323 err1: 6324 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 6325 err0: 6326 kfree(mr); 6327 6328 QL_DPRINT12(ha, "exit [%d]\n", rc); 6329 return (ERR_PTR(rc)); 6330 } 6331 6332 #endif /* #if __FreeBSD_version >= 1102000 */ 6333 6334 struct ib_ah * 6335 #if __FreeBSD_version >= 1102000 6336 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, 6337 struct ib_udata *udata) 6338 #else 6339 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) 6340 #endif /* #if __FreeBSD_version >= 1102000 */ 6341 { 6342 struct qlnxr_dev *dev; 6343 qlnx_host_t *ha; 6344 struct qlnxr_ah *ah; 6345 6346 dev = get_qlnxr_dev((ibpd->device)); 6347 ha = dev->ha; 6348 6349 QL_DPRINT12(ha, "in create_ah\n"); 6350 6351 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 6352 if (!ah) { 6353 QL_DPRINT12(ha, "no address handle can be allocated\n"); 6354 return ERR_PTR(-ENOMEM); 6355 } 6356 6357 ah->attr = *attr; 6358 6359 return &ah->ibah; 6360 } 6361 6362 int 6363 qlnxr_destroy_ah(struct ib_ah *ibah) 6364 { 6365 struct qlnxr_dev *dev; 6366 qlnx_host_t *ha; 6367 struct qlnxr_ah *ah = get_qlnxr_ah(ibah); 6368 6369 dev = get_qlnxr_dev((ibah->device)); 6370 ha = dev->ha; 6371 6372 QL_DPRINT12(ha, "in destroy_ah\n"); 6373 6374 kfree(ah); 6375 return 0; 6376 } 6377 6378 int 6379 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6380 { 6381 struct qlnxr_dev *dev; 6382 qlnx_host_t *ha; 6383 6384 dev = get_qlnxr_dev((ibah->device)); 6385 ha = dev->ha; 6386 QL_DPRINT12(ha, "Query AH not supported\n"); 6387 return -EINVAL; 6388 } 6389 6390 int 6391 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6392 { 6393 struct qlnxr_dev *dev; 6394 qlnx_host_t *ha; 6395 6396 dev = get_qlnxr_dev((ibah->device)); 6397 ha = dev->ha; 6398 QL_DPRINT12(ha, "Modify AH not supported\n"); 6399 return -ENOSYS; 6400 } 6401 6402 #if __FreeBSD_version >= 1102000 6403 int 6404 qlnxr_process_mad(struct ib_device *ibdev, 6405 int process_mad_flags, 6406 u8 port_num, 6407 const struct ib_wc *in_wc, 6408 const struct ib_grh *in_grh, 6409 const struct ib_mad_hdr *mad_hdr, 6410 size_t in_mad_size, 6411 struct ib_mad_hdr *out_mad, 6412 size_t *out_mad_size, 6413 u16 *out_mad_pkey_index) 6414 6415 #else 6416 6417 int 6418 qlnxr_process_mad(struct ib_device *ibdev, 6419 int process_mad_flags, 6420 u8 port_num, 6421 struct ib_wc *in_wc, 6422 struct ib_grh *in_grh, 6423 struct ib_mad *in_mad, 6424 struct ib_mad *out_mad) 6425 6426 #endif /* #if __FreeBSD_version >= 1102000 */ 6427 { 6428 struct qlnxr_dev *dev; 6429 qlnx_host_t *ha; 6430 6431 dev = get_qlnxr_dev(ibdev); 6432 ha = dev->ha; 6433 QL_DPRINT12(ha, "process mad not supported\n"); 6434 6435 return -ENOSYS; 6436 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", 6437 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, 6438 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, 6439 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, 6440 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); 6441 6442 // return IB_MAD_RESULT_SUCCESS; 6443 } 6444 6445 6446 #if __FreeBSD_version >= 1102000 6447 int 6448 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, 6449 struct ib_port_immutable *immutable) 6450 { 6451 struct qlnxr_dev *dev; 6452 qlnx_host_t *ha; 6453 struct ib_port_attr attr; 6454 int err; 6455 6456 dev = get_qlnxr_dev(ibdev); 6457 ha = dev->ha; 6458 6459 QL_DPRINT12(ha, "enter\n"); 6460 6461 err = qlnxr_query_port(ibdev, port_num, &attr); 6462 if (err) 6463 return err; 6464 6465 if (QLNX_IS_IWARP(dev)) { 6466 immutable->pkey_tbl_len = 1; 6467 immutable->gid_tbl_len = 1; 6468 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 6469 immutable->max_mad_size = 0; 6470 } else { 6471 immutable->pkey_tbl_len = attr.pkey_tbl_len; 6472 immutable->gid_tbl_len = attr.gid_tbl_len; 6473 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 6474 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 6475 } 6476 6477 QL_DPRINT12(ha, "exit\n"); 6478 return 0; 6479 } 6480 #endif /* #if __FreeBSD_version > 1102000 */ 6481 6482 6483 /***** iWARP related functions *************/ 6484 6485 6486 static void 6487 qlnxr_iw_mpa_request(void *context, 6488 struct ecore_iwarp_cm_event_params *params) 6489 { 6490 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; 6491 struct qlnxr_dev *dev = listener->dev; 6492 struct qlnxr_iw_ep *ep; 6493 struct iw_cm_event event; 6494 struct sockaddr_in *laddr; 6495 struct sockaddr_in *raddr; 6496 qlnx_host_t *ha; 6497 6498 ha = dev->ha; 6499 6500 QL_DPRINT12(ha, "enter\n"); 6501 6502 if (params->cm_info->ip_version != ECORE_TCP_IPV4) { 6503 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", 6504 params->cm_info->ip_version); 6505 return; 6506 } 6507 6508 ep = kzalloc(sizeof(*ep), GFP_ATOMIC); 6509 6510 if (!ep) { 6511 QL_DPRINT11(ha, "kzalloc{ep) failed\n"); 6512 return; 6513 } 6514 6515 ep->dev = dev; 6516 ep->ecore_context = params->ep_context; 6517 6518 memset(&event, 0, sizeof(event)); 6519 6520 event.event = IW_CM_EVENT_CONNECT_REQUEST; 6521 event.status = params->status; 6522 6523 laddr = (struct sockaddr_in *)&event.local_addr; 6524 raddr = (struct sockaddr_in *)&event.remote_addr; 6525 6526 laddr->sin_family = AF_INET; 6527 raddr->sin_family = AF_INET; 6528 6529 laddr->sin_port = htons(params->cm_info->local_port); 6530 raddr->sin_port = htons(params->cm_info->remote_port); 6531 6532 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); 6533 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); 6534 6535 event.provider_data = (void *)ep; 6536 event.private_data = (void *)params->cm_info->private_data; 6537 event.private_data_len = (u8)params->cm_info->private_data_len; 6538 6539 #if __FreeBSD_version >= 1100000 6540 event.ord = params->cm_info->ord; 6541 event.ird = params->cm_info->ird; 6542 #endif /* #if __FreeBSD_version >= 1100000 */ 6543 6544 listener->cm_id->event_handler(listener->cm_id, &event); 6545 6546 QL_DPRINT12(ha, "exit\n"); 6547 6548 return; 6549 } 6550 6551 static void 6552 qlnxr_iw_issue_event(void *context, 6553 struct ecore_iwarp_cm_event_params *params, 6554 enum iw_cm_event_type event_type, 6555 char *str) 6556 { 6557 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6558 struct qlnxr_dev *dev = ep->dev; 6559 struct iw_cm_event event; 6560 qlnx_host_t *ha; 6561 6562 ha = dev->ha; 6563 6564 QL_DPRINT12(ha, "enter\n"); 6565 6566 memset(&event, 0, sizeof(event)); 6567 event.status = params->status; 6568 event.event = event_type; 6569 6570 if (params->cm_info != NULL) { 6571 #if __FreeBSD_version >= 1100000 6572 event.ird = params->cm_info->ird; 6573 event.ord = params->cm_info->ord; 6574 QL_DPRINT12(ha, "ord=[%d] \n", event.ord); 6575 QL_DPRINT12(ha, "ird=[%d] \n", event.ird); 6576 #endif /* #if __FreeBSD_version >= 1100000 */ 6577 6578 event.private_data_len = params->cm_info->private_data_len; 6579 event.private_data = (void *)params->cm_info->private_data; 6580 QL_DPRINT12(ha, "private_data_len=[%d] \n", 6581 event.private_data_len); 6582 } 6583 6584 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); 6585 QL_DPRINT12(ha, "status=[%d] \n", event.status); 6586 6587 if (ep) { 6588 if (ep->cm_id) 6589 ep->cm_id->event_handler(ep->cm_id, &event); 6590 else 6591 QL_DPRINT11(ha, "ep->cm_id == NULL \n"); 6592 } else { 6593 QL_DPRINT11(ha, "ep == NULL \n"); 6594 } 6595 6596 QL_DPRINT12(ha, "exit\n"); 6597 6598 return; 6599 } 6600 6601 static void 6602 qlnxr_iw_close_event(void *context, 6603 struct ecore_iwarp_cm_event_params *params) 6604 { 6605 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6606 struct qlnxr_dev *dev = ep->dev; 6607 qlnx_host_t *ha; 6608 6609 ha = dev->ha; 6610 6611 QL_DPRINT12(ha, "enter\n"); 6612 6613 if (ep->cm_id) { 6614 qlnxr_iw_issue_event(context, 6615 params, 6616 IW_CM_EVENT_CLOSE, 6617 "IW_CM_EVENT_EVENT_CLOSE"); 6618 ep->cm_id->rem_ref(ep->cm_id); 6619 ep->cm_id = NULL; 6620 } 6621 6622 QL_DPRINT12(ha, "exit\n"); 6623 6624 return; 6625 } 6626 6627 #if __FreeBSD_version >= 1102000 6628 6629 static void 6630 qlnxr_iw_passive_complete(void *context, 6631 struct ecore_iwarp_cm_event_params *params) 6632 { 6633 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6634 struct qlnxr_dev *dev = ep->dev; 6635 qlnx_host_t *ha; 6636 6637 ha = dev->ha; 6638 6639 /* We will only reach the following state if MPA_REJECT was called on 6640 * passive. In this case there will be no associated QP. 6641 */ 6642 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { 6643 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); 6644 kfree(ep); 6645 return; 6646 } 6647 6648 /* We always issue an established event, however, ofed does not look 6649 * at event code for established. So if there was a failure, we follow 6650 * with close... 6651 */ 6652 qlnxr_iw_issue_event(context, 6653 params, 6654 IW_CM_EVENT_ESTABLISHED, 6655 "IW_CM_EVENT_ESTABLISHED"); 6656 6657 if (params->status < 0) { 6658 qlnxr_iw_close_event(context, params); 6659 } 6660 6661 return; 6662 } 6663 6664 struct qlnxr_discon_work { 6665 struct work_struct work; 6666 struct qlnxr_iw_ep *ep; 6667 enum ecore_iwarp_event_type event; 6668 int status; 6669 }; 6670 6671 static void 6672 qlnxr_iw_disconnect_worker(struct work_struct *work) 6673 { 6674 struct qlnxr_discon_work *dwork = 6675 container_of(work, struct qlnxr_discon_work, work); 6676 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 6677 struct qlnxr_iw_ep *ep = dwork->ep; 6678 struct qlnxr_dev *dev = ep->dev; 6679 struct qlnxr_qp *qp = ep->qp; 6680 struct iw_cm_event event; 6681 6682 if (qp->destroyed) { 6683 kfree(dwork); 6684 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6685 return; 6686 } 6687 6688 memset(&event, 0, sizeof(event)); 6689 event.status = dwork->status; 6690 event.event = IW_CM_EVENT_DISCONNECT; 6691 6692 /* Success means graceful disconnect was requested. modifying 6693 * to SQD is translated to graceful disconnect. O/w reset is sent 6694 */ 6695 if (dwork->status) 6696 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; 6697 else 6698 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; 6699 6700 kfree(dwork); 6701 6702 if (ep->cm_id) 6703 ep->cm_id->event_handler(ep->cm_id, &event); 6704 6705 SET_FIELD(qp_params.modify_flags, 6706 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 6707 6708 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 6709 6710 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6711 6712 return; 6713 } 6714 6715 void 6716 qlnxr_iw_disconnect_event(void *context, 6717 struct ecore_iwarp_cm_event_params *params) 6718 { 6719 struct qlnxr_discon_work *work; 6720 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6721 struct qlnxr_dev *dev = ep->dev; 6722 struct qlnxr_qp *qp = ep->qp; 6723 6724 work = kzalloc(sizeof(*work), GFP_ATOMIC); 6725 if (!work) 6726 return; 6727 6728 qlnxr_iw_qp_add_ref(&qp->ibqp); 6729 work->ep = ep; 6730 work->event = params->event; 6731 work->status = params->status; 6732 6733 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); 6734 queue_work(dev->iwarp_wq, &work->work); 6735 6736 return; 6737 } 6738 6739 #endif /* #if __FreeBSD_version >= 1102000 */ 6740 6741 static int 6742 qlnxr_iw_mpa_reply(void *context, 6743 struct ecore_iwarp_cm_event_params *params) 6744 { 6745 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6746 struct qlnxr_dev *dev = ep->dev; 6747 struct ecore_iwarp_send_rtr_in rtr_in; 6748 int rc; 6749 qlnx_host_t *ha; 6750 6751 ha = dev->ha; 6752 6753 QL_DPRINT12(ha, "enter\n"); 6754 6755 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6756 return -EINVAL; 6757 6758 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); 6759 rtr_in.ep_context = params->ep_context; 6760 6761 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); 6762 6763 QL_DPRINT12(ha, "exit rc = %d\n", rc); 6764 return rc; 6765 } 6766 6767 6768 void 6769 qlnxr_iw_qp_event(void *context, 6770 struct ecore_iwarp_cm_event_params *params, 6771 enum ib_event_type ib_event, 6772 char *str) 6773 { 6774 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6775 struct qlnxr_dev *dev = ep->dev; 6776 struct ib_qp *ibqp = &(ep->qp->ibqp); 6777 struct ib_event event; 6778 qlnx_host_t *ha; 6779 6780 ha = dev->ha; 6781 6782 QL_DPRINT12(ha, 6783 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", 6784 context, params->event, str, ibqp->event_handler); 6785 6786 if (ibqp->event_handler) { 6787 event.event = ib_event; 6788 event.device = ibqp->device; 6789 event.element.qp = ibqp; 6790 ibqp->event_handler(&event, ibqp->qp_context); 6791 } 6792 6793 return; 6794 } 6795 6796 int 6797 qlnxr_iw_event_handler(void *context, 6798 struct ecore_iwarp_cm_event_params *params) 6799 { 6800 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6801 struct qlnxr_dev *dev = ep->dev; 6802 qlnx_host_t *ha; 6803 6804 ha = dev->ha; 6805 6806 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6807 "enter\n", context, params->event); 6808 6809 switch (params->event) { 6810 6811 /* Passive side request received */ 6812 case ECORE_IWARP_EVENT_MPA_REQUEST: 6813 qlnxr_iw_mpa_request(context, params); 6814 break; 6815 6816 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: 6817 qlnxr_iw_mpa_reply(context, params); 6818 break; 6819 6820 /* Passive side established ( ack on mpa response ) */ 6821 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: 6822 6823 #if __FreeBSD_version >= 1102000 6824 6825 ep->during_connect = 0; 6826 qlnxr_iw_passive_complete(context, params); 6827 6828 #else 6829 qlnxr_iw_issue_event(context, 6830 params, 6831 IW_CM_EVENT_ESTABLISHED, 6832 "IW_CM_EVENT_ESTABLISHED"); 6833 #endif /* #if __FreeBSD_version >= 1102000 */ 6834 break; 6835 6836 /* Active side reply received */ 6837 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: 6838 ep->during_connect = 0; 6839 qlnxr_iw_issue_event(context, 6840 params, 6841 IW_CM_EVENT_CONNECT_REPLY, 6842 "IW_CM_EVENT_CONNECT_REPLY"); 6843 if (params->status < 0) { 6844 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6845 6846 ep->cm_id->rem_ref(ep->cm_id); 6847 ep->cm_id = NULL; 6848 } 6849 break; 6850 6851 case ECORE_IWARP_EVENT_DISCONNECT: 6852 6853 #if __FreeBSD_version >= 1102000 6854 qlnxr_iw_disconnect_event(context, params); 6855 #else 6856 qlnxr_iw_issue_event(context, 6857 params, 6858 IW_CM_EVENT_DISCONNECT, 6859 "IW_CM_EVENT_DISCONNECT"); 6860 qlnxr_iw_close_event(context, params); 6861 #endif /* #if __FreeBSD_version >= 1102000 */ 6862 break; 6863 6864 case ECORE_IWARP_EVENT_CLOSE: 6865 ep->during_connect = 0; 6866 qlnxr_iw_close_event(context, params); 6867 break; 6868 6869 case ECORE_IWARP_EVENT_RQ_EMPTY: 6870 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6871 "IWARP_EVENT_RQ_EMPTY"); 6872 break; 6873 6874 case ECORE_IWARP_EVENT_IRQ_FULL: 6875 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6876 "IWARP_EVENT_IRQ_FULL"); 6877 break; 6878 6879 case ECORE_IWARP_EVENT_LLP_TIMEOUT: 6880 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6881 "IWARP_EVENT_LLP_TIMEOUT"); 6882 break; 6883 6884 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: 6885 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6886 "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); 6887 break; 6888 6889 case ECORE_IWARP_EVENT_CQ_OVERFLOW: 6890 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6891 "QED_IWARP_EVENT_CQ_OVERFLOW"); 6892 break; 6893 6894 case ECORE_IWARP_EVENT_QP_CATASTROPHIC: 6895 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6896 "QED_IWARP_EVENT_QP_CATASTROPHIC"); 6897 break; 6898 6899 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: 6900 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6901 "IWARP_EVENT_LOCAL_ACCESS_ERROR"); 6902 break; 6903 6904 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: 6905 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6906 "IWARP_EVENT_REMOTE_OPERATION_ERROR"); 6907 break; 6908 6909 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: 6910 QL_DPRINT12(ha, "Got terminate message" 6911 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); 6912 break; 6913 6914 default: 6915 QL_DPRINT12(ha, 6916 "Unknown event [0x%x] received \n", params->event); 6917 break; 6918 }; 6919 6920 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6921 "exit\n", context, params->event); 6922 return 0; 6923 } 6924 6925 static int 6926 qlnxr_addr4_resolve(struct qlnxr_dev *dev, 6927 struct sockaddr_in *src_in, 6928 struct sockaddr_in *dst_in, 6929 u8 *dst_mac) 6930 { 6931 int rc; 6932 6933 #if __FreeBSD_version >= 1100000 6934 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, 6935 dst_mac, NULL, NULL); 6936 #else 6937 struct llentry *lle; 6938 6939 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in, 6940 dst_mac, &lle); 6941 #endif 6942 6943 QL_DPRINT12(dev->ha, "rc = %d " 6944 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " 6945 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, 6946 dst_in->sin_len, dst_in->sin_family, 6947 NIPQUAD((dst_in->sin_addr.s_addr)), 6948 dst_mac[0], dst_mac[1], dst_mac[2], 6949 dst_mac[3], dst_mac[4], dst_mac[5]); 6950 6951 return rc; 6952 } 6953 6954 int 6955 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 6956 { 6957 struct qlnxr_dev *dev; 6958 struct ecore_iwarp_connect_out out_params; 6959 struct ecore_iwarp_connect_in in_params; 6960 struct qlnxr_iw_ep *ep; 6961 struct qlnxr_qp *qp; 6962 struct sockaddr_in *laddr; 6963 struct sockaddr_in *raddr; 6964 int rc = 0; 6965 qlnx_host_t *ha; 6966 6967 dev = get_qlnxr_dev((cm_id->device)); 6968 ha = dev->ha; 6969 6970 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " 6971 "enter \n", cm_id, conn_param); 6972 6973 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6974 return -EINVAL; 6975 6976 qp = idr_find(&dev->qpidr, conn_param->qpn); 6977 6978 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6979 raddr = (struct sockaddr_in *)&cm_id->remote_addr; 6980 6981 QL_DPRINT12(ha, 6982 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", 6983 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, 6984 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); 6985 6986 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 6987 if (!ep) { 6988 QL_DPRINT11(ha, "struct qlnxr_iw_ep " 6989 "alloc memory failed\n"); 6990 return -ENOMEM; 6991 } 6992 6993 ep->dev = dev; 6994 ep->qp = qp; 6995 cm_id->add_ref(cm_id); 6996 ep->cm_id = cm_id; 6997 6998 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); 6999 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); 7000 7001 in_params.event_cb = qlnxr_iw_event_handler; 7002 in_params.cb_context = ep; 7003 7004 in_params.cm_info.ip_version = ECORE_TCP_IPV4; 7005 7006 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); 7007 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); 7008 in_params.cm_info.remote_port = ntohs(raddr->sin_port); 7009 in_params.cm_info.local_port = ntohs(laddr->sin_port); 7010 in_params.cm_info.vlan = 0; 7011 in_params.mss = dev->ha->ifp->if_mtu - 40; 7012 7013 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " 7014 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " 7015 "vlan = %d\n", 7016 NIPQUAD((in_params.cm_info.remote_ip[0])), 7017 NIPQUAD((in_params.cm_info.local_ip[0])), 7018 in_params.cm_info.remote_port, in_params.cm_info.local_port, 7019 in_params.cm_info.vlan); 7020 7021 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); 7022 7023 if (rc) { 7024 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); 7025 goto err; 7026 } 7027 7028 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" 7029 " private_data_len=%d rq_psn=%d\n", 7030 conn_param->ord, conn_param->ird, conn_param->private_data, 7031 conn_param->private_data_len, qp->rq_psn); 7032 7033 in_params.cm_info.ord = conn_param->ord; 7034 in_params.cm_info.ird = conn_param->ird; 7035 in_params.cm_info.private_data = conn_param->private_data; 7036 in_params.cm_info.private_data_len = conn_param->private_data_len; 7037 in_params.qp = qp->ecore_qp; 7038 7039 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 7040 7041 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); 7042 7043 if (rc) { 7044 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); 7045 goto err; 7046 } 7047 7048 QL_DPRINT12(ha, "exit\n"); 7049 7050 return rc; 7051 7052 err: 7053 cm_id->rem_ref(cm_id); 7054 kfree(ep); 7055 7056 QL_DPRINT12(ha, "exit [%d]\n", rc); 7057 return rc; 7058 } 7059 7060 int 7061 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) 7062 { 7063 struct qlnxr_dev *dev; 7064 struct qlnxr_iw_listener *listener; 7065 struct ecore_iwarp_listen_in iparams; 7066 struct ecore_iwarp_listen_out oparams; 7067 struct sockaddr_in *laddr; 7068 qlnx_host_t *ha; 7069 int rc; 7070 7071 dev = get_qlnxr_dev((cm_id->device)); 7072 ha = dev->ha; 7073 7074 QL_DPRINT12(ha, "enter\n"); 7075 7076 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7077 return -EINVAL; 7078 7079 laddr = (struct sockaddr_in *)&cm_id->local_addr; 7080 7081 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 7082 7083 if (listener == NULL) { 7084 QL_DPRINT11(ha, "listener memory alloc failed\n"); 7085 return -ENOMEM; 7086 } 7087 7088 listener->dev = dev; 7089 cm_id->add_ref(cm_id); 7090 listener->cm_id = cm_id; 7091 listener->backlog = backlog; 7092 7093 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); 7094 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); 7095 7096 iparams.cb_context = listener; 7097 iparams.event_cb = qlnxr_iw_event_handler; 7098 iparams.max_backlog = backlog; 7099 7100 iparams.ip_version = ECORE_TCP_IPV4; 7101 7102 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); 7103 iparams.port = ntohs(laddr->sin_port); 7104 iparams.vlan = 0; 7105 7106 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", 7107 NIPQUAD((laddr->sin_addr.s_addr)), 7108 laddr->sin_port, iparams.port); 7109 7110 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); 7111 if (rc) { 7112 QL_DPRINT11(ha, 7113 "ecore_iwarp_create_listen failed rc = %d\n", rc); 7114 goto err; 7115 } 7116 7117 listener->ecore_handle = oparams.handle; 7118 cm_id->provider_data = listener; 7119 7120 QL_DPRINT12(ha, "exit\n"); 7121 return rc; 7122 7123 err: 7124 cm_id->rem_ref(cm_id); 7125 kfree(listener); 7126 7127 QL_DPRINT12(ha, "exit [%d]\n", rc); 7128 return rc; 7129 } 7130 7131 void 7132 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) 7133 { 7134 struct qlnxr_iw_listener *listener = cm_id->provider_data; 7135 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); 7136 int rc = 0; 7137 qlnx_host_t *ha; 7138 7139 ha = dev->ha; 7140 7141 QL_DPRINT12(ha, "enter\n"); 7142 7143 if (listener->ecore_handle) 7144 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, 7145 listener->ecore_handle); 7146 7147 cm_id->rem_ref(cm_id); 7148 7149 QL_DPRINT12(ha, "exit [%d]\n", rc); 7150 return; 7151 } 7152 7153 int 7154 qlnxr_iw_accept(struct iw_cm_id *cm_id, 7155 struct iw_cm_conn_param *conn_param) 7156 { 7157 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7158 struct qlnxr_dev *dev = ep->dev; 7159 struct qlnxr_qp *qp; 7160 struct ecore_iwarp_accept_in params; 7161 int rc; 7162 qlnx_host_t *ha; 7163 7164 ha = dev->ha; 7165 7166 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); 7167 7168 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7169 return -EINVAL; 7170 7171 qp = idr_find(&dev->qpidr, conn_param->qpn); 7172 if (!qp) { 7173 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", 7174 conn_param->qpn); 7175 return -EINVAL; 7176 } 7177 ep->qp = qp; 7178 qp->ep = ep; 7179 cm_id->add_ref(cm_id); 7180 ep->cm_id = cm_id; 7181 7182 params.ep_context = ep->ecore_context; 7183 params.cb_context = ep; 7184 params.qp = ep->qp->ecore_qp; 7185 params.private_data = conn_param->private_data; 7186 params.private_data_len = conn_param->private_data_len; 7187 params.ird = conn_param->ird; 7188 params.ord = conn_param->ord; 7189 7190 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); 7191 if (rc) { 7192 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); 7193 goto err; 7194 } 7195 7196 QL_DPRINT12(ha, "exit\n"); 7197 return 0; 7198 err: 7199 cm_id->rem_ref(cm_id); 7200 QL_DPRINT12(ha, "exit rc = %d\n", rc); 7201 return rc; 7202 } 7203 7204 int 7205 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 7206 { 7207 #if __FreeBSD_version >= 1102000 7208 7209 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7210 struct qlnxr_dev *dev = ep->dev; 7211 struct ecore_iwarp_reject_in params; 7212 int rc; 7213 7214 params.ep_context = ep->ecore_context; 7215 params.cb_context = ep; 7216 params.private_data = pdata; 7217 params.private_data_len = pdata_len; 7218 ep->qp = NULL; 7219 7220 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); 7221 7222 return rc; 7223 7224 #else 7225 7226 printf("iWARP reject_cr not implemented\n"); 7227 return -EINVAL; 7228 7229 #endif /* #if __FreeBSD_version >= 1102000 */ 7230 } 7231 7232 void 7233 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) 7234 { 7235 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7236 qlnx_host_t *ha; 7237 7238 ha = qp->dev->ha; 7239 7240 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); 7241 7242 atomic_inc(&qp->refcnt); 7243 7244 QL_DPRINT12(ha, "exit \n"); 7245 return; 7246 } 7247 7248 void 7249 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) 7250 { 7251 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7252 qlnx_host_t *ha; 7253 7254 ha = qp->dev->ha; 7255 7256 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); 7257 7258 if (atomic_dec_and_test(&qp->refcnt)) { 7259 qlnxr_idr_remove(qp->dev, qp->qp_id); 7260 kfree(qp); 7261 } 7262 7263 QL_DPRINT12(ha, "exit \n"); 7264 return; 7265 } 7266 7267 struct ib_qp * 7268 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) 7269 { 7270 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 7271 struct ib_qp *qp; 7272 qlnx_host_t *ha; 7273 7274 ha = dev->ha; 7275 7276 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); 7277 7278 qp = idr_find(&dev->qpidr, qpn); 7279 7280 QL_DPRINT12(ha, "exit qp = %p\n", qp); 7281 7282 return (qp); 7283 } 7284