1 /* 2 * Copyright (c) 2018-2019 Cavium, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * File: qlnxr_verbs.c 30 */ 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "qlnxr_def.h" 35 #include "rdma_common.h" 36 #include "qlnxr_roce.h" 37 #include "qlnxr_cm.h" 38 39 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 40 41 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ 42 do { \ 43 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ 44 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ 45 } while (0) 46 47 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 48 do { \ 49 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 50 (sge)->length = cpu_to_le32(vlength); \ 51 (sge)->flags = cpu_to_le32(vflags); \ 52 } while (0) 53 54 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 55 do { \ 56 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ 57 (hdr)->num_sges = num_sge; \ 58 } while (0) 59 60 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 61 do { \ 62 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 63 (sge)->length = cpu_to_le32(vlength); \ 64 (sge)->l_key = cpu_to_le32(vlkey); \ 65 } while (0) 66 67 #define NIPQUAD(addr) \ 68 ((unsigned char *)&addr)[0], \ 69 ((unsigned char *)&addr)[1], \ 70 ((unsigned char *)&addr)[2], \ 71 ((unsigned char *)&addr)[3] 72 73 static int 74 qlnxr_check_srq_params(struct ib_pd *ibpd, 75 struct qlnxr_dev *dev, 76 struct ib_srq_init_attr *attrs); 77 78 static int 79 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 80 struct qlnxr_srq *srq, 81 struct qlnxr_create_srq_ureq *ureq, 82 int access, int dmasync); 83 84 static int 85 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 86 struct qlnxr_dev *dev, 87 struct ib_srq_init_attr *init_attr); 88 89 static int 90 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 91 struct qlnxr_srq *srq, 92 struct ib_udata *udata); 93 94 static void 95 qlnxr_free_srq_user_params(struct qlnxr_srq *srq); 96 97 static void 98 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); 99 100 static u32 101 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); 102 103 int 104 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, 105 union ib_gid *sgid) 106 { 107 struct qlnxr_dev *dev; 108 qlnx_host_t *ha; 109 110 dev = get_qlnxr_dev(ibdev); 111 ha = dev->ha; 112 113 QL_DPRINT12(ha, "enter\n"); 114 115 memset(sgid->raw, 0, sizeof(sgid->raw)); 116 117 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); 118 119 QL_DPRINT12(ha, "exit\n"); 120 121 return 0; 122 } 123 124 int 125 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, 126 union ib_gid *sgid) 127 { 128 struct qlnxr_dev *dev; 129 qlnx_host_t *ha; 130 131 dev = get_qlnxr_dev(ibdev); 132 ha = dev->ha; 133 QL_DPRINT12(ha, "enter index: %d\n", index); 134 #if 0 135 int ret = 0; 136 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ 137 //if (!rdma_cap_roce_gid_table(ibdev, port)) { 138 if (!(rdma_protocol_roce(ibdev, port) && 139 ibdev->add_gid && ibdev->del_gid)) { 140 QL_DPRINT11(ha, "acquire gid failed\n"); 141 return -ENODEV; 142 } 143 144 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 145 if (ret == -EAGAIN) { 146 memcpy(sgid, &zgid, sizeof(*sgid)); 147 return 0; 148 } 149 #endif 150 if ((index >= QLNXR_MAX_SGID) || (index < 0)) { 151 QL_DPRINT12(ha, "invalid gid index %d\n", index); 152 memset(sgid, 0, sizeof(*sgid)); 153 return -EINVAL; 154 } 155 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); 156 157 QL_DPRINT12(ha, "exit : %p\n", sgid); 158 159 return 0; 160 } 161 162 struct ib_srq * 163 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr, 164 struct ib_udata *udata) 165 { 166 struct qlnxr_dev *dev; 167 qlnx_host_t *ha; 168 struct ecore_rdma_destroy_srq_in_params destroy_in_params; 169 struct ecore_rdma_create_srq_out_params out_params; 170 struct ecore_rdma_create_srq_in_params in_params; 171 u64 pbl_base_addr, phy_prod_pair_addr; 172 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 173 struct ib_ucontext *ib_ctx = NULL; 174 struct qlnxr_srq_hwq_info *hw_srq; 175 struct qlnxr_ucontext *ctx = NULL; 176 struct qlnxr_create_srq_ureq ureq; 177 u32 page_cnt, page_size; 178 struct qlnxr_srq *srq; 179 int ret = 0; 180 181 dev = get_qlnxr_dev((ibpd->device)); 182 ha = dev->ha; 183 184 QL_DPRINT12(ha, "enter\n"); 185 186 ret = qlnxr_check_srq_params(ibpd, dev, init_attr); 187 188 srq = kzalloc(sizeof(*srq), GFP_KERNEL); 189 if (!srq) { 190 QL_DPRINT11(ha, "cannot allocate memory for srq\n"); 191 return NULL; //@@@ : TODO what to return here? 192 } 193 194 srq->dev = dev; 195 hw_srq = &srq->hw_srq; 196 spin_lock_init(&srq->lock); 197 memset(&in_params, 0, sizeof(in_params)); 198 199 if (udata && ibpd->uobject && ibpd->uobject->context) { 200 ib_ctx = ibpd->uobject->context; 201 ctx = get_qlnxr_ucontext(ib_ctx); 202 203 memset(&ureq, 0, sizeof(ureq)); 204 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 205 udata->inlen))) { 206 QL_DPRINT11(ha, "problem" 207 " copying data from user space\n"); 208 goto err0; 209 } 210 211 ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0); 212 if (ret) 213 goto err0; 214 215 page_cnt = srq->usrq.pbl_info.num_pbes; 216 pbl_base_addr = srq->usrq.pbl_tbl->pa; 217 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 218 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT 219 // page_size = BIT(srq->usrq.umem->page_shift); 220 // else 221 page_size = srq->usrq.umem->page_size; 222 } else { 223 struct ecore_chain *pbl; 224 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); 225 if (ret) 226 goto err0; 227 pbl = &hw_srq->pbl; 228 229 page_cnt = ecore_chain_get_page_cnt(pbl); 230 pbl_base_addr = ecore_chain_get_pbl_phys(pbl); 231 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 232 page_size = pbl->elem_per_page << 4; 233 } 234 235 in_params.pd_id = pd->pd_id; 236 in_params.pbl_base_addr = pbl_base_addr; 237 in_params.prod_pair_addr = phy_prod_pair_addr; 238 in_params.num_pages = page_cnt; 239 in_params.page_size = page_size; 240 241 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 242 if (ret) 243 goto err1; 244 245 srq->srq_id = out_params.srq_id; 246 247 if (udata) { 248 ret = qlnxr_copy_srq_uresp(dev, srq, udata); 249 if (ret) 250 goto err2; 251 } 252 253 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); 254 return &srq->ibsrq; 255 err2: 256 memset(&in_params, 0, sizeof(in_params)); 257 destroy_in_params.srq_id = srq->srq_id; 258 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 259 260 err1: 261 if (udata) 262 qlnxr_free_srq_user_params(srq); 263 else 264 qlnxr_free_srq_kernel_params(srq); 265 266 err0: 267 kfree(srq); 268 return ERR_PTR(-EFAULT); 269 } 270 271 int 272 qlnxr_destroy_srq(struct ib_srq *ibsrq) 273 { 274 struct qlnxr_dev *dev; 275 struct qlnxr_srq *srq; 276 qlnx_host_t *ha; 277 struct ecore_rdma_destroy_srq_in_params in_params; 278 279 srq = get_qlnxr_srq(ibsrq); 280 dev = srq->dev; 281 ha = dev->ha; 282 283 memset(&in_params, 0, sizeof(in_params)); 284 in_params.srq_id = srq->srq_id; 285 286 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); 287 288 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) 289 qlnxr_free_srq_user_params(srq); 290 else 291 qlnxr_free_srq_kernel_params(srq); 292 293 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); 294 kfree(srq); 295 return 0; 296 } 297 298 int 299 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 300 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 301 { 302 struct qlnxr_dev *dev; 303 struct qlnxr_srq *srq; 304 qlnx_host_t *ha; 305 struct ecore_rdma_modify_srq_in_params in_params; 306 int ret = 0; 307 308 srq = get_qlnxr_srq(ibsrq); 309 dev = srq->dev; 310 ha = dev->ha; 311 312 QL_DPRINT12(ha, "enter\n"); 313 if (attr_mask & IB_SRQ_MAX_WR) { 314 QL_DPRINT12(ha, "invalid attribute mask=0x%x" 315 " specified for %p\n", attr_mask, srq); 316 return -EINVAL; 317 } 318 319 if (attr_mask & IB_SRQ_LIMIT) { 320 if (attr->srq_limit >= srq->hw_srq.max_wr) { 321 QL_DPRINT12(ha, "invalid srq_limit=0x%x" 322 " (max_srq_limit = 0x%x)\n", 323 attr->srq_limit, srq->hw_srq.max_wr); 324 return -EINVAL; 325 } 326 memset(&in_params, 0, sizeof(in_params)); 327 in_params.srq_id = srq->srq_id; 328 in_params.wqe_limit = attr->srq_limit; 329 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); 330 if (ret) 331 return ret; 332 } 333 334 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); 335 return 0; 336 } 337 338 int 339 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 340 { 341 struct qlnxr_dev *dev; 342 struct qlnxr_srq *srq; 343 qlnx_host_t *ha; 344 struct ecore_rdma_device *qattr; 345 srq = get_qlnxr_srq(ibsrq); 346 dev = srq->dev; 347 ha = dev->ha; 348 //qattr = &dev->attr; 349 qattr = ecore_rdma_query_device(dev->rdma_ctx); 350 QL_DPRINT12(ha, "enter\n"); 351 352 if (!dev->rdma_ctx) { 353 QL_DPRINT12(ha, "called with invalid params" 354 " rdma_ctx is NULL\n"); 355 return -EINVAL; 356 } 357 358 srq_attr->srq_limit = qattr->max_srq; 359 srq_attr->max_wr = qattr->max_srq_wr; 360 srq_attr->max_sge = qattr->max_sge; 361 362 QL_DPRINT12(ha, "exit\n"); 363 return 0; 364 } 365 366 /* Increment srq wr producer by one */ 367 static 368 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) 369 { 370 info->wr_prod_cnt++; 371 } 372 373 /* Increment srq wr consumer by one */ 374 static 375 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) 376 { 377 info->wr_cons_cnt++; 378 } 379 380 /* get_port_immutable verb is not available in FreeBSD */ 381 #if 0 382 int 383 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 384 struct ib_port_immutable *immutable) 385 { 386 struct qlnxr_dev *dev; 387 qlnx_host_t *ha; 388 dev = get_qlnxr_dev(ibdev); 389 ha = dev->ha; 390 391 QL_DPRINT12(ha, "entered but not implemented!!!\n"); 392 } 393 #endif 394 395 int 396 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 397 struct ib_recv_wr **bad_wr) 398 { 399 struct qlnxr_dev *dev; 400 struct qlnxr_srq *srq; 401 qlnx_host_t *ha; 402 struct qlnxr_srq_hwq_info *hw_srq; 403 struct ecore_chain *pbl; 404 unsigned long flags; 405 int status = 0; 406 u32 num_sge, offset; 407 408 srq = get_qlnxr_srq(ibsrq); 409 dev = srq->dev; 410 ha = dev->ha; 411 hw_srq = &srq->hw_srq; 412 413 QL_DPRINT12(ha, "enter\n"); 414 spin_lock_irqsave(&srq->lock, flags); 415 416 pbl = &srq->hw_srq.pbl; 417 while (wr) { 418 struct rdma_srq_wqe_header *hdr; 419 int i; 420 421 if (!qlnxr_srq_elem_left(hw_srq) || 422 wr->num_sge > srq->hw_srq.max_sges) { 423 QL_DPRINT11(ha, "WR cannot be posted" 424 " (%d, %d) || (%d > %d)\n", 425 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 426 wr->num_sge, srq->hw_srq.max_sges); 427 status = -ENOMEM; 428 *bad_wr = wr; 429 break; 430 } 431 432 hdr = ecore_chain_produce(pbl); 433 num_sge = wr->num_sge; 434 /* Set number of sge and WR id in header */ 435 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 436 437 /* PBL is maintained in case of WR granularity. 438 * So increment WR producer in case we post a WR. 439 */ 440 qlnxr_inc_srq_wr_prod(hw_srq); 441 hw_srq->wqe_prod++; 442 hw_srq->sge_prod++; 443 444 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", 445 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 446 447 for (i = 0; i < wr->num_sge; i++) { 448 struct rdma_srq_sge *srq_sge = 449 ecore_chain_produce(pbl); 450 /* Set SGE length, lkey and address */ 451 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 452 wr->sg_list[i].length, wr->sg_list[i].lkey); 453 454 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", 455 i, srq_sge->length, srq_sge->l_key, 456 srq_sge->addr.hi, srq_sge->addr.lo); 457 hw_srq->sge_prod++; 458 } 459 wmb(); 460 /* 461 * SRQ prod is 8 bytes. Need to update SGE prod in index 462 * in first 4 bytes and need to update WQE prod in next 463 * 4 bytes. 464 */ 465 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; 466 offset = offsetof(struct rdma_srq_producers, wqe_prod); 467 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 468 hw_srq->wqe_prod; 469 /* Flush prod after updating it */ 470 wmb(); 471 wr = wr->next; 472 } 473 474 QL_DPRINT12(ha, "Elements in SRQ: %d\n", 475 ecore_chain_get_elem_left(pbl)); 476 477 spin_unlock_irqrestore(&srq->lock, flags); 478 QL_DPRINT12(ha, "exit\n"); 479 return status; 480 } 481 482 int 483 #if __FreeBSD_version < 1102000 484 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr) 485 #else 486 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 487 struct ib_udata *udata) 488 #endif /* #if __FreeBSD_version < 1102000 */ 489 490 { 491 struct qlnxr_dev *dev; 492 struct ecore_rdma_device *qattr; 493 qlnx_host_t *ha; 494 495 dev = get_qlnxr_dev(ibdev); 496 ha = dev->ha; 497 498 QL_DPRINT12(ha, "enter\n"); 499 500 #if __FreeBSD_version > 1102000 501 if (udata->inlen || udata->outlen) 502 return -EINVAL; 503 #endif /* #if __FreeBSD_version > 1102000 */ 504 505 if (dev->rdma_ctx == NULL) { 506 return -EINVAL; 507 } 508 509 qattr = ecore_rdma_query_device(dev->rdma_ctx); 510 511 memset(attr, 0, sizeof *attr); 512 513 attr->fw_ver = qattr->fw_ver; 514 attr->sys_image_guid = qattr->sys_image_guid; 515 attr->max_mr_size = qattr->max_mr_size; 516 attr->page_size_cap = qattr->page_size_caps; 517 attr->vendor_id = qattr->vendor_id; 518 attr->vendor_part_id = qattr->vendor_part_id; 519 attr->hw_ver = qattr->hw_ver; 520 attr->max_qp = qattr->max_qp; 521 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 522 IB_DEVICE_RC_RNR_NAK_GEN | 523 IB_DEVICE_LOCAL_DMA_LKEY | 524 IB_DEVICE_MEM_MGT_EXTENSIONS; 525 526 attr->max_sge = qattr->max_sge; 527 attr->max_sge_rd = qattr->max_sge; 528 attr->max_cq = qattr->max_cq; 529 attr->max_cqe = qattr->max_cqe; 530 attr->max_mr = qattr->max_mr; 531 attr->max_mw = qattr->max_mw; 532 attr->max_pd = qattr->max_pd; 533 attr->atomic_cap = dev->atomic_cap; 534 attr->max_fmr = qattr->max_fmr; 535 attr->max_map_per_fmr = 16; /* TBD: FMR */ 536 537 /* There is an implicit assumption in some of the ib_xxx apps that the 538 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in 539 * communication the qp_rd_atom is passed to the other side and used as 540 * init_rd_atom without check device capabilities for init_rd_atom. 541 * for this reason, we set the qp_rd_atom to be the minimum between the 542 * two...There is an additional assumption in mlx4 driver that the 543 * values are power of two, fls is performed on the value - 1, which 544 * in fact gives a larger power of two for values which are not a power 545 * of two. This should be fixed in mlx4 driver, but until then -> 546 * we provide a value that is a power of two in our code. 547 */ 548 attr->max_qp_init_rd_atom = 549 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 550 attr->max_qp_rd_atom = 551 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 552 attr->max_qp_init_rd_atom); 553 554 attr->max_srq = qattr->max_srq; 555 attr->max_srq_sge = qattr->max_srq_sge; 556 attr->max_srq_wr = qattr->max_srq_wr; 557 558 /* TODO: R&D to more properly configure the following */ 559 attr->local_ca_ack_delay = qattr->dev_ack_delay; 560 attr->max_fast_reg_page_list_len = qattr->max_mr/8; 561 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; 562 attr->max_ah = qattr->max_ah; 563 564 QL_DPRINT12(ha, "exit\n"); 565 return 0; 566 } 567 568 static inline void 569 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) 570 { 571 switch (speed) { 572 case 1000: 573 *ib_speed = IB_SPEED_SDR; 574 *ib_width = IB_WIDTH_1X; 575 break; 576 case 10000: 577 *ib_speed = IB_SPEED_QDR; 578 *ib_width = IB_WIDTH_1X; 579 break; 580 581 case 20000: 582 *ib_speed = IB_SPEED_DDR; 583 *ib_width = IB_WIDTH_4X; 584 break; 585 586 case 25000: 587 *ib_speed = IB_SPEED_EDR; 588 *ib_width = IB_WIDTH_1X; 589 break; 590 591 case 40000: 592 *ib_speed = IB_SPEED_QDR; 593 *ib_width = IB_WIDTH_4X; 594 break; 595 596 case 50000: 597 *ib_speed = IB_SPEED_QDR; 598 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... 599 break; 600 601 case 100000: 602 *ib_speed = IB_SPEED_EDR; 603 *ib_width = IB_WIDTH_4X; 604 break; 605 606 default: 607 /* Unsupported */ 608 *ib_speed = IB_SPEED_SDR; 609 *ib_width = IB_WIDTH_1X; 610 } 611 return; 612 } 613 614 int 615 qlnxr_query_port(struct ib_device *ibdev, uint8_t port, 616 struct ib_port_attr *attr) 617 { 618 struct qlnxr_dev *dev; 619 struct ecore_rdma_port *rdma_port; 620 qlnx_host_t *ha; 621 622 dev = get_qlnxr_dev(ibdev); 623 ha = dev->ha; 624 625 QL_DPRINT12(ha, "enter\n"); 626 627 if (port > 1) { 628 QL_DPRINT12(ha, "port [%d] > 1 \n", port); 629 return -EINVAL; 630 } 631 632 if (dev->rdma_ctx == NULL) { 633 QL_DPRINT12(ha, "rdma_ctx == NULL\n"); 634 return -EINVAL; 635 } 636 637 rdma_port = ecore_rdma_query_port(dev->rdma_ctx); 638 memset(attr, 0, sizeof *attr); 639 640 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { 641 attr->state = IB_PORT_ACTIVE; 642 attr->phys_state = 5; 643 } else { 644 attr->state = IB_PORT_DOWN; 645 attr->phys_state = 3; 646 } 647 648 attr->max_mtu = IB_MTU_4096; 649 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); 650 attr->lid = 0; 651 attr->lmc = 0; 652 attr->sm_lid = 0; 653 attr->sm_sl = 0; 654 attr->port_cap_flags = 0; 655 656 if (QLNX_IS_IWARP(dev)) { 657 attr->gid_tbl_len = 1; 658 attr->pkey_tbl_len = 1; 659 } else { 660 attr->gid_tbl_len = QLNXR_MAX_SGID; 661 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; 662 } 663 664 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 665 attr->qkey_viol_cntr = 0; 666 667 get_link_speed_and_width(rdma_port->link_speed, 668 &attr->active_speed, &attr->active_width); 669 670 attr->max_msg_sz = rdma_port->max_msg_size; 671 attr->max_vl_num = 4; /* TODO -> figure this one out... */ 672 673 QL_DPRINT12(ha, "state = %d phys_state = %d " 674 " link_speed = %d active_speed = %d active_width = %d" 675 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" 676 " max_msg_sz = 0x%x max_vl_num = 0x%x \n", 677 attr->state, attr->phys_state, 678 rdma_port->link_speed, attr->active_speed, 679 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, 680 attr->max_msg_sz, attr->max_vl_num); 681 682 QL_DPRINT12(ha, "exit\n"); 683 return 0; 684 } 685 686 int 687 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, 688 struct ib_port_modify *props) 689 { 690 struct qlnxr_dev *dev; 691 qlnx_host_t *ha; 692 693 dev = get_qlnxr_dev(ibdev); 694 ha = dev->ha; 695 696 QL_DPRINT12(ha, "enter\n"); 697 698 if (port > 1) { 699 QL_DPRINT12(ha, "port (%d) > 1\n", port); 700 return -EINVAL; 701 } 702 703 QL_DPRINT12(ha, "exit\n"); 704 return 0; 705 } 706 707 enum rdma_link_layer 708 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) 709 { 710 struct qlnxr_dev *dev; 711 qlnx_host_t *ha; 712 713 dev = get_qlnxr_dev(ibdev); 714 ha = dev->ha; 715 716 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); 717 718 return IB_LINK_LAYER_ETHERNET; 719 } 720 721 struct ib_pd * 722 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, 723 struct ib_udata *udata) 724 { 725 struct qlnxr_pd *pd = NULL; 726 u16 pd_id; 727 int rc; 728 struct qlnxr_dev *dev; 729 qlnx_host_t *ha; 730 731 dev = get_qlnxr_dev(ibdev); 732 ha = dev->ha; 733 734 QL_DPRINT12(ha, "ibdev = %p context = %p" 735 " udata = %p enter\n", ibdev, context, udata); 736 737 if (dev->rdma_ctx == NULL) { 738 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); 739 rc = -1; 740 goto err; 741 } 742 743 pd = kzalloc(sizeof(*pd), GFP_KERNEL); 744 if (!pd) { 745 rc = -ENOMEM; 746 QL_DPRINT11(ha, "kzalloc(pd) = NULL\n"); 747 goto err; 748 } 749 750 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); 751 if (rc) { 752 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); 753 goto err; 754 } 755 756 pd->pd_id = pd_id; 757 758 if (udata && context) { 759 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); 760 if (rc) { 761 QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); 762 ecore_rdma_free_pd(dev->rdma_ctx, pd_id); 763 goto err; 764 } 765 766 pd->uctx = get_qlnxr_ucontext(context); 767 pd->uctx->pd = pd; 768 } 769 770 atomic_add_rel_32(&dev->pd_count, 1); 771 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 772 pd, pd_id, dev->pd_count); 773 774 return &pd->ibpd; 775 776 err: 777 kfree(pd); 778 QL_DPRINT12(ha, "exit -1\n"); 779 return ERR_PTR(rc); 780 } 781 782 int 783 qlnxr_dealloc_pd(struct ib_pd *ibpd) 784 { 785 struct qlnxr_pd *pd; 786 struct qlnxr_dev *dev; 787 qlnx_host_t *ha; 788 789 pd = get_qlnxr_pd(ibpd); 790 dev = get_qlnxr_dev((ibpd->device)); 791 ha = dev->ha; 792 793 QL_DPRINT12(ha, "enter\n"); 794 795 if (pd == NULL) { 796 QL_DPRINT11(ha, "pd = NULL\n"); 797 } else { 798 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); 799 kfree(pd); 800 atomic_subtract_rel_32(&dev->pd_count, 1); 801 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 802 pd, pd->pd_id, dev->pd_count); 803 } 804 805 QL_DPRINT12(ha, "exit\n"); 806 return 0; 807 } 808 809 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) 810 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ 811 /* Should be part of HSI */ 812 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ 813 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 814 815 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); 816 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); 817 818 int 819 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 820 { 821 struct qlnxr_dev *dev; 822 qlnx_host_t *ha; 823 824 dev = get_qlnxr_dev(ibdev); 825 ha = dev->ha; 826 827 QL_DPRINT12(ha, "enter index = 0x%x\n", index); 828 829 if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 830 return -EINVAL; 831 832 *pkey = QLNXR_ROCE_PKEY_DEFAULT; 833 834 QL_DPRINT12(ha, "exit\n"); 835 return 0; 836 } 837 838 static inline bool 839 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, 840 u16 *vlan_id) 841 { 842 bool ret = false; 843 844 QL_DPRINT12(ha, "enter \n"); 845 846 *vlan_id = 0; 847 848 #if __FreeBSD_version >= 1100000 849 u16 tmp_vlan_id; 850 851 #if __FreeBSD_version >= 1102000 852 union ib_gid *dgid; 853 854 dgid = &attr->ah_attr.grh.dgid; 855 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; 856 857 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { 858 *vlan_id = tmp_vlan_id; 859 ret = true; 860 } 861 #else 862 tmp_vlan_id = attr->vlan_id; 863 864 if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) { 865 *vlan_id = tmp_vlan_id; 866 ret = true; 867 } 868 869 #endif /* #if __FreeBSD_version > 1102000 */ 870 871 #else 872 ret = true; 873 874 #endif /* #if __FreeBSD_version >= 1100000 */ 875 876 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); 877 878 return (ret); 879 } 880 881 static inline void 882 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, 883 int attr_mask, 884 struct qlnxr_dev *dev, 885 struct qlnxr_qp *qp, 886 struct ecore_rdma_modify_qp_in_params *qp_params) 887 { 888 int i; 889 qlnx_host_t *ha; 890 891 ha = dev->ha; 892 893 QL_DPRINT12(ha, "enter\n"); 894 895 memcpy(&qp_params->sgid.bytes[0], 896 &dev->sgid_tbl[qp->sgid_idx].raw[0], 897 sizeof(qp_params->sgid.bytes)); 898 memcpy(&qp_params->dgid.bytes[0], 899 &attr->ah_attr.grh.dgid.raw[0], 900 sizeof(qp_params->dgid)); 901 902 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); 903 904 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { 905 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 906 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 907 } 908 909 QL_DPRINT12(ha, "exit\n"); 910 return; 911 } 912 913 static int 914 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 915 { 916 struct qlnxr_mm *mm; 917 qlnx_host_t *ha; 918 919 ha = uctx->dev->ha; 920 921 QL_DPRINT12(ha, "enter\n"); 922 923 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 924 if (mm == NULL) { 925 QL_DPRINT11(ha, "mm = NULL\n"); 926 return -ENOMEM; 927 } 928 929 mm->key.phy_addr = phy_addr; 930 931 /* This function might be called with a length which is not a multiple 932 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 933 * forces this granularity by increasing the requested size if needed. 934 * When qedr_mmap is called, it will search the list with the updated 935 * length as a key. To prevent search failures, the length is rounded up 936 * in advance to PAGE_SIZE. 937 */ 938 mm->key.len = roundup(len, PAGE_SIZE); 939 INIT_LIST_HEAD(&mm->entry); 940 941 mutex_lock(&uctx->mm_list_lock); 942 list_add(&mm->entry, &uctx->mm_head); 943 mutex_unlock(&uctx->mm_list_lock); 944 945 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 946 (unsigned long long)mm->key.phy_addr, 947 (unsigned long)mm->key.len, uctx); 948 949 return 0; 950 } 951 952 static bool 953 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 954 { 955 bool found = false; 956 struct qlnxr_mm *mm; 957 qlnx_host_t *ha; 958 959 ha = uctx->dev->ha; 960 961 QL_DPRINT12(ha, "enter\n"); 962 963 mutex_lock(&uctx->mm_list_lock); 964 list_for_each_entry(mm, &uctx->mm_head, entry) { 965 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 966 continue; 967 968 found = true; 969 break; 970 } 971 mutex_unlock(&uctx->mm_list_lock); 972 973 QL_DPRINT12(ha, 974 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", 975 mm->key.phy_addr, mm->key.len, uctx, found); 976 977 return found; 978 } 979 980 struct 981 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev, 982 struct ib_udata *udata) 983 { 984 int rc; 985 struct qlnxr_ucontext *ctx; 986 struct qlnxr_alloc_ucontext_resp uresp; 987 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 988 qlnx_host_t *ha = dev->ha; 989 struct ecore_rdma_add_user_out_params oparams; 990 991 if (!udata) { 992 return ERR_PTR(-EFAULT); 993 } 994 995 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 996 if (!ctx) 997 return ERR_PTR(-ENOMEM); 998 999 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); 1000 if (rc) { 1001 QL_DPRINT12(ha, 1002 "Failed to allocate a DPI for a new RoCE application " 1003 ",rc = %d. To overcome this, consider to increase " 1004 "the number of DPIs, increase the doorbell BAR size " 1005 "or just close unnecessary RoCE applications. In " 1006 "order to increase the number of DPIs consult the " 1007 "README\n", rc); 1008 goto err; 1009 } 1010 1011 ctx->dpi = oparams.dpi; 1012 ctx->dpi_addr = oparams.dpi_addr; 1013 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 1014 ctx->dpi_size = oparams.dpi_size; 1015 INIT_LIST_HEAD(&ctx->mm_head); 1016 mutex_init(&ctx->mm_list_lock); 1017 1018 memset(&uresp, 0, sizeof(uresp)); 1019 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) 1020 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out 1021 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) 1022 < udata->outlen ? 1 : 0; //TODO: figure this out 1023 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) 1024 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 1025 uresp.db_pa = ctx->dpi_phys_addr; 1026 uresp.db_size = ctx->dpi_size; 1027 uresp.max_send_wr = dev->attr.max_sqe; 1028 uresp.max_recv_wr = dev->attr.max_rqe; 1029 uresp.max_srq_wr = dev->attr.max_srq_wr; 1030 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 1031 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 1032 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 1033 uresp.max_cqes = QLNXR_MAX_CQES; 1034 1035 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1036 if (rc) 1037 goto err; 1038 1039 ctx->dev = dev; 1040 1041 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 1042 if (rc) 1043 goto err; 1044 QL_DPRINT12(ha, "Allocated user context %p\n", 1045 &ctx->ibucontext); 1046 1047 return &ctx->ibucontext; 1048 err: 1049 kfree(ctx); 1050 return ERR_PTR(rc); 1051 } 1052 1053 int 1054 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) 1055 { 1056 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); 1057 struct qlnxr_dev *dev = uctx->dev; 1058 qlnx_host_t *ha = dev->ha; 1059 struct qlnxr_mm *mm, *tmp; 1060 int status = 0; 1061 1062 QL_DPRINT12(ha, "Deallocating user context %p\n", 1063 uctx); 1064 1065 if (dev) { 1066 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 1067 } 1068 1069 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 1070 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" 1071 " ctx=%p\n", 1072 mm->key.phy_addr, mm->key.len, uctx); 1073 list_del(&mm->entry); 1074 kfree(mm); 1075 } 1076 kfree(uctx); 1077 return status; 1078 } 1079 1080 int 1081 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 1082 { 1083 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); 1084 struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); 1085 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 1086 u64 unmapped_db; 1087 unsigned long len = (vma->vm_end - vma->vm_start); 1088 int rc = 0; 1089 bool found; 1090 qlnx_host_t *ha; 1091 1092 ha = dev->ha; 1093 1094 #if __FreeBSD_version > 1102000 1095 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 1096 #else 1097 unmapped_db = dev->db_phys_addr; 1098 #endif /* #if __FreeBSD_version > 1102000 */ 1099 1100 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" 1101 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 1102 vm_page, vma->vm_pgoff, unmapped_db, 1103 dev->db_size, len); 1104 1105 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 1106 QL_DPRINT11(ha, "Vma_start not page aligned " 1107 "vm_start = %ld vma_end = %ld\n", vma->vm_start, 1108 vma->vm_end); 1109 return -EINVAL; 1110 } 1111 1112 found = qlnxr_search_mmap(ucontext, vm_page, len); 1113 if (!found) { 1114 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", 1115 vma->vm_pgoff); 1116 return -EINVAL; 1117 } 1118 1119 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1120 1121 #if __FreeBSD_version > 1102000 1122 1123 if ((vm_page < unmapped_db) || 1124 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { 1125 QL_DPRINT11(ha, "failed pages are outside of dpi;" 1126 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", 1127 vm_page, unmapped_db, ucontext->dpi_size); 1128 return -EINVAL; 1129 } 1130 1131 if (vma->vm_flags & VM_READ) { 1132 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); 1133 return -EINVAL; 1134 } 1135 1136 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1137 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 1138 vma->vm_page_prot); 1139 1140 #else 1141 1142 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 1143 dev->db_size))) { 1144 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1145 1146 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1147 1148 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1149 PAGE_SIZE, vma->vm_page_prot); 1150 } else { 1151 QL_DPRINT12(ha, "Mapping chains\n"); 1152 rc = io_remap_pfn_range(vma, vma->vm_start, 1153 vma->vm_pgoff, len, vma->vm_page_prot); 1154 } 1155 1156 #endif /* #if __FreeBSD_version > 1102000 */ 1157 1158 QL_DPRINT12(ha, "exit [%d]\n", rc); 1159 return rc; 1160 } 1161 1162 struct ib_mr * 1163 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) 1164 { 1165 struct qlnxr_mr *mr; 1166 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1167 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 1168 int rc; 1169 qlnx_host_t *ha; 1170 1171 ha = dev->ha; 1172 1173 QL_DPRINT12(ha, "enter\n"); 1174 1175 if (acc & IB_ACCESS_MW_BIND) { 1176 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); 1177 } 1178 1179 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1180 if (!mr) { 1181 rc = -ENOMEM; 1182 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); 1183 goto err0; 1184 } 1185 1186 mr->type = QLNXR_MR_DMA; 1187 1188 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1189 if (rc) { 1190 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); 1191 goto err1; 1192 } 1193 1194 /* index only, 18 bit long, lkey = itid << 8 | key */ 1195 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1196 mr->hw_mr.pd = pd->pd_id; 1197 mr->hw_mr.local_read = 1; 1198 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1199 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1200 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1201 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1202 mr->hw_mr.dma_mr = true; 1203 1204 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1205 if (rc) { 1206 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); 1207 goto err2; 1208 } 1209 1210 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1211 1212 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1213 mr->hw_mr.remote_atomic) { 1214 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1215 } 1216 1217 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); 1218 1219 return &mr->ibmr; 1220 1221 err2: 1222 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1223 err1: 1224 kfree(mr); 1225 err0: 1226 QL_DPRINT12(ha, "exit [%d]\n", rc); 1227 1228 return ERR_PTR(rc); 1229 } 1230 1231 static void 1232 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, 1233 struct qlnxr_pbl *pbl) 1234 { 1235 int i; 1236 qlnx_host_t *ha; 1237 1238 ha = dev->ha; 1239 1240 QL_DPRINT12(ha, "enter\n"); 1241 1242 for (i = 0; i < pbl_info->num_pbls; i++) { 1243 if (!pbl[i].va) 1244 continue; 1245 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, 1246 pbl_info->pbl_size); 1247 } 1248 kfree(pbl); 1249 1250 QL_DPRINT12(ha, "exit\n"); 1251 return; 1252 } 1253 1254 #define MIN_FW_PBL_PAGE_SIZE (4*1024) 1255 #define MAX_FW_PBL_PAGE_SIZE (64*1024) 1256 1257 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 1258 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 1259 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) 1260 1261 static struct qlnxr_pbl * 1262 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, 1263 struct qlnxr_pbl_info *pbl_info, gfp_t flags) 1264 { 1265 void *va; 1266 dma_addr_t pa; 1267 dma_addr_t *pbl_main_tbl; 1268 struct qlnxr_pbl *pbl_table; 1269 int i, rc = 0; 1270 qlnx_host_t *ha; 1271 1272 ha = dev->ha; 1273 1274 QL_DPRINT12(ha, "enter\n"); 1275 1276 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); 1277 1278 if (!pbl_table) { 1279 QL_DPRINT12(ha, "pbl_table = NULL\n"); 1280 return NULL; 1281 } 1282 1283 for (i = 0; i < pbl_info->num_pbls; i++) { 1284 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); 1285 if (!va) { 1286 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); 1287 rc = -ENOMEM; 1288 goto err; 1289 } 1290 memset(va, 0, pbl_info->pbl_size); 1291 pbl_table[i].va = va; 1292 pbl_table[i].pa = pa; 1293 } 1294 1295 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 1296 * the first one with physical pointers to all of the rest 1297 */ 1298 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 1299 for (i = 0; i < pbl_info->num_pbls - 1; i++) 1300 pbl_main_tbl[i] = pbl_table[i + 1].pa; 1301 1302 QL_DPRINT12(ha, "exit\n"); 1303 return pbl_table; 1304 1305 err: 1306 qlnxr_free_pbl(dev, pbl_info, pbl_table); 1307 1308 QL_DPRINT12(ha, "exit with error\n"); 1309 return NULL; 1310 } 1311 1312 static int 1313 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, 1314 struct qlnxr_pbl_info *pbl_info, 1315 u32 num_pbes, 1316 int two_layer_capable) 1317 { 1318 u32 pbl_capacity; 1319 u32 pbl_size; 1320 u32 num_pbls; 1321 qlnx_host_t *ha; 1322 1323 ha = dev->ha; 1324 1325 QL_DPRINT12(ha, "enter\n"); 1326 1327 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 1328 if (num_pbes > MAX_PBES_TWO_LAYER) { 1329 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", 1330 num_pbes); 1331 return -EINVAL; 1332 } 1333 1334 /* calculate required pbl page size */ 1335 pbl_size = MIN_FW_PBL_PAGE_SIZE; 1336 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 1337 NUM_PBES_ON_PAGE(pbl_size); 1338 1339 while (pbl_capacity < num_pbes) { 1340 pbl_size *= 2; 1341 pbl_capacity = pbl_size / sizeof(u64); 1342 pbl_capacity = pbl_capacity * pbl_capacity; 1343 } 1344 1345 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 1346 num_pbls++; /* One for the layer0 ( points to the pbls) */ 1347 pbl_info->two_layered = true; 1348 } else { 1349 /* One layered PBL */ 1350 num_pbls = 1; 1351 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ 1352 roundup_pow_of_two((num_pbes * sizeof(u64)))); 1353 pbl_info->two_layered = false; 1354 } 1355 1356 pbl_info->num_pbls = num_pbls; 1357 pbl_info->pbl_size = pbl_size; 1358 pbl_info->num_pbes = num_pbes; 1359 1360 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", 1361 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 1362 1363 return 0; 1364 } 1365 1366 static void 1367 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, 1368 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 1369 { 1370 struct regpair *pbe; 1371 struct qlnxr_pbl *pbl_tbl; 1372 struct scatterlist *sg; 1373 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 1374 qlnx_host_t *ha; 1375 1376 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1377 int i; 1378 struct ib_umem_chunk *chunk = NULL; 1379 #else 1380 int entry; 1381 #endif 1382 1383 ha = dev->ha; 1384 1385 QL_DPRINT12(ha, "enter\n"); 1386 1387 if (!pbl_info) { 1388 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 1389 return; 1390 } 1391 1392 if (!pbl_info->num_pbes) { 1393 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 1394 return; 1395 } 1396 1397 /* If we have a two layered pbl, the first pbl points to the rest 1398 * of the pbls and the first entry lays on the second pbl in the table 1399 */ 1400 if (pbl_info->two_layered) 1401 pbl_tbl = &pbl[1]; 1402 else 1403 pbl_tbl = pbl; 1404 1405 pbe = (struct regpair *)pbl_tbl->va; 1406 if (!pbe) { 1407 QL_DPRINT12(ha, "pbe is NULL\n"); 1408 return; 1409 } 1410 1411 pbe_cnt = 0; 1412 1413 shift = ilog2(umem->page_size); 1414 1415 #ifndef DEFINE_IB_UMEM_WITH_CHUNK 1416 1417 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 1418 #else 1419 list_for_each_entry(chunk, &umem->chunk_list, list) { 1420 /* get all the dma regions from the chunk. */ 1421 for (i = 0; i < chunk->nmap; i++) { 1422 sg = &chunk->page_list[i]; 1423 #endif 1424 pages = sg_dma_len(sg) >> shift; 1425 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 1426 /* store the page address in pbe */ 1427 pbe->lo = 1428 cpu_to_le32(sg_dma_address(sg) + 1429 (umem->page_size * pg_cnt)); 1430 pbe->hi = 1431 cpu_to_le32(upper_32_bits 1432 ((sg_dma_address(sg) + 1433 umem->page_size * pg_cnt))); 1434 1435 QL_DPRINT12(ha, 1436 "Populate pbl table:" 1437 " pbe->addr=0x%x:0x%x " 1438 " pbe_cnt = %d total_num_pbes=%d" 1439 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 1440 total_num_pbes, pbe); 1441 1442 pbe_cnt ++; 1443 total_num_pbes ++; 1444 pbe++; 1445 1446 if (total_num_pbes == pbl_info->num_pbes) 1447 return; 1448 1449 /* if the given pbl is full storing the pbes, 1450 * move to next pbl. 1451 */ 1452 if (pbe_cnt == 1453 (pbl_info->pbl_size / sizeof(u64))) { 1454 pbl_tbl++; 1455 pbe = (struct regpair *)pbl_tbl->va; 1456 pbe_cnt = 0; 1457 } 1458 } 1459 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 1460 } 1461 #endif 1462 } 1463 QL_DPRINT12(ha, "exit\n"); 1464 return; 1465 } 1466 1467 static void 1468 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) 1469 { 1470 struct qlnxr_pbl *pbl, *tmp; 1471 qlnx_host_t *ha; 1472 1473 ha = dev->ha; 1474 1475 QL_DPRINT12(ha, "enter\n"); 1476 1477 if (info->pbl_table) 1478 list_add_tail(&info->pbl_table->list_entry, 1479 &info->free_pbl_list); 1480 1481 if (!list_empty(&info->inuse_pbl_list)) 1482 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 1483 1484 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 1485 list_del(&pbl->list_entry); 1486 qlnxr_free_pbl(dev, &info->pbl_info, pbl); 1487 } 1488 QL_DPRINT12(ha, "exit\n"); 1489 1490 return; 1491 } 1492 1493 static int 1494 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, 1495 size_t page_list_len, bool two_layered) 1496 { 1497 int rc; 1498 struct qlnxr_pbl *tmp; 1499 qlnx_host_t *ha; 1500 1501 ha = dev->ha; 1502 1503 QL_DPRINT12(ha, "enter\n"); 1504 1505 INIT_LIST_HEAD(&info->free_pbl_list); 1506 INIT_LIST_HEAD(&info->inuse_pbl_list); 1507 1508 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, 1509 page_list_len, two_layered); 1510 if (rc) { 1511 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); 1512 goto done; 1513 } 1514 1515 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1516 1517 if (!info->pbl_table) { 1518 rc = -ENOMEM; 1519 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); 1520 goto done; 1521 } 1522 1523 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); 1524 1525 /* in usual case we use 2 PBLs, so we add one to free 1526 * list and allocating another one 1527 */ 1528 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1529 1530 if (!tmp) { 1531 QL_DPRINT11(ha, "Extra PBL is not allocated\n"); 1532 goto done; /* it's OK if second allocation fails, so rc = 0*/ 1533 } 1534 1535 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 1536 1537 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); 1538 1539 done: 1540 if (rc) 1541 free_mr_info(dev, info); 1542 1543 QL_DPRINT12(ha, "exit [%d]\n", rc); 1544 1545 return rc; 1546 } 1547 1548 struct ib_mr * 1549 #if __FreeBSD_version >= 1102000 1550 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1551 u64 usr_addr, int acc, struct ib_udata *udata) 1552 #else 1553 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1554 u64 usr_addr, int acc, struct ib_udata *udata, int mr_id) 1555 #endif /* #if __FreeBSD_version >= 1102000 */ 1556 { 1557 int rc = -ENOMEM; 1558 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1559 struct qlnxr_mr *mr; 1560 struct qlnxr_pd *pd; 1561 qlnx_host_t *ha; 1562 1563 ha = dev->ha; 1564 1565 QL_DPRINT12(ha, "enter\n"); 1566 1567 pd = get_qlnxr_pd(ibpd); 1568 1569 QL_DPRINT12(ha, "qedr_register user mr pd = %d" 1570 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 1571 pd->pd_id, start, len, usr_addr, acc); 1572 1573 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 1574 QL_DPRINT11(ha, 1575 "(acc & IB_ACCESS_REMOTE_WRITE &&" 1576 " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 1577 return ERR_PTR(-EINVAL); 1578 } 1579 1580 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1581 if (!mr) { 1582 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 1583 return ERR_PTR(rc); 1584 } 1585 1586 mr->type = QLNXR_MR_USER; 1587 1588 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 1589 if (IS_ERR(mr->umem)) { 1590 rc = -EFAULT; 1591 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); 1592 goto err0; 1593 } 1594 1595 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 1596 if (rc) { 1597 QL_DPRINT11(ha, 1598 "qlnxr_init_mr_info failed [%d]\n", rc); 1599 goto err1; 1600 } 1601 1602 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 1603 &mr->info.pbl_info); 1604 1605 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1606 1607 if (rc) { 1608 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 1609 goto err1; 1610 } 1611 1612 /* index only, 18 bit long, lkey = itid << 8 | key */ 1613 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1614 mr->hw_mr.key = 0; 1615 mr->hw_mr.pd = pd->pd_id; 1616 mr->hw_mr.local_read = 1; 1617 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1618 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1619 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1620 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1621 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 1622 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 1623 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 1624 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 1625 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ 1626 1627 #if __FreeBSD_version >= 1102000 1628 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 1629 #else 1630 mr->hw_mr.fbo = mr->umem->offset; 1631 #endif 1632 mr->hw_mr.length = len; 1633 mr->hw_mr.vaddr = usr_addr; 1634 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 1635 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 1636 mr->hw_mr.dma_mr = false; 1637 1638 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1639 if (rc) { 1640 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 1641 goto err2; 1642 } 1643 1644 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1645 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1646 mr->hw_mr.remote_atomic) 1647 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1648 1649 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); 1650 1651 return (&mr->ibmr); 1652 1653 err2: 1654 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1655 err1: 1656 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1657 err0: 1658 kfree(mr); 1659 1660 QL_DPRINT12(ha, "exit [%d]\n", rc); 1661 return (ERR_PTR(rc)); 1662 } 1663 1664 int 1665 qlnxr_dereg_mr(struct ib_mr *ib_mr) 1666 { 1667 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); 1668 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); 1669 int rc = 0; 1670 qlnx_host_t *ha; 1671 1672 ha = dev->ha; 1673 1674 QL_DPRINT12(ha, "enter\n"); 1675 1676 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) 1677 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1678 1679 /* it could be user registered memory. */ 1680 if (mr->umem) 1681 ib_umem_release(mr->umem); 1682 1683 kfree(mr->pages); 1684 1685 kfree(mr); 1686 1687 QL_DPRINT12(ha, "exit\n"); 1688 return rc; 1689 } 1690 1691 static int 1692 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, 1693 struct qlnxr_cq *cq, struct ib_udata *udata) 1694 { 1695 struct qlnxr_create_cq_uresp uresp; 1696 int rc; 1697 qlnx_host_t *ha; 1698 1699 ha = dev->ha; 1700 1701 QL_DPRINT12(ha, "enter\n"); 1702 1703 memset(&uresp, 0, sizeof(uresp)); 1704 1705 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1706 uresp.icid = cq->icid; 1707 1708 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1709 1710 if (rc) { 1711 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", 1712 cq->icid, rc); 1713 } 1714 1715 QL_DPRINT12(ha, "exit [%d]\n", rc); 1716 return rc; 1717 } 1718 1719 static void 1720 consume_cqe(struct qlnxr_cq *cq) 1721 { 1722 1723 if (cq->latest_cqe == cq->toggle_cqe) 1724 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1725 1726 cq->latest_cqe = ecore_chain_consume(&cq->pbl); 1727 } 1728 1729 static inline int 1730 qlnxr_align_cq_entries(int entries) 1731 { 1732 u64 size, aligned_size; 1733 1734 /* We allocate an extra entry that we don't report to the FW. 1735 * Why? 1736 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes 1737 * (for performance purposes). Allocating an extra entry and telling 1738 * the FW we have less prevents overwriting the first entry in case of 1739 * a wrap i.e. when the FW writes the last entry and the application 1740 * hasn't read the first one. 1741 */ 1742 size = (entries + 1) * QLNXR_CQE_SIZE; 1743 1744 /* We align to PAGE_SIZE. 1745 * Why? 1746 * Since the CQ is going to be mapped and the mapping is anyhow in whole 1747 * kernel pages we benefit from the possibly extra CQEs. 1748 */ 1749 aligned_size = ALIGN(size, PAGE_SIZE); 1750 1751 /* note: for CQs created in user space the result of this function 1752 * should match the size mapped in user space 1753 */ 1754 return (aligned_size / QLNXR_CQE_SIZE); 1755 } 1756 1757 static inline int 1758 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, 1759 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, 1760 int access, int dmasync, int alloc_and_init) 1761 { 1762 int page_cnt; 1763 int rc; 1764 qlnx_host_t *ha; 1765 1766 ha = dev->ha; 1767 1768 QL_DPRINT12(ha, "enter\n"); 1769 1770 q->buf_addr = buf_addr; 1771 q->buf_len = buf_len; 1772 1773 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" 1774 " dmasync : %x\n", q->buf_addr, q->buf_len, 1775 access, dmasync); 1776 1777 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 1778 1779 if (IS_ERR(q->umem)) { 1780 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); 1781 return PTR_ERR(q->umem); 1782 } 1783 1784 page_cnt = ib_umem_page_count(q->umem); 1785 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 1786 0 /* SQ and RQ don't support dual layer pbl. 1787 * CQ may, but this is yet uncoded. 1788 */); 1789 if (rc) { 1790 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); 1791 goto err; 1792 } 1793 1794 if (alloc_and_init) { 1795 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 1796 1797 if (!q->pbl_tbl) { 1798 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1799 rc = -ENOMEM; 1800 goto err; 1801 } 1802 1803 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); 1804 } else { 1805 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 1806 1807 if (!q->pbl_tbl) { 1808 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1809 rc = -ENOMEM; 1810 goto err; 1811 } 1812 } 1813 1814 QL_DPRINT12(ha, "exit\n"); 1815 return 0; 1816 1817 err: 1818 ib_umem_release(q->umem); 1819 q->umem = NULL; 1820 1821 QL_DPRINT12(ha, "exit [%d]\n", rc); 1822 return rc; 1823 } 1824 1825 #if __FreeBSD_version >= 1102000 1826 1827 struct ib_cq * 1828 qlnxr_create_cq(struct ib_device *ibdev, 1829 const struct ib_cq_init_attr *attr, 1830 struct ib_ucontext *ib_ctx, 1831 struct ib_udata *udata) 1832 1833 #else 1834 1835 #if __FreeBSD_version >= 1100000 1836 1837 struct ib_cq * 1838 qlnxr_create_cq(struct ib_device *ibdev, 1839 struct ib_cq_init_attr *attr, 1840 struct ib_ucontext *ib_ctx, 1841 struct ib_udata *udata) 1842 1843 #else 1844 1845 struct ib_cq * 1846 qlnxr_create_cq(struct ib_device *ibdev, 1847 int entries, 1848 int vector, 1849 struct ib_ucontext *ib_ctx, 1850 struct ib_udata *udata) 1851 #endif /* #if __FreeBSD_version >= 1100000 */ 1852 1853 #endif /* #if __FreeBSD_version >= 1102000 */ 1854 { 1855 struct qlnxr_ucontext *ctx; 1856 struct ecore_rdma_destroy_cq_out_params destroy_oparams; 1857 struct ecore_rdma_destroy_cq_in_params destroy_iparams; 1858 struct qlnxr_dev *dev; 1859 struct ecore_rdma_create_cq_in_params params; 1860 struct qlnxr_create_cq_ureq ureq; 1861 1862 #if __FreeBSD_version >= 1100000 1863 int vector = attr->comp_vector; 1864 int entries = attr->cqe; 1865 #endif 1866 struct qlnxr_cq *cq; 1867 int chain_entries, rc, page_cnt; 1868 u64 pbl_ptr; 1869 u16 icid; 1870 qlnx_host_t *ha; 1871 1872 dev = get_qlnxr_dev(ibdev); 1873 ha = dev->ha; 1874 1875 QL_DPRINT12(ha, "called from %s. entries = %d, " 1876 "vector = %d\n", 1877 (udata ? "User Lib" : "Kernel"), entries, vector); 1878 1879 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); 1880 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); 1881 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); 1882 1883 if (entries > QLNXR_MAX_CQES) { 1884 QL_DPRINT11(ha, 1885 "the number of entries %d is too high. " 1886 "Must be equal or below %d.\n", 1887 entries, QLNXR_MAX_CQES); 1888 return ERR_PTR(-EINVAL); 1889 } 1890 chain_entries = qlnxr_align_cq_entries(entries); 1891 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); 1892 1893 cq = qlnx_zalloc((sizeof(struct qlnxr_cq))); 1894 1895 if (!cq) 1896 return ERR_PTR(-ENOMEM); 1897 1898 if (udata) { 1899 memset(&ureq, 0, sizeof(ureq)); 1900 1901 if (ib_copy_from_udata(&ureq, udata, 1902 min(sizeof(ureq), udata->inlen))) { 1903 QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); 1904 goto err0; 1905 } 1906 1907 if (!ureq.len) { 1908 QL_DPRINT11(ha, "ureq.len == 0\n"); 1909 goto err0; 1910 } 1911 1912 cq->cq_type = QLNXR_CQ_TYPE_USER; 1913 1914 qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len, 1915 IB_ACCESS_LOCAL_WRITE, 1, 1); 1916 1917 pbl_ptr = cq->q.pbl_tbl->pa; 1918 page_cnt = cq->q.pbl_info.num_pbes; 1919 cq->ibcq.cqe = chain_entries; 1920 } else { 1921 cq->cq_type = QLNXR_CQ_TYPE_KERNEL; 1922 1923 rc = ecore_chain_alloc(&dev->ha->cdev, 1924 ECORE_CHAIN_USE_TO_CONSUME, 1925 ECORE_CHAIN_MODE_PBL, 1926 ECORE_CHAIN_CNT_TYPE_U32, 1927 chain_entries, 1928 sizeof(union roce_cqe), 1929 &cq->pbl, NULL); 1930 1931 if (rc) 1932 goto err1; 1933 1934 page_cnt = ecore_chain_get_page_cnt(&cq->pbl); 1935 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); 1936 cq->ibcq.cqe = cq->pbl.capacity; 1937 } 1938 1939 params.cq_handle_hi = upper_32_bits((uintptr_t)cq); 1940 params.cq_handle_lo = lower_32_bits((uintptr_t)cq); 1941 params.cnq_id = vector; 1942 params.cq_size = chain_entries - 1; 1943 params.pbl_num_pages = page_cnt; 1944 params.pbl_ptr = pbl_ptr; 1945 params.pbl_two_level = 0; 1946 1947 if (ib_ctx != NULL) { 1948 ctx = get_qlnxr_ucontext(ib_ctx); 1949 params.dpi = ctx->dpi; 1950 } else { 1951 params.dpi = dev->dpi; 1952 } 1953 1954 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 1955 if (rc) 1956 goto err2; 1957 1958 cq->icid = icid; 1959 cq->sig = QLNXR_CQ_MAGIC_NUMBER; 1960 spin_lock_init(&cq->cq_lock); 1961 1962 if (ib_ctx) { 1963 rc = qlnxr_copy_cq_uresp(dev, cq, udata); 1964 if (rc) 1965 goto err3; 1966 } else { 1967 /* Generate doorbell address. 1968 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. 1969 * TODO: consider moving to device scope as it is a function of 1970 * the device. 1971 * TODO: add ifdef if plan to support 16 bit. 1972 */ 1973 cq->db_addr = dev->db_addr + 1974 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1975 cq->db.data.icid = cq->icid; 1976 cq->db.data.params = DB_AGG_CMD_SET << 1977 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1978 1979 /* point to the very last element, passing it we will toggle */ 1980 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); 1981 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1982 1983 /* must be different from pbl_toggle */ 1984 cq->latest_cqe = NULL; 1985 consume_cqe(cq); 1986 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 1987 } 1988 1989 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," 1990 " number of entries = 0x%x\n", 1991 cq->icid, cq, params.cq_size); 1992 QL_DPRINT12(ha,"cq_addr = %p\n", cq); 1993 return &cq->ibcq; 1994 1995 err3: 1996 destroy_iparams.icid = cq->icid; 1997 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); 1998 err2: 1999 if (udata) 2000 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2001 else 2002 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2003 err1: 2004 if (udata) 2005 ib_umem_release(cq->q.umem); 2006 err0: 2007 kfree(cq); 2008 2009 QL_DPRINT12(ha, "exit error\n"); 2010 2011 return ERR_PTR(-EINVAL); 2012 } 2013 2014 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 2015 { 2016 int status = 0; 2017 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2018 qlnx_host_t *ha; 2019 2020 ha = dev->ha; 2021 2022 QL_DPRINT12(ha, "enter/exit\n"); 2023 2024 return status; 2025 } 2026 2027 int 2028 qlnxr_destroy_cq(struct ib_cq *ibcq) 2029 { 2030 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 2031 struct ecore_rdma_destroy_cq_out_params oparams; 2032 struct ecore_rdma_destroy_cq_in_params iparams; 2033 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 2034 int rc = 0; 2035 qlnx_host_t *ha; 2036 2037 ha = dev->ha; 2038 2039 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); 2040 2041 cq->destroyed = 1; 2042 2043 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation 2044 * that all completions with notification are dealt with. The rest 2045 * of the completions are not interesting 2046 */ 2047 2048 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 2049 2050 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { 2051 iparams.icid = cq->icid; 2052 2053 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 2054 2055 if (rc) { 2056 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", 2057 cq->icid); 2058 return rc; 2059 } 2060 2061 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); 2062 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 2063 } 2064 2065 if (ibcq->uobject && ibcq->uobject->context) { 2066 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 2067 ib_umem_release(cq->q.umem); 2068 } 2069 2070 cq->sig = ~cq->sig; 2071 2072 kfree(cq); 2073 2074 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); 2075 2076 return rc; 2077 } 2078 2079 static int 2080 qlnxr_check_qp_attrs(struct ib_pd *ibpd, 2081 struct qlnxr_dev *dev, 2082 struct ib_qp_init_attr *attrs, 2083 struct ib_udata *udata) 2084 { 2085 struct ecore_rdma_device *qattr; 2086 qlnx_host_t *ha; 2087 2088 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2089 ha = dev->ha; 2090 2091 QL_DPRINT12(ha, "enter\n"); 2092 2093 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); 2094 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); 2095 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); 2096 2097 #if __FreeBSD_version < 1102000 2098 QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type); 2099 #endif 2100 2101 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); 2102 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); 2103 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); 2104 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); 2105 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); 2106 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", 2107 attrs->cap.max_inline_data); 2108 2109 #if __FreeBSD_version < 1102000 2110 QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n", 2111 attrs->cap.qpg_tss_mask_sz); 2112 #endif 2113 2114 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); 2115 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); 2116 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); 2117 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); 2118 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); 2119 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", 2120 (void *)qattr->sys_image_guid); 2121 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); 2122 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); 2123 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); 2124 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); 2125 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); 2126 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); 2127 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", 2128 qattr->max_qp_resp_rd_atomic_resc); 2129 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", 2130 qattr->max_qp_req_rd_atomic_resc); 2131 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", 2132 qattr->max_dev_resp_rd_atomic_resc); 2133 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); 2134 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); 2135 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); 2136 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); 2137 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); 2138 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); 2139 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); 2140 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); 2141 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", 2142 qattr->max_mr_mw_fmr_pbl); 2143 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", 2144 (void *)qattr->max_mr_mw_fmr_size); 2145 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); 2146 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); 2147 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); 2148 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); 2149 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", 2150 qattr->max_stats_queues); 2151 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); 2152 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", 2153 (void *)qattr->page_size_caps); 2154 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); 2155 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); 2156 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", 2157 qattr->bad_pkey_counter); 2158 2159 if ((attrs->qp_type == IB_QPT_GSI) && udata) { 2160 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); 2161 return -EINVAL; 2162 } 2163 2164 if (udata && !(ibpd->uobject && ibpd->uobject->context)) { 2165 QL_DPRINT12(ha, "called from user without context\n"); 2166 return -EINVAL; 2167 } 2168 2169 /* QP0... attrs->qp_type == IB_QPT_GSI */ 2170 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 2171 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 2172 attrs->qp_type); 2173 return -EINVAL; 2174 } 2175 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { 2176 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); 2177 return -EINVAL; 2178 } 2179 /* Skip the check for QP1 to support CM size of 128 */ 2180 if (attrs->cap.max_send_wr > qattr->max_wqe) { 2181 QL_DPRINT12(ha, "cannot create a SQ with %d elements " 2182 " (max_send_wr=0x%x)\n", 2183 attrs->cap.max_send_wr, qattr->max_wqe); 2184 return -EINVAL; 2185 } 2186 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { 2187 QL_DPRINT12(ha, "cannot create a RQ with %d elements" 2188 " (max_recv_wr=0x%x)\n", 2189 attrs->cap.max_recv_wr, qattr->max_wqe); 2190 return -EINVAL; 2191 } 2192 if (attrs->cap.max_inline_data > qattr->max_inline) { 2193 QL_DPRINT12(ha, 2194 "unsupported inline data size=0x%x " 2195 "requested (max_inline=0x%x)\n", 2196 attrs->cap.max_inline_data, qattr->max_inline); 2197 return -EINVAL; 2198 } 2199 if (attrs->cap.max_send_sge > qattr->max_sge) { 2200 QL_DPRINT12(ha, 2201 "unsupported send_sge=0x%x " 2202 "requested (max_send_sge=0x%x)\n", 2203 attrs->cap.max_send_sge, qattr->max_sge); 2204 return -EINVAL; 2205 } 2206 if (attrs->cap.max_recv_sge > qattr->max_sge) { 2207 QL_DPRINT12(ha, 2208 "unsupported recv_sge=0x%x requested " 2209 " (max_recv_sge=0x%x)\n", 2210 attrs->cap.max_recv_sge, qattr->max_sge); 2211 return -EINVAL; 2212 } 2213 /* unprivileged user space cannot create special QP */ 2214 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 2215 QL_DPRINT12(ha, 2216 "userspace can't create special QPs of type=0x%x\n", 2217 attrs->qp_type); 2218 return -EINVAL; 2219 } 2220 /* allow creating only one GSI type of QP */ 2221 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 2222 QL_DPRINT12(ha, 2223 "create qp: GSI special QPs already created.\n"); 2224 return -EINVAL; 2225 } 2226 2227 /* verify consumer QPs are not trying to use GSI QP's CQ */ 2228 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 2229 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); 2230 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); 2231 2232 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || 2233 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { 2234 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); 2235 return -EINVAL; 2236 } 2237 } 2238 QL_DPRINT12(ha, "exit\n"); 2239 return 0; 2240 } 2241 2242 static int 2243 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 2244 struct qlnxr_srq *srq, 2245 struct ib_udata *udata) 2246 { 2247 struct qlnxr_create_srq_uresp uresp; 2248 qlnx_host_t *ha; 2249 int rc; 2250 2251 ha = dev->ha; 2252 2253 QL_DPRINT12(ha, "enter\n"); 2254 2255 memset(&uresp, 0, sizeof(uresp)); 2256 2257 uresp.srq_id = srq->srq_id; 2258 2259 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2260 2261 QL_DPRINT12(ha, "exit [%d]\n", rc); 2262 return rc; 2263 } 2264 2265 static void 2266 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, 2267 struct qlnxr_create_qp_uresp *uresp, 2268 struct qlnxr_qp *qp) 2269 { 2270 qlnx_host_t *ha; 2271 2272 ha = dev->ha; 2273 2274 /* Return if QP is associated with SRQ instead of RQ */ 2275 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); 2276 2277 if (qp->srq) 2278 return; 2279 2280 /* iWARP requires two doorbells per RQ. */ 2281 if (QLNX_IS_IWARP(dev)) { 2282 uresp->rq_db_offset = 2283 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2284 uresp->rq_db2_offset = 2285 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2286 2287 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " 2288 "uresp->rq_db2_offset = 0x%x\n", 2289 uresp->rq_db_offset, uresp->rq_db2_offset); 2290 } else { 2291 uresp->rq_db_offset = 2292 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2293 } 2294 uresp->rq_icid = qp->icid; 2295 2296 QL_DPRINT12(ha, "exit\n"); 2297 return; 2298 } 2299 2300 static void 2301 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, 2302 struct qlnxr_create_qp_uresp *uresp, 2303 struct qlnxr_qp *qp) 2304 { 2305 qlnx_host_t *ha; 2306 2307 ha = dev->ha; 2308 2309 QL_DPRINT12(ha, "enter\n"); 2310 2311 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2312 2313 /* iWARP uses the same cid for rq and sq*/ 2314 if (QLNX_IS_IWARP(dev)) { 2315 uresp->sq_icid = qp->icid; 2316 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); 2317 } else 2318 uresp->sq_icid = qp->icid + 1; 2319 2320 QL_DPRINT12(ha, "exit\n"); 2321 return; 2322 } 2323 2324 static int 2325 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, 2326 struct qlnxr_qp *qp, 2327 struct ib_udata *udata) 2328 { 2329 int rc; 2330 struct qlnxr_create_qp_uresp uresp; 2331 qlnx_host_t *ha; 2332 2333 ha = dev->ha; 2334 2335 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); 2336 2337 memset(&uresp, 0, sizeof(uresp)); 2338 qlnxr_copy_sq_uresp(dev, &uresp, qp); 2339 qlnxr_copy_rq_uresp(dev, &uresp, qp); 2340 2341 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 2342 uresp.qp_id = qp->qp_id; 2343 2344 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2345 2346 QL_DPRINT12(ha, "exit [%d]\n", rc); 2347 return rc; 2348 } 2349 2350 static void 2351 qlnxr_set_common_qp_params(struct qlnxr_dev *dev, 2352 struct qlnxr_qp *qp, 2353 struct qlnxr_pd *pd, 2354 struct ib_qp_init_attr *attrs) 2355 { 2356 qlnx_host_t *ha; 2357 2358 ha = dev->ha; 2359 2360 QL_DPRINT12(ha, "enter\n"); 2361 2362 spin_lock_init(&qp->q_lock); 2363 2364 atomic_set(&qp->refcnt, 1); 2365 qp->pd = pd; 2366 qp->sig = QLNXR_QP_MAGIC_NUMBER; 2367 qp->qp_type = attrs->qp_type; 2368 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2369 qp->sq.max_sges = attrs->cap.max_send_sge; 2370 qp->state = ECORE_ROCE_QP_STATE_RESET; 2371 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 2372 qp->sq_cq = get_qlnxr_cq(attrs->send_cq); 2373 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); 2374 qp->dev = dev; 2375 2376 if (!attrs->srq) { 2377 /* QP is associated with RQ instead of SRQ */ 2378 qp->rq.max_sges = attrs->cap.max_recv_sge; 2379 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 2380 qp->rq.max_sges, qp->rq_cq->icid); 2381 } else { 2382 qp->srq = get_qlnxr_srq(attrs->srq); 2383 } 2384 2385 QL_DPRINT12(ha, 2386 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," 2387 " state = %d, signaled = %d, use_srq=%d\n", 2388 pd->pd_id, qp->qp_type, qp->max_inline_data, 2389 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); 2390 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 2391 qp->sq.max_sges, qp->sq_cq->icid); 2392 return; 2393 } 2394 2395 static int 2396 qlnxr_check_srq_params(struct ib_pd *ibpd, 2397 struct qlnxr_dev *dev, 2398 struct ib_srq_init_attr *attrs) 2399 { 2400 struct ecore_rdma_device *qattr; 2401 qlnx_host_t *ha; 2402 2403 ha = dev->ha; 2404 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2405 2406 QL_DPRINT12(ha, "enter\n"); 2407 2408 if (attrs->attr.max_wr > qattr->max_srq_wqe) { 2409 QL_DPRINT12(ha, "unsupported srq_wr=0x%x" 2410 " requested (max_srq_wr=0x%x)\n", 2411 attrs->attr.max_wr, qattr->max_srq_wr); 2412 return -EINVAL; 2413 } 2414 2415 if (attrs->attr.max_sge > qattr->max_sge) { 2416 QL_DPRINT12(ha, 2417 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 2418 attrs->attr.max_sge, qattr->max_sge); 2419 return -EINVAL; 2420 } 2421 2422 if (attrs->attr.srq_limit > attrs->attr.max_wr) { 2423 QL_DPRINT12(ha, 2424 "unsupported srq_limit=0x%x requested" 2425 " (max_srq_limit=0x%x)\n", 2426 attrs->attr.srq_limit, attrs->attr.srq_limit); 2427 return -EINVAL; 2428 } 2429 2430 QL_DPRINT12(ha, "exit\n"); 2431 return 0; 2432 } 2433 2434 static void 2435 qlnxr_free_srq_user_params(struct qlnxr_srq *srq) 2436 { 2437 struct qlnxr_dev *dev = srq->dev; 2438 qlnx_host_t *ha; 2439 2440 ha = dev->ha; 2441 2442 QL_DPRINT12(ha, "enter\n"); 2443 2444 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2445 ib_umem_release(srq->usrq.umem); 2446 ib_umem_release(srq->prod_umem); 2447 2448 QL_DPRINT12(ha, "exit\n"); 2449 return; 2450 } 2451 2452 static void 2453 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) 2454 { 2455 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2456 struct qlnxr_dev *dev = srq->dev; 2457 qlnx_host_t *ha; 2458 2459 ha = dev->ha; 2460 2461 QL_DPRINT12(ha, "enter\n"); 2462 2463 ecore_chain_free(dev->cdev, &hw_srq->pbl); 2464 2465 qlnx_dma_free_coherent(&dev->cdev, 2466 hw_srq->virt_prod_pair_addr, 2467 hw_srq->phy_prod_pair_addr, 2468 sizeof(struct rdma_srq_producers)); 2469 2470 QL_DPRINT12(ha, "exit\n"); 2471 2472 return; 2473 } 2474 2475 static int 2476 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 2477 struct qlnxr_srq *srq, 2478 struct qlnxr_create_srq_ureq *ureq, 2479 int access, int dmasync) 2480 { 2481 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2482 struct ib_umem_chunk *chunk; 2483 #endif 2484 struct scatterlist *sg; 2485 int rc; 2486 struct qlnxr_dev *dev = srq->dev; 2487 qlnx_host_t *ha; 2488 2489 ha = dev->ha; 2490 2491 QL_DPRINT12(ha, "enter\n"); 2492 2493 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 2494 ureq->srq_len, access, dmasync, 1); 2495 if (rc) 2496 return rc; 2497 2498 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 2499 sizeof(struct rdma_srq_producers), 2500 access, dmasync); 2501 if (IS_ERR(srq->prod_umem)) { 2502 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2503 ib_umem_release(srq->usrq.umem); 2504 2505 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", 2506 PTR_ERR(srq->prod_umem)); 2507 2508 return PTR_ERR(srq->prod_umem); 2509 } 2510 2511 #ifdef DEFINE_IB_UMEM_WITH_CHUNK 2512 chunk = container_of((&srq->prod_umem->chunk_list)->next, 2513 typeof(*chunk), list); 2514 sg = &chunk->page_list[0]; 2515 #else 2516 sg = srq->prod_umem->sg_head.sgl; 2517 #endif 2518 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 2519 2520 QL_DPRINT12(ha, "exit\n"); 2521 return 0; 2522 } 2523 2524 static int 2525 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 2526 struct qlnxr_dev *dev, 2527 struct ib_srq_init_attr *init_attr) 2528 { 2529 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2530 dma_addr_t phy_prod_pair_addr; 2531 u32 num_elems, max_wr; 2532 void *va; 2533 int rc; 2534 qlnx_host_t *ha; 2535 2536 ha = dev->ha; 2537 2538 QL_DPRINT12(ha, "enter\n"); 2539 2540 va = qlnx_dma_alloc_coherent(&dev->cdev, 2541 &phy_prod_pair_addr, 2542 sizeof(struct rdma_srq_producers)); 2543 if (!va) { 2544 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); 2545 return -ENOMEM; 2546 } 2547 2548 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 2549 hw_srq->virt_prod_pair_addr = va; 2550 2551 max_wr = init_attr->attr.max_wr; 2552 2553 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; 2554 2555 rc = ecore_chain_alloc(dev->cdev, 2556 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2557 ECORE_CHAIN_MODE_PBL, 2558 ECORE_CHAIN_CNT_TYPE_U32, 2559 num_elems, 2560 ECORE_RDMA_SRQ_WQE_ELEM_SIZE, 2561 &hw_srq->pbl, NULL); 2562 2563 if (rc) { 2564 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); 2565 goto err0; 2566 } 2567 2568 hw_srq->max_wr = max_wr; 2569 hw_srq->num_elems = num_elems; 2570 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; 2571 2572 QL_DPRINT12(ha, "exit\n"); 2573 return 0; 2574 2575 err0: 2576 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, 2577 sizeof(struct rdma_srq_producers)); 2578 2579 QL_DPRINT12(ha, "exit [%d]\n", rc); 2580 return rc; 2581 } 2582 2583 static inline void 2584 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, 2585 struct qlnxr_pd *pd, 2586 struct qlnxr_qp *qp, 2587 struct ib_qp_init_attr *attrs, 2588 bool fmr_and_reserved_lkey, 2589 struct ecore_rdma_create_qp_in_params *params) 2590 { 2591 qlnx_host_t *ha; 2592 2593 ha = dev->ha; 2594 2595 QL_DPRINT12(ha, "enter\n"); 2596 2597 /* QP handle to be written in an async event */ 2598 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 2599 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 2600 2601 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 2602 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 2603 params->pd = pd->pd_id; 2604 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 2605 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 2606 params->stats_queue = 0; 2607 2608 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 2609 2610 if (qp->srq) { 2611 /* QP is associated with SRQ instead of RQ */ 2612 params->srq_id = qp->srq->srq_id; 2613 params->use_srq = true; 2614 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 2615 params->srq_id, params->use_srq); 2616 return; 2617 } 2618 2619 params->srq_id = 0; 2620 params->use_srq = false; 2621 2622 QL_DPRINT12(ha, "exit\n"); 2623 return; 2624 } 2625 2626 static inline void 2627 qlnxr_qp_user_print( struct qlnxr_dev *dev, 2628 struct qlnxr_qp *qp) 2629 { 2630 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " 2631 "rq_addr=0x%llx, rq_len=%zd\n", 2632 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, 2633 qp->urq.buf_len); 2634 return; 2635 } 2636 2637 static int 2638 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) 2639 { 2640 u32 newid; 2641 int rc; 2642 qlnx_host_t *ha; 2643 2644 ha = dev->ha; 2645 2646 QL_DPRINT12(ha, "enter\n"); 2647 2648 if (!QLNX_IS_IWARP(dev)) 2649 return 0; 2650 2651 do { 2652 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { 2653 QL_DPRINT11(ha, "idr_pre_get failed\n"); 2654 return -ENOMEM; 2655 } 2656 2657 mtx_lock(&dev->idr_lock); 2658 2659 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); 2660 2661 mtx_unlock(&dev->idr_lock); 2662 2663 } while (rc == -EAGAIN); 2664 2665 QL_DPRINT12(ha, "exit [%d]\n", rc); 2666 2667 return rc; 2668 } 2669 2670 static void 2671 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) 2672 { 2673 qlnx_host_t *ha; 2674 2675 ha = dev->ha; 2676 2677 QL_DPRINT12(ha, "enter\n"); 2678 2679 if (!QLNX_IS_IWARP(dev)) 2680 return; 2681 2682 mtx_lock(&dev->idr_lock); 2683 idr_remove(&dev->qpidr, id); 2684 mtx_unlock(&dev->idr_lock); 2685 2686 QL_DPRINT12(ha, "exit \n"); 2687 2688 return; 2689 } 2690 2691 static inline void 2692 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, 2693 struct qlnxr_qp *qp, 2694 struct ecore_rdma_create_qp_out_params *out_params) 2695 { 2696 qlnx_host_t *ha; 2697 2698 ha = dev->ha; 2699 2700 QL_DPRINT12(ha, "enter\n"); 2701 2702 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 2703 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 2704 2705 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 2706 &qp->usq.pbl_info); 2707 2708 if (qp->srq) { 2709 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); 2710 return; 2711 } 2712 2713 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 2714 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 2715 2716 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 2717 &qp->urq.pbl_info); 2718 2719 QL_DPRINT12(ha, "exit\n"); 2720 return; 2721 } 2722 2723 static int 2724 qlnxr_create_user_qp(struct qlnxr_dev *dev, 2725 struct qlnxr_qp *qp, 2726 struct ib_pd *ibpd, 2727 struct ib_udata *udata, 2728 struct ib_qp_init_attr *attrs) 2729 { 2730 struct ecore_rdma_destroy_qp_out_params d_out_params; 2731 struct ecore_rdma_create_qp_in_params in_params; 2732 struct ecore_rdma_create_qp_out_params out_params; 2733 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2734 struct ib_ucontext *ib_ctx = NULL; 2735 struct qlnxr_ucontext *ctx = NULL; 2736 struct qlnxr_create_qp_ureq ureq; 2737 int alloc_and_init = QLNX_IS_ROCE(dev); 2738 int rc = -EINVAL; 2739 qlnx_host_t *ha; 2740 2741 ha = dev->ha; 2742 2743 QL_DPRINT12(ha, "enter\n"); 2744 2745 ib_ctx = ibpd->uobject->context; 2746 ctx = get_qlnxr_ucontext(ib_ctx); 2747 2748 memset(&ureq, 0, sizeof(ureq)); 2749 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 2750 2751 if (rc) { 2752 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); 2753 return rc; 2754 } 2755 2756 /* SQ - read access only (0), dma sync not required (0) */ 2757 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 2758 ureq.sq_len, 0, 0, 2759 alloc_and_init); 2760 if (rc) { 2761 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2762 return rc; 2763 } 2764 2765 if (!qp->srq) { 2766 /* RQ - read access only (0), dma sync not required (0) */ 2767 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 2768 ureq.rq_len, 0, 0, 2769 alloc_and_init); 2770 2771 if (rc) { 2772 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2773 return rc; 2774 } 2775 } 2776 2777 memset(&in_params, 0, sizeof(in_params)); 2778 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 2779 in_params.qp_handle_lo = ureq.qp_handle_lo; 2780 in_params.qp_handle_hi = ureq.qp_handle_hi; 2781 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 2782 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 2783 2784 if (!qp->srq) { 2785 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 2786 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 2787 } 2788 2789 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); 2790 2791 if (!qp->ecore_qp) { 2792 rc = -ENOMEM; 2793 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2794 goto err1; 2795 } 2796 2797 if (QLNX_IS_IWARP(dev)) 2798 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); 2799 2800 qp->qp_id = out_params.qp_id; 2801 qp->icid = out_params.icid; 2802 2803 rc = qlnxr_copy_qp_uresp(dev, qp, udata); 2804 2805 if (rc) { 2806 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); 2807 goto err; 2808 } 2809 2810 qlnxr_qp_user_print(dev, qp); 2811 2812 QL_DPRINT12(ha, "exit\n"); 2813 return 0; 2814 err: 2815 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2816 2817 if (rc) 2818 QL_DPRINT12(ha, "fatal fault\n"); 2819 2820 err1: 2821 qlnxr_cleanup_user(dev, qp); 2822 2823 QL_DPRINT12(ha, "exit[%d]\n", rc); 2824 return rc; 2825 } 2826 2827 static void 2828 qlnxr_set_roce_db_info(struct qlnxr_dev *dev, 2829 struct qlnxr_qp *qp) 2830 { 2831 qlnx_host_t *ha; 2832 2833 ha = dev->ha; 2834 2835 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2836 2837 qp->sq.db = dev->db_addr + 2838 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2839 qp->sq.db_data.data.icid = qp->icid + 1; 2840 2841 if (!qp->srq) { 2842 qp->rq.db = dev->db_addr + 2843 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2844 qp->rq.db_data.data.icid = qp->icid; 2845 } 2846 2847 QL_DPRINT12(ha, "exit\n"); 2848 return; 2849 } 2850 2851 static void 2852 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, 2853 struct qlnxr_qp *qp) 2854 2855 { 2856 qlnx_host_t *ha; 2857 2858 ha = dev->ha; 2859 2860 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2861 2862 qp->sq.db = dev->db_addr + 2863 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2864 qp->sq.db_data.data.icid = qp->icid; 2865 2866 if (!qp->srq) { 2867 qp->rq.db = dev->db_addr + 2868 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2869 qp->rq.db_data.data.icid = qp->icid; 2870 2871 qp->rq.iwarp_db2 = dev->db_addr + 2872 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2873 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2874 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2875 } 2876 2877 QL_DPRINT12(ha, 2878 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" 2879 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" 2880 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" 2881 " qp->rq.iwarp_db2.data.prod_val =0x%x\n", 2882 qp->sq.db, qp->sq.db_data.data.icid, 2883 qp->rq.db, qp->rq.db_data.data.icid, 2884 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, 2885 qp->rq.iwarp_db2_data.data.value); 2886 2887 QL_DPRINT12(ha, "exit\n"); 2888 return; 2889 } 2890 2891 static int 2892 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, 2893 struct qlnxr_qp *qp, 2894 struct ecore_rdma_create_qp_in_params *in_params, 2895 u32 n_sq_elems, 2896 u32 n_rq_elems) 2897 { 2898 struct ecore_rdma_create_qp_out_params out_params; 2899 int rc; 2900 qlnx_host_t *ha; 2901 2902 ha = dev->ha; 2903 2904 QL_DPRINT12(ha, "enter\n"); 2905 2906 rc = ecore_chain_alloc( 2907 dev->cdev, 2908 ECORE_CHAIN_USE_TO_PRODUCE, 2909 ECORE_CHAIN_MODE_PBL, 2910 ECORE_CHAIN_CNT_TYPE_U32, 2911 n_sq_elems, 2912 QLNXR_SQE_ELEMENT_SIZE, 2913 &qp->sq.pbl, 2914 NULL); 2915 2916 if (rc) { 2917 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); 2918 return rc; 2919 } 2920 2921 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); 2922 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); 2923 2924 if (!qp->srq) { 2925 rc = ecore_chain_alloc( 2926 dev->cdev, 2927 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2928 ECORE_CHAIN_MODE_PBL, 2929 ECORE_CHAIN_CNT_TYPE_U32, 2930 n_rq_elems, 2931 QLNXR_RQE_ELEMENT_SIZE, 2932 &qp->rq.pbl, 2933 NULL); 2934 2935 if (rc) { 2936 QL_DPRINT11(ha, 2937 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); 2938 return rc; 2939 } 2940 2941 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); 2942 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); 2943 } 2944 2945 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2946 2947 if (!qp->ecore_qp) { 2948 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); 2949 return -EINVAL; 2950 } 2951 2952 qp->qp_id = out_params.qp_id; 2953 qp->icid = out_params.icid; 2954 2955 qlnxr_set_roce_db_info(dev, qp); 2956 2957 QL_DPRINT12(ha, "exit\n"); 2958 return 0; 2959 } 2960 2961 static int 2962 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, 2963 struct qlnxr_qp *qp, 2964 struct ecore_rdma_create_qp_in_params *in_params, 2965 u32 n_sq_elems, 2966 u32 n_rq_elems) 2967 { 2968 struct ecore_rdma_destroy_qp_out_params d_out_params; 2969 struct ecore_rdma_create_qp_out_params out_params; 2970 struct ecore_chain_ext_pbl ext_pbl; 2971 int rc; 2972 qlnx_host_t *ha; 2973 2974 ha = dev->ha; 2975 2976 QL_DPRINT12(ha, "enter\n"); 2977 2978 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, 2979 QLNXR_SQE_ELEMENT_SIZE, 2980 ECORE_CHAIN_MODE_PBL); 2981 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, 2982 QLNXR_RQE_ELEMENT_SIZE, 2983 ECORE_CHAIN_MODE_PBL); 2984 2985 QL_DPRINT12(ha, "n_sq_elems = 0x%x" 2986 " n_rq_elems = 0x%x in_params\n" 2987 "\t\t\tqp_handle_lo\t\t= 0x%08x\n" 2988 "\t\t\tqp_handle_hi\t\t= 0x%08x\n" 2989 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" 2990 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" 2991 "\t\t\tuse_srq\t\t\t= 0x%x\n" 2992 "\t\t\tsignal_all\t\t= 0x%x\n" 2993 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" 2994 "\t\t\tpd\t\t\t= 0x%x\n" 2995 "\t\t\tdpi\t\t\t= 0x%x\n" 2996 "\t\t\tsq_cq_id\t\t\t= 0x%x\n" 2997 "\t\t\tsq_num_pages\t\t= 0x%x\n" 2998 "\t\t\tsq_pbl_ptr\t\t= %p\n" 2999 "\t\t\tmax_sq_sges\t\t= 0x%x\n" 3000 "\t\t\trq_cq_id\t\t\t= 0x%x\n" 3001 "\t\t\trq_num_pages\t\t= 0x%x\n" 3002 "\t\t\trq_pbl_ptr\t\t= %p\n" 3003 "\t\t\tsrq_id\t\t\t= 0x%x\n" 3004 "\t\t\tstats_queue\t\t= 0x%x\n", 3005 n_sq_elems, n_rq_elems, 3006 in_params->qp_handle_lo, 3007 in_params->qp_handle_hi, 3008 in_params->qp_handle_async_lo, 3009 in_params->qp_handle_async_hi, 3010 in_params->use_srq, 3011 in_params->signal_all, 3012 in_params->fmr_and_reserved_lkey, 3013 in_params->pd, 3014 in_params->dpi, 3015 in_params->sq_cq_id, 3016 in_params->sq_num_pages, 3017 (void *)in_params->sq_pbl_ptr, 3018 in_params->max_sq_sges, 3019 in_params->rq_cq_id, 3020 in_params->rq_num_pages, 3021 (void *)in_params->rq_pbl_ptr, 3022 in_params->srq_id, 3023 in_params->stats_queue ); 3024 3025 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); 3026 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); 3027 3028 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 3029 3030 if (!qp->ecore_qp) { 3031 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 3032 return -EINVAL; 3033 } 3034 3035 /* Now we allocate the chain */ 3036 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 3037 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 3038 3039 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3040 "ext_pbl.p_pbl_phys = %p\n", 3041 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3042 3043 rc = ecore_chain_alloc( 3044 dev->cdev, 3045 ECORE_CHAIN_USE_TO_PRODUCE, 3046 ECORE_CHAIN_MODE_PBL, 3047 ECORE_CHAIN_CNT_TYPE_U32, 3048 n_sq_elems, 3049 QLNXR_SQE_ELEMENT_SIZE, 3050 &qp->sq.pbl, 3051 &ext_pbl); 3052 3053 if (rc) { 3054 QL_DPRINT11(ha, 3055 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); 3056 goto err; 3057 } 3058 3059 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 3060 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 3061 3062 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 3063 "ext_pbl.p_pbl_phys = %p\n", 3064 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 3065 3066 if (!qp->srq) { 3067 rc = ecore_chain_alloc( 3068 dev->cdev, 3069 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 3070 ECORE_CHAIN_MODE_PBL, 3071 ECORE_CHAIN_CNT_TYPE_U32, 3072 n_rq_elems, 3073 QLNXR_RQE_ELEMENT_SIZE, 3074 &qp->rq.pbl, 3075 &ext_pbl); 3076 3077 if (rc) { 3078 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" 3079 " failed rc = %d\n", rc); 3080 goto err; 3081 } 3082 } 3083 3084 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", 3085 out_params.qp_id, out_params.icid); 3086 3087 qp->qp_id = out_params.qp_id; 3088 qp->icid = out_params.icid; 3089 3090 qlnxr_set_iwarp_db_info(dev, qp); 3091 3092 QL_DPRINT12(ha, "exit\n"); 3093 return 0; 3094 3095 err: 3096 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 3097 3098 QL_DPRINT12(ha, "exit rc = %d\n", rc); 3099 return rc; 3100 } 3101 3102 static int 3103 qlnxr_create_kernel_qp(struct qlnxr_dev *dev, 3104 struct qlnxr_qp *qp, 3105 struct ib_pd *ibpd, 3106 struct ib_qp_init_attr *attrs) 3107 { 3108 struct ecore_rdma_create_qp_in_params in_params; 3109 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3110 int rc = -EINVAL; 3111 u32 n_rq_elems; 3112 u32 n_sq_elems; 3113 u32 n_sq_entries; 3114 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3115 qlnx_host_t *ha; 3116 3117 ha = dev->ha; 3118 3119 QL_DPRINT12(ha, "enter\n"); 3120 3121 memset(&in_params, 0, sizeof(in_params)); 3122 3123 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in 3124 * the ring. The ring should allow at least a single WR, even if the 3125 * user requested none, due to allocation issues. 3126 * We should add an extra WR since the prod and cons indices of 3127 * wqe_wr_id are managed in such a way that the WQ is considered full 3128 * when (prod+1)%max_wr==cons. We currently don't do that because we 3129 * double the number of entries due an iSER issue that pushes far more 3130 * WRs than indicated. If we decline its ib_post_send() then we get 3131 * error prints in the dmesg we'd like to avoid. 3132 */ 3133 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 3134 qattr->max_wqe); 3135 3136 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 3137 GFP_KERNEL); 3138 if (!qp->wqe_wr_id) { 3139 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); 3140 return -ENOMEM; 3141 } 3142 3143 /* QP handle to be written in CQE */ 3144 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); 3145 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); 3146 3147 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in 3148 * the ring. There ring should allow at least a single WR, even if the 3149 * user requested none, due to allocation issues. 3150 */ 3151 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); 3152 3153 /* Allocate driver internal RQ array */ 3154 if (!qp->srq) { 3155 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 3156 GFP_KERNEL); 3157 if (!qp->rqe_wr_id) { 3158 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); 3159 kfree(qp->wqe_wr_id); 3160 return -ENOMEM; 3161 } 3162 } 3163 3164 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 3165 3166 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 3167 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 3168 3169 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 3170 in_params.fmr_and_reserved_lkey = true; 3171 in_params.pd = pd->pd_id; 3172 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 3173 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 3174 in_params.stats_queue = 0; 3175 3176 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 3177 3178 if (qp->srq) { 3179 /* QP is associated with SRQ instead of RQ */ 3180 in_params.srq_id = qp->srq->srq_id; 3181 in_params.use_srq = true; 3182 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 3183 in_params.srq_id, in_params.use_srq); 3184 } else { 3185 in_params.srq_id = 0; 3186 in_params.use_srq = false; 3187 } 3188 3189 n_sq_entries = attrs->cap.max_send_wr; 3190 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); 3191 n_sq_entries = max_t(u32, n_sq_entries, 1); 3192 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 3193 3194 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 3195 3196 if (QLNX_IS_ROCE(dev)) { 3197 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, 3198 n_sq_elems, n_rq_elems); 3199 } else { 3200 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, 3201 n_sq_elems, n_rq_elems); 3202 } 3203 3204 if (rc) 3205 qlnxr_cleanup_kernel(dev, qp); 3206 3207 QL_DPRINT12(ha, "exit [%d]\n", rc); 3208 return rc; 3209 } 3210 3211 struct ib_qp * 3212 qlnxr_create_qp(struct ib_pd *ibpd, 3213 struct ib_qp_init_attr *attrs, 3214 struct ib_udata *udata) 3215 { 3216 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); 3217 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3218 struct qlnxr_qp *qp; 3219 int rc = 0; 3220 qlnx_host_t *ha; 3221 3222 ha = dev->ha; 3223 3224 QL_DPRINT12(ha, "enter\n"); 3225 3226 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); 3227 if (rc) { 3228 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); 3229 return ERR_PTR(rc); 3230 } 3231 3232 QL_DPRINT12(ha, "called from %s, event_handle=%p," 3233 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 3234 (udata ? "user library" : "kernel"), 3235 attrs->event_handler, pd, 3236 get_qlnxr_cq(attrs->send_cq), 3237 get_qlnxr_cq(attrs->send_cq)->icid, 3238 get_qlnxr_cq(attrs->recv_cq), 3239 get_qlnxr_cq(attrs->recv_cq)->icid); 3240 3241 qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); 3242 3243 if (!qp) { 3244 QL_DPRINT11(ha, "kzalloc(qp) failed\n"); 3245 return ERR_PTR(-ENOMEM); 3246 } 3247 3248 qlnxr_set_common_qp_params(dev, qp, pd, attrs); 3249 3250 if (attrs->qp_type == IB_QPT_GSI) { 3251 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); 3252 return qlnxr_create_gsi_qp(dev, attrs, qp); 3253 } 3254 3255 if (udata) { 3256 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); 3257 3258 if (rc) { 3259 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); 3260 goto err; 3261 } 3262 } else { 3263 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); 3264 3265 if (rc) { 3266 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); 3267 goto err; 3268 } 3269 } 3270 3271 qp->ibqp.qp_num = qp->qp_id; 3272 3273 rc = qlnxr_idr_add(dev, qp, qp->qp_id); 3274 3275 if (rc) { 3276 QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); 3277 goto err; 3278 } 3279 3280 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); 3281 3282 return &qp->ibqp; 3283 err: 3284 kfree(qp); 3285 3286 QL_DPRINT12(ha, "failed exit\n"); 3287 return ERR_PTR(-EFAULT); 3288 } 3289 3290 static enum ib_qp_state 3291 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) 3292 { 3293 enum ib_qp_state state = IB_QPS_ERR; 3294 3295 switch (qp_state) { 3296 case ECORE_ROCE_QP_STATE_RESET: 3297 state = IB_QPS_RESET; 3298 break; 3299 3300 case ECORE_ROCE_QP_STATE_INIT: 3301 state = IB_QPS_INIT; 3302 break; 3303 3304 case ECORE_ROCE_QP_STATE_RTR: 3305 state = IB_QPS_RTR; 3306 break; 3307 3308 case ECORE_ROCE_QP_STATE_RTS: 3309 state = IB_QPS_RTS; 3310 break; 3311 3312 case ECORE_ROCE_QP_STATE_SQD: 3313 state = IB_QPS_SQD; 3314 break; 3315 3316 case ECORE_ROCE_QP_STATE_ERR: 3317 state = IB_QPS_ERR; 3318 break; 3319 3320 case ECORE_ROCE_QP_STATE_SQE: 3321 state = IB_QPS_SQE; 3322 break; 3323 } 3324 return state; 3325 } 3326 3327 static enum ecore_roce_qp_state 3328 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) 3329 { 3330 enum ecore_roce_qp_state ecore_qp_state; 3331 3332 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3333 3334 switch (qp_state) { 3335 case IB_QPS_RESET: 3336 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; 3337 break; 3338 3339 case IB_QPS_INIT: 3340 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; 3341 break; 3342 3343 case IB_QPS_RTR: 3344 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; 3345 break; 3346 3347 case IB_QPS_RTS: 3348 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; 3349 break; 3350 3351 case IB_QPS_SQD: 3352 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; 3353 break; 3354 3355 case IB_QPS_ERR: 3356 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3357 break; 3358 3359 default: 3360 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3361 break; 3362 } 3363 3364 return (ecore_qp_state); 3365 } 3366 3367 static void 3368 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) 3369 { 3370 ecore_chain_reset(&qph->pbl); 3371 qph->prod = qph->cons = 0; 3372 qph->wqe_cons = 0; 3373 qph->db_data.data.value = cpu_to_le16(0); 3374 3375 return; 3376 } 3377 3378 static int 3379 qlnxr_update_qp_state(struct qlnxr_dev *dev, 3380 struct qlnxr_qp *qp, 3381 enum ecore_roce_qp_state new_state) 3382 { 3383 int status = 0; 3384 uint32_t reg_addr; 3385 struct ecore_dev *cdev; 3386 qlnx_host_t *ha; 3387 3388 ha = dev->ha; 3389 cdev = &ha->cdev; 3390 3391 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", 3392 qp, new_state, qp->state); 3393 3394 if (new_state == qp->state) { 3395 return 0; 3396 } 3397 3398 switch (qp->state) { 3399 case ECORE_ROCE_QP_STATE_RESET: 3400 switch (new_state) { 3401 case ECORE_ROCE_QP_STATE_INIT: 3402 qp->prev_wqe_size = 0; 3403 qlnxr_reset_qp_hwq_info(&qp->sq); 3404 if (!(qp->srq)) 3405 qlnxr_reset_qp_hwq_info(&qp->rq); 3406 break; 3407 default: 3408 status = -EINVAL; 3409 break; 3410 }; 3411 break; 3412 case ECORE_ROCE_QP_STATE_INIT: 3413 /* INIT->XXX */ 3414 switch (new_state) { 3415 case ECORE_ROCE_QP_STATE_RTR: 3416 /* Update doorbell (in case post_recv was done before move to RTR) */ 3417 if (qp->srq) 3418 break; 3419 wmb(); 3420 //writel(qp->rq.db_data.raw, qp->rq.db); 3421 //if (QLNX_IS_IWARP(dev)) 3422 // writel(qp->rq.iwarp_db2_data.raw, 3423 // qp->rq.iwarp_db2); 3424 3425 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 3426 (uint8_t *)cdev->doorbells); 3427 3428 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 3429 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 3430 3431 if (QLNX_IS_IWARP(dev)) { 3432 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 3433 (uint8_t *)cdev->doorbells); 3434 bus_write_4(ha->pci_dbells, reg_addr,\ 3435 qp->rq.iwarp_db2_data.raw); 3436 bus_barrier(ha->pci_dbells, 0, 0,\ 3437 BUS_SPACE_BARRIER_READ); 3438 } 3439 3440 3441 mmiowb(); 3442 break; 3443 case ECORE_ROCE_QP_STATE_ERR: 3444 /* TBD:flush qps... */ 3445 break; 3446 default: 3447 /* invalid state change. */ 3448 status = -EINVAL; 3449 break; 3450 }; 3451 break; 3452 case ECORE_ROCE_QP_STATE_RTR: 3453 /* RTR->XXX */ 3454 switch (new_state) { 3455 case ECORE_ROCE_QP_STATE_RTS: 3456 break; 3457 case ECORE_ROCE_QP_STATE_ERR: 3458 break; 3459 default: 3460 /* invalid state change. */ 3461 status = -EINVAL; 3462 break; 3463 }; 3464 break; 3465 case ECORE_ROCE_QP_STATE_RTS: 3466 /* RTS->XXX */ 3467 switch (new_state) { 3468 case ECORE_ROCE_QP_STATE_SQD: 3469 break; 3470 case ECORE_ROCE_QP_STATE_ERR: 3471 break; 3472 default: 3473 /* invalid state change. */ 3474 status = -EINVAL; 3475 break; 3476 }; 3477 break; 3478 case ECORE_ROCE_QP_STATE_SQD: 3479 /* SQD->XXX */ 3480 switch (new_state) { 3481 case ECORE_ROCE_QP_STATE_RTS: 3482 case ECORE_ROCE_QP_STATE_ERR: 3483 break; 3484 default: 3485 /* invalid state change. */ 3486 status = -EINVAL; 3487 break; 3488 }; 3489 break; 3490 case ECORE_ROCE_QP_STATE_ERR: 3491 /* ERR->XXX */ 3492 switch (new_state) { 3493 case ECORE_ROCE_QP_STATE_RESET: 3494 if ((qp->rq.prod != qp->rq.cons) || 3495 (qp->sq.prod != qp->sq.cons)) { 3496 QL_DPRINT11(ha, 3497 "Error->Reset with rq/sq " 3498 "not empty rq.prod=0x%x rq.cons=0x%x" 3499 " sq.prod=0x%x sq.cons=0x%x\n", 3500 qp->rq.prod, qp->rq.cons, 3501 qp->sq.prod, qp->sq.cons); 3502 status = -EINVAL; 3503 } 3504 break; 3505 default: 3506 status = -EINVAL; 3507 break; 3508 }; 3509 break; 3510 default: 3511 status = -EINVAL; 3512 break; 3513 }; 3514 3515 QL_DPRINT12(ha, "exit\n"); 3516 return status; 3517 } 3518 3519 int 3520 qlnxr_modify_qp(struct ib_qp *ibqp, 3521 struct ib_qp_attr *attr, 3522 int attr_mask, 3523 struct ib_udata *udata) 3524 { 3525 int rc = 0; 3526 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3527 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); 3528 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 3529 enum ib_qp_state old_qp_state, new_qp_state; 3530 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3531 qlnx_host_t *ha; 3532 3533 ha = dev->ha; 3534 3535 QL_DPRINT12(ha, 3536 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", 3537 qp, attr_mask, attr->qp_state, udata); 3538 3539 old_qp_state = qlnxr_get_ibqp_state(qp->state); 3540 if (attr_mask & IB_QP_STATE) 3541 new_qp_state = attr->qp_state; 3542 else 3543 new_qp_state = old_qp_state; 3544 3545 if (QLNX_IS_ROCE(dev)) { 3546 #if __FreeBSD_version >= 1100000 3547 if (!ib_modify_qp_is_ok(old_qp_state, 3548 new_qp_state, 3549 ibqp->qp_type, 3550 attr_mask, 3551 IB_LINK_LAYER_ETHERNET)) { 3552 QL_DPRINT12(ha, 3553 "invalid attribute mask=0x%x" 3554 " specified for qpn=0x%x of type=0x%x \n" 3555 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3556 attr_mask, qp->qp_id, ibqp->qp_type, 3557 old_qp_state, new_qp_state); 3558 rc = -EINVAL; 3559 goto err; 3560 } 3561 #else 3562 if (!ib_modify_qp_is_ok(old_qp_state, 3563 new_qp_state, 3564 ibqp->qp_type, 3565 attr_mask )) { 3566 QL_DPRINT12(ha, 3567 "invalid attribute mask=0x%x" 3568 " specified for qpn=0x%x of type=0x%x \n" 3569 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3570 attr_mask, qp->qp_id, ibqp->qp_type, 3571 old_qp_state, new_qp_state); 3572 rc = -EINVAL; 3573 goto err; 3574 } 3575 3576 #endif /* #if __FreeBSD_version >= 1100000 */ 3577 } 3578 /* translate the masks... */ 3579 if (attr_mask & IB_QP_STATE) { 3580 SET_FIELD(qp_params.modify_flags, 3581 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 3582 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); 3583 } 3584 3585 // TBD consider changing ecore to be a flag as well... 3586 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 3587 qp_params.sqd_async = true; 3588 3589 if (attr_mask & IB_QP_PKEY_INDEX) { 3590 SET_FIELD(qp_params.modify_flags, 3591 ECORE_ROCE_MODIFY_QP_VALID_PKEY, 3592 1); 3593 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { 3594 rc = -EINVAL; 3595 goto err; 3596 } 3597 3598 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; 3599 } 3600 3601 if (attr_mask & IB_QP_QKEY) { 3602 qp->qkey = attr->qkey; 3603 } 3604 3605 /* tbd consider splitting in ecore.. */ 3606 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3607 SET_FIELD(qp_params.modify_flags, 3608 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 3609 qp_params.incoming_rdma_read_en = 3610 attr->qp_access_flags & IB_ACCESS_REMOTE_READ; 3611 qp_params.incoming_rdma_write_en = 3612 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; 3613 qp_params.incoming_atomic_en = 3614 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; 3615 } 3616 3617 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 3618 if (attr_mask & IB_QP_PATH_MTU) { 3619 if (attr->path_mtu < IB_MTU_256 || 3620 attr->path_mtu > IB_MTU_4096) { 3621 QL_DPRINT12(ha, 3622 "Only MTU sizes of 256, 512, 1024," 3623 " 2048 and 4096 are supported " 3624 " attr->path_mtu = [%d]\n", 3625 attr->path_mtu); 3626 3627 rc = -EINVAL; 3628 goto err; 3629 } 3630 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 3631 ib_mtu_enum_to_int( 3632 iboe_get_mtu(dev->ha->ifp->if_mtu))); 3633 } 3634 3635 if (qp->mtu == 0) { 3636 qp->mtu = ib_mtu_enum_to_int( 3637 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3638 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", 3639 qp->mtu); 3640 } 3641 3642 SET_FIELD(qp_params.modify_flags, 3643 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 3644 1); 3645 3646 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; 3647 qp_params.flow_label = attr->ah_attr.grh.flow_label; 3648 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; 3649 3650 qp->sgid_idx = attr->ah_attr.grh.sgid_index; 3651 3652 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); 3653 3654 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); 3655 if (rc) 3656 return rc; 3657 3658 qp_params.use_local_mac = true; 3659 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 3660 3661 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", 3662 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 3663 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 3664 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", 3665 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 3666 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 3667 QL_DPRINT12(ha, 3668 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", 3669 qp_params.remote_mac_addr[0], 3670 qp_params.remote_mac_addr[1], 3671 qp_params.remote_mac_addr[2], 3672 qp_params.remote_mac_addr[3], 3673 qp_params.remote_mac_addr[4], 3674 qp_params.remote_mac_addr[5]); 3675 3676 qp_params.mtu = qp->mtu; 3677 } 3678 3679 if (qp_params.mtu == 0) { 3680 /* stay with current MTU */ 3681 if (qp->mtu) { 3682 qp_params.mtu = qp->mtu; 3683 } else { 3684 qp_params.mtu = ib_mtu_enum_to_int( 3685 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3686 } 3687 } 3688 3689 if (attr_mask & IB_QP_TIMEOUT) { 3690 SET_FIELD(qp_params.modify_flags, \ 3691 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 3692 3693 qp_params.ack_timeout = attr->timeout; 3694 if (attr->timeout) { 3695 u32 temp; 3696 3697 /* 12.7.34 LOCAL ACK TIMEOUT 3698 * Value representing the transport (ACK) timeout for 3699 * use by the remote, expressed as (4.096 μS*2Local ACK 3700 * Timeout) 3701 */ 3702 /* We use 1UL since the temporal value may be overflow 3703 * 32 bits 3704 */ 3705 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 3706 qp_params.ack_timeout = temp; /* FW requires [msec] */ 3707 } 3708 else 3709 qp_params.ack_timeout = 0; /* infinite */ 3710 } 3711 if (attr_mask & IB_QP_RETRY_CNT) { 3712 SET_FIELD(qp_params.modify_flags,\ 3713 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 3714 qp_params.retry_cnt = attr->retry_cnt; 3715 } 3716 3717 if (attr_mask & IB_QP_RNR_RETRY) { 3718 SET_FIELD(qp_params.modify_flags, 3719 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 3720 1); 3721 qp_params.rnr_retry_cnt = attr->rnr_retry; 3722 } 3723 3724 if (attr_mask & IB_QP_RQ_PSN) { 3725 SET_FIELD(qp_params.modify_flags, 3726 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 3727 1); 3728 qp_params.rq_psn = attr->rq_psn; 3729 qp->rq_psn = attr->rq_psn; 3730 } 3731 3732 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 3733 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { 3734 rc = -EINVAL; 3735 QL_DPRINT12(ha, 3736 "unsupported max_rd_atomic=%d, supported=%d\n", 3737 attr->max_rd_atomic, 3738 qattr->max_qp_req_rd_atomic_resc); 3739 goto err; 3740 } 3741 3742 SET_FIELD(qp_params.modify_flags, 3743 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 3744 1); 3745 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 3746 } 3747 3748 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 3749 SET_FIELD(qp_params.modify_flags, 3750 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 3751 1); 3752 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 3753 } 3754 3755 if (attr_mask & IB_QP_SQ_PSN) { 3756 SET_FIELD(qp_params.modify_flags, 3757 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 3758 1); 3759 qp_params.sq_psn = attr->sq_psn; 3760 qp->sq_psn = attr->sq_psn; 3761 } 3762 3763 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 3764 if (attr->max_dest_rd_atomic > 3765 qattr->max_qp_resp_rd_atomic_resc) { 3766 QL_DPRINT12(ha, 3767 "unsupported max_dest_rd_atomic=%d, " 3768 "supported=%d\n", 3769 attr->max_dest_rd_atomic, 3770 qattr->max_qp_resp_rd_atomic_resc); 3771 3772 rc = -EINVAL; 3773 goto err; 3774 } 3775 3776 SET_FIELD(qp_params.modify_flags, 3777 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 3778 1); 3779 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 3780 } 3781 3782 if (attr_mask & IB_QP_DEST_QPN) { 3783 SET_FIELD(qp_params.modify_flags, 3784 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 3785 1); 3786 3787 qp_params.dest_qp = attr->dest_qp_num; 3788 qp->dest_qp_num = attr->dest_qp_num; 3789 } 3790 3791 /* 3792 * Update the QP state before the actual ramrod to prevent a race with 3793 * fast path. Modifying the QP state to error will cause the device to 3794 * flush the CQEs and while polling the flushed CQEs will considered as 3795 * a potential issue if the QP isn't in error state. 3796 */ 3797 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && 3798 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) 3799 qp->state = ECORE_ROCE_QP_STATE_ERR; 3800 3801 if (qp->qp_type != IB_QPT_GSI) 3802 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 3803 3804 if (attr_mask & IB_QP_STATE) { 3805 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 3806 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); 3807 qp->state = qp_params.new_state; 3808 } 3809 3810 err: 3811 QL_DPRINT12(ha, "exit\n"); 3812 return rc; 3813 } 3814 3815 static int 3816 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) 3817 { 3818 int ib_qp_acc_flags = 0; 3819 3820 if (params->incoming_rdma_write_en) 3821 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 3822 if (params->incoming_rdma_read_en) 3823 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 3824 if (params->incoming_atomic_en) 3825 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 3826 if (true) /* FIXME -> local write ?? */ 3827 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 3828 3829 return ib_qp_acc_flags; 3830 } 3831 3832 static enum ib_mtu 3833 qlnxr_mtu_int_to_enum(u16 mtu) 3834 { 3835 enum ib_mtu ib_mtu_size; 3836 3837 switch (mtu) { 3838 case 256: 3839 ib_mtu_size = IB_MTU_256; 3840 break; 3841 3842 case 512: 3843 ib_mtu_size = IB_MTU_512; 3844 break; 3845 3846 case 1024: 3847 ib_mtu_size = IB_MTU_1024; 3848 break; 3849 3850 case 2048: 3851 ib_mtu_size = IB_MTU_2048; 3852 break; 3853 3854 case 4096: 3855 ib_mtu_size = IB_MTU_4096; 3856 break; 3857 3858 default: 3859 ib_mtu_size = IB_MTU_1024; 3860 break; 3861 } 3862 return (ib_mtu_size); 3863 } 3864 3865 int 3866 qlnxr_query_qp(struct ib_qp *ibqp, 3867 struct ib_qp_attr *qp_attr, 3868 int attr_mask, 3869 struct ib_qp_init_attr *qp_init_attr) 3870 { 3871 int rc = 0; 3872 struct ecore_rdma_query_qp_out_params params; 3873 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3874 struct qlnxr_dev *dev = qp->dev; 3875 qlnx_host_t *ha; 3876 3877 ha = dev->ha; 3878 3879 QL_DPRINT12(ha, "enter\n"); 3880 3881 memset(¶ms, 0, sizeof(params)); 3882 3883 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); 3884 if (rc) 3885 goto err; 3886 3887 memset(qp_attr, 0, sizeof(*qp_attr)); 3888 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 3889 3890 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); 3891 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); 3892 3893 /* In some cases in iWARP qelr will ask for the state only */ 3894 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { 3895 QL_DPRINT11(ha, "only state requested\n"); 3896 return 0; 3897 } 3898 3899 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); 3900 qp_attr->path_mig_state = IB_MIG_MIGRATED; 3901 qp_attr->rq_psn = params.rq_psn; 3902 qp_attr->sq_psn = params.sq_psn; 3903 qp_attr->dest_qp_num = params.dest_qp; 3904 3905 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); 3906 3907 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " 3908 "path_mtu = %d qp_access_flags = 0x%x\n", 3909 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, 3910 qp_attr->qp_access_flags); 3911 3912 qp_attr->cap.max_send_wr = qp->sq.max_wr; 3913 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 3914 qp_attr->cap.max_send_sge = qp->sq.max_sges; 3915 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 3916 qp_attr->cap.max_inline_data = qp->max_inline_data; 3917 qp_init_attr->cap = qp_attr->cap; 3918 3919 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], 3920 sizeof(qp_attr->ah_attr.grh.dgid.raw)); 3921 3922 qp_attr->ah_attr.grh.flow_label = params.flow_label; 3923 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; 3924 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; 3925 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; 3926 3927 qp_attr->ah_attr.ah_flags = IB_AH_GRH; 3928 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ 3929 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ 3930 qp_attr->timeout = params.timeout; 3931 qp_attr->rnr_retry = params.rnr_retry; 3932 qp_attr->retry_cnt = params.retry_cnt; 3933 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 3934 qp_attr->pkey_index = params.pkey_index; 3935 qp_attr->port_num = 1; /* FIXME -> check this */ 3936 qp_attr->ah_attr.src_path_bits = 0; 3937 qp_attr->ah_attr.static_rate = 0; 3938 qp_attr->alt_pkey_index = 0; 3939 qp_attr->alt_port_num = 0; 3940 qp_attr->alt_timeout = 0; 3941 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 3942 3943 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; 3944 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 3945 qp_attr->max_rd_atomic = params.max_rd_atomic; 3946 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; 3947 3948 QL_DPRINT12(ha, "max_inline_data=%d\n", 3949 qp_attr->cap.max_inline_data); 3950 3951 err: 3952 QL_DPRINT12(ha, "exit\n"); 3953 return rc; 3954 } 3955 3956 static void 3957 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3958 { 3959 qlnx_host_t *ha; 3960 3961 ha = dev->ha; 3962 3963 QL_DPRINT12(ha, "enter\n"); 3964 3965 if (qp->usq.umem) 3966 ib_umem_release(qp->usq.umem); 3967 3968 qp->usq.umem = NULL; 3969 3970 if (qp->urq.umem) 3971 ib_umem_release(qp->urq.umem); 3972 3973 qp->urq.umem = NULL; 3974 3975 QL_DPRINT12(ha, "exit\n"); 3976 return; 3977 } 3978 3979 static void 3980 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3981 { 3982 qlnx_host_t *ha; 3983 3984 ha = dev->ha; 3985 3986 QL_DPRINT12(ha, "enter\n"); 3987 3988 if (qlnxr_qp_has_sq(qp)) { 3989 QL_DPRINT12(ha, "freeing SQ\n"); 3990 ha->qlnxr_debug = 1; 3991 // ecore_chain_free(dev->cdev, &qp->sq.pbl); 3992 ha->qlnxr_debug = 0; 3993 kfree(qp->wqe_wr_id); 3994 } 3995 3996 if (qlnxr_qp_has_rq(qp)) { 3997 QL_DPRINT12(ha, "freeing RQ\n"); 3998 ha->qlnxr_debug = 1; 3999 // ecore_chain_free(dev->cdev, &qp->rq.pbl); 4000 ha->qlnxr_debug = 0; 4001 kfree(qp->rqe_wr_id); 4002 } 4003 4004 QL_DPRINT12(ha, "exit\n"); 4005 return; 4006 } 4007 4008 int 4009 qlnxr_free_qp_resources(struct qlnxr_dev *dev, 4010 struct qlnxr_qp *qp) 4011 { 4012 int rc = 0; 4013 qlnx_host_t *ha; 4014 struct ecore_rdma_destroy_qp_out_params d_out_params; 4015 4016 ha = dev->ha; 4017 4018 QL_DPRINT12(ha, "enter\n"); 4019 4020 #if 0 4021 if (qp->qp_type != IB_QPT_GSI) { 4022 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4023 &d_out_params); 4024 if (rc) 4025 return rc; 4026 } 4027 4028 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4029 qlnxr_cleanup_user(dev, qp); 4030 else 4031 qlnxr_cleanup_kernel(dev, qp); 4032 #endif 4033 4034 if (qp->ibqp.uobject && qp->ibqp.uobject->context) 4035 qlnxr_cleanup_user(dev, qp); 4036 else 4037 qlnxr_cleanup_kernel(dev, qp); 4038 4039 if (qp->qp_type != IB_QPT_GSI) { 4040 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 4041 &d_out_params); 4042 if (rc) 4043 return rc; 4044 } 4045 4046 QL_DPRINT12(ha, "exit\n"); 4047 return 0; 4048 } 4049 4050 int 4051 qlnxr_destroy_qp(struct ib_qp *ibqp) 4052 { 4053 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4054 struct qlnxr_dev *dev = qp->dev; 4055 int rc = 0; 4056 struct ib_qp_attr attr; 4057 int attr_mask = 0; 4058 qlnx_host_t *ha; 4059 4060 ha = dev->ha; 4061 4062 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); 4063 4064 qp->destroyed = 1; 4065 4066 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | 4067 ECORE_ROCE_QP_STATE_ERR | 4068 ECORE_ROCE_QP_STATE_INIT))) { 4069 attr.qp_state = IB_QPS_ERR; 4070 attr_mask |= IB_QP_STATE; 4071 4072 /* change the QP state to ERROR */ 4073 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); 4074 } 4075 4076 if (qp->qp_type == IB_QPT_GSI) 4077 qlnxr_destroy_gsi_qp(dev); 4078 4079 qp->sig = ~qp->sig; 4080 4081 qlnxr_free_qp_resources(dev, qp); 4082 4083 if (atomic_dec_and_test(&qp->refcnt)) { 4084 /* TODO: only for iWARP? */ 4085 qlnxr_idr_remove(dev, qp->qp_id); 4086 kfree(qp); 4087 } 4088 4089 QL_DPRINT12(ha, "exit\n"); 4090 return rc; 4091 } 4092 4093 static inline int 4094 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) 4095 { 4096 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 4097 } 4098 4099 static int 4100 sge_data_len(struct ib_sge *sg_list, int num_sge) 4101 { 4102 int i, len = 0; 4103 for (i = 0; i < num_sge; i++) 4104 len += sg_list[i].length; 4105 return len; 4106 } 4107 4108 static void 4109 swap_wqe_data64(u64 *p) 4110 { 4111 int i; 4112 4113 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 4114 *p = cpu_to_be64(cpu_to_le64(*p)); 4115 } 4116 4117 static u32 4118 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, 4119 struct qlnxr_qp *qp, 4120 u8 *wqe_size, 4121 struct ib_send_wr *wr, 4122 struct ib_send_wr **bad_wr, 4123 u8 *bits, 4124 u8 bit) 4125 { 4126 int i, seg_siz; 4127 char *seg_prt, *wqe; 4128 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 4129 qlnx_host_t *ha; 4130 4131 ha = dev->ha; 4132 4133 QL_DPRINT12(ha, "enter[%d]\n", data_size); 4134 4135 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 4136 QL_DPRINT12(ha, 4137 "Too much inline data in WR:[%d, %d]\n", 4138 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); 4139 *bad_wr = wr; 4140 return 0; 4141 } 4142 4143 if (!data_size) 4144 return data_size; 4145 4146 /* set the bit */ 4147 *bits |= bit; 4148 4149 seg_prt = wqe = NULL; 4150 seg_siz = 0; 4151 4152 /* copy data inline */ 4153 for (i = 0; i < wr->num_sge; i++) { 4154 u32 len = wr->sg_list[i].length; 4155 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 4156 4157 while (len > 0) { 4158 u32 cur; 4159 4160 /* new segment required */ 4161 if (!seg_siz) { 4162 wqe = (char *)ecore_chain_produce(&qp->sq.pbl); 4163 seg_prt = wqe; 4164 seg_siz = sizeof(struct rdma_sq_common_wqe); 4165 (*wqe_size)++; 4166 } 4167 4168 /* calculate currently allowed length */ 4169 cur = MIN(len, seg_siz); 4170 4171 memcpy(seg_prt, src, cur); 4172 4173 /* update segment variables */ 4174 seg_prt += cur; 4175 seg_siz -= cur; 4176 /* update sge variables */ 4177 src += cur; 4178 len -= cur; 4179 4180 /* swap fully-completed segments */ 4181 if (!seg_siz) 4182 swap_wqe_data64((u64 *)wqe); 4183 } 4184 } 4185 4186 /* swap last not completed segment */ 4187 if (seg_siz) 4188 swap_wqe_data64((u64 *)wqe); 4189 4190 QL_DPRINT12(ha, "exit\n"); 4191 return data_size; 4192 } 4193 4194 static u32 4195 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, 4196 u8 *wqe_size, struct ib_send_wr *wr) 4197 { 4198 int i; 4199 u32 data_size = 0; 4200 qlnx_host_t *ha; 4201 4202 ha = dev->ha; 4203 4204 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); 4205 4206 for (i = 0; i < wr->num_sge; i++) { 4207 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); 4208 4209 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); 4210 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 4211 sge->length = cpu_to_le32(wr->sg_list[i].length); 4212 data_size += wr->sg_list[i].length; 4213 } 4214 4215 if (wqe_size) 4216 *wqe_size += wr->num_sge; 4217 4218 QL_DPRINT12(ha, "exit data_size = %d\n", data_size); 4219 return data_size; 4220 } 4221 4222 static u32 4223 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, 4224 struct qlnxr_qp *qp, 4225 struct rdma_sq_rdma_wqe_1st *rwqe, 4226 struct rdma_sq_rdma_wqe_2nd *rwqe2, 4227 struct ib_send_wr *wr, 4228 struct ib_send_wr **bad_wr) 4229 { 4230 qlnx_host_t *ha; 4231 u32 ret = 0; 4232 4233 ha = dev->ha; 4234 4235 QL_DPRINT12(ha, "enter\n"); 4236 4237 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 4238 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); 4239 4240 if (wr->send_flags & IB_SEND_INLINE) { 4241 u8 flags = 0; 4242 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 4243 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, 4244 wr, bad_wr, &rwqe->flags, flags); 4245 } 4246 4247 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); 4248 4249 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4250 4251 return (ret); 4252 } 4253 4254 static u32 4255 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, 4256 struct qlnxr_qp *qp, 4257 struct rdma_sq_send_wqe *swqe, 4258 struct rdma_sq_send_wqe *swqe2, 4259 struct ib_send_wr *wr, 4260 struct ib_send_wr **bad_wr) 4261 { 4262 qlnx_host_t *ha; 4263 u32 ret = 0; 4264 4265 ha = dev->ha; 4266 4267 QL_DPRINT12(ha, "enter\n"); 4268 4269 memset(swqe2, 0, sizeof(*swqe2)); 4270 4271 if (wr->send_flags & IB_SEND_INLINE) { 4272 u8 flags = 0; 4273 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 4274 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, 4275 wr, bad_wr, &swqe->flags, flags); 4276 } 4277 4278 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); 4279 4280 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4281 4282 return (ret); 4283 } 4284 4285 static void 4286 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) 4287 { 4288 qlnx_host_t *ha; 4289 4290 ha = dev->ha; 4291 4292 int work = info->completed - info->completed_handled - 1; 4293 4294 QL_DPRINT12(ha, "enter [%d]\n", work); 4295 4296 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 4297 struct qlnxr_pbl *pbl; 4298 4299 /* Free all the page list that are possible to be freed 4300 * (all the ones that were invalidated), under the assumption 4301 * that if an FMR was completed successfully that means that 4302 * if there was an invalidate operation before it also ended 4303 */ 4304 pbl = list_first_entry(&info->inuse_pbl_list, 4305 struct qlnxr_pbl, 4306 list_entry); 4307 list_del(&pbl->list_entry); 4308 list_add_tail(&pbl->list_entry, &info->free_pbl_list); 4309 info->completed_handled++; 4310 } 4311 4312 QL_DPRINT12(ha, "exit\n"); 4313 return; 4314 } 4315 4316 #if __FreeBSD_version >= 1102000 4317 4318 static int qlnxr_prepare_reg(struct qlnxr_qp *qp, 4319 struct rdma_sq_fmr_wqe_1st *fwqe1, 4320 struct ib_reg_wr *wr) 4321 { 4322 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); 4323 struct rdma_sq_fmr_wqe_2nd *fwqe2; 4324 4325 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); 4326 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 4327 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 4328 fwqe1->l_key = wr->key; 4329 4330 fwqe2->access_ctrl = 0; 4331 4332 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 4333 !!(wr->access & IB_ACCESS_REMOTE_READ)); 4334 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 4335 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 4336 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 4337 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 4338 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 4339 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 4340 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 4341 fwqe2->fmr_ctrl = 0; 4342 4343 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4344 ilog2(mr->ibmr.page_size) - 12); 4345 4346 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ 4347 fwqe2->length_lo = mr->ibmr.length; 4348 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 4349 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 4350 4351 qp->wqe_wr_id[qp->sq.prod].mr = mr; 4352 4353 return 0; 4354 } 4355 4356 #else 4357 4358 static void 4359 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr, 4360 struct mr_info *info) 4361 { 4362 int i; 4363 u64 buf_addr = 0; 4364 int num_pbes, total_num_pbes = 0; 4365 struct regpair *pbe; 4366 struct qlnxr_pbl *pbl_tbl = info->pbl_table; 4367 struct qlnxr_pbl_info *pbl_info = &info->pbl_info; 4368 qlnx_host_t *ha; 4369 4370 ha = dev->ha; 4371 4372 QL_DPRINT12(ha, "enter\n"); 4373 4374 pbe = (struct regpair *)pbl_tbl->va; 4375 num_pbes = 0; 4376 4377 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { 4378 buf_addr = wr->wr.fast_reg.page_list->page_list[i]; 4379 pbe->lo = cpu_to_le32((u32)buf_addr); 4380 pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr)); 4381 4382 num_pbes += 1; 4383 pbe++; 4384 total_num_pbes++; 4385 4386 if (total_num_pbes == pbl_info->num_pbes) 4387 return; 4388 4389 /* if the given pbl is full storing the pbes, 4390 * move to next pbl. 4391 */ 4392 if (num_pbes == 4393 (pbl_info->pbl_size / sizeof(u64))) { 4394 pbl_tbl++; 4395 pbe = (struct regpair *)pbl_tbl->va; 4396 num_pbes = 0; 4397 } 4398 } 4399 QL_DPRINT12(ha, "exit\n"); 4400 4401 return; 4402 } 4403 4404 static int 4405 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info) 4406 { 4407 int rc = 0; 4408 qlnx_host_t *ha; 4409 4410 ha = dev->ha; 4411 4412 QL_DPRINT12(ha, "enter\n"); 4413 4414 if (info->completed == 0) { 4415 //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n"); 4416 /* first fmr */ 4417 return 0; 4418 } 4419 4420 qlnx_handle_completed_mrs(dev, info); 4421 4422 list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list); 4423 4424 if (list_empty(&info->free_pbl_list)) { 4425 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, 4426 GFP_ATOMIC); 4427 } else { 4428 info->pbl_table = list_first_entry(&info->free_pbl_list, 4429 struct qlnxr_pbl, 4430 list_entry); 4431 list_del(&info->pbl_table->list_entry); 4432 } 4433 4434 if (!info->pbl_table) 4435 rc = -ENOMEM; 4436 4437 QL_DPRINT12(ha, "exit\n"); 4438 return rc; 4439 } 4440 4441 static inline int 4442 qlnxr_prepare_fmr(struct qlnxr_qp *qp, 4443 struct rdma_sq_fmr_wqe_1st *fwqe1, 4444 struct ib_send_wr *wr) 4445 { 4446 struct qlnxr_dev *dev = qp->dev; 4447 u64 fbo; 4448 struct qlnxr_fast_reg_page_list *frmr_list = 4449 get_qlnxr_frmr_list(wr->wr.fast_reg.page_list); 4450 struct rdma_sq_fmr_wqe *fwqe2 = 4451 (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl); 4452 int rc = 0; 4453 qlnx_host_t *ha; 4454 4455 ha = dev->ha; 4456 4457 QL_DPRINT12(ha, "enter\n"); 4458 4459 if (wr->wr.fast_reg.page_list_len == 0) 4460 BUG(); 4461 4462 rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info); 4463 if (rc) 4464 return rc; 4465 4466 fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start); 4467 fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start); 4468 fwqe1->l_key = wr->wr.fast_reg.rkey; 4469 4470 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ, 4471 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)); 4472 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE, 4473 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)); 4474 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC, 4475 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC)); 4476 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1); 4477 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE, 4478 !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)); 4479 4480 fwqe2->fmr_ctrl = 0; 4481 4482 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4483 ilog2(1 << wr->wr.fast_reg.page_shift) - 12); 4484 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0); 4485 4486 fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */ 4487 fwqe2->length_lo = wr->wr.fast_reg.length; 4488 fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa); 4489 fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa); 4490 4491 /* produce another wqe for fwqe3 */ 4492 ecore_chain_produce(&qp->sq.pbl); 4493 4494 fbo = wr->wr.fast_reg.iova_start - 4495 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK); 4496 4497 QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x" 4498 " length = %x pbl_addr %x:%x\n", 4499 wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey, 4500 fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo, 4501 fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo); 4502 4503 build_frmr_pbes(dev, wr, &frmr_list->info); 4504 4505 qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list; 4506 4507 QL_DPRINT12(ha, "exit\n"); 4508 return 0; 4509 } 4510 4511 #endif /* #if __FreeBSD_version >= 1102000 */ 4512 4513 static enum ib_wc_opcode 4514 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 4515 { 4516 switch (opcode) { 4517 case IB_WR_RDMA_WRITE: 4518 case IB_WR_RDMA_WRITE_WITH_IMM: 4519 return IB_WC_RDMA_WRITE; 4520 case IB_WR_SEND_WITH_IMM: 4521 case IB_WR_SEND: 4522 case IB_WR_SEND_WITH_INV: 4523 return IB_WC_SEND; 4524 case IB_WR_RDMA_READ: 4525 return IB_WC_RDMA_READ; 4526 case IB_WR_ATOMIC_CMP_AND_SWP: 4527 return IB_WC_COMP_SWAP; 4528 case IB_WR_ATOMIC_FETCH_AND_ADD: 4529 return IB_WC_FETCH_ADD; 4530 4531 #if __FreeBSD_version >= 1102000 4532 case IB_WR_REG_MR: 4533 return IB_WC_REG_MR; 4534 #else 4535 case IB_WR_FAST_REG_MR: 4536 return IB_WC_FAST_REG_MR; 4537 #endif /* #if __FreeBSD_version >= 1102000 */ 4538 4539 case IB_WR_LOCAL_INV: 4540 return IB_WC_LOCAL_INV; 4541 default: 4542 return IB_WC_SEND; 4543 } 4544 } 4545 static inline bool 4546 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr) 4547 { 4548 int wq_is_full, err_wr, pbl_is_full; 4549 struct qlnxr_dev *dev = qp->dev; 4550 qlnx_host_t *ha; 4551 4552 ha = dev->ha; 4553 4554 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); 4555 4556 /* prevent SQ overflow and/or processing of a bad WR */ 4557 err_wr = wr->num_sge > qp->sq.max_sges; 4558 wq_is_full = qlnxr_wq_is_full(&qp->sq); 4559 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < 4560 QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 4561 if (wq_is_full || err_wr || pbl_is_full) { 4562 if (wq_is_full && 4563 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { 4564 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; 4565 4566 QL_DPRINT12(ha, 4567 "error: WQ is full. Post send on QP failed" 4568 " (this error appears only once) " 4569 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4570 qp, wr, qp->err_bitmap); 4571 } 4572 4573 if (err_wr && 4574 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { 4575 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; 4576 4577 QL_DPRINT12(ha, 4578 "error: WQ is bad. Post send on QP failed" 4579 " (this error appears only once) " 4580 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4581 qp, wr, qp->err_bitmap); 4582 } 4583 4584 if (pbl_is_full && 4585 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { 4586 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; 4587 4588 QL_DPRINT12(ha, 4589 "error: WQ PBL is full. Post send on QP failed" 4590 " (this error appears only once) " 4591 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4592 qp, wr, qp->err_bitmap); 4593 } 4594 return false; 4595 } 4596 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); 4597 return true; 4598 } 4599 4600 int 4601 qlnxr_post_send(struct ib_qp *ibqp, 4602 struct ib_send_wr *wr, 4603 struct ib_send_wr **bad_wr) 4604 { 4605 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); 4606 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4607 unsigned long flags; 4608 int status = 0, rc = 0; 4609 bool comp; 4610 qlnx_host_t *ha; 4611 uint32_t reg_addr; 4612 4613 *bad_wr = NULL; 4614 ha = dev->ha; 4615 4616 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4617 ibqp, wr, bad_wr); 4618 4619 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 4620 return -EINVAL; 4621 4622 if (qp->qp_type == IB_QPT_GSI) 4623 return qlnxr_gsi_post_send(ibqp, wr, bad_wr); 4624 4625 spin_lock_irqsave(&qp->q_lock, flags); 4626 4627 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && 4628 (qp->state != ECORE_ROCE_QP_STATE_ERR) && 4629 (qp->state != ECORE_ROCE_QP_STATE_SQD)) { 4630 spin_unlock_irqrestore(&qp->q_lock, flags); 4631 *bad_wr = wr; 4632 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", 4633 qp->icid, qp->state); 4634 return -EINVAL; 4635 } 4636 4637 if (!wr) { 4638 QL_DPRINT11(ha, "Got an empty post send???\n"); 4639 } 4640 4641 while (wr) { 4642 struct rdma_sq_common_wqe *wqe; 4643 struct rdma_sq_send_wqe *swqe; 4644 struct rdma_sq_send_wqe *swqe2; 4645 struct rdma_sq_rdma_wqe_1st *rwqe; 4646 struct rdma_sq_rdma_wqe_2nd *rwqe2; 4647 struct rdma_sq_local_inv_wqe *iwqe; 4648 struct rdma_sq_atomic_wqe *awqe1; 4649 struct rdma_sq_atomic_wqe *awqe2; 4650 struct rdma_sq_atomic_wqe *awqe3; 4651 struct rdma_sq_fmr_wqe_1st *fwqe1; 4652 4653 if (!qlnxr_can_post_send(qp, wr)) { 4654 status = -ENOMEM; 4655 *bad_wr = wr; 4656 break; 4657 } 4658 4659 wqe = ecore_chain_produce(&qp->sq.pbl); 4660 4661 qp->wqe_wr_id[qp->sq.prod].signaled = 4662 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 4663 4664 /* common fields */ 4665 wqe->flags = 0; 4666 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << 4667 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); 4668 4669 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ 4670 !!(wr->send_flags & IB_SEND_SOLICITED)); 4671 4672 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || 4673 (qp->signaled); 4674 4675 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 4676 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ 4677 !!(wr->send_flags & IB_SEND_FENCE)); 4678 4679 wqe->prev_wqe_size = qp->prev_wqe_size; 4680 4681 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); 4682 4683 switch (wr->opcode) { 4684 case IB_WR_SEND_WITH_IMM: 4685 4686 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 4687 swqe = (struct rdma_sq_send_wqe *)wqe; 4688 swqe->wqe_size = 2; 4689 swqe2 = (struct rdma_sq_send_wqe *) 4690 ecore_chain_produce(&qp->sq.pbl); 4691 swqe->inv_key_or_imm_data = 4692 cpu_to_le32(wr->ex.imm_data); 4693 swqe->length = cpu_to_le32( 4694 qlnxr_prepare_sq_send_data(dev, 4695 qp, swqe, swqe2, wr, 4696 bad_wr)); 4697 4698 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4699 qp->prev_wqe_size = swqe->wqe_size; 4700 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4701 4702 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", 4703 swqe->length, wr->ex.imm_data); 4704 4705 break; 4706 4707 case IB_WR_SEND: 4708 4709 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 4710 swqe = (struct rdma_sq_send_wqe *)wqe; 4711 4712 swqe->wqe_size = 2; 4713 swqe2 = (struct rdma_sq_send_wqe *) 4714 ecore_chain_produce(&qp->sq.pbl); 4715 swqe->length = cpu_to_le32( 4716 qlnxr_prepare_sq_send_data(dev, 4717 qp, swqe, swqe2, wr, 4718 bad_wr)); 4719 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4720 qp->prev_wqe_size = swqe->wqe_size; 4721 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4722 4723 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", 4724 swqe->length); 4725 4726 break; 4727 4728 case IB_WR_SEND_WITH_INV: 4729 4730 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 4731 swqe = (struct rdma_sq_send_wqe *)wqe; 4732 swqe2 = (struct rdma_sq_send_wqe *) 4733 ecore_chain_produce(&qp->sq.pbl); 4734 swqe->wqe_size = 2; 4735 swqe->inv_key_or_imm_data = 4736 cpu_to_le32(wr->ex.invalidate_rkey); 4737 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, 4738 qp, swqe, swqe2, wr, bad_wr)); 4739 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4740 qp->prev_wqe_size = swqe->wqe_size; 4741 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4742 4743 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", 4744 swqe->length); 4745 break; 4746 4747 case IB_WR_RDMA_WRITE_WITH_IMM: 4748 4749 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 4750 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4751 4752 rwqe->wqe_size = 2; 4753 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 4754 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4755 ecore_chain_produce(&qp->sq.pbl); 4756 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4757 qp, rwqe, rwqe2, wr, bad_wr)); 4758 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4759 qp->prev_wqe_size = rwqe->wqe_size; 4760 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4761 4762 QL_DPRINT12(ha, 4763 "RDMA WRITE w/ IMM length = %d imm data=%x\n", 4764 rwqe->length, rwqe->imm_data); 4765 4766 break; 4767 4768 case IB_WR_RDMA_WRITE: 4769 4770 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 4771 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4772 4773 rwqe->wqe_size = 2; 4774 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4775 ecore_chain_produce(&qp->sq.pbl); 4776 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4777 qp, rwqe, rwqe2, wr, bad_wr)); 4778 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4779 qp->prev_wqe_size = rwqe->wqe_size; 4780 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4781 4782 QL_DPRINT12(ha, 4783 "RDMA WRITE w/o IMM length = %d\n", 4784 rwqe->length); 4785 4786 break; 4787 4788 case IB_WR_RDMA_READ_WITH_INV: 4789 4790 QL_DPRINT12(ha, 4791 "RDMA READ WITH INVALIDATE not supported\n"); 4792 4793 *bad_wr = wr; 4794 rc = -EINVAL; 4795 4796 break; 4797 4798 case IB_WR_RDMA_READ: 4799 4800 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 4801 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4802 4803 rwqe->wqe_size = 2; 4804 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4805 ecore_chain_produce(&qp->sq.pbl); 4806 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4807 qp, rwqe, rwqe2, wr, bad_wr)); 4808 4809 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4810 qp->prev_wqe_size = rwqe->wqe_size; 4811 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4812 4813 QL_DPRINT12(ha, "RDMA READ length = %d\n", 4814 rwqe->length); 4815 4816 break; 4817 4818 case IB_WR_ATOMIC_CMP_AND_SWP: 4819 case IB_WR_ATOMIC_FETCH_AND_ADD: 4820 4821 QL_DPRINT12(ha, 4822 "ATOMIC operation = %s\n", 4823 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? 4824 "IB_WR_ATOMIC_CMP_AND_SWP" : 4825 "IB_WR_ATOMIC_FETCH_AND_ADD")); 4826 4827 awqe1 = (struct rdma_sq_atomic_wqe *)wqe; 4828 awqe1->prev_wqe_size = 4; 4829 4830 awqe2 = (struct rdma_sq_atomic_wqe *) 4831 ecore_chain_produce(&qp->sq.pbl); 4832 4833 TYPEPTR_ADDR_SET(awqe2, remote_va, \ 4834 atomic_wr(wr)->remote_addr); 4835 4836 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 4837 4838 awqe3 = (struct rdma_sq_atomic_wqe *) 4839 ecore_chain_produce(&qp->sq.pbl); 4840 4841 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 4842 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 4843 TYPEPTR_ADDR_SET(awqe3, swap_data, 4844 atomic_wr(wr)->compare_add); 4845 } else { 4846 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 4847 TYPEPTR_ADDR_SET(awqe3, swap_data, 4848 atomic_wr(wr)->swap); 4849 TYPEPTR_ADDR_SET(awqe3, cmp_data, 4850 atomic_wr(wr)->compare_add); 4851 } 4852 4853 qlnxr_prepare_sq_sges(dev, qp, NULL, wr); 4854 4855 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; 4856 qp->prev_wqe_size = awqe1->prev_wqe_size; 4857 4858 break; 4859 4860 case IB_WR_LOCAL_INV: 4861 4862 QL_DPRINT12(ha, 4863 "INVALIDATE length (IB_WR_LOCAL_INV)\n"); 4864 4865 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 4866 iwqe->prev_wqe_size = 1; 4867 4868 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 4869 iwqe->inv_l_key = wr->ex.invalidate_rkey; 4870 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; 4871 qp->prev_wqe_size = iwqe->prev_wqe_size; 4872 4873 break; 4874 4875 #if __FreeBSD_version >= 1102000 4876 4877 case IB_WR_REG_MR: 4878 4879 QL_DPRINT12(ha, "IB_WR_REG_MR\n"); 4880 4881 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4882 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4883 fwqe1->wqe_size = 2; 4884 4885 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); 4886 if (rc) { 4887 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); 4888 *bad_wr = wr; 4889 break; 4890 } 4891 4892 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 4893 qp->prev_wqe_size = fwqe1->wqe_size; 4894 4895 break; 4896 #else 4897 case IB_WR_FAST_REG_MR: 4898 4899 QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n"); 4900 4901 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4902 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4903 fwqe1->prev_wqe_size = 3; 4904 4905 rc = qlnxr_prepare_fmr(qp, fwqe1, wr); 4906 4907 if (rc) { 4908 QL_DPRINT12(ha, 4909 "FAST_MR (IB_WR_FAST_REG_MR) failed" 4910 " rc = %d\n", rc); 4911 *bad_wr = wr; 4912 break; 4913 } 4914 4915 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size; 4916 qp->prev_wqe_size = fwqe1->prev_wqe_size; 4917 4918 break; 4919 #endif /* #if __FreeBSD_version >= 1102000 */ 4920 4921 default: 4922 4923 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); 4924 4925 rc = -EINVAL; 4926 *bad_wr = wr; 4927 break; 4928 } 4929 4930 if (*bad_wr) { 4931 /* 4932 * restore prod to its position before this WR was processed 4933 */ 4934 ecore_chain_set_prod(&qp->sq.pbl, 4935 le16_to_cpu(qp->sq.db_data.data.value), 4936 wqe); 4937 /* restore prev_wqe_size */ 4938 qp->prev_wqe_size = wqe->prev_wqe_size; 4939 status = rc; 4940 4941 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); 4942 break; /* out of the loop */ 4943 } 4944 4945 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 4946 4947 qlnxr_inc_sw_prod(&qp->sq); 4948 4949 qp->sq.db_data.data.value++; 4950 4951 wr = wr->next; 4952 } 4953 4954 /* Trigger doorbell 4955 * If there was a failure in the first WR then it will be triggered in 4956 * vane. However this is not harmful (as long as the producer value is 4957 * unchanged). For performance reasons we avoid checking for this 4958 * redundant doorbell. 4959 */ 4960 wmb(); 4961 //writel(qp->sq.db_data.raw, qp->sq.db); 4962 4963 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); 4964 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); 4965 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4966 4967 mmiowb(); 4968 4969 spin_unlock_irqrestore(&qp->q_lock, flags); 4970 4971 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4972 ibqp, wr, bad_wr); 4973 4974 return status; 4975 } 4976 4977 static u32 4978 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) 4979 { 4980 u32 used; 4981 4982 /* Calculate number of elements used based on producer 4983 * count and consumer count and subtract it from max 4984 * work request supported so that we get elements left. 4985 */ 4986 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 4987 4988 return hw_srq->max_wr - used; 4989 } 4990 4991 int 4992 qlnxr_post_recv(struct ib_qp *ibqp, 4993 struct ib_recv_wr *wr, 4994 struct ib_recv_wr **bad_wr) 4995 { 4996 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4997 struct qlnxr_dev *dev = qp->dev; 4998 unsigned long flags; 4999 int status = 0; 5000 qlnx_host_t *ha; 5001 uint32_t reg_addr; 5002 5003 ha = dev->ha; 5004 5005 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5006 return -EINVAL; 5007 5008 QL_DPRINT12(ha, "enter\n"); 5009 5010 if (qp->qp_type == IB_QPT_GSI) { 5011 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); 5012 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); 5013 } 5014 5015 if (qp->srq) { 5016 QL_DPRINT11(ha, "qp->srq [%p]" 5017 " QP is associated with SRQ, cannot post RQ buffers\n", 5018 qp->srq); 5019 return -EINVAL; 5020 } 5021 5022 spin_lock_irqsave(&qp->q_lock, flags); 5023 5024 if (qp->state == ECORE_ROCE_QP_STATE_RESET) { 5025 spin_unlock_irqrestore(&qp->q_lock, flags); 5026 *bad_wr = wr; 5027 5028 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); 5029 5030 return -EINVAL; 5031 } 5032 5033 while (wr) { 5034 int i; 5035 5036 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < 5037 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || 5038 (wr->num_sge > qp->rq.max_sges)) { 5039 status = -ENOMEM; 5040 *bad_wr = wr; 5041 break; 5042 } 5043 for (i = 0; i < wr->num_sge; i++) { 5044 u32 flags = 0; 5045 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5046 5047 /* first one must include the number of SGE in the list */ 5048 if (!i) 5049 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); 5050 5051 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); 5052 5053 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ 5054 wr->sg_list[i].length, flags); 5055 } 5056 /* Special case of no sges. FW requires between 1-4 sges... 5057 * in this case we need to post 1 sge with length zero. this is 5058 * because rdma write with immediate consumes an RQ. */ 5059 if (!wr->num_sge) { 5060 u32 flags = 0; 5061 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 5062 5063 /* first one must include the number of SGE in the list */ 5064 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 5065 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 5066 5067 //RQ_SGE_SET(rqe, 0, 0, flags); 5068 rqe->addr.hi = 0; 5069 rqe->addr.lo = 0; 5070 5071 rqe->length = 0; 5072 rqe->flags = cpu_to_le32(flags); 5073 5074 i = 1; 5075 } 5076 5077 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 5078 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 5079 5080 qlnxr_inc_sw_prod(&qp->rq); 5081 5082 wmb(); 5083 5084 qp->rq.db_data.data.value++; 5085 5086 // writel(qp->rq.db_data.raw, qp->rq.db); 5087 mmiowb(); 5088 // if (QLNX_IS_IWARP(dev)) { 5089 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 5090 // mmiowb(); /* for second doorbell */ 5091 // } 5092 5093 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 5094 (uint8_t *)ha->cdev.doorbells); 5095 5096 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 5097 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5098 5099 if (QLNX_IS_IWARP(dev)) { 5100 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 5101 (uint8_t *)ha->cdev.doorbells); 5102 bus_write_4(ha->pci_dbells, reg_addr, \ 5103 qp->rq.iwarp_db2_data.raw); 5104 bus_barrier(ha->pci_dbells, 0, 0, \ 5105 BUS_SPACE_BARRIER_READ); 5106 } 5107 5108 wr = wr->next; 5109 } 5110 5111 spin_unlock_irqrestore(&qp->q_lock, flags); 5112 5113 QL_DPRINT12(ha, "exit status = 0x%x\n", status); 5114 5115 return status; 5116 } 5117 5118 /* In fmr we need to increase the number of fmr completed counter for the fmr 5119 * algorithm determining whether we can free a pbl or not. 5120 * we need to perform this whether the work request was signaled or not. for 5121 * this purpose we call this function from the condition that checks if a wr 5122 * should be skipped, to make sure we don't miss it ( possibly this fmr 5123 * operation was not signalted) 5124 */ 5125 static inline void 5126 qlnxr_chk_if_fmr(struct qlnxr_qp *qp) 5127 { 5128 #if __FreeBSD_version >= 1102000 5129 5130 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 5131 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5132 #else 5133 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR) 5134 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5135 5136 #endif /* #if __FreeBSD_version >= 1102000 */ 5137 } 5138 5139 static int 5140 process_req(struct qlnxr_dev *dev, 5141 struct qlnxr_qp *qp, 5142 struct qlnxr_cq *cq, 5143 int num_entries, 5144 struct ib_wc *wc, 5145 u16 hw_cons, 5146 enum ib_wc_status status, 5147 int force) 5148 { 5149 u16 cnt = 0; 5150 qlnx_host_t *ha = dev->ha; 5151 5152 QL_DPRINT12(ha, "enter\n"); 5153 5154 while (num_entries && qp->sq.wqe_cons != hw_cons) { 5155 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 5156 qlnxr_chk_if_fmr(qp); 5157 /* skip WC */ 5158 goto next_cqe; 5159 } 5160 5161 /* fill WC */ 5162 wc->status = status; 5163 wc->vendor_err = 0; 5164 wc->wc_flags = 0; 5165 wc->src_qp = qp->id; 5166 wc->qp = &qp->ibqp; 5167 5168 // common section 5169 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 5170 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 5171 5172 switch (wc->opcode) { 5173 case IB_WC_RDMA_WRITE: 5174 5175 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 5176 5177 QL_DPRINT12(ha, 5178 "opcode = IB_WC_RDMA_WRITE bytes = %d\n", 5179 qp->wqe_wr_id[qp->sq.cons].bytes_len); 5180 break; 5181 5182 case IB_WC_COMP_SWAP: 5183 case IB_WC_FETCH_ADD: 5184 wc->byte_len = 8; 5185 break; 5186 5187 #if __FreeBSD_version >= 1102000 5188 case IB_WC_REG_MR: 5189 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 5190 break; 5191 #else 5192 case IB_WC_FAST_REG_MR: 5193 qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; 5194 break; 5195 #endif /* #if __FreeBSD_version >= 1102000 */ 5196 5197 case IB_WC_RDMA_READ: 5198 case IB_WC_SEND: 5199 5200 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); 5201 break; 5202 default: 5203 ;//DP_ERR("TBD ERROR"); 5204 } 5205 5206 num_entries--; 5207 wc++; 5208 cnt++; 5209 next_cqe: 5210 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 5211 ecore_chain_consume(&qp->sq.pbl); 5212 qlnxr_inc_sw_cons(&qp->sq); 5213 } 5214 5215 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5216 return cnt; 5217 } 5218 5219 static int 5220 qlnxr_poll_cq_req(struct qlnxr_dev *dev, 5221 struct qlnxr_qp *qp, 5222 struct qlnxr_cq *cq, 5223 int num_entries, 5224 struct ib_wc *wc, 5225 struct rdma_cqe_requester *req) 5226 { 5227 int cnt = 0; 5228 qlnx_host_t *ha = dev->ha; 5229 5230 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); 5231 5232 switch (req->status) { 5233 case RDMA_CQE_REQ_STS_OK: 5234 5235 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5236 IB_WC_SUCCESS, 0); 5237 break; 5238 5239 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 5240 5241 if (qp->state != ECORE_ROCE_QP_STATE_ERR) 5242 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 5243 IB_WC_WR_FLUSH_ERR, 1); 5244 break; 5245 5246 default: /* other errors case */ 5247 5248 /* process all WQE before the cosumer */ 5249 qp->state = ECORE_ROCE_QP_STATE_ERR; 5250 cnt = process_req(dev, qp, cq, num_entries, wc, 5251 req->sq_cons - 1, IB_WC_SUCCESS, 0); 5252 wc += cnt; 5253 /* if we have extra WC fill it with actual error info */ 5254 5255 if (cnt < num_entries) { 5256 enum ib_wc_status wc_status; 5257 5258 switch (req->status) { 5259 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 5260 wc_status = IB_WC_BAD_RESP_ERR; 5261 break; 5262 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 5263 wc_status = IB_WC_LOC_LEN_ERR; 5264 break; 5265 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 5266 wc_status = IB_WC_LOC_QP_OP_ERR; 5267 break; 5268 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 5269 wc_status = IB_WC_LOC_PROT_ERR; 5270 break; 5271 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 5272 wc_status = IB_WC_MW_BIND_ERR; 5273 break; 5274 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 5275 wc_status = IB_WC_REM_INV_REQ_ERR; 5276 break; 5277 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 5278 wc_status = IB_WC_REM_ACCESS_ERR; 5279 break; 5280 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 5281 wc_status = IB_WC_REM_OP_ERR; 5282 break; 5283 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 5284 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 5285 break; 5286 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 5287 wc_status = IB_WC_RETRY_EXC_ERR; 5288 break; 5289 default: 5290 wc_status = IB_WC_GENERAL_ERR; 5291 } 5292 5293 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 5294 wc_status, 1 /* force use of WC */); 5295 } 5296 } 5297 5298 QL_DPRINT12(ha, "exit cnt = %d\n", cnt); 5299 return cnt; 5300 } 5301 5302 static void 5303 __process_resp_one(struct qlnxr_dev *dev, 5304 struct qlnxr_qp *qp, 5305 struct qlnxr_cq *cq, 5306 struct ib_wc *wc, 5307 struct rdma_cqe_responder *resp, 5308 u64 wr_id) 5309 { 5310 enum ib_wc_status wc_status = IB_WC_SUCCESS; 5311 #if __FreeBSD_version < 1102000 5312 u8 flags; 5313 #endif 5314 qlnx_host_t *ha = dev->ha; 5315 5316 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", 5317 qp, resp->status); 5318 5319 wc->opcode = IB_WC_RECV; 5320 wc->wc_flags = 0; 5321 5322 switch (resp->status) { 5323 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 5324 wc_status = IB_WC_LOC_ACCESS_ERR; 5325 break; 5326 5327 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 5328 wc_status = IB_WC_LOC_LEN_ERR; 5329 break; 5330 5331 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 5332 wc_status = IB_WC_LOC_QP_OP_ERR; 5333 break; 5334 5335 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 5336 wc_status = IB_WC_LOC_PROT_ERR; 5337 break; 5338 5339 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 5340 wc_status = IB_WC_MW_BIND_ERR; 5341 break; 5342 5343 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 5344 wc_status = IB_WC_REM_INV_RD_REQ_ERR; 5345 break; 5346 5347 case RDMA_CQE_RESP_STS_OK: 5348 5349 #if __FreeBSD_version >= 1102000 5350 if (resp->flags & QLNXR_RESP_IMM) { 5351 wc->ex.imm_data = 5352 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5353 wc->wc_flags |= IB_WC_WITH_IMM; 5354 5355 if (resp->flags & QLNXR_RESP_RDMA) 5356 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5357 5358 if (resp->flags & QLNXR_RESP_INV) { 5359 QL_DPRINT11(ha, 5360 "Invalid flags QLNXR_RESP_INV [0x%x]" 5361 "qp = %p qp->id = 0x%x cq = %p" 5362 " cq->icid = 0x%x\n", 5363 resp->flags, qp, qp->id, cq, cq->icid ); 5364 } 5365 } else if (resp->flags & QLNXR_RESP_INV) { 5366 wc->ex.imm_data = 5367 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5368 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 5369 5370 if (resp->flags & QLNXR_RESP_RDMA) { 5371 QL_DPRINT11(ha, 5372 "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5373 "qp = %p qp->id = 0x%x cq = %p" 5374 " cq->icid = 0x%x\n", 5375 resp->flags, qp, qp->id, cq, cq->icid ); 5376 } 5377 } else if (resp->flags & QLNXR_RESP_RDMA) { 5378 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5379 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5380 resp->flags, qp, qp->id, cq, cq->icid ); 5381 } 5382 #else 5383 wc_status = IB_WC_SUCCESS; 5384 wc->byte_len = le32_to_cpu(resp->length); 5385 5386 flags = resp->flags & QLNXR_RESP_RDMA_IMM; 5387 5388 switch (flags) { 5389 case QLNXR_RESP_RDMA_IMM: 5390 /* update opcode */ 5391 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 5392 /* fall to set imm data */ 5393 case QLNXR_RESP_IMM: 5394 wc->ex.imm_data = 5395 le32_to_cpu(resp->imm_data_or_inv_r_Key); 5396 wc->wc_flags |= IB_WC_WITH_IMM; 5397 break; 5398 case QLNXR_RESP_RDMA: 5399 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 5400 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 5401 resp->flags, qp, qp->id, cq, cq->icid ); 5402 break; 5403 default: 5404 /* valid configuration, but nothing todo here */ 5405 ; 5406 } 5407 #endif /* #if __FreeBSD_version >= 1102000 */ 5408 5409 break; 5410 default: 5411 wc_status = IB_WC_GENERAL_ERR; 5412 } 5413 5414 /* fill WC */ 5415 wc->status = wc_status; 5416 wc->vendor_err = 0; 5417 wc->src_qp = qp->id; 5418 wc->qp = &qp->ibqp; 5419 wc->wr_id = wr_id; 5420 5421 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); 5422 5423 return; 5424 } 5425 5426 static int 5427 process_resp_one_srq(struct qlnxr_dev *dev, 5428 struct qlnxr_qp *qp, 5429 struct qlnxr_cq *cq, 5430 struct ib_wc *wc, 5431 struct rdma_cqe_responder *resp) 5432 { 5433 struct qlnxr_srq *srq = qp->srq; 5434 u64 wr_id; 5435 qlnx_host_t *ha = dev->ha; 5436 5437 QL_DPRINT12(ha, "enter\n"); 5438 5439 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); 5440 5441 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5442 wc->status = IB_WC_WR_FLUSH_ERR; 5443 wc->vendor_err = 0; 5444 wc->wr_id = wr_id; 5445 wc->byte_len = 0; 5446 wc->src_qp = qp->id; 5447 wc->qp = &qp->ibqp; 5448 wc->wr_id = wr_id; 5449 } else { 5450 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5451 } 5452 5453 /* PBL is maintained in case of WR granularity. 5454 * So increment WR consumer after consuming WR 5455 */ 5456 srq->hw_srq.wr_cons_cnt++; 5457 5458 QL_DPRINT12(ha, "exit\n"); 5459 return 1; 5460 } 5461 5462 static int 5463 process_resp_one(struct qlnxr_dev *dev, 5464 struct qlnxr_qp *qp, 5465 struct qlnxr_cq *cq, 5466 struct ib_wc *wc, 5467 struct rdma_cqe_responder *resp) 5468 { 5469 qlnx_host_t *ha = dev->ha; 5470 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5471 5472 QL_DPRINT12(ha, "enter\n"); 5473 5474 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5475 5476 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5477 ecore_chain_consume(&qp->rq.pbl); 5478 qlnxr_inc_sw_cons(&qp->rq); 5479 5480 QL_DPRINT12(ha, "exit\n"); 5481 return 1; 5482 } 5483 5484 static int 5485 process_resp_flush(struct qlnxr_qp *qp, 5486 int num_entries, 5487 struct ib_wc *wc, 5488 u16 hw_cons) 5489 { 5490 u16 cnt = 0; 5491 qlnx_host_t *ha = qp->dev->ha; 5492 5493 QL_DPRINT12(ha, "enter\n"); 5494 5495 while (num_entries && qp->rq.wqe_cons != hw_cons) { 5496 /* fill WC */ 5497 wc->status = IB_WC_WR_FLUSH_ERR; 5498 wc->vendor_err = 0; 5499 wc->wc_flags = 0; 5500 wc->src_qp = qp->id; 5501 wc->byte_len = 0; 5502 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5503 wc->qp = &qp->ibqp; 5504 num_entries--; 5505 wc++; 5506 cnt++; 5507 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5508 ecore_chain_consume(&qp->rq.pbl); 5509 qlnxr_inc_sw_cons(&qp->rq); 5510 } 5511 5512 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5513 return cnt; 5514 } 5515 5516 static void 5517 try_consume_resp_cqe(struct qlnxr_cq *cq, 5518 struct qlnxr_qp *qp, 5519 struct rdma_cqe_responder *resp, 5520 int *update) 5521 { 5522 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 5523 consume_cqe(cq); 5524 *update |= 1; 5525 } 5526 } 5527 5528 static int 5529 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, 5530 struct qlnxr_qp *qp, 5531 struct qlnxr_cq *cq, 5532 int num_entries, 5533 struct ib_wc *wc, 5534 struct rdma_cqe_responder *resp, 5535 int *update) 5536 { 5537 int cnt; 5538 qlnx_host_t *ha = dev->ha; 5539 5540 QL_DPRINT12(ha, "enter\n"); 5541 5542 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 5543 consume_cqe(cq); 5544 *update |= 1; 5545 5546 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5547 return cnt; 5548 } 5549 5550 static int 5551 qlnxr_poll_cq_resp(struct qlnxr_dev *dev, 5552 struct qlnxr_qp *qp, 5553 struct qlnxr_cq *cq, 5554 int num_entries, 5555 struct ib_wc *wc, 5556 struct rdma_cqe_responder *resp, 5557 int *update) 5558 { 5559 int cnt; 5560 qlnx_host_t *ha = dev->ha; 5561 5562 QL_DPRINT12(ha, "enter\n"); 5563 5564 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5565 cnt = process_resp_flush(qp, num_entries, wc, 5566 resp->rq_cons); 5567 try_consume_resp_cqe(cq, qp, resp, update); 5568 } else { 5569 cnt = process_resp_one(dev, qp, cq, wc, resp); 5570 consume_cqe(cq); 5571 *update |= 1; 5572 } 5573 5574 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5575 return cnt; 5576 } 5577 5578 static void 5579 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, 5580 struct rdma_cqe_requester *req, int *update) 5581 { 5582 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 5583 consume_cqe(cq); 5584 *update |= 1; 5585 } 5586 } 5587 5588 static void 5589 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) 5590 { 5591 uint64_t reg_addr; 5592 qlnx_host_t *ha = dev->ha; 5593 5594 QL_DPRINT12(ha, "enter\n"); 5595 5596 wmb(); 5597 cq->db.data.agg_flags = flags; 5598 cq->db.data.value = cpu_to_le32(cons); 5599 5600 reg_addr = (uint64_t)((uint8_t *)cq->db_addr - 5601 (uint8_t *)(ha->cdev.doorbells)); 5602 5603 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); 5604 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5605 5606 QL_DPRINT12(ha, "exit\n"); 5607 return; 5608 5609 //#ifdef __LP64__ 5610 // writeq(cq->db.raw, cq->db_addr); 5611 //#else 5612 /* Note that since the FW allows 64 bit write only, in 32bit systems 5613 * the value of db_addr must be low enough. This is currently not 5614 * enforced. 5615 */ 5616 // writel(cq->db.raw & 0xffffffff, cq->db_addr); 5617 // mmiowb(); 5618 //#endif 5619 } 5620 5621 static int 5622 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) 5623 { 5624 struct rdma_cqe_requester *resp_cqe = &cqe->req; 5625 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == 5626 cq->pbl_toggle; 5627 } 5628 5629 int 5630 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 5631 { 5632 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5633 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 5634 int done = 0; 5635 union rdma_cqe *cqe = cq->latest_cqe; 5636 int update = 0; 5637 u32 old_cons, new_cons; 5638 unsigned long flags; 5639 qlnx_host_t *ha = dev->ha; 5640 5641 QL_DPRINT12(ha, "enter\n"); 5642 5643 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5644 return -EINVAL; 5645 5646 if (cq->destroyed) { 5647 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", 5648 cq, cq->icid); 5649 return 0; 5650 } 5651 5652 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) 5653 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); 5654 5655 spin_lock_irqsave(&cq->cq_lock, flags); 5656 5657 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5658 5659 while (num_entries && is_valid_cqe(cq, cqe)) { 5660 int cnt = 0; 5661 struct qlnxr_qp *qp; 5662 struct rdma_cqe_requester *resp_cqe; 5663 enum rdma_cqe_type cqe_type; 5664 5665 /* prevent speculative reads of any field of CQE */ 5666 rmb(); 5667 5668 resp_cqe = &cqe->req; 5669 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, 5670 resp_cqe->qp_handle.lo); 5671 5672 if (!qp) { 5673 QL_DPRINT11(ha, "qp = NULL\n"); 5674 break; 5675 } 5676 5677 wc->qp = &qp->ibqp; 5678 5679 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 5680 5681 switch (cqe_type) { 5682 case RDMA_CQE_TYPE_REQUESTER: 5683 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, 5684 wc, &cqe->req); 5685 try_consume_req_cqe(cq, qp, &cqe->req, &update); 5686 break; 5687 case RDMA_CQE_TYPE_RESPONDER_RQ: 5688 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, 5689 wc, &cqe->resp, &update); 5690 break; 5691 case RDMA_CQE_TYPE_RESPONDER_SRQ: 5692 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, 5693 wc, &cqe->resp, &update); 5694 break; 5695 case RDMA_CQE_TYPE_INVALID: 5696 default: 5697 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); 5698 break; 5699 } 5700 num_entries -= cnt; 5701 wc += cnt; 5702 done += cnt; 5703 5704 cqe = cq->latest_cqe; 5705 } 5706 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5707 5708 cq->cq_cons += new_cons - old_cons; 5709 5710 if (update) { 5711 /* doorbell notifies abount latest VALID entry, 5712 * but chain already point to the next INVALID one 5713 */ 5714 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); 5715 QL_DPRINT12(ha, "cq = %p cons = 0x%x " 5716 "arm_flags = 0x%x db.icid = 0x%x\n", cq, 5717 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); 5718 } 5719 5720 spin_unlock_irqrestore(&cq->cq_lock, flags); 5721 5722 QL_DPRINT12(ha, "exit\n"); 5723 5724 return done; 5725 } 5726 5727 int 5728 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 5729 { 5730 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5731 unsigned long sflags; 5732 struct qlnxr_dev *dev; 5733 qlnx_host_t *ha; 5734 5735 dev = get_qlnxr_dev((ibcq->device)); 5736 ha = dev->ha; 5737 5738 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " 5739 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, 5740 flags, cq, cq->cq_cons, cq->cq_type); 5741 5742 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5743 return -EINVAL; 5744 5745 if (cq->destroyed) { 5746 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", 5747 cq, cq->icid); 5748 return -EINVAL; 5749 } 5750 5751 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { 5752 return 0; 5753 } 5754 5755 spin_lock_irqsave(&cq->cq_lock, sflags); 5756 5757 cq->arm_flags = 0; 5758 5759 if (flags & IB_CQ_SOLICITED) { 5760 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 5761 } 5762 if (flags & IB_CQ_NEXT_COMP) { 5763 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 5764 } 5765 5766 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); 5767 5768 spin_unlock_irqrestore(&cq->cq_lock, sflags); 5769 5770 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); 5771 return 0; 5772 } 5773 5774 static struct qlnxr_mr * 5775 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) 5776 { 5777 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 5778 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 5779 struct qlnxr_mr *mr; 5780 int rc = -ENOMEM; 5781 qlnx_host_t *ha; 5782 5783 ha = dev->ha; 5784 5785 QL_DPRINT12(ha, "enter ibpd = %p pd = %p " 5786 " pd_id = %d max_page_list_len = %d\n", 5787 ibpd, pd, pd->pd_id, max_page_list_len); 5788 5789 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5790 if (!mr) { 5791 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 5792 return ERR_PTR(rc); 5793 } 5794 5795 mr->dev = dev; 5796 mr->type = QLNXR_MR_FRMR; 5797 5798 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 5799 1 /* allow dual layer pbl */); 5800 if (rc) { 5801 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); 5802 goto err0; 5803 } 5804 5805 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 5806 if (rc) { 5807 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); 5808 goto err0; 5809 } 5810 5811 /* index only, 18 bit long, lkey = itid << 8 | key */ 5812 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; 5813 mr->hw_mr.key = 0; 5814 mr->hw_mr.pd = pd->pd_id; 5815 mr->hw_mr.local_read = 1; 5816 mr->hw_mr.local_write = 0; 5817 mr->hw_mr.remote_read = 0; 5818 mr->hw_mr.remote_write = 0; 5819 mr->hw_mr.remote_atomic = 0; 5820 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 5821 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ 5822 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 5823 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 5824 mr->hw_mr.fbo = 0; 5825 mr->hw_mr.length = 0; 5826 mr->hw_mr.vaddr = 0; 5827 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 5828 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ 5829 mr->hw_mr.dma_mr = false; 5830 5831 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 5832 if (rc) { 5833 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); 5834 goto err1; 5835 } 5836 5837 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 5838 mr->ibmr.rkey = mr->ibmr.lkey; 5839 5840 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", 5841 mr, mr->ibmr.lkey); 5842 5843 return mr; 5844 5845 err1: 5846 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 5847 err0: 5848 kfree(mr); 5849 5850 QL_DPRINT12(ha, "exit\n"); 5851 5852 return ERR_PTR(rc); 5853 } 5854 5855 #if __FreeBSD_version >= 1102000 5856 5857 struct ib_mr * 5858 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) 5859 { 5860 struct qlnxr_dev *dev; 5861 struct qlnxr_mr *mr; 5862 qlnx_host_t *ha; 5863 5864 dev = get_qlnxr_dev(ibpd->device); 5865 ha = dev->ha; 5866 5867 QL_DPRINT12(ha, "enter\n"); 5868 5869 if (mr_type != IB_MR_TYPE_MEM_REG) 5870 return ERR_PTR(-EINVAL); 5871 5872 mr = __qlnxr_alloc_mr(ibpd, max_num_sg); 5873 5874 if (IS_ERR(mr)) 5875 return ERR_PTR(-EINVAL); 5876 5877 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); 5878 5879 return &mr->ibmr; 5880 } 5881 5882 static int 5883 qlnxr_set_page(struct ib_mr *ibmr, u64 addr) 5884 { 5885 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5886 struct qlnxr_pbl *pbl_table; 5887 struct regpair *pbe; 5888 struct qlnxr_dev *dev; 5889 qlnx_host_t *ha; 5890 u32 pbes_in_page; 5891 5892 dev = mr->dev; 5893 ha = dev->ha; 5894 5895 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 5896 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); 5897 return -ENOMEM; 5898 } 5899 5900 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, 5901 ((void *)addr)); 5902 5903 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 5904 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 5905 pbe = (struct regpair *)pbl_table->va; 5906 pbe += mr->npages % pbes_in_page; 5907 pbe->lo = cpu_to_le32((u32)addr); 5908 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 5909 5910 mr->npages++; 5911 5912 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, 5913 ((void *)addr)); 5914 return 0; 5915 } 5916 5917 int 5918 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 5919 int sg_nents, unsigned int *sg_offset) 5920 { 5921 int ret; 5922 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5923 qlnx_host_t *ha; 5924 5925 if (mr == NULL) 5926 return (-1); 5927 5928 if (mr->dev == NULL) 5929 return (-1); 5930 5931 ha = mr->dev->ha; 5932 5933 QL_DPRINT12(ha, "enter\n"); 5934 5935 mr->npages = 0; 5936 qlnx_handle_completed_mrs(mr->dev, &mr->info); 5937 5938 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); 5939 5940 QL_DPRINT12(ha, "exit ret = %d\n", ret); 5941 5942 return (ret); 5943 } 5944 5945 #else 5946 5947 struct ib_mr * 5948 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len) 5949 { 5950 struct qlnxr_dev *dev; 5951 struct qlnxr_mr *mr; 5952 qlnx_host_t *ha; 5953 struct ib_mr *ibmr = NULL; 5954 5955 dev = get_qlnxr_dev((ibpd->device)); 5956 ha = dev->ha; 5957 5958 QL_DPRINT12(ha, "enter\n"); 5959 5960 mr = __qlnxr_alloc_mr(ibpd, max_page_list_len); 5961 5962 if (IS_ERR(mr)) { 5963 ibmr = ERR_PTR(-EINVAL); 5964 } else { 5965 ibmr = &mr->ibmr; 5966 } 5967 5968 QL_DPRINT12(ha, "exit %p\n", ibmr); 5969 return (ibmr); 5970 } 5971 5972 void 5973 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list) 5974 { 5975 struct qlnxr_fast_reg_page_list *frmr_list; 5976 5977 frmr_list = get_qlnxr_frmr_list(page_list); 5978 5979 free_mr_info(frmr_list->dev, &frmr_list->info); 5980 5981 kfree(frmr_list->ibfrpl.page_list); 5982 kfree(frmr_list); 5983 5984 return; 5985 } 5986 5987 struct ib_fast_reg_page_list * 5988 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len) 5989 { 5990 struct qlnxr_fast_reg_page_list *frmr_list = NULL; 5991 struct qlnxr_dev *dev; 5992 int size = page_list_len * sizeof(u64); 5993 int rc = -ENOMEM; 5994 qlnx_host_t *ha; 5995 5996 dev = get_qlnxr_dev(ibdev); 5997 ha = dev->ha; 5998 5999 QL_DPRINT12(ha, "enter\n"); 6000 6001 frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL); 6002 if (!frmr_list) { 6003 QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n"); 6004 goto err; 6005 } 6006 6007 frmr_list->dev = dev; 6008 frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL); 6009 if (!frmr_list->ibfrpl.page_list) { 6010 QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n"); 6011 goto err0; 6012 } 6013 6014 rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len, 6015 1 /* allow dual layer pbl */); 6016 if (rc) 6017 goto err1; 6018 6019 QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl); 6020 6021 return &frmr_list->ibfrpl; 6022 6023 err1: 6024 kfree(frmr_list->ibfrpl.page_list); 6025 err0: 6026 kfree(frmr_list); 6027 err: 6028 QL_DPRINT12(ha, "exit with error\n"); 6029 6030 return ERR_PTR(rc); 6031 } 6032 6033 static int 6034 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6035 int buf_cnt, uint64_t *total_size) 6036 { 6037 u64 size = 0; 6038 6039 *total_size = 0; 6040 6041 if (!buf_cnt || buf_list == NULL) { 6042 QL_DPRINT11(ha, 6043 "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt); 6044 return (-1); 6045 } 6046 6047 size = buf_list->size; 6048 6049 if (!size) { 6050 QL_DPRINT11(ha, 6051 "failed buf_list = %p buf_cnt = %d" 6052 " buf_list->size = 0\n", buf_list, buf_cnt); 6053 return (-1); 6054 } 6055 6056 while (buf_cnt) { 6057 *total_size += buf_list->size; 6058 6059 if (buf_list->size != size) { 6060 QL_DPRINT11(ha, 6061 "failed buf_list = %p buf_cnt = %d" 6062 " all buffers should have same size\n", 6063 buf_list, buf_cnt); 6064 return (-1); 6065 } 6066 6067 buf_list++; 6068 buf_cnt--; 6069 } 6070 return (0); 6071 } 6072 6073 static size_t 6074 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list, 6075 int buf_cnt) 6076 { 6077 int i; 6078 size_t num_pages = 0; 6079 u64 size; 6080 6081 for (i = 0; i < buf_cnt; i++) { 6082 size = 0; 6083 while (size < buf_list->size) { 6084 size += PAGE_SIZE; 6085 num_pages++; 6086 } 6087 buf_list++; 6088 } 6089 return (num_pages); 6090 } 6091 6092 static void 6093 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev, 6094 struct ib_phys_buf *buf_list, int buf_cnt, 6095 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 6096 { 6097 struct regpair *pbe; 6098 struct qlnxr_pbl *pbl_tbl; 6099 int pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 6100 qlnx_host_t *ha; 6101 int i; 6102 u64 pbe_addr; 6103 6104 ha = dev->ha; 6105 6106 QL_DPRINT12(ha, "enter\n"); 6107 6108 if (!pbl_info) { 6109 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 6110 return; 6111 } 6112 6113 if (!pbl_info->num_pbes) { 6114 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 6115 return; 6116 } 6117 6118 /* If we have a two layered pbl, the first pbl points to the rest 6119 * of the pbls and the first entry lays on the second pbl in the table 6120 */ 6121 if (pbl_info->two_layered) 6122 pbl_tbl = &pbl[1]; 6123 else 6124 pbl_tbl = pbl; 6125 6126 pbe = (struct regpair *)pbl_tbl->va; 6127 if (!pbe) { 6128 QL_DPRINT12(ha, "pbe is NULL\n"); 6129 return; 6130 } 6131 6132 pbe_cnt = 0; 6133 6134 for (i = 0; i < buf_cnt; i++) { 6135 pages = buf_list->size >> PAGE_SHIFT; 6136 6137 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 6138 /* store the page address in pbe */ 6139 6140 pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt); 6141 6142 pbe->lo = cpu_to_le32((u32)pbe_addr); 6143 pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32))); 6144 6145 QL_DPRINT12(ha, "Populate pbl table:" 6146 " pbe->addr=0x%x:0x%x " 6147 " pbe_cnt = %d total_num_pbes=%d" 6148 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 6149 total_num_pbes, pbe); 6150 6151 pbe_cnt ++; 6152 total_num_pbes ++; 6153 pbe++; 6154 6155 if (total_num_pbes == pbl_info->num_pbes) 6156 return; 6157 6158 /* if the given pbl is full storing the pbes, 6159 * move to next pbl. */ 6160 6161 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 6162 pbl_tbl++; 6163 pbe = (struct regpair *)pbl_tbl->va; 6164 pbe_cnt = 0; 6165 } 6166 } 6167 buf_list++; 6168 } 6169 QL_DPRINT12(ha, "exit\n"); 6170 return; 6171 } 6172 6173 struct ib_mr * 6174 qlnxr_reg_kernel_mr(struct ib_pd *ibpd, 6175 struct ib_phys_buf *buf_list, 6176 int buf_cnt, int acc, u64 *iova_start) 6177 { 6178 int rc = -ENOMEM; 6179 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 6180 struct qlnxr_mr *mr; 6181 struct qlnxr_pd *pd; 6182 qlnx_host_t *ha; 6183 size_t num_pages = 0; 6184 uint64_t length; 6185 6186 ha = dev->ha; 6187 6188 QL_DPRINT12(ha, "enter\n"); 6189 6190 pd = get_qlnxr_pd(ibpd); 6191 6192 QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d," 6193 " iova_start = %p, acc = %d\n", 6194 pd->pd_id, buf_list, buf_cnt, iova_start, acc); 6195 6196 //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 6197 // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&" 6198 // " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 6199 // return ERR_PTR(-EINVAL); 6200 //} 6201 6202 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 6203 if (!mr) { 6204 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 6205 return ERR_PTR(rc); 6206 } 6207 6208 mr->type = QLNXR_MR_KERNEL; 6209 mr->iova_start = iova_start; 6210 6211 rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length); 6212 if (rc) 6213 goto err0; 6214 6215 num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt); 6216 if (!num_pages) 6217 goto err0; 6218 6219 rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1); 6220 if (rc) { 6221 QL_DPRINT11(ha, 6222 "qlnxr_init_mr_info failed [%d]\n", rc); 6223 goto err1; 6224 } 6225 6226 qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table, 6227 &mr->info.pbl_info); 6228 6229 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 6230 6231 if (rc) { 6232 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 6233 goto err1; 6234 } 6235 6236 /* index only, 18 bit long, lkey = itid << 8 | key */ 6237 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 6238 mr->hw_mr.key = 0; 6239 mr->hw_mr.pd = pd->pd_id; 6240 mr->hw_mr.local_read = 1; 6241 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 6242 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 6243 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 6244 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 6245 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 6246 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 6247 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 6248 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 6249 mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */ 6250 6251 mr->hw_mr.fbo = 0; 6252 6253 mr->hw_mr.length = length; 6254 mr->hw_mr.vaddr = (uint64_t)iova_start; 6255 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 6256 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 6257 mr->hw_mr.dma_mr = false; 6258 6259 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 6260 if (rc) { 6261 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 6262 goto err2; 6263 } 6264 6265 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6266 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 6267 mr->hw_mr.remote_atomic) 6268 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 6269 6270 QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey); 6271 6272 return (&mr->ibmr); 6273 6274 err2: 6275 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 6276 err1: 6277 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 6278 err0: 6279 kfree(mr); 6280 6281 QL_DPRINT12(ha, "exit [%d]\n", rc); 6282 return (ERR_PTR(rc)); 6283 } 6284 6285 #endif /* #if __FreeBSD_version >= 1102000 */ 6286 6287 struct ib_ah * 6288 #if __FreeBSD_version >= 1102000 6289 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, 6290 struct ib_udata *udata) 6291 #else 6292 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) 6293 #endif /* #if __FreeBSD_version >= 1102000 */ 6294 { 6295 struct qlnxr_dev *dev; 6296 qlnx_host_t *ha; 6297 struct qlnxr_ah *ah; 6298 6299 dev = get_qlnxr_dev((ibpd->device)); 6300 ha = dev->ha; 6301 6302 QL_DPRINT12(ha, "in create_ah\n"); 6303 6304 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 6305 if (!ah) { 6306 QL_DPRINT12(ha, "no address handle can be allocated\n"); 6307 return ERR_PTR(-ENOMEM); 6308 } 6309 6310 ah->attr = *attr; 6311 6312 return &ah->ibah; 6313 } 6314 6315 int 6316 qlnxr_destroy_ah(struct ib_ah *ibah) 6317 { 6318 struct qlnxr_dev *dev; 6319 qlnx_host_t *ha; 6320 struct qlnxr_ah *ah = get_qlnxr_ah(ibah); 6321 6322 dev = get_qlnxr_dev((ibah->device)); 6323 ha = dev->ha; 6324 6325 QL_DPRINT12(ha, "in destroy_ah\n"); 6326 6327 kfree(ah); 6328 return 0; 6329 } 6330 6331 int 6332 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6333 { 6334 struct qlnxr_dev *dev; 6335 qlnx_host_t *ha; 6336 6337 dev = get_qlnxr_dev((ibah->device)); 6338 ha = dev->ha; 6339 QL_DPRINT12(ha, "Query AH not supported\n"); 6340 return -EINVAL; 6341 } 6342 6343 int 6344 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 6345 { 6346 struct qlnxr_dev *dev; 6347 qlnx_host_t *ha; 6348 6349 dev = get_qlnxr_dev((ibah->device)); 6350 ha = dev->ha; 6351 QL_DPRINT12(ha, "Modify AH not supported\n"); 6352 return -ENOSYS; 6353 } 6354 6355 #if __FreeBSD_version >= 1102000 6356 int 6357 qlnxr_process_mad(struct ib_device *ibdev, 6358 int process_mad_flags, 6359 u8 port_num, 6360 const struct ib_wc *in_wc, 6361 const struct ib_grh *in_grh, 6362 const struct ib_mad_hdr *mad_hdr, 6363 size_t in_mad_size, 6364 struct ib_mad_hdr *out_mad, 6365 size_t *out_mad_size, 6366 u16 *out_mad_pkey_index) 6367 6368 #else 6369 6370 int 6371 qlnxr_process_mad(struct ib_device *ibdev, 6372 int process_mad_flags, 6373 u8 port_num, 6374 struct ib_wc *in_wc, 6375 struct ib_grh *in_grh, 6376 struct ib_mad *in_mad, 6377 struct ib_mad *out_mad) 6378 6379 #endif /* #if __FreeBSD_version >= 1102000 */ 6380 { 6381 struct qlnxr_dev *dev; 6382 qlnx_host_t *ha; 6383 6384 dev = get_qlnxr_dev(ibdev); 6385 ha = dev->ha; 6386 QL_DPRINT12(ha, "process mad not supported\n"); 6387 6388 return -ENOSYS; 6389 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", 6390 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, 6391 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, 6392 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, 6393 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); 6394 6395 // return IB_MAD_RESULT_SUCCESS; 6396 } 6397 6398 #if __FreeBSD_version >= 1102000 6399 int 6400 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, 6401 struct ib_port_immutable *immutable) 6402 { 6403 struct qlnxr_dev *dev; 6404 qlnx_host_t *ha; 6405 struct ib_port_attr attr; 6406 int err; 6407 6408 dev = get_qlnxr_dev(ibdev); 6409 ha = dev->ha; 6410 6411 QL_DPRINT12(ha, "enter\n"); 6412 6413 err = qlnxr_query_port(ibdev, port_num, &attr); 6414 if (err) 6415 return err; 6416 6417 if (QLNX_IS_IWARP(dev)) { 6418 immutable->pkey_tbl_len = 1; 6419 immutable->gid_tbl_len = 1; 6420 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 6421 immutable->max_mad_size = 0; 6422 } else { 6423 immutable->pkey_tbl_len = attr.pkey_tbl_len; 6424 immutable->gid_tbl_len = attr.gid_tbl_len; 6425 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 6426 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 6427 } 6428 6429 QL_DPRINT12(ha, "exit\n"); 6430 return 0; 6431 } 6432 #endif /* #if __FreeBSD_version > 1102000 */ 6433 6434 /***** iWARP related functions *************/ 6435 6436 static void 6437 qlnxr_iw_mpa_request(void *context, 6438 struct ecore_iwarp_cm_event_params *params) 6439 { 6440 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; 6441 struct qlnxr_dev *dev = listener->dev; 6442 struct qlnxr_iw_ep *ep; 6443 struct iw_cm_event event; 6444 struct sockaddr_in *laddr; 6445 struct sockaddr_in *raddr; 6446 qlnx_host_t *ha; 6447 6448 ha = dev->ha; 6449 6450 QL_DPRINT12(ha, "enter\n"); 6451 6452 if (params->cm_info->ip_version != ECORE_TCP_IPV4) { 6453 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", 6454 params->cm_info->ip_version); 6455 return; 6456 } 6457 6458 ep = kzalloc(sizeof(*ep), GFP_ATOMIC); 6459 6460 if (!ep) { 6461 QL_DPRINT11(ha, "kzalloc{ep) failed\n"); 6462 return; 6463 } 6464 6465 ep->dev = dev; 6466 ep->ecore_context = params->ep_context; 6467 6468 memset(&event, 0, sizeof(event)); 6469 6470 event.event = IW_CM_EVENT_CONNECT_REQUEST; 6471 event.status = params->status; 6472 6473 laddr = (struct sockaddr_in *)&event.local_addr; 6474 raddr = (struct sockaddr_in *)&event.remote_addr; 6475 6476 laddr->sin_family = AF_INET; 6477 raddr->sin_family = AF_INET; 6478 6479 laddr->sin_port = htons(params->cm_info->local_port); 6480 raddr->sin_port = htons(params->cm_info->remote_port); 6481 6482 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); 6483 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); 6484 6485 event.provider_data = (void *)ep; 6486 event.private_data = (void *)params->cm_info->private_data; 6487 event.private_data_len = (u8)params->cm_info->private_data_len; 6488 6489 #if __FreeBSD_version >= 1100000 6490 event.ord = params->cm_info->ord; 6491 event.ird = params->cm_info->ird; 6492 #endif /* #if __FreeBSD_version >= 1100000 */ 6493 6494 listener->cm_id->event_handler(listener->cm_id, &event); 6495 6496 QL_DPRINT12(ha, "exit\n"); 6497 6498 return; 6499 } 6500 6501 static void 6502 qlnxr_iw_issue_event(void *context, 6503 struct ecore_iwarp_cm_event_params *params, 6504 enum iw_cm_event_type event_type, 6505 char *str) 6506 { 6507 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6508 struct qlnxr_dev *dev = ep->dev; 6509 struct iw_cm_event event; 6510 qlnx_host_t *ha; 6511 6512 ha = dev->ha; 6513 6514 QL_DPRINT12(ha, "enter\n"); 6515 6516 memset(&event, 0, sizeof(event)); 6517 event.status = params->status; 6518 event.event = event_type; 6519 6520 if (params->cm_info != NULL) { 6521 #if __FreeBSD_version >= 1100000 6522 event.ird = params->cm_info->ird; 6523 event.ord = params->cm_info->ord; 6524 QL_DPRINT12(ha, "ord=[%d] \n", event.ord); 6525 QL_DPRINT12(ha, "ird=[%d] \n", event.ird); 6526 #endif /* #if __FreeBSD_version >= 1100000 */ 6527 6528 event.private_data_len = params->cm_info->private_data_len; 6529 event.private_data = (void *)params->cm_info->private_data; 6530 QL_DPRINT12(ha, "private_data_len=[%d] \n", 6531 event.private_data_len); 6532 } 6533 6534 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); 6535 QL_DPRINT12(ha, "status=[%d] \n", event.status); 6536 6537 if (ep) { 6538 if (ep->cm_id) 6539 ep->cm_id->event_handler(ep->cm_id, &event); 6540 else 6541 QL_DPRINT11(ha, "ep->cm_id == NULL \n"); 6542 } else { 6543 QL_DPRINT11(ha, "ep == NULL \n"); 6544 } 6545 6546 QL_DPRINT12(ha, "exit\n"); 6547 6548 return; 6549 } 6550 6551 static void 6552 qlnxr_iw_close_event(void *context, 6553 struct ecore_iwarp_cm_event_params *params) 6554 { 6555 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6556 struct qlnxr_dev *dev = ep->dev; 6557 qlnx_host_t *ha; 6558 6559 ha = dev->ha; 6560 6561 QL_DPRINT12(ha, "enter\n"); 6562 6563 if (ep->cm_id) { 6564 qlnxr_iw_issue_event(context, 6565 params, 6566 IW_CM_EVENT_CLOSE, 6567 "IW_CM_EVENT_EVENT_CLOSE"); 6568 ep->cm_id->rem_ref(ep->cm_id); 6569 ep->cm_id = NULL; 6570 } 6571 6572 QL_DPRINT12(ha, "exit\n"); 6573 6574 return; 6575 } 6576 6577 #if __FreeBSD_version >= 1102000 6578 6579 static void 6580 qlnxr_iw_passive_complete(void *context, 6581 struct ecore_iwarp_cm_event_params *params) 6582 { 6583 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6584 struct qlnxr_dev *dev = ep->dev; 6585 qlnx_host_t *ha; 6586 6587 ha = dev->ha; 6588 6589 /* We will only reach the following state if MPA_REJECT was called on 6590 * passive. In this case there will be no associated QP. 6591 */ 6592 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { 6593 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); 6594 kfree(ep); 6595 return; 6596 } 6597 6598 /* We always issue an established event, however, ofed does not look 6599 * at event code for established. So if there was a failure, we follow 6600 * with close... 6601 */ 6602 qlnxr_iw_issue_event(context, 6603 params, 6604 IW_CM_EVENT_ESTABLISHED, 6605 "IW_CM_EVENT_ESTABLISHED"); 6606 6607 if (params->status < 0) { 6608 qlnxr_iw_close_event(context, params); 6609 } 6610 6611 return; 6612 } 6613 6614 struct qlnxr_discon_work { 6615 struct work_struct work; 6616 struct qlnxr_iw_ep *ep; 6617 enum ecore_iwarp_event_type event; 6618 int status; 6619 }; 6620 6621 static void 6622 qlnxr_iw_disconnect_worker(struct work_struct *work) 6623 { 6624 struct qlnxr_discon_work *dwork = 6625 container_of(work, struct qlnxr_discon_work, work); 6626 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 6627 struct qlnxr_iw_ep *ep = dwork->ep; 6628 struct qlnxr_dev *dev = ep->dev; 6629 struct qlnxr_qp *qp = ep->qp; 6630 struct iw_cm_event event; 6631 6632 if (qp->destroyed) { 6633 kfree(dwork); 6634 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6635 return; 6636 } 6637 6638 memset(&event, 0, sizeof(event)); 6639 event.status = dwork->status; 6640 event.event = IW_CM_EVENT_DISCONNECT; 6641 6642 /* Success means graceful disconnect was requested. modifying 6643 * to SQD is translated to graceful disconnect. O/w reset is sent 6644 */ 6645 if (dwork->status) 6646 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; 6647 else 6648 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; 6649 6650 kfree(dwork); 6651 6652 if (ep->cm_id) 6653 ep->cm_id->event_handler(ep->cm_id, &event); 6654 6655 SET_FIELD(qp_params.modify_flags, 6656 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 6657 6658 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 6659 6660 qlnxr_iw_qp_rem_ref(&qp->ibqp); 6661 6662 return; 6663 } 6664 6665 void 6666 qlnxr_iw_disconnect_event(void *context, 6667 struct ecore_iwarp_cm_event_params *params) 6668 { 6669 struct qlnxr_discon_work *work; 6670 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6671 struct qlnxr_dev *dev = ep->dev; 6672 struct qlnxr_qp *qp = ep->qp; 6673 6674 work = kzalloc(sizeof(*work), GFP_ATOMIC); 6675 if (!work) 6676 return; 6677 6678 qlnxr_iw_qp_add_ref(&qp->ibqp); 6679 work->ep = ep; 6680 work->event = params->event; 6681 work->status = params->status; 6682 6683 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); 6684 queue_work(dev->iwarp_wq, &work->work); 6685 6686 return; 6687 } 6688 6689 #endif /* #if __FreeBSD_version >= 1102000 */ 6690 6691 static int 6692 qlnxr_iw_mpa_reply(void *context, 6693 struct ecore_iwarp_cm_event_params *params) 6694 { 6695 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6696 struct qlnxr_dev *dev = ep->dev; 6697 struct ecore_iwarp_send_rtr_in rtr_in; 6698 int rc; 6699 qlnx_host_t *ha; 6700 6701 ha = dev->ha; 6702 6703 QL_DPRINT12(ha, "enter\n"); 6704 6705 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6706 return -EINVAL; 6707 6708 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); 6709 rtr_in.ep_context = params->ep_context; 6710 6711 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); 6712 6713 QL_DPRINT12(ha, "exit rc = %d\n", rc); 6714 return rc; 6715 } 6716 6717 void 6718 qlnxr_iw_qp_event(void *context, 6719 struct ecore_iwarp_cm_event_params *params, 6720 enum ib_event_type ib_event, 6721 char *str) 6722 { 6723 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6724 struct qlnxr_dev *dev = ep->dev; 6725 struct ib_qp *ibqp = &(ep->qp->ibqp); 6726 struct ib_event event; 6727 qlnx_host_t *ha; 6728 6729 ha = dev->ha; 6730 6731 QL_DPRINT12(ha, 6732 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", 6733 context, params->event, str, ibqp->event_handler); 6734 6735 if (ibqp->event_handler) { 6736 event.event = ib_event; 6737 event.device = ibqp->device; 6738 event.element.qp = ibqp; 6739 ibqp->event_handler(&event, ibqp->qp_context); 6740 } 6741 6742 return; 6743 } 6744 6745 int 6746 qlnxr_iw_event_handler(void *context, 6747 struct ecore_iwarp_cm_event_params *params) 6748 { 6749 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6750 struct qlnxr_dev *dev = ep->dev; 6751 qlnx_host_t *ha; 6752 6753 ha = dev->ha; 6754 6755 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6756 "enter\n", context, params->event); 6757 6758 switch (params->event) { 6759 /* Passive side request received */ 6760 case ECORE_IWARP_EVENT_MPA_REQUEST: 6761 qlnxr_iw_mpa_request(context, params); 6762 break; 6763 6764 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: 6765 qlnxr_iw_mpa_reply(context, params); 6766 break; 6767 6768 /* Passive side established ( ack on mpa response ) */ 6769 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: 6770 6771 #if __FreeBSD_version >= 1102000 6772 6773 ep->during_connect = 0; 6774 qlnxr_iw_passive_complete(context, params); 6775 6776 #else 6777 qlnxr_iw_issue_event(context, 6778 params, 6779 IW_CM_EVENT_ESTABLISHED, 6780 "IW_CM_EVENT_ESTABLISHED"); 6781 #endif /* #if __FreeBSD_version >= 1102000 */ 6782 break; 6783 6784 /* Active side reply received */ 6785 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: 6786 ep->during_connect = 0; 6787 qlnxr_iw_issue_event(context, 6788 params, 6789 IW_CM_EVENT_CONNECT_REPLY, 6790 "IW_CM_EVENT_CONNECT_REPLY"); 6791 if (params->status < 0) { 6792 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 6793 6794 ep->cm_id->rem_ref(ep->cm_id); 6795 ep->cm_id = NULL; 6796 } 6797 break; 6798 6799 case ECORE_IWARP_EVENT_DISCONNECT: 6800 6801 #if __FreeBSD_version >= 1102000 6802 qlnxr_iw_disconnect_event(context, params); 6803 #else 6804 qlnxr_iw_issue_event(context, 6805 params, 6806 IW_CM_EVENT_DISCONNECT, 6807 "IW_CM_EVENT_DISCONNECT"); 6808 qlnxr_iw_close_event(context, params); 6809 #endif /* #if __FreeBSD_version >= 1102000 */ 6810 break; 6811 6812 case ECORE_IWARP_EVENT_CLOSE: 6813 ep->during_connect = 0; 6814 qlnxr_iw_close_event(context, params); 6815 break; 6816 6817 case ECORE_IWARP_EVENT_RQ_EMPTY: 6818 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6819 "IWARP_EVENT_RQ_EMPTY"); 6820 break; 6821 6822 case ECORE_IWARP_EVENT_IRQ_FULL: 6823 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6824 "IWARP_EVENT_IRQ_FULL"); 6825 break; 6826 6827 case ECORE_IWARP_EVENT_LLP_TIMEOUT: 6828 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6829 "IWARP_EVENT_LLP_TIMEOUT"); 6830 break; 6831 6832 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: 6833 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6834 "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); 6835 break; 6836 6837 case ECORE_IWARP_EVENT_CQ_OVERFLOW: 6838 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6839 "QED_IWARP_EVENT_CQ_OVERFLOW"); 6840 break; 6841 6842 case ECORE_IWARP_EVENT_QP_CATASTROPHIC: 6843 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6844 "QED_IWARP_EVENT_QP_CATASTROPHIC"); 6845 break; 6846 6847 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: 6848 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6849 "IWARP_EVENT_LOCAL_ACCESS_ERROR"); 6850 break; 6851 6852 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: 6853 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6854 "IWARP_EVENT_REMOTE_OPERATION_ERROR"); 6855 break; 6856 6857 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: 6858 QL_DPRINT12(ha, "Got terminate message" 6859 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); 6860 break; 6861 6862 default: 6863 QL_DPRINT12(ha, 6864 "Unknown event [0x%x] received \n", params->event); 6865 break; 6866 }; 6867 6868 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6869 "exit\n", context, params->event); 6870 return 0; 6871 } 6872 6873 static int 6874 qlnxr_addr4_resolve(struct qlnxr_dev *dev, 6875 struct sockaddr_in *src_in, 6876 struct sockaddr_in *dst_in, 6877 u8 *dst_mac) 6878 { 6879 int rc; 6880 6881 #if __FreeBSD_version >= 1100000 6882 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, 6883 dst_mac, NULL, NULL); 6884 #else 6885 struct llentry *lle; 6886 6887 rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in, 6888 dst_mac, &lle); 6889 #endif 6890 6891 QL_DPRINT12(dev->ha, "rc = %d " 6892 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " 6893 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, 6894 dst_in->sin_len, dst_in->sin_family, 6895 NIPQUAD((dst_in->sin_addr.s_addr)), 6896 dst_mac[0], dst_mac[1], dst_mac[2], 6897 dst_mac[3], dst_mac[4], dst_mac[5]); 6898 6899 return rc; 6900 } 6901 6902 int 6903 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 6904 { 6905 struct qlnxr_dev *dev; 6906 struct ecore_iwarp_connect_out out_params; 6907 struct ecore_iwarp_connect_in in_params; 6908 struct qlnxr_iw_ep *ep; 6909 struct qlnxr_qp *qp; 6910 struct sockaddr_in *laddr; 6911 struct sockaddr_in *raddr; 6912 int rc = 0; 6913 qlnx_host_t *ha; 6914 6915 dev = get_qlnxr_dev((cm_id->device)); 6916 ha = dev->ha; 6917 6918 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " 6919 "enter \n", cm_id, conn_param); 6920 6921 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6922 return -EINVAL; 6923 6924 qp = idr_find(&dev->qpidr, conn_param->qpn); 6925 6926 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6927 raddr = (struct sockaddr_in *)&cm_id->remote_addr; 6928 6929 QL_DPRINT12(ha, 6930 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", 6931 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, 6932 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); 6933 6934 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 6935 if (!ep) { 6936 QL_DPRINT11(ha, "struct qlnxr_iw_ep " 6937 "alloc memory failed\n"); 6938 return -ENOMEM; 6939 } 6940 6941 ep->dev = dev; 6942 ep->qp = qp; 6943 cm_id->add_ref(cm_id); 6944 ep->cm_id = cm_id; 6945 6946 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); 6947 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); 6948 6949 in_params.event_cb = qlnxr_iw_event_handler; 6950 in_params.cb_context = ep; 6951 6952 in_params.cm_info.ip_version = ECORE_TCP_IPV4; 6953 6954 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); 6955 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); 6956 in_params.cm_info.remote_port = ntohs(raddr->sin_port); 6957 in_params.cm_info.local_port = ntohs(laddr->sin_port); 6958 in_params.cm_info.vlan = 0; 6959 in_params.mss = dev->ha->ifp->if_mtu - 40; 6960 6961 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " 6962 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " 6963 "vlan = %d\n", 6964 NIPQUAD((in_params.cm_info.remote_ip[0])), 6965 NIPQUAD((in_params.cm_info.local_ip[0])), 6966 in_params.cm_info.remote_port, in_params.cm_info.local_port, 6967 in_params.cm_info.vlan); 6968 6969 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); 6970 6971 if (rc) { 6972 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); 6973 goto err; 6974 } 6975 6976 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" 6977 " private_data_len=%d rq_psn=%d\n", 6978 conn_param->ord, conn_param->ird, conn_param->private_data, 6979 conn_param->private_data_len, qp->rq_psn); 6980 6981 in_params.cm_info.ord = conn_param->ord; 6982 in_params.cm_info.ird = conn_param->ird; 6983 in_params.cm_info.private_data = conn_param->private_data; 6984 in_params.cm_info.private_data_len = conn_param->private_data_len; 6985 in_params.qp = qp->ecore_qp; 6986 6987 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 6988 6989 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); 6990 6991 if (rc) { 6992 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); 6993 goto err; 6994 } 6995 6996 QL_DPRINT12(ha, "exit\n"); 6997 6998 return rc; 6999 7000 err: 7001 cm_id->rem_ref(cm_id); 7002 kfree(ep); 7003 7004 QL_DPRINT12(ha, "exit [%d]\n", rc); 7005 return rc; 7006 } 7007 7008 int 7009 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) 7010 { 7011 struct qlnxr_dev *dev; 7012 struct qlnxr_iw_listener *listener; 7013 struct ecore_iwarp_listen_in iparams; 7014 struct ecore_iwarp_listen_out oparams; 7015 struct sockaddr_in *laddr; 7016 qlnx_host_t *ha; 7017 int rc; 7018 7019 dev = get_qlnxr_dev((cm_id->device)); 7020 ha = dev->ha; 7021 7022 QL_DPRINT12(ha, "enter\n"); 7023 7024 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7025 return -EINVAL; 7026 7027 laddr = (struct sockaddr_in *)&cm_id->local_addr; 7028 7029 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 7030 7031 if (listener == NULL) { 7032 QL_DPRINT11(ha, "listener memory alloc failed\n"); 7033 return -ENOMEM; 7034 } 7035 7036 listener->dev = dev; 7037 cm_id->add_ref(cm_id); 7038 listener->cm_id = cm_id; 7039 listener->backlog = backlog; 7040 7041 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); 7042 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); 7043 7044 iparams.cb_context = listener; 7045 iparams.event_cb = qlnxr_iw_event_handler; 7046 iparams.max_backlog = backlog; 7047 7048 iparams.ip_version = ECORE_TCP_IPV4; 7049 7050 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); 7051 iparams.port = ntohs(laddr->sin_port); 7052 iparams.vlan = 0; 7053 7054 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", 7055 NIPQUAD((laddr->sin_addr.s_addr)), 7056 laddr->sin_port, iparams.port); 7057 7058 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); 7059 if (rc) { 7060 QL_DPRINT11(ha, 7061 "ecore_iwarp_create_listen failed rc = %d\n", rc); 7062 goto err; 7063 } 7064 7065 listener->ecore_handle = oparams.handle; 7066 cm_id->provider_data = listener; 7067 7068 QL_DPRINT12(ha, "exit\n"); 7069 return rc; 7070 7071 err: 7072 cm_id->rem_ref(cm_id); 7073 kfree(listener); 7074 7075 QL_DPRINT12(ha, "exit [%d]\n", rc); 7076 return rc; 7077 } 7078 7079 void 7080 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) 7081 { 7082 struct qlnxr_iw_listener *listener = cm_id->provider_data; 7083 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); 7084 int rc = 0; 7085 qlnx_host_t *ha; 7086 7087 ha = dev->ha; 7088 7089 QL_DPRINT12(ha, "enter\n"); 7090 7091 if (listener->ecore_handle) 7092 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, 7093 listener->ecore_handle); 7094 7095 cm_id->rem_ref(cm_id); 7096 7097 QL_DPRINT12(ha, "exit [%d]\n", rc); 7098 return; 7099 } 7100 7101 int 7102 qlnxr_iw_accept(struct iw_cm_id *cm_id, 7103 struct iw_cm_conn_param *conn_param) 7104 { 7105 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7106 struct qlnxr_dev *dev = ep->dev; 7107 struct qlnxr_qp *qp; 7108 struct ecore_iwarp_accept_in params; 7109 int rc; 7110 qlnx_host_t *ha; 7111 7112 ha = dev->ha; 7113 7114 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); 7115 7116 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 7117 return -EINVAL; 7118 7119 qp = idr_find(&dev->qpidr, conn_param->qpn); 7120 if (!qp) { 7121 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", 7122 conn_param->qpn); 7123 return -EINVAL; 7124 } 7125 ep->qp = qp; 7126 qp->ep = ep; 7127 cm_id->add_ref(cm_id); 7128 ep->cm_id = cm_id; 7129 7130 params.ep_context = ep->ecore_context; 7131 params.cb_context = ep; 7132 params.qp = ep->qp->ecore_qp; 7133 params.private_data = conn_param->private_data; 7134 params.private_data_len = conn_param->private_data_len; 7135 params.ird = conn_param->ird; 7136 params.ord = conn_param->ord; 7137 7138 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); 7139 if (rc) { 7140 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); 7141 goto err; 7142 } 7143 7144 QL_DPRINT12(ha, "exit\n"); 7145 return 0; 7146 err: 7147 cm_id->rem_ref(cm_id); 7148 QL_DPRINT12(ha, "exit rc = %d\n", rc); 7149 return rc; 7150 } 7151 7152 int 7153 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 7154 { 7155 #if __FreeBSD_version >= 1102000 7156 7157 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 7158 struct qlnxr_dev *dev = ep->dev; 7159 struct ecore_iwarp_reject_in params; 7160 int rc; 7161 7162 params.ep_context = ep->ecore_context; 7163 params.cb_context = ep; 7164 params.private_data = pdata; 7165 params.private_data_len = pdata_len; 7166 ep->qp = NULL; 7167 7168 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); 7169 7170 return rc; 7171 7172 #else 7173 7174 printf("iWARP reject_cr not implemented\n"); 7175 return -EINVAL; 7176 7177 #endif /* #if __FreeBSD_version >= 1102000 */ 7178 } 7179 7180 void 7181 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) 7182 { 7183 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7184 qlnx_host_t *ha; 7185 7186 ha = qp->dev->ha; 7187 7188 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); 7189 7190 atomic_inc(&qp->refcnt); 7191 7192 QL_DPRINT12(ha, "exit \n"); 7193 return; 7194 } 7195 7196 void 7197 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) 7198 { 7199 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 7200 qlnx_host_t *ha; 7201 7202 ha = qp->dev->ha; 7203 7204 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); 7205 7206 if (atomic_dec_and_test(&qp->refcnt)) { 7207 qlnxr_idr_remove(qp->dev, qp->qp_id); 7208 kfree(qp); 7209 } 7210 7211 QL_DPRINT12(ha, "exit \n"); 7212 return; 7213 } 7214 7215 struct ib_qp * 7216 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) 7217 { 7218 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 7219 struct ib_qp *qp; 7220 qlnx_host_t *ha; 7221 7222 ha = dev->ha; 7223 7224 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); 7225 7226 qp = idr_find(&dev->qpidr, qpn); 7227 7228 QL_DPRINT12(ha, "exit qp = %p\n", qp); 7229 7230 return (qp); 7231 } 7232