1 /* 2 * Copyright (c) 2018-2019 Cavium, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * File: qlnxr_verbs.c 30 */ 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "qlnxr_def.h" 35 #include "rdma_common.h" 36 #include "qlnxr_roce.h" 37 #include "qlnxr_cm.h" 38 39 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 40 41 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ 42 do { \ 43 (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ 44 (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ 45 } while (0) 46 47 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 48 do { \ 49 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 50 (sge)->length = cpu_to_le32(vlength); \ 51 (sge)->flags = cpu_to_le32(vflags); \ 52 } while (0) 53 54 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 55 do { \ 56 TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ 57 (hdr)->num_sges = num_sge; \ 58 } while (0) 59 60 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 61 do { \ 62 TYPEPTR_ADDR_SET(sge, addr, vaddr); \ 63 (sge)->length = cpu_to_le32(vlength); \ 64 (sge)->l_key = cpu_to_le32(vlkey); \ 65 } while (0) 66 67 #define NIPQUAD(addr) \ 68 ((unsigned char *)&addr)[0], \ 69 ((unsigned char *)&addr)[1], \ 70 ((unsigned char *)&addr)[2], \ 71 ((unsigned char *)&addr)[3] 72 73 static int 74 qlnxr_check_srq_params(struct qlnxr_dev *dev, 75 struct ib_srq_init_attr *attrs); 76 77 static int 78 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 79 struct qlnxr_srq *srq, 80 struct qlnxr_create_srq_ureq *ureq, 81 int access, int dmasync); 82 83 static int 84 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 85 struct qlnxr_dev *dev, 86 struct ib_srq_init_attr *init_attr); 87 88 static int 89 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 90 struct qlnxr_srq *srq, 91 struct ib_udata *udata); 92 93 static void 94 qlnxr_free_srq_user_params(struct qlnxr_srq *srq); 95 96 static void 97 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); 98 99 static u32 100 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); 101 102 int 103 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, 104 union ib_gid *sgid) 105 { 106 struct qlnxr_dev *dev; 107 qlnx_host_t *ha; 108 109 dev = get_qlnxr_dev(ibdev); 110 ha = dev->ha; 111 112 QL_DPRINT12(ha, "enter\n"); 113 114 memset(sgid->raw, 0, sizeof(sgid->raw)); 115 116 memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); 117 118 QL_DPRINT12(ha, "exit\n"); 119 120 return 0; 121 } 122 123 int 124 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, 125 union ib_gid *sgid) 126 { 127 struct qlnxr_dev *dev; 128 qlnx_host_t *ha; 129 130 dev = get_qlnxr_dev(ibdev); 131 ha = dev->ha; 132 QL_DPRINT12(ha, "enter index: %d\n", index); 133 #if 0 134 int ret = 0; 135 /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ 136 //if (!rdma_cap_roce_gid_table(ibdev, port)) { 137 if (!(rdma_protocol_roce(ibdev, port) && 138 ibdev->add_gid && ibdev->del_gid)) { 139 QL_DPRINT11(ha, "acquire gid failed\n"); 140 return -ENODEV; 141 } 142 143 ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); 144 if (ret == -EAGAIN) { 145 memcpy(sgid, &zgid, sizeof(*sgid)); 146 return 0; 147 } 148 #endif 149 if ((index >= QLNXR_MAX_SGID) || (index < 0)) { 150 QL_DPRINT12(ha, "invalid gid index %d\n", index); 151 memset(sgid, 0, sizeof(*sgid)); 152 return -EINVAL; 153 } 154 memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); 155 156 QL_DPRINT12(ha, "exit : %p\n", sgid); 157 158 return 0; 159 } 160 161 int 162 qlnxr_create_srq(struct ib_srq *ibsrq, 163 struct ib_srq_init_attr *init_attr, 164 struct ib_udata *udata) 165 { 166 struct qlnxr_dev *dev; 167 qlnx_host_t *ha; 168 struct ecore_rdma_destroy_srq_in_params destroy_in_params; 169 struct ecore_rdma_create_srq_out_params out_params; 170 struct ecore_rdma_create_srq_in_params in_params; 171 u64 pbl_base_addr, phy_prod_pair_addr; 172 struct qlnxr_srq_hwq_info *hw_srq; 173 struct qlnxr_ucontext *ctx; 174 struct qlnxr_create_srq_ureq ureq; 175 u32 page_cnt, page_size; 176 struct qlnxr_srq *srq = get_qlnxr_srq(ibsrq); 177 int ret = 0; 178 179 dev = get_qlnxr_dev(ibsrq->device); 180 ha = dev->ha; 181 182 QL_DPRINT12(ha, "enter\n"); 183 184 ret = qlnxr_check_srq_params(dev, init_attr); 185 186 srq->dev = dev; 187 hw_srq = &srq->hw_srq; 188 spin_lock_init(&srq->lock); 189 memset(&in_params, 0, sizeof(in_params)); 190 191 if (udata) { 192 ctx = rdma_udata_to_drv_context( 193 udata, struct qlnxr_ucontext, ibucontext); 194 195 memset(&ureq, 0, sizeof(ureq)); 196 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 197 udata->inlen))) { 198 QL_DPRINT11(ha, "problem" 199 " copying data from user space\n"); 200 goto err0; 201 } 202 203 ret = qlnxr_init_srq_user_params(&ctx->ibucontext, srq, &ureq, 0, 0); 204 if (ret) 205 goto err0; 206 207 page_cnt = srq->usrq.pbl_info.num_pbes; 208 pbl_base_addr = srq->usrq.pbl_tbl->pa; 209 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 210 // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT 211 // page_size = BIT(srq->usrq.umem->page_shift); 212 // else 213 page_size = srq->usrq.umem->page_size; 214 } else { 215 struct ecore_chain *pbl; 216 ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); 217 if (ret) 218 goto err0; 219 pbl = &hw_srq->pbl; 220 221 page_cnt = ecore_chain_get_page_cnt(pbl); 222 pbl_base_addr = ecore_chain_get_pbl_phys(pbl); 223 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 224 page_size = pbl->elem_per_page << 4; 225 } 226 227 in_params.pd_id = get_qlnxr_pd(ibsrq->pd)->pd_id; 228 in_params.pbl_base_addr = pbl_base_addr; 229 in_params.prod_pair_addr = phy_prod_pair_addr; 230 in_params.num_pages = page_cnt; 231 in_params.page_size = page_size; 232 233 ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 234 if (ret) 235 goto err1; 236 237 srq->srq_id = out_params.srq_id; 238 239 if (udata) { 240 ret = qlnxr_copy_srq_uresp(dev, srq, udata); 241 if (ret) 242 goto err2; 243 } 244 245 QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); 246 return (0); 247 err2: 248 memset(&in_params, 0, sizeof(in_params)); 249 destroy_in_params.srq_id = srq->srq_id; 250 ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 251 252 err1: 253 if (udata) 254 qlnxr_free_srq_user_params(srq); 255 else 256 qlnxr_free_srq_kernel_params(srq); 257 258 err0: 259 return (-EFAULT); 260 } 261 262 void 263 qlnxr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 264 { 265 struct qlnxr_dev *dev; 266 struct qlnxr_srq *srq; 267 qlnx_host_t *ha; 268 struct ecore_rdma_destroy_srq_in_params in_params; 269 270 srq = get_qlnxr_srq(ibsrq); 271 dev = srq->dev; 272 ha = dev->ha; 273 274 memset(&in_params, 0, sizeof(in_params)); 275 in_params.srq_id = srq->srq_id; 276 277 ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); 278 279 if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) 280 qlnxr_free_srq_user_params(srq); 281 else 282 qlnxr_free_srq_kernel_params(srq); 283 284 QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); 285 } 286 287 int 288 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 289 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 290 { 291 struct qlnxr_dev *dev; 292 struct qlnxr_srq *srq; 293 qlnx_host_t *ha; 294 struct ecore_rdma_modify_srq_in_params in_params; 295 int ret = 0; 296 297 srq = get_qlnxr_srq(ibsrq); 298 dev = srq->dev; 299 ha = dev->ha; 300 301 QL_DPRINT12(ha, "enter\n"); 302 if (attr_mask & IB_SRQ_MAX_WR) { 303 QL_DPRINT12(ha, "invalid attribute mask=0x%x" 304 " specified for %p\n", attr_mask, srq); 305 return -EINVAL; 306 } 307 308 if (attr_mask & IB_SRQ_LIMIT) { 309 if (attr->srq_limit >= srq->hw_srq.max_wr) { 310 QL_DPRINT12(ha, "invalid srq_limit=0x%x" 311 " (max_srq_limit = 0x%x)\n", 312 attr->srq_limit, srq->hw_srq.max_wr); 313 return -EINVAL; 314 } 315 memset(&in_params, 0, sizeof(in_params)); 316 in_params.srq_id = srq->srq_id; 317 in_params.wqe_limit = attr->srq_limit; 318 ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); 319 if (ret) 320 return ret; 321 } 322 323 QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); 324 return 0; 325 } 326 327 int 328 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 329 { 330 struct qlnxr_dev *dev; 331 struct qlnxr_srq *srq; 332 qlnx_host_t *ha; 333 struct ecore_rdma_device *qattr; 334 srq = get_qlnxr_srq(ibsrq); 335 dev = srq->dev; 336 ha = dev->ha; 337 //qattr = &dev->attr; 338 qattr = ecore_rdma_query_device(dev->rdma_ctx); 339 QL_DPRINT12(ha, "enter\n"); 340 341 if (!dev->rdma_ctx) { 342 QL_DPRINT12(ha, "called with invalid params" 343 " rdma_ctx is NULL\n"); 344 return -EINVAL; 345 } 346 347 srq_attr->srq_limit = qattr->max_srq; 348 srq_attr->max_wr = qattr->max_srq_wr; 349 srq_attr->max_sge = qattr->max_sge; 350 351 QL_DPRINT12(ha, "exit\n"); 352 return 0; 353 } 354 355 /* Increment srq wr producer by one */ 356 static 357 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) 358 { 359 info->wr_prod_cnt++; 360 } 361 362 /* Increment srq wr consumer by one */ 363 static 364 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) 365 { 366 info->wr_cons_cnt++; 367 } 368 369 /* get_port_immutable verb is not available in FreeBSD */ 370 #if 0 371 int 372 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 373 struct ib_port_immutable *immutable) 374 { 375 struct qlnxr_dev *dev; 376 qlnx_host_t *ha; 377 dev = get_qlnxr_dev(ibdev); 378 ha = dev->ha; 379 380 QL_DPRINT12(ha, "entered but not implemented!!!\n"); 381 } 382 #endif 383 384 int 385 qlnxr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 386 const struct ib_recv_wr **bad_wr) 387 { 388 struct qlnxr_dev *dev; 389 struct qlnxr_srq *srq; 390 qlnx_host_t *ha; 391 struct qlnxr_srq_hwq_info *hw_srq; 392 struct ecore_chain *pbl; 393 unsigned long flags; 394 int status = 0; 395 u32 num_sge, offset; 396 397 srq = get_qlnxr_srq(ibsrq); 398 dev = srq->dev; 399 ha = dev->ha; 400 hw_srq = &srq->hw_srq; 401 402 QL_DPRINT12(ha, "enter\n"); 403 spin_lock_irqsave(&srq->lock, flags); 404 405 pbl = &srq->hw_srq.pbl; 406 while (wr) { 407 struct rdma_srq_wqe_header *hdr; 408 int i; 409 410 if (!qlnxr_srq_elem_left(hw_srq) || 411 wr->num_sge > srq->hw_srq.max_sges) { 412 QL_DPRINT11(ha, "WR cannot be posted" 413 " (%d, %d) || (%d > %d)\n", 414 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 415 wr->num_sge, srq->hw_srq.max_sges); 416 status = -ENOMEM; 417 *bad_wr = wr; 418 break; 419 } 420 421 hdr = ecore_chain_produce(pbl); 422 num_sge = wr->num_sge; 423 /* Set number of sge and WR id in header */ 424 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 425 426 /* PBL is maintained in case of WR granularity. 427 * So increment WR producer in case we post a WR. 428 */ 429 qlnxr_inc_srq_wr_prod(hw_srq); 430 hw_srq->wqe_prod++; 431 hw_srq->sge_prod++; 432 433 QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", 434 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 435 436 for (i = 0; i < wr->num_sge; i++) { 437 struct rdma_srq_sge *srq_sge = 438 ecore_chain_produce(pbl); 439 /* Set SGE length, lkey and address */ 440 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 441 wr->sg_list[i].length, wr->sg_list[i].lkey); 442 443 QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", 444 i, srq_sge->length, srq_sge->l_key, 445 srq_sge->addr.hi, srq_sge->addr.lo); 446 hw_srq->sge_prod++; 447 } 448 wmb(); 449 /* 450 * SRQ prod is 8 bytes. Need to update SGE prod in index 451 * in first 4 bytes and need to update WQE prod in next 452 * 4 bytes. 453 */ 454 *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; 455 offset = offsetof(struct rdma_srq_producers, wqe_prod); 456 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 457 hw_srq->wqe_prod; 458 /* Flush prod after updating it */ 459 wmb(); 460 wr = wr->next; 461 } 462 463 QL_DPRINT12(ha, "Elements in SRQ: %d\n", 464 ecore_chain_get_elem_left(pbl)); 465 466 spin_unlock_irqrestore(&srq->lock, flags); 467 QL_DPRINT12(ha, "exit\n"); 468 return status; 469 } 470 471 int 472 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 473 struct ib_udata *udata) 474 { 475 struct qlnxr_dev *dev; 476 struct ecore_rdma_device *qattr; 477 qlnx_host_t *ha; 478 479 dev = get_qlnxr_dev(ibdev); 480 ha = dev->ha; 481 482 QL_DPRINT12(ha, "enter\n"); 483 484 if (udata->inlen || udata->outlen) 485 return -EINVAL; 486 487 if (dev->rdma_ctx == NULL) { 488 return -EINVAL; 489 } 490 491 qattr = ecore_rdma_query_device(dev->rdma_ctx); 492 493 memset(attr, 0, sizeof *attr); 494 495 attr->fw_ver = qattr->fw_ver; 496 attr->sys_image_guid = qattr->sys_image_guid; 497 attr->max_mr_size = qattr->max_mr_size; 498 attr->page_size_cap = qattr->page_size_caps; 499 attr->vendor_id = qattr->vendor_id; 500 attr->vendor_part_id = qattr->vendor_part_id; 501 attr->hw_ver = qattr->hw_ver; 502 attr->max_qp = qattr->max_qp; 503 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 504 IB_DEVICE_RC_RNR_NAK_GEN | 505 IB_DEVICE_LOCAL_DMA_LKEY | 506 IB_DEVICE_MEM_MGT_EXTENSIONS; 507 508 attr->max_sge = qattr->max_sge; 509 attr->max_sge_rd = qattr->max_sge; 510 attr->max_cq = qattr->max_cq; 511 attr->max_cqe = qattr->max_cqe; 512 attr->max_mr = qattr->max_mr; 513 attr->max_mw = qattr->max_mw; 514 attr->max_pd = qattr->max_pd; 515 attr->atomic_cap = dev->atomic_cap; 516 attr->max_fmr = qattr->max_fmr; 517 attr->max_map_per_fmr = 16; /* TBD: FMR */ 518 519 /* There is an implicit assumption in some of the ib_xxx apps that the 520 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in 521 * communication the qp_rd_atom is passed to the other side and used as 522 * init_rd_atom without check device capabilities for init_rd_atom. 523 * for this reason, we set the qp_rd_atom to be the minimum between the 524 * two...There is an additional assumption in mlx4 driver that the 525 * values are power of two, fls is performed on the value - 1, which 526 * in fact gives a larger power of two for values which are not a power 527 * of two. This should be fixed in mlx4 driver, but until then -> 528 * we provide a value that is a power of two in our code. 529 */ 530 attr->max_qp_init_rd_atom = 531 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 532 attr->max_qp_rd_atom = 533 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 534 attr->max_qp_init_rd_atom); 535 536 attr->max_srq = qattr->max_srq; 537 attr->max_srq_sge = qattr->max_srq_sge; 538 attr->max_srq_wr = qattr->max_srq_wr; 539 540 /* TODO: R&D to more properly configure the following */ 541 attr->local_ca_ack_delay = qattr->dev_ack_delay; 542 attr->max_fast_reg_page_list_len = qattr->max_mr/8; 543 attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; 544 attr->max_ah = qattr->max_ah; 545 546 QL_DPRINT12(ha, "exit\n"); 547 return 0; 548 } 549 550 static inline void 551 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) 552 { 553 switch (speed) { 554 case 1000: 555 *ib_speed = IB_SPEED_SDR; 556 *ib_width = IB_WIDTH_1X; 557 break; 558 case 10000: 559 *ib_speed = IB_SPEED_QDR; 560 *ib_width = IB_WIDTH_1X; 561 break; 562 563 case 20000: 564 *ib_speed = IB_SPEED_DDR; 565 *ib_width = IB_WIDTH_4X; 566 break; 567 568 case 25000: 569 *ib_speed = IB_SPEED_EDR; 570 *ib_width = IB_WIDTH_1X; 571 break; 572 573 case 40000: 574 *ib_speed = IB_SPEED_QDR; 575 *ib_width = IB_WIDTH_4X; 576 break; 577 578 case 50000: 579 *ib_speed = IB_SPEED_QDR; 580 *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... 581 break; 582 583 case 100000: 584 *ib_speed = IB_SPEED_EDR; 585 *ib_width = IB_WIDTH_4X; 586 break; 587 588 default: 589 /* Unsupported */ 590 *ib_speed = IB_SPEED_SDR; 591 *ib_width = IB_WIDTH_1X; 592 } 593 return; 594 } 595 596 int 597 qlnxr_query_port(struct ib_device *ibdev, uint8_t port, 598 struct ib_port_attr *attr) 599 { 600 struct qlnxr_dev *dev; 601 struct ecore_rdma_port *rdma_port; 602 qlnx_host_t *ha; 603 604 dev = get_qlnxr_dev(ibdev); 605 ha = dev->ha; 606 607 QL_DPRINT12(ha, "enter\n"); 608 609 if (port > 1) { 610 QL_DPRINT12(ha, "port [%d] > 1 \n", port); 611 return -EINVAL; 612 } 613 614 if (dev->rdma_ctx == NULL) { 615 QL_DPRINT12(ha, "rdma_ctx == NULL\n"); 616 return -EINVAL; 617 } 618 619 rdma_port = ecore_rdma_query_port(dev->rdma_ctx); 620 memset(attr, 0, sizeof *attr); 621 622 if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { 623 attr->state = IB_PORT_ACTIVE; 624 attr->phys_state = 5; 625 } else { 626 attr->state = IB_PORT_DOWN; 627 attr->phys_state = 3; 628 } 629 630 attr->max_mtu = IB_MTU_4096; 631 attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); 632 attr->lid = 0; 633 attr->lmc = 0; 634 attr->sm_lid = 0; 635 attr->sm_sl = 0; 636 attr->port_cap_flags = 0; 637 638 if (QLNX_IS_IWARP(dev)) { 639 attr->gid_tbl_len = 1; 640 attr->pkey_tbl_len = 1; 641 } else { 642 attr->gid_tbl_len = QLNXR_MAX_SGID; 643 attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; 644 } 645 646 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 647 attr->qkey_viol_cntr = 0; 648 649 get_link_speed_and_width(rdma_port->link_speed, 650 &attr->active_speed, &attr->active_width); 651 652 attr->max_msg_sz = rdma_port->max_msg_size; 653 attr->max_vl_num = 4; /* TODO -> figure this one out... */ 654 655 QL_DPRINT12(ha, "state = %d phys_state = %d " 656 " link_speed = %d active_speed = %d active_width = %d" 657 " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" 658 " max_msg_sz = 0x%x max_vl_num = 0x%x \n", 659 attr->state, attr->phys_state, 660 rdma_port->link_speed, attr->active_speed, 661 attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, 662 attr->max_msg_sz, attr->max_vl_num); 663 664 QL_DPRINT12(ha, "exit\n"); 665 return 0; 666 } 667 668 int 669 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, 670 struct ib_port_modify *props) 671 { 672 struct qlnxr_dev *dev; 673 qlnx_host_t *ha; 674 675 dev = get_qlnxr_dev(ibdev); 676 ha = dev->ha; 677 678 QL_DPRINT12(ha, "enter\n"); 679 680 if (port > 1) { 681 QL_DPRINT12(ha, "port (%d) > 1\n", port); 682 return -EINVAL; 683 } 684 685 QL_DPRINT12(ha, "exit\n"); 686 return 0; 687 } 688 689 enum rdma_link_layer 690 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) 691 { 692 struct qlnxr_dev *dev; 693 qlnx_host_t *ha; 694 695 dev = get_qlnxr_dev(ibdev); 696 ha = dev->ha; 697 698 QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); 699 700 return IB_LINK_LAYER_ETHERNET; 701 } 702 703 int 704 qlnxr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 705 { 706 struct ib_device *ibdev = ibpd->device; 707 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 708 u16 pd_id; 709 int rc; 710 struct qlnxr_dev *dev; 711 qlnx_host_t *ha; 712 713 dev = get_qlnxr_dev(ibdev); 714 ha = dev->ha; 715 716 QL_DPRINT12(ha, "ibdev = %p udata = %p enter\n", ibdev, udata); 717 718 if (dev->rdma_ctx == NULL) { 719 QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); 720 rc = -1; 721 goto err; 722 } 723 724 rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); 725 if (rc) { 726 QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); 727 goto err; 728 } 729 730 pd->pd_id = pd_id; 731 732 if (udata) { 733 rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); 734 if (rc) { 735 QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); 736 ecore_rdma_free_pd(dev->rdma_ctx, pd_id); 737 goto err; 738 } 739 740 pd->uctx = rdma_udata_to_drv_context( 741 udata, struct qlnxr_ucontext, ibucontext); 742 pd->uctx->pd = pd; 743 } 744 745 atomic_add_rel_32(&dev->pd_count, 1); 746 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 747 pd, pd_id, dev->pd_count); 748 749 return (0); 750 751 err: 752 QL_DPRINT12(ha, "exit -1\n"); 753 return (rc); 754 } 755 756 void 757 qlnxr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 758 { 759 struct qlnxr_pd *pd; 760 struct qlnxr_dev *dev; 761 qlnx_host_t *ha; 762 763 pd = get_qlnxr_pd(ibpd); 764 dev = get_qlnxr_dev((ibpd->device)); 765 ha = dev->ha; 766 767 QL_DPRINT12(ha, "enter\n"); 768 769 if (pd == NULL) { 770 QL_DPRINT11(ha, "pd = NULL\n"); 771 } else { 772 ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); 773 atomic_subtract_rel_32(&dev->pd_count, 1); 774 QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", 775 pd, pd->pd_id, dev->pd_count); 776 } 777 778 QL_DPRINT12(ha, "exit\n"); 779 } 780 781 #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) 782 #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ 783 /* Should be part of HSI */ 784 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ 785 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 786 787 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); 788 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); 789 790 int 791 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 792 { 793 struct qlnxr_dev *dev; 794 qlnx_host_t *ha; 795 796 dev = get_qlnxr_dev(ibdev); 797 ha = dev->ha; 798 799 QL_DPRINT12(ha, "enter index = 0x%x\n", index); 800 801 if (index > QLNXR_ROCE_PKEY_TABLE_LEN) 802 return -EINVAL; 803 804 *pkey = QLNXR_ROCE_PKEY_DEFAULT; 805 806 QL_DPRINT12(ha, "exit\n"); 807 return 0; 808 } 809 810 static inline bool 811 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, 812 u16 *vlan_id) 813 { 814 bool ret = false; 815 816 u16 tmp_vlan_id; 817 818 union ib_gid *dgid; 819 820 QL_DPRINT12(ha, "enter \n"); 821 822 *vlan_id = 0; 823 824 dgid = &attr->ah_attr.grh.dgid; 825 tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; 826 827 if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { 828 *vlan_id = tmp_vlan_id; 829 ret = true; 830 } 831 832 QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); 833 834 return (ret); 835 } 836 837 static inline void 838 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, 839 int attr_mask, 840 struct qlnxr_dev *dev, 841 struct qlnxr_qp *qp, 842 struct ecore_rdma_modify_qp_in_params *qp_params) 843 { 844 int i; 845 qlnx_host_t *ha; 846 847 ha = dev->ha; 848 849 QL_DPRINT12(ha, "enter\n"); 850 851 memcpy(&qp_params->sgid.bytes[0], 852 &dev->sgid_tbl[qp->sgid_idx].raw[0], 853 sizeof(qp_params->sgid.bytes)); 854 memcpy(&qp_params->dgid.bytes[0], 855 &attr->ah_attr.grh.dgid.raw[0], 856 sizeof(qp_params->dgid)); 857 858 qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); 859 860 for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { 861 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 862 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 863 } 864 865 QL_DPRINT12(ha, "exit\n"); 866 return; 867 } 868 869 static int 870 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 871 { 872 struct qlnxr_mm *mm; 873 qlnx_host_t *ha; 874 875 ha = uctx->dev->ha; 876 877 QL_DPRINT12(ha, "enter\n"); 878 879 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 880 if (mm == NULL) { 881 QL_DPRINT11(ha, "mm = NULL\n"); 882 return -ENOMEM; 883 } 884 885 mm->key.phy_addr = phy_addr; 886 887 /* This function might be called with a length which is not a multiple 888 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel 889 * forces this granularity by increasing the requested size if needed. 890 * When qedr_mmap is called, it will search the list with the updated 891 * length as a key. To prevent search failures, the length is rounded up 892 * in advance to PAGE_SIZE. 893 */ 894 mm->key.len = roundup(len, PAGE_SIZE); 895 INIT_LIST_HEAD(&mm->entry); 896 897 mutex_lock(&uctx->mm_list_lock); 898 list_add(&mm->entry, &uctx->mm_head); 899 mutex_unlock(&uctx->mm_list_lock); 900 901 QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", 902 (unsigned long long)mm->key.phy_addr, 903 (unsigned long)mm->key.len, uctx); 904 905 return 0; 906 } 907 908 static bool 909 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) 910 { 911 bool found = false; 912 struct qlnxr_mm *mm; 913 qlnx_host_t *ha; 914 915 ha = uctx->dev->ha; 916 917 QL_DPRINT12(ha, "enter\n"); 918 919 mutex_lock(&uctx->mm_list_lock); 920 list_for_each_entry(mm, &uctx->mm_head, entry) { 921 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 922 continue; 923 924 found = true; 925 break; 926 } 927 mutex_unlock(&uctx->mm_list_lock); 928 929 QL_DPRINT12(ha, 930 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", 931 mm->key.phy_addr, mm->key.len, uctx, found); 932 933 return found; 934 } 935 936 int 937 qlnxr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 938 { 939 int rc; 940 struct qlnxr_ucontext *ctx = get_qlnxr_ucontext(uctx); 941 struct qlnxr_alloc_ucontext_resp uresp; 942 struct qlnxr_dev *dev = get_qlnxr_dev(uctx->device); 943 qlnx_host_t *ha = dev->ha; 944 struct ecore_rdma_add_user_out_params oparams; 945 946 if (!udata) 947 return -EFAULT; 948 949 rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); 950 if (rc) { 951 QL_DPRINT12(ha, 952 "Failed to allocate a DPI for a new RoCE application " 953 ",rc = %d. To overcome this, consider to increase " 954 "the number of DPIs, increase the doorbell BAR size " 955 "or just close unnecessary RoCE applications. In " 956 "order to increase the number of DPIs consult the " 957 "README\n", rc); 958 goto err; 959 } 960 961 ctx->dpi = oparams.dpi; 962 ctx->dpi_addr = oparams.dpi_addr; 963 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 964 ctx->dpi_size = oparams.dpi_size; 965 INIT_LIST_HEAD(&ctx->mm_head); 966 mutex_init(&ctx->mm_list_lock); 967 968 memset(&uresp, 0, sizeof(uresp)); 969 uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) 970 < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out 971 uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) 972 < udata->outlen ? 1 : 0; //TODO: figure this out 973 uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) 974 < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out 975 uresp.db_pa = ctx->dpi_phys_addr; 976 uresp.db_size = ctx->dpi_size; 977 uresp.max_send_wr = dev->attr.max_sqe; 978 uresp.max_recv_wr = dev->attr.max_rqe; 979 uresp.max_srq_wr = dev->attr.max_srq_wr; 980 uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 981 uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 982 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 983 uresp.max_cqes = QLNXR_MAX_CQES; 984 985 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 986 if (rc) 987 goto err; 988 989 ctx->dev = dev; 990 991 rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); 992 if (rc) 993 goto err; 994 QL_DPRINT12(ha, "Allocated user context %p\n", 995 &ctx->ibucontext); 996 997 return (0); 998 err: 999 return (rc); 1000 } 1001 1002 void 1003 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) 1004 { 1005 struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); 1006 struct qlnxr_dev *dev = uctx->dev; 1007 qlnx_host_t *ha = dev->ha; 1008 struct qlnxr_mm *mm, *tmp; 1009 1010 QL_DPRINT12(ha, "Deallocating user context %p\n", 1011 uctx); 1012 1013 if (dev) { 1014 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); 1015 } 1016 1017 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 1018 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" 1019 " ctx=%p\n", 1020 mm->key.phy_addr, mm->key.len, uctx); 1021 list_del(&mm->entry); 1022 kfree(mm); 1023 } 1024 } 1025 1026 int 1027 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 1028 { 1029 struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); 1030 struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); 1031 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 1032 u64 unmapped_db; 1033 unsigned long len = (vma->vm_end - vma->vm_start); 1034 int rc = 0; 1035 bool found; 1036 qlnx_host_t *ha; 1037 1038 ha = dev->ha; 1039 1040 unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); 1041 1042 QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" 1043 " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", 1044 vm_page, vma->vm_pgoff, unmapped_db, 1045 dev->db_size, len); 1046 1047 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { 1048 QL_DPRINT11(ha, "Vma_start not page aligned " 1049 "vm_start = %ld vma_end = %ld\n", vma->vm_start, 1050 vma->vm_end); 1051 return -EINVAL; 1052 } 1053 1054 found = qlnxr_search_mmap(ucontext, vm_page, len); 1055 if (!found) { 1056 QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", 1057 vma->vm_pgoff); 1058 return -EINVAL; 1059 } 1060 1061 QL_DPRINT12(ha, "Mapping doorbell bar\n"); 1062 1063 if ((vm_page < unmapped_db) || 1064 ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { 1065 QL_DPRINT11(ha, "failed pages are outside of dpi;" 1066 "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", 1067 vm_page, unmapped_db, ucontext->dpi_size); 1068 return -EINVAL; 1069 } 1070 1071 if (vma->vm_flags & VM_READ) { 1072 QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); 1073 return -EINVAL; 1074 } 1075 1076 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 1077 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, 1078 vma->vm_page_prot); 1079 1080 QL_DPRINT12(ha, "exit [%d]\n", rc); 1081 return rc; 1082 } 1083 1084 struct ib_mr * 1085 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) 1086 { 1087 struct qlnxr_mr *mr; 1088 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1089 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 1090 int rc; 1091 qlnx_host_t *ha; 1092 1093 ha = dev->ha; 1094 1095 QL_DPRINT12(ha, "enter\n"); 1096 1097 if (acc & IB_ACCESS_MW_BIND) { 1098 QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); 1099 } 1100 1101 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1102 if (!mr) { 1103 rc = -ENOMEM; 1104 QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); 1105 goto err0; 1106 } 1107 1108 mr->type = QLNXR_MR_DMA; 1109 1110 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1111 if (rc) { 1112 QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); 1113 goto err1; 1114 } 1115 1116 /* index only, 18 bit long, lkey = itid << 8 | key */ 1117 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1118 mr->hw_mr.pd = pd->pd_id; 1119 mr->hw_mr.local_read = 1; 1120 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1121 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1122 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1123 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1124 mr->hw_mr.dma_mr = true; 1125 1126 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1127 if (rc) { 1128 QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); 1129 goto err2; 1130 } 1131 1132 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1133 1134 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1135 mr->hw_mr.remote_atomic) { 1136 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1137 } 1138 1139 QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); 1140 1141 return &mr->ibmr; 1142 1143 err2: 1144 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1145 err1: 1146 kfree(mr); 1147 err0: 1148 QL_DPRINT12(ha, "exit [%d]\n", rc); 1149 1150 return ERR_PTR(rc); 1151 } 1152 1153 static void 1154 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, 1155 struct qlnxr_pbl *pbl) 1156 { 1157 int i; 1158 qlnx_host_t *ha; 1159 1160 ha = dev->ha; 1161 1162 QL_DPRINT12(ha, "enter\n"); 1163 1164 for (i = 0; i < pbl_info->num_pbls; i++) { 1165 if (!pbl[i].va) 1166 continue; 1167 qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, 1168 pbl_info->pbl_size); 1169 } 1170 kfree(pbl); 1171 1172 QL_DPRINT12(ha, "exit\n"); 1173 return; 1174 } 1175 1176 #define MIN_FW_PBL_PAGE_SIZE (4*1024) 1177 #define MAX_FW_PBL_PAGE_SIZE (64*1024) 1178 1179 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 1180 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 1181 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) 1182 1183 static struct qlnxr_pbl * 1184 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, 1185 struct qlnxr_pbl_info *pbl_info, gfp_t flags) 1186 { 1187 void *va; 1188 dma_addr_t pa; 1189 dma_addr_t *pbl_main_tbl; 1190 struct qlnxr_pbl *pbl_table; 1191 int i; 1192 qlnx_host_t *ha; 1193 1194 ha = dev->ha; 1195 1196 QL_DPRINT12(ha, "enter\n"); 1197 1198 pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); 1199 1200 if (!pbl_table) { 1201 QL_DPRINT12(ha, "pbl_table = NULL\n"); 1202 return NULL; 1203 } 1204 1205 for (i = 0; i < pbl_info->num_pbls; i++) { 1206 va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); 1207 if (!va) { 1208 QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); 1209 goto err; 1210 } 1211 memset(va, 0, pbl_info->pbl_size); 1212 pbl_table[i].va = va; 1213 pbl_table[i].pa = pa; 1214 } 1215 1216 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 1217 * the first one with physical pointers to all of the rest 1218 */ 1219 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 1220 for (i = 0; i < pbl_info->num_pbls - 1; i++) 1221 pbl_main_tbl[i] = pbl_table[i + 1].pa; 1222 1223 QL_DPRINT12(ha, "exit\n"); 1224 return pbl_table; 1225 1226 err: 1227 qlnxr_free_pbl(dev, pbl_info, pbl_table); 1228 1229 QL_DPRINT12(ha, "exit with error\n"); 1230 return NULL; 1231 } 1232 1233 static int 1234 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, 1235 struct qlnxr_pbl_info *pbl_info, 1236 u32 num_pbes, 1237 int two_layer_capable) 1238 { 1239 u32 pbl_capacity; 1240 u32 pbl_size; 1241 u32 num_pbls; 1242 qlnx_host_t *ha; 1243 1244 ha = dev->ha; 1245 1246 QL_DPRINT12(ha, "enter\n"); 1247 1248 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 1249 if (num_pbes > MAX_PBES_TWO_LAYER) { 1250 QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", 1251 num_pbes); 1252 return -EINVAL; 1253 } 1254 1255 /* calculate required pbl page size */ 1256 pbl_size = MIN_FW_PBL_PAGE_SIZE; 1257 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 1258 NUM_PBES_ON_PAGE(pbl_size); 1259 1260 while (pbl_capacity < num_pbes) { 1261 pbl_size *= 2; 1262 pbl_capacity = pbl_size / sizeof(u64); 1263 pbl_capacity = pbl_capacity * pbl_capacity; 1264 } 1265 1266 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 1267 num_pbls++; /* One for the layer0 ( points to the pbls) */ 1268 pbl_info->two_layered = true; 1269 } else { 1270 /* One layered PBL */ 1271 num_pbls = 1; 1272 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ 1273 roundup_pow_of_two((num_pbes * sizeof(u64)))); 1274 pbl_info->two_layered = false; 1275 } 1276 1277 pbl_info->num_pbls = num_pbls; 1278 pbl_info->pbl_size = pbl_size; 1279 pbl_info->num_pbes = num_pbes; 1280 1281 QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", 1282 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 1283 1284 return 0; 1285 } 1286 1287 static void 1288 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, 1289 struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) 1290 { 1291 struct regpair *pbe; 1292 struct qlnxr_pbl *pbl_tbl; 1293 struct scatterlist *sg; 1294 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; 1295 qlnx_host_t *ha; 1296 int entry; 1297 1298 ha = dev->ha; 1299 1300 QL_DPRINT12(ha, "enter\n"); 1301 1302 if (!pbl_info) { 1303 QL_DPRINT11(ha, "PBL_INFO not initialized\n"); 1304 return; 1305 } 1306 1307 if (!pbl_info->num_pbes) { 1308 QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); 1309 return; 1310 } 1311 1312 /* If we have a two layered pbl, the first pbl points to the rest 1313 * of the pbls and the first entry lays on the second pbl in the table 1314 */ 1315 if (pbl_info->two_layered) 1316 pbl_tbl = &pbl[1]; 1317 else 1318 pbl_tbl = pbl; 1319 1320 pbe = (struct regpair *)pbl_tbl->va; 1321 if (!pbe) { 1322 QL_DPRINT12(ha, "pbe is NULL\n"); 1323 return; 1324 } 1325 1326 pbe_cnt = 0; 1327 1328 shift = ilog2(umem->page_size); 1329 1330 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 1331 pages = sg_dma_len(sg) >> shift; 1332 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { 1333 /* store the page address in pbe */ 1334 pbe->lo = 1335 cpu_to_le32(sg_dma_address(sg) + 1336 (umem->page_size * pg_cnt)); 1337 pbe->hi = 1338 cpu_to_le32(upper_32_bits 1339 ((sg_dma_address(sg) + 1340 umem->page_size * pg_cnt))); 1341 1342 QL_DPRINT12(ha, 1343 "Populate pbl table:" 1344 " pbe->addr=0x%x:0x%x " 1345 " pbe_cnt = %d total_num_pbes=%d" 1346 " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, 1347 total_num_pbes, pbe); 1348 1349 pbe_cnt ++; 1350 total_num_pbes ++; 1351 pbe++; 1352 1353 if (total_num_pbes == pbl_info->num_pbes) 1354 return; 1355 1356 /* if the given pbl is full storing the pbes, 1357 * move to next pbl. 1358 */ 1359 if (pbe_cnt == 1360 (pbl_info->pbl_size / sizeof(u64))) { 1361 pbl_tbl++; 1362 pbe = (struct regpair *)pbl_tbl->va; 1363 pbe_cnt = 0; 1364 } 1365 } 1366 } 1367 QL_DPRINT12(ha, "exit\n"); 1368 return; 1369 } 1370 1371 static void 1372 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) 1373 { 1374 struct qlnxr_pbl *pbl, *tmp; 1375 qlnx_host_t *ha; 1376 1377 ha = dev->ha; 1378 1379 QL_DPRINT12(ha, "enter\n"); 1380 1381 if (info->pbl_table) 1382 list_add_tail(&info->pbl_table->list_entry, 1383 &info->free_pbl_list); 1384 1385 if (!list_empty(&info->inuse_pbl_list)) 1386 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 1387 1388 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 1389 list_del(&pbl->list_entry); 1390 qlnxr_free_pbl(dev, &info->pbl_info, pbl); 1391 } 1392 QL_DPRINT12(ha, "exit\n"); 1393 1394 return; 1395 } 1396 1397 static int 1398 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, 1399 size_t page_list_len, bool two_layered) 1400 { 1401 int rc; 1402 struct qlnxr_pbl *tmp; 1403 qlnx_host_t *ha; 1404 1405 ha = dev->ha; 1406 1407 QL_DPRINT12(ha, "enter\n"); 1408 1409 INIT_LIST_HEAD(&info->free_pbl_list); 1410 INIT_LIST_HEAD(&info->inuse_pbl_list); 1411 1412 rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, 1413 page_list_len, two_layered); 1414 if (rc) { 1415 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); 1416 goto done; 1417 } 1418 1419 info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1420 1421 if (!info->pbl_table) { 1422 rc = -ENOMEM; 1423 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); 1424 goto done; 1425 } 1426 1427 QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); 1428 1429 /* in usual case we use 2 PBLs, so we add one to free 1430 * list and allocating another one 1431 */ 1432 tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 1433 1434 if (!tmp) { 1435 QL_DPRINT11(ha, "Extra PBL is not allocated\n"); 1436 goto done; /* it's OK if second allocation fails, so rc = 0*/ 1437 } 1438 1439 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 1440 1441 QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); 1442 1443 done: 1444 if (rc) 1445 free_mr_info(dev, info); 1446 1447 QL_DPRINT12(ha, "exit [%d]\n", rc); 1448 1449 return rc; 1450 } 1451 1452 struct ib_mr * 1453 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1454 u64 usr_addr, int acc, struct ib_udata *udata) 1455 { 1456 int rc = -ENOMEM; 1457 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 1458 struct qlnxr_mr *mr; 1459 struct qlnxr_pd *pd; 1460 qlnx_host_t *ha; 1461 1462 ha = dev->ha; 1463 1464 QL_DPRINT12(ha, "enter\n"); 1465 1466 pd = get_qlnxr_pd(ibpd); 1467 1468 QL_DPRINT12(ha, "qedr_register user mr pd = %d" 1469 " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 1470 pd->pd_id, start, len, usr_addr, acc); 1471 1472 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 1473 QL_DPRINT11(ha, 1474 "(acc & IB_ACCESS_REMOTE_WRITE &&" 1475 " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); 1476 return ERR_PTR(-EINVAL); 1477 } 1478 1479 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1480 if (!mr) { 1481 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 1482 return ERR_PTR(rc); 1483 } 1484 1485 mr->type = QLNXR_MR_USER; 1486 1487 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 1488 if (IS_ERR(mr->umem)) { 1489 rc = -EFAULT; 1490 QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); 1491 goto err0; 1492 } 1493 1494 rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 1495 if (rc) { 1496 QL_DPRINT11(ha, 1497 "qlnxr_init_mr_info failed [%d]\n", rc); 1498 goto err1; 1499 } 1500 1501 qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 1502 &mr->info.pbl_info); 1503 1504 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 1505 1506 if (rc) { 1507 QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); 1508 goto err1; 1509 } 1510 1511 /* index only, 18 bit long, lkey = itid << 8 | key */ 1512 mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; 1513 mr->hw_mr.key = 0; 1514 mr->hw_mr.pd = pd->pd_id; 1515 mr->hw_mr.local_read = 1; 1516 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 1517 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 1518 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1519 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 1520 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 1521 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 1522 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 1523 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 1524 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ 1525 1526 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 1527 mr->hw_mr.length = len; 1528 mr->hw_mr.vaddr = usr_addr; 1529 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 1530 mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ 1531 mr->hw_mr.dma_mr = false; 1532 1533 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 1534 if (rc) { 1535 QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); 1536 goto err2; 1537 } 1538 1539 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1540 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 1541 mr->hw_mr.remote_atomic) 1542 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 1543 1544 QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); 1545 1546 return (&mr->ibmr); 1547 1548 err2: 1549 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 1550 err1: 1551 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1552 err0: 1553 kfree(mr); 1554 1555 QL_DPRINT12(ha, "exit [%d]\n", rc); 1556 return (ERR_PTR(rc)); 1557 } 1558 1559 int 1560 qlnxr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 1561 { 1562 struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); 1563 struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); 1564 int rc = 0; 1565 qlnx_host_t *ha; 1566 1567 ha = dev->ha; 1568 1569 QL_DPRINT12(ha, "enter\n"); 1570 1571 if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) 1572 qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 1573 1574 /* it could be user registered memory. */ 1575 if (mr->umem) 1576 ib_umem_release(mr->umem); 1577 1578 kfree(mr->pages); 1579 1580 kfree(mr); 1581 1582 QL_DPRINT12(ha, "exit\n"); 1583 return rc; 1584 } 1585 1586 static int 1587 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, 1588 struct qlnxr_cq *cq, struct ib_udata *udata) 1589 { 1590 struct qlnxr_create_cq_uresp uresp; 1591 int rc; 1592 qlnx_host_t *ha; 1593 1594 ha = dev->ha; 1595 1596 QL_DPRINT12(ha, "enter\n"); 1597 1598 memset(&uresp, 0, sizeof(uresp)); 1599 1600 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1601 uresp.icid = cq->icid; 1602 1603 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1604 1605 if (rc) { 1606 QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", 1607 cq->icid, rc); 1608 } 1609 1610 QL_DPRINT12(ha, "exit [%d]\n", rc); 1611 return rc; 1612 } 1613 1614 static void 1615 consume_cqe(struct qlnxr_cq *cq) 1616 { 1617 1618 if (cq->latest_cqe == cq->toggle_cqe) 1619 cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1620 1621 cq->latest_cqe = ecore_chain_consume(&cq->pbl); 1622 } 1623 1624 static inline int 1625 qlnxr_align_cq_entries(int entries) 1626 { 1627 u64 size, aligned_size; 1628 1629 /* We allocate an extra entry that we don't report to the FW. 1630 * Why? 1631 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes 1632 * (for performance purposes). Allocating an extra entry and telling 1633 * the FW we have less prevents overwriting the first entry in case of 1634 * a wrap i.e. when the FW writes the last entry and the application 1635 * hasn't read the first one. 1636 */ 1637 size = (entries + 1) * QLNXR_CQE_SIZE; 1638 1639 /* We align to PAGE_SIZE. 1640 * Why? 1641 * Since the CQ is going to be mapped and the mapping is anyhow in whole 1642 * kernel pages we benefit from the possibly extra CQEs. 1643 */ 1644 aligned_size = ALIGN(size, PAGE_SIZE); 1645 1646 /* note: for CQs created in user space the result of this function 1647 * should match the size mapped in user space 1648 */ 1649 return (aligned_size / QLNXR_CQE_SIZE); 1650 } 1651 1652 static inline int 1653 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, 1654 struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, 1655 int access, int dmasync, int alloc_and_init) 1656 { 1657 int page_cnt; 1658 int rc; 1659 qlnx_host_t *ha; 1660 1661 ha = dev->ha; 1662 1663 QL_DPRINT12(ha, "enter\n"); 1664 1665 q->buf_addr = buf_addr; 1666 q->buf_len = buf_len; 1667 1668 QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" 1669 " dmasync : %x\n", q->buf_addr, q->buf_len, 1670 access, dmasync); 1671 1672 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); 1673 1674 if (IS_ERR(q->umem)) { 1675 QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); 1676 return PTR_ERR(q->umem); 1677 } 1678 1679 page_cnt = ib_umem_page_count(q->umem); 1680 rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 1681 0 /* SQ and RQ don't support dual layer pbl. 1682 * CQ may, but this is yet uncoded. 1683 */); 1684 if (rc) { 1685 QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); 1686 goto err; 1687 } 1688 1689 if (alloc_and_init) { 1690 q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 1691 1692 if (!q->pbl_tbl) { 1693 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1694 rc = -ENOMEM; 1695 goto err; 1696 } 1697 1698 qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); 1699 } else { 1700 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 1701 1702 if (!q->pbl_tbl) { 1703 QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); 1704 rc = -ENOMEM; 1705 goto err; 1706 } 1707 } 1708 1709 QL_DPRINT12(ha, "exit\n"); 1710 return 0; 1711 1712 err: 1713 ib_umem_release(q->umem); 1714 q->umem = NULL; 1715 1716 QL_DPRINT12(ha, "exit [%d]\n", rc); 1717 return rc; 1718 } 1719 1720 int 1721 qlnxr_create_cq(struct ib_cq *ibcq, 1722 const struct ib_cq_init_attr *attr, 1723 struct ib_udata *udata) 1724 { 1725 struct qlnxr_ucontext *ctx; 1726 struct ecore_rdma_destroy_cq_out_params destroy_oparams; 1727 struct ecore_rdma_destroy_cq_in_params destroy_iparams; 1728 struct qlnxr_dev *dev; 1729 struct ecore_rdma_create_cq_in_params params; 1730 struct qlnxr_create_cq_ureq ureq; 1731 1732 int vector = attr->comp_vector; 1733 int entries = attr->cqe; 1734 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 1735 int chain_entries, rc, page_cnt; 1736 u64 pbl_ptr; 1737 u16 icid; 1738 qlnx_host_t *ha; 1739 1740 dev = get_qlnxr_dev(ibcq->device); 1741 ha = dev->ha; 1742 1743 QL_DPRINT12(ha, "called from %s. entries = %d, " 1744 "vector = %d\n", 1745 (udata ? "User Lib" : "Kernel"), entries, vector); 1746 1747 memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); 1748 memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); 1749 memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); 1750 1751 if (entries > QLNXR_MAX_CQES) { 1752 QL_DPRINT11(ha, 1753 "the number of entries %d is too high. " 1754 "Must be equal or below %d.\n", 1755 entries, QLNXR_MAX_CQES); 1756 return -EINVAL; 1757 } 1758 chain_entries = qlnxr_align_cq_entries(entries); 1759 chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); 1760 1761 if (udata) { 1762 ctx = rdma_udata_to_drv_context( 1763 udata, struct qlnxr_ucontext, ibucontext); 1764 1765 memset(&ureq, 0, sizeof(ureq)); 1766 1767 if (ib_copy_from_udata(&ureq, udata, 1768 min(sizeof(ureq), udata->inlen))) { 1769 QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); 1770 goto err0; 1771 } 1772 1773 if (!ureq.len) { 1774 QL_DPRINT11(ha, "ureq.len == 0\n"); 1775 goto err0; 1776 } 1777 1778 cq->cq_type = QLNXR_CQ_TYPE_USER; 1779 1780 qlnxr_init_user_queue(&ctx->ibucontext, dev, &cq->q, ureq.addr, ureq.len, 1781 IB_ACCESS_LOCAL_WRITE, 1, 1); 1782 1783 pbl_ptr = cq->q.pbl_tbl->pa; 1784 page_cnt = cq->q.pbl_info.num_pbes; 1785 cq->ibcq.cqe = chain_entries; 1786 } else { 1787 ctx = NULL; 1788 1789 cq->cq_type = QLNXR_CQ_TYPE_KERNEL; 1790 1791 rc = ecore_chain_alloc(&dev->ha->cdev, 1792 ECORE_CHAIN_USE_TO_CONSUME, 1793 ECORE_CHAIN_MODE_PBL, 1794 ECORE_CHAIN_CNT_TYPE_U32, 1795 chain_entries, 1796 sizeof(union roce_cqe), 1797 &cq->pbl, NULL); 1798 1799 if (rc) 1800 goto err1; 1801 1802 page_cnt = ecore_chain_get_page_cnt(&cq->pbl); 1803 pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); 1804 cq->ibcq.cqe = cq->pbl.capacity; 1805 } 1806 1807 params.cq_handle_hi = upper_32_bits((uintptr_t)cq); 1808 params.cq_handle_lo = lower_32_bits((uintptr_t)cq); 1809 params.cnq_id = vector; 1810 params.cq_size = chain_entries - 1; 1811 params.pbl_num_pages = page_cnt; 1812 params.pbl_ptr = pbl_ptr; 1813 params.pbl_two_level = 0; 1814 1815 if (udata) { 1816 params.dpi = ctx->dpi; 1817 } else { 1818 params.dpi = dev->dpi; 1819 } 1820 1821 rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 1822 if (rc) 1823 goto err2; 1824 1825 cq->icid = icid; 1826 cq->sig = QLNXR_CQ_MAGIC_NUMBER; 1827 spin_lock_init(&cq->cq_lock); 1828 1829 if (udata) { 1830 rc = qlnxr_copy_cq_uresp(dev, cq, udata); 1831 if (rc) 1832 goto err3; 1833 } else { 1834 /* Generate doorbell address. 1835 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. 1836 * TODO: consider moving to device scope as it is a function of 1837 * the device. 1838 * TODO: add ifdef if plan to support 16 bit. 1839 */ 1840 cq->db_addr = dev->db_addr + 1841 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 1842 cq->db.data.icid = cq->icid; 1843 cq->db.data.params = DB_AGG_CMD_SET << 1844 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1845 1846 /* point to the very last element, passing it we will toggle */ 1847 cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); 1848 cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; 1849 1850 /* must be different from pbl_toggle */ 1851 cq->latest_cqe = NULL; 1852 consume_cqe(cq); 1853 cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 1854 } 1855 1856 QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," 1857 " number of entries = 0x%x\n", 1858 cq->icid, cq, params.cq_size); 1859 QL_DPRINT12(ha,"cq_addr = %p\n", cq); 1860 return (0); 1861 1862 err3: 1863 destroy_iparams.icid = cq->icid; 1864 ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); 1865 err2: 1866 if (udata) 1867 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1868 else 1869 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 1870 err1: 1871 if (udata) 1872 ib_umem_release(cq->q.umem); 1873 err0: 1874 QL_DPRINT12(ha, "exit error\n"); 1875 1876 return (-EINVAL); 1877 } 1878 1879 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1880 { 1881 int status = 0; 1882 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 1883 qlnx_host_t *ha; 1884 1885 ha = dev->ha; 1886 1887 QL_DPRINT12(ha, "enter/exit\n"); 1888 1889 return status; 1890 } 1891 1892 void 1893 qlnxr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1894 { 1895 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 1896 struct ecore_rdma_destroy_cq_out_params oparams; 1897 struct ecore_rdma_destroy_cq_in_params iparams; 1898 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 1899 int rc = 0; 1900 qlnx_host_t *ha; 1901 1902 ha = dev->ha; 1903 1904 QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); 1905 1906 cq->destroyed = 1; 1907 1908 /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation 1909 * that all completions with notification are dealt with. The rest 1910 * of the completions are not interesting 1911 */ 1912 1913 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1914 1915 if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { 1916 iparams.icid = cq->icid; 1917 1918 rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1919 1920 if (rc) { 1921 QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", 1922 cq->icid); 1923 return; 1924 } 1925 1926 QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); 1927 ecore_chain_free(&dev->ha->cdev, &cq->pbl); 1928 } 1929 1930 if (udata) { 1931 qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1932 ib_umem_release(cq->q.umem); 1933 } 1934 1935 cq->sig = ~cq->sig; 1936 1937 QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); 1938 } 1939 1940 static int 1941 qlnxr_check_qp_attrs(struct ib_pd *ibpd, 1942 struct qlnxr_dev *dev, 1943 struct ib_qp_init_attr *attrs, 1944 struct ib_udata *udata) 1945 { 1946 struct ecore_rdma_device *qattr; 1947 qlnx_host_t *ha; 1948 1949 qattr = ecore_rdma_query_device(dev->rdma_ctx); 1950 ha = dev->ha; 1951 1952 QL_DPRINT12(ha, "enter\n"); 1953 1954 QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); 1955 QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); 1956 QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); 1957 1958 QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); 1959 QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); 1960 QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); 1961 QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); 1962 QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); 1963 QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", 1964 attrs->cap.max_inline_data); 1965 1966 QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); 1967 QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); 1968 QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); 1969 QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); 1970 QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); 1971 QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", 1972 (void *)qattr->sys_image_guid); 1973 QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); 1974 QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); 1975 QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); 1976 QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); 1977 QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); 1978 QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); 1979 QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", 1980 qattr->max_qp_resp_rd_atomic_resc); 1981 QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", 1982 qattr->max_qp_req_rd_atomic_resc); 1983 QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", 1984 qattr->max_dev_resp_rd_atomic_resc); 1985 QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); 1986 QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); 1987 QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); 1988 QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); 1989 QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); 1990 QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); 1991 QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); 1992 QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); 1993 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", 1994 qattr->max_mr_mw_fmr_pbl); 1995 QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", 1996 (void *)qattr->max_mr_mw_fmr_size); 1997 QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); 1998 QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); 1999 QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); 2000 QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); 2001 QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", 2002 qattr->max_stats_queues); 2003 //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); 2004 QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", 2005 (void *)qattr->page_size_caps); 2006 QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); 2007 QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); 2008 QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", 2009 qattr->bad_pkey_counter); 2010 2011 if ((attrs->qp_type == IB_QPT_GSI) && udata) { 2012 QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); 2013 return -EINVAL; 2014 } 2015 2016 if (udata && !(ibpd->uobject && ibpd->uobject->context)) { 2017 QL_DPRINT12(ha, "called from user without context\n"); 2018 return -EINVAL; 2019 } 2020 2021 /* QP0... attrs->qp_type == IB_QPT_GSI */ 2022 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 2023 QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", 2024 attrs->qp_type); 2025 return -EINVAL; 2026 } 2027 if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { 2028 QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); 2029 return -EINVAL; 2030 } 2031 /* Skip the check for QP1 to support CM size of 128 */ 2032 if (attrs->cap.max_send_wr > qattr->max_wqe) { 2033 QL_DPRINT12(ha, "cannot create a SQ with %d elements " 2034 " (max_send_wr=0x%x)\n", 2035 attrs->cap.max_send_wr, qattr->max_wqe); 2036 return -EINVAL; 2037 } 2038 if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { 2039 QL_DPRINT12(ha, "cannot create a RQ with %d elements" 2040 " (max_recv_wr=0x%x)\n", 2041 attrs->cap.max_recv_wr, qattr->max_wqe); 2042 return -EINVAL; 2043 } 2044 if (attrs->cap.max_inline_data > qattr->max_inline) { 2045 QL_DPRINT12(ha, 2046 "unsupported inline data size=0x%x " 2047 "requested (max_inline=0x%x)\n", 2048 attrs->cap.max_inline_data, qattr->max_inline); 2049 return -EINVAL; 2050 } 2051 if (attrs->cap.max_send_sge > qattr->max_sge) { 2052 QL_DPRINT12(ha, 2053 "unsupported send_sge=0x%x " 2054 "requested (max_send_sge=0x%x)\n", 2055 attrs->cap.max_send_sge, qattr->max_sge); 2056 return -EINVAL; 2057 } 2058 if (attrs->cap.max_recv_sge > qattr->max_sge) { 2059 QL_DPRINT12(ha, 2060 "unsupported recv_sge=0x%x requested " 2061 " (max_recv_sge=0x%x)\n", 2062 attrs->cap.max_recv_sge, qattr->max_sge); 2063 return -EINVAL; 2064 } 2065 /* unprivileged user space cannot create special QP */ 2066 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { 2067 QL_DPRINT12(ha, 2068 "userspace can't create special QPs of type=0x%x\n", 2069 attrs->qp_type); 2070 return -EINVAL; 2071 } 2072 /* allow creating only one GSI type of QP */ 2073 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 2074 QL_DPRINT12(ha, 2075 "create qp: GSI special QPs already created.\n"); 2076 return -EINVAL; 2077 } 2078 2079 /* verify consumer QPs are not trying to use GSI QP's CQ */ 2080 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 2081 struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); 2082 struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); 2083 2084 if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || 2085 (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { 2086 QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); 2087 return -EINVAL; 2088 } 2089 } 2090 QL_DPRINT12(ha, "exit\n"); 2091 return 0; 2092 } 2093 2094 static int 2095 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, 2096 struct qlnxr_srq *srq, 2097 struct ib_udata *udata) 2098 { 2099 struct qlnxr_create_srq_uresp uresp; 2100 qlnx_host_t *ha; 2101 int rc; 2102 2103 ha = dev->ha; 2104 2105 QL_DPRINT12(ha, "enter\n"); 2106 2107 memset(&uresp, 0, sizeof(uresp)); 2108 2109 uresp.srq_id = srq->srq_id; 2110 2111 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2112 2113 QL_DPRINT12(ha, "exit [%d]\n", rc); 2114 return rc; 2115 } 2116 2117 static void 2118 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, 2119 struct qlnxr_create_qp_uresp *uresp, 2120 struct qlnxr_qp *qp) 2121 { 2122 qlnx_host_t *ha; 2123 2124 ha = dev->ha; 2125 2126 /* Return if QP is associated with SRQ instead of RQ */ 2127 QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); 2128 2129 if (qp->srq) 2130 return; 2131 2132 /* iWARP requires two doorbells per RQ. */ 2133 if (QLNX_IS_IWARP(dev)) { 2134 uresp->rq_db_offset = 2135 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2136 uresp->rq_db2_offset = 2137 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2138 2139 QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " 2140 "uresp->rq_db2_offset = 0x%x\n", 2141 uresp->rq_db_offset, uresp->rq_db2_offset); 2142 } else { 2143 uresp->rq_db_offset = 2144 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2145 } 2146 uresp->rq_icid = qp->icid; 2147 2148 QL_DPRINT12(ha, "exit\n"); 2149 return; 2150 } 2151 2152 static void 2153 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, 2154 struct qlnxr_create_qp_uresp *uresp, 2155 struct qlnxr_qp *qp) 2156 { 2157 qlnx_host_t *ha; 2158 2159 ha = dev->ha; 2160 2161 QL_DPRINT12(ha, "enter\n"); 2162 2163 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2164 2165 /* iWARP uses the same cid for rq and sq*/ 2166 if (QLNX_IS_IWARP(dev)) { 2167 uresp->sq_icid = qp->icid; 2168 QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); 2169 } else 2170 uresp->sq_icid = qp->icid + 1; 2171 2172 QL_DPRINT12(ha, "exit\n"); 2173 return; 2174 } 2175 2176 static int 2177 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, 2178 struct qlnxr_qp *qp, 2179 struct ib_udata *udata) 2180 { 2181 int rc; 2182 struct qlnxr_create_qp_uresp uresp; 2183 qlnx_host_t *ha; 2184 2185 ha = dev->ha; 2186 2187 QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); 2188 2189 memset(&uresp, 0, sizeof(uresp)); 2190 qlnxr_copy_sq_uresp(dev, &uresp, qp); 2191 qlnxr_copy_rq_uresp(dev, &uresp, qp); 2192 2193 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 2194 uresp.qp_id = qp->qp_id; 2195 2196 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 2197 2198 QL_DPRINT12(ha, "exit [%d]\n", rc); 2199 return rc; 2200 } 2201 2202 static void 2203 qlnxr_set_common_qp_params(struct qlnxr_dev *dev, 2204 struct qlnxr_qp *qp, 2205 struct qlnxr_pd *pd, 2206 struct ib_qp_init_attr *attrs) 2207 { 2208 qlnx_host_t *ha; 2209 2210 ha = dev->ha; 2211 2212 QL_DPRINT12(ha, "enter\n"); 2213 2214 spin_lock_init(&qp->q_lock); 2215 2216 atomic_set(&qp->refcnt, 1); 2217 qp->pd = pd; 2218 qp->sig = QLNXR_QP_MAGIC_NUMBER; 2219 qp->qp_type = attrs->qp_type; 2220 qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2221 qp->sq.max_sges = attrs->cap.max_send_sge; 2222 qp->state = ECORE_ROCE_QP_STATE_RESET; 2223 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 2224 qp->sq_cq = get_qlnxr_cq(attrs->send_cq); 2225 qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); 2226 qp->dev = dev; 2227 2228 if (!attrs->srq) { 2229 /* QP is associated with RQ instead of SRQ */ 2230 qp->rq.max_sges = attrs->cap.max_recv_sge; 2231 QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 2232 qp->rq.max_sges, qp->rq_cq->icid); 2233 } else { 2234 qp->srq = get_qlnxr_srq(attrs->srq); 2235 } 2236 2237 QL_DPRINT12(ha, 2238 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," 2239 " state = %d, signaled = %d, use_srq=%d\n", 2240 pd->pd_id, qp->qp_type, qp->max_inline_data, 2241 qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); 2242 QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 2243 qp->sq.max_sges, qp->sq_cq->icid); 2244 return; 2245 } 2246 2247 static int 2248 qlnxr_check_srq_params(struct qlnxr_dev *dev, 2249 struct ib_srq_init_attr *attrs) 2250 { 2251 struct ecore_rdma_device *qattr; 2252 qlnx_host_t *ha; 2253 2254 ha = dev->ha; 2255 qattr = ecore_rdma_query_device(dev->rdma_ctx); 2256 2257 QL_DPRINT12(ha, "enter\n"); 2258 2259 if (attrs->attr.max_wr > qattr->max_srq_wqe) { 2260 QL_DPRINT12(ha, "unsupported srq_wr=0x%x" 2261 " requested (max_srq_wr=0x%x)\n", 2262 attrs->attr.max_wr, qattr->max_srq_wr); 2263 return -EINVAL; 2264 } 2265 2266 if (attrs->attr.max_sge > qattr->max_sge) { 2267 QL_DPRINT12(ha, 2268 "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 2269 attrs->attr.max_sge, qattr->max_sge); 2270 return -EINVAL; 2271 } 2272 2273 if (attrs->attr.srq_limit > attrs->attr.max_wr) { 2274 QL_DPRINT12(ha, 2275 "unsupported srq_limit=0x%x requested" 2276 " (max_srq_limit=0x%x)\n", 2277 attrs->attr.srq_limit, attrs->attr.srq_limit); 2278 return -EINVAL; 2279 } 2280 2281 QL_DPRINT12(ha, "exit\n"); 2282 return 0; 2283 } 2284 2285 static void 2286 qlnxr_free_srq_user_params(struct qlnxr_srq *srq) 2287 { 2288 struct qlnxr_dev *dev = srq->dev; 2289 qlnx_host_t *ha; 2290 2291 ha = dev->ha; 2292 2293 QL_DPRINT12(ha, "enter\n"); 2294 2295 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2296 ib_umem_release(srq->usrq.umem); 2297 ib_umem_release(srq->prod_umem); 2298 2299 QL_DPRINT12(ha, "exit\n"); 2300 return; 2301 } 2302 2303 static void 2304 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) 2305 { 2306 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2307 struct qlnxr_dev *dev = srq->dev; 2308 qlnx_host_t *ha; 2309 2310 ha = dev->ha; 2311 2312 QL_DPRINT12(ha, "enter\n"); 2313 2314 ecore_chain_free(dev->cdev, &hw_srq->pbl); 2315 2316 qlnx_dma_free_coherent(&dev->cdev, 2317 hw_srq->virt_prod_pair_addr, 2318 hw_srq->phy_prod_pair_addr, 2319 sizeof(struct rdma_srq_producers)); 2320 2321 QL_DPRINT12(ha, "exit\n"); 2322 2323 return; 2324 } 2325 2326 static int 2327 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, 2328 struct qlnxr_srq *srq, 2329 struct qlnxr_create_srq_ureq *ureq, 2330 int access, int dmasync) 2331 { 2332 struct scatterlist *sg; 2333 int rc; 2334 struct qlnxr_dev *dev = srq->dev; 2335 qlnx_host_t *ha; 2336 2337 ha = dev->ha; 2338 2339 QL_DPRINT12(ha, "enter\n"); 2340 2341 rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, 2342 ureq->srq_len, access, dmasync, 1); 2343 if (rc) 2344 return rc; 2345 2346 srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, 2347 sizeof(struct rdma_srq_producers), 2348 access, dmasync); 2349 if (IS_ERR(srq->prod_umem)) { 2350 qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 2351 ib_umem_release(srq->usrq.umem); 2352 2353 QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", 2354 PTR_ERR(srq->prod_umem)); 2355 2356 return PTR_ERR(srq->prod_umem); 2357 } 2358 2359 sg = srq->prod_umem->sg_head.sgl; 2360 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 2361 2362 QL_DPRINT12(ha, "exit\n"); 2363 return 0; 2364 } 2365 2366 static int 2367 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, 2368 struct qlnxr_dev *dev, 2369 struct ib_srq_init_attr *init_attr) 2370 { 2371 struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; 2372 dma_addr_t phy_prod_pair_addr; 2373 u32 num_elems, max_wr; 2374 void *va; 2375 int rc; 2376 qlnx_host_t *ha; 2377 2378 ha = dev->ha; 2379 2380 QL_DPRINT12(ha, "enter\n"); 2381 2382 va = qlnx_dma_alloc_coherent(&dev->cdev, 2383 &phy_prod_pair_addr, 2384 sizeof(struct rdma_srq_producers)); 2385 if (!va) { 2386 QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); 2387 return -ENOMEM; 2388 } 2389 2390 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 2391 hw_srq->virt_prod_pair_addr = va; 2392 2393 max_wr = init_attr->attr.max_wr; 2394 2395 num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; 2396 2397 rc = ecore_chain_alloc(dev->cdev, 2398 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2399 ECORE_CHAIN_MODE_PBL, 2400 ECORE_CHAIN_CNT_TYPE_U32, 2401 num_elems, 2402 ECORE_RDMA_SRQ_WQE_ELEM_SIZE, 2403 &hw_srq->pbl, NULL); 2404 2405 if (rc) { 2406 QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); 2407 goto err0; 2408 } 2409 2410 hw_srq->max_wr = max_wr; 2411 hw_srq->num_elems = num_elems; 2412 hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; 2413 2414 QL_DPRINT12(ha, "exit\n"); 2415 return 0; 2416 2417 err0: 2418 qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, 2419 sizeof(struct rdma_srq_producers)); 2420 2421 QL_DPRINT12(ha, "exit [%d]\n", rc); 2422 return rc; 2423 } 2424 2425 static inline void 2426 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, 2427 struct qlnxr_pd *pd, 2428 struct qlnxr_qp *qp, 2429 struct ib_qp_init_attr *attrs, 2430 bool fmr_and_reserved_lkey, 2431 struct ecore_rdma_create_qp_in_params *params) 2432 { 2433 qlnx_host_t *ha; 2434 2435 ha = dev->ha; 2436 2437 QL_DPRINT12(ha, "enter\n"); 2438 2439 /* QP handle to be written in an async event */ 2440 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 2441 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 2442 2443 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 2444 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 2445 params->pd = pd->pd_id; 2446 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 2447 params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 2448 params->stats_queue = 0; 2449 2450 params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 2451 2452 if (qp->srq) { 2453 /* QP is associated with SRQ instead of RQ */ 2454 params->srq_id = qp->srq->srq_id; 2455 params->use_srq = true; 2456 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 2457 params->srq_id, params->use_srq); 2458 return; 2459 } 2460 2461 params->srq_id = 0; 2462 params->use_srq = false; 2463 2464 QL_DPRINT12(ha, "exit\n"); 2465 return; 2466 } 2467 2468 static inline void 2469 qlnxr_qp_user_print( struct qlnxr_dev *dev, 2470 struct qlnxr_qp *qp) 2471 { 2472 QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " 2473 "rq_addr=0x%llx, rq_len=%zd\n", 2474 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, 2475 qp->urq.buf_len); 2476 return; 2477 } 2478 2479 static int 2480 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) 2481 { 2482 u32 newid; 2483 int rc; 2484 qlnx_host_t *ha; 2485 2486 ha = dev->ha; 2487 2488 QL_DPRINT12(ha, "enter\n"); 2489 2490 if (!QLNX_IS_IWARP(dev)) 2491 return 0; 2492 2493 do { 2494 if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { 2495 QL_DPRINT11(ha, "idr_pre_get failed\n"); 2496 return -ENOMEM; 2497 } 2498 2499 mtx_lock(&dev->idr_lock); 2500 2501 rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); 2502 2503 mtx_unlock(&dev->idr_lock); 2504 2505 } while (rc == -EAGAIN); 2506 2507 QL_DPRINT12(ha, "exit [%d]\n", rc); 2508 2509 return rc; 2510 } 2511 2512 static void 2513 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) 2514 { 2515 qlnx_host_t *ha; 2516 2517 ha = dev->ha; 2518 2519 QL_DPRINT12(ha, "enter\n"); 2520 2521 if (!QLNX_IS_IWARP(dev)) 2522 return; 2523 2524 mtx_lock(&dev->idr_lock); 2525 idr_remove(&dev->qpidr, id); 2526 mtx_unlock(&dev->idr_lock); 2527 2528 QL_DPRINT12(ha, "exit \n"); 2529 2530 return; 2531 } 2532 2533 static inline void 2534 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, 2535 struct qlnxr_qp *qp, 2536 struct ecore_rdma_create_qp_out_params *out_params) 2537 { 2538 qlnx_host_t *ha; 2539 2540 ha = dev->ha; 2541 2542 QL_DPRINT12(ha, "enter\n"); 2543 2544 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 2545 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 2546 2547 qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 2548 &qp->usq.pbl_info); 2549 2550 if (qp->srq) { 2551 QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); 2552 return; 2553 } 2554 2555 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 2556 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 2557 2558 qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 2559 &qp->urq.pbl_info); 2560 2561 QL_DPRINT12(ha, "exit\n"); 2562 return; 2563 } 2564 2565 static int 2566 qlnxr_create_user_qp(struct qlnxr_dev *dev, 2567 struct qlnxr_qp *qp, 2568 struct ib_pd *ibpd, 2569 struct ib_udata *udata, 2570 struct ib_qp_init_attr *attrs) 2571 { 2572 struct ecore_rdma_destroy_qp_out_params d_out_params; 2573 struct ecore_rdma_create_qp_in_params in_params; 2574 struct ecore_rdma_create_qp_out_params out_params; 2575 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2576 struct ib_ucontext *ib_ctx = NULL; 2577 struct qlnxr_create_qp_ureq ureq; 2578 int alloc_and_init = QLNX_IS_ROCE(dev); 2579 int rc = -EINVAL; 2580 qlnx_host_t *ha; 2581 2582 ha = dev->ha; 2583 2584 QL_DPRINT12(ha, "enter\n"); 2585 2586 ib_ctx = ibpd->uobject->context; 2587 2588 memset(&ureq, 0, sizeof(ureq)); 2589 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); 2590 2591 if (rc) { 2592 QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); 2593 return rc; 2594 } 2595 2596 /* SQ - read access only (0), dma sync not required (0) */ 2597 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, 2598 ureq.sq_len, 0, 0, 2599 alloc_and_init); 2600 if (rc) { 2601 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2602 return rc; 2603 } 2604 2605 if (!qp->srq) { 2606 /* RQ - read access only (0), dma sync not required (0) */ 2607 rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, 2608 ureq.rq_len, 0, 0, 2609 alloc_and_init); 2610 2611 if (rc) { 2612 QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); 2613 return rc; 2614 } 2615 } 2616 2617 memset(&in_params, 0, sizeof(in_params)); 2618 qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 2619 in_params.qp_handle_lo = ureq.qp_handle_lo; 2620 in_params.qp_handle_hi = ureq.qp_handle_hi; 2621 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 2622 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 2623 2624 if (!qp->srq) { 2625 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 2626 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 2627 } 2628 2629 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); 2630 2631 if (!qp->ecore_qp) { 2632 rc = -ENOMEM; 2633 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2634 goto err1; 2635 } 2636 2637 if (QLNX_IS_IWARP(dev)) 2638 qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); 2639 2640 qp->qp_id = out_params.qp_id; 2641 qp->icid = out_params.icid; 2642 2643 rc = qlnxr_copy_qp_uresp(dev, qp, udata); 2644 2645 if (rc) { 2646 QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); 2647 goto err; 2648 } 2649 2650 qlnxr_qp_user_print(dev, qp); 2651 2652 QL_DPRINT12(ha, "exit\n"); 2653 return 0; 2654 err: 2655 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2656 2657 if (rc) 2658 QL_DPRINT12(ha, "fatal fault\n"); 2659 2660 err1: 2661 qlnxr_cleanup_user(dev, qp); 2662 2663 QL_DPRINT12(ha, "exit[%d]\n", rc); 2664 return rc; 2665 } 2666 2667 static void 2668 qlnxr_set_roce_db_info(struct qlnxr_dev *dev, 2669 struct qlnxr_qp *qp) 2670 { 2671 qlnx_host_t *ha; 2672 2673 ha = dev->ha; 2674 2675 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2676 2677 qp->sq.db = dev->db_addr + 2678 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2679 qp->sq.db_data.data.icid = qp->icid + 1; 2680 2681 if (!qp->srq) { 2682 qp->rq.db = dev->db_addr + 2683 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 2684 qp->rq.db_data.data.icid = qp->icid; 2685 } 2686 2687 QL_DPRINT12(ha, "exit\n"); 2688 return; 2689 } 2690 2691 static void 2692 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, 2693 struct qlnxr_qp *qp) 2694 2695 { 2696 qlnx_host_t *ha; 2697 2698 ha = dev->ha; 2699 2700 QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); 2701 2702 qp->sq.db = dev->db_addr + 2703 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2704 qp->sq.db_data.data.icid = qp->icid; 2705 2706 if (!qp->srq) { 2707 qp->rq.db = dev->db_addr + 2708 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2709 qp->rq.db_data.data.icid = qp->icid; 2710 2711 qp->rq.iwarp_db2 = dev->db_addr + 2712 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2713 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2714 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2715 } 2716 2717 QL_DPRINT12(ha, 2718 "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" 2719 "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" 2720 "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" 2721 " qp->rq.iwarp_db2.data.prod_val =0x%x\n", 2722 qp->sq.db, qp->sq.db_data.data.icid, 2723 qp->rq.db, qp->rq.db_data.data.icid, 2724 qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, 2725 qp->rq.iwarp_db2_data.data.value); 2726 2727 QL_DPRINT12(ha, "exit\n"); 2728 return; 2729 } 2730 2731 static int 2732 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, 2733 struct qlnxr_qp *qp, 2734 struct ecore_rdma_create_qp_in_params *in_params, 2735 u32 n_sq_elems, 2736 u32 n_rq_elems) 2737 { 2738 struct ecore_rdma_create_qp_out_params out_params; 2739 int rc; 2740 qlnx_host_t *ha; 2741 2742 ha = dev->ha; 2743 2744 QL_DPRINT12(ha, "enter\n"); 2745 2746 rc = ecore_chain_alloc( 2747 dev->cdev, 2748 ECORE_CHAIN_USE_TO_PRODUCE, 2749 ECORE_CHAIN_MODE_PBL, 2750 ECORE_CHAIN_CNT_TYPE_U32, 2751 n_sq_elems, 2752 QLNXR_SQE_ELEMENT_SIZE, 2753 &qp->sq.pbl, 2754 NULL); 2755 2756 if (rc) { 2757 QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); 2758 return rc; 2759 } 2760 2761 in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); 2762 in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); 2763 2764 if (!qp->srq) { 2765 rc = ecore_chain_alloc( 2766 dev->cdev, 2767 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2768 ECORE_CHAIN_MODE_PBL, 2769 ECORE_CHAIN_CNT_TYPE_U32, 2770 n_rq_elems, 2771 QLNXR_RQE_ELEMENT_SIZE, 2772 &qp->rq.pbl, 2773 NULL); 2774 2775 if (rc) { 2776 QL_DPRINT11(ha, 2777 "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); 2778 return rc; 2779 } 2780 2781 in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); 2782 in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); 2783 } 2784 2785 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2786 2787 if (!qp->ecore_qp) { 2788 QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); 2789 return -EINVAL; 2790 } 2791 2792 qp->qp_id = out_params.qp_id; 2793 qp->icid = out_params.icid; 2794 2795 qlnxr_set_roce_db_info(dev, qp); 2796 2797 QL_DPRINT12(ha, "exit\n"); 2798 return 0; 2799 } 2800 2801 static int 2802 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, 2803 struct qlnxr_qp *qp, 2804 struct ecore_rdma_create_qp_in_params *in_params, 2805 u32 n_sq_elems, 2806 u32 n_rq_elems) 2807 { 2808 struct ecore_rdma_destroy_qp_out_params d_out_params; 2809 struct ecore_rdma_create_qp_out_params out_params; 2810 struct ecore_chain_ext_pbl ext_pbl; 2811 int rc; 2812 qlnx_host_t *ha; 2813 2814 ha = dev->ha; 2815 2816 QL_DPRINT12(ha, "enter\n"); 2817 2818 in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, 2819 QLNXR_SQE_ELEMENT_SIZE, 2820 ECORE_CHAIN_MODE_PBL); 2821 in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, 2822 QLNXR_RQE_ELEMENT_SIZE, 2823 ECORE_CHAIN_MODE_PBL); 2824 2825 QL_DPRINT12(ha, "n_sq_elems = 0x%x" 2826 " n_rq_elems = 0x%x in_params\n" 2827 "\t\t\tqp_handle_lo\t\t= 0x%08x\n" 2828 "\t\t\tqp_handle_hi\t\t= 0x%08x\n" 2829 "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" 2830 "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" 2831 "\t\t\tuse_srq\t\t\t= 0x%x\n" 2832 "\t\t\tsignal_all\t\t= 0x%x\n" 2833 "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" 2834 "\t\t\tpd\t\t\t= 0x%x\n" 2835 "\t\t\tdpi\t\t\t= 0x%x\n" 2836 "\t\t\tsq_cq_id\t\t\t= 0x%x\n" 2837 "\t\t\tsq_num_pages\t\t= 0x%x\n" 2838 "\t\t\tsq_pbl_ptr\t\t= %p\n" 2839 "\t\t\tmax_sq_sges\t\t= 0x%x\n" 2840 "\t\t\trq_cq_id\t\t\t= 0x%x\n" 2841 "\t\t\trq_num_pages\t\t= 0x%x\n" 2842 "\t\t\trq_pbl_ptr\t\t= %p\n" 2843 "\t\t\tsrq_id\t\t\t= 0x%x\n" 2844 "\t\t\tstats_queue\t\t= 0x%x\n", 2845 n_sq_elems, n_rq_elems, 2846 in_params->qp_handle_lo, 2847 in_params->qp_handle_hi, 2848 in_params->qp_handle_async_lo, 2849 in_params->qp_handle_async_hi, 2850 in_params->use_srq, 2851 in_params->signal_all, 2852 in_params->fmr_and_reserved_lkey, 2853 in_params->pd, 2854 in_params->dpi, 2855 in_params->sq_cq_id, 2856 in_params->sq_num_pages, 2857 (void *)in_params->sq_pbl_ptr, 2858 in_params->max_sq_sges, 2859 in_params->rq_cq_id, 2860 in_params->rq_num_pages, 2861 (void *)in_params->rq_pbl_ptr, 2862 in_params->srq_id, 2863 in_params->stats_queue ); 2864 2865 memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); 2866 memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); 2867 2868 qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); 2869 2870 if (!qp->ecore_qp) { 2871 QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); 2872 return -EINVAL; 2873 } 2874 2875 /* Now we allocate the chain */ 2876 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 2877 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 2878 2879 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 2880 "ext_pbl.p_pbl_phys = %p\n", 2881 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 2882 2883 rc = ecore_chain_alloc( 2884 dev->cdev, 2885 ECORE_CHAIN_USE_TO_PRODUCE, 2886 ECORE_CHAIN_MODE_PBL, 2887 ECORE_CHAIN_CNT_TYPE_U32, 2888 n_sq_elems, 2889 QLNXR_SQE_ELEMENT_SIZE, 2890 &qp->sq.pbl, 2891 &ext_pbl); 2892 2893 if (rc) { 2894 QL_DPRINT11(ha, 2895 "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); 2896 goto err; 2897 } 2898 2899 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 2900 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 2901 2902 QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " 2903 "ext_pbl.p_pbl_phys = %p\n", 2904 ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); 2905 2906 if (!qp->srq) { 2907 rc = ecore_chain_alloc( 2908 dev->cdev, 2909 ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, 2910 ECORE_CHAIN_MODE_PBL, 2911 ECORE_CHAIN_CNT_TYPE_U32, 2912 n_rq_elems, 2913 QLNXR_RQE_ELEMENT_SIZE, 2914 &qp->rq.pbl, 2915 &ext_pbl); 2916 2917 if (rc) { 2918 QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" 2919 " failed rc = %d\n", rc); 2920 goto err; 2921 } 2922 } 2923 2924 QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", 2925 out_params.qp_id, out_params.icid); 2926 2927 qp->qp_id = out_params.qp_id; 2928 qp->icid = out_params.icid; 2929 2930 qlnxr_set_iwarp_db_info(dev, qp); 2931 2932 QL_DPRINT12(ha, "exit\n"); 2933 return 0; 2934 2935 err: 2936 ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); 2937 2938 QL_DPRINT12(ha, "exit rc = %d\n", rc); 2939 return rc; 2940 } 2941 2942 static int 2943 qlnxr_create_kernel_qp(struct qlnxr_dev *dev, 2944 struct qlnxr_qp *qp, 2945 struct ib_pd *ibpd, 2946 struct ib_qp_init_attr *attrs) 2947 { 2948 struct ecore_rdma_create_qp_in_params in_params; 2949 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 2950 int rc = -EINVAL; 2951 u32 n_rq_elems; 2952 u32 n_sq_elems; 2953 u32 n_sq_entries; 2954 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 2955 qlnx_host_t *ha; 2956 2957 ha = dev->ha; 2958 2959 QL_DPRINT12(ha, "enter\n"); 2960 2961 memset(&in_params, 0, sizeof(in_params)); 2962 2963 /* A single work request may take up to MAX_SQ_WQE_SIZE elements in 2964 * the ring. The ring should allow at least a single WR, even if the 2965 * user requested none, due to allocation issues. 2966 * We should add an extra WR since the prod and cons indices of 2967 * wqe_wr_id are managed in such a way that the WQ is considered full 2968 * when (prod+1)%max_wr==cons. We currently don't do that because we 2969 * double the number of entries due an iSER issue that pushes far more 2970 * WRs than indicated. If we decline its ib_post_send() then we get 2971 * error prints in the dmesg we'd like to avoid. 2972 */ 2973 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2974 qattr->max_wqe); 2975 2976 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), 2977 GFP_KERNEL); 2978 if (!qp->wqe_wr_id) { 2979 QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); 2980 return -ENOMEM; 2981 } 2982 2983 /* QP handle to be written in CQE */ 2984 in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); 2985 in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); 2986 2987 /* A single work request may take up to MAX_RQ_WQE_SIZE elements in 2988 * the ring. There ring should allow at least a single WR, even if the 2989 * user requested none, due to allocation issues. 2990 */ 2991 qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); 2992 2993 /* Allocate driver internal RQ array */ 2994 if (!qp->srq) { 2995 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), 2996 GFP_KERNEL); 2997 if (!qp->rqe_wr_id) { 2998 QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); 2999 kfree(qp->wqe_wr_id); 3000 return -ENOMEM; 3001 } 3002 } 3003 3004 //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 3005 3006 in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); 3007 in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); 3008 3009 in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 3010 in_params.fmr_and_reserved_lkey = true; 3011 in_params.pd = pd->pd_id; 3012 in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 3013 in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; 3014 in_params.stats_queue = 0; 3015 3016 in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; 3017 3018 if (qp->srq) { 3019 /* QP is associated with SRQ instead of RQ */ 3020 in_params.srq_id = qp->srq->srq_id; 3021 in_params.use_srq = true; 3022 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", 3023 in_params.srq_id, in_params.use_srq); 3024 } else { 3025 in_params.srq_id = 0; 3026 in_params.use_srq = false; 3027 } 3028 3029 n_sq_entries = attrs->cap.max_send_wr; 3030 n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); 3031 n_sq_entries = max_t(u32, n_sq_entries, 1); 3032 n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 3033 3034 n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; 3035 3036 if (QLNX_IS_ROCE(dev)) { 3037 rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, 3038 n_sq_elems, n_rq_elems); 3039 } else { 3040 rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, 3041 n_sq_elems, n_rq_elems); 3042 } 3043 3044 if (rc) 3045 qlnxr_cleanup_kernel(dev, qp); 3046 3047 QL_DPRINT12(ha, "exit [%d]\n", rc); 3048 return rc; 3049 } 3050 3051 struct ib_qp * 3052 qlnxr_create_qp(struct ib_pd *ibpd, 3053 struct ib_qp_init_attr *attrs, 3054 struct ib_udata *udata) 3055 { 3056 struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); 3057 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 3058 struct qlnxr_qp *qp; 3059 int rc = 0; 3060 qlnx_host_t *ha; 3061 3062 ha = dev->ha; 3063 3064 QL_DPRINT12(ha, "enter\n"); 3065 3066 rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); 3067 if (rc) { 3068 QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); 3069 return ERR_PTR(rc); 3070 } 3071 3072 QL_DPRINT12(ha, "called from %s, event_handle=%p," 3073 " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 3074 (udata ? "user library" : "kernel"), 3075 attrs->event_handler, pd, 3076 get_qlnxr_cq(attrs->send_cq), 3077 get_qlnxr_cq(attrs->send_cq)->icid, 3078 get_qlnxr_cq(attrs->recv_cq), 3079 get_qlnxr_cq(attrs->recv_cq)->icid); 3080 3081 qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); 3082 3083 if (!qp) { 3084 QL_DPRINT11(ha, "kzalloc(qp) failed\n"); 3085 return ERR_PTR(-ENOMEM); 3086 } 3087 3088 qlnxr_set_common_qp_params(dev, qp, pd, attrs); 3089 3090 if (attrs->qp_type == IB_QPT_GSI) { 3091 QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); 3092 return qlnxr_create_gsi_qp(dev, attrs, qp); 3093 } 3094 3095 if (udata) { 3096 rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); 3097 3098 if (rc) { 3099 QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); 3100 goto err; 3101 } 3102 } else { 3103 rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); 3104 3105 if (rc) { 3106 QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); 3107 goto err; 3108 } 3109 } 3110 3111 qp->ibqp.qp_num = qp->qp_id; 3112 3113 rc = qlnxr_idr_add(dev, qp, qp->qp_id); 3114 3115 if (rc) { 3116 QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); 3117 goto err; 3118 } 3119 3120 QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); 3121 3122 return &qp->ibqp; 3123 err: 3124 QL_DPRINT12(ha, "failed exit\n"); 3125 return ERR_PTR(-EFAULT); 3126 } 3127 3128 static enum ib_qp_state 3129 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) 3130 { 3131 enum ib_qp_state state = IB_QPS_ERR; 3132 3133 switch (qp_state) { 3134 case ECORE_ROCE_QP_STATE_RESET: 3135 state = IB_QPS_RESET; 3136 break; 3137 3138 case ECORE_ROCE_QP_STATE_INIT: 3139 state = IB_QPS_INIT; 3140 break; 3141 3142 case ECORE_ROCE_QP_STATE_RTR: 3143 state = IB_QPS_RTR; 3144 break; 3145 3146 case ECORE_ROCE_QP_STATE_RTS: 3147 state = IB_QPS_RTS; 3148 break; 3149 3150 case ECORE_ROCE_QP_STATE_SQD: 3151 state = IB_QPS_SQD; 3152 break; 3153 3154 case ECORE_ROCE_QP_STATE_ERR: 3155 state = IB_QPS_ERR; 3156 break; 3157 3158 case ECORE_ROCE_QP_STATE_SQE: 3159 state = IB_QPS_SQE; 3160 break; 3161 } 3162 return state; 3163 } 3164 3165 static enum ecore_roce_qp_state 3166 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) 3167 { 3168 enum ecore_roce_qp_state ecore_qp_state; 3169 3170 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3171 3172 switch (qp_state) { 3173 case IB_QPS_RESET: 3174 ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; 3175 break; 3176 3177 case IB_QPS_INIT: 3178 ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; 3179 break; 3180 3181 case IB_QPS_RTR: 3182 ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; 3183 break; 3184 3185 case IB_QPS_RTS: 3186 ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; 3187 break; 3188 3189 case IB_QPS_SQD: 3190 ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; 3191 break; 3192 3193 case IB_QPS_ERR: 3194 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3195 break; 3196 3197 default: 3198 ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; 3199 break; 3200 } 3201 3202 return (ecore_qp_state); 3203 } 3204 3205 static void 3206 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) 3207 { 3208 ecore_chain_reset(&qph->pbl); 3209 qph->prod = qph->cons = 0; 3210 qph->wqe_cons = 0; 3211 qph->db_data.data.value = cpu_to_le16(0); 3212 3213 return; 3214 } 3215 3216 static int 3217 qlnxr_update_qp_state(struct qlnxr_dev *dev, 3218 struct qlnxr_qp *qp, 3219 enum ecore_roce_qp_state new_state) 3220 { 3221 int status = 0; 3222 uint32_t reg_addr; 3223 struct ecore_dev *cdev; 3224 qlnx_host_t *ha; 3225 3226 ha = dev->ha; 3227 cdev = &ha->cdev; 3228 3229 QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", 3230 qp, new_state, qp->state); 3231 3232 if (new_state == qp->state) { 3233 return 0; 3234 } 3235 3236 switch (qp->state) { 3237 case ECORE_ROCE_QP_STATE_RESET: 3238 switch (new_state) { 3239 case ECORE_ROCE_QP_STATE_INIT: 3240 qp->prev_wqe_size = 0; 3241 qlnxr_reset_qp_hwq_info(&qp->sq); 3242 if (!(qp->srq)) 3243 qlnxr_reset_qp_hwq_info(&qp->rq); 3244 break; 3245 default: 3246 status = -EINVAL; 3247 break; 3248 }; 3249 break; 3250 case ECORE_ROCE_QP_STATE_INIT: 3251 /* INIT->XXX */ 3252 switch (new_state) { 3253 case ECORE_ROCE_QP_STATE_RTR: 3254 /* Update doorbell (in case post_recv was done before move to RTR) */ 3255 if (qp->srq) 3256 break; 3257 wmb(); 3258 //writel(qp->rq.db_data.raw, qp->rq.db); 3259 //if (QLNX_IS_IWARP(dev)) 3260 // writel(qp->rq.iwarp_db2_data.raw, 3261 // qp->rq.iwarp_db2); 3262 3263 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 3264 (uint8_t *)cdev->doorbells); 3265 3266 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 3267 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 3268 3269 if (QLNX_IS_IWARP(dev)) { 3270 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 3271 (uint8_t *)cdev->doorbells); 3272 bus_write_4(ha->pci_dbells, reg_addr,\ 3273 qp->rq.iwarp_db2_data.raw); 3274 bus_barrier(ha->pci_dbells, 0, 0,\ 3275 BUS_SPACE_BARRIER_READ); 3276 } 3277 3278 3279 mmiowb(); 3280 break; 3281 case ECORE_ROCE_QP_STATE_ERR: 3282 /* TBD:flush qps... */ 3283 break; 3284 default: 3285 /* invalid state change. */ 3286 status = -EINVAL; 3287 break; 3288 }; 3289 break; 3290 case ECORE_ROCE_QP_STATE_RTR: 3291 /* RTR->XXX */ 3292 switch (new_state) { 3293 case ECORE_ROCE_QP_STATE_RTS: 3294 break; 3295 case ECORE_ROCE_QP_STATE_ERR: 3296 break; 3297 default: 3298 /* invalid state change. */ 3299 status = -EINVAL; 3300 break; 3301 }; 3302 break; 3303 case ECORE_ROCE_QP_STATE_RTS: 3304 /* RTS->XXX */ 3305 switch (new_state) { 3306 case ECORE_ROCE_QP_STATE_SQD: 3307 break; 3308 case ECORE_ROCE_QP_STATE_ERR: 3309 break; 3310 default: 3311 /* invalid state change. */ 3312 status = -EINVAL; 3313 break; 3314 }; 3315 break; 3316 case ECORE_ROCE_QP_STATE_SQD: 3317 /* SQD->XXX */ 3318 switch (new_state) { 3319 case ECORE_ROCE_QP_STATE_RTS: 3320 case ECORE_ROCE_QP_STATE_ERR: 3321 break; 3322 default: 3323 /* invalid state change. */ 3324 status = -EINVAL; 3325 break; 3326 }; 3327 break; 3328 case ECORE_ROCE_QP_STATE_ERR: 3329 /* ERR->XXX */ 3330 switch (new_state) { 3331 case ECORE_ROCE_QP_STATE_RESET: 3332 if ((qp->rq.prod != qp->rq.cons) || 3333 (qp->sq.prod != qp->sq.cons)) { 3334 QL_DPRINT11(ha, 3335 "Error->Reset with rq/sq " 3336 "not empty rq.prod=0x%x rq.cons=0x%x" 3337 " sq.prod=0x%x sq.cons=0x%x\n", 3338 qp->rq.prod, qp->rq.cons, 3339 qp->sq.prod, qp->sq.cons); 3340 status = -EINVAL; 3341 } 3342 break; 3343 default: 3344 status = -EINVAL; 3345 break; 3346 }; 3347 break; 3348 default: 3349 status = -EINVAL; 3350 break; 3351 }; 3352 3353 QL_DPRINT12(ha, "exit\n"); 3354 return status; 3355 } 3356 3357 int 3358 qlnxr_modify_qp(struct ib_qp *ibqp, 3359 struct ib_qp_attr *attr, 3360 int attr_mask, 3361 struct ib_udata *udata) 3362 { 3363 int rc = 0; 3364 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3365 struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); 3366 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 3367 enum ib_qp_state old_qp_state, new_qp_state; 3368 struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); 3369 qlnx_host_t *ha; 3370 3371 ha = dev->ha; 3372 3373 QL_DPRINT12(ha, 3374 "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", 3375 qp, attr_mask, attr->qp_state, udata); 3376 3377 old_qp_state = qlnxr_get_ibqp_state(qp->state); 3378 if (attr_mask & IB_QP_STATE) 3379 new_qp_state = attr->qp_state; 3380 else 3381 new_qp_state = old_qp_state; 3382 3383 if (QLNX_IS_ROCE(dev)) { 3384 if (!ib_modify_qp_is_ok(old_qp_state, 3385 new_qp_state, 3386 ibqp->qp_type, 3387 attr_mask )) { 3388 QL_DPRINT12(ha, 3389 "invalid attribute mask=0x%x" 3390 " specified for qpn=0x%x of type=0x%x \n" 3391 " old_qp_state=0x%x, new_qp_state=0x%x\n", 3392 attr_mask, qp->qp_id, ibqp->qp_type, 3393 old_qp_state, new_qp_state); 3394 rc = -EINVAL; 3395 goto err; 3396 } 3397 } 3398 /* translate the masks... */ 3399 if (attr_mask & IB_QP_STATE) { 3400 SET_FIELD(qp_params.modify_flags, 3401 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 3402 qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); 3403 } 3404 3405 // TBD consider changing ecore to be a flag as well... 3406 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 3407 qp_params.sqd_async = true; 3408 3409 if (attr_mask & IB_QP_PKEY_INDEX) { 3410 SET_FIELD(qp_params.modify_flags, 3411 ECORE_ROCE_MODIFY_QP_VALID_PKEY, 3412 1); 3413 if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { 3414 rc = -EINVAL; 3415 goto err; 3416 } 3417 3418 qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; 3419 } 3420 3421 if (attr_mask & IB_QP_QKEY) { 3422 qp->qkey = attr->qkey; 3423 } 3424 3425 /* tbd consider splitting in ecore.. */ 3426 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3427 SET_FIELD(qp_params.modify_flags, 3428 ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 3429 qp_params.incoming_rdma_read_en = 3430 attr->qp_access_flags & IB_ACCESS_REMOTE_READ; 3431 qp_params.incoming_rdma_write_en = 3432 attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; 3433 qp_params.incoming_atomic_en = 3434 attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; 3435 } 3436 3437 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 3438 if (attr_mask & IB_QP_PATH_MTU) { 3439 if (attr->path_mtu < IB_MTU_256 || 3440 attr->path_mtu > IB_MTU_4096) { 3441 QL_DPRINT12(ha, 3442 "Only MTU sizes of 256, 512, 1024," 3443 " 2048 and 4096 are supported " 3444 " attr->path_mtu = [%d]\n", 3445 attr->path_mtu); 3446 3447 rc = -EINVAL; 3448 goto err; 3449 } 3450 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 3451 ib_mtu_enum_to_int( 3452 iboe_get_mtu(dev->ha->ifp->if_mtu))); 3453 } 3454 3455 if (qp->mtu == 0) { 3456 qp->mtu = ib_mtu_enum_to_int( 3457 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3458 QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", 3459 qp->mtu); 3460 } 3461 3462 SET_FIELD(qp_params.modify_flags, 3463 ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 3464 1); 3465 3466 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; 3467 qp_params.flow_label = attr->ah_attr.grh.flow_label; 3468 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; 3469 3470 qp->sgid_idx = attr->ah_attr.grh.sgid_index; 3471 3472 get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); 3473 3474 rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); 3475 if (rc) 3476 return rc; 3477 3478 qp_params.use_local_mac = true; 3479 memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 3480 3481 QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", 3482 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 3483 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 3484 QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", 3485 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 3486 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 3487 QL_DPRINT12(ha, 3488 "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", 3489 qp_params.remote_mac_addr[0], 3490 qp_params.remote_mac_addr[1], 3491 qp_params.remote_mac_addr[2], 3492 qp_params.remote_mac_addr[3], 3493 qp_params.remote_mac_addr[4], 3494 qp_params.remote_mac_addr[5]); 3495 3496 qp_params.mtu = qp->mtu; 3497 } 3498 3499 if (qp_params.mtu == 0) { 3500 /* stay with current MTU */ 3501 if (qp->mtu) { 3502 qp_params.mtu = qp->mtu; 3503 } else { 3504 qp_params.mtu = ib_mtu_enum_to_int( 3505 iboe_get_mtu(dev->ha->ifp->if_mtu)); 3506 } 3507 } 3508 3509 if (attr_mask & IB_QP_TIMEOUT) { 3510 SET_FIELD(qp_params.modify_flags, \ 3511 ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 3512 3513 qp_params.ack_timeout = attr->timeout; 3514 if (attr->timeout) { 3515 u32 temp; 3516 3517 /* 12.7.34 LOCAL ACK TIMEOUT 3518 * Value representing the transport (ACK) timeout for 3519 * use by the remote, expressed as (4.096 μS*2Local ACK 3520 * Timeout) 3521 */ 3522 /* We use 1UL since the temporal value may be overflow 3523 * 32 bits 3524 */ 3525 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; 3526 qp_params.ack_timeout = temp; /* FW requires [msec] */ 3527 } 3528 else 3529 qp_params.ack_timeout = 0; /* infinite */ 3530 } 3531 if (attr_mask & IB_QP_RETRY_CNT) { 3532 SET_FIELD(qp_params.modify_flags,\ 3533 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 3534 qp_params.retry_cnt = attr->retry_cnt; 3535 } 3536 3537 if (attr_mask & IB_QP_RNR_RETRY) { 3538 SET_FIELD(qp_params.modify_flags, 3539 ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 3540 1); 3541 qp_params.rnr_retry_cnt = attr->rnr_retry; 3542 } 3543 3544 if (attr_mask & IB_QP_RQ_PSN) { 3545 SET_FIELD(qp_params.modify_flags, 3546 ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 3547 1); 3548 qp_params.rq_psn = attr->rq_psn; 3549 qp->rq_psn = attr->rq_psn; 3550 } 3551 3552 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 3553 if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { 3554 rc = -EINVAL; 3555 QL_DPRINT12(ha, 3556 "unsupported max_rd_atomic=%d, supported=%d\n", 3557 attr->max_rd_atomic, 3558 qattr->max_qp_req_rd_atomic_resc); 3559 goto err; 3560 } 3561 3562 SET_FIELD(qp_params.modify_flags, 3563 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 3564 1); 3565 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 3566 } 3567 3568 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 3569 SET_FIELD(qp_params.modify_flags, 3570 ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 3571 1); 3572 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 3573 } 3574 3575 if (attr_mask & IB_QP_SQ_PSN) { 3576 SET_FIELD(qp_params.modify_flags, 3577 ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 3578 1); 3579 qp_params.sq_psn = attr->sq_psn; 3580 qp->sq_psn = attr->sq_psn; 3581 } 3582 3583 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 3584 if (attr->max_dest_rd_atomic > 3585 qattr->max_qp_resp_rd_atomic_resc) { 3586 QL_DPRINT12(ha, 3587 "unsupported max_dest_rd_atomic=%d, " 3588 "supported=%d\n", 3589 attr->max_dest_rd_atomic, 3590 qattr->max_qp_resp_rd_atomic_resc); 3591 3592 rc = -EINVAL; 3593 goto err; 3594 } 3595 3596 SET_FIELD(qp_params.modify_flags, 3597 ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 3598 1); 3599 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 3600 } 3601 3602 if (attr_mask & IB_QP_DEST_QPN) { 3603 SET_FIELD(qp_params.modify_flags, 3604 ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 3605 1); 3606 3607 qp_params.dest_qp = attr->dest_qp_num; 3608 qp->dest_qp_num = attr->dest_qp_num; 3609 } 3610 3611 /* 3612 * Update the QP state before the actual ramrod to prevent a race with 3613 * fast path. Modifying the QP state to error will cause the device to 3614 * flush the CQEs and while polling the flushed CQEs will considered as 3615 * a potential issue if the QP isn't in error state. 3616 */ 3617 if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && 3618 (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) 3619 qp->state = ECORE_ROCE_QP_STATE_ERR; 3620 3621 if (qp->qp_type != IB_QPT_GSI) 3622 rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 3623 3624 if (attr_mask & IB_QP_STATE) { 3625 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 3626 rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); 3627 qp->state = qp_params.new_state; 3628 } 3629 3630 err: 3631 QL_DPRINT12(ha, "exit\n"); 3632 return rc; 3633 } 3634 3635 static int 3636 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) 3637 { 3638 int ib_qp_acc_flags = 0; 3639 3640 if (params->incoming_rdma_write_en) 3641 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 3642 if (params->incoming_rdma_read_en) 3643 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 3644 if (params->incoming_atomic_en) 3645 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 3646 if (true) /* FIXME -> local write ?? */ 3647 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 3648 3649 return ib_qp_acc_flags; 3650 } 3651 3652 static enum ib_mtu 3653 qlnxr_mtu_int_to_enum(u16 mtu) 3654 { 3655 enum ib_mtu ib_mtu_size; 3656 3657 switch (mtu) { 3658 case 256: 3659 ib_mtu_size = IB_MTU_256; 3660 break; 3661 3662 case 512: 3663 ib_mtu_size = IB_MTU_512; 3664 break; 3665 3666 case 1024: 3667 ib_mtu_size = IB_MTU_1024; 3668 break; 3669 3670 case 2048: 3671 ib_mtu_size = IB_MTU_2048; 3672 break; 3673 3674 case 4096: 3675 ib_mtu_size = IB_MTU_4096; 3676 break; 3677 3678 default: 3679 ib_mtu_size = IB_MTU_1024; 3680 break; 3681 } 3682 return (ib_mtu_size); 3683 } 3684 3685 int 3686 qlnxr_query_qp(struct ib_qp *ibqp, 3687 struct ib_qp_attr *qp_attr, 3688 int attr_mask, 3689 struct ib_qp_init_attr *qp_init_attr) 3690 { 3691 int rc = 0; 3692 struct ecore_rdma_query_qp_out_params params; 3693 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3694 struct qlnxr_dev *dev = qp->dev; 3695 qlnx_host_t *ha; 3696 3697 ha = dev->ha; 3698 3699 QL_DPRINT12(ha, "enter\n"); 3700 3701 memset(¶ms, 0, sizeof(params)); 3702 3703 rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); 3704 if (rc) 3705 goto err; 3706 3707 memset(qp_attr, 0, sizeof(*qp_attr)); 3708 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 3709 3710 qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); 3711 qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); 3712 3713 /* In some cases in iWARP qelr will ask for the state only */ 3714 if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { 3715 QL_DPRINT11(ha, "only state requested\n"); 3716 return 0; 3717 } 3718 3719 qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); 3720 qp_attr->path_mig_state = IB_MIG_MIGRATED; 3721 qp_attr->rq_psn = params.rq_psn; 3722 qp_attr->sq_psn = params.sq_psn; 3723 qp_attr->dest_qp_num = params.dest_qp; 3724 3725 qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); 3726 3727 QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " 3728 "path_mtu = %d qp_access_flags = 0x%x\n", 3729 qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, 3730 qp_attr->qp_access_flags); 3731 3732 qp_attr->cap.max_send_wr = qp->sq.max_wr; 3733 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 3734 qp_attr->cap.max_send_sge = qp->sq.max_sges; 3735 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 3736 qp_attr->cap.max_inline_data = qp->max_inline_data; 3737 qp_init_attr->cap = qp_attr->cap; 3738 3739 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], 3740 sizeof(qp_attr->ah_attr.grh.dgid.raw)); 3741 3742 qp_attr->ah_attr.grh.flow_label = params.flow_label; 3743 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; 3744 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; 3745 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; 3746 3747 qp_attr->ah_attr.ah_flags = IB_AH_GRH; 3748 qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ 3749 qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ 3750 qp_attr->timeout = params.timeout; 3751 qp_attr->rnr_retry = params.rnr_retry; 3752 qp_attr->retry_cnt = params.retry_cnt; 3753 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 3754 qp_attr->pkey_index = params.pkey_index; 3755 qp_attr->port_num = 1; /* FIXME -> check this */ 3756 qp_attr->ah_attr.src_path_bits = 0; 3757 qp_attr->ah_attr.static_rate = 0; 3758 qp_attr->alt_pkey_index = 0; 3759 qp_attr->alt_port_num = 0; 3760 qp_attr->alt_timeout = 0; 3761 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 3762 3763 qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; 3764 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 3765 qp_attr->max_rd_atomic = params.max_rd_atomic; 3766 qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; 3767 3768 QL_DPRINT12(ha, "max_inline_data=%d\n", 3769 qp_attr->cap.max_inline_data); 3770 3771 err: 3772 QL_DPRINT12(ha, "exit\n"); 3773 return rc; 3774 } 3775 3776 static void 3777 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3778 { 3779 qlnx_host_t *ha; 3780 3781 ha = dev->ha; 3782 3783 QL_DPRINT12(ha, "enter\n"); 3784 3785 if (qp->usq.umem) 3786 ib_umem_release(qp->usq.umem); 3787 3788 qp->usq.umem = NULL; 3789 3790 if (qp->urq.umem) 3791 ib_umem_release(qp->urq.umem); 3792 3793 qp->urq.umem = NULL; 3794 3795 QL_DPRINT12(ha, "exit\n"); 3796 return; 3797 } 3798 3799 static void 3800 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) 3801 { 3802 qlnx_host_t *ha; 3803 3804 ha = dev->ha; 3805 3806 QL_DPRINT12(ha, "enter\n"); 3807 3808 if (qlnxr_qp_has_sq(qp)) { 3809 QL_DPRINT12(ha, "freeing SQ\n"); 3810 ha->qlnxr_debug = 1; 3811 // ecore_chain_free(dev->cdev, &qp->sq.pbl); 3812 ha->qlnxr_debug = 0; 3813 kfree(qp->wqe_wr_id); 3814 } 3815 3816 if (qlnxr_qp_has_rq(qp)) { 3817 QL_DPRINT12(ha, "freeing RQ\n"); 3818 ha->qlnxr_debug = 1; 3819 // ecore_chain_free(dev->cdev, &qp->rq.pbl); 3820 ha->qlnxr_debug = 0; 3821 kfree(qp->rqe_wr_id); 3822 } 3823 3824 QL_DPRINT12(ha, "exit\n"); 3825 return; 3826 } 3827 3828 static int 3829 qlnxr_free_qp_resources(struct qlnxr_dev *dev, 3830 struct qlnxr_qp *qp, struct ib_udata *udata) 3831 { 3832 int rc = 0; 3833 qlnx_host_t *ha; 3834 struct ecore_rdma_destroy_qp_out_params d_out_params; 3835 3836 ha = dev->ha; 3837 3838 QL_DPRINT12(ha, "enter\n"); 3839 3840 #if 0 3841 if (qp->qp_type != IB_QPT_GSI) { 3842 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 3843 &d_out_params); 3844 if (rc) 3845 return rc; 3846 } 3847 3848 if (udata) 3849 qlnxr_cleanup_user(dev, qp); 3850 else 3851 qlnxr_cleanup_kernel(dev, qp); 3852 #endif 3853 3854 if (udata) 3855 qlnxr_cleanup_user(dev, qp); 3856 else 3857 qlnxr_cleanup_kernel(dev, qp); 3858 3859 if (qp->qp_type != IB_QPT_GSI) { 3860 rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, 3861 &d_out_params); 3862 if (rc) 3863 return rc; 3864 } 3865 3866 QL_DPRINT12(ha, "exit\n"); 3867 return 0; 3868 } 3869 3870 int 3871 qlnxr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 3872 { 3873 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 3874 struct qlnxr_dev *dev = qp->dev; 3875 int rc = 0; 3876 struct ib_qp_attr attr; 3877 int attr_mask = 0; 3878 qlnx_host_t *ha; 3879 3880 ha = dev->ha; 3881 3882 QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); 3883 3884 qp->destroyed = 1; 3885 3886 if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | 3887 ECORE_ROCE_QP_STATE_ERR | 3888 ECORE_ROCE_QP_STATE_INIT))) { 3889 attr.qp_state = IB_QPS_ERR; 3890 attr_mask |= IB_QP_STATE; 3891 3892 /* change the QP state to ERROR */ 3893 qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); 3894 } 3895 3896 if (qp->qp_type == IB_QPT_GSI) 3897 qlnxr_destroy_gsi_qp(dev); 3898 3899 qp->sig = ~qp->sig; 3900 3901 qlnxr_free_qp_resources(dev, qp, udata); 3902 3903 if (atomic_dec_and_test(&qp->refcnt)) { 3904 /* TODO: only for iWARP? */ 3905 qlnxr_idr_remove(dev, qp->qp_id); 3906 } 3907 3908 QL_DPRINT12(ha, "exit\n"); 3909 return rc; 3910 } 3911 3912 static inline int 3913 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) 3914 { 3915 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3916 } 3917 3918 static int 3919 sge_data_len(struct ib_sge *sg_list, int num_sge) 3920 { 3921 int i, len = 0; 3922 for (i = 0; i < num_sge; i++) 3923 len += sg_list[i].length; 3924 return len; 3925 } 3926 3927 static void 3928 swap_wqe_data64(u64 *p) 3929 { 3930 int i; 3931 3932 for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3933 *p = cpu_to_be64(cpu_to_le64(*p)); 3934 } 3935 3936 static u32 3937 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, 3938 struct qlnxr_qp *qp, 3939 u8 *wqe_size, 3940 const struct ib_send_wr *wr, 3941 const struct ib_send_wr **bad_wr, 3942 u8 *bits, 3943 u8 bit) 3944 { 3945 int i, seg_siz; 3946 char *seg_prt, *wqe; 3947 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3948 qlnx_host_t *ha; 3949 3950 ha = dev->ha; 3951 3952 QL_DPRINT12(ha, "enter[%d]\n", data_size); 3953 3954 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3955 QL_DPRINT12(ha, 3956 "Too much inline data in WR:[%d, %d]\n", 3957 data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); 3958 *bad_wr = wr; 3959 return 0; 3960 } 3961 3962 if (!data_size) 3963 return data_size; 3964 3965 /* set the bit */ 3966 *bits |= bit; 3967 3968 seg_prt = wqe = NULL; 3969 seg_siz = 0; 3970 3971 /* copy data inline */ 3972 for (i = 0; i < wr->num_sge; i++) { 3973 u32 len = wr->sg_list[i].length; 3974 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3975 3976 while (len > 0) { 3977 u32 cur; 3978 3979 /* new segment required */ 3980 if (!seg_siz) { 3981 wqe = (char *)ecore_chain_produce(&qp->sq.pbl); 3982 seg_prt = wqe; 3983 seg_siz = sizeof(struct rdma_sq_common_wqe); 3984 (*wqe_size)++; 3985 } 3986 3987 /* calculate currently allowed length */ 3988 cur = MIN(len, seg_siz); 3989 3990 memcpy(seg_prt, src, cur); 3991 3992 /* update segment variables */ 3993 seg_prt += cur; 3994 seg_siz -= cur; 3995 /* update sge variables */ 3996 src += cur; 3997 len -= cur; 3998 3999 /* swap fully-completed segments */ 4000 if (!seg_siz) 4001 swap_wqe_data64((u64 *)wqe); 4002 } 4003 } 4004 4005 /* swap last not completed segment */ 4006 if (seg_siz) 4007 swap_wqe_data64((u64 *)wqe); 4008 4009 QL_DPRINT12(ha, "exit\n"); 4010 return data_size; 4011 } 4012 4013 static u32 4014 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, 4015 u8 *wqe_size, const struct ib_send_wr *wr) 4016 { 4017 int i; 4018 u32 data_size = 0; 4019 qlnx_host_t *ha; 4020 4021 ha = dev->ha; 4022 4023 QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); 4024 4025 for (i = 0; i < wr->num_sge; i++) { 4026 struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); 4027 4028 TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); 4029 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 4030 sge->length = cpu_to_le32(wr->sg_list[i].length); 4031 data_size += wr->sg_list[i].length; 4032 } 4033 4034 if (wqe_size) 4035 *wqe_size += wr->num_sge; 4036 4037 QL_DPRINT12(ha, "exit data_size = %d\n", data_size); 4038 return data_size; 4039 } 4040 4041 static u32 4042 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, 4043 struct qlnxr_qp *qp, 4044 struct rdma_sq_rdma_wqe_1st *rwqe, 4045 struct rdma_sq_rdma_wqe_2nd *rwqe2, 4046 const struct ib_send_wr *wr, 4047 const struct ib_send_wr **bad_wr) 4048 { 4049 qlnx_host_t *ha; 4050 u32 ret = 0; 4051 4052 ha = dev->ha; 4053 4054 QL_DPRINT12(ha, "enter\n"); 4055 4056 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 4057 TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); 4058 4059 if (wr->send_flags & IB_SEND_INLINE) { 4060 u8 flags = 0; 4061 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 4062 return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, 4063 wr, bad_wr, &rwqe->flags, flags); 4064 } 4065 4066 ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); 4067 4068 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4069 4070 return (ret); 4071 } 4072 4073 static u32 4074 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, 4075 struct qlnxr_qp *qp, 4076 struct rdma_sq_send_wqe *swqe, 4077 struct rdma_sq_send_wqe *swqe2, 4078 const struct ib_send_wr *wr, 4079 const struct ib_send_wr **bad_wr) 4080 { 4081 qlnx_host_t *ha; 4082 u32 ret = 0; 4083 4084 ha = dev->ha; 4085 4086 QL_DPRINT12(ha, "enter\n"); 4087 4088 memset(swqe2, 0, sizeof(*swqe2)); 4089 4090 if (wr->send_flags & IB_SEND_INLINE) { 4091 u8 flags = 0; 4092 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 4093 return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, 4094 wr, bad_wr, &swqe->flags, flags); 4095 } 4096 4097 ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); 4098 4099 QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); 4100 4101 return (ret); 4102 } 4103 4104 static void 4105 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) 4106 { 4107 qlnx_host_t *ha; 4108 4109 ha = dev->ha; 4110 4111 int work = info->completed - info->completed_handled - 1; 4112 4113 QL_DPRINT12(ha, "enter [%d]\n", work); 4114 4115 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 4116 struct qlnxr_pbl *pbl; 4117 4118 /* Free all the page list that are possible to be freed 4119 * (all the ones that were invalidated), under the assumption 4120 * that if an FMR was completed successfully that means that 4121 * if there was an invalidate operation before it also ended 4122 */ 4123 pbl = list_first_entry(&info->inuse_pbl_list, 4124 struct qlnxr_pbl, 4125 list_entry); 4126 list_del(&pbl->list_entry); 4127 list_add_tail(&pbl->list_entry, &info->free_pbl_list); 4128 info->completed_handled++; 4129 } 4130 4131 QL_DPRINT12(ha, "exit\n"); 4132 return; 4133 } 4134 4135 static int qlnxr_prepare_reg(struct qlnxr_qp *qp, 4136 struct rdma_sq_fmr_wqe_1st *fwqe1, 4137 const struct ib_reg_wr *wr) 4138 { 4139 struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); 4140 struct rdma_sq_fmr_wqe_2nd *fwqe2; 4141 4142 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); 4143 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 4144 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 4145 fwqe1->l_key = wr->key; 4146 4147 fwqe2->access_ctrl = 0; 4148 4149 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 4150 !!(wr->access & IB_ACCESS_REMOTE_READ)); 4151 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 4152 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 4153 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 4154 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 4155 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 4156 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 4157 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 4158 fwqe2->fmr_ctrl = 0; 4159 4160 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 4161 ilog2(mr->ibmr.page_size) - 12); 4162 4163 fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ 4164 fwqe2->length_lo = mr->ibmr.length; 4165 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 4166 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 4167 4168 qp->wqe_wr_id[qp->sq.prod].mr = mr; 4169 4170 return 0; 4171 } 4172 4173 static enum ib_wc_opcode 4174 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 4175 { 4176 switch (opcode) { 4177 case IB_WR_RDMA_WRITE: 4178 case IB_WR_RDMA_WRITE_WITH_IMM: 4179 return IB_WC_RDMA_WRITE; 4180 case IB_WR_SEND_WITH_IMM: 4181 case IB_WR_SEND: 4182 case IB_WR_SEND_WITH_INV: 4183 return IB_WC_SEND; 4184 case IB_WR_RDMA_READ: 4185 return IB_WC_RDMA_READ; 4186 case IB_WR_ATOMIC_CMP_AND_SWP: 4187 return IB_WC_COMP_SWAP; 4188 case IB_WR_ATOMIC_FETCH_AND_ADD: 4189 return IB_WC_FETCH_ADD; 4190 case IB_WR_REG_MR: 4191 return IB_WC_REG_MR; 4192 case IB_WR_LOCAL_INV: 4193 return IB_WC_LOCAL_INV; 4194 default: 4195 return IB_WC_SEND; 4196 } 4197 } 4198 static inline bool 4199 qlnxr_can_post_send(struct qlnxr_qp *qp, const struct ib_send_wr *wr) 4200 { 4201 int wq_is_full, err_wr, pbl_is_full; 4202 struct qlnxr_dev *dev = qp->dev; 4203 qlnx_host_t *ha; 4204 4205 ha = dev->ha; 4206 4207 QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); 4208 4209 /* prevent SQ overflow and/or processing of a bad WR */ 4210 err_wr = wr->num_sge > qp->sq.max_sges; 4211 wq_is_full = qlnxr_wq_is_full(&qp->sq); 4212 pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < 4213 QLNXR_MAX_SQE_ELEMENTS_PER_SQE; 4214 if (wq_is_full || err_wr || pbl_is_full) { 4215 if (wq_is_full && 4216 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { 4217 qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; 4218 4219 QL_DPRINT12(ha, 4220 "error: WQ is full. Post send on QP failed" 4221 " (this error appears only once) " 4222 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4223 qp, wr, qp->err_bitmap); 4224 } 4225 4226 if (err_wr && 4227 !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { 4228 qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; 4229 4230 QL_DPRINT12(ha, 4231 "error: WQ is bad. Post send on QP failed" 4232 " (this error appears only once) " 4233 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4234 qp, wr, qp->err_bitmap); 4235 } 4236 4237 if (pbl_is_full && 4238 !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { 4239 qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; 4240 4241 QL_DPRINT12(ha, 4242 "error: WQ PBL is full. Post send on QP failed" 4243 " (this error appears only once) " 4244 "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", 4245 qp, wr, qp->err_bitmap); 4246 } 4247 return false; 4248 } 4249 QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); 4250 return true; 4251 } 4252 4253 int 4254 qlnxr_post_send(struct ib_qp *ibqp, 4255 const struct ib_send_wr *wr, 4256 const struct ib_send_wr **bad_wr) 4257 { 4258 struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); 4259 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4260 unsigned long flags; 4261 int status = 0, rc = 0; 4262 bool comp; 4263 qlnx_host_t *ha; 4264 uint32_t reg_addr; 4265 4266 *bad_wr = NULL; 4267 ha = dev->ha; 4268 4269 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4270 ibqp, wr, bad_wr); 4271 4272 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 4273 return -EINVAL; 4274 4275 if (qp->qp_type == IB_QPT_GSI) 4276 return qlnxr_gsi_post_send(ibqp, wr, bad_wr); 4277 4278 spin_lock_irqsave(&qp->q_lock, flags); 4279 4280 if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && 4281 (qp->state != ECORE_ROCE_QP_STATE_ERR) && 4282 (qp->state != ECORE_ROCE_QP_STATE_SQD)) { 4283 spin_unlock_irqrestore(&qp->q_lock, flags); 4284 *bad_wr = wr; 4285 QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", 4286 qp->icid, qp->state); 4287 return -EINVAL; 4288 } 4289 4290 if (!wr) { 4291 QL_DPRINT11(ha, "Got an empty post send???\n"); 4292 } 4293 4294 while (wr) { 4295 struct rdma_sq_common_wqe *wqe; 4296 struct rdma_sq_send_wqe *swqe; 4297 struct rdma_sq_send_wqe *swqe2; 4298 struct rdma_sq_rdma_wqe_1st *rwqe; 4299 struct rdma_sq_rdma_wqe_2nd *rwqe2; 4300 struct rdma_sq_local_inv_wqe *iwqe; 4301 struct rdma_sq_atomic_wqe *awqe1; 4302 struct rdma_sq_atomic_wqe *awqe2; 4303 struct rdma_sq_atomic_wqe *awqe3; 4304 struct rdma_sq_fmr_wqe_1st *fwqe1; 4305 4306 if (!qlnxr_can_post_send(qp, wr)) { 4307 status = -ENOMEM; 4308 *bad_wr = wr; 4309 break; 4310 } 4311 4312 wqe = ecore_chain_produce(&qp->sq.pbl); 4313 4314 qp->wqe_wr_id[qp->sq.prod].signaled = 4315 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 4316 4317 /* common fields */ 4318 wqe->flags = 0; 4319 wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << 4320 RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); 4321 4322 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ 4323 !!(wr->send_flags & IB_SEND_SOLICITED)); 4324 4325 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || 4326 (qp->signaled); 4327 4328 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 4329 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ 4330 !!(wr->send_flags & IB_SEND_FENCE)); 4331 4332 wqe->prev_wqe_size = qp->prev_wqe_size; 4333 4334 qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); 4335 4336 switch (wr->opcode) { 4337 case IB_WR_SEND_WITH_IMM: 4338 4339 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 4340 swqe = (struct rdma_sq_send_wqe *)wqe; 4341 swqe->wqe_size = 2; 4342 swqe2 = (struct rdma_sq_send_wqe *) 4343 ecore_chain_produce(&qp->sq.pbl); 4344 swqe->inv_key_or_imm_data = 4345 cpu_to_le32(wr->ex.imm_data); 4346 swqe->length = cpu_to_le32( 4347 qlnxr_prepare_sq_send_data(dev, 4348 qp, swqe, swqe2, wr, 4349 bad_wr)); 4350 4351 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4352 qp->prev_wqe_size = swqe->wqe_size; 4353 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4354 4355 QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", 4356 swqe->length, wr->ex.imm_data); 4357 4358 break; 4359 4360 case IB_WR_SEND: 4361 4362 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 4363 swqe = (struct rdma_sq_send_wqe *)wqe; 4364 4365 swqe->wqe_size = 2; 4366 swqe2 = (struct rdma_sq_send_wqe *) 4367 ecore_chain_produce(&qp->sq.pbl); 4368 swqe->length = cpu_to_le32( 4369 qlnxr_prepare_sq_send_data(dev, 4370 qp, swqe, swqe2, wr, 4371 bad_wr)); 4372 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4373 qp->prev_wqe_size = swqe->wqe_size; 4374 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4375 4376 QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", 4377 swqe->length); 4378 4379 break; 4380 4381 case IB_WR_SEND_WITH_INV: 4382 4383 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 4384 swqe = (struct rdma_sq_send_wqe *)wqe; 4385 swqe2 = (struct rdma_sq_send_wqe *) 4386 ecore_chain_produce(&qp->sq.pbl); 4387 swqe->wqe_size = 2; 4388 swqe->inv_key_or_imm_data = 4389 cpu_to_le32(wr->ex.invalidate_rkey); 4390 swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, 4391 qp, swqe, swqe2, wr, bad_wr)); 4392 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 4393 qp->prev_wqe_size = swqe->wqe_size; 4394 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 4395 4396 QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", 4397 swqe->length); 4398 break; 4399 4400 case IB_WR_RDMA_WRITE_WITH_IMM: 4401 4402 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 4403 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4404 4405 rwqe->wqe_size = 2; 4406 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 4407 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4408 ecore_chain_produce(&qp->sq.pbl); 4409 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4410 qp, rwqe, rwqe2, wr, bad_wr)); 4411 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4412 qp->prev_wqe_size = rwqe->wqe_size; 4413 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4414 4415 QL_DPRINT12(ha, 4416 "RDMA WRITE w/ IMM length = %d imm data=%x\n", 4417 rwqe->length, rwqe->imm_data); 4418 4419 break; 4420 4421 case IB_WR_RDMA_WRITE: 4422 4423 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 4424 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4425 4426 rwqe->wqe_size = 2; 4427 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4428 ecore_chain_produce(&qp->sq.pbl); 4429 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4430 qp, rwqe, rwqe2, wr, bad_wr)); 4431 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4432 qp->prev_wqe_size = rwqe->wqe_size; 4433 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4434 4435 QL_DPRINT12(ha, 4436 "RDMA WRITE w/o IMM length = %d\n", 4437 rwqe->length); 4438 4439 break; 4440 4441 case IB_WR_RDMA_READ_WITH_INV: 4442 4443 QL_DPRINT12(ha, 4444 "RDMA READ WITH INVALIDATE not supported\n"); 4445 4446 *bad_wr = wr; 4447 rc = -EINVAL; 4448 4449 break; 4450 4451 case IB_WR_RDMA_READ: 4452 4453 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 4454 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 4455 4456 rwqe->wqe_size = 2; 4457 rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) 4458 ecore_chain_produce(&qp->sq.pbl); 4459 rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, 4460 qp, rwqe, rwqe2, wr, bad_wr)); 4461 4462 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 4463 qp->prev_wqe_size = rwqe->wqe_size; 4464 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 4465 4466 QL_DPRINT12(ha, "RDMA READ length = %d\n", 4467 rwqe->length); 4468 4469 break; 4470 4471 case IB_WR_ATOMIC_CMP_AND_SWP: 4472 case IB_WR_ATOMIC_FETCH_AND_ADD: 4473 4474 QL_DPRINT12(ha, 4475 "ATOMIC operation = %s\n", 4476 ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? 4477 "IB_WR_ATOMIC_CMP_AND_SWP" : 4478 "IB_WR_ATOMIC_FETCH_AND_ADD")); 4479 4480 awqe1 = (struct rdma_sq_atomic_wqe *)wqe; 4481 awqe1->prev_wqe_size = 4; 4482 4483 awqe2 = (struct rdma_sq_atomic_wqe *) 4484 ecore_chain_produce(&qp->sq.pbl); 4485 4486 TYPEPTR_ADDR_SET(awqe2, remote_va, \ 4487 atomic_wr(wr)->remote_addr); 4488 4489 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 4490 4491 awqe3 = (struct rdma_sq_atomic_wqe *) 4492 ecore_chain_produce(&qp->sq.pbl); 4493 4494 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 4495 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 4496 TYPEPTR_ADDR_SET(awqe3, swap_data, 4497 atomic_wr(wr)->compare_add); 4498 } else { 4499 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 4500 TYPEPTR_ADDR_SET(awqe3, swap_data, 4501 atomic_wr(wr)->swap); 4502 TYPEPTR_ADDR_SET(awqe3, cmp_data, 4503 atomic_wr(wr)->compare_add); 4504 } 4505 4506 qlnxr_prepare_sq_sges(dev, qp, NULL, wr); 4507 4508 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; 4509 qp->prev_wqe_size = awqe1->prev_wqe_size; 4510 4511 break; 4512 4513 case IB_WR_LOCAL_INV: 4514 4515 QL_DPRINT12(ha, 4516 "INVALIDATE length (IB_WR_LOCAL_INV)\n"); 4517 4518 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 4519 iwqe->prev_wqe_size = 1; 4520 4521 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 4522 iwqe->inv_l_key = wr->ex.invalidate_rkey; 4523 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; 4524 qp->prev_wqe_size = iwqe->prev_wqe_size; 4525 4526 break; 4527 4528 case IB_WR_REG_MR: 4529 4530 QL_DPRINT12(ha, "IB_WR_REG_MR\n"); 4531 4532 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 4533 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 4534 fwqe1->wqe_size = 2; 4535 4536 rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); 4537 if (rc) { 4538 QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); 4539 *bad_wr = wr; 4540 break; 4541 } 4542 4543 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 4544 qp->prev_wqe_size = fwqe1->wqe_size; 4545 4546 break; 4547 4548 default: 4549 4550 QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); 4551 4552 rc = -EINVAL; 4553 *bad_wr = wr; 4554 break; 4555 } 4556 4557 if (*bad_wr) { 4558 /* 4559 * restore prod to its position before this WR was processed 4560 */ 4561 ecore_chain_set_prod(&qp->sq.pbl, 4562 le16_to_cpu(qp->sq.db_data.data.value), 4563 wqe); 4564 /* restore prev_wqe_size */ 4565 qp->prev_wqe_size = wqe->prev_wqe_size; 4566 status = rc; 4567 4568 QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); 4569 break; /* out of the loop */ 4570 } 4571 4572 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 4573 4574 qlnxr_inc_sw_prod(&qp->sq); 4575 4576 qp->sq.db_data.data.value++; 4577 4578 wr = wr->next; 4579 } 4580 4581 /* Trigger doorbell 4582 * If there was a failure in the first WR then it will be triggered in 4583 * vane. However this is not harmful (as long as the producer value is 4584 * unchanged). For performance reasons we avoid checking for this 4585 * redundant doorbell. 4586 */ 4587 wmb(); 4588 //writel(qp->sq.db_data.raw, qp->sq.db); 4589 4590 reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); 4591 bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); 4592 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4593 4594 mmiowb(); 4595 4596 spin_unlock_irqrestore(&qp->q_lock, flags); 4597 4598 QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", 4599 ibqp, wr, bad_wr); 4600 4601 return status; 4602 } 4603 4604 static u32 4605 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) 4606 { 4607 u32 used; 4608 4609 /* Calculate number of elements used based on producer 4610 * count and consumer count and subtract it from max 4611 * work request supported so that we get elements left. 4612 */ 4613 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 4614 4615 return hw_srq->max_wr - used; 4616 } 4617 4618 int 4619 qlnxr_post_recv(struct ib_qp *ibqp, 4620 const struct ib_recv_wr *wr, 4621 const struct ib_recv_wr **bad_wr) 4622 { 4623 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 4624 struct qlnxr_dev *dev = qp->dev; 4625 unsigned long flags; 4626 int status = 0; 4627 qlnx_host_t *ha; 4628 uint32_t reg_addr; 4629 4630 ha = dev->ha; 4631 4632 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 4633 return -EINVAL; 4634 4635 QL_DPRINT12(ha, "enter\n"); 4636 4637 if (qp->qp_type == IB_QPT_GSI) { 4638 QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); 4639 return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); 4640 } 4641 4642 if (qp->srq) { 4643 QL_DPRINT11(ha, "qp->srq [%p]" 4644 " QP is associated with SRQ, cannot post RQ buffers\n", 4645 qp->srq); 4646 return -EINVAL; 4647 } 4648 4649 spin_lock_irqsave(&qp->q_lock, flags); 4650 4651 if (qp->state == ECORE_ROCE_QP_STATE_RESET) { 4652 spin_unlock_irqrestore(&qp->q_lock, flags); 4653 *bad_wr = wr; 4654 4655 QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); 4656 4657 return -EINVAL; 4658 } 4659 4660 while (wr) { 4661 int i; 4662 4663 if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < 4664 QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || 4665 (wr->num_sge > qp->rq.max_sges)) { 4666 status = -ENOMEM; 4667 *bad_wr = wr; 4668 break; 4669 } 4670 for (i = 0; i < wr->num_sge; i++) { 4671 u32 flags = 0; 4672 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 4673 4674 /* first one must include the number of SGE in the list */ 4675 if (!i) 4676 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); 4677 4678 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); 4679 4680 RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ 4681 wr->sg_list[i].length, flags); 4682 } 4683 /* Special case of no sges. FW requires between 1-4 sges... 4684 * in this case we need to post 1 sge with length zero. this is 4685 * because rdma write with immediate consumes an RQ. */ 4686 if (!wr->num_sge) { 4687 u32 flags = 0; 4688 struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); 4689 4690 /* first one must include the number of SGE in the list */ 4691 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); 4692 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 4693 4694 //RQ_SGE_SET(rqe, 0, 0, flags); 4695 rqe->addr.hi = 0; 4696 rqe->addr.lo = 0; 4697 4698 rqe->length = 0; 4699 rqe->flags = cpu_to_le32(flags); 4700 4701 i = 1; 4702 } 4703 4704 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 4705 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 4706 4707 qlnxr_inc_sw_prod(&qp->rq); 4708 4709 wmb(); 4710 4711 qp->rq.db_data.data.value++; 4712 4713 // writel(qp->rq.db_data.raw, qp->rq.db); 4714 mmiowb(); 4715 // if (QLNX_IS_IWARP(dev)) { 4716 // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 4717 // mmiowb(); /* for second doorbell */ 4718 // } 4719 4720 reg_addr = (uint32_t)((uint8_t *)qp->rq.db - 4721 (uint8_t *)ha->cdev.doorbells); 4722 4723 bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); 4724 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 4725 4726 if (QLNX_IS_IWARP(dev)) { 4727 reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - 4728 (uint8_t *)ha->cdev.doorbells); 4729 bus_write_4(ha->pci_dbells, reg_addr, \ 4730 qp->rq.iwarp_db2_data.raw); 4731 bus_barrier(ha->pci_dbells, 0, 0, \ 4732 BUS_SPACE_BARRIER_READ); 4733 } 4734 4735 wr = wr->next; 4736 } 4737 4738 spin_unlock_irqrestore(&qp->q_lock, flags); 4739 4740 QL_DPRINT12(ha, "exit status = 0x%x\n", status); 4741 4742 return status; 4743 } 4744 4745 /* In fmr we need to increase the number of fmr completed counter for the fmr 4746 * algorithm determining whether we can free a pbl or not. 4747 * we need to perform this whether the work request was signaled or not. for 4748 * this purpose we call this function from the condition that checks if a wr 4749 * should be skipped, to make sure we don't miss it ( possibly this fmr 4750 * operation was not signalted) 4751 */ 4752 static inline void 4753 qlnxr_chk_if_fmr(struct qlnxr_qp *qp) 4754 { 4755 4756 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4757 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4758 } 4759 4760 static int 4761 process_req(struct qlnxr_dev *dev, 4762 struct qlnxr_qp *qp, 4763 struct qlnxr_cq *cq, 4764 int num_entries, 4765 struct ib_wc *wc, 4766 u16 hw_cons, 4767 enum ib_wc_status status, 4768 int force) 4769 { 4770 u16 cnt = 0; 4771 qlnx_host_t *ha = dev->ha; 4772 4773 QL_DPRINT12(ha, "enter\n"); 4774 4775 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4776 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4777 qlnxr_chk_if_fmr(qp); 4778 /* skip WC */ 4779 goto next_cqe; 4780 } 4781 4782 /* fill WC */ 4783 wc->status = status; 4784 wc->vendor_err = 0; 4785 wc->wc_flags = 0; 4786 wc->src_qp = qp->id; 4787 wc->qp = &qp->ibqp; 4788 4789 // common section 4790 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4791 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4792 4793 switch (wc->opcode) { 4794 case IB_WC_RDMA_WRITE: 4795 4796 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4797 4798 QL_DPRINT12(ha, 4799 "opcode = IB_WC_RDMA_WRITE bytes = %d\n", 4800 qp->wqe_wr_id[qp->sq.cons].bytes_len); 4801 break; 4802 4803 case IB_WC_COMP_SWAP: 4804 case IB_WC_FETCH_ADD: 4805 wc->byte_len = 8; 4806 break; 4807 4808 case IB_WC_REG_MR: 4809 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4810 break; 4811 4812 case IB_WC_RDMA_READ: 4813 case IB_WC_SEND: 4814 4815 QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); 4816 break; 4817 default: 4818 ;//DP_ERR("TBD ERROR"); 4819 } 4820 4821 num_entries--; 4822 wc++; 4823 cnt++; 4824 next_cqe: 4825 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4826 ecore_chain_consume(&qp->sq.pbl); 4827 qlnxr_inc_sw_cons(&qp->sq); 4828 } 4829 4830 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 4831 return cnt; 4832 } 4833 4834 static int 4835 qlnxr_poll_cq_req(struct qlnxr_dev *dev, 4836 struct qlnxr_qp *qp, 4837 struct qlnxr_cq *cq, 4838 int num_entries, 4839 struct ib_wc *wc, 4840 struct rdma_cqe_requester *req) 4841 { 4842 int cnt = 0; 4843 qlnx_host_t *ha = dev->ha; 4844 4845 QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); 4846 4847 switch (req->status) { 4848 case RDMA_CQE_REQ_STS_OK: 4849 4850 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4851 IB_WC_SUCCESS, 0); 4852 break; 4853 4854 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4855 4856 if (qp->state != ECORE_ROCE_QP_STATE_ERR) 4857 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4858 IB_WC_WR_FLUSH_ERR, 1); 4859 break; 4860 4861 default: /* other errors case */ 4862 4863 /* process all WQE before the cosumer */ 4864 qp->state = ECORE_ROCE_QP_STATE_ERR; 4865 cnt = process_req(dev, qp, cq, num_entries, wc, 4866 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4867 wc += cnt; 4868 /* if we have extra WC fill it with actual error info */ 4869 4870 if (cnt < num_entries) { 4871 enum ib_wc_status wc_status; 4872 4873 switch (req->status) { 4874 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4875 wc_status = IB_WC_BAD_RESP_ERR; 4876 break; 4877 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4878 wc_status = IB_WC_LOC_LEN_ERR; 4879 break; 4880 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4881 wc_status = IB_WC_LOC_QP_OP_ERR; 4882 break; 4883 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4884 wc_status = IB_WC_LOC_PROT_ERR; 4885 break; 4886 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4887 wc_status = IB_WC_MW_BIND_ERR; 4888 break; 4889 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4890 wc_status = IB_WC_REM_INV_REQ_ERR; 4891 break; 4892 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4893 wc_status = IB_WC_REM_ACCESS_ERR; 4894 break; 4895 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4896 wc_status = IB_WC_REM_OP_ERR; 4897 break; 4898 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4899 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4900 break; 4901 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4902 wc_status = IB_WC_RETRY_EXC_ERR; 4903 break; 4904 default: 4905 wc_status = IB_WC_GENERAL_ERR; 4906 } 4907 4908 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4909 wc_status, 1 /* force use of WC */); 4910 } 4911 } 4912 4913 QL_DPRINT12(ha, "exit cnt = %d\n", cnt); 4914 return cnt; 4915 } 4916 4917 static void 4918 __process_resp_one(struct qlnxr_dev *dev, 4919 struct qlnxr_qp *qp, 4920 struct qlnxr_cq *cq, 4921 struct ib_wc *wc, 4922 struct rdma_cqe_responder *resp, 4923 u64 wr_id) 4924 { 4925 enum ib_wc_status wc_status = IB_WC_SUCCESS; 4926 qlnx_host_t *ha = dev->ha; 4927 4928 QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", 4929 qp, resp->status); 4930 4931 wc->opcode = IB_WC_RECV; 4932 wc->wc_flags = 0; 4933 4934 switch (resp->status) { 4935 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4936 wc_status = IB_WC_LOC_ACCESS_ERR; 4937 break; 4938 4939 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4940 wc_status = IB_WC_LOC_LEN_ERR; 4941 break; 4942 4943 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4944 wc_status = IB_WC_LOC_QP_OP_ERR; 4945 break; 4946 4947 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4948 wc_status = IB_WC_LOC_PROT_ERR; 4949 break; 4950 4951 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4952 wc_status = IB_WC_MW_BIND_ERR; 4953 break; 4954 4955 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4956 wc_status = IB_WC_REM_INV_RD_REQ_ERR; 4957 break; 4958 4959 case RDMA_CQE_RESP_STS_OK: 4960 if (resp->flags & QLNXR_RESP_IMM) { 4961 wc->ex.imm_data = 4962 le32_to_cpu(resp->imm_data_or_inv_r_Key); 4963 wc->wc_flags |= IB_WC_WITH_IMM; 4964 4965 if (resp->flags & QLNXR_RESP_RDMA) 4966 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4967 4968 if (resp->flags & QLNXR_RESP_INV) { 4969 QL_DPRINT11(ha, 4970 "Invalid flags QLNXR_RESP_INV [0x%x]" 4971 "qp = %p qp->id = 0x%x cq = %p" 4972 " cq->icid = 0x%x\n", 4973 resp->flags, qp, qp->id, cq, cq->icid ); 4974 } 4975 } else if (resp->flags & QLNXR_RESP_INV) { 4976 wc->ex.imm_data = 4977 le32_to_cpu(resp->imm_data_or_inv_r_Key); 4978 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4979 4980 if (resp->flags & QLNXR_RESP_RDMA) { 4981 QL_DPRINT11(ha, 4982 "Invalid flags QLNXR_RESP_RDMA [0x%x]" 4983 "qp = %p qp->id = 0x%x cq = %p" 4984 " cq->icid = 0x%x\n", 4985 resp->flags, qp, qp->id, cq, cq->icid ); 4986 } 4987 } else if (resp->flags & QLNXR_RESP_RDMA) { 4988 QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" 4989 "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", 4990 resp->flags, qp, qp->id, cq, cq->icid ); 4991 } 4992 break; 4993 4994 default: 4995 wc_status = IB_WC_GENERAL_ERR; 4996 } 4997 4998 /* fill WC */ 4999 wc->status = wc_status; 5000 wc->vendor_err = 0; 5001 wc->src_qp = qp->id; 5002 wc->qp = &qp->ibqp; 5003 wc->wr_id = wr_id; 5004 5005 QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); 5006 5007 return; 5008 } 5009 5010 static int 5011 process_resp_one_srq(struct qlnxr_dev *dev, 5012 struct qlnxr_qp *qp, 5013 struct qlnxr_cq *cq, 5014 struct ib_wc *wc, 5015 struct rdma_cqe_responder *resp) 5016 { 5017 struct qlnxr_srq *srq = qp->srq; 5018 u64 wr_id; 5019 qlnx_host_t *ha = dev->ha; 5020 5021 QL_DPRINT12(ha, "enter\n"); 5022 5023 wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); 5024 5025 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5026 wc->status = IB_WC_WR_FLUSH_ERR; 5027 wc->vendor_err = 0; 5028 wc->wr_id = wr_id; 5029 wc->byte_len = 0; 5030 wc->src_qp = qp->id; 5031 wc->qp = &qp->ibqp; 5032 wc->wr_id = wr_id; 5033 } else { 5034 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5035 } 5036 5037 /* PBL is maintained in case of WR granularity. 5038 * So increment WR consumer after consuming WR 5039 */ 5040 srq->hw_srq.wr_cons_cnt++; 5041 5042 QL_DPRINT12(ha, "exit\n"); 5043 return 1; 5044 } 5045 5046 static int 5047 process_resp_one(struct qlnxr_dev *dev, 5048 struct qlnxr_qp *qp, 5049 struct qlnxr_cq *cq, 5050 struct ib_wc *wc, 5051 struct rdma_cqe_responder *resp) 5052 { 5053 qlnx_host_t *ha = dev->ha; 5054 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5055 5056 QL_DPRINT12(ha, "enter\n"); 5057 5058 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 5059 5060 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5061 ecore_chain_consume(&qp->rq.pbl); 5062 qlnxr_inc_sw_cons(&qp->rq); 5063 5064 QL_DPRINT12(ha, "exit\n"); 5065 return 1; 5066 } 5067 5068 static int 5069 process_resp_flush(struct qlnxr_qp *qp, 5070 int num_entries, 5071 struct ib_wc *wc, 5072 u16 hw_cons) 5073 { 5074 u16 cnt = 0; 5075 qlnx_host_t *ha = qp->dev->ha; 5076 5077 QL_DPRINT12(ha, "enter\n"); 5078 5079 while (num_entries && qp->rq.wqe_cons != hw_cons) { 5080 /* fill WC */ 5081 wc->status = IB_WC_WR_FLUSH_ERR; 5082 wc->vendor_err = 0; 5083 wc->wc_flags = 0; 5084 wc->src_qp = qp->id; 5085 wc->byte_len = 0; 5086 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 5087 wc->qp = &qp->ibqp; 5088 num_entries--; 5089 wc++; 5090 cnt++; 5091 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 5092 ecore_chain_consume(&qp->rq.pbl); 5093 qlnxr_inc_sw_cons(&qp->rq); 5094 } 5095 5096 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5097 return cnt; 5098 } 5099 5100 static void 5101 try_consume_resp_cqe(struct qlnxr_cq *cq, 5102 struct qlnxr_qp *qp, 5103 struct rdma_cqe_responder *resp, 5104 int *update) 5105 { 5106 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { 5107 consume_cqe(cq); 5108 *update |= 1; 5109 } 5110 } 5111 5112 static int 5113 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, 5114 struct qlnxr_qp *qp, 5115 struct qlnxr_cq *cq, 5116 int num_entries, 5117 struct ib_wc *wc, 5118 struct rdma_cqe_responder *resp, 5119 int *update) 5120 { 5121 int cnt; 5122 qlnx_host_t *ha = dev->ha; 5123 5124 QL_DPRINT12(ha, "enter\n"); 5125 5126 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 5127 consume_cqe(cq); 5128 *update |= 1; 5129 5130 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5131 return cnt; 5132 } 5133 5134 static int 5135 qlnxr_poll_cq_resp(struct qlnxr_dev *dev, 5136 struct qlnxr_qp *qp, 5137 struct qlnxr_cq *cq, 5138 int num_entries, 5139 struct ib_wc *wc, 5140 struct rdma_cqe_responder *resp, 5141 int *update) 5142 { 5143 int cnt; 5144 qlnx_host_t *ha = dev->ha; 5145 5146 QL_DPRINT12(ha, "enter\n"); 5147 5148 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 5149 cnt = process_resp_flush(qp, num_entries, wc, 5150 resp->rq_cons); 5151 try_consume_resp_cqe(cq, qp, resp, update); 5152 } else { 5153 cnt = process_resp_one(dev, qp, cq, wc, resp); 5154 consume_cqe(cq); 5155 *update |= 1; 5156 } 5157 5158 QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); 5159 return cnt; 5160 } 5161 5162 static void 5163 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, 5164 struct rdma_cqe_requester *req, int *update) 5165 { 5166 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 5167 consume_cqe(cq); 5168 *update |= 1; 5169 } 5170 } 5171 5172 static void 5173 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) 5174 { 5175 uint64_t reg_addr; 5176 qlnx_host_t *ha = dev->ha; 5177 5178 QL_DPRINT12(ha, "enter\n"); 5179 5180 wmb(); 5181 cq->db.data.agg_flags = flags; 5182 cq->db.data.value = cpu_to_le32(cons); 5183 5184 reg_addr = (uint64_t)((uint8_t *)cq->db_addr - 5185 (uint8_t *)(ha->cdev.doorbells)); 5186 5187 bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); 5188 bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); 5189 5190 QL_DPRINT12(ha, "exit\n"); 5191 return; 5192 5193 //#ifdef __LP64__ 5194 // writeq(cq->db.raw, cq->db_addr); 5195 //#else 5196 /* Note that since the FW allows 64 bit write only, in 32bit systems 5197 * the value of db_addr must be low enough. This is currently not 5198 * enforced. 5199 */ 5200 // writel(cq->db.raw & 0xffffffff, cq->db_addr); 5201 // mmiowb(); 5202 //#endif 5203 } 5204 5205 static int 5206 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) 5207 { 5208 struct rdma_cqe_requester *resp_cqe = &cqe->req; 5209 return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == 5210 cq->pbl_toggle; 5211 } 5212 5213 int 5214 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 5215 { 5216 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5217 struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); 5218 int done = 0; 5219 union rdma_cqe *cqe = cq->latest_cqe; 5220 int update = 0; 5221 u32 old_cons, new_cons; 5222 unsigned long flags; 5223 qlnx_host_t *ha = dev->ha; 5224 5225 QL_DPRINT12(ha, "enter\n"); 5226 5227 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5228 return -EINVAL; 5229 5230 if (cq->destroyed) { 5231 QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", 5232 cq, cq->icid); 5233 return 0; 5234 } 5235 5236 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) 5237 return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); 5238 5239 spin_lock_irqsave(&cq->cq_lock, flags); 5240 5241 old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5242 5243 while (num_entries && is_valid_cqe(cq, cqe)) { 5244 int cnt = 0; 5245 struct qlnxr_qp *qp; 5246 struct rdma_cqe_requester *resp_cqe; 5247 enum rdma_cqe_type cqe_type; 5248 5249 /* prevent speculative reads of any field of CQE */ 5250 rmb(); 5251 5252 resp_cqe = &cqe->req; 5253 qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, 5254 resp_cqe->qp_handle.lo); 5255 5256 if (!qp) { 5257 QL_DPRINT11(ha, "qp = NULL\n"); 5258 break; 5259 } 5260 5261 wc->qp = &qp->ibqp; 5262 5263 cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 5264 5265 switch (cqe_type) { 5266 case RDMA_CQE_TYPE_REQUESTER: 5267 cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, 5268 wc, &cqe->req); 5269 try_consume_req_cqe(cq, qp, &cqe->req, &update); 5270 break; 5271 case RDMA_CQE_TYPE_RESPONDER_RQ: 5272 cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, 5273 wc, &cqe->resp, &update); 5274 break; 5275 case RDMA_CQE_TYPE_RESPONDER_SRQ: 5276 cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, 5277 wc, &cqe->resp, &update); 5278 break; 5279 case RDMA_CQE_TYPE_INVALID: 5280 default: 5281 QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); 5282 break; 5283 } 5284 num_entries -= cnt; 5285 wc += cnt; 5286 done += cnt; 5287 5288 cqe = cq->latest_cqe; 5289 } 5290 new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); 5291 5292 cq->cq_cons += new_cons - old_cons; 5293 5294 if (update) { 5295 /* doorbell notifies abount latest VALID entry, 5296 * but chain already point to the next INVALID one 5297 */ 5298 doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); 5299 QL_DPRINT12(ha, "cq = %p cons = 0x%x " 5300 "arm_flags = 0x%x db.icid = 0x%x\n", cq, 5301 (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); 5302 } 5303 5304 spin_unlock_irqrestore(&cq->cq_lock, flags); 5305 5306 QL_DPRINT12(ha, "exit\n"); 5307 5308 return done; 5309 } 5310 5311 int 5312 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 5313 { 5314 struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); 5315 unsigned long sflags; 5316 struct qlnxr_dev *dev; 5317 qlnx_host_t *ha; 5318 5319 dev = get_qlnxr_dev((ibcq->device)); 5320 ha = dev->ha; 5321 5322 QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " 5323 "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, 5324 flags, cq, cq->cq_cons, cq->cq_type); 5325 5326 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5327 return -EINVAL; 5328 5329 if (cq->destroyed) { 5330 QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", 5331 cq, cq->icid); 5332 return -EINVAL; 5333 } 5334 5335 if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { 5336 return 0; 5337 } 5338 5339 spin_lock_irqsave(&cq->cq_lock, sflags); 5340 5341 cq->arm_flags = 0; 5342 5343 if (flags & IB_CQ_SOLICITED) { 5344 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 5345 } 5346 if (flags & IB_CQ_NEXT_COMP) { 5347 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 5348 } 5349 5350 doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); 5351 5352 spin_unlock_irqrestore(&cq->cq_lock, sflags); 5353 5354 QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); 5355 return 0; 5356 } 5357 5358 static struct qlnxr_mr * 5359 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) 5360 { 5361 struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); 5362 struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); 5363 struct qlnxr_mr *mr; 5364 int rc = -ENOMEM; 5365 qlnx_host_t *ha; 5366 5367 ha = dev->ha; 5368 5369 QL_DPRINT12(ha, "enter ibpd = %p pd = %p " 5370 " pd_id = %d max_page_list_len = %d\n", 5371 ibpd, pd, pd->pd_id, max_page_list_len); 5372 5373 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5374 if (!mr) { 5375 QL_DPRINT11(ha, "kzalloc(mr) failed\n"); 5376 return ERR_PTR(rc); 5377 } 5378 5379 mr->dev = dev; 5380 mr->type = QLNXR_MR_FRMR; 5381 5382 rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 5383 1 /* allow dual layer pbl */); 5384 if (rc) { 5385 QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); 5386 goto err0; 5387 } 5388 5389 rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 5390 if (rc) { 5391 QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); 5392 goto err0; 5393 } 5394 5395 /* index only, 18 bit long, lkey = itid << 8 | key */ 5396 mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; 5397 mr->hw_mr.key = 0; 5398 mr->hw_mr.pd = pd->pd_id; 5399 mr->hw_mr.local_read = 1; 5400 mr->hw_mr.local_write = 0; 5401 mr->hw_mr.remote_read = 0; 5402 mr->hw_mr.remote_write = 0; 5403 mr->hw_mr.remote_atomic = 0; 5404 mr->hw_mr.mw_bind = false; /* TBD MW BIND */ 5405 mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ 5406 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 5407 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 5408 mr->hw_mr.fbo = 0; 5409 mr->hw_mr.length = 0; 5410 mr->hw_mr.vaddr = 0; 5411 mr->hw_mr.zbva = false; /* TBD figure when this should be true */ 5412 mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ 5413 mr->hw_mr.dma_mr = false; 5414 5415 rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 5416 if (rc) { 5417 QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); 5418 goto err1; 5419 } 5420 5421 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 5422 mr->ibmr.rkey = mr->ibmr.lkey; 5423 5424 QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", 5425 mr, mr->ibmr.lkey); 5426 5427 return mr; 5428 5429 err1: 5430 ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 5431 err0: 5432 kfree(mr); 5433 5434 QL_DPRINT12(ha, "exit\n"); 5435 5436 return ERR_PTR(rc); 5437 } 5438 5439 struct ib_mr * 5440 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 5441 u32 max_num_sg, struct ib_udata *udata) 5442 { 5443 struct qlnxr_dev *dev; 5444 struct qlnxr_mr *mr; 5445 qlnx_host_t *ha; 5446 5447 dev = get_qlnxr_dev(ibpd->device); 5448 ha = dev->ha; 5449 5450 QL_DPRINT12(ha, "enter\n"); 5451 5452 if (mr_type != IB_MR_TYPE_MEM_REG) 5453 return ERR_PTR(-EINVAL); 5454 5455 mr = __qlnxr_alloc_mr(ibpd, max_num_sg); 5456 5457 if (IS_ERR(mr)) 5458 return ERR_PTR(-EINVAL); 5459 5460 QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); 5461 5462 return &mr->ibmr; 5463 } 5464 5465 static int 5466 qlnxr_set_page(struct ib_mr *ibmr, u64 addr) 5467 { 5468 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5469 struct qlnxr_pbl *pbl_table; 5470 struct regpair *pbe; 5471 struct qlnxr_dev *dev; 5472 qlnx_host_t *ha; 5473 u32 pbes_in_page; 5474 5475 dev = mr->dev; 5476 ha = dev->ha; 5477 5478 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 5479 QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); 5480 return -ENOMEM; 5481 } 5482 5483 QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, 5484 ((void *)addr)); 5485 5486 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 5487 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 5488 pbe = (struct regpair *)pbl_table->va; 5489 pbe += mr->npages % pbes_in_page; 5490 pbe->lo = cpu_to_le32((u32)addr); 5491 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 5492 5493 mr->npages++; 5494 5495 QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, 5496 ((void *)addr)); 5497 return 0; 5498 } 5499 5500 int 5501 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 5502 int sg_nents, unsigned int *sg_offset) 5503 { 5504 int ret; 5505 struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); 5506 qlnx_host_t *ha; 5507 5508 if (mr == NULL) 5509 return (-1); 5510 5511 if (mr->dev == NULL) 5512 return (-1); 5513 5514 ha = mr->dev->ha; 5515 5516 QL_DPRINT12(ha, "enter\n"); 5517 5518 mr->npages = 0; 5519 qlnx_handle_completed_mrs(mr->dev, &mr->info); 5520 5521 ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); 5522 5523 QL_DPRINT12(ha, "exit ret = %d\n", ret); 5524 5525 return (ret); 5526 } 5527 5528 int 5529 qlnxr_create_ah(struct ib_ah *ibah, 5530 struct ib_ah_attr *attr, u32 flags, 5531 struct ib_udata *udata) 5532 { 5533 struct qlnxr_dev *dev; 5534 qlnx_host_t *ha; 5535 struct qlnxr_ah *ah = get_qlnxr_ah(ibah); 5536 5537 dev = get_qlnxr_dev(ibah->device); 5538 ha = dev->ha; 5539 5540 QL_DPRINT12(ha, "in create_ah\n"); 5541 5542 ah->attr = *attr; 5543 5544 return (0); 5545 } 5546 5547 void 5548 qlnxr_destroy_ah(struct ib_ah *ibah, u32 flags) 5549 { 5550 } 5551 5552 int 5553 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 5554 { 5555 struct qlnxr_dev *dev; 5556 qlnx_host_t *ha; 5557 5558 dev = get_qlnxr_dev((ibah->device)); 5559 ha = dev->ha; 5560 QL_DPRINT12(ha, "Query AH not supported\n"); 5561 return -EINVAL; 5562 } 5563 5564 int 5565 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) 5566 { 5567 struct qlnxr_dev *dev; 5568 qlnx_host_t *ha; 5569 5570 dev = get_qlnxr_dev((ibah->device)); 5571 ha = dev->ha; 5572 QL_DPRINT12(ha, "Modify AH not supported\n"); 5573 return -ENOSYS; 5574 } 5575 5576 int 5577 qlnxr_process_mad(struct ib_device *ibdev, 5578 int process_mad_flags, 5579 u8 port_num, 5580 const struct ib_wc *in_wc, 5581 const struct ib_grh *in_grh, 5582 const struct ib_mad_hdr *mad_hdr, 5583 size_t in_mad_size, 5584 struct ib_mad_hdr *out_mad, 5585 size_t *out_mad_size, 5586 u16 *out_mad_pkey_index) 5587 { 5588 struct qlnxr_dev *dev; 5589 qlnx_host_t *ha; 5590 5591 dev = get_qlnxr_dev(ibdev); 5592 ha = dev->ha; 5593 QL_DPRINT12(ha, "process mad not supported\n"); 5594 5595 return -ENOSYS; 5596 // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", 5597 // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, 5598 // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, 5599 // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, 5600 // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); 5601 5602 // return IB_MAD_RESULT_SUCCESS; 5603 } 5604 5605 int 5606 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, 5607 struct ib_port_immutable *immutable) 5608 { 5609 struct qlnxr_dev *dev; 5610 qlnx_host_t *ha; 5611 struct ib_port_attr attr; 5612 int err; 5613 5614 dev = get_qlnxr_dev(ibdev); 5615 ha = dev->ha; 5616 5617 QL_DPRINT12(ha, "enter\n"); 5618 5619 err = qlnxr_query_port(ibdev, port_num, &attr); 5620 if (err) 5621 return err; 5622 5623 if (QLNX_IS_IWARP(dev)) { 5624 immutable->pkey_tbl_len = 1; 5625 immutable->gid_tbl_len = 1; 5626 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 5627 immutable->max_mad_size = 0; 5628 } else { 5629 immutable->pkey_tbl_len = attr.pkey_tbl_len; 5630 immutable->gid_tbl_len = attr.gid_tbl_len; 5631 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 5632 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 5633 } 5634 5635 QL_DPRINT12(ha, "exit\n"); 5636 return 0; 5637 } 5638 5639 /***** iWARP related functions *************/ 5640 5641 static void 5642 qlnxr_iw_mpa_request(void *context, 5643 struct ecore_iwarp_cm_event_params *params) 5644 { 5645 struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; 5646 struct qlnxr_dev *dev = listener->dev; 5647 struct qlnxr_iw_ep *ep; 5648 struct iw_cm_event event; 5649 struct sockaddr_in *laddr; 5650 struct sockaddr_in *raddr; 5651 qlnx_host_t *ha; 5652 5653 ha = dev->ha; 5654 5655 QL_DPRINT12(ha, "enter\n"); 5656 5657 if (params->cm_info->ip_version != ECORE_TCP_IPV4) { 5658 QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", 5659 params->cm_info->ip_version); 5660 return; 5661 } 5662 5663 ep = kzalloc(sizeof(*ep), GFP_ATOMIC); 5664 5665 if (!ep) { 5666 QL_DPRINT11(ha, "kzalloc{ep) failed\n"); 5667 return; 5668 } 5669 5670 ep->dev = dev; 5671 ep->ecore_context = params->ep_context; 5672 5673 memset(&event, 0, sizeof(event)); 5674 5675 event.event = IW_CM_EVENT_CONNECT_REQUEST; 5676 event.status = params->status; 5677 5678 laddr = (struct sockaddr_in *)&event.local_addr; 5679 raddr = (struct sockaddr_in *)&event.remote_addr; 5680 5681 laddr->sin_family = AF_INET; 5682 raddr->sin_family = AF_INET; 5683 5684 laddr->sin_port = htons(params->cm_info->local_port); 5685 raddr->sin_port = htons(params->cm_info->remote_port); 5686 5687 laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); 5688 raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); 5689 5690 event.provider_data = (void *)ep; 5691 event.private_data = (void *)params->cm_info->private_data; 5692 event.private_data_len = (u8)params->cm_info->private_data_len; 5693 5694 event.ord = params->cm_info->ord; 5695 event.ird = params->cm_info->ird; 5696 5697 listener->cm_id->event_handler(listener->cm_id, &event); 5698 5699 QL_DPRINT12(ha, "exit\n"); 5700 5701 return; 5702 } 5703 5704 static void 5705 qlnxr_iw_issue_event(void *context, 5706 struct ecore_iwarp_cm_event_params *params, 5707 enum iw_cm_event_type event_type, 5708 char *str) 5709 { 5710 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5711 struct qlnxr_dev *dev = ep->dev; 5712 struct iw_cm_event event; 5713 qlnx_host_t *ha; 5714 5715 ha = dev->ha; 5716 5717 QL_DPRINT12(ha, "enter\n"); 5718 5719 memset(&event, 0, sizeof(event)); 5720 event.status = params->status; 5721 event.event = event_type; 5722 5723 if (params->cm_info != NULL) { 5724 event.ird = params->cm_info->ird; 5725 event.ord = params->cm_info->ord; 5726 QL_DPRINT12(ha, "ord=[%d] \n", event.ord); 5727 QL_DPRINT12(ha, "ird=[%d] \n", event.ird); 5728 5729 event.private_data_len = params->cm_info->private_data_len; 5730 event.private_data = (void *)params->cm_info->private_data; 5731 QL_DPRINT12(ha, "private_data_len=[%d] \n", 5732 event.private_data_len); 5733 } 5734 5735 QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); 5736 QL_DPRINT12(ha, "status=[%d] \n", event.status); 5737 5738 if (ep) { 5739 if (ep->cm_id) 5740 ep->cm_id->event_handler(ep->cm_id, &event); 5741 else 5742 QL_DPRINT11(ha, "ep->cm_id == NULL \n"); 5743 } else { 5744 QL_DPRINT11(ha, "ep == NULL \n"); 5745 } 5746 5747 QL_DPRINT12(ha, "exit\n"); 5748 5749 return; 5750 } 5751 5752 static void 5753 qlnxr_iw_close_event(void *context, 5754 struct ecore_iwarp_cm_event_params *params) 5755 { 5756 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5757 struct qlnxr_dev *dev = ep->dev; 5758 qlnx_host_t *ha; 5759 5760 ha = dev->ha; 5761 5762 QL_DPRINT12(ha, "enter\n"); 5763 5764 if (ep->cm_id) { 5765 qlnxr_iw_issue_event(context, 5766 params, 5767 IW_CM_EVENT_CLOSE, 5768 "IW_CM_EVENT_EVENT_CLOSE"); 5769 ep->cm_id->rem_ref(ep->cm_id); 5770 ep->cm_id = NULL; 5771 } 5772 5773 QL_DPRINT12(ha, "exit\n"); 5774 5775 return; 5776 } 5777 5778 static void 5779 qlnxr_iw_passive_complete(void *context, 5780 struct ecore_iwarp_cm_event_params *params) 5781 { 5782 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5783 struct qlnxr_dev *dev = ep->dev; 5784 qlnx_host_t *ha; 5785 5786 ha = dev->ha; 5787 5788 /* We will only reach the following state if MPA_REJECT was called on 5789 * passive. In this case there will be no associated QP. 5790 */ 5791 if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { 5792 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); 5793 kfree(ep); 5794 return; 5795 } 5796 5797 /* We always issue an established event, however, ofed does not look 5798 * at event code for established. So if there was a failure, we follow 5799 * with close... 5800 */ 5801 qlnxr_iw_issue_event(context, 5802 params, 5803 IW_CM_EVENT_ESTABLISHED, 5804 "IW_CM_EVENT_ESTABLISHED"); 5805 5806 if (params->status < 0) { 5807 qlnxr_iw_close_event(context, params); 5808 } 5809 5810 return; 5811 } 5812 5813 struct qlnxr_discon_work { 5814 struct work_struct work; 5815 struct qlnxr_iw_ep *ep; 5816 enum ecore_iwarp_event_type event; 5817 int status; 5818 }; 5819 5820 static void 5821 qlnxr_iw_disconnect_worker(struct work_struct *work) 5822 { 5823 struct qlnxr_discon_work *dwork = 5824 container_of(work, struct qlnxr_discon_work, work); 5825 struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; 5826 struct qlnxr_iw_ep *ep = dwork->ep; 5827 struct qlnxr_dev *dev = ep->dev; 5828 struct qlnxr_qp *qp = ep->qp; 5829 struct iw_cm_event event; 5830 5831 if (qp->destroyed) { 5832 kfree(dwork); 5833 qlnxr_iw_qp_rem_ref(&qp->ibqp); 5834 return; 5835 } 5836 5837 memset(&event, 0, sizeof(event)); 5838 event.status = dwork->status; 5839 event.event = IW_CM_EVENT_DISCONNECT; 5840 5841 /* Success means graceful disconnect was requested. modifying 5842 * to SQD is translated to graceful disconnect. O/w reset is sent 5843 */ 5844 if (dwork->status) 5845 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; 5846 else 5847 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; 5848 5849 kfree(dwork); 5850 5851 if (ep->cm_id) 5852 ep->cm_id->event_handler(ep->cm_id, &event); 5853 5854 SET_FIELD(qp_params.modify_flags, 5855 ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 5856 5857 ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); 5858 5859 qlnxr_iw_qp_rem_ref(&qp->ibqp); 5860 5861 return; 5862 } 5863 5864 void 5865 qlnxr_iw_disconnect_event(void *context, 5866 struct ecore_iwarp_cm_event_params *params) 5867 { 5868 struct qlnxr_discon_work *work; 5869 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5870 struct qlnxr_dev *dev = ep->dev; 5871 struct qlnxr_qp *qp = ep->qp; 5872 5873 work = kzalloc(sizeof(*work), GFP_ATOMIC); 5874 if (!work) 5875 return; 5876 5877 qlnxr_iw_qp_add_ref(&qp->ibqp); 5878 work->ep = ep; 5879 work->event = params->event; 5880 work->status = params->status; 5881 5882 INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); 5883 queue_work(dev->iwarp_wq, &work->work); 5884 5885 return; 5886 } 5887 5888 static int 5889 qlnxr_iw_mpa_reply(void *context, 5890 struct ecore_iwarp_cm_event_params *params) 5891 { 5892 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5893 struct qlnxr_dev *dev = ep->dev; 5894 struct ecore_iwarp_send_rtr_in rtr_in; 5895 int rc; 5896 qlnx_host_t *ha; 5897 5898 ha = dev->ha; 5899 5900 QL_DPRINT12(ha, "enter\n"); 5901 5902 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 5903 return -EINVAL; 5904 5905 bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); 5906 rtr_in.ep_context = params->ep_context; 5907 5908 rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); 5909 5910 QL_DPRINT12(ha, "exit rc = %d\n", rc); 5911 return rc; 5912 } 5913 5914 void 5915 qlnxr_iw_qp_event(void *context, 5916 struct ecore_iwarp_cm_event_params *params, 5917 enum ib_event_type ib_event, 5918 char *str) 5919 { 5920 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5921 struct qlnxr_dev *dev = ep->dev; 5922 struct ib_qp *ibqp = &(ep->qp->ibqp); 5923 struct ib_event event; 5924 qlnx_host_t *ha; 5925 5926 ha = dev->ha; 5927 5928 QL_DPRINT12(ha, 5929 "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", 5930 context, params->event, str, ibqp->event_handler); 5931 5932 if (ibqp->event_handler) { 5933 event.event = ib_event; 5934 event.device = ibqp->device; 5935 event.element.qp = ibqp; 5936 ibqp->event_handler(&event, ibqp->qp_context); 5937 } 5938 5939 return; 5940 } 5941 5942 int 5943 qlnxr_iw_event_handler(void *context, 5944 struct ecore_iwarp_cm_event_params *params) 5945 { 5946 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5947 struct qlnxr_dev *dev = ep->dev; 5948 qlnx_host_t *ha; 5949 5950 ha = dev->ha; 5951 5952 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 5953 "enter\n", context, params->event); 5954 5955 switch (params->event) { 5956 /* Passive side request received */ 5957 case ECORE_IWARP_EVENT_MPA_REQUEST: 5958 qlnxr_iw_mpa_request(context, params); 5959 break; 5960 5961 case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: 5962 qlnxr_iw_mpa_reply(context, params); 5963 break; 5964 5965 /* Passive side established ( ack on mpa response ) */ 5966 case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: 5967 ep->during_connect = 0; 5968 qlnxr_iw_passive_complete(context, params); 5969 break; 5970 5971 /* Active side reply received */ 5972 case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: 5973 ep->during_connect = 0; 5974 qlnxr_iw_issue_event(context, 5975 params, 5976 IW_CM_EVENT_CONNECT_REPLY, 5977 "IW_CM_EVENT_CONNECT_REPLY"); 5978 if (params->status < 0) { 5979 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; 5980 5981 ep->cm_id->rem_ref(ep->cm_id); 5982 ep->cm_id = NULL; 5983 } 5984 break; 5985 5986 case ECORE_IWARP_EVENT_DISCONNECT: 5987 qlnxr_iw_disconnect_event(context, params); 5988 break; 5989 5990 case ECORE_IWARP_EVENT_CLOSE: 5991 ep->during_connect = 0; 5992 qlnxr_iw_close_event(context, params); 5993 break; 5994 5995 case ECORE_IWARP_EVENT_RQ_EMPTY: 5996 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 5997 "IWARP_EVENT_RQ_EMPTY"); 5998 break; 5999 6000 case ECORE_IWARP_EVENT_IRQ_FULL: 6001 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6002 "IWARP_EVENT_IRQ_FULL"); 6003 break; 6004 6005 case ECORE_IWARP_EVENT_LLP_TIMEOUT: 6006 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6007 "IWARP_EVENT_LLP_TIMEOUT"); 6008 break; 6009 6010 case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: 6011 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6012 "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); 6013 break; 6014 6015 case ECORE_IWARP_EVENT_CQ_OVERFLOW: 6016 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6017 "QED_IWARP_EVENT_CQ_OVERFLOW"); 6018 break; 6019 6020 case ECORE_IWARP_EVENT_QP_CATASTROPHIC: 6021 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6022 "QED_IWARP_EVENT_QP_CATASTROPHIC"); 6023 break; 6024 6025 case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: 6026 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, 6027 "IWARP_EVENT_LOCAL_ACCESS_ERROR"); 6028 break; 6029 6030 case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: 6031 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, 6032 "IWARP_EVENT_REMOTE_OPERATION_ERROR"); 6033 break; 6034 6035 case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: 6036 QL_DPRINT12(ha, "Got terminate message" 6037 " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); 6038 break; 6039 6040 default: 6041 QL_DPRINT12(ha, 6042 "Unknown event [0x%x] received \n", params->event); 6043 break; 6044 }; 6045 6046 QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " 6047 "exit\n", context, params->event); 6048 return 0; 6049 } 6050 6051 static int 6052 qlnxr_addr4_resolve(struct qlnxr_dev *dev, 6053 struct sockaddr_in *src_in, 6054 struct sockaddr_in *dst_in, 6055 u8 *dst_mac) 6056 { 6057 int rc; 6058 6059 rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, 6060 dst_mac, NULL, NULL); 6061 6062 QL_DPRINT12(dev->ha, "rc = %d " 6063 "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " 6064 "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, 6065 dst_in->sin_len, dst_in->sin_family, 6066 NIPQUAD((dst_in->sin_addr.s_addr)), 6067 dst_mac[0], dst_mac[1], dst_mac[2], 6068 dst_mac[3], dst_mac[4], dst_mac[5]); 6069 6070 return rc; 6071 } 6072 6073 int 6074 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 6075 { 6076 struct qlnxr_dev *dev; 6077 struct ecore_iwarp_connect_out out_params; 6078 struct ecore_iwarp_connect_in in_params; 6079 struct qlnxr_iw_ep *ep; 6080 struct qlnxr_qp *qp; 6081 struct sockaddr_in *laddr; 6082 struct sockaddr_in *raddr; 6083 int rc = 0; 6084 qlnx_host_t *ha; 6085 6086 dev = get_qlnxr_dev((cm_id->device)); 6087 ha = dev->ha; 6088 6089 QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " 6090 "enter \n", cm_id, conn_param); 6091 6092 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6093 return -EINVAL; 6094 6095 qp = idr_find(&dev->qpidr, conn_param->qpn); 6096 6097 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6098 raddr = (struct sockaddr_in *)&cm_id->remote_addr; 6099 6100 QL_DPRINT12(ha, 6101 "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", 6102 NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, 6103 NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); 6104 6105 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 6106 if (!ep) { 6107 QL_DPRINT11(ha, "struct qlnxr_iw_ep " 6108 "alloc memory failed\n"); 6109 return -ENOMEM; 6110 } 6111 6112 ep->dev = dev; 6113 ep->qp = qp; 6114 cm_id->add_ref(cm_id); 6115 ep->cm_id = cm_id; 6116 6117 memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); 6118 memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); 6119 6120 in_params.event_cb = qlnxr_iw_event_handler; 6121 in_params.cb_context = ep; 6122 6123 in_params.cm_info.ip_version = ECORE_TCP_IPV4; 6124 6125 in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); 6126 in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); 6127 in_params.cm_info.remote_port = ntohs(raddr->sin_port); 6128 in_params.cm_info.local_port = ntohs(laddr->sin_port); 6129 in_params.cm_info.vlan = 0; 6130 in_params.mss = dev->ha->ifp->if_mtu - 40; 6131 6132 QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " 6133 "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " 6134 "vlan = %d\n", 6135 NIPQUAD((in_params.cm_info.remote_ip[0])), 6136 NIPQUAD((in_params.cm_info.local_ip[0])), 6137 in_params.cm_info.remote_port, in_params.cm_info.local_port, 6138 in_params.cm_info.vlan); 6139 6140 rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); 6141 6142 if (rc) { 6143 QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); 6144 goto err; 6145 } 6146 6147 QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" 6148 " private_data_len=%d rq_psn=%d\n", 6149 conn_param->ord, conn_param->ird, conn_param->private_data, 6150 conn_param->private_data_len, qp->rq_psn); 6151 6152 in_params.cm_info.ord = conn_param->ord; 6153 in_params.cm_info.ird = conn_param->ird; 6154 in_params.cm_info.private_data = conn_param->private_data; 6155 in_params.cm_info.private_data_len = conn_param->private_data_len; 6156 in_params.qp = qp->ecore_qp; 6157 6158 memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); 6159 6160 rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); 6161 6162 if (rc) { 6163 QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); 6164 goto err; 6165 } 6166 6167 QL_DPRINT12(ha, "exit\n"); 6168 6169 return rc; 6170 6171 err: 6172 cm_id->rem_ref(cm_id); 6173 kfree(ep); 6174 6175 QL_DPRINT12(ha, "exit [%d]\n", rc); 6176 return rc; 6177 } 6178 6179 int 6180 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) 6181 { 6182 struct qlnxr_dev *dev; 6183 struct qlnxr_iw_listener *listener; 6184 struct ecore_iwarp_listen_in iparams; 6185 struct ecore_iwarp_listen_out oparams; 6186 struct sockaddr_in *laddr; 6187 qlnx_host_t *ha; 6188 int rc; 6189 6190 dev = get_qlnxr_dev((cm_id->device)); 6191 ha = dev->ha; 6192 6193 QL_DPRINT12(ha, "enter\n"); 6194 6195 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6196 return -EINVAL; 6197 6198 laddr = (struct sockaddr_in *)&cm_id->local_addr; 6199 6200 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 6201 6202 if (listener == NULL) { 6203 QL_DPRINT11(ha, "listener memory alloc failed\n"); 6204 return -ENOMEM; 6205 } 6206 6207 listener->dev = dev; 6208 cm_id->add_ref(cm_id); 6209 listener->cm_id = cm_id; 6210 listener->backlog = backlog; 6211 6212 memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); 6213 memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); 6214 6215 iparams.cb_context = listener; 6216 iparams.event_cb = qlnxr_iw_event_handler; 6217 iparams.max_backlog = backlog; 6218 6219 iparams.ip_version = ECORE_TCP_IPV4; 6220 6221 iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); 6222 iparams.port = ntohs(laddr->sin_port); 6223 iparams.vlan = 0; 6224 6225 QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", 6226 NIPQUAD((laddr->sin_addr.s_addr)), 6227 laddr->sin_port, iparams.port); 6228 6229 rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); 6230 if (rc) { 6231 QL_DPRINT11(ha, 6232 "ecore_iwarp_create_listen failed rc = %d\n", rc); 6233 goto err; 6234 } 6235 6236 listener->ecore_handle = oparams.handle; 6237 cm_id->provider_data = listener; 6238 6239 QL_DPRINT12(ha, "exit\n"); 6240 return rc; 6241 6242 err: 6243 cm_id->rem_ref(cm_id); 6244 kfree(listener); 6245 6246 QL_DPRINT12(ha, "exit [%d]\n", rc); 6247 return rc; 6248 } 6249 6250 void 6251 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) 6252 { 6253 struct qlnxr_iw_listener *listener = cm_id->provider_data; 6254 struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); 6255 int rc = 0; 6256 qlnx_host_t *ha; 6257 6258 ha = dev->ha; 6259 6260 QL_DPRINT12(ha, "enter\n"); 6261 6262 if (listener->ecore_handle) 6263 rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, 6264 listener->ecore_handle); 6265 6266 cm_id->rem_ref(cm_id); 6267 6268 QL_DPRINT12(ha, "exit [%d]\n", rc); 6269 return; 6270 } 6271 6272 int 6273 qlnxr_iw_accept(struct iw_cm_id *cm_id, 6274 struct iw_cm_conn_param *conn_param) 6275 { 6276 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 6277 struct qlnxr_dev *dev = ep->dev; 6278 struct qlnxr_qp *qp; 6279 struct ecore_iwarp_accept_in params; 6280 int rc; 6281 qlnx_host_t *ha; 6282 6283 ha = dev->ha; 6284 6285 QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); 6286 6287 if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) 6288 return -EINVAL; 6289 6290 qp = idr_find(&dev->qpidr, conn_param->qpn); 6291 if (!qp) { 6292 QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", 6293 conn_param->qpn); 6294 return -EINVAL; 6295 } 6296 ep->qp = qp; 6297 qp->ep = ep; 6298 cm_id->add_ref(cm_id); 6299 ep->cm_id = cm_id; 6300 6301 params.ep_context = ep->ecore_context; 6302 params.cb_context = ep; 6303 params.qp = ep->qp->ecore_qp; 6304 params.private_data = conn_param->private_data; 6305 params.private_data_len = conn_param->private_data_len; 6306 params.ird = conn_param->ird; 6307 params.ord = conn_param->ord; 6308 6309 rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); 6310 if (rc) { 6311 QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); 6312 goto err; 6313 } 6314 6315 QL_DPRINT12(ha, "exit\n"); 6316 return 0; 6317 err: 6318 cm_id->rem_ref(cm_id); 6319 QL_DPRINT12(ha, "exit rc = %d\n", rc); 6320 return rc; 6321 } 6322 6323 int 6324 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 6325 { 6326 struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; 6327 struct qlnxr_dev *dev = ep->dev; 6328 struct ecore_iwarp_reject_in params; 6329 int rc; 6330 6331 params.ep_context = ep->ecore_context; 6332 params.cb_context = ep; 6333 params.private_data = pdata; 6334 params.private_data_len = pdata_len; 6335 ep->qp = NULL; 6336 6337 rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); 6338 6339 return rc; 6340 } 6341 6342 void 6343 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) 6344 { 6345 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 6346 qlnx_host_t *ha; 6347 6348 ha = qp->dev->ha; 6349 6350 QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); 6351 6352 atomic_inc(&qp->refcnt); 6353 6354 QL_DPRINT12(ha, "exit \n"); 6355 return; 6356 } 6357 6358 void 6359 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) 6360 { 6361 struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); 6362 qlnx_host_t *ha; 6363 6364 ha = qp->dev->ha; 6365 6366 QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); 6367 6368 if (atomic_dec_and_test(&qp->refcnt)) { 6369 qlnxr_idr_remove(qp->dev, qp->qp_id); 6370 } 6371 6372 QL_DPRINT12(ha, "exit \n"); 6373 return; 6374 } 6375 6376 struct ib_qp * 6377 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) 6378 { 6379 struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); 6380 struct ib_qp *qp; 6381 qlnx_host_t *ha; 6382 6383 ha = dev->ha; 6384 6385 QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); 6386 6387 qp = idr_find(&dev->qpidr, qpn); 6388 6389 QL_DPRINT12(ha, "exit qp = %p\n", qp); 6390 6391 return (qp); 6392 } 6393