1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 #include <rdma/ib_addr.h> 7 #include <rdma/ib_cache.h> 8 #include <rdma/ib_user_verbs.h> 9 #include <ionic_api.h> 10 11 #include "ionic_fw.h" 12 #include "ionic_ibdev.h" 13 14 #define ionic_set_ecn(tos) (((tos) | 2u) & ~1u) 15 #define ionic_clear_ecn(tos) ((tos) & ~3u) 16 17 static int ionic_validate_qdesc(struct ionic_qdesc *q) 18 { 19 if (!q->addr || !q->size || !q->mask || 20 !q->depth_log2 || !q->stride_log2) 21 return -EINVAL; 22 23 if (q->addr & (PAGE_SIZE - 1)) 24 return -EINVAL; 25 26 if (q->mask != BIT(q->depth_log2) - 1) 27 return -EINVAL; 28 29 if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2)) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx) 36 { 37 /* EQ per vector per udma, and the first eqs reserved for async events. 38 * The rest of the vectors can be requested for completions. 39 */ 40 u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1; 41 42 return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx; 43 } 44 45 static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx) 46 { 47 unsigned int size, base, bound; 48 int rc; 49 50 size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; 51 base = size * udma_idx; 52 bound = base + size; 53 54 rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound); 55 if (rc >= 0) { 56 /* cq_base is zero or a multiple of two queue groups */ 57 *cqid = dev->lif_cfg.cq_base + 58 ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift, 59 dev->half_cqid_udma_shift); 60 61 rc = 0; 62 } 63 64 return rc; 65 } 66 67 static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid) 68 { 69 u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base, 70 dev->lif_cfg.udma_qgrp_shift, 71 dev->half_cqid_udma_shift); 72 73 ionic_resid_put(&dev->inuse_cqid, bitid); 74 } 75 76 int ionic_create_cq_common(struct ionic_vcq *vcq, 77 struct ionic_tbl_buf *buf, 78 const struct ib_cq_init_attr *attr, 79 struct ionic_ctx *ctx, 80 struct ib_udata *udata, 81 struct ionic_qdesc *req_cq, 82 int udma_idx) 83 { 84 struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); 85 struct ionic_cq *cq = &vcq->cq[udma_idx]; 86 int rc; 87 88 cq->vcq = vcq; 89 90 if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) { 91 rc = -EINVAL; 92 goto err_args; 93 } 94 95 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { 96 rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); 97 if (rc) 98 goto err_args; 99 } 100 101 cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); 102 103 spin_lock_init(&cq->lock); 104 INIT_LIST_HEAD(&cq->poll_sq); 105 INIT_LIST_HEAD(&cq->flush_sq); 106 INIT_LIST_HEAD(&cq->flush_rq); 107 108 if (udata) { 109 rc = ionic_validate_qdesc(req_cq); 110 if (rc) 111 goto err_qdesc; 112 113 cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, 114 req_cq->addr, req_cq->size, 115 IB_ACCESS_LOCAL_WRITE); 116 if (IS_ERR(cq->umem)) { 117 rc = PTR_ERR(cq->umem); 118 goto err_qdesc; 119 } 120 121 cq->q.ptr = NULL; 122 cq->q.size = req_cq->size; 123 cq->q.mask = req_cq->mask; 124 cq->q.depth_log2 = req_cq->depth_log2; 125 cq->q.stride_log2 = req_cq->stride_log2; 126 } else { 127 rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, 128 attr->cqe + IONIC_CQ_GRACE, 129 sizeof(struct ionic_v1_cqe)); 130 if (rc) 131 goto err_q_init; 132 133 cq->color = true; 134 cq->credit = cq->q.mask; 135 } 136 137 rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE); 138 if (rc) 139 goto err_pgtbl_init; 140 141 init_completion(&cq->cq_rel_comp); 142 kref_init(&cq->cq_kref); 143 144 return 0; 145 146 err_pgtbl_init: 147 if (!udata) 148 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 149 err_q_init: 150 if (cq->umem) 151 ib_umem_release(cq->umem); 152 err_qdesc: 153 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) 154 ionic_put_cqid(dev, cq->cqid); 155 err_args: 156 cq->vcq = NULL; 157 158 return rc; 159 } 160 161 int ionic_post_create_cq_cmd(struct ionic_cq *cq, 162 struct ib_udata *udata, 163 __u32 *resp_cqid) 164 { 165 struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device); 166 167 if (udata && resp_cqid) 168 *resp_cqid = cq->cqid; 169 else 170 ionic_queue_dbell_init(&cq->q, cq->cqid); 171 172 return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); 173 } 174 175 void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq) 176 { 177 if (!cq->vcq) 178 return; 179 180 if (!xa_erase_irq(&dev->cq_tbl, cq->cqid)) 181 return; 182 183 kref_put(&cq->cq_kref, ionic_cq_complete); 184 wait_for_completion(&cq->cq_rel_comp); 185 } 186 187 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq) 188 { 189 if (!cq->vcq) 190 return; 191 192 if (cq->umem) 193 ib_umem_release(cq->umem); 194 else 195 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 196 197 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) 198 ionic_put_cqid(dev, cq->cqid); 199 200 cq->vcq = NULL; 201 } 202 203 static int ionic_validate_qdesc_zero(struct ionic_qdesc *q) 204 { 205 if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2) 206 return -EINVAL; 207 208 return 0; 209 } 210 211 static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid) 212 { 213 int rc; 214 215 rc = ionic_resid_get(&dev->inuse_pdid); 216 if (rc < 0) 217 return rc; 218 219 *pdid = rc; 220 return 0; 221 } 222 223 static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid) 224 { 225 int rc; 226 227 rc = ionic_resid_get(&dev->inuse_ahid); 228 if (rc < 0) 229 return rc; 230 231 *ahid = rc; 232 return 0; 233 } 234 235 static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid) 236 { 237 int rc; 238 239 /* wrap to 1, skip reserved lkey */ 240 rc = ionic_resid_get_shared(&dev->inuse_mrid, 1, 241 dev->inuse_mrid.inuse_size); 242 if (rc < 0) 243 return rc; 244 245 *mrid = ionic_mrid(rc, dev->next_mrkey++); 246 return 0; 247 } 248 249 static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid) 250 { 251 int rc = 0; 252 253 rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1); 254 if (rc < 0) 255 return rc; 256 257 *qpid = IB_QPT_GSI; 258 return 0; 259 } 260 261 static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid, 262 u8 *udma_idx, u8 udma_mask) 263 { 264 unsigned int size, base, bound; 265 int udma_i, udma_x, udma_ix; 266 int rc = -EINVAL; 267 268 udma_x = dev->next_qpid_udma_idx; 269 270 dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1; 271 272 for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) { 273 udma_ix = udma_i ^ udma_x; 274 275 if (!(udma_mask & BIT(udma_ix))) 276 continue; 277 278 size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count; 279 base = size * udma_ix; 280 bound = base + size; 281 282 /* skip reserved SMI and GSI qpids in group zero */ 283 if (!base) 284 base = 2; 285 286 rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound); 287 if (rc >= 0) { 288 *qpid = ionic_bitid_to_qid(rc, 289 dev->lif_cfg.udma_qgrp_shift, 290 dev->half_qpid_udma_shift); 291 *udma_idx = udma_ix; 292 293 rc = 0; 294 break; 295 } 296 } 297 298 return rc; 299 } 300 301 static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr) 302 { 303 int rc, dbpage_num; 304 305 /* wrap to 1, skip kernel reserved */ 306 rc = ionic_resid_get_shared(&dev->inuse_dbid, 1, 307 dev->inuse_dbid.inuse_size); 308 if (rc < 0) 309 return rc; 310 311 dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc; 312 *addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT); 313 314 *dbid = rc; 315 316 return 0; 317 } 318 319 static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid) 320 { 321 ionic_resid_put(&dev->inuse_pdid, pdid); 322 } 323 324 static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid) 325 { 326 ionic_resid_put(&dev->inuse_ahid, ahid); 327 } 328 329 static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid) 330 { 331 ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid)); 332 } 333 334 static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid) 335 { 336 u32 bitid = ionic_qid_to_bitid(qpid, 337 dev->lif_cfg.udma_qgrp_shift, 338 dev->half_qpid_udma_shift); 339 340 ionic_resid_put(&dev->inuse_qpid, bitid); 341 } 342 343 static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid) 344 { 345 ionic_resid_put(&dev->inuse_dbid, dbid); 346 } 347 348 static struct rdma_user_mmap_entry* 349 ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size, 350 unsigned long pfn, u8 mmap_flags, u64 *offset) 351 { 352 struct ionic_mmap_entry *entry; 353 int rc; 354 355 entry = kzalloc_obj(*entry); 356 if (!entry) 357 return NULL; 358 359 entry->size = size; 360 entry->pfn = pfn; 361 entry->mmap_flags = mmap_flags; 362 363 rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry, 364 entry->size); 365 if (rc) { 366 kfree(entry); 367 return NULL; 368 } 369 370 if (offset) 371 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 372 373 return &entry->rdma_entry; 374 } 375 376 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 377 { 378 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 379 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 380 struct ionic_ctx_resp resp = {}; 381 struct ionic_ctx_req req; 382 phys_addr_t db_phys = 0; 383 int rc; 384 385 rc = ib_copy_validate_udata_in(udata, req, rsvd); 386 if (rc) 387 return rc; 388 389 /* try to allocate dbid for user ctx */ 390 rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys); 391 if (rc < 0) 392 return rc; 393 394 ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid); 395 396 ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 397 PHYS_PFN(db_phys), 0, NULL); 398 if (!ctx->mmap_dbell) { 399 rc = -ENOMEM; 400 goto err_mmap_dbell; 401 } 402 403 if (dev->lif_cfg.phc_state) { 404 ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0, 405 IONIC_MMAP_PHC, 406 &resp.phc_offset); 407 if (!ctx->mmap_phc) { 408 rc = -ENOMEM; 409 goto err_mmap_phc; 410 } 411 } 412 413 resp.page_shift = PAGE_SHIFT; 414 415 resp.dbell_offset = db_phys & ~PAGE_MASK; 416 417 resp.version = dev->lif_cfg.rdma_version; 418 resp.qp_opcodes = dev->lif_cfg.qp_opcodes; 419 resp.admin_opcodes = dev->lif_cfg.admin_opcodes; 420 421 resp.sq_qtype = dev->lif_cfg.sq_qtype; 422 resp.rq_qtype = dev->lif_cfg.rq_qtype; 423 resp.cq_qtype = dev->lif_cfg.cq_qtype; 424 resp.admin_qtype = dev->lif_cfg.aq_qtype; 425 resp.max_stride = dev->lif_cfg.max_stride; 426 resp.max_spec = IONIC_SPEC_HIGH; 427 428 resp.udma_count = dev->lif_cfg.udma_count; 429 resp.expdb_mask = dev->lif_cfg.expdb_mask; 430 resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit; 431 resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS; 432 433 if (dev->lif_cfg.sq_expdb) 434 resp.expdb_qtypes |= IONIC_EXPDB_SQ; 435 if (dev->lif_cfg.rq_expdb) 436 resp.expdb_qtypes |= IONIC_EXPDB_RQ; 437 438 rc = ib_respond_udata(udata, resp); 439 if (rc) 440 goto err_resp; 441 442 return 0; 443 444 err_resp: 445 rdma_user_mmap_entry_remove(ctx->mmap_phc); 446 err_mmap_phc: 447 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 448 err_mmap_dbell: 449 ionic_put_dbid(dev, ctx->dbid); 450 451 return rc; 452 } 453 454 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx) 455 { 456 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 457 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 458 459 rdma_user_mmap_entry_remove(ctx->mmap_phc); 460 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 461 ionic_put_dbid(dev, ctx->dbid); 462 } 463 464 static int ionic_mmap_phc_state(struct ionic_ibdev *dev, 465 struct vm_area_struct *vma) 466 { 467 if (!(vma->vm_flags & VM_SHARED)) 468 return -EINVAL; 469 470 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 471 return -EPERM; 472 473 vm_flags_clear(vma, VM_MAYWRITE); 474 475 return vm_insert_page(vma, vma->vm_start, 476 virt_to_page(dev->lif_cfg.phc_state)); 477 } 478 479 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma) 480 { 481 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 482 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 483 struct rdma_user_mmap_entry *rdma_entry; 484 struct ionic_mmap_entry *ionic_entry; 485 int rc = 0; 486 487 rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma); 488 if (!rdma_entry) { 489 ibdev_dbg(&dev->ibdev, "not found %#lx\n", 490 vma->vm_pgoff << PAGE_SHIFT); 491 return -EINVAL; 492 } 493 494 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 495 rdma_entry); 496 497 if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) { 498 rc = ionic_mmap_phc_state(dev, vma); 499 rdma_user_mmap_entry_put(rdma_entry); 500 return rc; 501 } 502 503 ibdev_dbg(&dev->ibdev, "writecombine? %d\n", 504 ionic_entry->mmap_flags & IONIC_MMAP_WC); 505 if (ionic_entry->mmap_flags & IONIC_MMAP_WC) 506 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 507 else 508 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 509 510 ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n", 511 vma->vm_start, ionic_entry->pfn, ionic_entry->size); 512 rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn, 513 ionic_entry->size, vma->vm_page_prot, 514 rdma_entry); 515 if (rc) 516 ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc); 517 518 rdma_user_mmap_entry_put(rdma_entry); 519 return rc; 520 } 521 522 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 523 { 524 struct ionic_mmap_entry *ionic_entry; 525 526 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 527 rdma_entry); 528 kfree(ionic_entry); 529 } 530 531 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 532 { 533 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 534 struct ionic_pd *pd = to_ionic_pd(ibpd); 535 int rc; 536 537 rc = ib_no_udata_io(udata); 538 if (rc) 539 return rc; 540 541 return ionic_get_pdid(dev, &pd->pdid); 542 } 543 544 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 545 { 546 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 547 struct ionic_pd *pd = to_ionic_pd(ibpd); 548 int rc; 549 550 rc = ib_no_udata_io(udata); 551 if (rc) 552 return rc; 553 554 ionic_put_pdid(dev, pd->pdid); 555 556 return 0; 557 } 558 559 static int ionic_build_hdr(struct ionic_ibdev *dev, 560 struct ib_ud_header *hdr, 561 const struct rdma_ah_attr *attr, 562 u16 sport, bool want_ecn) 563 { 564 const struct ib_global_route *grh; 565 enum rdma_network_type net; 566 u8 smac[ETH_ALEN]; 567 u16 vlan; 568 int rc; 569 570 if (attr->ah_flags != IB_AH_GRH) 571 return -EINVAL; 572 if (attr->type != RDMA_AH_ATTR_TYPE_ROCE) 573 return -EINVAL; 574 575 grh = rdma_ah_read_grh(attr); 576 577 rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); 578 if (rc) 579 return rc; 580 581 net = rdma_gid_attr_network_type(grh->sgid_attr); 582 583 rc = ib_ud_header_init(0, /* no payload */ 584 0, /* no lrh */ 585 1, /* yes eth */ 586 vlan != 0xffff, 587 0, /* no grh */ 588 net == RDMA_NETWORK_IPV4 ? 4 : 6, 589 1, /* yes udp */ 590 0, /* no imm */ 591 hdr); 592 if (rc) 593 return rc; 594 595 ether_addr_copy(hdr->eth.smac_h, smac); 596 ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); 597 598 if (net == RDMA_NETWORK_IPV4) { 599 hdr->eth.type = cpu_to_be16(ETH_P_IP); 600 hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */ 601 hdr->ip4.ttl = grh->hop_limit; 602 hdr->ip4.tot_len = cpu_to_be16(0xffff); 603 hdr->ip4.saddr = 604 *(const __be32 *)(grh->sgid_attr->gid.raw + 12); 605 hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12); 606 607 if (want_ecn) 608 hdr->ip4.tos = ionic_set_ecn(grh->traffic_class); 609 else 610 hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class); 611 } else { 612 hdr->eth.type = cpu_to_be16(ETH_P_IPV6); 613 hdr->grh.flow_label = cpu_to_be32(grh->flow_label); 614 hdr->grh.hop_limit = grh->hop_limit; 615 hdr->grh.source_gid = grh->sgid_attr->gid; 616 hdr->grh.destination_gid = grh->dgid; 617 618 if (want_ecn) 619 hdr->grh.traffic_class = 620 ionic_set_ecn(grh->traffic_class); 621 else 622 hdr->grh.traffic_class = 623 ionic_clear_ecn(grh->traffic_class); 624 } 625 626 if (vlan != 0xffff) { 627 vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT; 628 hdr->vlan.tag = cpu_to_be16(vlan); 629 hdr->vlan.type = hdr->eth.type; 630 hdr->eth.type = cpu_to_be16(ETH_P_8021Q); 631 } 632 633 hdr->udp.sport = cpu_to_be16(sport); 634 hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT); 635 636 return 0; 637 } 638 639 static void ionic_set_ah_attr(struct ionic_ibdev *dev, 640 struct rdma_ah_attr *ah_attr, 641 struct ib_ud_header *hdr, 642 int sgid_index) 643 { 644 u32 flow_label; 645 u16 vlan = 0; 646 u8 tos, ttl; 647 648 if (hdr->vlan_present) 649 vlan = be16_to_cpu(hdr->vlan.tag); 650 651 if (hdr->ipv4_present) { 652 flow_label = 0; 653 ttl = hdr->ip4.ttl; 654 tos = hdr->ip4.tos; 655 *(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff); 656 *(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr; 657 } else { 658 flow_label = be32_to_cpu(hdr->grh.flow_label); 659 ttl = hdr->grh.hop_limit; 660 tos = hdr->grh.traffic_class; 661 } 662 663 memset(ah_attr, 0, sizeof(*ah_attr)); 664 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE; 665 if (hdr->eth_present) 666 ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h); 667 rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT); 668 rdma_ah_set_port_num(ah_attr, 1); 669 rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos); 670 rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid); 671 } 672 673 static int ionic_create_ah_cmd(struct ionic_ibdev *dev, 674 struct ionic_ah *ah, 675 struct ionic_pd *pd, 676 struct rdma_ah_attr *attr, 677 u32 flags) 678 { 679 struct ionic_admin_wr wr = { 680 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 681 .wqe = { 682 .op = IONIC_V1_ADMIN_CREATE_AH, 683 .len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN), 684 .cmd.create_ah = { 685 .pd_id = cpu_to_le32(pd->pdid), 686 .dbid_flags = cpu_to_le16(dev->lif_cfg.dbid), 687 .id_ver = cpu_to_le32(ah->ahid), 688 } 689 } 690 }; 691 enum ionic_admin_flags admin_flags = 0; 692 dma_addr_t hdr_dma = 0; 693 void *hdr_buf; 694 gfp_t gfp = GFP_ATOMIC; 695 int rc, hdr_len = 0; 696 697 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH) 698 return -EBADRQC; 699 700 if (flags & RDMA_CREATE_AH_SLEEPABLE) 701 gfp = GFP_KERNEL; 702 else 703 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 704 705 rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false); 706 if (rc) 707 return rc; 708 709 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) { 710 if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP)) 711 wr.wqe.cmd.create_ah.csum_profile = 712 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP; 713 else 714 wr.wqe.cmd.create_ah.csum_profile = 715 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP; 716 } else { 717 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP)) 718 wr.wqe.cmd.create_ah.csum_profile = 719 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 720 else 721 wr.wqe.cmd.create_ah.csum_profile = 722 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 723 } 724 725 ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index; 726 727 hdr_buf = kmalloc(PAGE_SIZE, gfp); 728 if (!hdr_buf) 729 return -ENOMEM; 730 731 hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf); 732 hdr_len -= IB_BTH_BYTES; 733 hdr_len -= IB_DETH_BYTES; 734 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 735 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 736 hdr_buf, hdr_len, true); 737 738 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 739 DMA_TO_DEVICE); 740 741 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 742 if (rc) 743 goto err_dma; 744 745 wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma); 746 wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len); 747 748 ionic_admin_post(dev, &wr); 749 rc = ionic_admin_wait(dev, &wr, admin_flags); 750 751 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 752 DMA_TO_DEVICE); 753 err_dma: 754 kfree(hdr_buf); 755 756 return rc; 757 } 758 759 static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags) 760 { 761 struct ionic_admin_wr wr = { 762 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 763 .wqe = { 764 .op = IONIC_V1_ADMIN_DESTROY_AH, 765 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN), 766 .cmd.destroy_ah = { 767 .ah_id = cpu_to_le32(ahid), 768 }, 769 } 770 }; 771 enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN; 772 773 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH) 774 return -EBADRQC; 775 776 if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) 777 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 778 779 ionic_admin_post(dev, &wr); 780 ionic_admin_wait(dev, &wr, admin_flags); 781 782 /* No host-memory resource is associated with ah, so it is ok 783 * to "succeed" and complete this destroy ah on the host. 784 */ 785 return 0; 786 } 787 788 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 789 struct ib_udata *udata) 790 { 791 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 792 struct rdma_ah_attr *attr = init_attr->ah_attr; 793 struct ionic_pd *pd = to_ionic_pd(ibah->pd); 794 struct ionic_ah *ah = to_ionic_ah(ibah); 795 struct ionic_ah_resp resp = {}; 796 u32 flags = init_attr->flags; 797 int rc; 798 799 rc = ib_is_udata_in_empty(udata); 800 if (rc) 801 return rc; 802 803 rc = ionic_get_ahid(dev, &ah->ahid); 804 if (rc) 805 return rc; 806 807 rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags); 808 if (rc) 809 goto err_cmd; 810 811 if (udata) { 812 resp.ahid = ah->ahid; 813 814 rc = ib_respond_udata(udata, resp); 815 if (rc) 816 goto err_resp; 817 } 818 819 return 0; 820 821 err_resp: 822 ionic_destroy_ah_cmd(dev, ah->ahid, flags); 823 err_cmd: 824 ionic_put_ahid(dev, ah->ahid); 825 return rc; 826 } 827 828 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 829 { 830 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 831 struct ionic_ah *ah = to_ionic_ah(ibah); 832 833 ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index); 834 835 return 0; 836 } 837 838 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags) 839 { 840 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 841 struct ionic_ah *ah = to_ionic_ah(ibah); 842 int rc; 843 844 rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags); 845 if (rc) 846 return rc; 847 848 ionic_put_ahid(dev, ah->ahid); 849 850 return 0; 851 } 852 853 static int ionic_create_mr_cmd(struct ionic_ibdev *dev, 854 struct ionic_pd *pd, 855 struct ionic_mr *mr, 856 u64 addr, 857 u64 length) 858 { 859 struct ionic_admin_wr wr = { 860 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 861 .wqe = { 862 .op = IONIC_V1_ADMIN_CREATE_MR, 863 .len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN), 864 .cmd.create_mr = { 865 .va = cpu_to_le64(addr), 866 .length = cpu_to_le64(length), 867 .pd_id = cpu_to_le32(pd->pdid), 868 .page_size_log2 = mr->buf.page_size_log2, 869 .tbl_index = cpu_to_le32(~0), 870 .map_count = cpu_to_le32(mr->buf.tbl_pages), 871 .dma_addr = ionic_pgtbl_dma(&mr->buf, addr), 872 .dbid_flags = cpu_to_le16(mr->flags), 873 .id_ver = cpu_to_le32(mr->mrid), 874 } 875 } 876 }; 877 int rc; 878 879 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR) 880 return -EBADRQC; 881 882 ionic_admin_post(dev, &wr); 883 rc = ionic_admin_wait(dev, &wr, 0); 884 if (!rc) 885 mr->created = true; 886 887 return rc; 888 } 889 890 static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid) 891 { 892 struct ionic_admin_wr wr = { 893 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 894 .wqe = { 895 .op = IONIC_V1_ADMIN_DESTROY_MR, 896 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN), 897 .cmd.destroy_mr = { 898 .mr_id = cpu_to_le32(mrid), 899 }, 900 } 901 }; 902 903 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR) 904 return -EBADRQC; 905 906 ionic_admin_post(dev, &wr); 907 908 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 909 } 910 911 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access) 912 { 913 struct ionic_pd *pd = to_ionic_pd(ibpd); 914 struct ionic_mr *mr; 915 916 mr = kzalloc_obj(*mr); 917 if (!mr) 918 return ERR_PTR(-ENOMEM); 919 920 mr->ibmr.lkey = IONIC_DMA_LKEY; 921 mr->ibmr.rkey = IONIC_DMA_RKEY; 922 923 if (pd) 924 pd->flags |= IONIC_QPF_PRIVILEGED; 925 926 return &mr->ibmr; 927 } 928 929 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 930 u64 addr, int access, struct ib_dmah *dmah, 931 struct ib_udata *udata) 932 { 933 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 934 struct ionic_pd *pd = to_ionic_pd(ibpd); 935 struct ionic_mr *mr; 936 unsigned long pg_sz; 937 int rc; 938 939 rc = ib_no_udata_io(udata); 940 if (rc) 941 return ERR_PTR(rc); 942 943 if (dmah) 944 return ERR_PTR(-EOPNOTSUPP); 945 946 mr = kzalloc_obj(*mr); 947 if (!mr) 948 return ERR_PTR(-ENOMEM); 949 950 rc = ionic_get_mrid(dev, &mr->mrid); 951 if (rc) 952 goto err_mrid; 953 954 mr->ibmr.lkey = mr->mrid; 955 mr->ibmr.rkey = mr->mrid; 956 mr->ibmr.iova = addr; 957 mr->ibmr.length = length; 958 959 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 960 961 mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); 962 if (IS_ERR(mr->umem)) { 963 rc = PTR_ERR(mr->umem); 964 goto err_umem; 965 } 966 967 pg_sz = ib_umem_find_best_pgsz(mr->umem, 968 dev->lif_cfg.page_size_supported, 969 addr); 970 if (!pg_sz) { 971 rc = -EINVAL; 972 goto err_pgtbl; 973 } 974 975 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 976 if (rc) 977 goto err_pgtbl; 978 979 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 980 if (rc) 981 goto err_cmd; 982 983 ionic_pgtbl_unbuf(dev, &mr->buf); 984 985 return &mr->ibmr; 986 987 err_cmd: 988 ionic_pgtbl_unbuf(dev, &mr->buf); 989 err_pgtbl: 990 ib_umem_release(mr->umem); 991 err_umem: 992 ionic_put_mrid(dev, mr->mrid); 993 err_mrid: 994 kfree(mr); 995 return ERR_PTR(rc); 996 } 997 998 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 999 u64 length, u64 addr, int fd, int access, 1000 struct ib_dmah *dmah, 1001 struct uverbs_attr_bundle *attrs) 1002 { 1003 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 1004 struct ionic_pd *pd = to_ionic_pd(ibpd); 1005 struct ib_umem_dmabuf *umem_dmabuf; 1006 struct ionic_mr *mr; 1007 u64 pg_sz; 1008 int rc; 1009 1010 if (dmah) 1011 return ERR_PTR(-EOPNOTSUPP); 1012 1013 mr = kzalloc_obj(*mr); 1014 if (!mr) 1015 return ERR_PTR(-ENOMEM); 1016 1017 rc = ionic_get_mrid(dev, &mr->mrid); 1018 if (rc) 1019 goto err_mrid; 1020 1021 mr->ibmr.lkey = mr->mrid; 1022 mr->ibmr.rkey = mr->mrid; 1023 mr->ibmr.iova = addr; 1024 mr->ibmr.length = length; 1025 1026 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 1027 1028 umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length, 1029 fd, access); 1030 if (IS_ERR(umem_dmabuf)) { 1031 rc = PTR_ERR(umem_dmabuf); 1032 goto err_umem; 1033 } 1034 1035 mr->umem = &umem_dmabuf->umem; 1036 1037 pg_sz = ib_umem_find_best_pgsz(mr->umem, 1038 dev->lif_cfg.page_size_supported, 1039 addr); 1040 if (!pg_sz) { 1041 rc = -EINVAL; 1042 goto err_pgtbl; 1043 } 1044 1045 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 1046 if (rc) 1047 goto err_pgtbl; 1048 1049 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 1050 if (rc) 1051 goto err_cmd; 1052 1053 ionic_pgtbl_unbuf(dev, &mr->buf); 1054 1055 return &mr->ibmr; 1056 1057 err_cmd: 1058 ionic_pgtbl_unbuf(dev, &mr->buf); 1059 err_pgtbl: 1060 ib_umem_release(mr->umem); 1061 err_umem: 1062 ionic_put_mrid(dev, mr->mrid); 1063 err_mrid: 1064 kfree(mr); 1065 return ERR_PTR(rc); 1066 } 1067 1068 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1069 { 1070 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1071 struct ionic_mr *mr = to_ionic_mr(ibmr); 1072 int rc; 1073 1074 rc = ib_no_udata_io(udata); 1075 if (rc) 1076 return rc; 1077 1078 if (!mr->ibmr.lkey) 1079 goto out; 1080 1081 if (mr->created) { 1082 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1083 if (rc) 1084 return rc; 1085 } 1086 1087 ionic_pgtbl_unbuf(dev, &mr->buf); 1088 1089 if (mr->umem) 1090 ib_umem_release(mr->umem); 1091 1092 ionic_put_mrid(dev, mr->mrid); 1093 1094 out: 1095 kfree(mr); 1096 1097 return 0; 1098 } 1099 1100 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 1101 u32 max_sg) 1102 { 1103 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 1104 struct ionic_pd *pd = to_ionic_pd(ibpd); 1105 struct ionic_mr *mr; 1106 int rc; 1107 1108 if (type != IB_MR_TYPE_MEM_REG) 1109 return ERR_PTR(-EINVAL); 1110 1111 mr = kzalloc_obj(*mr); 1112 if (!mr) 1113 return ERR_PTR(-ENOMEM); 1114 1115 rc = ionic_get_mrid(dev, &mr->mrid); 1116 if (rc) 1117 goto err_mrid; 1118 1119 mr->ibmr.lkey = mr->mrid; 1120 mr->ibmr.rkey = mr->mrid; 1121 1122 mr->flags = IONIC_MRF_PHYS_MR; 1123 1124 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE); 1125 if (rc) 1126 goto err_pgtbl; 1127 1128 mr->buf.tbl_pages = 0; 1129 1130 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1131 if (rc) 1132 goto err_cmd; 1133 1134 return &mr->ibmr; 1135 1136 err_cmd: 1137 ionic_pgtbl_unbuf(dev, &mr->buf); 1138 err_pgtbl: 1139 ionic_put_mrid(dev, mr->mrid); 1140 err_mrid: 1141 kfree(mr); 1142 return ERR_PTR(rc); 1143 } 1144 1145 static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma) 1146 { 1147 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1148 struct ionic_mr *mr = to_ionic_mr(ibmr); 1149 1150 ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma); 1151 return ionic_pgtbl_page(&mr->buf, dma); 1152 } 1153 1154 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1155 unsigned int *sg_offset) 1156 { 1157 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1158 struct ionic_mr *mr = to_ionic_mr(ibmr); 1159 int rc; 1160 1161 /* mr must be allocated using ib_alloc_mr() */ 1162 if (unlikely(!mr->buf.tbl_limit)) 1163 return -EINVAL; 1164 1165 mr->buf.tbl_pages = 0; 1166 1167 if (mr->buf.tbl_buf) 1168 dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1169 mr->buf.tbl_size, DMA_TO_DEVICE); 1170 1171 ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents); 1172 rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page); 1173 1174 mr->buf.page_size_log2 = order_base_2(ibmr->page_size); 1175 1176 if (mr->buf.tbl_buf) 1177 dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1178 mr->buf.tbl_size, DMA_TO_DEVICE); 1179 1180 return rc; 1181 } 1182 1183 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 1184 { 1185 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1186 struct ionic_pd *pd = to_ionic_pd(ibmw->pd); 1187 struct ionic_mr *mr = to_ionic_mw(ibmw); 1188 int rc; 1189 1190 rc = ib_no_udata_io(udata); 1191 if (rc) 1192 return rc; 1193 1194 rc = ionic_get_mrid(dev, &mr->mrid); 1195 if (rc) 1196 return rc; 1197 1198 mr->ibmw.rkey = mr->mrid; 1199 1200 if (mr->ibmw.type == IB_MW_TYPE_1) 1201 mr->flags = IONIC_MRF_MW_1; 1202 else 1203 mr->flags = IONIC_MRF_MW_2; 1204 1205 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1206 if (rc) 1207 goto err_cmd; 1208 1209 return 0; 1210 1211 err_cmd: 1212 ionic_put_mrid(dev, mr->mrid); 1213 return rc; 1214 } 1215 1216 int ionic_dealloc_mw(struct ib_mw *ibmw) 1217 { 1218 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1219 struct ionic_mr *mr = to_ionic_mw(ibmw); 1220 int rc; 1221 1222 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1223 if (rc) 1224 return rc; 1225 1226 ionic_put_mrid(dev, mr->mrid); 1227 1228 return 0; 1229 } 1230 1231 static int ionic_create_cq_cmd(struct ionic_ibdev *dev, 1232 struct ionic_ctx *ctx, 1233 struct ionic_cq *cq, 1234 struct ionic_tbl_buf *buf, 1235 int udma_idx, 1236 int *out_udma_idx) 1237 { 1238 struct ionic_admin_create_cq_resp *resp_buf; 1239 const u16 dbid = ionic_ctx_dbid(dev, ctx); 1240 struct ionic_admin_wr wr = { 1241 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1242 .wqe = { 1243 .op = IONIC_V1_ADMIN_CREATE_CQ, 1244 .len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN), 1245 .cmd.create_cq = { 1246 .eq_id = cpu_to_le32(cq->eqid), 1247 .depth_log2 = cq->q.depth_log2, 1248 .stride_log2 = cq->q.stride_log2, 1249 .page_size_log2 = buf->page_size_log2, 1250 .tbl_index = cpu_to_le32(~0), 1251 .map_count = cpu_to_le32(buf->tbl_pages), 1252 .dma_addr = ionic_pgtbl_dma(buf, 0), 1253 .dbid_flags = cpu_to_le16(dbid), 1254 .id_ver = cpu_to_le32(cq->cqid), 1255 } 1256 } 1257 }; 1258 dma_addr_t resp_buf_dma; 1259 int rc; 1260 1261 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) 1262 return -EBADRQC; 1263 1264 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { 1265 ionic_admin_post(dev, &wr); 1266 return ionic_admin_wait(dev, &wr, 0); 1267 } 1268 1269 resp_buf = kzalloc_obj(*resp_buf); 1270 if (!resp_buf) 1271 return -ENOMEM; 1272 1273 resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, 1274 sizeof(*resp_buf), 1275 DMA_FROM_DEVICE); 1276 1277 rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); 1278 if (rc) { 1279 rc = -ENOMEM; 1280 goto err_dma; 1281 } 1282 1283 wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); 1284 wr.wqe.cmd.create_cq.udma_idx = udma_idx; 1285 wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma); 1286 wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); 1287 1288 ionic_admin_post(dev, &wr); 1289 rc = ionic_admin_wait(dev, &wr, 0); 1290 1291 dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), 1292 DMA_FROM_DEVICE); 1293 if (rc) 1294 goto err_dma; 1295 1296 if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) { 1297 rc = -EOPNOTSUPP; 1298 goto err_dma; 1299 } 1300 1301 cq->cqid = le32_to_cpu(resp_buf->id); 1302 if (out_udma_idx) 1303 *out_udma_idx = resp_buf->udma_idx; 1304 1305 err_dma: 1306 kfree(resp_buf); 1307 1308 return rc; 1309 } 1310 1311 static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid) 1312 { 1313 struct ionic_admin_wr wr = { 1314 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1315 .wqe = { 1316 .op = IONIC_V1_ADMIN_DESTROY_CQ, 1317 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN), 1318 .cmd.destroy_cq = { 1319 .cq_id = cpu_to_le32(cqid), 1320 }, 1321 } 1322 }; 1323 1324 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ) 1325 return -EBADRQC; 1326 1327 ionic_admin_post(dev, &wr); 1328 1329 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1330 } 1331 1332 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1333 struct uverbs_attr_bundle *attrs) 1334 { 1335 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1336 struct ib_udata *udata = &attrs->driver_udata; 1337 struct ionic_ctx *ctx = 1338 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 1339 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1340 struct ionic_tbl_buf buf = {}; 1341 struct ionic_cq_resp resp = {}; 1342 struct ionic_cq_req req; 1343 int udma_idx = 0, rc; 1344 1345 if (udata) { 1346 rc = ib_copy_validate_udata_in(udata, req, rsvd); 1347 if (rc) 1348 return rc; 1349 } 1350 1351 vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 1352 1353 if (udata) 1354 vcq->udma_mask &= req.udma_mask; 1355 1356 if (!vcq->udma_mask) { 1357 rc = -EINVAL; 1358 goto err_init; 1359 } 1360 1361 for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) { 1362 if (!(vcq->udma_mask & BIT(udma_idx))) 1363 continue; 1364 1365 rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, 1366 &req.cq[udma_idx], 1367 udma_idx); 1368 if (rc) 1369 goto err_init; 1370 1371 rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf, 1372 udma_idx, NULL); 1373 if (rc) 1374 goto err_cmd; 1375 1376 ionic_pgtbl_unbuf(dev, &buf); 1377 1378 rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata, 1379 &resp.cqid[udma_idx]); 1380 if (rc) 1381 goto err_post; 1382 } 1383 1384 vcq->ibcq.cqe = attr->cqe; 1385 1386 if (udata) { 1387 resp.udma_mask = vcq->udma_mask; 1388 1389 rc = ib_respond_udata(udata, resp); 1390 if (rc) 1391 goto err_resp; 1392 } 1393 1394 return 0; 1395 1396 err_resp: 1397 while (udma_idx) { 1398 --udma_idx; 1399 if (!(vcq->udma_mask & BIT(udma_idx))) 1400 continue; 1401 ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); 1402 err_post: 1403 ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1404 err_cmd: 1405 ionic_pgtbl_unbuf(dev, &buf); 1406 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1407 err_init: 1408 ; 1409 } 1410 1411 return rc; 1412 } 1413 1414 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1415 { 1416 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1417 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1418 int udma_idx, rc_tmp, rc = 0; 1419 1420 rc = ib_no_udata_io(udata); 1421 if (rc) 1422 return rc; 1423 1424 for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) { 1425 --udma_idx; 1426 1427 if (!(vcq->udma_mask & BIT(udma_idx))) 1428 continue; 1429 1430 /* 1431 * Untrack the CQ before releasing its hardware ID below, so a 1432 * concurrent create that gets the same ID reused by firmware 1433 * cannot have its fresh XArray entry erased by this destroy. 1434 */ 1435 ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); 1436 1437 rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1438 if (rc_tmp) { 1439 if (!rc) 1440 rc = rc_tmp; 1441 1442 continue; 1443 } 1444 1445 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1446 } 1447 1448 return rc; 1449 } 1450 1451 static bool pd_remote_privileged(struct ib_pd *pd) 1452 { 1453 return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; 1454 } 1455 1456 static int ionic_create_qp_cmd(struct ionic_ibdev *dev, 1457 struct ionic_pd *pd, 1458 struct ionic_cq *send_cq, 1459 struct ionic_cq *recv_cq, 1460 struct ionic_qp *qp, 1461 struct ionic_tbl_buf *sq_buf, 1462 struct ionic_tbl_buf *rq_buf, 1463 struct ib_qp_init_attr *attr, 1464 u32 ionic_flags) 1465 { 1466 const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); 1467 const u32 flags = to_ionic_qp_flags(0, 0, 1468 qp->sq_cmb & IONIC_CMB_ENABLE, 1469 qp->rq.cmb & IONIC_CMB_ENABLE, 1470 qp->sq_spec, qp->rq.spec, 1471 pd->flags & IONIC_QPF_PRIVILEGED, 1472 pd_remote_privileged(&pd->ibpd)); 1473 struct ionic_admin_wr wr = { 1474 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1475 .wqe = { 1476 .op = IONIC_V1_ADMIN_CREATE_QP, 1477 .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), 1478 .cmd.create_qp = { 1479 .pd_id = cpu_to_le32(pd->pdid), 1480 /* User-supplied ionic_flags are passed through to 1481 * firmware, which validates and rejects any 1482 * unsupported or unauthorized bits. 1483 */ 1484 .priv_flags = cpu_to_be32(flags | 1485 (ionic_flags & IONIC_QP_USER_FLAGS_MASK)), 1486 .type_state = to_ionic_qp_type(attr->qp_type), 1487 .dbid_flags = cpu_to_le16(dbid), 1488 .id_ver = cpu_to_le32(qp->qpid), 1489 } 1490 } 1491 }; 1492 1493 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP) 1494 return -EBADRQC; 1495 1496 if (qp->has_sq) { 1497 wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid); 1498 wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2; 1499 wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2; 1500 wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2; 1501 wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0); 1502 wr.wqe.cmd.create_qp.sq_map_count = 1503 cpu_to_le32(sq_buf->tbl_pages); 1504 wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); 1505 } 1506 1507 if (!qp->srq) { 1508 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); 1509 wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; 1510 wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; 1511 wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; 1512 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); 1513 wr.wqe.cmd.create_qp.rq_map_count = 1514 cpu_to_le32(rq_buf->tbl_pages); 1515 wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); 1516 } else { 1517 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid); 1518 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); 1519 } 1520 1521 ionic_admin_post(dev, &wr); 1522 1523 return ionic_admin_wait(dev, &wr, 0); 1524 } 1525 1526 static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, 1527 struct ionic_pd *pd, 1528 struct ionic_qp *qp, 1529 struct ib_qp_attr *attr, 1530 int mask) 1531 { 1532 const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, 1533 attr->en_sqd_async_notify, 1534 qp->sq_cmb & IONIC_CMB_ENABLE, 1535 qp->rq.cmb & IONIC_CMB_ENABLE, 1536 qp->sq_spec, qp->rq.spec, 1537 pd->flags & IONIC_QPF_PRIVILEGED, 1538 pd_remote_privileged(qp->ibqp.pd)); 1539 const u8 state = to_ionic_qp_modify_state(attr->qp_state, 1540 attr->cur_qp_state); 1541 struct ionic_admin_wr wr = { 1542 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1543 .wqe = { 1544 .op = IONIC_V1_ADMIN_MODIFY_QP, 1545 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN), 1546 .cmd.mod_qp = { 1547 .attr_mask = cpu_to_be32(mask), 1548 .access_flags = cpu_to_be16(flags), 1549 .rq_psn = cpu_to_le32(attr->rq_psn), 1550 .sq_psn = cpu_to_le32(attr->sq_psn), 1551 .rate_limit_kbps = 1552 cpu_to_le32(attr->rate_limit), 1553 .pmtu = (attr->path_mtu + 7), 1554 .retry = (attr->retry_cnt | 1555 (attr->rnr_retry << 4)), 1556 .rnr_timer = attr->min_rnr_timer, 1557 .retry_timeout = attr->timeout, 1558 .type_state = state, 1559 .id_ver = cpu_to_le32(qp->qpid), 1560 } 1561 } 1562 }; 1563 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1564 void *hdr_buf = NULL; 1565 dma_addr_t hdr_dma = 0; 1566 int rc, hdr_len = 0; 1567 u16 sport; 1568 1569 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP) 1570 return -EBADRQC; 1571 1572 if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) { 1573 /* Note, round up/down was already done for allocating 1574 * resources on the device. The allocation order is in cache 1575 * line size. We can't use the order of the resource 1576 * allocation to determine the order wqes here, because for 1577 * queue length <= one cache line it is not distinct. 1578 * 1579 * Therefore, order wqes is computed again here. 1580 * 1581 * Account for hole and round up to the next order. 1582 */ 1583 wr.wqe.cmd.mod_qp.rsq_depth = 1584 order_base_2(attr->max_dest_rd_atomic + 1); 1585 wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0); 1586 } 1587 1588 if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 1589 /* Account for hole and round down to the next order */ 1590 wr.wqe.cmd.mod_qp.rrq_depth = 1591 order_base_2(attr->max_rd_atomic + 2) - 1; 1592 wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0); 1593 } 1594 1595 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) 1596 wr.wqe.cmd.mod_qp.qkey_dest_qpn = 1597 cpu_to_le32(attr->dest_qp_num); 1598 else 1599 wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey); 1600 1601 if (mask & IB_QP_AV) { 1602 if (!qp->hdr) 1603 return -ENOMEM; 1604 1605 sport = rdma_get_udp_sport(grh->flow_label, 1606 qp->qpid, 1607 attr->dest_qp_num); 1608 1609 rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true); 1610 if (rc) 1611 return rc; 1612 1613 qp->sgid_index = grh->sgid_index; 1614 1615 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1616 if (!hdr_buf) 1617 return -ENOMEM; 1618 1619 hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf); 1620 hdr_len -= IB_BTH_BYTES; 1621 hdr_len -= IB_DETH_BYTES; 1622 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 1623 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 1624 hdr_buf, hdr_len, true); 1625 1626 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 1627 DMA_TO_DEVICE); 1628 1629 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1630 if (rc) 1631 goto err_dma; 1632 1633 if (qp->hdr->ipv4_present) { 1634 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1635 qp->hdr->vlan_present ? 1636 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP : 1637 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 1638 } else { 1639 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1640 qp->hdr->vlan_present ? 1641 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP : 1642 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 1643 } 1644 1645 wr.wqe.cmd.mod_qp.ah_id_len = 1646 cpu_to_le32(qp->ahid | (hdr_len << 24)); 1647 wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma); 1648 1649 wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl; 1650 wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2; 1651 } 1652 1653 ionic_admin_post(dev, &wr); 1654 1655 rc = ionic_admin_wait(dev, &wr, 0); 1656 1657 if (mask & IB_QP_AV) 1658 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 1659 DMA_TO_DEVICE); 1660 err_dma: 1661 if (mask & IB_QP_AV) 1662 kfree(hdr_buf); 1663 1664 return rc; 1665 } 1666 1667 static int ionic_query_qp_cmd(struct ionic_ibdev *dev, 1668 struct ionic_qp *qp, 1669 struct ib_qp_attr *attr, 1670 int mask) 1671 { 1672 struct ionic_admin_wr wr = { 1673 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1674 .wqe = { 1675 .op = IONIC_V1_ADMIN_QUERY_QP, 1676 .len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN), 1677 .cmd.query_qp = { 1678 .id_ver = cpu_to_le32(qp->qpid), 1679 }, 1680 } 1681 }; 1682 struct ionic_v1_admin_query_qp_sq *query_sqbuf; 1683 struct ionic_v1_admin_query_qp_rq *query_rqbuf; 1684 dma_addr_t query_sqdma; 1685 dma_addr_t query_rqdma; 1686 dma_addr_t hdr_dma = 0; 1687 void *hdr_buf = NULL; 1688 int flags, rc; 1689 1690 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP) 1691 return -EBADRQC; 1692 1693 if (qp->has_sq) { 1694 bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB); 1695 1696 attr->cap.max_send_sge = 1697 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 1698 qp->sq_spec, 1699 expdb); 1700 attr->cap.max_inline_data = 1701 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); 1702 } 1703 1704 if (!qp->srq) { 1705 attr->cap.max_recv_sge = 1706 ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, 1707 qp->rq.spec, 1708 qp->rq.cmb & IONIC_CMB_EXPDB); 1709 } 1710 1711 query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1712 if (!query_sqbuf) 1713 return -ENOMEM; 1714 1715 query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1716 if (!query_rqbuf) { 1717 rc = -ENOMEM; 1718 goto err_rqbuf; 1719 } 1720 1721 query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE, 1722 DMA_FROM_DEVICE); 1723 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma); 1724 if (rc) 1725 goto err_sqdma; 1726 1727 query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE, 1728 DMA_FROM_DEVICE); 1729 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma); 1730 if (rc) 1731 goto err_rqdma; 1732 1733 if (mask & IB_QP_AV) { 1734 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1735 if (!hdr_buf) { 1736 rc = -ENOMEM; 1737 goto err_hdrbuf; 1738 } 1739 1740 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, 1741 PAGE_SIZE, DMA_FROM_DEVICE); 1742 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1743 if (rc) 1744 goto err_hdrdma; 1745 } 1746 1747 wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma); 1748 wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma); 1749 wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma); 1750 wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid); 1751 1752 ionic_admin_post(dev, &wr); 1753 1754 rc = ionic_admin_wait(dev, &wr, 0); 1755 1756 if (rc) 1757 goto err_hdrdma; 1758 1759 flags = be16_to_cpu(query_sqbuf->access_perms_flags | 1760 query_rqbuf->access_perms_flags); 1761 1762 print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1763 query_sqbuf, sizeof(*query_sqbuf), true); 1764 print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1765 query_rqbuf, sizeof(*query_rqbuf), true); 1766 ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x", 1767 qp->qpid, query_rqbuf->state_pmtu, flags); 1768 1769 attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4); 1770 attr->cur_qp_state = attr->qp_state; 1771 attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7; 1772 attr->path_mig_state = IB_MIG_MIGRATED; 1773 attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn); 1774 attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn); 1775 attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn); 1776 attr->dest_qp_num = attr->qkey; 1777 attr->qp_access_flags = from_ionic_qp_flags(flags); 1778 attr->pkey_index = 0; 1779 attr->alt_pkey_index = 0; 1780 attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY); 1781 attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING); 1782 attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1; 1783 attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1; 1784 attr->min_rnr_timer = query_sqbuf->rnr_timer; 1785 attr->port_num = 0; 1786 attr->timeout = query_sqbuf->retry_timeout; 1787 attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf; 1788 attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4; 1789 attr->alt_port_num = 0; 1790 attr->alt_timeout = 0; 1791 attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps); 1792 1793 if (mask & IB_QP_AV) 1794 ionic_set_ah_attr(dev, &attr->ah_attr, 1795 qp->hdr, qp->sgid_index); 1796 1797 err_hdrdma: 1798 if (mask & IB_QP_AV) { 1799 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, 1800 PAGE_SIZE, DMA_FROM_DEVICE); 1801 kfree(hdr_buf); 1802 } 1803 err_hdrbuf: 1804 dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf), 1805 DMA_FROM_DEVICE); 1806 err_rqdma: 1807 dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf), 1808 DMA_FROM_DEVICE); 1809 err_sqdma: 1810 kfree(query_rqbuf); 1811 err_rqbuf: 1812 kfree(query_sqbuf); 1813 1814 return rc; 1815 } 1816 1817 static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid) 1818 { 1819 struct ionic_admin_wr wr = { 1820 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1821 .wqe = { 1822 .op = IONIC_V1_ADMIN_DESTROY_QP, 1823 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN), 1824 .cmd.destroy_qp = { 1825 .qp_id = cpu_to_le32(qpid), 1826 }, 1827 } 1828 }; 1829 1830 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP) 1831 return -EBADRQC; 1832 1833 ionic_admin_post(dev, &wr); 1834 1835 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1836 } 1837 1838 static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev, 1839 uint32_t wqe_size) 1840 { 1841 switch (wqe_size) { 1842 case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64; 1843 case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128; 1844 case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256; 1845 case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512; 1846 } 1847 1848 return false; 1849 } 1850 1851 static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev, 1852 struct ionic_qp *qp, 1853 struct ib_udata *udata, 1854 int max_data) 1855 { 1856 u8 expdb_stride_log2 = 0; 1857 bool expdb; 1858 int rc; 1859 1860 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1861 goto not_in_cmb; 1862 1863 if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) { 1864 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1865 goto not_in_cmb; 1866 1867 qp->sq_cmb &= IONIC_CMB_SUPPORTED; 1868 } 1869 1870 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) { 1871 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1872 goto not_in_cmb; 1873 1874 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1875 } 1876 1877 qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE); 1878 1879 if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER) 1880 goto not_in_cmb; 1881 1882 if (qp->sq_cmb & IONIC_CMB_EXPDB) 1883 expdb_stride_log2 = qp->sq.stride_log2; 1884 1885 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid, 1886 &qp->sq_cmb_addr, qp->sq_cmb_order, 1887 expdb_stride_log2, &expdb); 1888 if (rc) 1889 goto not_in_cmb; 1890 1891 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) { 1892 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1893 goto err_map; 1894 1895 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1896 } 1897 1898 return; 1899 1900 err_map: 1901 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1902 not_in_cmb: 1903 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1904 ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n"); 1905 1906 qp->sq_cmb = 0; 1907 qp->sq_cmb_order = IONIC_RES_INVALID; 1908 qp->sq_cmb_pgid = 0; 1909 qp->sq_cmb_addr = 0; 1910 } 1911 1912 static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev, 1913 struct ionic_ctx *ctx, 1914 struct ionic_qp *qp) 1915 { 1916 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1917 return; 1918 1919 if (ctx) 1920 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); 1921 1922 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1923 } 1924 1925 static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 1926 struct ionic_qp *qp, struct ionic_qdesc *sq, 1927 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 1928 int max_data, int sq_spec, struct ib_udata *udata) 1929 { 1930 u32 wqe_size; 1931 int rc = 0; 1932 1933 qp->sq_msn_prod = 0; 1934 qp->sq_msn_cons = 0; 1935 1936 if (!qp->has_sq) { 1937 if (buf) { 1938 buf->tbl_buf = NULL; 1939 buf->tbl_limit = 0; 1940 buf->tbl_pages = 0; 1941 } 1942 if (udata) 1943 rc = ionic_validate_qdesc_zero(sq); 1944 1945 return rc; 1946 } 1947 1948 rc = -EINVAL; 1949 1950 if (max_wr < 0 || max_wr > 0xffff) 1951 return rc; 1952 1953 if (max_sge < 1) 1954 return rc; 1955 1956 if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, 1957 qp->sq_cmb & 1958 IONIC_CMB_EXPDB), 1959 IONIC_SPEC_HIGH)) 1960 return rc; 1961 1962 if (max_data < 0) 1963 return rc; 1964 1965 if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride, 1966 qp->sq_cmb & IONIC_CMB_EXPDB)) 1967 return rc; 1968 1969 if (udata) { 1970 rc = ionic_validate_qdesc(sq); 1971 if (rc) 1972 return rc; 1973 1974 qp->sq_spec = sq_spec; 1975 1976 qp->sq.ptr = NULL; 1977 qp->sq.size = sq->size; 1978 qp->sq.mask = sq->mask; 1979 qp->sq.depth_log2 = sq->depth_log2; 1980 qp->sq.stride_log2 = sq->stride_log2; 1981 1982 qp->sq_meta = NULL; 1983 qp->sq_msn_idx = NULL; 1984 1985 qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); 1986 if (IS_ERR(qp->sq_umem)) 1987 return PTR_ERR(qp->sq_umem); 1988 } else { 1989 qp->sq_umem = NULL; 1990 1991 qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec); 1992 if (sq_spec && !qp->sq_spec) 1993 ibdev_dbg(&dev->ibdev, 1994 "init sq: max_sge %u disables spec\n", 1995 max_sge); 1996 1997 if (qp->sq_cmb & IONIC_CMB_EXPDB) { 1998 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 1999 qp->sq_spec, 2000 true); 2001 2002 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 2003 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 2004 } 2005 2006 if (!(qp->sq_cmb & IONIC_CMB_EXPDB)) 2007 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 2008 qp->sq_spec, 2009 false); 2010 2011 rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev, 2012 max_wr, wqe_size); 2013 if (rc) 2014 return rc; 2015 2016 ionic_queue_dbell_init(&qp->sq, qp->qpid); 2017 2018 qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1); 2019 if (!qp->sq_meta) { 2020 rc = -ENOMEM; 2021 goto err_sq_meta; 2022 } 2023 2024 qp->sq_msn_idx = kmalloc_objs(*qp->sq_msn_idx, 2025 (u32)qp->sq.mask + 1); 2026 if (!qp->sq_msn_idx) { 2027 rc = -ENOMEM; 2028 goto err_sq_msn; 2029 } 2030 } 2031 2032 ionic_qp_sq_init_cmb(dev, qp, udata, max_data); 2033 2034 if (qp->sq_cmb & IONIC_CMB_ENABLE) 2035 rc = ionic_pgtbl_init(dev, buf, NULL, 2036 (u64)qp->sq_cmb_pgid << PAGE_SHIFT, 2037 1, PAGE_SIZE); 2038 else 2039 rc = ionic_pgtbl_init(dev, buf, 2040 qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE); 2041 if (rc) 2042 goto err_sq_tbl; 2043 2044 return 0; 2045 2046 err_sq_tbl: 2047 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 2048 kfree(qp->sq_msn_idx); 2049 err_sq_msn: 2050 kfree(qp->sq_meta); 2051 err_sq_meta: 2052 if (qp->sq_umem) 2053 ib_umem_release(qp->sq_umem); 2054 else 2055 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 2056 return rc; 2057 } 2058 2059 static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, 2060 struct ionic_ctx *ctx, 2061 struct ionic_qp *qp) 2062 { 2063 if (!qp->has_sq) 2064 return; 2065 2066 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 2067 2068 kfree(qp->sq_msn_idx); 2069 kfree(qp->sq_meta); 2070 2071 if (qp->sq_umem) 2072 ib_umem_release(qp->sq_umem); 2073 else 2074 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 2075 } 2076 2077 static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev, 2078 struct ionic_ctx *ctx, 2079 struct ionic_rq *rq, 2080 u64 *cmb_offset) 2081 { 2082 bool wc; 2083 2084 /* set mapping by default to uncached for 2085 * expdb (to guarantee writes order) otherwise 2086 * writecombine, unless this default is 2087 * overridden by userspace 2088 */ 2089 if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2090 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2091 ibdev_dbg(&dev->ibdev, 2092 "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n"); 2093 rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2094 } 2095 2096 if (rq->cmb & IONIC_CMB_EXPDB) 2097 wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; 2098 else 2099 wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC; 2100 2101 /* let userspace know the mapping */ 2102 if (wc) 2103 rq->cmb |= IONIC_CMB_WC; 2104 else 2105 rq->cmb |= IONIC_CMB_UC; 2106 2107 rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size, 2108 PHYS_PFN(rq->cmb_addr), 2109 wc ? IONIC_MMAP_WC : 0, 2110 cmb_offset); 2111 } 2112 2113 static void ionic_rq_init_cmb(struct ionic_ibdev *dev, 2114 struct ionic_rq *rq, 2115 struct ib_udata *udata) 2116 { 2117 u8 expdb_stride_log2 = 0; 2118 bool expdb; 2119 int rc; 2120 2121 if (!(rq->cmb & IONIC_CMB_ENABLE)) 2122 goto not_in_cmb; 2123 2124 if (rq->cmb & ~IONIC_CMB_SUPPORTED) { 2125 if (rq->cmb & IONIC_CMB_REQUIRE) 2126 goto not_in_cmb; 2127 2128 rq->cmb &= IONIC_CMB_SUPPORTED; 2129 } 2130 2131 if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { 2132 if (rq->cmb & IONIC_CMB_REQUIRE) 2133 goto not_in_cmb; 2134 2135 rq->cmb &= ~IONIC_CMB_EXPDB; 2136 } 2137 2138 rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE); 2139 2140 if (rq->cmb_order >= IONIC_RQCMB_ORDER) 2141 goto not_in_cmb; 2142 2143 if (rq->cmb & IONIC_CMB_EXPDB) 2144 expdb_stride_log2 = rq->q.stride_log2; 2145 2146 rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid, 2147 &rq->cmb_addr, rq->cmb_order, 2148 expdb_stride_log2, &expdb); 2149 if (rc) 2150 goto not_in_cmb; 2151 2152 if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) { 2153 if (rq->cmb & IONIC_CMB_REQUIRE) 2154 goto err_map; 2155 2156 rq->cmb &= ~IONIC_CMB_EXPDB; 2157 } 2158 2159 return; 2160 2161 err_map: 2162 ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); 2163 not_in_cmb: 2164 if (rq->cmb & IONIC_CMB_REQUIRE) 2165 ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n"); 2166 2167 rq->cmb = 0; 2168 rq->cmb_order = IONIC_RES_INVALID; 2169 rq->cmb_pgid = 0; 2170 rq->cmb_addr = 0; 2171 } 2172 2173 static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev, 2174 struct ionic_ctx *ctx, 2175 struct ionic_rq *rq) 2176 { 2177 if (!(rq->cmb & IONIC_CMB_ENABLE)) 2178 return; 2179 2180 if (ctx) 2181 rdma_user_mmap_entry_remove(rq->mmap_cmb); 2182 2183 ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); 2184 } 2185 2186 static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 2187 struct ionic_rq *rq, struct ionic_qdesc *qdesc, 2188 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 2189 int rq_spec, struct ib_udata *udata) 2190 { 2191 int rc = 0, i; 2192 u32 wqe_size; 2193 2194 rc = -EINVAL; 2195 2196 if (max_wr < 0 || max_wr > 0xffff) 2197 return rc; 2198 2199 if (max_sge < 1) 2200 return rc; 2201 2202 if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), 2203 IONIC_SPEC_HIGH)) 2204 return rc; 2205 2206 if (udata) { 2207 rc = ionic_validate_qdesc(qdesc); 2208 if (rc) 2209 return rc; 2210 2211 rq->spec = rq_spec; 2212 2213 rq->q.ptr = NULL; 2214 rq->q.size = qdesc->size; 2215 rq->q.mask = qdesc->mask; 2216 rq->q.depth_log2 = qdesc->depth_log2; 2217 rq->q.stride_log2 = qdesc->stride_log2; 2218 2219 rq->meta = NULL; 2220 2221 rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0); 2222 if (IS_ERR(rq->umem)) 2223 return PTR_ERR(rq->umem); 2224 } else { 2225 rq->umem = NULL; 2226 2227 rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec); 2228 if (rq_spec && !rq->spec) 2229 ibdev_dbg(&dev->ibdev, 2230 "init rq: max_sge %u disables spec\n", 2231 max_sge); 2232 2233 if (rq->cmb & IONIC_CMB_EXPDB) { 2234 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2235 rq->spec, 2236 true); 2237 2238 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 2239 rq->cmb &= ~IONIC_CMB_EXPDB; 2240 } 2241 2242 if (!(rq->cmb & IONIC_CMB_EXPDB)) 2243 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2244 rq->spec, 2245 false); 2246 2247 rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev, 2248 max_wr, wqe_size); 2249 if (rc) 2250 return rc; 2251 2252 ionic_queue_dbell_init(&rq->q, rq->qid); 2253 2254 rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1); 2255 if (!rq->meta) { 2256 rc = -ENOMEM; 2257 goto err_rq_meta; 2258 } 2259 2260 for (i = 0; i < rq->q.mask; ++i) 2261 rq->meta[i].next = &rq->meta[i + 1]; 2262 rq->meta[i].next = IONIC_META_LAST; 2263 rq->meta_head = &rq->meta[0]; 2264 } 2265 2266 ionic_rq_init_cmb(dev, rq, udata); 2267 2268 if (rq->cmb & IONIC_CMB_ENABLE) 2269 rc = ionic_pgtbl_init(dev, buf, NULL, 2270 (u64)rq->cmb_pgid << PAGE_SHIFT, 2271 1, PAGE_SIZE); 2272 else 2273 rc = ionic_pgtbl_init(dev, buf, rq->umem, 2274 rq->q.dma, 1, PAGE_SIZE); 2275 if (rc) 2276 goto err_rq_tbl; 2277 2278 return 0; 2279 2280 err_rq_tbl: 2281 ionic_rq_destroy_cmb(dev, ctx, rq); 2282 kfree(rq->meta); 2283 err_rq_meta: 2284 if (rq->umem) 2285 ib_umem_release(rq->umem); 2286 else 2287 ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); 2288 return rc; 2289 } 2290 2291 static void ionic_rq_destroy(struct ionic_ibdev *dev, 2292 struct ionic_ctx *ctx, 2293 struct ionic_rq *rq) 2294 { 2295 ionic_rq_destroy_cmb(dev, ctx, rq); 2296 2297 kfree(rq->meta); 2298 2299 if (rq->umem) 2300 ib_umem_release(rq->umem); 2301 else 2302 ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); 2303 } 2304 2305 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 2306 struct ib_udata *udata) 2307 { 2308 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2309 struct ionic_tbl_buf sq_buf = {}, rq_buf = {}; 2310 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2311 struct ionic_qp *qp = to_ionic_qp(ibqp); 2312 struct ionic_ctx *ctx = 2313 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2314 struct ionic_qp_resp resp = {}; 2315 struct ionic_qp_req req = {}; 2316 struct ionic_cq *cq; 2317 u8 udma_mask; 2318 void *entry; 2319 int rc; 2320 2321 if (udata) { 2322 rc = ib_copy_validate_udata_in(udata, req, rsvd); 2323 if (rc) 2324 return rc; 2325 2326 if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK) 2327 return -EINVAL; 2328 2329 if (req.rsvd_pad) 2330 return -EINVAL; 2331 } else { 2332 req.sq_spec = IONIC_SPEC_HIGH; 2333 req.rq_spec = IONIC_SPEC_HIGH; 2334 } 2335 2336 if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD) 2337 return -EOPNOTSUPP; 2338 2339 qp->state = IB_QPS_RESET; 2340 2341 INIT_LIST_HEAD(&qp->cq_poll_sq); 2342 INIT_LIST_HEAD(&qp->cq_flush_sq); 2343 INIT_LIST_HEAD(&qp->rq.cq_flush); 2344 2345 spin_lock_init(&qp->sq_lock); 2346 spin_lock_init(&qp->rq.lock); 2347 2348 qp->has_sq = 1; 2349 2350 if (attr->srq) 2351 qp->srq = to_ionic_srq(attr->srq); 2352 else 2353 qp->srq = NULL; 2354 2355 if (attr->qp_type == IB_QPT_GSI) { 2356 rc = ionic_get_gsi_qpid(dev, &qp->qpid); 2357 } else { 2358 udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 2359 2360 if (attr->send_cq) 2361 udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask; 2362 2363 if (attr->recv_cq) 2364 udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask; 2365 2366 if (attr->srq) 2367 udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx); 2368 2369 if (udata && req.udma_mask) 2370 udma_mask &= req.udma_mask; 2371 2372 if (!udma_mask) 2373 return -EINVAL; 2374 2375 rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask); 2376 } 2377 if (rc) 2378 return rc; 2379 2380 qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2381 qp->has_ah = attr->qp_type == IB_QPT_RC; 2382 2383 if (qp->has_ah) { 2384 qp->hdr = kzalloc_obj(*qp->hdr); 2385 if (!qp->hdr) { 2386 rc = -ENOMEM; 2387 goto err_ah_alloc; 2388 } 2389 2390 rc = ionic_get_ahid(dev, &qp->ahid); 2391 if (rc) 2392 goto err_ahid; 2393 } 2394 2395 if (udata && (req.sq_cmb & IONIC_CMB_ENABLE)) 2396 qp->sq_cmb = req.sq_cmb; 2397 2398 rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, 2399 attr->cap.max_send_wr, attr->cap.max_send_sge, 2400 attr->cap.max_inline_data, req.sq_spec, udata); 2401 if (rc) 2402 goto err_sq; 2403 2404 if (qp->srq) { 2405 rq_buf.tbl_buf = NULL; 2406 rq_buf.tbl_limit = 0; 2407 rq_buf.tbl_pages = 0; 2408 2409 if (udata) 2410 rc = ionic_validate_qdesc_zero(&req.rq); 2411 } else { 2412 if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) 2413 qp->rq.cmb = req.rq_cmb; 2414 2415 /* for non-srq qps, rq qid is same as qpid */ 2416 qp->rq.qid = qp->qpid; 2417 rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, 2418 attr->cap.max_recv_wr, attr->cap.max_recv_sge, 2419 req.rq_spec, udata); 2420 } 2421 if (rc) 2422 goto err_rq; 2423 2424 rc = ionic_create_qp_cmd(dev, pd, 2425 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), 2426 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), 2427 qp, &sq_buf, &rq_buf, attr, req.ionic_flags); 2428 if (rc) 2429 goto err_cmd; 2430 2431 if (udata) { 2432 resp.qpid = qp->qpid; 2433 resp.udma_idx = qp->udma_idx; 2434 2435 if (qp->sq_cmb & IONIC_CMB_ENABLE) { 2436 bool wc; 2437 2438 if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2439 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2440 ibdev_dbg(&dev->ibdev, 2441 "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); 2442 qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2443 } 2444 2445 wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) 2446 != IONIC_CMB_UC; 2447 2448 /* let userspace know the mapping */ 2449 if (wc) 2450 qp->sq_cmb |= IONIC_CMB_WC; 2451 else 2452 qp->sq_cmb |= IONIC_CMB_UC; 2453 2454 qp->mmap_sq_cmb = 2455 ionic_mmap_entry_insert(ctx, 2456 qp->sq.size, 2457 PHYS_PFN(qp->sq_cmb_addr), 2458 wc ? IONIC_MMAP_WC : 0, 2459 &resp.sq_cmb_offset); 2460 if (!qp->mmap_sq_cmb) { 2461 rc = -ENOMEM; 2462 goto err_qp_cmd; 2463 } 2464 2465 resp.sq_cmb = qp->sq_cmb; 2466 } 2467 2468 if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) { 2469 ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset); 2470 2471 if (!qp->rq.mmap_cmb) { 2472 rc = -ENOMEM; 2473 goto err_qp_cmd; 2474 } 2475 2476 resp.rq_cmb = qp->rq.cmb; 2477 } 2478 2479 rc = ib_respond_udata(udata, resp); 2480 if (rc) 2481 goto err_qp_cmd; 2482 } 2483 2484 ionic_pgtbl_unbuf(dev, &rq_buf); 2485 ionic_pgtbl_unbuf(dev, &sq_buf); 2486 2487 qp->ibqp.qp_num = qp->qpid; 2488 2489 init_completion(&qp->qp_rel_comp); 2490 kref_init(&qp->qp_kref); 2491 2492 entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL); 2493 if (entry) { 2494 if (!xa_is_err(entry)) 2495 rc = -EINVAL; 2496 else 2497 rc = xa_err(entry); 2498 2499 goto err_qp_cmd; 2500 } 2501 2502 if (qp->has_sq) { 2503 cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx); 2504 2505 attr->cap.max_send_wr = qp->sq.mask; 2506 attr->cap.max_send_sge = 2507 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 2508 qp->sq_spec, 2509 qp->sq_cmb & IONIC_CMB_EXPDB); 2510 attr->cap.max_inline_data = 2511 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, 2512 qp->sq_cmb & 2513 IONIC_CMB_EXPDB); 2514 qp->sq_cqid = cq->cqid; 2515 } 2516 2517 if (!qp->srq) { 2518 cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); 2519 2520 attr->cap.max_recv_wr = qp->rq.q.mask; 2521 attr->cap.max_recv_sge = 2522 ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, 2523 qp->rq.spec, 2524 qp->rq.cmb & IONIC_CMB_EXPDB); 2525 qp->rq.cqid = cq->cqid; 2526 } 2527 2528 return 0; 2529 2530 err_qp_cmd: 2531 ionic_destroy_qp_cmd(dev, qp->qpid); 2532 err_cmd: 2533 if (!qp->srq) { 2534 ionic_pgtbl_unbuf(dev, &rq_buf); 2535 ionic_rq_destroy(dev, ctx, &qp->rq); 2536 } 2537 err_rq: 2538 ionic_pgtbl_unbuf(dev, &sq_buf); 2539 ionic_qp_sq_destroy(dev, ctx, qp); 2540 err_sq: 2541 if (qp->has_ah) 2542 ionic_put_ahid(dev, qp->ahid); 2543 err_ahid: 2544 kfree(qp->hdr); 2545 err_ah_alloc: 2546 ionic_put_qpid(dev, qp->qpid); 2547 return rc; 2548 } 2549 2550 void ionic_notify_flush_cq(struct ionic_cq *cq) 2551 { 2552 if (cq->flush && cq->vcq->ibcq.comp_handler) 2553 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, 2554 cq->vcq->ibcq.cq_context); 2555 } 2556 2557 static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp) 2558 { 2559 if (qp->ibqp.send_cq) 2560 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq, 2561 qp->udma_idx)); 2562 if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq) 2563 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq, 2564 qp->udma_idx)); 2565 } 2566 2567 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2568 { 2569 unsigned long irqflags; 2570 struct ionic_cq *cq; 2571 2572 if (qp->ibqp.send_cq) { 2573 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2574 2575 /* Hold the CQ lock and QP sq_lock to set up flush */ 2576 spin_lock_irqsave(&cq->lock, irqflags); 2577 spin_lock(&qp->sq_lock); 2578 qp->sq_flush = true; 2579 if (!ionic_queue_empty(&qp->sq)) { 2580 cq->flush = true; 2581 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); 2582 } 2583 spin_unlock(&qp->sq_lock); 2584 spin_unlock_irqrestore(&cq->lock, irqflags); 2585 } 2586 2587 if (qp->ibqp.recv_cq && !qp->srq) { 2588 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2589 2590 /* Hold the CQ lock and QP rq.lock to set up flush */ 2591 spin_lock_irqsave(&cq->lock, irqflags); 2592 spin_lock(&qp->rq.lock); 2593 qp->rq.flush = true; 2594 if (!ionic_queue_empty(&qp->rq.q)) { 2595 cq->flush = true; 2596 list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); 2597 } 2598 spin_unlock(&qp->rq.lock); 2599 spin_unlock_irqrestore(&cq->lock, irqflags); 2600 } 2601 } 2602 2603 static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid) 2604 { 2605 struct ionic_v1_cqe *qcqe; 2606 int prod, qtf, qid, type; 2607 bool color; 2608 2609 if (!cq->q.ptr) 2610 return; 2611 2612 color = cq->color; 2613 prod = cq->q.prod; 2614 qcqe = ionic_queue_at(&cq->q, prod); 2615 2616 while (color == ionic_v1_cqe_color(qcqe)) { 2617 qtf = ionic_v1_cqe_qtf(qcqe); 2618 qid = ionic_v1_cqe_qtf_qid(qtf); 2619 type = ionic_v1_cqe_qtf_type(qtf); 2620 2621 if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN) 2622 ionic_v1_cqe_clean(qcqe); 2623 2624 prod = ionic_queue_next(&cq->q, prod); 2625 qcqe = ionic_queue_at(&cq->q, prod); 2626 color = ionic_color_wrap(prod, color); 2627 } 2628 } 2629 2630 static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2631 { 2632 unsigned long irqflags; 2633 struct ionic_cq *cq; 2634 int i; 2635 2636 local_irq_save(irqflags); 2637 2638 if (qp->ibqp.send_cq) { 2639 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2640 spin_lock(&cq->lock); 2641 ionic_clean_cq(cq, qp->qpid); 2642 spin_unlock(&cq->lock); 2643 } 2644 2645 if (qp->ibqp.recv_cq) { 2646 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2647 spin_lock(&cq->lock); 2648 ionic_clean_cq(cq, qp->qpid); 2649 spin_unlock(&cq->lock); 2650 } 2651 2652 if (qp->has_sq) { 2653 spin_lock(&qp->sq_lock); 2654 qp->sq_flush = false; 2655 qp->sq_flush_rcvd = false; 2656 qp->sq_msn_prod = 0; 2657 qp->sq_msn_cons = 0; 2658 qp->sq.prod = 0; 2659 qp->sq.cons = 0; 2660 spin_unlock(&qp->sq_lock); 2661 } 2662 2663 if (!qp->srq) { 2664 spin_lock(&qp->rq.lock); 2665 qp->rq.flush = false; 2666 qp->rq.q.prod = 0; 2667 qp->rq.q.cons = 0; 2668 if (qp->rq.meta) { 2669 for (i = 0; i < qp->rq.q.mask; ++i) 2670 qp->rq.meta[i].next = &qp->rq.meta[i + 1]; 2671 qp->rq.meta[i].next = IONIC_META_LAST; 2672 } 2673 qp->rq.meta_head = &qp->rq.meta[0]; 2674 spin_unlock(&qp->rq.lock); 2675 } 2676 2677 local_irq_restore(irqflags); 2678 } 2679 2680 static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, 2681 enum ib_qp_state attr_state) 2682 { 2683 if (q_state == attr_state) 2684 return true; 2685 2686 if (attr_state == IB_QPS_ERR) 2687 return true; 2688 2689 if (attr_state == IB_QPS_SQE) 2690 return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD; 2691 2692 return false; 2693 } 2694 2695 static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, 2696 enum ib_qp_type type, enum ib_qp_state cur, 2697 enum ib_qp_state next) 2698 { 2699 if (!ext_mask) 2700 return true; 2701 2702 if (ext_mask & ~IB_QP_RATE_LIMIT) 2703 return false; 2704 2705 /* Rate limit is only supported for RC QPs during specific transitions */ 2706 return type == IB_QPT_RC && 2707 ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || 2708 (cur == IB_QPS_RTR && next == IB_QPS_RTS) || 2709 (cur == IB_QPS_RTS && next == IB_QPS_RTS)); 2710 } 2711 2712 static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, 2713 int mask) 2714 { 2715 enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ? 2716 attr->cur_qp_state : qp->state; 2717 enum ib_qp_state next_state = (mask & IB_QP_STATE) ? 2718 attr->qp_state : cur_state; 2719 2720 if ((mask & IB_QP_CUR_STATE) && 2721 !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) 2722 return -EINVAL; 2723 2724 if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || 2725 !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, 2726 qp->ibqp.qp_type, cur_state, next_state)) 2727 return -EINVAL; 2728 2729 /* unprivileged qp not allowed privileged qkey */ 2730 if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) && 2731 qp->ibqp.uobject) 2732 return -EPERM; 2733 2734 return 0; 2735 } 2736 2737 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 2738 struct ib_udata *udata) 2739 { 2740 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2741 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2742 struct ionic_qp *qp = to_ionic_qp(ibqp); 2743 int rc; 2744 2745 rc = ib_no_udata_io(udata); 2746 if (rc) 2747 return rc; 2748 2749 rc = ionic_check_modify_qp(qp, attr, mask); 2750 if (rc) 2751 return rc; 2752 2753 if (mask & IB_QP_CAP) 2754 return -EINVAL; 2755 2756 rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask); 2757 if (rc) 2758 return rc; 2759 2760 if (mask & IB_QP_STATE) { 2761 qp->state = attr->qp_state; 2762 2763 if (attr->qp_state == IB_QPS_ERR) { 2764 ionic_flush_qp(dev, qp); 2765 ionic_notify_qp_cqs(dev, qp); 2766 } else if (attr->qp_state == IB_QPS_RESET) { 2767 ionic_reset_qp(dev, qp); 2768 } 2769 } 2770 2771 return 0; 2772 } 2773 2774 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2775 int mask, struct ib_qp_init_attr *init_attr) 2776 { 2777 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2778 struct ionic_qp *qp = to_ionic_qp(ibqp); 2779 int rc; 2780 2781 memset(attr, 0, sizeof(*attr)); 2782 memset(init_attr, 0, sizeof(*init_attr)); 2783 2784 rc = ionic_query_qp_cmd(dev, qp, attr, mask); 2785 if (rc) 2786 return rc; 2787 2788 if (qp->has_sq) 2789 attr->cap.max_send_wr = qp->sq.mask; 2790 2791 if (!qp->srq) 2792 attr->cap.max_recv_wr = qp->rq.q.mask; 2793 2794 init_attr->event_handler = ibqp->event_handler; 2795 init_attr->qp_context = ibqp->qp_context; 2796 init_attr->send_cq = ibqp->send_cq; 2797 init_attr->recv_cq = ibqp->recv_cq; 2798 init_attr->srq = ibqp->srq; 2799 init_attr->xrcd = ibqp->xrcd; 2800 init_attr->cap = attr->cap; 2801 init_attr->sq_sig_type = qp->sig_all ? 2802 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2803 init_attr->qp_type = ibqp->qp_type; 2804 init_attr->create_flags = 0; 2805 init_attr->port_num = 0; 2806 init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl; 2807 init_attr->source_qpn = 0; 2808 2809 return rc; 2810 } 2811 2812 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2813 { 2814 struct ionic_ctx *ctx = 2815 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2816 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2817 struct ionic_qp *qp = to_ionic_qp(ibqp); 2818 unsigned long irqflags; 2819 struct ionic_cq *cq; 2820 int rc; 2821 2822 rc = ib_no_udata_io(udata); 2823 if (rc) 2824 return rc; 2825 2826 rc = ionic_destroy_qp_cmd(dev, qp->qpid); 2827 if (rc) 2828 return rc; 2829 2830 xa_erase_irq(&dev->qp_tbl, qp->qpid); 2831 2832 kref_put(&qp->qp_kref, ionic_qp_complete); 2833 wait_for_completion(&qp->qp_rel_comp); 2834 2835 if (qp->ibqp.send_cq) { 2836 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2837 spin_lock_irqsave(&cq->lock, irqflags); 2838 ionic_clean_cq(cq, qp->qpid); 2839 list_del(&qp->cq_poll_sq); 2840 list_del(&qp->cq_flush_sq); 2841 spin_unlock_irqrestore(&cq->lock, irqflags); 2842 } 2843 2844 if (qp->ibqp.recv_cq) { 2845 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2846 spin_lock_irqsave(&cq->lock, irqflags); 2847 ionic_clean_cq(cq, qp->qpid); 2848 list_del(&qp->rq.cq_flush); 2849 spin_unlock_irqrestore(&cq->lock, irqflags); 2850 } 2851 2852 if (!qp->srq) 2853 ionic_rq_destroy(dev, ctx, &qp->rq); 2854 2855 ionic_qp_sq_destroy(dev, ctx, qp); 2856 if (qp->has_ah) { 2857 ionic_put_ahid(dev, qp->ahid); 2858 kfree(qp->hdr); 2859 } 2860 ionic_put_qpid(dev, qp->qpid); 2861 2862 return 0; 2863 } 2864 2865 static int ionic_create_srq_cmd(struct ionic_ibdev *dev, 2866 struct ionic_ctx *ctx, 2867 struct ionic_srq *srq, 2868 struct ionic_pd *pd, 2869 struct ionic_tbl_buf *buf, 2870 u8 udma_mask) 2871 { 2872 const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0; 2873 struct ionic_admin_create_srq_resp *resp_buf; 2874 const u16 dbid = ionic_ctx_dbid(dev, ctx); 2875 struct ionic_admin_wr wr = { 2876 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2877 .wqe = { 2878 .op = IONIC_V1_ADMIN_CREATE_SRQ, 2879 .len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN), 2880 .cmd.create_srq = { 2881 .pd_id = cpu_to_le32(pd->pdid), 2882 .depth_log2 = srq->rq.q.depth_log2, 2883 .stride_log2 = srq->rq.q.stride_log2, 2884 .page_size_log2 = buf->page_size_log2, 2885 .map_count = cpu_to_le32(buf->tbl_pages), 2886 .dma_addr = ionic_pgtbl_dma(buf, 0), 2887 .dbid = cpu_to_le16(dbid), 2888 .qid = cpu_to_le32(srq->rq.qid), 2889 .low_wqes_limit = cpu_to_le16(srq->srq_limit), 2890 .udma_mask = udma_mask, 2891 .flags = cpu_to_le16(flags), 2892 } 2893 } 2894 }; 2895 dma_addr_t resp_buf_dma; 2896 int rc; 2897 2898 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ) 2899 return -EOPNOTSUPP; 2900 2901 resp_buf = kzalloc_obj(*resp_buf); 2902 if (!resp_buf) 2903 return -ENOMEM; 2904 2905 resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, 2906 sizeof(*resp_buf), 2907 DMA_FROM_DEVICE); 2908 2909 rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); 2910 if (rc) 2911 goto err_dma; 2912 2913 wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma); 2914 wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); 2915 2916 ionic_admin_post(dev, &wr); 2917 2918 rc = ionic_admin_wait(dev, &wr, 0); 2919 2920 dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), 2921 DMA_FROM_DEVICE); 2922 if (rc) 2923 goto err_dma; 2924 2925 if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) { 2926 rc = -EOPNOTSUPP; 2927 goto err_dma; 2928 } 2929 2930 srq->rq.qid = le32_to_cpu(resp_buf->id); 2931 srq->udma_idx = resp_buf->udma_idx; 2932 2933 err_dma: 2934 kfree(resp_buf); 2935 2936 return rc; 2937 } 2938 2939 static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid) 2940 { 2941 struct ionic_admin_wr wr = { 2942 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2943 .wqe = { 2944 .op = IONIC_V1_ADMIN_DESTROY_SRQ, 2945 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN), 2946 .cmd.destroy_srq = { 2947 .qid = cpu_to_le32(srqid), 2948 }, 2949 } 2950 }; 2951 2952 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ) 2953 return -EOPNOTSUPP; 2954 2955 ionic_admin_post(dev, &wr); 2956 2957 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 2958 } 2959 2960 static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid, 2961 u16 srq_limit) 2962 { 2963 struct ionic_admin_wr wr = { 2964 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2965 .wqe = { 2966 .op = IONIC_V1_ADMIN_MODIFY_SRQ, 2967 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN), 2968 .cmd.modify_srq = { 2969 .qid = cpu_to_le32(srqid), 2970 .low_wqes_limit = cpu_to_le16(srq_limit), 2971 }, 2972 } 2973 }; 2974 2975 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ) 2976 return -EOPNOTSUPP; 2977 2978 ionic_admin_post(dev, &wr); 2979 2980 return ionic_admin_wait(dev, &wr, 0); 2981 } 2982 2983 int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, 2984 struct ib_udata *udata) 2985 { 2986 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 2987 struct ionic_pd *pd = to_ionic_pd(ibsrq->pd); 2988 struct ionic_srq *srq = to_ionic_srq(ibsrq); 2989 struct ionic_ctx *ctx = 2990 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2991 struct ionic_srq_resp resp = {}; 2992 struct ionic_srq_req req = {}; 2993 struct ionic_tbl_buf buf = {}; 2994 u8 udma_mask; 2995 int rc; 2996 2997 if (!ctx) 2998 return -EOPNOTSUPP; 2999 3000 if (attr->srq_type != IB_SRQT_BASIC) 3001 return -EOPNOTSUPP; 3002 3003 if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES) 3004 return -EINVAL; 3005 3006 if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT) 3007 return -EINVAL; 3008 3009 udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 3010 if (udata) { 3011 rc = ib_copy_validate_udata_in(udata, req, rsvd); 3012 if (rc) 3013 return rc; 3014 3015 if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd))) 3016 return -EINVAL; 3017 3018 udma_mask &= req.udma_mask; 3019 } 3020 3021 if (!udma_mask) 3022 return -EINVAL; 3023 3024 if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) 3025 srq->rq.cmb = req.rq_cmb; 3026 3027 rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf, 3028 attr->attr.max_wr, attr->attr.max_sge, 3029 req.rq_spec, udata); 3030 if (rc) 3031 return rc; 3032 3033 srq->srq_limit = attr->attr.srq_limit; 3034 3035 rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask); 3036 if (rc) 3037 goto err_cmd; 3038 3039 ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid); 3040 3041 if (udata) { 3042 resp.srqid = srq->rq.qid; 3043 resp.udma_idx = srq->udma_idx; 3044 3045 if (srq->rq.cmb & IONIC_CMB_ENABLE) { 3046 ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset); 3047 if (!srq->rq.mmap_cmb) { 3048 rc = -ENOMEM; 3049 goto err_srq_cmd; 3050 } 3051 3052 resp.rq_cmb = srq->rq.cmb; 3053 } 3054 3055 rc = ib_respond_udata(udata, resp); 3056 if (rc) 3057 goto err_srq_cmd; 3058 } 3059 3060 kref_init(&srq->kref); 3061 init_completion(&srq->rel_comp); 3062 3063 rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL); 3064 if (rc) 3065 goto err_srq_cmd; 3066 3067 ionic_pgtbl_unbuf(dev, &buf); 3068 3069 attr->attr.max_wr = srq->rq.q.mask; 3070 3071 return 0; 3072 3073 err_srq_cmd: 3074 ionic_destroy_srq_cmd(dev, srq->rq.qid); 3075 err_cmd: 3076 ionic_pgtbl_unbuf(dev, &buf); 3077 ionic_rq_destroy(dev, ctx, &srq->rq); 3078 3079 return rc; 3080 } 3081 3082 int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 3083 { 3084 struct ionic_ctx *ctx = 3085 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 3086 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 3087 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3088 int rc; 3089 3090 if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) { 3091 kref_put(&srq->kref, ionic_srq_complete); 3092 wait_for_completion(&srq->rel_comp); 3093 } 3094 3095 rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); 3096 if (rc) 3097 return rc; 3098 3099 ionic_rq_destroy(dev, ctx, &srq->rq); 3100 3101 return 0; 3102 } 3103 3104 int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 3105 { 3106 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3107 3108 srq_attr->max_wr = srq->rq.q.mask; 3109 srq_attr->max_sge = 3110 ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec, 3111 srq->rq.cmb & IONIC_CMB_EXPDB); 3112 srq_attr->srq_limit = srq->srq_limit; 3113 3114 return 0; 3115 } 3116 3117 int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 3118 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 3119 { 3120 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 3121 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3122 int rc; 3123 3124 if (attr_mask & IB_SRQ_MAX_WR) 3125 return -EINVAL; 3126 3127 if (attr_mask & IB_SRQ_LIMIT) { 3128 if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT) 3129 return -EINVAL; 3130 3131 rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit); 3132 if (rc) 3133 return rc; 3134 3135 srq->srq_limit = attr->srq_limit; 3136 } 3137 3138 return 0; 3139 } 3140