1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 #include <rdma/ib_addr.h> 7 #include <rdma/ib_cache.h> 8 #include <rdma/ib_user_verbs.h> 9 #include <ionic_api.h> 10 11 #include "ionic_fw.h" 12 #include "ionic_ibdev.h" 13 14 #define ionic_set_ecn(tos) (((tos) | 2u) & ~1u) 15 #define ionic_clear_ecn(tos) ((tos) & ~3u) 16 17 static int ionic_validate_qdesc(struct ionic_qdesc *q) 18 { 19 if (!q->addr || !q->size || !q->mask || 20 !q->depth_log2 || !q->stride_log2) 21 return -EINVAL; 22 23 if (q->addr & (PAGE_SIZE - 1)) 24 return -EINVAL; 25 26 if (q->mask != BIT(q->depth_log2) - 1) 27 return -EINVAL; 28 29 if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2)) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx) 36 { 37 /* EQ per vector per udma, and the first eqs reserved for async events. 38 * The rest of the vectors can be requested for completions. 39 */ 40 u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1; 41 42 return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx; 43 } 44 45 static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx) 46 { 47 unsigned int size, base, bound; 48 int rc; 49 50 size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; 51 base = size * udma_idx; 52 bound = base + size; 53 54 rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound); 55 if (rc >= 0) { 56 /* cq_base is zero or a multiple of two queue groups */ 57 *cqid = dev->lif_cfg.cq_base + 58 ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift, 59 dev->half_cqid_udma_shift); 60 61 rc = 0; 62 } 63 64 return rc; 65 } 66 67 static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid) 68 { 69 u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base, 70 dev->lif_cfg.udma_qgrp_shift, 71 dev->half_cqid_udma_shift); 72 73 ionic_resid_put(&dev->inuse_cqid, bitid); 74 } 75 76 int ionic_create_cq_common(struct ionic_vcq *vcq, 77 struct ionic_tbl_buf *buf, 78 const struct ib_cq_init_attr *attr, 79 struct ionic_ctx *ctx, 80 struct ib_udata *udata, 81 struct ionic_qdesc *req_cq, 82 int udma_idx) 83 { 84 struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); 85 struct ionic_cq *cq = &vcq->cq[udma_idx]; 86 int rc; 87 88 cq->vcq = vcq; 89 90 if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) { 91 rc = -EINVAL; 92 goto err_args; 93 } 94 95 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { 96 rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); 97 if (rc) 98 goto err_args; 99 } 100 101 cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); 102 103 spin_lock_init(&cq->lock); 104 INIT_LIST_HEAD(&cq->poll_sq); 105 INIT_LIST_HEAD(&cq->flush_sq); 106 INIT_LIST_HEAD(&cq->flush_rq); 107 108 if (udata) { 109 rc = ionic_validate_qdesc(req_cq); 110 if (rc) 111 goto err_qdesc; 112 113 cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, 114 req_cq->addr, req_cq->size, 115 IB_ACCESS_LOCAL_WRITE); 116 if (IS_ERR(cq->umem)) { 117 rc = PTR_ERR(cq->umem); 118 goto err_qdesc; 119 } 120 121 cq->q.ptr = NULL; 122 cq->q.size = req_cq->size; 123 cq->q.mask = req_cq->mask; 124 cq->q.depth_log2 = req_cq->depth_log2; 125 cq->q.stride_log2 = req_cq->stride_log2; 126 } else { 127 rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, 128 attr->cqe + IONIC_CQ_GRACE, 129 sizeof(struct ionic_v1_cqe)); 130 if (rc) 131 goto err_q_init; 132 133 cq->color = true; 134 cq->credit = cq->q.mask; 135 } 136 137 rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE); 138 if (rc) 139 goto err_pgtbl_init; 140 141 init_completion(&cq->cq_rel_comp); 142 kref_init(&cq->cq_kref); 143 144 return 0; 145 146 err_pgtbl_init: 147 if (!udata) 148 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 149 err_q_init: 150 if (cq->umem) 151 ib_umem_release(cq->umem); 152 err_qdesc: 153 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) 154 ionic_put_cqid(dev, cq->cqid); 155 err_args: 156 cq->vcq = NULL; 157 158 return rc; 159 } 160 161 int ionic_post_create_cq_cmd(struct ionic_cq *cq, 162 struct ib_udata *udata, 163 __u32 *resp_cqid) 164 { 165 struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device); 166 167 if (udata && resp_cqid) 168 *resp_cqid = cq->cqid; 169 else 170 ionic_queue_dbell_init(&cq->q, cq->cqid); 171 172 return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); 173 } 174 175 void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq) 176 { 177 if (!cq->vcq) 178 return; 179 180 if (!xa_erase_irq(&dev->cq_tbl, cq->cqid)) 181 return; 182 183 kref_put(&cq->cq_kref, ionic_cq_complete); 184 wait_for_completion(&cq->cq_rel_comp); 185 } 186 187 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq) 188 { 189 if (!cq->vcq) 190 return; 191 192 if (cq->umem) 193 ib_umem_release(cq->umem); 194 else 195 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 196 197 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) 198 ionic_put_cqid(dev, cq->cqid); 199 200 cq->vcq = NULL; 201 } 202 203 static int ionic_validate_qdesc_zero(struct ionic_qdesc *q) 204 { 205 if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2) 206 return -EINVAL; 207 208 return 0; 209 } 210 211 static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid) 212 { 213 int rc; 214 215 rc = ionic_resid_get(&dev->inuse_pdid); 216 if (rc < 0) 217 return rc; 218 219 *pdid = rc; 220 return 0; 221 } 222 223 static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid) 224 { 225 int rc; 226 227 rc = ionic_resid_get(&dev->inuse_ahid); 228 if (rc < 0) 229 return rc; 230 231 *ahid = rc; 232 return 0; 233 } 234 235 static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid) 236 { 237 int rc; 238 239 /* wrap to 1, skip reserved lkey */ 240 rc = ionic_resid_get_shared(&dev->inuse_mrid, 1, 241 dev->inuse_mrid.inuse_size); 242 if (rc < 0) 243 return rc; 244 245 *mrid = ionic_mrid(rc, dev->next_mrkey++); 246 return 0; 247 } 248 249 static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid) 250 { 251 int rc = 0; 252 253 rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1); 254 if (rc < 0) 255 return rc; 256 257 *qpid = IB_QPT_GSI; 258 return 0; 259 } 260 261 static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid, 262 u8 *udma_idx, u8 udma_mask) 263 { 264 unsigned int size, base, bound; 265 int udma_i, udma_x, udma_ix; 266 int rc = -EINVAL; 267 268 udma_x = dev->next_qpid_udma_idx; 269 270 dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1; 271 272 for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) { 273 udma_ix = udma_i ^ udma_x; 274 275 if (!(udma_mask & BIT(udma_ix))) 276 continue; 277 278 size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count; 279 base = size * udma_ix; 280 bound = base + size; 281 282 /* skip reserved SMI and GSI qpids in group zero */ 283 if (!base) 284 base = 2; 285 286 rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound); 287 if (rc >= 0) { 288 *qpid = ionic_bitid_to_qid(rc, 289 dev->lif_cfg.udma_qgrp_shift, 290 dev->half_qpid_udma_shift); 291 *udma_idx = udma_ix; 292 293 rc = 0; 294 break; 295 } 296 } 297 298 return rc; 299 } 300 301 static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr) 302 { 303 int rc, dbpage_num; 304 305 /* wrap to 1, skip kernel reserved */ 306 rc = ionic_resid_get_shared(&dev->inuse_dbid, 1, 307 dev->inuse_dbid.inuse_size); 308 if (rc < 0) 309 return rc; 310 311 dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc; 312 *addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT); 313 314 *dbid = rc; 315 316 return 0; 317 } 318 319 static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid) 320 { 321 ionic_resid_put(&dev->inuse_pdid, pdid); 322 } 323 324 static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid) 325 { 326 ionic_resid_put(&dev->inuse_ahid, ahid); 327 } 328 329 static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid) 330 { 331 ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid)); 332 } 333 334 static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid) 335 { 336 u32 bitid = ionic_qid_to_bitid(qpid, 337 dev->lif_cfg.udma_qgrp_shift, 338 dev->half_qpid_udma_shift); 339 340 ionic_resid_put(&dev->inuse_qpid, bitid); 341 } 342 343 static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid) 344 { 345 ionic_resid_put(&dev->inuse_dbid, dbid); 346 } 347 348 static struct rdma_user_mmap_entry* 349 ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size, 350 unsigned long pfn, u8 mmap_flags, u64 *offset) 351 { 352 struct ionic_mmap_entry *entry; 353 int rc; 354 355 entry = kzalloc_obj(*entry); 356 if (!entry) 357 return NULL; 358 359 entry->size = size; 360 entry->pfn = pfn; 361 entry->mmap_flags = mmap_flags; 362 363 rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry, 364 entry->size); 365 if (rc) { 366 kfree(entry); 367 return NULL; 368 } 369 370 if (offset) 371 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 372 373 return &entry->rdma_entry; 374 } 375 376 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 377 { 378 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 379 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 380 struct ionic_ctx_resp resp = {}; 381 struct ionic_ctx_req req; 382 phys_addr_t db_phys = 0; 383 int rc; 384 385 rc = ib_copy_validate_udata_in(udata, req, rsvd); 386 if (rc) 387 return rc; 388 389 /* try to allocate dbid for user ctx */ 390 rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys); 391 if (rc < 0) 392 return rc; 393 394 ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid); 395 396 ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 397 PHYS_PFN(db_phys), 0, NULL); 398 if (!ctx->mmap_dbell) { 399 rc = -ENOMEM; 400 goto err_mmap_dbell; 401 } 402 403 if (dev->lif_cfg.phc_state) { 404 ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0, 405 IONIC_MMAP_PHC, 406 &resp.phc_offset); 407 if (!ctx->mmap_phc) { 408 rc = -ENOMEM; 409 goto err_mmap_phc; 410 } 411 } 412 413 resp.page_shift = PAGE_SHIFT; 414 415 resp.dbell_offset = db_phys & ~PAGE_MASK; 416 417 resp.version = dev->lif_cfg.rdma_version; 418 resp.qp_opcodes = dev->lif_cfg.qp_opcodes; 419 resp.admin_opcodes = dev->lif_cfg.admin_opcodes; 420 421 resp.sq_qtype = dev->lif_cfg.sq_qtype; 422 resp.rq_qtype = dev->lif_cfg.rq_qtype; 423 resp.cq_qtype = dev->lif_cfg.cq_qtype; 424 resp.admin_qtype = dev->lif_cfg.aq_qtype; 425 resp.max_stride = dev->lif_cfg.max_stride; 426 resp.max_spec = IONIC_SPEC_HIGH; 427 428 resp.udma_count = dev->lif_cfg.udma_count; 429 resp.expdb_mask = dev->lif_cfg.expdb_mask; 430 resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit; 431 resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS; 432 433 if (dev->lif_cfg.sq_expdb) 434 resp.expdb_qtypes |= IONIC_EXPDB_SQ; 435 if (dev->lif_cfg.rq_expdb) 436 resp.expdb_qtypes |= IONIC_EXPDB_RQ; 437 438 rc = ib_respond_udata(udata, resp); 439 if (rc) 440 goto err_resp; 441 442 return 0; 443 444 err_resp: 445 rdma_user_mmap_entry_remove(ctx->mmap_phc); 446 err_mmap_phc: 447 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 448 err_mmap_dbell: 449 ionic_put_dbid(dev, ctx->dbid); 450 451 return rc; 452 } 453 454 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx) 455 { 456 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 457 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 458 459 rdma_user_mmap_entry_remove(ctx->mmap_phc); 460 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 461 ionic_put_dbid(dev, ctx->dbid); 462 } 463 464 static int ionic_mmap_phc_state(struct ionic_ibdev *dev, 465 struct vm_area_struct *vma) 466 { 467 if (!(vma->vm_flags & VM_SHARED)) 468 return -EINVAL; 469 470 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 471 return -EPERM; 472 473 vm_flags_clear(vma, VM_MAYWRITE); 474 475 return vm_insert_page(vma, vma->vm_start, 476 virt_to_page(dev->lif_cfg.phc_state)); 477 } 478 479 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma) 480 { 481 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 482 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 483 struct rdma_user_mmap_entry *rdma_entry; 484 struct ionic_mmap_entry *ionic_entry; 485 int rc = 0; 486 487 rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma); 488 if (!rdma_entry) { 489 ibdev_dbg(&dev->ibdev, "not found %#lx\n", 490 vma->vm_pgoff << PAGE_SHIFT); 491 return -EINVAL; 492 } 493 494 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 495 rdma_entry); 496 497 if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) { 498 rc = ionic_mmap_phc_state(dev, vma); 499 rdma_user_mmap_entry_put(rdma_entry); 500 return rc; 501 } 502 503 ibdev_dbg(&dev->ibdev, "writecombine? %d\n", 504 ionic_entry->mmap_flags & IONIC_MMAP_WC); 505 if (ionic_entry->mmap_flags & IONIC_MMAP_WC) 506 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 507 else 508 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 509 510 ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n", 511 vma->vm_start, ionic_entry->pfn, ionic_entry->size); 512 rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn, 513 ionic_entry->size, vma->vm_page_prot, 514 rdma_entry); 515 if (rc) 516 ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc); 517 518 rdma_user_mmap_entry_put(rdma_entry); 519 return rc; 520 } 521 522 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 523 { 524 struct ionic_mmap_entry *ionic_entry; 525 526 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 527 rdma_entry); 528 kfree(ionic_entry); 529 } 530 531 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 532 { 533 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 534 struct ionic_pd *pd = to_ionic_pd(ibpd); 535 int rc; 536 537 rc = ib_no_udata_io(udata); 538 if (rc) 539 return rc; 540 541 return ionic_get_pdid(dev, &pd->pdid); 542 } 543 544 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 545 { 546 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 547 struct ionic_pd *pd = to_ionic_pd(ibpd); 548 int rc; 549 550 rc = ib_no_udata_io(udata); 551 if (rc) 552 return rc; 553 554 ionic_put_pdid(dev, pd->pdid); 555 556 return 0; 557 } 558 559 static int ionic_build_hdr(struct ionic_ibdev *dev, 560 struct ib_ud_header *hdr, 561 const struct rdma_ah_attr *attr, 562 u16 sport, bool want_ecn) 563 { 564 const struct ib_global_route *grh; 565 enum rdma_network_type net; 566 u8 smac[ETH_ALEN]; 567 u16 vlan; 568 int rc; 569 570 if (attr->ah_flags != IB_AH_GRH) 571 return -EINVAL; 572 if (attr->type != RDMA_AH_ATTR_TYPE_ROCE) 573 return -EINVAL; 574 575 grh = rdma_ah_read_grh(attr); 576 577 rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); 578 if (rc) 579 return rc; 580 581 net = rdma_gid_attr_network_type(grh->sgid_attr); 582 583 rc = ib_ud_header_init(0, /* no payload */ 584 0, /* no lrh */ 585 1, /* yes eth */ 586 vlan != 0xffff, 587 0, /* no grh */ 588 net == RDMA_NETWORK_IPV4 ? 4 : 6, 589 1, /* yes udp */ 590 0, /* no imm */ 591 hdr); 592 if (rc) 593 return rc; 594 595 ether_addr_copy(hdr->eth.smac_h, smac); 596 ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); 597 598 if (net == RDMA_NETWORK_IPV4) { 599 hdr->eth.type = cpu_to_be16(ETH_P_IP); 600 hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */ 601 hdr->ip4.ttl = grh->hop_limit; 602 hdr->ip4.tot_len = cpu_to_be16(0xffff); 603 hdr->ip4.saddr = 604 *(const __be32 *)(grh->sgid_attr->gid.raw + 12); 605 hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12); 606 607 if (want_ecn) 608 hdr->ip4.tos = ionic_set_ecn(grh->traffic_class); 609 else 610 hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class); 611 } else { 612 hdr->eth.type = cpu_to_be16(ETH_P_IPV6); 613 hdr->grh.flow_label = cpu_to_be32(grh->flow_label); 614 hdr->grh.hop_limit = grh->hop_limit; 615 hdr->grh.source_gid = grh->sgid_attr->gid; 616 hdr->grh.destination_gid = grh->dgid; 617 618 if (want_ecn) 619 hdr->grh.traffic_class = 620 ionic_set_ecn(grh->traffic_class); 621 else 622 hdr->grh.traffic_class = 623 ionic_clear_ecn(grh->traffic_class); 624 } 625 626 if (vlan != 0xffff) { 627 vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT; 628 hdr->vlan.tag = cpu_to_be16(vlan); 629 hdr->vlan.type = hdr->eth.type; 630 hdr->eth.type = cpu_to_be16(ETH_P_8021Q); 631 } 632 633 hdr->udp.sport = cpu_to_be16(sport); 634 hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT); 635 636 return 0; 637 } 638 639 static void ionic_set_ah_attr(struct ionic_ibdev *dev, 640 struct rdma_ah_attr *ah_attr, 641 struct ib_ud_header *hdr, 642 int sgid_index) 643 { 644 u32 flow_label; 645 u16 vlan = 0; 646 u8 tos, ttl; 647 648 if (hdr->vlan_present) 649 vlan = be16_to_cpu(hdr->vlan.tag); 650 651 if (hdr->ipv4_present) { 652 flow_label = 0; 653 ttl = hdr->ip4.ttl; 654 tos = hdr->ip4.tos; 655 *(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff); 656 *(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr; 657 } else { 658 flow_label = be32_to_cpu(hdr->grh.flow_label); 659 ttl = hdr->grh.hop_limit; 660 tos = hdr->grh.traffic_class; 661 } 662 663 memset(ah_attr, 0, sizeof(*ah_attr)); 664 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE; 665 if (hdr->eth_present) 666 ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h); 667 rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT); 668 rdma_ah_set_port_num(ah_attr, 1); 669 rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos); 670 rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid); 671 } 672 673 static int ionic_create_ah_cmd(struct ionic_ibdev *dev, 674 struct ionic_ah *ah, 675 struct ionic_pd *pd, 676 struct rdma_ah_attr *attr, 677 u32 flags) 678 { 679 struct ionic_admin_wr wr = { 680 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 681 .wqe = { 682 .op = IONIC_V1_ADMIN_CREATE_AH, 683 .len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN), 684 .cmd.create_ah = { 685 .pd_id = cpu_to_le32(pd->pdid), 686 .dbid_flags = cpu_to_le16(dev->lif_cfg.dbid), 687 .id_ver = cpu_to_le32(ah->ahid), 688 } 689 } 690 }; 691 enum ionic_admin_flags admin_flags = 0; 692 dma_addr_t hdr_dma = 0; 693 void *hdr_buf; 694 gfp_t gfp = GFP_ATOMIC; 695 int rc, hdr_len = 0; 696 697 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH) 698 return -EBADRQC; 699 700 if (flags & RDMA_CREATE_AH_SLEEPABLE) 701 gfp = GFP_KERNEL; 702 else 703 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 704 705 rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false); 706 if (rc) 707 return rc; 708 709 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) { 710 if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP)) 711 wr.wqe.cmd.create_ah.csum_profile = 712 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP; 713 else 714 wr.wqe.cmd.create_ah.csum_profile = 715 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP; 716 } else { 717 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP)) 718 wr.wqe.cmd.create_ah.csum_profile = 719 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 720 else 721 wr.wqe.cmd.create_ah.csum_profile = 722 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 723 } 724 725 ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index; 726 727 hdr_buf = kmalloc(PAGE_SIZE, gfp); 728 if (!hdr_buf) 729 return -ENOMEM; 730 731 hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf); 732 hdr_len -= IB_BTH_BYTES; 733 hdr_len -= IB_DETH_BYTES; 734 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 735 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 736 hdr_buf, hdr_len, true); 737 738 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 739 DMA_TO_DEVICE); 740 741 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 742 if (rc) 743 goto err_dma; 744 745 wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma); 746 wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len); 747 748 ionic_admin_post(dev, &wr); 749 rc = ionic_admin_wait(dev, &wr, admin_flags); 750 751 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 752 DMA_TO_DEVICE); 753 err_dma: 754 kfree(hdr_buf); 755 756 return rc; 757 } 758 759 static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags) 760 { 761 struct ionic_admin_wr wr = { 762 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 763 .wqe = { 764 .op = IONIC_V1_ADMIN_DESTROY_AH, 765 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN), 766 .cmd.destroy_ah = { 767 .ah_id = cpu_to_le32(ahid), 768 }, 769 } 770 }; 771 enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN; 772 773 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH) 774 return -EBADRQC; 775 776 if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) 777 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 778 779 ionic_admin_post(dev, &wr); 780 ionic_admin_wait(dev, &wr, admin_flags); 781 782 /* No host-memory resource is associated with ah, so it is ok 783 * to "succeed" and complete this destroy ah on the host. 784 */ 785 return 0; 786 } 787 788 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 789 struct ib_udata *udata) 790 { 791 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 792 struct rdma_ah_attr *attr = init_attr->ah_attr; 793 struct ionic_pd *pd = to_ionic_pd(ibah->pd); 794 struct ionic_ah *ah = to_ionic_ah(ibah); 795 struct ionic_ah_resp resp = {}; 796 u32 flags = init_attr->flags; 797 int rc; 798 799 rc = ib_is_udata_in_empty(udata); 800 if (rc) 801 return rc; 802 803 rc = ionic_get_ahid(dev, &ah->ahid); 804 if (rc) 805 return rc; 806 807 rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags); 808 if (rc) 809 goto err_cmd; 810 811 if (udata) { 812 resp.ahid = ah->ahid; 813 814 rc = ib_respond_udata(udata, resp); 815 if (rc) 816 goto err_resp; 817 } 818 819 return 0; 820 821 err_resp: 822 ionic_destroy_ah_cmd(dev, ah->ahid, flags); 823 err_cmd: 824 ionic_put_ahid(dev, ah->ahid); 825 return rc; 826 } 827 828 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 829 { 830 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 831 struct ionic_ah *ah = to_ionic_ah(ibah); 832 833 ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index); 834 835 return 0; 836 } 837 838 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags) 839 { 840 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 841 struct ionic_ah *ah = to_ionic_ah(ibah); 842 int rc; 843 844 rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags); 845 if (rc) 846 return rc; 847 848 ionic_put_ahid(dev, ah->ahid); 849 850 return 0; 851 } 852 853 static int ionic_create_mr_cmd(struct ionic_ibdev *dev, 854 struct ionic_pd *pd, 855 struct ionic_mr *mr, 856 u64 addr, 857 u64 length) 858 { 859 struct ionic_admin_wr wr = { 860 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 861 .wqe = { 862 .op = IONIC_V1_ADMIN_CREATE_MR, 863 .len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN), 864 .cmd.create_mr = { 865 .va = cpu_to_le64(addr), 866 .length = cpu_to_le64(length), 867 .pd_id = cpu_to_le32(pd->pdid), 868 .page_size_log2 = mr->buf.page_size_log2, 869 .tbl_index = cpu_to_le32(~0), 870 .map_count = cpu_to_le32(mr->buf.tbl_pages), 871 .dma_addr = ionic_pgtbl_dma(&mr->buf, addr), 872 .dbid_flags = cpu_to_le16(mr->flags), 873 .id_ver = cpu_to_le32(mr->mrid), 874 } 875 } 876 }; 877 int rc; 878 879 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR) 880 return -EBADRQC; 881 882 ionic_admin_post(dev, &wr); 883 rc = ionic_admin_wait(dev, &wr, 0); 884 if (!rc) 885 mr->created = true; 886 887 return rc; 888 } 889 890 static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid) 891 { 892 struct ionic_admin_wr wr = { 893 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 894 .wqe = { 895 .op = IONIC_V1_ADMIN_DESTROY_MR, 896 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN), 897 .cmd.destroy_mr = { 898 .mr_id = cpu_to_le32(mrid), 899 }, 900 } 901 }; 902 903 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR) 904 return -EBADRQC; 905 906 ionic_admin_post(dev, &wr); 907 908 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 909 } 910 911 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access) 912 { 913 struct ionic_pd *pd = to_ionic_pd(ibpd); 914 struct ionic_mr *mr; 915 916 mr = kzalloc_obj(*mr); 917 if (!mr) 918 return ERR_PTR(-ENOMEM); 919 920 mr->ibmr.lkey = IONIC_DMA_LKEY; 921 mr->ibmr.rkey = IONIC_DMA_RKEY; 922 923 if (pd) 924 pd->flags |= IONIC_QPF_PRIVILEGED; 925 926 return &mr->ibmr; 927 } 928 929 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 930 u64 addr, int access, struct ib_dmah *dmah, 931 struct ib_udata *udata) 932 { 933 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 934 struct ionic_pd *pd = to_ionic_pd(ibpd); 935 struct ionic_mr *mr; 936 unsigned long pg_sz; 937 int rc; 938 939 rc = ib_no_udata_io(udata); 940 if (rc) 941 return ERR_PTR(rc); 942 943 if (dmah) 944 return ERR_PTR(-EOPNOTSUPP); 945 946 mr = kzalloc_obj(*mr); 947 if (!mr) 948 return ERR_PTR(-ENOMEM); 949 950 rc = ionic_get_mrid(dev, &mr->mrid); 951 if (rc) 952 goto err_mrid; 953 954 mr->ibmr.lkey = mr->mrid; 955 mr->ibmr.rkey = mr->mrid; 956 mr->ibmr.iova = addr; 957 mr->ibmr.length = length; 958 959 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 960 961 mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); 962 if (IS_ERR(mr->umem)) { 963 rc = PTR_ERR(mr->umem); 964 goto err_umem; 965 } 966 967 pg_sz = ib_umem_find_best_pgsz(mr->umem, 968 dev->lif_cfg.page_size_supported, 969 addr); 970 if (!pg_sz) { 971 rc = -EINVAL; 972 goto err_pgtbl; 973 } 974 975 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 976 if (rc) 977 goto err_pgtbl; 978 979 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 980 if (rc) 981 goto err_cmd; 982 983 ionic_pgtbl_unbuf(dev, &mr->buf); 984 985 return &mr->ibmr; 986 987 err_cmd: 988 ionic_pgtbl_unbuf(dev, &mr->buf); 989 err_pgtbl: 990 ib_umem_release(mr->umem); 991 err_umem: 992 ionic_put_mrid(dev, mr->mrid); 993 err_mrid: 994 kfree(mr); 995 return ERR_PTR(rc); 996 } 997 998 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 999 u64 length, u64 addr, int fd, int access, 1000 struct ib_dmah *dmah, 1001 struct uverbs_attr_bundle *attrs) 1002 { 1003 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 1004 struct ionic_pd *pd = to_ionic_pd(ibpd); 1005 struct ib_umem_dmabuf *umem_dmabuf; 1006 struct ionic_mr *mr; 1007 u64 pg_sz; 1008 int rc; 1009 1010 if (dmah) 1011 return ERR_PTR(-EOPNOTSUPP); 1012 1013 mr = kzalloc_obj(*mr); 1014 if (!mr) 1015 return ERR_PTR(-ENOMEM); 1016 1017 rc = ionic_get_mrid(dev, &mr->mrid); 1018 if (rc) 1019 goto err_mrid; 1020 1021 mr->ibmr.lkey = mr->mrid; 1022 mr->ibmr.rkey = mr->mrid; 1023 mr->ibmr.iova = addr; 1024 mr->ibmr.length = length; 1025 1026 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 1027 1028 umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length, 1029 fd, access); 1030 if (IS_ERR(umem_dmabuf)) { 1031 rc = PTR_ERR(umem_dmabuf); 1032 goto err_umem; 1033 } 1034 1035 mr->umem = &umem_dmabuf->umem; 1036 1037 pg_sz = ib_umem_find_best_pgsz(mr->umem, 1038 dev->lif_cfg.page_size_supported, 1039 addr); 1040 if (!pg_sz) { 1041 rc = -EINVAL; 1042 goto err_pgtbl; 1043 } 1044 1045 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 1046 if (rc) 1047 goto err_pgtbl; 1048 1049 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 1050 if (rc) 1051 goto err_cmd; 1052 1053 ionic_pgtbl_unbuf(dev, &mr->buf); 1054 1055 return &mr->ibmr; 1056 1057 err_cmd: 1058 ionic_pgtbl_unbuf(dev, &mr->buf); 1059 err_pgtbl: 1060 ib_umem_release(mr->umem); 1061 err_umem: 1062 ionic_put_mrid(dev, mr->mrid); 1063 err_mrid: 1064 kfree(mr); 1065 return ERR_PTR(rc); 1066 } 1067 1068 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1069 { 1070 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1071 struct ionic_mr *mr = to_ionic_mr(ibmr); 1072 int rc; 1073 1074 rc = ib_no_udata_io(udata); 1075 if (rc) 1076 return rc; 1077 1078 if (!mr->ibmr.lkey) 1079 goto out; 1080 1081 if (mr->created) { 1082 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1083 if (rc) 1084 return rc; 1085 } 1086 1087 ionic_pgtbl_unbuf(dev, &mr->buf); 1088 1089 if (mr->umem) 1090 ib_umem_release(mr->umem); 1091 1092 ionic_put_mrid(dev, mr->mrid); 1093 1094 out: 1095 kfree(mr); 1096 1097 return 0; 1098 } 1099 1100 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 1101 u32 max_sg) 1102 { 1103 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 1104 struct ionic_pd *pd = to_ionic_pd(ibpd); 1105 struct ionic_mr *mr; 1106 int rc; 1107 1108 if (type != IB_MR_TYPE_MEM_REG) 1109 return ERR_PTR(-EINVAL); 1110 1111 mr = kzalloc_obj(*mr); 1112 if (!mr) 1113 return ERR_PTR(-ENOMEM); 1114 1115 rc = ionic_get_mrid(dev, &mr->mrid); 1116 if (rc) 1117 goto err_mrid; 1118 1119 mr->ibmr.lkey = mr->mrid; 1120 mr->ibmr.rkey = mr->mrid; 1121 1122 mr->flags = IONIC_MRF_PHYS_MR; 1123 1124 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE); 1125 if (rc) 1126 goto err_pgtbl; 1127 1128 mr->buf.tbl_pages = 0; 1129 1130 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1131 if (rc) 1132 goto err_cmd; 1133 1134 return &mr->ibmr; 1135 1136 err_cmd: 1137 ionic_pgtbl_unbuf(dev, &mr->buf); 1138 err_pgtbl: 1139 ionic_put_mrid(dev, mr->mrid); 1140 err_mrid: 1141 kfree(mr); 1142 return ERR_PTR(rc); 1143 } 1144 1145 static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma) 1146 { 1147 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1148 struct ionic_mr *mr = to_ionic_mr(ibmr); 1149 1150 ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma); 1151 return ionic_pgtbl_page(&mr->buf, dma); 1152 } 1153 1154 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1155 unsigned int *sg_offset) 1156 { 1157 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1158 struct ionic_mr *mr = to_ionic_mr(ibmr); 1159 int rc; 1160 1161 /* mr must be allocated using ib_alloc_mr() */ 1162 if (unlikely(!mr->buf.tbl_limit)) 1163 return -EINVAL; 1164 1165 mr->buf.tbl_pages = 0; 1166 1167 if (mr->buf.tbl_buf) 1168 dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1169 mr->buf.tbl_size, DMA_TO_DEVICE); 1170 1171 ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents); 1172 rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page); 1173 1174 mr->buf.page_size_log2 = order_base_2(ibmr->page_size); 1175 1176 if (mr->buf.tbl_buf) 1177 dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1178 mr->buf.tbl_size, DMA_TO_DEVICE); 1179 1180 return rc; 1181 } 1182 1183 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 1184 { 1185 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1186 struct ionic_pd *pd = to_ionic_pd(ibmw->pd); 1187 struct ionic_mr *mr = to_ionic_mw(ibmw); 1188 int rc; 1189 1190 rc = ib_no_udata_io(udata); 1191 if (rc) 1192 return rc; 1193 1194 rc = ionic_get_mrid(dev, &mr->mrid); 1195 if (rc) 1196 return rc; 1197 1198 mr->ibmw.rkey = mr->mrid; 1199 1200 if (mr->ibmw.type == IB_MW_TYPE_1) 1201 mr->flags = IONIC_MRF_MW_1; 1202 else 1203 mr->flags = IONIC_MRF_MW_2; 1204 1205 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1206 if (rc) 1207 goto err_cmd; 1208 1209 return 0; 1210 1211 err_cmd: 1212 ionic_put_mrid(dev, mr->mrid); 1213 return rc; 1214 } 1215 1216 int ionic_dealloc_mw(struct ib_mw *ibmw) 1217 { 1218 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1219 struct ionic_mr *mr = to_ionic_mw(ibmw); 1220 int rc; 1221 1222 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1223 if (rc) 1224 return rc; 1225 1226 ionic_put_mrid(dev, mr->mrid); 1227 1228 return 0; 1229 } 1230 1231 static int ionic_create_cq_cmd(struct ionic_ibdev *dev, 1232 struct ionic_ctx *ctx, 1233 struct ionic_cq *cq, 1234 struct ionic_tbl_buf *buf, 1235 int udma_idx, 1236 int *out_udma_idx) 1237 { 1238 struct ionic_admin_create_cq_resp *resp_buf; 1239 const u16 dbid = ionic_ctx_dbid(dev, ctx); 1240 struct ionic_admin_wr wr = { 1241 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1242 .wqe = { 1243 .op = IONIC_V1_ADMIN_CREATE_CQ, 1244 .len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN), 1245 .cmd.create_cq = { 1246 .eq_id = cpu_to_le32(cq->eqid), 1247 .depth_log2 = cq->q.depth_log2, 1248 .stride_log2 = cq->q.stride_log2, 1249 .page_size_log2 = buf->page_size_log2, 1250 .tbl_index = cpu_to_le32(~0), 1251 .map_count = cpu_to_le32(buf->tbl_pages), 1252 .dma_addr = ionic_pgtbl_dma(buf, 0), 1253 .dbid_flags = cpu_to_le16(dbid), 1254 .id_ver = cpu_to_le32(cq->cqid), 1255 } 1256 } 1257 }; 1258 dma_addr_t resp_buf_dma; 1259 int rc; 1260 1261 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) 1262 return -EBADRQC; 1263 1264 if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { 1265 ionic_admin_post(dev, &wr); 1266 return ionic_admin_wait(dev, &wr, 0); 1267 } 1268 1269 resp_buf = kzalloc_obj(*resp_buf); 1270 if (!resp_buf) 1271 return -ENOMEM; 1272 1273 resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, 1274 sizeof(*resp_buf), 1275 DMA_FROM_DEVICE); 1276 1277 rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); 1278 if (rc) { 1279 rc = -ENOMEM; 1280 goto err_dma; 1281 } 1282 1283 wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); 1284 wr.wqe.cmd.create_cq.udma_idx = udma_idx; 1285 wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma); 1286 wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); 1287 1288 ionic_admin_post(dev, &wr); 1289 rc = ionic_admin_wait(dev, &wr, 0); 1290 1291 dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), 1292 DMA_FROM_DEVICE); 1293 if (rc) 1294 goto err_dma; 1295 1296 if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) { 1297 rc = -EOPNOTSUPP; 1298 goto err_dma; 1299 } 1300 1301 cq->cqid = le32_to_cpu(resp_buf->id); 1302 if (out_udma_idx) 1303 *out_udma_idx = resp_buf->udma_idx; 1304 1305 err_dma: 1306 kfree(resp_buf); 1307 1308 return rc; 1309 } 1310 1311 static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid) 1312 { 1313 struct ionic_admin_wr wr = { 1314 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1315 .wqe = { 1316 .op = IONIC_V1_ADMIN_DESTROY_CQ, 1317 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN), 1318 .cmd.destroy_cq = { 1319 .cq_id = cpu_to_le32(cqid), 1320 }, 1321 } 1322 }; 1323 1324 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ) 1325 return -EBADRQC; 1326 1327 ionic_admin_post(dev, &wr); 1328 1329 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1330 } 1331 1332 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1333 struct uverbs_attr_bundle *attrs) 1334 { 1335 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1336 struct ib_udata *udata = &attrs->driver_udata; 1337 struct ionic_ctx *ctx = 1338 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 1339 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1340 struct ionic_tbl_buf buf = {}; 1341 struct ionic_cq_resp resp = {}; 1342 struct ionic_cq_req req; 1343 int udma_idx = 0, rc; 1344 1345 if (udata) { 1346 rc = ib_copy_validate_udata_in(udata, req, rsvd); 1347 if (rc) 1348 return rc; 1349 } 1350 1351 vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 1352 1353 if (udata) 1354 vcq->udma_mask &= req.udma_mask; 1355 1356 if (!vcq->udma_mask) { 1357 rc = -EINVAL; 1358 goto err_init; 1359 } 1360 1361 for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) { 1362 if (!(vcq->udma_mask & BIT(udma_idx))) 1363 continue; 1364 1365 rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, 1366 &req.cq[udma_idx], 1367 udma_idx); 1368 if (rc) 1369 goto err_init; 1370 1371 rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf, 1372 udma_idx, NULL); 1373 if (rc) 1374 goto err_cmd; 1375 1376 ionic_pgtbl_unbuf(dev, &buf); 1377 1378 rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata, 1379 &resp.cqid[udma_idx]); 1380 if (rc) 1381 goto err_post; 1382 } 1383 1384 vcq->ibcq.cqe = attr->cqe; 1385 1386 if (udata) { 1387 resp.udma_mask = vcq->udma_mask; 1388 1389 rc = ib_respond_udata(udata, resp); 1390 if (rc) 1391 goto err_resp; 1392 } 1393 1394 return 0; 1395 1396 err_resp: 1397 while (udma_idx) { 1398 --udma_idx; 1399 if (!(vcq->udma_mask & BIT(udma_idx))) 1400 continue; 1401 ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); 1402 err_post: 1403 ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1404 err_cmd: 1405 ionic_pgtbl_unbuf(dev, &buf); 1406 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1407 err_init: 1408 ; 1409 } 1410 1411 return rc; 1412 } 1413 1414 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1415 { 1416 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1417 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1418 int udma_idx, rc_tmp, rc = 0; 1419 1420 rc = ib_no_udata_io(udata); 1421 if (rc) 1422 return rc; 1423 1424 for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) { 1425 --udma_idx; 1426 1427 if (!(vcq->udma_mask & BIT(udma_idx))) 1428 continue; 1429 1430 /* 1431 * Untrack the CQ before releasing its hardware ID below, so a 1432 * concurrent create that gets the same ID reused by firmware 1433 * cannot have its fresh XArray entry erased by this destroy. 1434 */ 1435 ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); 1436 1437 rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1438 if (rc_tmp) { 1439 if (!rc) 1440 rc = rc_tmp; 1441 1442 continue; 1443 } 1444 1445 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1446 } 1447 1448 return rc; 1449 } 1450 1451 static bool pd_remote_privileged(struct ib_pd *pd) 1452 { 1453 return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; 1454 } 1455 1456 static int ionic_create_qp_cmd(struct ionic_ibdev *dev, 1457 struct ionic_pd *pd, 1458 struct ionic_cq *send_cq, 1459 struct ionic_cq *recv_cq, 1460 struct ionic_qp *qp, 1461 struct ionic_tbl_buf *sq_buf, 1462 struct ionic_tbl_buf *rq_buf, 1463 struct ib_qp_init_attr *attr, 1464 u32 ionic_flags) 1465 { 1466 const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); 1467 const u32 flags = to_ionic_qp_flags(0, 0, 1468 qp->sq_cmb & IONIC_CMB_ENABLE, 1469 qp->rq.cmb & IONIC_CMB_ENABLE, 1470 qp->sq_spec, qp->rq.spec, 1471 pd->flags & IONIC_QPF_PRIVILEGED, 1472 pd_remote_privileged(&pd->ibpd)); 1473 struct ionic_admin_wr wr = { 1474 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1475 .wqe = { 1476 .op = IONIC_V1_ADMIN_CREATE_QP, 1477 .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), 1478 .cmd.create_qp = { 1479 .pd_id = cpu_to_le32(pd->pdid), 1480 /* User-supplied ionic_flags are passed through to 1481 * firmware, which validates and rejects any 1482 * unsupported or unauthorized bits. 1483 */ 1484 .priv_flags = cpu_to_be32(flags | 1485 (ionic_flags & IONIC_QP_USER_FLAGS_MASK)), 1486 .type_state = to_ionic_qp_type(attr->qp_type), 1487 .dbid_flags = cpu_to_le16(dbid), 1488 .id_ver = cpu_to_le32(qp->qpid), 1489 } 1490 } 1491 }; 1492 1493 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP) 1494 return -EBADRQC; 1495 1496 if (qp->has_sq) { 1497 wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid); 1498 wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2; 1499 wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2; 1500 wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2; 1501 wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0); 1502 wr.wqe.cmd.create_qp.sq_map_count = 1503 cpu_to_le32(sq_buf->tbl_pages); 1504 wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); 1505 } 1506 1507 if (!qp->srq) { 1508 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); 1509 wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; 1510 wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; 1511 wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; 1512 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); 1513 wr.wqe.cmd.create_qp.rq_map_count = 1514 cpu_to_le32(rq_buf->tbl_pages); 1515 wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); 1516 } else { 1517 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid); 1518 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); 1519 } 1520 1521 ionic_admin_post(dev, &wr); 1522 1523 return ionic_admin_wait(dev, &wr, 0); 1524 } 1525 1526 static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, 1527 struct ionic_pd *pd, 1528 struct ionic_qp *qp, 1529 struct ib_qp_attr *attr, 1530 int mask) 1531 { 1532 const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, 1533 attr->en_sqd_async_notify, 1534 qp->sq_cmb & IONIC_CMB_ENABLE, 1535 qp->rq.cmb & IONIC_CMB_ENABLE, 1536 qp->sq_spec, qp->rq.spec, 1537 pd->flags & IONIC_QPF_PRIVILEGED, 1538 pd_remote_privileged(qp->ibqp.pd)); 1539 const u8 state = to_ionic_qp_modify_state(attr->qp_state, 1540 attr->cur_qp_state); 1541 struct ionic_admin_wr wr = { 1542 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1543 .wqe = { 1544 .op = IONIC_V1_ADMIN_MODIFY_QP, 1545 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN), 1546 .cmd.mod_qp = { 1547 .attr_mask = cpu_to_be32(mask), 1548 .access_flags = cpu_to_be16(flags), 1549 .rq_psn = cpu_to_le32(attr->rq_psn), 1550 .sq_psn = cpu_to_le32(attr->sq_psn), 1551 .rate_limit_kbps = 1552 cpu_to_le32(attr->rate_limit), 1553 .pmtu = (attr->path_mtu + 7), 1554 .retry = (attr->retry_cnt | 1555 (attr->rnr_retry << 4)), 1556 .rnr_timer = attr->min_rnr_timer, 1557 .retry_timeout = attr->timeout, 1558 .type_state = state, 1559 .id_ver = cpu_to_le32(qp->qpid), 1560 } 1561 } 1562 }; 1563 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1564 void *hdr_buf = NULL; 1565 dma_addr_t hdr_dma = 0; 1566 int rc, hdr_len = 0; 1567 u16 sport; 1568 1569 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP) 1570 return -EBADRQC; 1571 1572 if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) { 1573 /* Note, round up/down was already done for allocating 1574 * resources on the device. The allocation order is in cache 1575 * line size. We can't use the order of the resource 1576 * allocation to determine the order wqes here, because for 1577 * queue length <= one cache line it is not distinct. 1578 * 1579 * Therefore, order wqes is computed again here. 1580 * 1581 * Account for hole and round up to the next order. 1582 */ 1583 wr.wqe.cmd.mod_qp.rsq_depth = 1584 order_base_2(attr->max_dest_rd_atomic + 1); 1585 wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0); 1586 } 1587 1588 if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 1589 /* Account for hole and round down to the next order */ 1590 wr.wqe.cmd.mod_qp.rrq_depth = 1591 order_base_2(attr->max_rd_atomic + 2) - 1; 1592 wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0); 1593 } 1594 1595 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) 1596 wr.wqe.cmd.mod_qp.qkey_dest_qpn = 1597 cpu_to_le32(attr->dest_qp_num); 1598 else 1599 wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey); 1600 1601 if (mask & IB_QP_AV) { 1602 if (!qp->hdr) 1603 return -ENOMEM; 1604 1605 sport = rdma_get_udp_sport(grh->flow_label, 1606 qp->qpid, 1607 attr->dest_qp_num); 1608 1609 rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true); 1610 if (rc) 1611 return rc; 1612 1613 qp->sgid_index = grh->sgid_index; 1614 1615 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1616 if (!hdr_buf) 1617 return -ENOMEM; 1618 1619 hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf); 1620 hdr_len -= IB_BTH_BYTES; 1621 hdr_len -= IB_DETH_BYTES; 1622 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 1623 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 1624 hdr_buf, hdr_len, true); 1625 1626 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 1627 DMA_TO_DEVICE); 1628 1629 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1630 if (rc) 1631 goto err_dma; 1632 1633 if (qp->hdr->ipv4_present) { 1634 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1635 qp->hdr->vlan_present ? 1636 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP : 1637 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 1638 } else { 1639 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1640 qp->hdr->vlan_present ? 1641 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP : 1642 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 1643 } 1644 1645 wr.wqe.cmd.mod_qp.ah_id_len = 1646 cpu_to_le32(qp->ahid | (hdr_len << 24)); 1647 wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma); 1648 1649 wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl; 1650 wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2; 1651 } 1652 1653 ionic_admin_post(dev, &wr); 1654 1655 rc = ionic_admin_wait(dev, &wr, 0); 1656 1657 if (mask & IB_QP_AV) 1658 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 1659 DMA_TO_DEVICE); 1660 err_dma: 1661 if (mask & IB_QP_AV) 1662 kfree(hdr_buf); 1663 1664 return rc; 1665 } 1666 1667 static int ionic_query_qp_cmd(struct ionic_ibdev *dev, 1668 struct ionic_qp *qp, 1669 struct ib_qp_attr *attr, 1670 int mask) 1671 { 1672 struct ionic_admin_wr wr = { 1673 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1674 .wqe = { 1675 .op = IONIC_V1_ADMIN_QUERY_QP, 1676 .len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN), 1677 .cmd.query_qp = { 1678 .id_ver = cpu_to_le32(qp->qpid), 1679 }, 1680 } 1681 }; 1682 struct ionic_v1_admin_query_qp_sq *query_sqbuf; 1683 struct ionic_v1_admin_query_qp_rq *query_rqbuf; 1684 dma_addr_t query_sqdma; 1685 dma_addr_t query_rqdma; 1686 dma_addr_t hdr_dma = 0; 1687 void *hdr_buf = NULL; 1688 int flags, rc; 1689 1690 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP) 1691 return -EBADRQC; 1692 1693 if (qp->has_sq) { 1694 bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB); 1695 1696 attr->cap.max_send_sge = 1697 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 1698 qp->sq_spec, 1699 expdb); 1700 attr->cap.max_inline_data = 1701 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); 1702 } 1703 1704 if (!qp->srq) { 1705 attr->cap.max_recv_sge = 1706 ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, 1707 qp->rq.spec, 1708 qp->rq.cmb & IONIC_CMB_EXPDB); 1709 } 1710 1711 query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1712 if (!query_sqbuf) 1713 return -ENOMEM; 1714 1715 query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1716 if (!query_rqbuf) { 1717 rc = -ENOMEM; 1718 goto err_rqbuf; 1719 } 1720 1721 query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE, 1722 DMA_FROM_DEVICE); 1723 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma); 1724 if (rc) 1725 goto err_sqdma; 1726 1727 query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE, 1728 DMA_FROM_DEVICE); 1729 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma); 1730 if (rc) 1731 goto err_rqdma; 1732 1733 if (mask & IB_QP_AV) { 1734 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1735 if (!hdr_buf) { 1736 rc = -ENOMEM; 1737 goto err_hdrbuf; 1738 } 1739 1740 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, 1741 PAGE_SIZE, DMA_FROM_DEVICE); 1742 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1743 if (rc) 1744 goto err_hdrdma; 1745 } 1746 1747 wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma); 1748 wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma); 1749 wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma); 1750 wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid); 1751 1752 ionic_admin_post(dev, &wr); 1753 1754 rc = ionic_admin_wait(dev, &wr, 0); 1755 1756 if (rc) 1757 goto err_hdrdma; 1758 1759 flags = be16_to_cpu(query_sqbuf->access_perms_flags | 1760 query_rqbuf->access_perms_flags); 1761 1762 print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1763 query_sqbuf, sizeof(*query_sqbuf), true); 1764 print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1765 query_rqbuf, sizeof(*query_rqbuf), true); 1766 ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x", 1767 qp->qpid, query_rqbuf->state_pmtu, flags); 1768 1769 attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4); 1770 attr->cur_qp_state = attr->qp_state; 1771 attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7; 1772 attr->path_mig_state = IB_MIG_MIGRATED; 1773 attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn); 1774 attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn); 1775 attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn); 1776 attr->dest_qp_num = attr->qkey; 1777 attr->qp_access_flags = from_ionic_qp_flags(flags); 1778 attr->pkey_index = 0; 1779 attr->alt_pkey_index = 0; 1780 attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY); 1781 attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING); 1782 attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1; 1783 attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1; 1784 attr->min_rnr_timer = query_sqbuf->rnr_timer; 1785 attr->port_num = 0; 1786 attr->timeout = query_sqbuf->retry_timeout; 1787 attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf; 1788 attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4; 1789 attr->alt_port_num = 0; 1790 attr->alt_timeout = 0; 1791 attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps); 1792 1793 if (mask & IB_QP_AV) 1794 ionic_set_ah_attr(dev, &attr->ah_attr, 1795 qp->hdr, qp->sgid_index); 1796 1797 err_hdrdma: 1798 if (mask & IB_QP_AV) { 1799 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, 1800 PAGE_SIZE, DMA_FROM_DEVICE); 1801 kfree(hdr_buf); 1802 } 1803 err_hdrbuf: 1804 dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf), 1805 DMA_FROM_DEVICE); 1806 err_rqdma: 1807 dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf), 1808 DMA_FROM_DEVICE); 1809 err_sqdma: 1810 kfree(query_rqbuf); 1811 err_rqbuf: 1812 kfree(query_sqbuf); 1813 1814 return rc; 1815 } 1816 1817 static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid) 1818 { 1819 struct ionic_admin_wr wr = { 1820 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1821 .wqe = { 1822 .op = IONIC_V1_ADMIN_DESTROY_QP, 1823 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN), 1824 .cmd.destroy_qp = { 1825 .qp_id = cpu_to_le32(qpid), 1826 }, 1827 } 1828 }; 1829 1830 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP) 1831 return -EBADRQC; 1832 1833 ionic_admin_post(dev, &wr); 1834 1835 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1836 } 1837 1838 static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev, 1839 uint32_t wqe_size) 1840 { 1841 switch (wqe_size) { 1842 case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64; 1843 case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128; 1844 case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256; 1845 case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512; 1846 } 1847 1848 return false; 1849 } 1850 1851 static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev, 1852 struct ionic_qp *qp, 1853 struct ib_udata *udata, 1854 int max_data) 1855 { 1856 u8 expdb_stride_log2 = 0; 1857 bool expdb; 1858 int rc; 1859 1860 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1861 goto not_in_cmb; 1862 1863 if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) { 1864 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1865 goto not_in_cmb; 1866 1867 qp->sq_cmb &= IONIC_CMB_SUPPORTED; 1868 } 1869 1870 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) { 1871 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1872 goto not_in_cmb; 1873 1874 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1875 } 1876 1877 qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE); 1878 1879 if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER) 1880 goto not_in_cmb; 1881 1882 if (qp->sq_cmb & IONIC_CMB_EXPDB) 1883 expdb_stride_log2 = qp->sq.stride_log2; 1884 1885 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid, 1886 &qp->sq_cmb_addr, qp->sq_cmb_order, 1887 expdb_stride_log2, &expdb); 1888 if (rc) 1889 goto not_in_cmb; 1890 1891 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) { 1892 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1893 goto err_map; 1894 1895 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1896 } 1897 1898 return; 1899 1900 err_map: 1901 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1902 not_in_cmb: 1903 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1904 ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n"); 1905 1906 qp->sq_cmb = 0; 1907 qp->sq_cmb_order = IONIC_RES_INVALID; 1908 qp->sq_cmb_pgid = 0; 1909 qp->sq_cmb_addr = 0; 1910 } 1911 1912 static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev, 1913 struct ionic_ctx *ctx, 1914 struct ionic_qp *qp) 1915 { 1916 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1917 return; 1918 1919 if (ctx) 1920 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); 1921 1922 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1923 } 1924 1925 static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 1926 struct ionic_qp *qp, struct ionic_qdesc *sq, 1927 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 1928 int max_data, int sq_spec, struct ib_udata *udata) 1929 { 1930 u32 wqe_size; 1931 int rc = 0; 1932 1933 qp->sq_msn_prod = 0; 1934 qp->sq_msn_cons = 0; 1935 1936 if (!qp->has_sq) { 1937 if (buf) { 1938 buf->tbl_buf = NULL; 1939 buf->tbl_limit = 0; 1940 buf->tbl_pages = 0; 1941 } 1942 if (udata) 1943 rc = ionic_validate_qdesc_zero(sq); 1944 1945 return rc; 1946 } 1947 1948 rc = -EINVAL; 1949 1950 if (max_wr < 0 || max_wr > 0xffff) 1951 return rc; 1952 1953 if (max_sge < 1) 1954 return rc; 1955 1956 if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, 1957 qp->sq_cmb & 1958 IONIC_CMB_EXPDB), 1959 IONIC_SPEC_HIGH)) 1960 return rc; 1961 1962 if (max_data < 0) 1963 return rc; 1964 1965 if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride, 1966 qp->sq_cmb & IONIC_CMB_EXPDB)) 1967 return rc; 1968 1969 if (udata) { 1970 rc = ionic_validate_qdesc(sq); 1971 if (rc) 1972 return rc; 1973 1974 qp->sq_spec = sq_spec; 1975 1976 qp->sq.ptr = NULL; 1977 qp->sq.size = sq->size; 1978 qp->sq.mask = sq->mask; 1979 qp->sq.depth_log2 = sq->depth_log2; 1980 qp->sq.stride_log2 = sq->stride_log2; 1981 1982 qp->sq_meta = NULL; 1983 qp->sq_msn_idx = NULL; 1984 1985 qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); 1986 if (IS_ERR(qp->sq_umem)) 1987 return PTR_ERR(qp->sq_umem); 1988 } else { 1989 qp->sq_umem = NULL; 1990 1991 qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec); 1992 if (sq_spec && !qp->sq_spec) 1993 ibdev_dbg(&dev->ibdev, 1994 "init sq: max_sge %u disables spec\n", 1995 max_sge); 1996 1997 if (qp->sq_cmb & IONIC_CMB_EXPDB) { 1998 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 1999 qp->sq_spec, 2000 true); 2001 2002 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 2003 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 2004 } 2005 2006 if (!(qp->sq_cmb & IONIC_CMB_EXPDB)) 2007 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 2008 qp->sq_spec, 2009 false); 2010 2011 rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev, 2012 max_wr, wqe_size); 2013 if (rc) 2014 return rc; 2015 2016 ionic_queue_dbell_init(&qp->sq, qp->qpid); 2017 2018 qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1); 2019 if (!qp->sq_meta) { 2020 rc = -ENOMEM; 2021 goto err_sq_meta; 2022 } 2023 2024 qp->sq_msn_idx = kmalloc_array((u32)qp->sq.mask + 1, 2025 sizeof(*qp->sq_msn_idx), 2026 GFP_KERNEL); 2027 if (!qp->sq_msn_idx) { 2028 rc = -ENOMEM; 2029 goto err_sq_msn; 2030 } 2031 } 2032 2033 ionic_qp_sq_init_cmb(dev, qp, udata, max_data); 2034 2035 if (qp->sq_cmb & IONIC_CMB_ENABLE) 2036 rc = ionic_pgtbl_init(dev, buf, NULL, 2037 (u64)qp->sq_cmb_pgid << PAGE_SHIFT, 2038 1, PAGE_SIZE); 2039 else 2040 rc = ionic_pgtbl_init(dev, buf, 2041 qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE); 2042 if (rc) 2043 goto err_sq_tbl; 2044 2045 return 0; 2046 2047 err_sq_tbl: 2048 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 2049 kfree(qp->sq_msn_idx); 2050 err_sq_msn: 2051 kfree(qp->sq_meta); 2052 err_sq_meta: 2053 if (qp->sq_umem) 2054 ib_umem_release(qp->sq_umem); 2055 else 2056 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 2057 return rc; 2058 } 2059 2060 static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, 2061 struct ionic_ctx *ctx, 2062 struct ionic_qp *qp) 2063 { 2064 if (!qp->has_sq) 2065 return; 2066 2067 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 2068 2069 kfree(qp->sq_msn_idx); 2070 kfree(qp->sq_meta); 2071 2072 if (qp->sq_umem) 2073 ib_umem_release(qp->sq_umem); 2074 else 2075 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 2076 } 2077 2078 static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev, 2079 struct ionic_ctx *ctx, 2080 struct ionic_rq *rq, 2081 u64 *cmb_offset) 2082 { 2083 bool wc; 2084 2085 /* set mapping by default to uncached for 2086 * expdb (to guarantee writes order) otherwise 2087 * writecombine, unless this default is 2088 * overridden by userspace 2089 */ 2090 if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2091 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2092 ibdev_dbg(&dev->ibdev, 2093 "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n"); 2094 rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2095 } 2096 2097 if (rq->cmb & IONIC_CMB_EXPDB) 2098 wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; 2099 else 2100 wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC; 2101 2102 /* let userspace know the mapping */ 2103 if (wc) 2104 rq->cmb |= IONIC_CMB_WC; 2105 else 2106 rq->cmb |= IONIC_CMB_UC; 2107 2108 rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size, 2109 PHYS_PFN(rq->cmb_addr), 2110 wc ? IONIC_MMAP_WC : 0, 2111 cmb_offset); 2112 } 2113 2114 static void ionic_rq_init_cmb(struct ionic_ibdev *dev, 2115 struct ionic_rq *rq, 2116 struct ib_udata *udata) 2117 { 2118 u8 expdb_stride_log2 = 0; 2119 bool expdb; 2120 int rc; 2121 2122 if (!(rq->cmb & IONIC_CMB_ENABLE)) 2123 goto not_in_cmb; 2124 2125 if (rq->cmb & ~IONIC_CMB_SUPPORTED) { 2126 if (rq->cmb & IONIC_CMB_REQUIRE) 2127 goto not_in_cmb; 2128 2129 rq->cmb &= IONIC_CMB_SUPPORTED; 2130 } 2131 2132 if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { 2133 if (rq->cmb & IONIC_CMB_REQUIRE) 2134 goto not_in_cmb; 2135 2136 rq->cmb &= ~IONIC_CMB_EXPDB; 2137 } 2138 2139 rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE); 2140 2141 if (rq->cmb_order >= IONIC_RQCMB_ORDER) 2142 goto not_in_cmb; 2143 2144 if (rq->cmb & IONIC_CMB_EXPDB) 2145 expdb_stride_log2 = rq->q.stride_log2; 2146 2147 rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid, 2148 &rq->cmb_addr, rq->cmb_order, 2149 expdb_stride_log2, &expdb); 2150 if (rc) 2151 goto not_in_cmb; 2152 2153 if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) { 2154 if (rq->cmb & IONIC_CMB_REQUIRE) 2155 goto err_map; 2156 2157 rq->cmb &= ~IONIC_CMB_EXPDB; 2158 } 2159 2160 return; 2161 2162 err_map: 2163 ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); 2164 not_in_cmb: 2165 if (rq->cmb & IONIC_CMB_REQUIRE) 2166 ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n"); 2167 2168 rq->cmb = 0; 2169 rq->cmb_order = IONIC_RES_INVALID; 2170 rq->cmb_pgid = 0; 2171 rq->cmb_addr = 0; 2172 } 2173 2174 static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev, 2175 struct ionic_ctx *ctx, 2176 struct ionic_rq *rq) 2177 { 2178 if (!(rq->cmb & IONIC_CMB_ENABLE)) 2179 return; 2180 2181 if (ctx) 2182 rdma_user_mmap_entry_remove(rq->mmap_cmb); 2183 2184 ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); 2185 } 2186 2187 static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 2188 struct ionic_rq *rq, struct ionic_qdesc *qdesc, 2189 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 2190 int rq_spec, struct ib_udata *udata) 2191 { 2192 int rc = 0, i; 2193 u32 wqe_size; 2194 2195 rc = -EINVAL; 2196 2197 if (max_wr < 0 || max_wr > 0xffff) 2198 return rc; 2199 2200 if (max_sge < 1) 2201 return rc; 2202 2203 if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), 2204 IONIC_SPEC_HIGH)) 2205 return rc; 2206 2207 if (udata) { 2208 rc = ionic_validate_qdesc(qdesc); 2209 if (rc) 2210 return rc; 2211 2212 rq->spec = rq_spec; 2213 2214 rq->q.ptr = NULL; 2215 rq->q.size = qdesc->size; 2216 rq->q.mask = qdesc->mask; 2217 rq->q.depth_log2 = qdesc->depth_log2; 2218 rq->q.stride_log2 = qdesc->stride_log2; 2219 2220 rq->meta = NULL; 2221 2222 rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0); 2223 if (IS_ERR(rq->umem)) 2224 return PTR_ERR(rq->umem); 2225 } else { 2226 rq->umem = NULL; 2227 2228 rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec); 2229 if (rq_spec && !rq->spec) 2230 ibdev_dbg(&dev->ibdev, 2231 "init rq: max_sge %u disables spec\n", 2232 max_sge); 2233 2234 if (rq->cmb & IONIC_CMB_EXPDB) { 2235 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2236 rq->spec, 2237 true); 2238 2239 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 2240 rq->cmb &= ~IONIC_CMB_EXPDB; 2241 } 2242 2243 if (!(rq->cmb & IONIC_CMB_EXPDB)) 2244 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2245 rq->spec, 2246 false); 2247 2248 rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev, 2249 max_wr, wqe_size); 2250 if (rc) 2251 return rc; 2252 2253 ionic_queue_dbell_init(&rq->q, rq->qid); 2254 2255 rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1); 2256 if (!rq->meta) { 2257 rc = -ENOMEM; 2258 goto err_rq_meta; 2259 } 2260 2261 for (i = 0; i < rq->q.mask; ++i) 2262 rq->meta[i].next = &rq->meta[i + 1]; 2263 rq->meta[i].next = IONIC_META_LAST; 2264 rq->meta_head = &rq->meta[0]; 2265 } 2266 2267 ionic_rq_init_cmb(dev, rq, udata); 2268 2269 if (rq->cmb & IONIC_CMB_ENABLE) 2270 rc = ionic_pgtbl_init(dev, buf, NULL, 2271 (u64)rq->cmb_pgid << PAGE_SHIFT, 2272 1, PAGE_SIZE); 2273 else 2274 rc = ionic_pgtbl_init(dev, buf, rq->umem, 2275 rq->q.dma, 1, PAGE_SIZE); 2276 if (rc) 2277 goto err_rq_tbl; 2278 2279 return 0; 2280 2281 err_rq_tbl: 2282 ionic_rq_destroy_cmb(dev, ctx, rq); 2283 kfree(rq->meta); 2284 err_rq_meta: 2285 if (rq->umem) 2286 ib_umem_release(rq->umem); 2287 else 2288 ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); 2289 return rc; 2290 } 2291 2292 static void ionic_rq_destroy(struct ionic_ibdev *dev, 2293 struct ionic_ctx *ctx, 2294 struct ionic_rq *rq) 2295 { 2296 ionic_rq_destroy_cmb(dev, ctx, rq); 2297 2298 kfree(rq->meta); 2299 2300 if (rq->umem) 2301 ib_umem_release(rq->umem); 2302 else 2303 ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); 2304 } 2305 2306 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 2307 struct ib_udata *udata) 2308 { 2309 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2310 struct ionic_tbl_buf sq_buf = {}, rq_buf = {}; 2311 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2312 struct ionic_qp *qp = to_ionic_qp(ibqp); 2313 struct ionic_ctx *ctx = 2314 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2315 struct ionic_qp_resp resp = {}; 2316 struct ionic_qp_req req = {}; 2317 struct ionic_cq *cq; 2318 u8 udma_mask; 2319 void *entry; 2320 int rc; 2321 2322 if (udata) { 2323 rc = ib_copy_validate_udata_in(udata, req, rsvd); 2324 if (rc) 2325 return rc; 2326 2327 if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK) 2328 return -EINVAL; 2329 2330 if (req.rsvd_pad) 2331 return -EINVAL; 2332 } else { 2333 req.sq_spec = IONIC_SPEC_HIGH; 2334 req.rq_spec = IONIC_SPEC_HIGH; 2335 } 2336 2337 if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD) 2338 return -EOPNOTSUPP; 2339 2340 qp->state = IB_QPS_RESET; 2341 2342 INIT_LIST_HEAD(&qp->cq_poll_sq); 2343 INIT_LIST_HEAD(&qp->cq_flush_sq); 2344 INIT_LIST_HEAD(&qp->rq.cq_flush); 2345 2346 spin_lock_init(&qp->sq_lock); 2347 spin_lock_init(&qp->rq.lock); 2348 2349 qp->has_sq = 1; 2350 2351 if (attr->srq) 2352 qp->srq = to_ionic_srq(attr->srq); 2353 else 2354 qp->srq = NULL; 2355 2356 if (attr->qp_type == IB_QPT_GSI) { 2357 rc = ionic_get_gsi_qpid(dev, &qp->qpid); 2358 } else { 2359 udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 2360 2361 if (attr->send_cq) 2362 udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask; 2363 2364 if (attr->recv_cq) 2365 udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask; 2366 2367 if (attr->srq) 2368 udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx); 2369 2370 if (udata && req.udma_mask) 2371 udma_mask &= req.udma_mask; 2372 2373 if (!udma_mask) 2374 return -EINVAL; 2375 2376 rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask); 2377 } 2378 if (rc) 2379 return rc; 2380 2381 qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2382 qp->has_ah = attr->qp_type == IB_QPT_RC; 2383 2384 if (qp->has_ah) { 2385 qp->hdr = kzalloc_obj(*qp->hdr); 2386 if (!qp->hdr) { 2387 rc = -ENOMEM; 2388 goto err_ah_alloc; 2389 } 2390 2391 rc = ionic_get_ahid(dev, &qp->ahid); 2392 if (rc) 2393 goto err_ahid; 2394 } 2395 2396 if (udata && (req.sq_cmb & IONIC_CMB_ENABLE)) 2397 qp->sq_cmb = req.sq_cmb; 2398 2399 rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, 2400 attr->cap.max_send_wr, attr->cap.max_send_sge, 2401 attr->cap.max_inline_data, req.sq_spec, udata); 2402 if (rc) 2403 goto err_sq; 2404 2405 if (qp->srq) { 2406 rq_buf.tbl_buf = NULL; 2407 rq_buf.tbl_limit = 0; 2408 rq_buf.tbl_pages = 0; 2409 2410 if (udata) 2411 rc = ionic_validate_qdesc_zero(&req.rq); 2412 } else { 2413 if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) 2414 qp->rq.cmb = req.rq_cmb; 2415 2416 /* for non-srq qps, rq qid is same as qpid */ 2417 qp->rq.qid = qp->qpid; 2418 rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, 2419 attr->cap.max_recv_wr, attr->cap.max_recv_sge, 2420 req.rq_spec, udata); 2421 } 2422 if (rc) 2423 goto err_rq; 2424 2425 rc = ionic_create_qp_cmd(dev, pd, 2426 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), 2427 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), 2428 qp, &sq_buf, &rq_buf, attr, req.ionic_flags); 2429 if (rc) 2430 goto err_cmd; 2431 2432 if (udata) { 2433 resp.qpid = qp->qpid; 2434 resp.udma_idx = qp->udma_idx; 2435 2436 if (qp->sq_cmb & IONIC_CMB_ENABLE) { 2437 bool wc; 2438 2439 if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2440 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2441 ibdev_dbg(&dev->ibdev, 2442 "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); 2443 qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2444 } 2445 2446 wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) 2447 != IONIC_CMB_UC; 2448 2449 /* let userspace know the mapping */ 2450 if (wc) 2451 qp->sq_cmb |= IONIC_CMB_WC; 2452 else 2453 qp->sq_cmb |= IONIC_CMB_UC; 2454 2455 qp->mmap_sq_cmb = 2456 ionic_mmap_entry_insert(ctx, 2457 qp->sq.size, 2458 PHYS_PFN(qp->sq_cmb_addr), 2459 wc ? IONIC_MMAP_WC : 0, 2460 &resp.sq_cmb_offset); 2461 if (!qp->mmap_sq_cmb) { 2462 rc = -ENOMEM; 2463 goto err_qp_cmd; 2464 } 2465 2466 resp.sq_cmb = qp->sq_cmb; 2467 } 2468 2469 if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) { 2470 ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset); 2471 2472 if (!qp->rq.mmap_cmb) { 2473 rc = -ENOMEM; 2474 goto err_qp_cmd; 2475 } 2476 2477 resp.rq_cmb = qp->rq.cmb; 2478 } 2479 2480 rc = ib_respond_udata(udata, resp); 2481 if (rc) 2482 goto err_qp_cmd; 2483 } 2484 2485 ionic_pgtbl_unbuf(dev, &rq_buf); 2486 ionic_pgtbl_unbuf(dev, &sq_buf); 2487 2488 qp->ibqp.qp_num = qp->qpid; 2489 2490 init_completion(&qp->qp_rel_comp); 2491 kref_init(&qp->qp_kref); 2492 2493 entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL); 2494 if (entry) { 2495 if (!xa_is_err(entry)) 2496 rc = -EINVAL; 2497 else 2498 rc = xa_err(entry); 2499 2500 goto err_qp_cmd; 2501 } 2502 2503 if (qp->has_sq) { 2504 cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx); 2505 2506 attr->cap.max_send_wr = qp->sq.mask; 2507 attr->cap.max_send_sge = 2508 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 2509 qp->sq_spec, 2510 qp->sq_cmb & IONIC_CMB_EXPDB); 2511 attr->cap.max_inline_data = 2512 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, 2513 qp->sq_cmb & 2514 IONIC_CMB_EXPDB); 2515 qp->sq_cqid = cq->cqid; 2516 } 2517 2518 if (!qp->srq) { 2519 cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); 2520 2521 attr->cap.max_recv_wr = qp->rq.q.mask; 2522 attr->cap.max_recv_sge = 2523 ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, 2524 qp->rq.spec, 2525 qp->rq.cmb & IONIC_CMB_EXPDB); 2526 qp->rq.cqid = cq->cqid; 2527 } 2528 2529 return 0; 2530 2531 err_qp_cmd: 2532 ionic_destroy_qp_cmd(dev, qp->qpid); 2533 err_cmd: 2534 if (!qp->srq) { 2535 ionic_pgtbl_unbuf(dev, &rq_buf); 2536 ionic_rq_destroy(dev, ctx, &qp->rq); 2537 } 2538 err_rq: 2539 ionic_pgtbl_unbuf(dev, &sq_buf); 2540 ionic_qp_sq_destroy(dev, ctx, qp); 2541 err_sq: 2542 if (qp->has_ah) 2543 ionic_put_ahid(dev, qp->ahid); 2544 err_ahid: 2545 kfree(qp->hdr); 2546 err_ah_alloc: 2547 ionic_put_qpid(dev, qp->qpid); 2548 return rc; 2549 } 2550 2551 void ionic_notify_flush_cq(struct ionic_cq *cq) 2552 { 2553 if (cq->flush && cq->vcq->ibcq.comp_handler) 2554 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, 2555 cq->vcq->ibcq.cq_context); 2556 } 2557 2558 static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp) 2559 { 2560 if (qp->ibqp.send_cq) 2561 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq, 2562 qp->udma_idx)); 2563 if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq) 2564 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq, 2565 qp->udma_idx)); 2566 } 2567 2568 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2569 { 2570 unsigned long irqflags; 2571 struct ionic_cq *cq; 2572 2573 if (qp->ibqp.send_cq) { 2574 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2575 2576 /* Hold the CQ lock and QP sq_lock to set up flush */ 2577 spin_lock_irqsave(&cq->lock, irqflags); 2578 spin_lock(&qp->sq_lock); 2579 qp->sq_flush = true; 2580 if (!ionic_queue_empty(&qp->sq)) { 2581 cq->flush = true; 2582 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); 2583 } 2584 spin_unlock(&qp->sq_lock); 2585 spin_unlock_irqrestore(&cq->lock, irqflags); 2586 } 2587 2588 if (qp->ibqp.recv_cq && !qp->srq) { 2589 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2590 2591 /* Hold the CQ lock and QP rq.lock to set up flush */ 2592 spin_lock_irqsave(&cq->lock, irqflags); 2593 spin_lock(&qp->rq.lock); 2594 qp->rq.flush = true; 2595 if (!ionic_queue_empty(&qp->rq.q)) { 2596 cq->flush = true; 2597 list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); 2598 } 2599 spin_unlock(&qp->rq.lock); 2600 spin_unlock_irqrestore(&cq->lock, irqflags); 2601 } 2602 } 2603 2604 static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid) 2605 { 2606 struct ionic_v1_cqe *qcqe; 2607 int prod, qtf, qid, type; 2608 bool color; 2609 2610 if (!cq->q.ptr) 2611 return; 2612 2613 color = cq->color; 2614 prod = cq->q.prod; 2615 qcqe = ionic_queue_at(&cq->q, prod); 2616 2617 while (color == ionic_v1_cqe_color(qcqe)) { 2618 qtf = ionic_v1_cqe_qtf(qcqe); 2619 qid = ionic_v1_cqe_qtf_qid(qtf); 2620 type = ionic_v1_cqe_qtf_type(qtf); 2621 2622 if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN) 2623 ionic_v1_cqe_clean(qcqe); 2624 2625 prod = ionic_queue_next(&cq->q, prod); 2626 qcqe = ionic_queue_at(&cq->q, prod); 2627 color = ionic_color_wrap(prod, color); 2628 } 2629 } 2630 2631 static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2632 { 2633 unsigned long irqflags; 2634 struct ionic_cq *cq; 2635 int i; 2636 2637 local_irq_save(irqflags); 2638 2639 if (qp->ibqp.send_cq) { 2640 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2641 spin_lock(&cq->lock); 2642 ionic_clean_cq(cq, qp->qpid); 2643 spin_unlock(&cq->lock); 2644 } 2645 2646 if (qp->ibqp.recv_cq) { 2647 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2648 spin_lock(&cq->lock); 2649 ionic_clean_cq(cq, qp->qpid); 2650 spin_unlock(&cq->lock); 2651 } 2652 2653 if (qp->has_sq) { 2654 spin_lock(&qp->sq_lock); 2655 qp->sq_flush = false; 2656 qp->sq_flush_rcvd = false; 2657 qp->sq_msn_prod = 0; 2658 qp->sq_msn_cons = 0; 2659 qp->sq.prod = 0; 2660 qp->sq.cons = 0; 2661 spin_unlock(&qp->sq_lock); 2662 } 2663 2664 if (!qp->srq) { 2665 spin_lock(&qp->rq.lock); 2666 qp->rq.flush = false; 2667 qp->rq.q.prod = 0; 2668 qp->rq.q.cons = 0; 2669 if (qp->rq.meta) { 2670 for (i = 0; i < qp->rq.q.mask; ++i) 2671 qp->rq.meta[i].next = &qp->rq.meta[i + 1]; 2672 qp->rq.meta[i].next = IONIC_META_LAST; 2673 } 2674 qp->rq.meta_head = &qp->rq.meta[0]; 2675 spin_unlock(&qp->rq.lock); 2676 } 2677 2678 local_irq_restore(irqflags); 2679 } 2680 2681 static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, 2682 enum ib_qp_state attr_state) 2683 { 2684 if (q_state == attr_state) 2685 return true; 2686 2687 if (attr_state == IB_QPS_ERR) 2688 return true; 2689 2690 if (attr_state == IB_QPS_SQE) 2691 return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD; 2692 2693 return false; 2694 } 2695 2696 static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, 2697 enum ib_qp_type type, enum ib_qp_state cur, 2698 enum ib_qp_state next) 2699 { 2700 if (!ext_mask) 2701 return true; 2702 2703 if (ext_mask & ~IB_QP_RATE_LIMIT) 2704 return false; 2705 2706 /* Rate limit is only supported for RC QPs during specific transitions */ 2707 return type == IB_QPT_RC && 2708 ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || 2709 (cur == IB_QPS_RTR && next == IB_QPS_RTS) || 2710 (cur == IB_QPS_RTS && next == IB_QPS_RTS)); 2711 } 2712 2713 static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, 2714 int mask) 2715 { 2716 enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ? 2717 attr->cur_qp_state : qp->state; 2718 enum ib_qp_state next_state = (mask & IB_QP_STATE) ? 2719 attr->qp_state : cur_state; 2720 2721 if ((mask & IB_QP_CUR_STATE) && 2722 !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) 2723 return -EINVAL; 2724 2725 if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || 2726 !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, 2727 qp->ibqp.qp_type, cur_state, next_state)) 2728 return -EINVAL; 2729 2730 /* unprivileged qp not allowed privileged qkey */ 2731 if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) && 2732 qp->ibqp.uobject) 2733 return -EPERM; 2734 2735 return 0; 2736 } 2737 2738 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 2739 struct ib_udata *udata) 2740 { 2741 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2742 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2743 struct ionic_qp *qp = to_ionic_qp(ibqp); 2744 int rc; 2745 2746 rc = ib_no_udata_io(udata); 2747 if (rc) 2748 return rc; 2749 2750 rc = ionic_check_modify_qp(qp, attr, mask); 2751 if (rc) 2752 return rc; 2753 2754 if (mask & IB_QP_CAP) 2755 return -EINVAL; 2756 2757 rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask); 2758 if (rc) 2759 return rc; 2760 2761 if (mask & IB_QP_STATE) { 2762 qp->state = attr->qp_state; 2763 2764 if (attr->qp_state == IB_QPS_ERR) { 2765 ionic_flush_qp(dev, qp); 2766 ionic_notify_qp_cqs(dev, qp); 2767 } else if (attr->qp_state == IB_QPS_RESET) { 2768 ionic_reset_qp(dev, qp); 2769 } 2770 } 2771 2772 return 0; 2773 } 2774 2775 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2776 int mask, struct ib_qp_init_attr *init_attr) 2777 { 2778 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2779 struct ionic_qp *qp = to_ionic_qp(ibqp); 2780 int rc; 2781 2782 memset(attr, 0, sizeof(*attr)); 2783 memset(init_attr, 0, sizeof(*init_attr)); 2784 2785 rc = ionic_query_qp_cmd(dev, qp, attr, mask); 2786 if (rc) 2787 return rc; 2788 2789 if (qp->has_sq) 2790 attr->cap.max_send_wr = qp->sq.mask; 2791 2792 if (!qp->srq) 2793 attr->cap.max_recv_wr = qp->rq.q.mask; 2794 2795 init_attr->event_handler = ibqp->event_handler; 2796 init_attr->qp_context = ibqp->qp_context; 2797 init_attr->send_cq = ibqp->send_cq; 2798 init_attr->recv_cq = ibqp->recv_cq; 2799 init_attr->srq = ibqp->srq; 2800 init_attr->xrcd = ibqp->xrcd; 2801 init_attr->cap = attr->cap; 2802 init_attr->sq_sig_type = qp->sig_all ? 2803 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2804 init_attr->qp_type = ibqp->qp_type; 2805 init_attr->create_flags = 0; 2806 init_attr->port_num = 0; 2807 init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl; 2808 init_attr->source_qpn = 0; 2809 2810 return rc; 2811 } 2812 2813 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2814 { 2815 struct ionic_ctx *ctx = 2816 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2817 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2818 struct ionic_qp *qp = to_ionic_qp(ibqp); 2819 unsigned long irqflags; 2820 struct ionic_cq *cq; 2821 int rc; 2822 2823 rc = ib_no_udata_io(udata); 2824 if (rc) 2825 return rc; 2826 2827 rc = ionic_destroy_qp_cmd(dev, qp->qpid); 2828 if (rc) 2829 return rc; 2830 2831 xa_erase_irq(&dev->qp_tbl, qp->qpid); 2832 2833 kref_put(&qp->qp_kref, ionic_qp_complete); 2834 wait_for_completion(&qp->qp_rel_comp); 2835 2836 if (qp->ibqp.send_cq) { 2837 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2838 spin_lock_irqsave(&cq->lock, irqflags); 2839 ionic_clean_cq(cq, qp->qpid); 2840 list_del(&qp->cq_poll_sq); 2841 list_del(&qp->cq_flush_sq); 2842 spin_unlock_irqrestore(&cq->lock, irqflags); 2843 } 2844 2845 if (qp->ibqp.recv_cq) { 2846 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2847 spin_lock_irqsave(&cq->lock, irqflags); 2848 ionic_clean_cq(cq, qp->qpid); 2849 list_del(&qp->rq.cq_flush); 2850 spin_unlock_irqrestore(&cq->lock, irqflags); 2851 } 2852 2853 if (!qp->srq) 2854 ionic_rq_destroy(dev, ctx, &qp->rq); 2855 2856 ionic_qp_sq_destroy(dev, ctx, qp); 2857 if (qp->has_ah) { 2858 ionic_put_ahid(dev, qp->ahid); 2859 kfree(qp->hdr); 2860 } 2861 ionic_put_qpid(dev, qp->qpid); 2862 2863 return 0; 2864 } 2865 2866 static int ionic_create_srq_cmd(struct ionic_ibdev *dev, 2867 struct ionic_ctx *ctx, 2868 struct ionic_srq *srq, 2869 struct ionic_pd *pd, 2870 struct ionic_tbl_buf *buf, 2871 u8 udma_mask) 2872 { 2873 const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0; 2874 struct ionic_admin_create_srq_resp *resp_buf; 2875 const u16 dbid = ionic_ctx_dbid(dev, ctx); 2876 struct ionic_admin_wr wr = { 2877 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2878 .wqe = { 2879 .op = IONIC_V1_ADMIN_CREATE_SRQ, 2880 .len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN), 2881 .cmd.create_srq = { 2882 .pd_id = cpu_to_le32(pd->pdid), 2883 .depth_log2 = srq->rq.q.depth_log2, 2884 .stride_log2 = srq->rq.q.stride_log2, 2885 .page_size_log2 = buf->page_size_log2, 2886 .map_count = cpu_to_le32(buf->tbl_pages), 2887 .dma_addr = ionic_pgtbl_dma(buf, 0), 2888 .dbid = cpu_to_le16(dbid), 2889 .qid = cpu_to_le32(srq->rq.qid), 2890 .low_wqes_limit = cpu_to_le16(srq->srq_limit), 2891 .udma_mask = udma_mask, 2892 .flags = cpu_to_le16(flags), 2893 } 2894 } 2895 }; 2896 dma_addr_t resp_buf_dma; 2897 int rc; 2898 2899 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ) 2900 return -EOPNOTSUPP; 2901 2902 resp_buf = kzalloc_obj(*resp_buf); 2903 if (!resp_buf) 2904 return -ENOMEM; 2905 2906 resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, 2907 sizeof(*resp_buf), 2908 DMA_FROM_DEVICE); 2909 2910 rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); 2911 if (rc) 2912 goto err_dma; 2913 2914 wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma); 2915 wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); 2916 2917 ionic_admin_post(dev, &wr); 2918 2919 rc = ionic_admin_wait(dev, &wr, 0); 2920 2921 dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), 2922 DMA_FROM_DEVICE); 2923 if (rc) 2924 goto err_dma; 2925 2926 if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) { 2927 rc = -EOPNOTSUPP; 2928 goto err_dma; 2929 } 2930 2931 srq->rq.qid = le32_to_cpu(resp_buf->id); 2932 srq->udma_idx = resp_buf->udma_idx; 2933 2934 err_dma: 2935 kfree(resp_buf); 2936 2937 return rc; 2938 } 2939 2940 static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid) 2941 { 2942 struct ionic_admin_wr wr = { 2943 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2944 .wqe = { 2945 .op = IONIC_V1_ADMIN_DESTROY_SRQ, 2946 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN), 2947 .cmd.destroy_srq = { 2948 .qid = cpu_to_le32(srqid), 2949 }, 2950 } 2951 }; 2952 2953 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ) 2954 return -EOPNOTSUPP; 2955 2956 ionic_admin_post(dev, &wr); 2957 2958 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 2959 } 2960 2961 static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid, 2962 u16 srq_limit) 2963 { 2964 struct ionic_admin_wr wr = { 2965 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 2966 .wqe = { 2967 .op = IONIC_V1_ADMIN_MODIFY_SRQ, 2968 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN), 2969 .cmd.modify_srq = { 2970 .qid = cpu_to_le32(srqid), 2971 .low_wqes_limit = cpu_to_le16(srq_limit), 2972 }, 2973 } 2974 }; 2975 2976 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ) 2977 return -EOPNOTSUPP; 2978 2979 ionic_admin_post(dev, &wr); 2980 2981 return ionic_admin_wait(dev, &wr, 0); 2982 } 2983 2984 int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, 2985 struct ib_udata *udata) 2986 { 2987 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 2988 struct ionic_pd *pd = to_ionic_pd(ibsrq->pd); 2989 struct ionic_srq *srq = to_ionic_srq(ibsrq); 2990 struct ionic_ctx *ctx = 2991 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2992 struct ionic_srq_resp resp = {}; 2993 struct ionic_srq_req req = {}; 2994 struct ionic_tbl_buf buf = {}; 2995 u8 udma_mask; 2996 int rc; 2997 2998 if (!ctx) 2999 return -EOPNOTSUPP; 3000 3001 if (attr->srq_type != IB_SRQT_BASIC) 3002 return -EOPNOTSUPP; 3003 3004 if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES) 3005 return -EINVAL; 3006 3007 if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT) 3008 return -EINVAL; 3009 3010 udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 3011 if (udata) { 3012 rc = ib_copy_validate_udata_in(udata, req, rsvd); 3013 if (rc) 3014 return rc; 3015 3016 if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd))) 3017 return -EINVAL; 3018 3019 udma_mask &= req.udma_mask; 3020 } 3021 3022 if (!udma_mask) 3023 return -EINVAL; 3024 3025 if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) 3026 srq->rq.cmb = req.rq_cmb; 3027 3028 rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf, 3029 attr->attr.max_wr, attr->attr.max_sge, 3030 req.rq_spec, udata); 3031 if (rc) 3032 return rc; 3033 3034 srq->srq_limit = attr->attr.srq_limit; 3035 3036 rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask); 3037 if (rc) 3038 goto err_cmd; 3039 3040 ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid); 3041 3042 if (udata) { 3043 resp.srqid = srq->rq.qid; 3044 resp.udma_idx = srq->udma_idx; 3045 3046 if (srq->rq.cmb & IONIC_CMB_ENABLE) { 3047 ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset); 3048 if (!srq->rq.mmap_cmb) { 3049 rc = -ENOMEM; 3050 goto err_srq_cmd; 3051 } 3052 3053 resp.rq_cmb = srq->rq.cmb; 3054 } 3055 3056 rc = ib_respond_udata(udata, resp); 3057 if (rc) 3058 goto err_srq_cmd; 3059 } 3060 3061 kref_init(&srq->kref); 3062 init_completion(&srq->rel_comp); 3063 3064 rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL); 3065 if (rc) 3066 goto err_srq_cmd; 3067 3068 ionic_pgtbl_unbuf(dev, &buf); 3069 3070 attr->attr.max_wr = srq->rq.q.mask; 3071 3072 return 0; 3073 3074 err_srq_cmd: 3075 ionic_destroy_srq_cmd(dev, srq->rq.qid); 3076 err_cmd: 3077 ionic_pgtbl_unbuf(dev, &buf); 3078 ionic_rq_destroy(dev, ctx, &srq->rq); 3079 3080 return rc; 3081 } 3082 3083 int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 3084 { 3085 struct ionic_ctx *ctx = 3086 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 3087 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 3088 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3089 int rc; 3090 3091 if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) { 3092 kref_put(&srq->kref, ionic_srq_complete); 3093 wait_for_completion(&srq->rel_comp); 3094 } 3095 3096 rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); 3097 if (rc) 3098 return rc; 3099 3100 ionic_rq_destroy(dev, ctx, &srq->rq); 3101 3102 return 0; 3103 } 3104 3105 int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 3106 { 3107 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3108 3109 srq_attr->max_wr = srq->rq.q.mask; 3110 srq_attr->max_sge = 3111 ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec, 3112 srq->rq.cmb & IONIC_CMB_EXPDB); 3113 srq_attr->srq_limit = srq->srq_limit; 3114 3115 return 0; 3116 } 3117 3118 int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 3119 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 3120 { 3121 struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); 3122 struct ionic_srq *srq = to_ionic_srq(ibsrq); 3123 int rc; 3124 3125 if (attr_mask & IB_SRQ_MAX_WR) 3126 return -EINVAL; 3127 3128 if (attr_mask & IB_SRQ_LIMIT) { 3129 if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT) 3130 return -EINVAL; 3131 3132 rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit); 3133 if (rc) 3134 return rc; 3135 3136 srq->srq_limit = attr->srq_limit; 3137 } 3138 3139 return 0; 3140 } 3141