1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 #include <rdma/ib_addr.h> 7 #include <rdma/ib_cache.h> 8 #include <rdma/ib_user_verbs.h> 9 #include <ionic_api.h> 10 11 #include "ionic_fw.h" 12 #include "ionic_ibdev.h" 13 14 #define ionic_set_ecn(tos) (((tos) | 2u) & ~1u) 15 #define ionic_clear_ecn(tos) ((tos) & ~3u) 16 17 static int ionic_validate_qdesc(struct ionic_qdesc *q) 18 { 19 if (!q->addr || !q->size || !q->mask || 20 !q->depth_log2 || !q->stride_log2) 21 return -EINVAL; 22 23 if (q->addr & (PAGE_SIZE - 1)) 24 return -EINVAL; 25 26 if (q->mask != BIT(q->depth_log2) - 1) 27 return -EINVAL; 28 29 if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2)) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx) 36 { 37 /* EQ per vector per udma, and the first eqs reserved for async events. 38 * The rest of the vectors can be requested for completions. 39 */ 40 u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1; 41 42 return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx; 43 } 44 45 static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx) 46 { 47 unsigned int size, base, bound; 48 int rc; 49 50 size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; 51 base = size * udma_idx; 52 bound = base + size; 53 54 rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound); 55 if (rc >= 0) { 56 /* cq_base is zero or a multiple of two queue groups */ 57 *cqid = dev->lif_cfg.cq_base + 58 ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift, 59 dev->half_cqid_udma_shift); 60 61 rc = 0; 62 } 63 64 return rc; 65 } 66 67 static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid) 68 { 69 u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base, 70 dev->lif_cfg.udma_qgrp_shift, 71 dev->half_cqid_udma_shift); 72 73 ionic_resid_put(&dev->inuse_cqid, bitid); 74 } 75 76 int ionic_create_cq_common(struct ionic_vcq *vcq, 77 struct ionic_tbl_buf *buf, 78 const struct ib_cq_init_attr *attr, 79 struct ionic_ctx *ctx, 80 struct ib_udata *udata, 81 struct ionic_qdesc *req_cq, 82 __u32 *resp_cqid, 83 int udma_idx) 84 { 85 struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); 86 struct ionic_cq *cq = &vcq->cq[udma_idx]; 87 void *entry; 88 int rc; 89 90 cq->vcq = vcq; 91 92 if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) { 93 rc = -EINVAL; 94 goto err_args; 95 } 96 97 rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); 98 if (rc) 99 goto err_args; 100 101 cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); 102 103 spin_lock_init(&cq->lock); 104 INIT_LIST_HEAD(&cq->poll_sq); 105 INIT_LIST_HEAD(&cq->flush_sq); 106 INIT_LIST_HEAD(&cq->flush_rq); 107 108 if (udata) { 109 rc = ionic_validate_qdesc(req_cq); 110 if (rc) 111 goto err_qdesc; 112 113 cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr, 114 req_cq->size, IB_ACCESS_LOCAL_WRITE); 115 if (IS_ERR(cq->umem)) { 116 rc = PTR_ERR(cq->umem); 117 goto err_qdesc; 118 } 119 120 cq->q.ptr = NULL; 121 cq->q.size = req_cq->size; 122 cq->q.mask = req_cq->mask; 123 cq->q.depth_log2 = req_cq->depth_log2; 124 cq->q.stride_log2 = req_cq->stride_log2; 125 126 *resp_cqid = cq->cqid; 127 } else { 128 rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, 129 attr->cqe + IONIC_CQ_GRACE, 130 sizeof(struct ionic_v1_cqe)); 131 if (rc) 132 goto err_q_init; 133 134 ionic_queue_dbell_init(&cq->q, cq->cqid); 135 cq->color = true; 136 cq->credit = cq->q.mask; 137 } 138 139 rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE); 140 if (rc) 141 goto err_pgtbl_init; 142 143 init_completion(&cq->cq_rel_comp); 144 kref_init(&cq->cq_kref); 145 146 entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); 147 if (entry) { 148 if (!xa_is_err(entry)) 149 rc = -EINVAL; 150 else 151 rc = xa_err(entry); 152 153 goto err_xa; 154 } 155 156 return 0; 157 158 err_xa: 159 ionic_pgtbl_unbuf(dev, buf); 160 err_pgtbl_init: 161 if (!udata) 162 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 163 err_q_init: 164 if (cq->umem) 165 ib_umem_release(cq->umem); 166 err_qdesc: 167 ionic_put_cqid(dev, cq->cqid); 168 err_args: 169 cq->vcq = NULL; 170 171 return rc; 172 } 173 174 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq) 175 { 176 if (!cq->vcq) 177 return; 178 179 xa_erase_irq(&dev->cq_tbl, cq->cqid); 180 181 kref_put(&cq->cq_kref, ionic_cq_complete); 182 wait_for_completion(&cq->cq_rel_comp); 183 184 if (cq->umem) 185 ib_umem_release(cq->umem); 186 else 187 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 188 189 ionic_put_cqid(dev, cq->cqid); 190 191 cq->vcq = NULL; 192 } 193 194 static int ionic_validate_qdesc_zero(struct ionic_qdesc *q) 195 { 196 if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2) 197 return -EINVAL; 198 199 return 0; 200 } 201 202 static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid) 203 { 204 int rc; 205 206 rc = ionic_resid_get(&dev->inuse_pdid); 207 if (rc < 0) 208 return rc; 209 210 *pdid = rc; 211 return 0; 212 } 213 214 static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid) 215 { 216 int rc; 217 218 rc = ionic_resid_get(&dev->inuse_ahid); 219 if (rc < 0) 220 return rc; 221 222 *ahid = rc; 223 return 0; 224 } 225 226 static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid) 227 { 228 int rc; 229 230 /* wrap to 1, skip reserved lkey */ 231 rc = ionic_resid_get_shared(&dev->inuse_mrid, 1, 232 dev->inuse_mrid.inuse_size); 233 if (rc < 0) 234 return rc; 235 236 *mrid = ionic_mrid(rc, dev->next_mrkey++); 237 return 0; 238 } 239 240 static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid) 241 { 242 int rc = 0; 243 244 rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1); 245 if (rc < 0) 246 return rc; 247 248 *qpid = IB_QPT_GSI; 249 return 0; 250 } 251 252 static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid, 253 u8 *udma_idx, u8 udma_mask) 254 { 255 unsigned int size, base, bound; 256 int udma_i, udma_x, udma_ix; 257 int rc = -EINVAL; 258 259 udma_x = dev->next_qpid_udma_idx; 260 261 dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1; 262 263 for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) { 264 udma_ix = udma_i ^ udma_x; 265 266 if (!(udma_mask & BIT(udma_ix))) 267 continue; 268 269 size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count; 270 base = size * udma_ix; 271 bound = base + size; 272 273 /* skip reserved SMI and GSI qpids in group zero */ 274 if (!base) 275 base = 2; 276 277 rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound); 278 if (rc >= 0) { 279 *qpid = ionic_bitid_to_qid(rc, 280 dev->lif_cfg.udma_qgrp_shift, 281 dev->half_qpid_udma_shift); 282 *udma_idx = udma_ix; 283 284 rc = 0; 285 break; 286 } 287 } 288 289 return rc; 290 } 291 292 static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr) 293 { 294 int rc, dbpage_num; 295 296 /* wrap to 1, skip kernel reserved */ 297 rc = ionic_resid_get_shared(&dev->inuse_dbid, 1, 298 dev->inuse_dbid.inuse_size); 299 if (rc < 0) 300 return rc; 301 302 dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc; 303 *addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT); 304 305 *dbid = rc; 306 307 return 0; 308 } 309 310 static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid) 311 { 312 ionic_resid_put(&dev->inuse_pdid, pdid); 313 } 314 315 static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid) 316 { 317 ionic_resid_put(&dev->inuse_ahid, ahid); 318 } 319 320 static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid) 321 { 322 ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid)); 323 } 324 325 static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid) 326 { 327 u32 bitid = ionic_qid_to_bitid(qpid, 328 dev->lif_cfg.udma_qgrp_shift, 329 dev->half_qpid_udma_shift); 330 331 ionic_resid_put(&dev->inuse_qpid, bitid); 332 } 333 334 static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid) 335 { 336 ionic_resid_put(&dev->inuse_dbid, dbid); 337 } 338 339 static struct rdma_user_mmap_entry* 340 ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size, 341 unsigned long pfn, u8 mmap_flags, u64 *offset) 342 { 343 struct ionic_mmap_entry *entry; 344 int rc; 345 346 entry = kzalloc_obj(*entry); 347 if (!entry) 348 return NULL; 349 350 entry->size = size; 351 entry->pfn = pfn; 352 entry->mmap_flags = mmap_flags; 353 354 rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry, 355 entry->size); 356 if (rc) { 357 kfree(entry); 358 return NULL; 359 } 360 361 if (offset) 362 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 363 364 return &entry->rdma_entry; 365 } 366 367 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 368 { 369 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 370 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 371 struct ionic_ctx_resp resp = {}; 372 struct ionic_ctx_req req; 373 phys_addr_t db_phys = 0; 374 int rc; 375 376 rc = ib_copy_validate_udata_in(udata, req, rsvd); 377 if (rc) 378 return rc; 379 380 /* try to allocate dbid for user ctx */ 381 rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys); 382 if (rc < 0) 383 return rc; 384 385 ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid); 386 387 ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 388 PHYS_PFN(db_phys), 0, NULL); 389 if (!ctx->mmap_dbell) { 390 rc = -ENOMEM; 391 goto err_mmap_dbell; 392 } 393 394 if (dev->lif_cfg.phc_state) { 395 ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0, 396 IONIC_MMAP_PHC, 397 &resp.phc_offset); 398 if (!ctx->mmap_phc) { 399 rc = -ENOMEM; 400 goto err_mmap_phc; 401 } 402 } 403 404 resp.page_shift = PAGE_SHIFT; 405 406 resp.dbell_offset = db_phys & ~PAGE_MASK; 407 408 resp.version = dev->lif_cfg.rdma_version; 409 resp.qp_opcodes = dev->lif_cfg.qp_opcodes; 410 resp.admin_opcodes = dev->lif_cfg.admin_opcodes; 411 412 resp.sq_qtype = dev->lif_cfg.sq_qtype; 413 resp.rq_qtype = dev->lif_cfg.rq_qtype; 414 resp.cq_qtype = dev->lif_cfg.cq_qtype; 415 resp.admin_qtype = dev->lif_cfg.aq_qtype; 416 resp.max_stride = dev->lif_cfg.max_stride; 417 resp.max_spec = IONIC_SPEC_HIGH; 418 419 resp.udma_count = dev->lif_cfg.udma_count; 420 resp.expdb_mask = dev->lif_cfg.expdb_mask; 421 422 if (dev->lif_cfg.sq_expdb) 423 resp.expdb_qtypes |= IONIC_EXPDB_SQ; 424 if (dev->lif_cfg.rq_expdb) 425 resp.expdb_qtypes |= IONIC_EXPDB_RQ; 426 427 rc = ib_respond_udata(udata, resp); 428 if (rc) 429 goto err_resp; 430 431 return 0; 432 433 err_resp: 434 rdma_user_mmap_entry_remove(ctx->mmap_phc); 435 err_mmap_phc: 436 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 437 err_mmap_dbell: 438 ionic_put_dbid(dev, ctx->dbid); 439 440 return rc; 441 } 442 443 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx) 444 { 445 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 446 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 447 448 rdma_user_mmap_entry_remove(ctx->mmap_phc); 449 rdma_user_mmap_entry_remove(ctx->mmap_dbell); 450 ionic_put_dbid(dev, ctx->dbid); 451 } 452 453 static int ionic_mmap_phc_state(struct ionic_ibdev *dev, 454 struct vm_area_struct *vma) 455 { 456 if (!(vma->vm_flags & VM_SHARED)) 457 return -EINVAL; 458 459 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) 460 return -EPERM; 461 462 vm_flags_clear(vma, VM_MAYWRITE); 463 464 return vm_insert_page(vma, vma->vm_start, 465 virt_to_page(dev->lif_cfg.phc_state)); 466 } 467 468 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma) 469 { 470 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); 471 struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 472 struct rdma_user_mmap_entry *rdma_entry; 473 struct ionic_mmap_entry *ionic_entry; 474 int rc = 0; 475 476 rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma); 477 if (!rdma_entry) { 478 ibdev_dbg(&dev->ibdev, "not found %#lx\n", 479 vma->vm_pgoff << PAGE_SHIFT); 480 return -EINVAL; 481 } 482 483 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 484 rdma_entry); 485 486 if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) { 487 rc = ionic_mmap_phc_state(dev, vma); 488 rdma_user_mmap_entry_put(rdma_entry); 489 return rc; 490 } 491 492 ibdev_dbg(&dev->ibdev, "writecombine? %d\n", 493 ionic_entry->mmap_flags & IONIC_MMAP_WC); 494 if (ionic_entry->mmap_flags & IONIC_MMAP_WC) 495 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 496 else 497 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 498 499 ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n", 500 vma->vm_start, ionic_entry->pfn, ionic_entry->size); 501 rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn, 502 ionic_entry->size, vma->vm_page_prot, 503 rdma_entry); 504 if (rc) 505 ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc); 506 507 rdma_user_mmap_entry_put(rdma_entry); 508 return rc; 509 } 510 511 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 512 { 513 struct ionic_mmap_entry *ionic_entry; 514 515 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, 516 rdma_entry); 517 kfree(ionic_entry); 518 } 519 520 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 521 { 522 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 523 struct ionic_pd *pd = to_ionic_pd(ibpd); 524 int rc; 525 526 rc = ib_no_udata_io(udata); 527 if (rc) 528 return rc; 529 530 return ionic_get_pdid(dev, &pd->pdid); 531 } 532 533 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 534 { 535 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 536 struct ionic_pd *pd = to_ionic_pd(ibpd); 537 int rc; 538 539 rc = ib_no_udata_io(udata); 540 if (rc) 541 return rc; 542 543 ionic_put_pdid(dev, pd->pdid); 544 545 return 0; 546 } 547 548 static int ionic_build_hdr(struct ionic_ibdev *dev, 549 struct ib_ud_header *hdr, 550 const struct rdma_ah_attr *attr, 551 u16 sport, bool want_ecn) 552 { 553 const struct ib_global_route *grh; 554 enum rdma_network_type net; 555 u8 smac[ETH_ALEN]; 556 u16 vlan; 557 int rc; 558 559 if (attr->ah_flags != IB_AH_GRH) 560 return -EINVAL; 561 if (attr->type != RDMA_AH_ATTR_TYPE_ROCE) 562 return -EINVAL; 563 564 grh = rdma_ah_read_grh(attr); 565 566 rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); 567 if (rc) 568 return rc; 569 570 net = rdma_gid_attr_network_type(grh->sgid_attr); 571 572 rc = ib_ud_header_init(0, /* no payload */ 573 0, /* no lrh */ 574 1, /* yes eth */ 575 vlan != 0xffff, 576 0, /* no grh */ 577 net == RDMA_NETWORK_IPV4 ? 4 : 6, 578 1, /* yes udp */ 579 0, /* no imm */ 580 hdr); 581 if (rc) 582 return rc; 583 584 ether_addr_copy(hdr->eth.smac_h, smac); 585 ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); 586 587 if (net == RDMA_NETWORK_IPV4) { 588 hdr->eth.type = cpu_to_be16(ETH_P_IP); 589 hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */ 590 hdr->ip4.ttl = grh->hop_limit; 591 hdr->ip4.tot_len = cpu_to_be16(0xffff); 592 hdr->ip4.saddr = 593 *(const __be32 *)(grh->sgid_attr->gid.raw + 12); 594 hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12); 595 596 if (want_ecn) 597 hdr->ip4.tos = ionic_set_ecn(grh->traffic_class); 598 else 599 hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class); 600 } else { 601 hdr->eth.type = cpu_to_be16(ETH_P_IPV6); 602 hdr->grh.flow_label = cpu_to_be32(grh->flow_label); 603 hdr->grh.hop_limit = grh->hop_limit; 604 hdr->grh.source_gid = grh->sgid_attr->gid; 605 hdr->grh.destination_gid = grh->dgid; 606 607 if (want_ecn) 608 hdr->grh.traffic_class = 609 ionic_set_ecn(grh->traffic_class); 610 else 611 hdr->grh.traffic_class = 612 ionic_clear_ecn(grh->traffic_class); 613 } 614 615 if (vlan != 0xffff) { 616 vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT; 617 hdr->vlan.tag = cpu_to_be16(vlan); 618 hdr->vlan.type = hdr->eth.type; 619 hdr->eth.type = cpu_to_be16(ETH_P_8021Q); 620 } 621 622 hdr->udp.sport = cpu_to_be16(sport); 623 hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT); 624 625 return 0; 626 } 627 628 static void ionic_set_ah_attr(struct ionic_ibdev *dev, 629 struct rdma_ah_attr *ah_attr, 630 struct ib_ud_header *hdr, 631 int sgid_index) 632 { 633 u32 flow_label; 634 u16 vlan = 0; 635 u8 tos, ttl; 636 637 if (hdr->vlan_present) 638 vlan = be16_to_cpu(hdr->vlan.tag); 639 640 if (hdr->ipv4_present) { 641 flow_label = 0; 642 ttl = hdr->ip4.ttl; 643 tos = hdr->ip4.tos; 644 *(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff); 645 *(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr; 646 } else { 647 flow_label = be32_to_cpu(hdr->grh.flow_label); 648 ttl = hdr->grh.hop_limit; 649 tos = hdr->grh.traffic_class; 650 } 651 652 memset(ah_attr, 0, sizeof(*ah_attr)); 653 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE; 654 if (hdr->eth_present) 655 ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h); 656 rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT); 657 rdma_ah_set_port_num(ah_attr, 1); 658 rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos); 659 rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid); 660 } 661 662 static int ionic_create_ah_cmd(struct ionic_ibdev *dev, 663 struct ionic_ah *ah, 664 struct ionic_pd *pd, 665 struct rdma_ah_attr *attr, 666 u32 flags) 667 { 668 struct ionic_admin_wr wr = { 669 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 670 .wqe = { 671 .op = IONIC_V1_ADMIN_CREATE_AH, 672 .len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN), 673 .cmd.create_ah = { 674 .pd_id = cpu_to_le32(pd->pdid), 675 .dbid_flags = cpu_to_le16(dev->lif_cfg.dbid), 676 .id_ver = cpu_to_le32(ah->ahid), 677 } 678 } 679 }; 680 enum ionic_admin_flags admin_flags = 0; 681 dma_addr_t hdr_dma = 0; 682 void *hdr_buf; 683 gfp_t gfp = GFP_ATOMIC; 684 int rc, hdr_len = 0; 685 686 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH) 687 return -EBADRQC; 688 689 if (flags & RDMA_CREATE_AH_SLEEPABLE) 690 gfp = GFP_KERNEL; 691 else 692 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 693 694 rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false); 695 if (rc) 696 return rc; 697 698 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) { 699 if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP)) 700 wr.wqe.cmd.create_ah.csum_profile = 701 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP; 702 else 703 wr.wqe.cmd.create_ah.csum_profile = 704 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP; 705 } else { 706 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP)) 707 wr.wqe.cmd.create_ah.csum_profile = 708 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 709 else 710 wr.wqe.cmd.create_ah.csum_profile = 711 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 712 } 713 714 ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index; 715 716 hdr_buf = kmalloc(PAGE_SIZE, gfp); 717 if (!hdr_buf) 718 return -ENOMEM; 719 720 hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf); 721 hdr_len -= IB_BTH_BYTES; 722 hdr_len -= IB_DETH_BYTES; 723 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 724 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 725 hdr_buf, hdr_len, true); 726 727 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 728 DMA_TO_DEVICE); 729 730 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 731 if (rc) 732 goto err_dma; 733 734 wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma); 735 wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len); 736 737 ionic_admin_post(dev, &wr); 738 rc = ionic_admin_wait(dev, &wr, admin_flags); 739 740 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 741 DMA_TO_DEVICE); 742 err_dma: 743 kfree(hdr_buf); 744 745 return rc; 746 } 747 748 static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags) 749 { 750 struct ionic_admin_wr wr = { 751 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 752 .wqe = { 753 .op = IONIC_V1_ADMIN_DESTROY_AH, 754 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN), 755 .cmd.destroy_ah = { 756 .ah_id = cpu_to_le32(ahid), 757 }, 758 } 759 }; 760 enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN; 761 762 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH) 763 return -EBADRQC; 764 765 if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) 766 admin_flags |= IONIC_ADMIN_F_BUSYWAIT; 767 768 ionic_admin_post(dev, &wr); 769 ionic_admin_wait(dev, &wr, admin_flags); 770 771 /* No host-memory resource is associated with ah, so it is ok 772 * to "succeed" and complete this destroy ah on the host. 773 */ 774 return 0; 775 } 776 777 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 778 struct ib_udata *udata) 779 { 780 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 781 struct rdma_ah_attr *attr = init_attr->ah_attr; 782 struct ionic_pd *pd = to_ionic_pd(ibah->pd); 783 struct ionic_ah *ah = to_ionic_ah(ibah); 784 struct ionic_ah_resp resp = {}; 785 u32 flags = init_attr->flags; 786 int rc; 787 788 rc = ib_is_udata_in_empty(udata); 789 if (rc) 790 return rc; 791 792 rc = ionic_get_ahid(dev, &ah->ahid); 793 if (rc) 794 return rc; 795 796 rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags); 797 if (rc) 798 goto err_cmd; 799 800 if (udata) { 801 resp.ahid = ah->ahid; 802 803 rc = ib_respond_udata(udata, resp); 804 if (rc) 805 goto err_resp; 806 } 807 808 return 0; 809 810 err_resp: 811 ionic_destroy_ah_cmd(dev, ah->ahid, flags); 812 err_cmd: 813 ionic_put_ahid(dev, ah->ahid); 814 return rc; 815 } 816 817 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 818 { 819 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 820 struct ionic_ah *ah = to_ionic_ah(ibah); 821 822 ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index); 823 824 return 0; 825 } 826 827 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags) 828 { 829 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); 830 struct ionic_ah *ah = to_ionic_ah(ibah); 831 int rc; 832 833 rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags); 834 if (rc) 835 return rc; 836 837 ionic_put_ahid(dev, ah->ahid); 838 839 return 0; 840 } 841 842 static int ionic_create_mr_cmd(struct ionic_ibdev *dev, 843 struct ionic_pd *pd, 844 struct ionic_mr *mr, 845 u64 addr, 846 u64 length) 847 { 848 struct ionic_admin_wr wr = { 849 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 850 .wqe = { 851 .op = IONIC_V1_ADMIN_CREATE_MR, 852 .len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN), 853 .cmd.create_mr = { 854 .va = cpu_to_le64(addr), 855 .length = cpu_to_le64(length), 856 .pd_id = cpu_to_le32(pd->pdid), 857 .page_size_log2 = mr->buf.page_size_log2, 858 .tbl_index = cpu_to_le32(~0), 859 .map_count = cpu_to_le32(mr->buf.tbl_pages), 860 .dma_addr = ionic_pgtbl_dma(&mr->buf, addr), 861 .dbid_flags = cpu_to_le16(mr->flags), 862 .id_ver = cpu_to_le32(mr->mrid), 863 } 864 } 865 }; 866 int rc; 867 868 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR) 869 return -EBADRQC; 870 871 ionic_admin_post(dev, &wr); 872 rc = ionic_admin_wait(dev, &wr, 0); 873 if (!rc) 874 mr->created = true; 875 876 return rc; 877 } 878 879 static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid) 880 { 881 struct ionic_admin_wr wr = { 882 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 883 .wqe = { 884 .op = IONIC_V1_ADMIN_DESTROY_MR, 885 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN), 886 .cmd.destroy_mr = { 887 .mr_id = cpu_to_le32(mrid), 888 }, 889 } 890 }; 891 892 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR) 893 return -EBADRQC; 894 895 ionic_admin_post(dev, &wr); 896 897 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 898 } 899 900 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access) 901 { 902 struct ionic_pd *pd = to_ionic_pd(ibpd); 903 struct ionic_mr *mr; 904 905 mr = kzalloc_obj(*mr); 906 if (!mr) 907 return ERR_PTR(-ENOMEM); 908 909 mr->ibmr.lkey = IONIC_DMA_LKEY; 910 mr->ibmr.rkey = IONIC_DMA_RKEY; 911 912 if (pd) 913 pd->flags |= IONIC_QPF_PRIVILEGED; 914 915 return &mr->ibmr; 916 } 917 918 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 919 u64 addr, int access, struct ib_dmah *dmah, 920 struct ib_udata *udata) 921 { 922 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 923 struct ionic_pd *pd = to_ionic_pd(ibpd); 924 struct ionic_mr *mr; 925 unsigned long pg_sz; 926 int rc; 927 928 rc = ib_no_udata_io(udata); 929 if (rc) 930 return ERR_PTR(rc); 931 932 if (dmah) 933 return ERR_PTR(-EOPNOTSUPP); 934 935 mr = kzalloc_obj(*mr); 936 if (!mr) 937 return ERR_PTR(-ENOMEM); 938 939 rc = ionic_get_mrid(dev, &mr->mrid); 940 if (rc) 941 goto err_mrid; 942 943 mr->ibmr.lkey = mr->mrid; 944 mr->ibmr.rkey = mr->mrid; 945 mr->ibmr.iova = addr; 946 mr->ibmr.length = length; 947 948 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 949 950 mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); 951 if (IS_ERR(mr->umem)) { 952 rc = PTR_ERR(mr->umem); 953 goto err_umem; 954 } 955 956 pg_sz = ib_umem_find_best_pgsz(mr->umem, 957 dev->lif_cfg.page_size_supported, 958 addr); 959 if (!pg_sz) { 960 rc = -EINVAL; 961 goto err_pgtbl; 962 } 963 964 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 965 if (rc) 966 goto err_pgtbl; 967 968 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 969 if (rc) 970 goto err_cmd; 971 972 ionic_pgtbl_unbuf(dev, &mr->buf); 973 974 return &mr->ibmr; 975 976 err_cmd: 977 ionic_pgtbl_unbuf(dev, &mr->buf); 978 err_pgtbl: 979 ib_umem_release(mr->umem); 980 err_umem: 981 ionic_put_mrid(dev, mr->mrid); 982 err_mrid: 983 kfree(mr); 984 return ERR_PTR(rc); 985 } 986 987 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 988 u64 length, u64 addr, int fd, int access, 989 struct ib_dmah *dmah, 990 struct uverbs_attr_bundle *attrs) 991 { 992 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 993 struct ionic_pd *pd = to_ionic_pd(ibpd); 994 struct ib_umem_dmabuf *umem_dmabuf; 995 struct ionic_mr *mr; 996 u64 pg_sz; 997 int rc; 998 999 if (dmah) 1000 return ERR_PTR(-EOPNOTSUPP); 1001 1002 mr = kzalloc_obj(*mr); 1003 if (!mr) 1004 return ERR_PTR(-ENOMEM); 1005 1006 rc = ionic_get_mrid(dev, &mr->mrid); 1007 if (rc) 1008 goto err_mrid; 1009 1010 mr->ibmr.lkey = mr->mrid; 1011 mr->ibmr.rkey = mr->mrid; 1012 mr->ibmr.iova = addr; 1013 mr->ibmr.length = length; 1014 1015 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access); 1016 1017 umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length, 1018 fd, access); 1019 if (IS_ERR(umem_dmabuf)) { 1020 rc = PTR_ERR(umem_dmabuf); 1021 goto err_umem; 1022 } 1023 1024 mr->umem = &umem_dmabuf->umem; 1025 1026 pg_sz = ib_umem_find_best_pgsz(mr->umem, 1027 dev->lif_cfg.page_size_supported, 1028 addr); 1029 if (!pg_sz) { 1030 rc = -EINVAL; 1031 goto err_pgtbl; 1032 } 1033 1034 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); 1035 if (rc) 1036 goto err_pgtbl; 1037 1038 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); 1039 if (rc) 1040 goto err_cmd; 1041 1042 ionic_pgtbl_unbuf(dev, &mr->buf); 1043 1044 return &mr->ibmr; 1045 1046 err_cmd: 1047 ionic_pgtbl_unbuf(dev, &mr->buf); 1048 err_pgtbl: 1049 ib_umem_release(mr->umem); 1050 err_umem: 1051 ionic_put_mrid(dev, mr->mrid); 1052 err_mrid: 1053 kfree(mr); 1054 return ERR_PTR(rc); 1055 } 1056 1057 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1058 { 1059 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1060 struct ionic_mr *mr = to_ionic_mr(ibmr); 1061 int rc; 1062 1063 rc = ib_no_udata_io(udata); 1064 if (rc) 1065 return rc; 1066 1067 if (!mr->ibmr.lkey) 1068 goto out; 1069 1070 if (mr->created) { 1071 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1072 if (rc) 1073 return rc; 1074 } 1075 1076 ionic_pgtbl_unbuf(dev, &mr->buf); 1077 1078 if (mr->umem) 1079 ib_umem_release(mr->umem); 1080 1081 ionic_put_mrid(dev, mr->mrid); 1082 1083 out: 1084 kfree(mr); 1085 1086 return 0; 1087 } 1088 1089 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 1090 u32 max_sg) 1091 { 1092 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); 1093 struct ionic_pd *pd = to_ionic_pd(ibpd); 1094 struct ionic_mr *mr; 1095 int rc; 1096 1097 if (type != IB_MR_TYPE_MEM_REG) 1098 return ERR_PTR(-EINVAL); 1099 1100 mr = kzalloc_obj(*mr); 1101 if (!mr) 1102 return ERR_PTR(-ENOMEM); 1103 1104 rc = ionic_get_mrid(dev, &mr->mrid); 1105 if (rc) 1106 goto err_mrid; 1107 1108 mr->ibmr.lkey = mr->mrid; 1109 mr->ibmr.rkey = mr->mrid; 1110 1111 mr->flags = IONIC_MRF_PHYS_MR; 1112 1113 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE); 1114 if (rc) 1115 goto err_pgtbl; 1116 1117 mr->buf.tbl_pages = 0; 1118 1119 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1120 if (rc) 1121 goto err_cmd; 1122 1123 return &mr->ibmr; 1124 1125 err_cmd: 1126 ionic_pgtbl_unbuf(dev, &mr->buf); 1127 err_pgtbl: 1128 ionic_put_mrid(dev, mr->mrid); 1129 err_mrid: 1130 kfree(mr); 1131 return ERR_PTR(rc); 1132 } 1133 1134 static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma) 1135 { 1136 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1137 struct ionic_mr *mr = to_ionic_mr(ibmr); 1138 1139 ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma); 1140 return ionic_pgtbl_page(&mr->buf, dma); 1141 } 1142 1143 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1144 unsigned int *sg_offset) 1145 { 1146 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); 1147 struct ionic_mr *mr = to_ionic_mr(ibmr); 1148 int rc; 1149 1150 /* mr must be allocated using ib_alloc_mr() */ 1151 if (unlikely(!mr->buf.tbl_limit)) 1152 return -EINVAL; 1153 1154 mr->buf.tbl_pages = 0; 1155 1156 if (mr->buf.tbl_buf) 1157 dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1158 mr->buf.tbl_size, DMA_TO_DEVICE); 1159 1160 ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents); 1161 rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page); 1162 1163 mr->buf.page_size_log2 = order_base_2(ibmr->page_size); 1164 1165 if (mr->buf.tbl_buf) 1166 dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma, 1167 mr->buf.tbl_size, DMA_TO_DEVICE); 1168 1169 return rc; 1170 } 1171 1172 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 1173 { 1174 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1175 struct ionic_pd *pd = to_ionic_pd(ibmw->pd); 1176 struct ionic_mr *mr = to_ionic_mw(ibmw); 1177 int rc; 1178 1179 rc = ib_no_udata_io(udata); 1180 if (rc) 1181 return rc; 1182 1183 rc = ionic_get_mrid(dev, &mr->mrid); 1184 if (rc) 1185 return rc; 1186 1187 mr->ibmw.rkey = mr->mrid; 1188 1189 if (mr->ibmw.type == IB_MW_TYPE_1) 1190 mr->flags = IONIC_MRF_MW_1; 1191 else 1192 mr->flags = IONIC_MRF_MW_2; 1193 1194 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); 1195 if (rc) 1196 goto err_cmd; 1197 1198 return 0; 1199 1200 err_cmd: 1201 ionic_put_mrid(dev, mr->mrid); 1202 return rc; 1203 } 1204 1205 int ionic_dealloc_mw(struct ib_mw *ibmw) 1206 { 1207 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); 1208 struct ionic_mr *mr = to_ionic_mw(ibmw); 1209 int rc; 1210 1211 rc = ionic_destroy_mr_cmd(dev, mr->mrid); 1212 if (rc) 1213 return rc; 1214 1215 ionic_put_mrid(dev, mr->mrid); 1216 1217 return 0; 1218 } 1219 1220 static int ionic_create_cq_cmd(struct ionic_ibdev *dev, 1221 struct ionic_ctx *ctx, 1222 struct ionic_cq *cq, 1223 struct ionic_tbl_buf *buf) 1224 { 1225 const u16 dbid = ionic_ctx_dbid(dev, ctx); 1226 struct ionic_admin_wr wr = { 1227 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1228 .wqe = { 1229 .op = IONIC_V1_ADMIN_CREATE_CQ, 1230 .len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN), 1231 .cmd.create_cq = { 1232 .eq_id = cpu_to_le32(cq->eqid), 1233 .depth_log2 = cq->q.depth_log2, 1234 .stride_log2 = cq->q.stride_log2, 1235 .page_size_log2 = buf->page_size_log2, 1236 .tbl_index = cpu_to_le32(~0), 1237 .map_count = cpu_to_le32(buf->tbl_pages), 1238 .dma_addr = ionic_pgtbl_dma(buf, 0), 1239 .dbid_flags = cpu_to_le16(dbid), 1240 .id_ver = cpu_to_le32(cq->cqid), 1241 } 1242 } 1243 }; 1244 1245 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) 1246 return -EBADRQC; 1247 1248 ionic_admin_post(dev, &wr); 1249 1250 return ionic_admin_wait(dev, &wr, 0); 1251 } 1252 1253 static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid) 1254 { 1255 struct ionic_admin_wr wr = { 1256 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1257 .wqe = { 1258 .op = IONIC_V1_ADMIN_DESTROY_CQ, 1259 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN), 1260 .cmd.destroy_cq = { 1261 .cq_id = cpu_to_le32(cqid), 1262 }, 1263 } 1264 }; 1265 1266 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ) 1267 return -EBADRQC; 1268 1269 ionic_admin_post(dev, &wr); 1270 1271 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1272 } 1273 1274 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1275 struct uverbs_attr_bundle *attrs) 1276 { 1277 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1278 struct ib_udata *udata = &attrs->driver_udata; 1279 struct ionic_ctx *ctx = 1280 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 1281 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1282 struct ionic_tbl_buf buf = {}; 1283 struct ionic_cq_resp resp = {}; 1284 struct ionic_cq_req req; 1285 int udma_idx = 0, rc; 1286 1287 if (udata) { 1288 rc = ib_copy_validate_udata_in(udata, req, rsvd); 1289 if (rc) 1290 return rc; 1291 } 1292 1293 vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 1294 1295 if (udata) 1296 vcq->udma_mask &= req.udma_mask; 1297 1298 if (!vcq->udma_mask) { 1299 rc = -EINVAL; 1300 goto err_init; 1301 } 1302 1303 for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) { 1304 if (!(vcq->udma_mask & BIT(udma_idx))) 1305 continue; 1306 1307 rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, 1308 &req.cq[udma_idx], 1309 &resp.cqid[udma_idx], 1310 udma_idx); 1311 if (rc) 1312 goto err_init; 1313 1314 rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf); 1315 if (rc) 1316 goto err_cmd; 1317 1318 ionic_pgtbl_unbuf(dev, &buf); 1319 } 1320 1321 vcq->ibcq.cqe = attr->cqe; 1322 1323 if (udata) { 1324 resp.udma_mask = vcq->udma_mask; 1325 1326 rc = ib_respond_udata(udata, resp); 1327 if (rc) 1328 goto err_resp; 1329 } 1330 1331 return 0; 1332 1333 err_resp: 1334 while (udma_idx) { 1335 --udma_idx; 1336 if (!(vcq->udma_mask & BIT(udma_idx))) 1337 continue; 1338 ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1339 err_cmd: 1340 ionic_pgtbl_unbuf(dev, &buf); 1341 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1342 err_init: 1343 ; 1344 } 1345 1346 return rc; 1347 } 1348 1349 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1350 { 1351 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); 1352 struct ionic_vcq *vcq = to_ionic_vcq(ibcq); 1353 int udma_idx, rc_tmp, rc = 0; 1354 1355 rc = ib_no_udata_io(udata); 1356 if (rc) 1357 return rc; 1358 1359 for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) { 1360 --udma_idx; 1361 1362 if (!(vcq->udma_mask & BIT(udma_idx))) 1363 continue; 1364 1365 rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); 1366 if (rc_tmp) { 1367 if (!rc) 1368 rc = rc_tmp; 1369 1370 continue; 1371 } 1372 1373 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); 1374 } 1375 1376 return rc; 1377 } 1378 1379 static bool pd_remote_privileged(struct ib_pd *pd) 1380 { 1381 return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; 1382 } 1383 1384 static int ionic_create_qp_cmd(struct ionic_ibdev *dev, 1385 struct ionic_pd *pd, 1386 struct ionic_cq *send_cq, 1387 struct ionic_cq *recv_cq, 1388 struct ionic_qp *qp, 1389 struct ionic_tbl_buf *sq_buf, 1390 struct ionic_tbl_buf *rq_buf, 1391 struct ib_qp_init_attr *attr) 1392 { 1393 const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); 1394 const u32 flags = to_ionic_qp_flags(0, 0, 1395 qp->sq_cmb & IONIC_CMB_ENABLE, 1396 qp->rq_cmb & IONIC_CMB_ENABLE, 1397 qp->sq_spec, qp->rq_spec, 1398 pd->flags & IONIC_QPF_PRIVILEGED, 1399 pd_remote_privileged(&pd->ibpd)); 1400 struct ionic_admin_wr wr = { 1401 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1402 .wqe = { 1403 .op = IONIC_V1_ADMIN_CREATE_QP, 1404 .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), 1405 .cmd.create_qp = { 1406 .pd_id = cpu_to_le32(pd->pdid), 1407 .priv_flags = cpu_to_be32(flags), 1408 .type_state = to_ionic_qp_type(attr->qp_type), 1409 .dbid_flags = cpu_to_le16(dbid), 1410 .id_ver = cpu_to_le32(qp->qpid), 1411 } 1412 } 1413 }; 1414 1415 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP) 1416 return -EBADRQC; 1417 1418 if (qp->has_sq) { 1419 wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid); 1420 wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2; 1421 wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2; 1422 wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2; 1423 wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0); 1424 wr.wqe.cmd.create_qp.sq_map_count = 1425 cpu_to_le32(sq_buf->tbl_pages); 1426 wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); 1427 } 1428 1429 if (qp->has_rq) { 1430 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); 1431 wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.depth_log2; 1432 wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.stride_log2; 1433 wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; 1434 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); 1435 wr.wqe.cmd.create_qp.rq_map_count = 1436 cpu_to_le32(rq_buf->tbl_pages); 1437 wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); 1438 } 1439 1440 ionic_admin_post(dev, &wr); 1441 1442 return ionic_admin_wait(dev, &wr, 0); 1443 } 1444 1445 static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, 1446 struct ionic_pd *pd, 1447 struct ionic_qp *qp, 1448 struct ib_qp_attr *attr, 1449 int mask) 1450 { 1451 const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, 1452 attr->en_sqd_async_notify, 1453 qp->sq_cmb & IONIC_CMB_ENABLE, 1454 qp->rq_cmb & IONIC_CMB_ENABLE, 1455 qp->sq_spec, qp->rq_spec, 1456 pd->flags & IONIC_QPF_PRIVILEGED, 1457 pd_remote_privileged(qp->ibqp.pd)); 1458 const u8 state = to_ionic_qp_modify_state(attr->qp_state, 1459 attr->cur_qp_state); 1460 struct ionic_admin_wr wr = { 1461 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1462 .wqe = { 1463 .op = IONIC_V1_ADMIN_MODIFY_QP, 1464 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN), 1465 .cmd.mod_qp = { 1466 .attr_mask = cpu_to_be32(mask), 1467 .access_flags = cpu_to_be16(flags), 1468 .rq_psn = cpu_to_le32(attr->rq_psn), 1469 .sq_psn = cpu_to_le32(attr->sq_psn), 1470 .rate_limit_kbps = 1471 cpu_to_le32(attr->rate_limit), 1472 .pmtu = (attr->path_mtu + 7), 1473 .retry = (attr->retry_cnt | 1474 (attr->rnr_retry << 4)), 1475 .rnr_timer = attr->min_rnr_timer, 1476 .retry_timeout = attr->timeout, 1477 .type_state = state, 1478 .id_ver = cpu_to_le32(qp->qpid), 1479 } 1480 } 1481 }; 1482 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1483 void *hdr_buf = NULL; 1484 dma_addr_t hdr_dma = 0; 1485 int rc, hdr_len = 0; 1486 u16 sport; 1487 1488 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP) 1489 return -EBADRQC; 1490 1491 if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) { 1492 /* Note, round up/down was already done for allocating 1493 * resources on the device. The allocation order is in cache 1494 * line size. We can't use the order of the resource 1495 * allocation to determine the order wqes here, because for 1496 * queue length <= one cache line it is not distinct. 1497 * 1498 * Therefore, order wqes is computed again here. 1499 * 1500 * Account for hole and round up to the next order. 1501 */ 1502 wr.wqe.cmd.mod_qp.rsq_depth = 1503 order_base_2(attr->max_dest_rd_atomic + 1); 1504 wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0); 1505 } 1506 1507 if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 1508 /* Account for hole and round down to the next order */ 1509 wr.wqe.cmd.mod_qp.rrq_depth = 1510 order_base_2(attr->max_rd_atomic + 2) - 1; 1511 wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0); 1512 } 1513 1514 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) 1515 wr.wqe.cmd.mod_qp.qkey_dest_qpn = 1516 cpu_to_le32(attr->dest_qp_num); 1517 else 1518 wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey); 1519 1520 if (mask & IB_QP_AV) { 1521 if (!qp->hdr) 1522 return -ENOMEM; 1523 1524 sport = rdma_get_udp_sport(grh->flow_label, 1525 qp->qpid, 1526 attr->dest_qp_num); 1527 1528 rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true); 1529 if (rc) 1530 return rc; 1531 1532 qp->sgid_index = grh->sgid_index; 1533 1534 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1535 if (!hdr_buf) 1536 return -ENOMEM; 1537 1538 hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf); 1539 hdr_len -= IB_BTH_BYTES; 1540 hdr_len -= IB_DETH_BYTES; 1541 ibdev_dbg(&dev->ibdev, "roce packet header template\n"); 1542 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, 1543 hdr_buf, hdr_len, true); 1544 1545 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, 1546 DMA_TO_DEVICE); 1547 1548 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1549 if (rc) 1550 goto err_dma; 1551 1552 if (qp->hdr->ipv4_present) { 1553 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1554 qp->hdr->vlan_present ? 1555 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP : 1556 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; 1557 } else { 1558 wr.wqe.cmd.mod_qp.tfp_csum_profile = 1559 qp->hdr->vlan_present ? 1560 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP : 1561 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; 1562 } 1563 1564 wr.wqe.cmd.mod_qp.ah_id_len = 1565 cpu_to_le32(qp->ahid | (hdr_len << 24)); 1566 wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma); 1567 1568 wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl; 1569 wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2; 1570 } 1571 1572 ionic_admin_post(dev, &wr); 1573 1574 rc = ionic_admin_wait(dev, &wr, 0); 1575 1576 if (mask & IB_QP_AV) 1577 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, 1578 DMA_TO_DEVICE); 1579 err_dma: 1580 if (mask & IB_QP_AV) 1581 kfree(hdr_buf); 1582 1583 return rc; 1584 } 1585 1586 static int ionic_query_qp_cmd(struct ionic_ibdev *dev, 1587 struct ionic_qp *qp, 1588 struct ib_qp_attr *attr, 1589 int mask) 1590 { 1591 struct ionic_admin_wr wr = { 1592 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1593 .wqe = { 1594 .op = IONIC_V1_ADMIN_QUERY_QP, 1595 .len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN), 1596 .cmd.query_qp = { 1597 .id_ver = cpu_to_le32(qp->qpid), 1598 }, 1599 } 1600 }; 1601 struct ionic_v1_admin_query_qp_sq *query_sqbuf; 1602 struct ionic_v1_admin_query_qp_rq *query_rqbuf; 1603 dma_addr_t query_sqdma; 1604 dma_addr_t query_rqdma; 1605 dma_addr_t hdr_dma = 0; 1606 void *hdr_buf = NULL; 1607 int flags, rc; 1608 1609 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP) 1610 return -EBADRQC; 1611 1612 if (qp->has_sq) { 1613 bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB); 1614 1615 attr->cap.max_send_sge = 1616 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 1617 qp->sq_spec, 1618 expdb); 1619 attr->cap.max_inline_data = 1620 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); 1621 } 1622 1623 if (qp->has_rq) { 1624 attr->cap.max_recv_sge = 1625 ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, 1626 qp->rq_spec, 1627 qp->rq_cmb & IONIC_CMB_EXPDB); 1628 } 1629 1630 query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1631 if (!query_sqbuf) 1632 return -ENOMEM; 1633 1634 query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); 1635 if (!query_rqbuf) { 1636 rc = -ENOMEM; 1637 goto err_rqbuf; 1638 } 1639 1640 query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE, 1641 DMA_FROM_DEVICE); 1642 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma); 1643 if (rc) 1644 goto err_sqdma; 1645 1646 query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE, 1647 DMA_FROM_DEVICE); 1648 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma); 1649 if (rc) 1650 goto err_rqdma; 1651 1652 if (mask & IB_QP_AV) { 1653 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); 1654 if (!hdr_buf) { 1655 rc = -ENOMEM; 1656 goto err_hdrbuf; 1657 } 1658 1659 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, 1660 PAGE_SIZE, DMA_FROM_DEVICE); 1661 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); 1662 if (rc) 1663 goto err_hdrdma; 1664 } 1665 1666 wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma); 1667 wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma); 1668 wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma); 1669 wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid); 1670 1671 ionic_admin_post(dev, &wr); 1672 1673 rc = ionic_admin_wait(dev, &wr, 0); 1674 1675 if (rc) 1676 goto err_hdrdma; 1677 1678 flags = be16_to_cpu(query_sqbuf->access_perms_flags | 1679 query_rqbuf->access_perms_flags); 1680 1681 print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1682 query_sqbuf, sizeof(*query_sqbuf), true); 1683 print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1, 1684 query_rqbuf, sizeof(*query_rqbuf), true); 1685 ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x", 1686 qp->qpid, query_rqbuf->state_pmtu, flags); 1687 1688 attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4); 1689 attr->cur_qp_state = attr->qp_state; 1690 attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7; 1691 attr->path_mig_state = IB_MIG_MIGRATED; 1692 attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn); 1693 attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn); 1694 attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn); 1695 attr->dest_qp_num = attr->qkey; 1696 attr->qp_access_flags = from_ionic_qp_flags(flags); 1697 attr->pkey_index = 0; 1698 attr->alt_pkey_index = 0; 1699 attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY); 1700 attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING); 1701 attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1; 1702 attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1; 1703 attr->min_rnr_timer = query_sqbuf->rnr_timer; 1704 attr->port_num = 0; 1705 attr->timeout = query_sqbuf->retry_timeout; 1706 attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf; 1707 attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4; 1708 attr->alt_port_num = 0; 1709 attr->alt_timeout = 0; 1710 attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps); 1711 1712 if (mask & IB_QP_AV) 1713 ionic_set_ah_attr(dev, &attr->ah_attr, 1714 qp->hdr, qp->sgid_index); 1715 1716 err_hdrdma: 1717 if (mask & IB_QP_AV) { 1718 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, 1719 PAGE_SIZE, DMA_FROM_DEVICE); 1720 kfree(hdr_buf); 1721 } 1722 err_hdrbuf: 1723 dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf), 1724 DMA_FROM_DEVICE); 1725 err_rqdma: 1726 dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf), 1727 DMA_FROM_DEVICE); 1728 err_sqdma: 1729 kfree(query_rqbuf); 1730 err_rqbuf: 1731 kfree(query_sqbuf); 1732 1733 return rc; 1734 } 1735 1736 static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid) 1737 { 1738 struct ionic_admin_wr wr = { 1739 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), 1740 .wqe = { 1741 .op = IONIC_V1_ADMIN_DESTROY_QP, 1742 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN), 1743 .cmd.destroy_qp = { 1744 .qp_id = cpu_to_le32(qpid), 1745 }, 1746 } 1747 }; 1748 1749 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP) 1750 return -EBADRQC; 1751 1752 ionic_admin_post(dev, &wr); 1753 1754 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); 1755 } 1756 1757 static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev, 1758 uint32_t wqe_size) 1759 { 1760 switch (wqe_size) { 1761 case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64; 1762 case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128; 1763 case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256; 1764 case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512; 1765 } 1766 1767 return false; 1768 } 1769 1770 static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev, 1771 struct ionic_qp *qp, 1772 struct ib_udata *udata, 1773 int max_data) 1774 { 1775 u8 expdb_stride_log2 = 0; 1776 bool expdb; 1777 int rc; 1778 1779 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1780 goto not_in_cmb; 1781 1782 if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) { 1783 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1784 goto not_in_cmb; 1785 1786 qp->sq_cmb &= IONIC_CMB_SUPPORTED; 1787 } 1788 1789 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) { 1790 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1791 goto not_in_cmb; 1792 1793 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1794 } 1795 1796 qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE); 1797 1798 if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER) 1799 goto not_in_cmb; 1800 1801 if (qp->sq_cmb & IONIC_CMB_EXPDB) 1802 expdb_stride_log2 = qp->sq.stride_log2; 1803 1804 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid, 1805 &qp->sq_cmb_addr, qp->sq_cmb_order, 1806 expdb_stride_log2, &expdb); 1807 if (rc) 1808 goto not_in_cmb; 1809 1810 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) { 1811 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1812 goto err_map; 1813 1814 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1815 } 1816 1817 return; 1818 1819 err_map: 1820 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1821 not_in_cmb: 1822 if (qp->sq_cmb & IONIC_CMB_REQUIRE) 1823 ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n"); 1824 1825 qp->sq_cmb = 0; 1826 qp->sq_cmb_order = IONIC_RES_INVALID; 1827 qp->sq_cmb_pgid = 0; 1828 qp->sq_cmb_addr = 0; 1829 } 1830 1831 static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev, 1832 struct ionic_ctx *ctx, 1833 struct ionic_qp *qp) 1834 { 1835 if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) 1836 return; 1837 1838 if (ctx) 1839 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); 1840 1841 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order); 1842 } 1843 1844 static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 1845 struct ionic_qp *qp, struct ionic_qdesc *sq, 1846 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 1847 int max_data, int sq_spec, struct ib_udata *udata) 1848 { 1849 u32 wqe_size; 1850 int rc = 0; 1851 1852 qp->sq_msn_prod = 0; 1853 qp->sq_msn_cons = 0; 1854 1855 if (!qp->has_sq) { 1856 if (buf) { 1857 buf->tbl_buf = NULL; 1858 buf->tbl_limit = 0; 1859 buf->tbl_pages = 0; 1860 } 1861 if (udata) 1862 rc = ionic_validate_qdesc_zero(sq); 1863 1864 return rc; 1865 } 1866 1867 rc = -EINVAL; 1868 1869 if (max_wr < 0 || max_wr > 0xffff) 1870 return rc; 1871 1872 if (max_sge < 1) 1873 return rc; 1874 1875 if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, 1876 qp->sq_cmb & 1877 IONIC_CMB_EXPDB), 1878 IONIC_SPEC_HIGH)) 1879 return rc; 1880 1881 if (max_data < 0) 1882 return rc; 1883 1884 if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride, 1885 qp->sq_cmb & IONIC_CMB_EXPDB)) 1886 return rc; 1887 1888 if (udata) { 1889 rc = ionic_validate_qdesc(sq); 1890 if (rc) 1891 return rc; 1892 1893 qp->sq_spec = sq_spec; 1894 1895 qp->sq.ptr = NULL; 1896 qp->sq.size = sq->size; 1897 qp->sq.mask = sq->mask; 1898 qp->sq.depth_log2 = sq->depth_log2; 1899 qp->sq.stride_log2 = sq->stride_log2; 1900 1901 qp->sq_meta = NULL; 1902 qp->sq_msn_idx = NULL; 1903 1904 qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); 1905 if (IS_ERR(qp->sq_umem)) 1906 return PTR_ERR(qp->sq_umem); 1907 } else { 1908 qp->sq_umem = NULL; 1909 1910 qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec); 1911 if (sq_spec && !qp->sq_spec) 1912 ibdev_dbg(&dev->ibdev, 1913 "init sq: max_sge %u disables spec\n", 1914 max_sge); 1915 1916 if (qp->sq_cmb & IONIC_CMB_EXPDB) { 1917 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 1918 qp->sq_spec, 1919 true); 1920 1921 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 1922 qp->sq_cmb &= ~IONIC_CMB_EXPDB; 1923 } 1924 1925 if (!(qp->sq_cmb & IONIC_CMB_EXPDB)) 1926 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, 1927 qp->sq_spec, 1928 false); 1929 1930 rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev, 1931 max_wr, wqe_size); 1932 if (rc) 1933 return rc; 1934 1935 ionic_queue_dbell_init(&qp->sq, qp->qpid); 1936 1937 qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1); 1938 if (!qp->sq_meta) { 1939 rc = -ENOMEM; 1940 goto err_sq_meta; 1941 } 1942 1943 qp->sq_msn_idx = kmalloc_array((u32)qp->sq.mask + 1, 1944 sizeof(*qp->sq_msn_idx), 1945 GFP_KERNEL); 1946 if (!qp->sq_msn_idx) { 1947 rc = -ENOMEM; 1948 goto err_sq_msn; 1949 } 1950 } 1951 1952 ionic_qp_sq_init_cmb(dev, qp, udata, max_data); 1953 1954 if (qp->sq_cmb & IONIC_CMB_ENABLE) 1955 rc = ionic_pgtbl_init(dev, buf, NULL, 1956 (u64)qp->sq_cmb_pgid << PAGE_SHIFT, 1957 1, PAGE_SIZE); 1958 else 1959 rc = ionic_pgtbl_init(dev, buf, 1960 qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE); 1961 if (rc) 1962 goto err_sq_tbl; 1963 1964 return 0; 1965 1966 err_sq_tbl: 1967 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 1968 kfree(qp->sq_msn_idx); 1969 err_sq_msn: 1970 kfree(qp->sq_meta); 1971 err_sq_meta: 1972 if (qp->sq_umem) 1973 ib_umem_release(qp->sq_umem); 1974 else 1975 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 1976 return rc; 1977 } 1978 1979 static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, 1980 struct ionic_ctx *ctx, 1981 struct ionic_qp *qp) 1982 { 1983 if (!qp->has_sq) 1984 return; 1985 1986 ionic_qp_sq_destroy_cmb(dev, ctx, qp); 1987 1988 kfree(qp->sq_msn_idx); 1989 kfree(qp->sq_meta); 1990 1991 if (qp->sq_umem) 1992 ib_umem_release(qp->sq_umem); 1993 else 1994 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); 1995 } 1996 1997 static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev, 1998 struct ionic_qp *qp, 1999 struct ib_udata *udata) 2000 { 2001 u8 expdb_stride_log2 = 0; 2002 bool expdb; 2003 int rc; 2004 2005 if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) 2006 goto not_in_cmb; 2007 2008 if (qp->rq_cmb & ~IONIC_CMB_SUPPORTED) { 2009 if (qp->rq_cmb & IONIC_CMB_REQUIRE) 2010 goto not_in_cmb; 2011 2012 qp->rq_cmb &= IONIC_CMB_SUPPORTED; 2013 } 2014 2015 if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { 2016 if (qp->rq_cmb & IONIC_CMB_REQUIRE) 2017 goto not_in_cmb; 2018 2019 qp->rq_cmb &= ~IONIC_CMB_EXPDB; 2020 } 2021 2022 qp->rq_cmb_order = order_base_2(qp->rq.size / PAGE_SIZE); 2023 2024 if (qp->rq_cmb_order >= IONIC_RQCMB_ORDER) 2025 goto not_in_cmb; 2026 2027 if (qp->rq_cmb & IONIC_CMB_EXPDB) 2028 expdb_stride_log2 = qp->rq.stride_log2; 2029 2030 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->rq_cmb_pgid, 2031 &qp->rq_cmb_addr, qp->rq_cmb_order, 2032 expdb_stride_log2, &expdb); 2033 if (rc) 2034 goto not_in_cmb; 2035 2036 if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !expdb) { 2037 if (qp->rq_cmb & IONIC_CMB_REQUIRE) 2038 goto err_map; 2039 2040 qp->rq_cmb &= ~IONIC_CMB_EXPDB; 2041 } 2042 2043 return; 2044 2045 err_map: 2046 ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); 2047 not_in_cmb: 2048 if (qp->rq_cmb & IONIC_CMB_REQUIRE) 2049 ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n"); 2050 2051 qp->rq_cmb = 0; 2052 qp->rq_cmb_order = IONIC_RES_INVALID; 2053 qp->rq_cmb_pgid = 0; 2054 qp->rq_cmb_addr = 0; 2055 } 2056 2057 static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev, 2058 struct ionic_ctx *ctx, 2059 struct ionic_qp *qp) 2060 { 2061 if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) 2062 return; 2063 2064 if (ctx) 2065 rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); 2066 2067 ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); 2068 } 2069 2070 static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, 2071 struct ionic_qp *qp, struct ionic_qdesc *rq, 2072 struct ionic_tbl_buf *buf, int max_wr, int max_sge, 2073 int rq_spec, struct ib_udata *udata) 2074 { 2075 int rc = 0, i; 2076 u32 wqe_size; 2077 2078 if (!qp->has_rq) { 2079 if (buf) { 2080 buf->tbl_buf = NULL; 2081 buf->tbl_limit = 0; 2082 buf->tbl_pages = 0; 2083 } 2084 if (udata) 2085 rc = ionic_validate_qdesc_zero(rq); 2086 2087 return rc; 2088 } 2089 2090 rc = -EINVAL; 2091 2092 if (max_wr < 0 || max_wr > 0xffff) 2093 return rc; 2094 2095 if (max_sge < 1) 2096 return rc; 2097 2098 if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), 2099 IONIC_SPEC_HIGH)) 2100 return rc; 2101 2102 if (udata) { 2103 rc = ionic_validate_qdesc(rq); 2104 if (rc) 2105 return rc; 2106 2107 qp->rq_spec = rq_spec; 2108 2109 qp->rq.ptr = NULL; 2110 qp->rq.size = rq->size; 2111 qp->rq.mask = rq->mask; 2112 qp->rq.depth_log2 = rq->depth_log2; 2113 qp->rq.stride_log2 = rq->stride_log2; 2114 2115 qp->rq_meta = NULL; 2116 2117 qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0); 2118 if (IS_ERR(qp->rq_umem)) 2119 return PTR_ERR(qp->rq_umem); 2120 } else { 2121 qp->rq_umem = NULL; 2122 2123 qp->rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec); 2124 if (rq_spec && !qp->rq_spec) 2125 ibdev_dbg(&dev->ibdev, 2126 "init rq: max_sge %u disables spec\n", 2127 max_sge); 2128 2129 if (qp->rq_cmb & IONIC_CMB_EXPDB) { 2130 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2131 qp->rq_spec, 2132 true); 2133 2134 if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) 2135 qp->rq_cmb &= ~IONIC_CMB_EXPDB; 2136 } 2137 2138 if (!(qp->rq_cmb & IONIC_CMB_EXPDB)) 2139 wqe_size = ionic_v1_recv_wqe_min_size(max_sge, 2140 qp->rq_spec, 2141 false); 2142 2143 rc = ionic_queue_init(&qp->rq, dev->lif_cfg.hwdev, 2144 max_wr, wqe_size); 2145 if (rc) 2146 return rc; 2147 2148 ionic_queue_dbell_init(&qp->rq, qp->qpid); 2149 2150 qp->rq_meta = kmalloc_objs(*qp->rq_meta, (u32)qp->rq.mask + 1); 2151 if (!qp->rq_meta) { 2152 rc = -ENOMEM; 2153 goto err_rq_meta; 2154 } 2155 2156 for (i = 0; i < qp->rq.mask; ++i) 2157 qp->rq_meta[i].next = &qp->rq_meta[i + 1]; 2158 qp->rq_meta[i].next = IONIC_META_LAST; 2159 qp->rq_meta_head = &qp->rq_meta[0]; 2160 } 2161 2162 ionic_qp_rq_init_cmb(dev, qp, udata); 2163 2164 if (qp->rq_cmb & IONIC_CMB_ENABLE) 2165 rc = ionic_pgtbl_init(dev, buf, NULL, 2166 (u64)qp->rq_cmb_pgid << PAGE_SHIFT, 2167 1, PAGE_SIZE); 2168 else 2169 rc = ionic_pgtbl_init(dev, buf, 2170 qp->rq_umem, qp->rq.dma, 1, PAGE_SIZE); 2171 if (rc) 2172 goto err_rq_tbl; 2173 2174 return 0; 2175 2176 err_rq_tbl: 2177 ionic_qp_rq_destroy_cmb(dev, ctx, qp); 2178 kfree(qp->rq_meta); 2179 err_rq_meta: 2180 if (qp->rq_umem) 2181 ib_umem_release(qp->rq_umem); 2182 else 2183 ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); 2184 return rc; 2185 } 2186 2187 static void ionic_qp_rq_destroy(struct ionic_ibdev *dev, 2188 struct ionic_ctx *ctx, 2189 struct ionic_qp *qp) 2190 { 2191 if (!qp->has_rq) 2192 return; 2193 2194 ionic_qp_rq_destroy_cmb(dev, ctx, qp); 2195 2196 kfree(qp->rq_meta); 2197 2198 if (qp->rq_umem) 2199 ib_umem_release(qp->rq_umem); 2200 else 2201 ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); 2202 } 2203 2204 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 2205 struct ib_udata *udata) 2206 { 2207 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2208 struct ionic_tbl_buf sq_buf = {}, rq_buf = {}; 2209 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2210 struct ionic_qp *qp = to_ionic_qp(ibqp); 2211 struct ionic_ctx *ctx = 2212 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2213 struct ionic_qp_resp resp = {}; 2214 struct ionic_qp_req req = {}; 2215 struct ionic_cq *cq; 2216 u8 udma_mask; 2217 void *entry; 2218 int rc; 2219 2220 if (udata) { 2221 rc = ib_copy_validate_udata_in(udata, req, rsvd); 2222 if (rc) 2223 return rc; 2224 } else { 2225 req.sq_spec = IONIC_SPEC_HIGH; 2226 req.rq_spec = IONIC_SPEC_HIGH; 2227 } 2228 2229 if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD) 2230 return -EOPNOTSUPP; 2231 2232 qp->state = IB_QPS_RESET; 2233 2234 INIT_LIST_HEAD(&qp->cq_poll_sq); 2235 INIT_LIST_HEAD(&qp->cq_flush_sq); 2236 INIT_LIST_HEAD(&qp->cq_flush_rq); 2237 2238 spin_lock_init(&qp->sq_lock); 2239 spin_lock_init(&qp->rq_lock); 2240 2241 qp->has_sq = 1; 2242 qp->has_rq = 1; 2243 2244 if (attr->qp_type == IB_QPT_GSI) { 2245 rc = ionic_get_gsi_qpid(dev, &qp->qpid); 2246 } else { 2247 udma_mask = BIT(dev->lif_cfg.udma_count) - 1; 2248 2249 if (qp->has_sq) 2250 udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask; 2251 2252 if (qp->has_rq) 2253 udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask; 2254 2255 if (udata && req.udma_mask) 2256 udma_mask &= req.udma_mask; 2257 2258 if (!udma_mask) 2259 return -EINVAL; 2260 2261 rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask); 2262 } 2263 if (rc) 2264 return rc; 2265 2266 qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2267 qp->has_ah = attr->qp_type == IB_QPT_RC; 2268 2269 if (qp->has_ah) { 2270 qp->hdr = kzalloc_obj(*qp->hdr); 2271 if (!qp->hdr) { 2272 rc = -ENOMEM; 2273 goto err_ah_alloc; 2274 } 2275 2276 rc = ionic_get_ahid(dev, &qp->ahid); 2277 if (rc) 2278 goto err_ahid; 2279 } 2280 2281 if (udata) { 2282 if (req.rq_cmb & IONIC_CMB_ENABLE) 2283 qp->rq_cmb = req.rq_cmb; 2284 2285 if (req.sq_cmb & IONIC_CMB_ENABLE) 2286 qp->sq_cmb = req.sq_cmb; 2287 } 2288 2289 rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, 2290 attr->cap.max_send_wr, attr->cap.max_send_sge, 2291 attr->cap.max_inline_data, req.sq_spec, udata); 2292 if (rc) 2293 goto err_sq; 2294 2295 rc = ionic_qp_rq_init(dev, ctx, qp, &req.rq, &rq_buf, 2296 attr->cap.max_recv_wr, attr->cap.max_recv_sge, 2297 req.rq_spec, udata); 2298 if (rc) 2299 goto err_rq; 2300 2301 rc = ionic_create_qp_cmd(dev, pd, 2302 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), 2303 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), 2304 qp, &sq_buf, &rq_buf, attr); 2305 if (rc) 2306 goto err_cmd; 2307 2308 if (udata) { 2309 resp.qpid = qp->qpid; 2310 resp.udma_idx = qp->udma_idx; 2311 2312 if (qp->sq_cmb & IONIC_CMB_ENABLE) { 2313 bool wc; 2314 2315 if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2316 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2317 ibdev_dbg(&dev->ibdev, 2318 "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); 2319 qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2320 } 2321 2322 wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) 2323 != IONIC_CMB_UC; 2324 2325 /* let userspace know the mapping */ 2326 if (wc) 2327 qp->sq_cmb |= IONIC_CMB_WC; 2328 else 2329 qp->sq_cmb |= IONIC_CMB_UC; 2330 2331 qp->mmap_sq_cmb = 2332 ionic_mmap_entry_insert(ctx, 2333 qp->sq.size, 2334 PHYS_PFN(qp->sq_cmb_addr), 2335 wc ? IONIC_MMAP_WC : 0, 2336 &resp.sq_cmb_offset); 2337 if (!qp->mmap_sq_cmb) { 2338 rc = -ENOMEM; 2339 goto err_mmap_sq; 2340 } 2341 2342 resp.sq_cmb = qp->sq_cmb; 2343 } 2344 2345 if (qp->rq_cmb & IONIC_CMB_ENABLE) { 2346 bool wc; 2347 2348 if ((qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == 2349 (IONIC_CMB_WC | IONIC_CMB_UC)) { 2350 ibdev_dbg(&dev->ibdev, 2351 "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); 2352 qp->rq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); 2353 } 2354 2355 if (qp->rq_cmb & IONIC_CMB_EXPDB) 2356 wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) 2357 == IONIC_CMB_WC; 2358 else 2359 wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) 2360 != IONIC_CMB_UC; 2361 2362 /* let userspace know the mapping */ 2363 if (wc) 2364 qp->rq_cmb |= IONIC_CMB_WC; 2365 else 2366 qp->rq_cmb |= IONIC_CMB_UC; 2367 2368 qp->mmap_rq_cmb = 2369 ionic_mmap_entry_insert(ctx, 2370 qp->rq.size, 2371 PHYS_PFN(qp->rq_cmb_addr), 2372 wc ? IONIC_MMAP_WC : 0, 2373 &resp.rq_cmb_offset); 2374 if (!qp->mmap_rq_cmb) { 2375 rc = -ENOMEM; 2376 goto err_mmap_rq; 2377 } 2378 2379 resp.rq_cmb = qp->rq_cmb; 2380 } 2381 2382 rc = ib_respond_udata(udata, resp); 2383 if (rc) 2384 goto err_resp; 2385 } 2386 2387 ionic_pgtbl_unbuf(dev, &rq_buf); 2388 ionic_pgtbl_unbuf(dev, &sq_buf); 2389 2390 qp->ibqp.qp_num = qp->qpid; 2391 2392 init_completion(&qp->qp_rel_comp); 2393 kref_init(&qp->qp_kref); 2394 2395 entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL); 2396 if (entry) { 2397 if (!xa_is_err(entry)) 2398 rc = -EINVAL; 2399 else 2400 rc = xa_err(entry); 2401 2402 goto err_resp; 2403 } 2404 2405 if (qp->has_sq) { 2406 cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx); 2407 2408 attr->cap.max_send_wr = qp->sq.mask; 2409 attr->cap.max_send_sge = 2410 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, 2411 qp->sq_spec, 2412 qp->sq_cmb & IONIC_CMB_EXPDB); 2413 attr->cap.max_inline_data = 2414 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, 2415 qp->sq_cmb & 2416 IONIC_CMB_EXPDB); 2417 qp->sq_cqid = cq->cqid; 2418 } 2419 2420 if (qp->has_rq) { 2421 cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); 2422 2423 attr->cap.max_recv_wr = qp->rq.mask; 2424 attr->cap.max_recv_sge = 2425 ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, 2426 qp->rq_spec, 2427 qp->rq_cmb & IONIC_CMB_EXPDB); 2428 qp->rq_cqid = cq->cqid; 2429 } 2430 2431 return 0; 2432 2433 err_resp: 2434 if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE)) 2435 rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); 2436 err_mmap_rq: 2437 if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE)) 2438 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); 2439 err_mmap_sq: 2440 ionic_destroy_qp_cmd(dev, qp->qpid); 2441 err_cmd: 2442 ionic_pgtbl_unbuf(dev, &rq_buf); 2443 ionic_qp_rq_destroy(dev, ctx, qp); 2444 err_rq: 2445 ionic_pgtbl_unbuf(dev, &sq_buf); 2446 ionic_qp_sq_destroy(dev, ctx, qp); 2447 err_sq: 2448 if (qp->has_ah) 2449 ionic_put_ahid(dev, qp->ahid); 2450 err_ahid: 2451 kfree(qp->hdr); 2452 err_ah_alloc: 2453 ionic_put_qpid(dev, qp->qpid); 2454 return rc; 2455 } 2456 2457 void ionic_notify_flush_cq(struct ionic_cq *cq) 2458 { 2459 if (cq->flush && cq->vcq->ibcq.comp_handler) 2460 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, 2461 cq->vcq->ibcq.cq_context); 2462 } 2463 2464 static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp) 2465 { 2466 if (qp->ibqp.send_cq) 2467 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq, 2468 qp->udma_idx)); 2469 if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq) 2470 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq, 2471 qp->udma_idx)); 2472 } 2473 2474 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2475 { 2476 unsigned long irqflags; 2477 struct ionic_cq *cq; 2478 2479 if (qp->ibqp.send_cq) { 2480 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2481 2482 /* Hold the CQ lock and QP sq_lock to set up flush */ 2483 spin_lock_irqsave(&cq->lock, irqflags); 2484 spin_lock(&qp->sq_lock); 2485 qp->sq_flush = true; 2486 if (!ionic_queue_empty(&qp->sq)) { 2487 cq->flush = true; 2488 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); 2489 } 2490 spin_unlock(&qp->sq_lock); 2491 spin_unlock_irqrestore(&cq->lock, irqflags); 2492 } 2493 2494 if (qp->ibqp.recv_cq) { 2495 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2496 2497 /* Hold the CQ lock and QP rq_lock to set up flush */ 2498 spin_lock_irqsave(&cq->lock, irqflags); 2499 spin_lock(&qp->rq_lock); 2500 qp->rq_flush = true; 2501 if (!ionic_queue_empty(&qp->rq)) { 2502 cq->flush = true; 2503 list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); 2504 } 2505 spin_unlock(&qp->rq_lock); 2506 spin_unlock_irqrestore(&cq->lock, irqflags); 2507 } 2508 } 2509 2510 static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid) 2511 { 2512 struct ionic_v1_cqe *qcqe; 2513 int prod, qtf, qid, type; 2514 bool color; 2515 2516 if (!cq->q.ptr) 2517 return; 2518 2519 color = cq->color; 2520 prod = cq->q.prod; 2521 qcqe = ionic_queue_at(&cq->q, prod); 2522 2523 while (color == ionic_v1_cqe_color(qcqe)) { 2524 qtf = ionic_v1_cqe_qtf(qcqe); 2525 qid = ionic_v1_cqe_qtf_qid(qtf); 2526 type = ionic_v1_cqe_qtf_type(qtf); 2527 2528 if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN) 2529 ionic_v1_cqe_clean(qcqe); 2530 2531 prod = ionic_queue_next(&cq->q, prod); 2532 qcqe = ionic_queue_at(&cq->q, prod); 2533 color = ionic_color_wrap(prod, color); 2534 } 2535 } 2536 2537 static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) 2538 { 2539 unsigned long irqflags; 2540 struct ionic_cq *cq; 2541 int i; 2542 2543 local_irq_save(irqflags); 2544 2545 if (qp->ibqp.send_cq) { 2546 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2547 spin_lock(&cq->lock); 2548 ionic_clean_cq(cq, qp->qpid); 2549 spin_unlock(&cq->lock); 2550 } 2551 2552 if (qp->ibqp.recv_cq) { 2553 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2554 spin_lock(&cq->lock); 2555 ionic_clean_cq(cq, qp->qpid); 2556 spin_unlock(&cq->lock); 2557 } 2558 2559 if (qp->has_sq) { 2560 spin_lock(&qp->sq_lock); 2561 qp->sq_flush = false; 2562 qp->sq_flush_rcvd = false; 2563 qp->sq_msn_prod = 0; 2564 qp->sq_msn_cons = 0; 2565 qp->sq.prod = 0; 2566 qp->sq.cons = 0; 2567 spin_unlock(&qp->sq_lock); 2568 } 2569 2570 if (qp->has_rq) { 2571 spin_lock(&qp->rq_lock); 2572 qp->rq_flush = false; 2573 qp->rq.prod = 0; 2574 qp->rq.cons = 0; 2575 if (qp->rq_meta) { 2576 for (i = 0; i < qp->rq.mask; ++i) 2577 qp->rq_meta[i].next = &qp->rq_meta[i + 1]; 2578 qp->rq_meta[i].next = IONIC_META_LAST; 2579 } 2580 qp->rq_meta_head = &qp->rq_meta[0]; 2581 spin_unlock(&qp->rq_lock); 2582 } 2583 2584 local_irq_restore(irqflags); 2585 } 2586 2587 static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, 2588 enum ib_qp_state attr_state) 2589 { 2590 if (q_state == attr_state) 2591 return true; 2592 2593 if (attr_state == IB_QPS_ERR) 2594 return true; 2595 2596 if (attr_state == IB_QPS_SQE) 2597 return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD; 2598 2599 return false; 2600 } 2601 2602 static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, 2603 enum ib_qp_type type, enum ib_qp_state cur, 2604 enum ib_qp_state next) 2605 { 2606 if (!ext_mask) 2607 return true; 2608 2609 if (ext_mask & ~IB_QP_RATE_LIMIT) 2610 return false; 2611 2612 /* Rate limit is only supported for RC QPs during specific transitions */ 2613 return type == IB_QPT_RC && 2614 ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || 2615 (cur == IB_QPS_RTR && next == IB_QPS_RTS) || 2616 (cur == IB_QPS_RTS && next == IB_QPS_RTS)); 2617 } 2618 2619 static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, 2620 int mask) 2621 { 2622 enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ? 2623 attr->cur_qp_state : qp->state; 2624 enum ib_qp_state next_state = (mask & IB_QP_STATE) ? 2625 attr->qp_state : cur_state; 2626 2627 if ((mask & IB_QP_CUR_STATE) && 2628 !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) 2629 return -EINVAL; 2630 2631 if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || 2632 !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, 2633 qp->ibqp.qp_type, cur_state, next_state)) 2634 return -EINVAL; 2635 2636 /* unprivileged qp not allowed privileged qkey */ 2637 if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) && 2638 qp->ibqp.uobject) 2639 return -EPERM; 2640 2641 return 0; 2642 } 2643 2644 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 2645 struct ib_udata *udata) 2646 { 2647 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2648 struct ionic_pd *pd = to_ionic_pd(ibqp->pd); 2649 struct ionic_qp *qp = to_ionic_qp(ibqp); 2650 int rc; 2651 2652 rc = ib_no_udata_io(udata); 2653 if (rc) 2654 return rc; 2655 2656 rc = ionic_check_modify_qp(qp, attr, mask); 2657 if (rc) 2658 return rc; 2659 2660 if (mask & IB_QP_CAP) 2661 return -EINVAL; 2662 2663 rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask); 2664 if (rc) 2665 return rc; 2666 2667 if (mask & IB_QP_STATE) { 2668 qp->state = attr->qp_state; 2669 2670 if (attr->qp_state == IB_QPS_ERR) { 2671 ionic_flush_qp(dev, qp); 2672 ionic_notify_qp_cqs(dev, qp); 2673 } else if (attr->qp_state == IB_QPS_RESET) { 2674 ionic_reset_qp(dev, qp); 2675 } 2676 } 2677 2678 return 0; 2679 } 2680 2681 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2682 int mask, struct ib_qp_init_attr *init_attr) 2683 { 2684 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2685 struct ionic_qp *qp = to_ionic_qp(ibqp); 2686 int rc; 2687 2688 memset(attr, 0, sizeof(*attr)); 2689 memset(init_attr, 0, sizeof(*init_attr)); 2690 2691 rc = ionic_query_qp_cmd(dev, qp, attr, mask); 2692 if (rc) 2693 return rc; 2694 2695 if (qp->has_sq) 2696 attr->cap.max_send_wr = qp->sq.mask; 2697 2698 if (qp->has_rq) 2699 attr->cap.max_recv_wr = qp->rq.mask; 2700 2701 init_attr->event_handler = ibqp->event_handler; 2702 init_attr->qp_context = ibqp->qp_context; 2703 init_attr->send_cq = ibqp->send_cq; 2704 init_attr->recv_cq = ibqp->recv_cq; 2705 init_attr->srq = ibqp->srq; 2706 init_attr->xrcd = ibqp->xrcd; 2707 init_attr->cap = attr->cap; 2708 init_attr->sq_sig_type = qp->sig_all ? 2709 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2710 init_attr->qp_type = ibqp->qp_type; 2711 init_attr->create_flags = 0; 2712 init_attr->port_num = 0; 2713 init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl; 2714 init_attr->source_qpn = 0; 2715 2716 return rc; 2717 } 2718 2719 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2720 { 2721 struct ionic_ctx *ctx = 2722 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); 2723 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); 2724 struct ionic_qp *qp = to_ionic_qp(ibqp); 2725 unsigned long irqflags; 2726 struct ionic_cq *cq; 2727 int rc; 2728 2729 rc = ib_no_udata_io(udata); 2730 if (rc) 2731 return rc; 2732 2733 rc = ionic_destroy_qp_cmd(dev, qp->qpid); 2734 if (rc) 2735 return rc; 2736 2737 xa_erase_irq(&dev->qp_tbl, qp->qpid); 2738 2739 kref_put(&qp->qp_kref, ionic_qp_complete); 2740 wait_for_completion(&qp->qp_rel_comp); 2741 2742 if (qp->ibqp.send_cq) { 2743 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); 2744 spin_lock_irqsave(&cq->lock, irqflags); 2745 ionic_clean_cq(cq, qp->qpid); 2746 list_del(&qp->cq_poll_sq); 2747 list_del(&qp->cq_flush_sq); 2748 spin_unlock_irqrestore(&cq->lock, irqflags); 2749 } 2750 2751 if (qp->ibqp.recv_cq) { 2752 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); 2753 spin_lock_irqsave(&cq->lock, irqflags); 2754 ionic_clean_cq(cq, qp->qpid); 2755 list_del(&qp->cq_flush_rq); 2756 spin_unlock_irqrestore(&cq->lock, irqflags); 2757 } 2758 2759 ionic_qp_rq_destroy(dev, ctx, qp); 2760 ionic_qp_sq_destroy(dev, ctx, qp); 2761 if (qp->has_ah) { 2762 ionic_put_ahid(dev, qp->ahid); 2763 kfree(qp->hdr); 2764 } 2765 ionic_put_qpid(dev, qp->qpid); 2766 2767 return 0; 2768 } 2769