Lines Matching +full:px +full:- +full:supply
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
42 *nmap = umem->nmap; in get_ib_umem_sgl()
43 return umem->sg_head.sgl; in get_ib_umem_sgl()
54 int rc = -1; in bnxt_re_resolve_dmac_task()
60 rdev = dmac_work->rdev; in bnxt_re_resolve_dmac_task()
61 ah_attr = dmac_work->ah_attr; in bnxt_re_resolve_dmac_task()
62 rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr); in bnxt_re_resolve_dmac_task()
64 dev_err(rdev_to_dev(dmac_work->rdev), in bnxt_re_resolve_dmac_task()
66 atomic_set(&dmac_work->status_wait, rc << 8); in bnxt_re_resolve_dmac_task()
133 if (!rdev || !rdev->netdev) in bnxt_re_get_netdev()
136 netdev = rdev->netdev; in bnxt_re_get_netdev()
138 /* In case of active-backup bond mode, return active slave */ in bnxt_re_get_netdev()
152 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; in bnxt_re_query_device()
156 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); in bnxt_re_query_device()
157 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); in bnxt_re_query_device()
158 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; in bnxt_re_query_device()
159 ib_attr->page_size_cap = dev_attr->page_size_cap; in bnxt_re_query_device()
160 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; in bnxt_re_query_device()
161 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; in bnxt_re_query_device()
162 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; in bnxt_re_query_device()
163 ib_attr->max_qp = dev_attr->max_qp; in bnxt_re_query_device()
164 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; in bnxt_re_query_device()
169 if (rdev->min_tx_depth == 1 && in bnxt_re_query_device()
170 min_tx_depth < dev_attr->max_qp_wqes) in bnxt_re_query_device()
171 rdev->min_tx_depth = min_tx_depth; in bnxt_re_query_device()
172 ib_attr->device_cap_flags = in bnxt_re_query_device()
184 ib_attr->max_send_sge = dev_attr->max_qp_sges; in bnxt_re_query_device()
185 ib_attr->max_recv_sge = dev_attr->max_qp_sges; in bnxt_re_query_device()
186 ib_attr->max_sge_rd = dev_attr->max_qp_sges; in bnxt_re_query_device()
187 ib_attr->max_cq = dev_attr->max_cq; in bnxt_re_query_device()
188 ib_attr->max_cqe = dev_attr->max_cq_wqes; in bnxt_re_query_device()
189 ib_attr->max_mr = dev_attr->max_mr; in bnxt_re_query_device()
190 ib_attr->max_pd = dev_attr->max_pd; in bnxt_re_query_device()
191 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; in bnxt_re_query_device()
192 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; in bnxt_re_query_device()
193 if (dev_attr->is_atomic) { in bnxt_re_query_device()
194 ib_attr->atomic_cap = IB_ATOMIC_GLOB; in bnxt_re_query_device()
195 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; in bnxt_re_query_device()
197 ib_attr->max_ee_rd_atom = 0; in bnxt_re_query_device()
198 ib_attr->max_res_rd_atom = 0; in bnxt_re_query_device()
199 ib_attr->max_ee_init_rd_atom = 0; in bnxt_re_query_device()
200 ib_attr->max_ee = 0; in bnxt_re_query_device()
201 ib_attr->max_rdd = 0; in bnxt_re_query_device()
202 ib_attr->max_mw = dev_attr->max_mw; in bnxt_re_query_device()
203 ib_attr->max_raw_ipv6_qp = 0; in bnxt_re_query_device()
204 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; in bnxt_re_query_device()
205 ib_attr->max_mcast_grp = 0; in bnxt_re_query_device()
206 ib_attr->max_mcast_qp_attach = 0; in bnxt_re_query_device()
207 ib_attr->max_total_mcast_qp_attach = 0; in bnxt_re_query_device()
208 ib_attr->max_ah = dev_attr->max_ah; in bnxt_re_query_device()
209 ib_attr->max_srq = dev_attr->max_srq; in bnxt_re_query_device()
210 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; in bnxt_re_query_device()
211 ib_attr->max_srq_sge = dev_attr->max_srq_sges; in bnxt_re_query_device()
213 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; in bnxt_re_query_device()
214 ib_attr->max_pkeys = 1; in bnxt_re_query_device()
215 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; in bnxt_re_query_device()
216 ib_attr->sig_prot_cap = 0; in bnxt_re_query_device()
217 ib_attr->sig_guard_cap = 0; in bnxt_re_query_device()
218 ib_attr->odp_caps.general_caps = 0; in bnxt_re_query_device()
233 /* GUID should be made as READ-ONLY */ in bnxt_re_modify_device()
236 /* Node Desc should be made as READ-ONLY */ in bnxt_re_modify_device()
291 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; in bnxt_re_query_port()
297 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; in bnxt_re_query_port()
298 port_attr->state = bnxt_re_get_link_state(rdev); in bnxt_re_query_port()
299 if (port_attr->state == IB_PORT_ACTIVE) in bnxt_re_query_port()
300 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; in bnxt_re_query_port()
301 port_attr->max_mtu = IB_MTU_4096; in bnxt_re_query_port()
302 port_attr->active_mtu = iboe_get_mtu(if_getmtu(rdev->netdev)); in bnxt_re_query_port()
303 port_attr->gid_tbl_len = dev_attr->max_sgid; in bnxt_re_query_port()
304 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | in bnxt_re_query_port()
309 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; in bnxt_re_query_port()
310 port_attr->bad_pkey_cntr = 0; in bnxt_re_query_port()
311 port_attr->qkey_viol_cntr = 0; in bnxt_re_query_port()
312 port_attr->pkey_tbl_len = dev_attr->max_pkey; in bnxt_re_query_port()
313 port_attr->lid = 0; in bnxt_re_query_port()
314 port_attr->sm_lid = 0; in bnxt_re_query_port()
315 port_attr->lmc = 0; in bnxt_re_query_port()
316 port_attr->max_vl_num = 4; in bnxt_re_query_port()
317 port_attr->sm_sl = 0; in bnxt_re_query_port()
318 port_attr->subnet_timeout = 0; in bnxt_re_query_port()
319 port_attr->init_type_reply = 0; in bnxt_re_query_port()
320 rdev->espeed = rdev->en_dev->espeed; in bnxt_re_query_port()
322 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) in bnxt_re_query_port()
323 __to_ib_speed_width(rdev->espeed, &active_speed, in bnxt_re_query_port()
326 port_attr->active_speed = active_speed; in bnxt_re_query_port()
327 port_attr->active_width = active_width; in bnxt_re_query_port()
359 return -EINVAL; in bnxt_re_get_port_immutable()
361 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; in bnxt_re_get_port_immutable()
362 immutable->gid_tbl_len = port_attr.gid_tbl_len; in bnxt_re_get_port_immutable()
363 if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) in bnxt_re_get_port_immutable()
364 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; in bnxt_re_get_port_immutable()
365 else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) in bnxt_re_get_port_immutable()
366 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; in bnxt_re_get_port_immutable()
368 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | in bnxt_re_get_port_immutable()
370 immutable->max_mad_size = IB_MGMT_MAD_SIZE; in bnxt_re_get_port_immutable()
378 sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], in bnxt_re_compat_qfwstr()
379 rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], in bnxt_re_compat_qfwstr()
380 rdev->dev_attr->fw_ver[3]); in bnxt_re_compat_qfwstr()
387 return -EINVAL; in bnxt_re_query_pkey()
402 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, in bnxt_re_query_gid()
403 &rdev->qplib_res.sgid_tbl, index, in bnxt_re_query_gid()
414 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; in bnxt_re_del_gid()
422 return -EINVAL; in bnxt_re_del_gid()
424 if (sgid_tbl && sgid_tbl->active) { in bnxt_re_del_gid()
425 if (ctx->idx >= sgid_tbl->max) { in bnxt_re_del_gid()
427 return -EINVAL; in bnxt_re_del_gid()
429 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; in bnxt_re_del_gid()
430 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; in bnxt_re_del_gid()
431 ctx->refcnt--; in bnxt_re_del_gid()
438 * where we issue MODIFY_GID cmd to update the GID entry -- TBD in bnxt_re_del_gid()
440 if (ctx->idx == 0 && in bnxt_re_del_gid()
442 (rdev->gsi_ctx.gsi_sqp || in bnxt_re_del_gid()
443 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { in bnxt_re_del_gid()
446 if (!ctx->refcnt) { in bnxt_re_del_gid()
447 rdev->gid_map[index] = -1; in bnxt_re_del_gid()
448 ctx_tbl = sgid_tbl->ctx; in bnxt_re_del_gid()
449 ctx_tbl[ctx->idx] = NULL; in bnxt_re_del_gid()
454 rdev->gid_map[index] = -1; in bnxt_re_del_gid()
455 if (!ctx->refcnt) { in bnxt_re_del_gid()
460 ctx_tbl = sgid_tbl->ctx; in bnxt_re_del_gid()
461 ctx_tbl[ctx->idx] = NULL; in bnxt_re_del_gid()
470 return -EINVAL; in bnxt_re_del_gid()
484 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; in bnxt_re_add_gid()
485 if ((attr->ndev) && is_vlan_dev(attr->ndev)) in bnxt_re_add_gid()
486 vlan_id = vlan_dev_vlan_id(attr->ndev); in bnxt_re_add_gid()
489 rdev->dev_addr, in bnxt_re_add_gid()
491 if (rc == -EALREADY) { in bnxt_re_add_gid()
493 ctx_tbl = sgid_tbl->ctx; in bnxt_re_add_gid()
497 return -ENOMEM; in bnxt_re_add_gid()
498 ctx->idx = tbl_idx; in bnxt_re_add_gid()
499 ctx->refcnt = 1; in bnxt_re_add_gid()
502 ctx_tbl[tbl_idx]->refcnt++; in bnxt_re_add_gid()
506 rdev->gid_map[index] = tbl_idx; in bnxt_re_add_gid()
515 return -ENOMEM; in bnxt_re_add_gid()
517 ctx_tbl = sgid_tbl->ctx; in bnxt_re_add_gid()
518 ctx->idx = tbl_idx; in bnxt_re_add_gid()
519 ctx->refcnt = 1; in bnxt_re_add_gid()
522 rdev->gid_map[index] = tbl_idx; in bnxt_re_add_gid()
536 struct bnxt_re_legacy_fence_data *fence = &pd->fence; in bnxt_re_legacy_create_fence_wqe()
537 struct ib_mr *ib_mr = &fence->mr->ib_mr; in bnxt_re_legacy_create_fence_wqe()
538 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; in bnxt_re_legacy_create_fence_wqe()
539 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_legacy_create_fence_wqe()
541 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) in bnxt_re_legacy_create_fence_wqe()
545 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; in bnxt_re_legacy_create_fence_wqe()
546 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; in bnxt_re_legacy_create_fence_wqe()
547 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_legacy_create_fence_wqe()
548 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_legacy_create_fence_wqe()
549 wqe->bind.zero_based = false; in bnxt_re_legacy_create_fence_wqe()
550 wqe->bind.parent_l_key = ib_mr->lkey; in bnxt_re_legacy_create_fence_wqe()
551 wqe->bind.va = (u64)fence->va; in bnxt_re_legacy_create_fence_wqe()
552 wqe->bind.length = fence->size; in bnxt_re_legacy_create_fence_wqe()
553 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); in bnxt_re_legacy_create_fence_wqe()
554 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; in bnxt_re_legacy_create_fence_wqe()
557 * wqe->bind.r_key will be set at (re)bind time. in bnxt_re_legacy_create_fence_wqe()
559 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); in bnxt_re_legacy_create_fence_wqe()
566 struct ib_pd *ib_pd = qp->ib_qp.pd; in bnxt_re_legacy_bind_fence_mw()
568 struct bnxt_re_legacy_fence_data *fence = &pd->fence; in bnxt_re_legacy_bind_fence_mw()
569 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; in bnxt_re_legacy_bind_fence_mw()
577 wqe.bind.r_key = fence->bind_rkey; in bnxt_re_legacy_bind_fence_mw()
578 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); in bnxt_re_legacy_bind_fence_mw()
580 dev_dbg(rdev_to_dev(qp->rdev), in bnxt_re_legacy_bind_fence_mw()
581 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", in bnxt_re_legacy_bind_fence_mw()
582 wqe.bind.r_key, qp->qplib_qp.id, pd); in bnxt_re_legacy_bind_fence_mw()
583 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); in bnxt_re_legacy_bind_fence_mw()
585 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); in bnxt_re_legacy_bind_fence_mw()
588 bnxt_qplib_post_send_db(&qp->qplib_qp); in bnxt_re_legacy_bind_fence_mw()
596 struct bnxt_re_legacy_fence_data *fence = &pd->fence; in bnxt_re_legacy_create_fence_mr()
597 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_legacy_create_fence_mr()
606 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) in bnxt_re_legacy_create_fence_mr()
610 /* Allocate a small chunk of memory and dma-map it */ in bnxt_re_legacy_create_fence_mr()
611 fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); in bnxt_re_legacy_create_fence_mr()
612 if (!fence->va) in bnxt_re_legacy_create_fence_mr()
613 return -ENOMEM; in bnxt_re_legacy_create_fence_mr()
614 dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, in bnxt_re_legacy_create_fence_mr()
617 rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); in bnxt_re_legacy_create_fence_mr()
619 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); in bnxt_re_legacy_create_fence_mr()
620 rc = -EIO; in bnxt_re_legacy_create_fence_mr()
621 fence->dma_addr = 0; in bnxt_re_legacy_create_fence_mr()
624 fence->dma_addr = dma_addr; in bnxt_re_legacy_create_fence_mr()
630 fence->mr = mr; in bnxt_re_legacy_create_fence_mr()
631 mr->rdev = rdev; in bnxt_re_legacy_create_fence_mr()
632 mr->qplib_mr.pd = &pd->qplib_pd; in bnxt_re_legacy_create_fence_mr()
633 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; in bnxt_re_legacy_create_fence_mr()
634 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); in bnxt_re_legacy_create_fence_mr()
635 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { in bnxt_re_legacy_create_fence_mr()
636 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_legacy_create_fence_mr()
638 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); in bnxt_re_legacy_create_fence_mr()
642 mr->ib_mr.lkey = mr->qplib_mr.lkey; in bnxt_re_legacy_create_fence_mr()
644 mr->qplib_mr.va = (u64)fence->va; in bnxt_re_legacy_create_fence_mr()
645 mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; in bnxt_re_legacy_create_fence_mr()
648 mrinfo.mrw = &mr->qplib_mr; in bnxt_re_legacy_create_fence_mr()
656 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); in bnxt_re_legacy_create_fence_mr()
658 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); in bnxt_re_legacy_create_fence_mr()
661 mr->ib_mr.lkey = mr->qplib_mr.lkey; in bnxt_re_legacy_create_fence_mr()
662 mr->ib_mr.rkey = mr->qplib_mr.rkey; in bnxt_re_legacy_create_fence_mr()
663 atomic_inc(&rdev->stats.rsors.mr_count); in bnxt_re_legacy_create_fence_mr()
664 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); in bnxt_re_legacy_create_fence_mr()
665 if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) in bnxt_re_legacy_create_fence_mr()
666 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); in bnxt_re_legacy_create_fence_mr()
668 ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); in bnxt_re_legacy_create_fence_mr()
672 "Failed to create fence-MW for PD: %p\n", pd); in bnxt_re_legacy_create_fence_mr()
673 rc = -EINVAL; in bnxt_re_legacy_create_fence_mr()
676 fence->mw = ib_mw; in bnxt_re_legacy_create_fence_mr()
682 if (mr->ib_mr.lkey) { in bnxt_re_legacy_create_fence_mr()
683 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_legacy_create_fence_mr()
684 atomic_dec(&rdev->stats.rsors.mr_count); in bnxt_re_legacy_create_fence_mr()
687 fence->mr = NULL; in bnxt_re_legacy_create_fence_mr()
690 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, in bnxt_re_legacy_create_fence_mr()
692 fence->dma_addr = 0; in bnxt_re_legacy_create_fence_mr()
695 kfree(fence->va); in bnxt_re_legacy_create_fence_mr()
696 fence->va = NULL; in bnxt_re_legacy_create_fence_mr()
702 struct bnxt_re_legacy_fence_data *fence = &pd->fence; in bnxt_re_legacy_destroy_fence_mr()
703 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_legacy_destroy_fence_mr()
704 struct bnxt_re_mr *mr = fence->mr; in bnxt_re_legacy_destroy_fence_mr()
706 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) in bnxt_re_legacy_destroy_fence_mr()
709 if (fence->mw) { in bnxt_re_legacy_destroy_fence_mr()
710 bnxt_re_dealloc_mw(fence->mw); in bnxt_re_legacy_destroy_fence_mr()
711 fence->mw = NULL; in bnxt_re_legacy_destroy_fence_mr()
714 if (mr->ib_mr.rkey) in bnxt_re_legacy_destroy_fence_mr()
715 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, in bnxt_re_legacy_destroy_fence_mr()
717 if (mr->ib_mr.lkey) in bnxt_re_legacy_destroy_fence_mr()
718 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_legacy_destroy_fence_mr()
720 fence->mr = NULL; in bnxt_re_legacy_destroy_fence_mr()
721 atomic_dec(&rdev->stats.rsors.mr_count); in bnxt_re_legacy_destroy_fence_mr()
723 if (fence->dma_addr) { in bnxt_re_legacy_destroy_fence_mr()
724 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, in bnxt_re_legacy_destroy_fence_mr()
727 fence->dma_addr = 0; in bnxt_re_legacy_destroy_fence_mr()
729 kfree(fence->va); in bnxt_re_legacy_destroy_fence_mr()
730 fence->va = NULL; in bnxt_re_legacy_destroy_fence_mr()
737 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; in bnxt_re_get_user_dpi()
744 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); in bnxt_re_get_user_dpi()
750 if (cctx->modes.db_push) { in bnxt_re_get_user_dpi()
752 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, in bnxt_re_get_user_dpi()
765 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_dealloc_pd()
770 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, in bnxt_re_dealloc_pd()
771 &rdev->qplib_res.pd_tbl, in bnxt_re_dealloc_pd()
772 &pd->qplib_pd); in bnxt_re_dealloc_pd()
776 atomic_dec(&rdev->stats.rsors.pd_count); in bnxt_re_dealloc_pd()
785 struct ib_device *ibdev = ibpd->device; in bnxt_re_alloc_pd()
794 pd->rdev = rdev; in bnxt_re_alloc_pd()
795 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { in bnxt_re_alloc_pd()
798 rc = -ENOMEM; in bnxt_re_alloc_pd()
805 if (!ucntx->dpi.dbr) { in bnxt_re_alloc_pd()
811 resp.pdid = pd->qplib_pd.id; in bnxt_re_alloc_pd()
813 resp.dpi = ucntx->dpi.dpi; in bnxt_re_alloc_pd()
814 resp.dbr = (u64)ucntx->dpi.umdbr; in bnxt_re_alloc_pd()
816 if (ucntx->wcdpi.dpi) { in bnxt_re_alloc_pd()
817 resp.wcdpi = ucntx->wcdpi.dpi; in bnxt_re_alloc_pd()
820 if (rdev->dbr_pacing) { in bnxt_re_alloc_pd()
821 WARN_ON(!rdev->dbr_bar_addr); in bnxt_re_alloc_pd()
822 resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; in bnxt_re_alloc_pd()
827 min(udata->outlen, sizeof(resp)), in bnxt_re_alloc_pd()
836 "Failed to create Fence-MR\n"); in bnxt_re_alloc_pd()
838 atomic_inc(&rdev->stats.rsors.pd_count); in bnxt_re_alloc_pd()
839 max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); in bnxt_re_alloc_pd()
840 if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) in bnxt_re_alloc_pd()
841 atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); in bnxt_re_alloc_pd()
845 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, in bnxt_re_alloc_pd()
846 &pd->qplib_pd); in bnxt_re_alloc_pd()
855 struct bnxt_re_dev *rdev = ah->rdev; in bnxt_re_destroy_ah()
861 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); in bnxt_re_destroy_ah()
865 __func__, ah->qplib_ah.id, block, rc); in bnxt_re_destroy_ah()
866 atomic_dec(&rdev->stats.rsors.ah_count); in bnxt_re_destroy_ah()
903 return ib_gid_to_network_type(sgid_attr->gid_type, sgid); in bnxt_re_gid_to_network_type()
916 gid = &ah_info->sgid; in bnxt_re_get_ah_info()
917 gattr = &ah_info->sgid_attr; in bnxt_re_get_ah_info()
919 rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index, in bnxt_re_get_ah_info()
920 gid, &gattr, &ah_attr->grh, NULL); in bnxt_re_get_ah_info()
925 if (gattr->ndev) { in bnxt_re_get_ah_info()
926 if (is_vlan_dev(gattr->ndev)) in bnxt_re_get_ah_info()
927 ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); in bnxt_re_get_ah_info()
928 if_rele(gattr->ndev); in bnxt_re_get_ah_info()
935 ah_info->nw_type = ntype; in bnxt_re_get_ah_info()
942 gindx = rdev->gid_map[gindx]; in _get_sgid_index()
954 ah_attr->grh.dgid.raw) && in bnxt_re_init_dmac()
955 !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { in bnxt_re_init_dmac()
963 resolve_dmac_work->rdev = rdev; in bnxt_re_init_dmac()
964 resolve_dmac_work->ah_attr = ah_attr; in bnxt_re_init_dmac()
965 resolve_dmac_work->ah_info = ah_info; in bnxt_re_init_dmac()
967 atomic_set(&resolve_dmac_work->status_wait, 1); in bnxt_re_init_dmac()
968 INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task); in bnxt_re_init_dmac()
969 queue_work(rdev->resolve_wq, &resolve_dmac_work->work); in bnxt_re_init_dmac()
972 rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF; in bnxt_re_init_dmac()
976 } while (--retry_count); in bnxt_re_init_dmac()
977 if (atomic_read(&resolve_dmac_work->status_wait)) { in bnxt_re_init_dmac()
978 INIT_LIST_HEAD(&resolve_dmac_work->list); in bnxt_re_init_dmac()
979 list_add_tail(&resolve_dmac_work->list, in bnxt_re_init_dmac()
980 &rdev->mac_wq_list); in bnxt_re_init_dmac()
981 return -EFAULT; in bnxt_re_init_dmac()
987 memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN); in bnxt_re_init_dmac()
996 struct ib_pd *ib_pd = ib_ah->pd; in bnxt_re_create_ah()
999 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_create_ah()
1008 if (!(ah_attr->ah_flags & IB_AH_GRH)) in bnxt_re_create_ah()
1009 dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); in bnxt_re_create_ah()
1011 ah->rdev = rdev; in bnxt_re_create_ah()
1012 ah->qplib_ah.pd = &pd->qplib_pd; in bnxt_re_create_ah()
1013 is_user = ib_pd->uobject ? true : false; in bnxt_re_create_ah()
1015 /* Supply the configuration for the HW */ in bnxt_re_create_ah()
1016 memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, in bnxt_re_create_ah()
1018 ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index); in bnxt_re_create_ah()
1019 if (ah->qplib_ah.sgid_index == 0xFF) { in bnxt_re_create_ah()
1021 rc = -EINVAL; in bnxt_re_create_ah()
1024 ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; in bnxt_re_create_ah()
1025 ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; in bnxt_re_create_ah()
1026 ah->qplib_ah.flow_label = ah_attr->grh.flow_label; in bnxt_re_create_ah()
1027 ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; in bnxt_re_create_ah()
1028 ah->qplib_ah.sl = ah_attr->sl; in bnxt_re_create_ah()
1032 ah->qplib_ah.nw_type = ah_info.nw_type; in bnxt_re_create_ah()
1038 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); in bnxt_re_create_ah()
1046 if (ib_pd->uobject) { in bnxt_re_create_ah()
1047 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; in bnxt_re_create_ah()
1053 spin_lock_irqsave(&uctx->sh_lock, flag); in bnxt_re_create_ah()
1054 wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); in bnxt_re_create_ah()
1055 *wrptr = ah->qplib_ah.id; in bnxt_re_create_ah()
1057 spin_unlock_irqrestore(&uctx->sh_lock, flag); in bnxt_re_create_ah()
1059 atomic_inc(&rdev->stats.rsors.ah_count); in bnxt_re_create_ah()
1060 max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); in bnxt_re_create_ah()
1061 if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) in bnxt_re_create_ah()
1062 atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); in bnxt_re_create_ah()
1078 memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, in bnxt_re_query_ah()
1080 ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; in bnxt_re_query_ah()
1081 ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; in bnxt_re_query_ah()
1082 ah_attr->sl = ah->qplib_ah.sl; in bnxt_re_query_ah()
1083 memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); in bnxt_re_query_ah()
1084 ah_attr->ah_flags = IB_AH_GRH; in bnxt_re_query_ah()
1085 ah_attr->port_num = 1; in bnxt_re_query_ah()
1086 ah_attr->static_rate = 0; in bnxt_re_query_ah()
1096 struct bnxt_re_dev *rdev = srq->rdev; in bnxt_re_destroy_srq()
1097 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; in bnxt_re_destroy_srq()
1101 rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); in bnxt_re_destroy_srq()
1105 __func__, qplib_srq->id, rc); in bnxt_re_destroy_srq()
1107 if (srq->umem && !IS_ERR(srq->umem)) in bnxt_re_destroy_srq()
1108 ib_umem_release(srq->umem); in bnxt_re_destroy_srq()
1110 atomic_dec(&rdev->stats.rsors.srq_count); in bnxt_re_destroy_srq()
1123 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) in bnxt_re_get_rwqe_size()
1168 context = pd->ibpd.uobject->context; in bnxt_re_init_user_srq()
1170 qplib_srq = &srq->qplib_srq; in bnxt_re_init_user_srq()
1171 sginfo = &qplib_srq->sginfo; in bnxt_re_init_user_srq()
1173 if (udata->inlen < sizeof(ureq)) in bnxt_re_init_user_srq()
1176 (unsigned int)udata->inlen, in bnxt_re_init_user_srq()
1180 min(udata->inlen, sizeof(ureq))); in bnxt_re_init_user_srq()
1184 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); in bnxt_re_init_user_srq()
1194 srq->umem = umem; in bnxt_re_init_user_srq()
1195 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); in bnxt_re_init_user_srq()
1196 sginfo->npages = ib_umem_num_pages_compat(umem); in bnxt_re_init_user_srq()
1197 qplib_srq->srq_handle = ureq.srq_handle; in bnxt_re_init_user_srq()
1198 qplib_srq->dpi = &cntx->dpi; in bnxt_re_init_user_srq()
1199 qplib_srq->is_user = true; in bnxt_re_init_user_srq()
1214 struct ib_pd *ib_pd = ib_srq->pd; in bnxt_re_create_srq()
1220 rdev = pd->rdev; in bnxt_re_create_srq()
1221 dev_attr = rdev->dev_attr; in bnxt_re_create_srq()
1223 if (rdev->mod_exit) { in bnxt_re_create_srq()
1225 rc = -EIO; in bnxt_re_create_srq()
1229 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { in bnxt_re_create_srq()
1231 rc = -ENOTSUPP; in bnxt_re_create_srq()
1236 context = pd->ibpd.uobject->context; in bnxt_re_create_srq()
1240 if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { in bnxt_re_create_srq()
1241 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); in bnxt_re_create_srq()
1242 rc = -EINVAL; in bnxt_re_create_srq()
1246 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { in bnxt_re_create_srq()
1247 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); in bnxt_re_create_srq()
1248 rc = -EINVAL; in bnxt_re_create_srq()
1252 srq->rdev = rdev; in bnxt_re_create_srq()
1253 srq->qplib_srq.pd = &pd->qplib_pd; in bnxt_re_create_srq()
1254 srq->qplib_srq.dpi = &rdev->dpi_privileged; in bnxt_re_create_srq()
1258 entries = srq_init_attr->attr.max_wr + 1; in bnxt_re_create_srq()
1260 if (entries > dev_attr->max_srq_wqes + 1) in bnxt_re_create_srq()
1261 entries = dev_attr->max_srq_wqes + 1; in bnxt_re_create_srq()
1263 srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ in bnxt_re_create_srq()
1264 srq->qplib_srq.max_wqe = entries; in bnxt_re_create_srq()
1265 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; in bnxt_re_create_srq()
1266 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; in bnxt_re_create_srq()
1267 srq->srq_limit = srq_init_attr->attr.srq_limit; in bnxt_re_create_srq()
1268 srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; in bnxt_re_create_srq()
1269 srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; in bnxt_re_create_srq()
1270 srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; in bnxt_re_create_srq()
1278 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); in bnxt_re_create_srq()
1287 resp.srqid = srq->qplib_srq.id; in bnxt_re_create_srq()
1289 min(udata->outlen, sizeof(resp)), in bnxt_re_create_srq()
1292 bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); in bnxt_re_create_srq()
1296 atomic_inc(&rdev->stats.rsors.srq_count); in bnxt_re_create_srq()
1297 max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); in bnxt_re_create_srq()
1298 if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) in bnxt_re_create_srq()
1299 atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); in bnxt_re_create_srq()
1300 spin_lock_init(&srq->lock); in bnxt_re_create_srq()
1304 if (udata && srq->umem && !IS_ERR(srq->umem)) { in bnxt_re_create_srq()
1305 ib_umem_release(srq->umem); in bnxt_re_create_srq()
1306 srq->umem = NULL; in bnxt_re_create_srq()
1318 struct bnxt_re_dev *rdev = srq->rdev; in bnxt_re_modify_srq()
1327 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) in bnxt_re_modify_srq()
1328 return -EINVAL; in bnxt_re_modify_srq()
1330 srq->qplib_srq.threshold = srq_attr->srq_limit; in bnxt_re_modify_srq()
1331 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); in bnxt_re_modify_srq()
1337 srq->srq_limit = srq_attr->srq_limit; in bnxt_re_modify_srq()
1349 return -EINVAL; in bnxt_re_modify_srq()
1358 struct bnxt_re_dev *rdev = srq->rdev; in bnxt_re_query_srq()
1361 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); in bnxt_re_query_srq()
1364 srq->qplib_srq.id, rc); in bnxt_re_query_srq()
1367 srq_attr->max_wr = srq->qplib_srq.max_wqe; in bnxt_re_query_srq()
1368 srq_attr->max_sge = srq->qplib_srq.max_sge; in bnxt_re_query_srq()
1369 srq_attr->srq_limit = srq->qplib_srq.threshold; in bnxt_re_query_srq()
1383 spin_lock_irqsave(&srq->lock, flags); in bnxt_re_post_srq_recv()
1386 wqe.num_sge = wr->num_sge; in bnxt_re_post_srq_recv()
1387 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; in bnxt_re_post_srq_recv()
1388 wqe.wr_id = wr->wr_id; in bnxt_re_post_srq_recv()
1390 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); in bnxt_re_post_srq_recv()
1395 wr = wr->next; in bnxt_re_post_srq_recv()
1397 spin_unlock_irqrestore(&srq->lock, flags); in bnxt_re_post_srq_recv()
1406 spin_lock_irqsave(&qp->scq->cq_lock, flags); in bnxt_re_lock_cqs()
1407 if (qp->rcq && qp->rcq != qp->scq) in bnxt_re_lock_cqs()
1408 spin_lock(&qp->rcq->cq_lock); in bnxt_re_lock_cqs()
1416 if (qp->rcq && qp->rcq != qp->scq) in bnxt_re_unlock_cqs()
1417 spin_unlock(&qp->rcq->cq_lock); in bnxt_re_unlock_cqs()
1418 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); in bnxt_re_unlock_cqs()
1430 rdev = qp->rdev; in bnxt_re_destroy_gsi_sqp()
1431 gsi_sqp = rdev->gsi_ctx.gsi_sqp; in bnxt_re_destroy_gsi_sqp()
1432 gsi_sah = rdev->gsi_ctx.gsi_sah; in bnxt_re_destroy_gsi_sqp()
1435 mutex_lock(&rdev->qp_lock); in bnxt_re_destroy_gsi_sqp()
1436 list_del(&gsi_sqp->list); in bnxt_re_destroy_gsi_sqp()
1437 mutex_unlock(&rdev->qp_lock); in bnxt_re_destroy_gsi_sqp()
1441 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, in bnxt_re_destroy_gsi_sqp()
1446 atomic_dec(&rdev->stats.rsors.ah_count); in bnxt_re_destroy_gsi_sqp()
1450 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); in bnxt_re_destroy_gsi_sqp()
1456 bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); in bnxt_re_destroy_gsi_sqp()
1459 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); in bnxt_re_destroy_gsi_sqp()
1460 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); in bnxt_re_destroy_gsi_sqp()
1461 kfree(rdev->gsi_ctx.sqp_tbl); in bnxt_re_destroy_gsi_sqp()
1464 rdev->gsi_ctx.gsi_sqp = NULL; in bnxt_re_destroy_gsi_sqp()
1465 rdev->gsi_ctx.gsi_sah = NULL; in bnxt_re_destroy_gsi_sqp()
1466 rdev->gsi_ctx.sqp_tbl = NULL; in bnxt_re_destroy_gsi_sqp()
1467 atomic_dec(&rdev->stats.rsors.qp_count); in bnxt_re_destroy_gsi_sqp()
1478 if (!rdev->rcfw.sp_perf_stats_enabled) in bnxt_re_dump_debug_stats()
1485 if (rdev->rcfw.qp_destroy_stats[i]) { in bnxt_re_dump_debug_stats()
1487 avg_time += rdev->rcfw.qp_destroy_stats[i]; in bnxt_re_dump_debug_stats()
1510 atomic_read(&rdev->stats.rsors.max_qp_count)); in bnxt_re_dump_debug_stats()
1518 struct bnxt_re_dev *rdev = qp->rdev; in bnxt_re_destroy_qp()
1523 mutex_lock(&rdev->qp_lock); in bnxt_re_destroy_qp()
1524 list_del(&qp->list); in bnxt_re_destroy_qp()
1525 active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); in bnxt_re_destroy_qp()
1526 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) in bnxt_re_destroy_qp()
1527 atomic_dec(&rdev->stats.rsors.rc_qp_count); in bnxt_re_destroy_qp()
1528 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) in bnxt_re_destroy_qp()
1529 atomic_dec(&rdev->stats.rsors.ud_qp_count); in bnxt_re_destroy_qp()
1530 mutex_unlock(&rdev->qp_lock); in bnxt_re_destroy_qp()
1532 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_destroy_qp()
1536 __func__, qp->qplib_qp.id, rc); in bnxt_re_destroy_qp()
1538 if (!ib_qp->uobject) { in bnxt_re_destroy_qp()
1540 bnxt_qplib_clean_qp(&qp->qplib_qp); in bnxt_re_destroy_qp()
1544 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_destroy_qp()
1545 if (ib_qp->qp_type == IB_QPT_GSI && in bnxt_re_destroy_qp()
1546 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { in bnxt_re_destroy_qp()
1547 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && in bnxt_re_destroy_qp()
1548 rdev->gsi_ctx.gsi_sqp) { in bnxt_re_destroy_qp()
1551 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_destroy_qp()
1554 if (qp->rumem && !IS_ERR(qp->rumem)) in bnxt_re_destroy_qp()
1555 ib_umem_release(qp->rumem); in bnxt_re_destroy_qp()
1556 if (qp->sumem && !IS_ERR(qp->sumem)) in bnxt_re_destroy_qp()
1557 ib_umem_release(qp->sumem); in bnxt_re_destroy_qp()
1608 rdev = qp->rdev; in bnxt_re_setup_swqe_size()
1609 qplqp = &qp->qplib_qp; in bnxt_re_setup_swqe_size()
1610 sq = &qplqp->sq; in bnxt_re_setup_swqe_size()
1611 dev_attr = rdev->dev_attr; in bnxt_re_setup_swqe_size()
1614 ilsize = ALIGN(init_attr->cap.max_inline_data, align); in bnxt_re_setup_swqe_size()
1616 sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); in bnxt_re_setup_swqe_size()
1617 if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) in bnxt_re_setup_swqe_size()
1618 return -EINVAL; in bnxt_re_setup_swqe_size()
1622 if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && in bnxt_re_setup_swqe_size()
1623 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) in bnxt_re_setup_swqe_size()
1624 sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); in bnxt_re_setup_swqe_size()
1626 if (init_attr->cap.max_inline_data) { in bnxt_re_setup_swqe_size()
1627 qplqp->max_inline_data = sq->wqe_size - in bnxt_re_setup_swqe_size()
1629 init_attr->cap.max_inline_data = qplqp->max_inline_data; in bnxt_re_setup_swqe_size()
1630 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) in bnxt_re_setup_swqe_size()
1631 sq->max_sge = qplqp->max_inline_data / in bnxt_re_setup_swqe_size()
1652 qplib_qp = &qp->qplib_qp; in bnxt_re_init_user_qp()
1653 context = pd->ibpd.uobject->context; in bnxt_re_init_user_qp()
1655 sginfo = &qplib_qp->sq.sginfo; in bnxt_re_init_user_qp()
1657 if (udata->inlen < sizeof(ureq)) in bnxt_re_init_user_qp()
1660 (unsigned int)udata->inlen, in bnxt_re_init_user_qp()
1664 min(udata->inlen, sizeof(ureq))); in bnxt_re_init_user_qp()
1668 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); in bnxt_re_init_user_qp()
1670 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { in bnxt_re_init_user_qp()
1671 psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? in bnxt_re_init_user_qp()
1674 if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) in bnxt_re_init_user_qp()
1676 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? in bnxt_re_init_user_qp()
1677 qplib_qp->sq.max_wqe : in bnxt_re_init_user_qp()
1678 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / in bnxt_re_init_user_qp()
1680 if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) in bnxt_re_init_user_qp()
1694 qp->sumem = umem; in bnxt_re_init_user_qp()
1696 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); in bnxt_re_init_user_qp()
1697 sginfo->npages = ib_umem_num_pages_compat(umem); in bnxt_re_init_user_qp()
1698 qplib_qp->qp_handle = ureq.qp_handle; in bnxt_re_init_user_qp()
1700 if (!qp->qplib_qp.srq) { in bnxt_re_init_user_qp()
1701 sginfo = &qplib_qp->rq.sginfo; in bnxt_re_init_user_qp()
1702 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); in bnxt_re_init_user_qp()
1713 qp->rumem = umem; in bnxt_re_init_user_qp()
1715 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); in bnxt_re_init_user_qp()
1716 sginfo->npages = ib_umem_num_pages_compat(umem); in bnxt_re_init_user_qp()
1719 qplib_qp->dpi = &cntx->dpi; in bnxt_re_init_user_qp()
1720 qplib_qp->is_user = true; in bnxt_re_init_user_qp()
1724 ib_umem_release(qp->sumem); in bnxt_re_init_user_qp()
1725 qp->sumem = NULL; in bnxt_re_init_user_qp()
1726 qplib_qp->sq.sginfo.sghead = NULL; in bnxt_re_init_user_qp()
1727 qplib_qp->sq.sginfo.nmap = 0; in bnxt_re_init_user_qp()
1736 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_create_shadow_qp_ah()
1747 ah->rdev = rdev; in bnxt_re_create_shadow_qp_ah()
1748 ah->qplib_ah.pd = &pd->qplib_pd; in bnxt_re_create_shadow_qp_ah()
1750 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); in bnxt_re_create_shadow_qp_ah()
1754 /* supply the dgid data same as sgid */ in bnxt_re_create_shadow_qp_ah()
1755 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, in bnxt_re_create_shadow_qp_ah()
1757 ah->qplib_ah.sgid_index = 0; in bnxt_re_create_shadow_qp_ah()
1759 ah->qplib_ah.traffic_class = 0; in bnxt_re_create_shadow_qp_ah()
1760 ah->qplib_ah.flow_label = 0; in bnxt_re_create_shadow_qp_ah()
1761 ah->qplib_ah.hop_limit = 1; in bnxt_re_create_shadow_qp_ah()
1762 ah->qplib_ah.sl = 0; in bnxt_re_create_shadow_qp_ah()
1764 ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); in bnxt_re_create_shadow_qp_ah()
1765 dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", in bnxt_re_create_shadow_qp_ah()
1766 ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], in bnxt_re_create_shadow_qp_ah()
1767 ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); in bnxt_re_create_shadow_qp_ah()
1769 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); in bnxt_re_create_shadow_qp_ah()
1775 dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); in bnxt_re_create_shadow_qp_ah()
1776 atomic_inc(&rdev->stats.rsors.ah_count); in bnxt_re_create_shadow_qp_ah()
1795 sah = rdev->gsi_ctx.gsi_sah; in bnxt_re_update_shadow_ah()
1800 if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { in bnxt_re_update_shadow_ah()
1806 gsi_qp = rdev->gsi_ctx.gsi_qp; in bnxt_re_update_shadow_ah()
1807 ib_pd = gsi_qp->ib_qp.pd; in bnxt_re_update_shadow_ah()
1809 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, in bnxt_re_update_shadow_ah()
1810 &sah->qplib_ah, false); in bnxt_re_update_shadow_ah()
1816 atomic_dec(&rdev->stats.rsors.ah_count); in bnxt_re_update_shadow_ah()
1818 rdev->gsi_ctx.gsi_sah = NULL; in bnxt_re_update_shadow_ah()
1820 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, in bnxt_re_update_shadow_ah()
1821 &gsi_qp->qplib_qp); in bnxt_re_update_shadow_ah()
1827 rdev->gsi_ctx.gsi_sah = sah; in bnxt_re_update_shadow_ah()
1828 atomic_inc(&rdev->stats.rsors.ah_count); in bnxt_re_update_shadow_ah()
1836 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_create_shadow_qp()
1846 qp->rdev = rdev; in bnxt_re_create_shadow_qp()
1849 ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); in bnxt_re_create_shadow_qp()
1850 qp->qplib_qp.pd = &pd->qplib_pd; in bnxt_re_create_shadow_qp()
1851 qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; in bnxt_re_create_shadow_qp()
1852 qp->qplib_qp.type = IB_QPT_UD; in bnxt_re_create_shadow_qp()
1854 qp->qplib_qp.max_inline_data = 0; in bnxt_re_create_shadow_qp()
1855 qp->qplib_qp.sig_type = true; in bnxt_re_create_shadow_qp()
1858 qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); in bnxt_re_create_shadow_qp()
1859 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; in bnxt_re_create_shadow_qp()
1860 qp->qplib_qp.sq.max_sge = 2; in bnxt_re_create_shadow_qp()
1862 qp->qplib_qp.sq.q_full_delta = 1; in bnxt_re_create_shadow_qp()
1863 qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; in bnxt_re_create_shadow_qp()
1864 qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; in bnxt_re_create_shadow_qp()
1866 qp->qplib_qp.scq = qp1_qp->scq; in bnxt_re_create_shadow_qp()
1867 qp->qplib_qp.rcq = qp1_qp->rcq; in bnxt_re_create_shadow_qp()
1869 qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ in bnxt_re_create_shadow_qp()
1870 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; in bnxt_re_create_shadow_qp()
1871 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; in bnxt_re_create_shadow_qp()
1872 qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; in bnxt_re_create_shadow_qp()
1873 qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; in bnxt_re_create_shadow_qp()
1875 qp->qplib_qp.rq.q_full_delta = 1; in bnxt_re_create_shadow_qp()
1876 qp->qplib_qp.mtu = qp1_qp->mtu; in bnxt_re_create_shadow_qp()
1877 qp->qplib_qp.dpi = &rdev->dpi_privileged; in bnxt_re_create_shadow_qp()
1879 rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, in bnxt_re_create_shadow_qp()
1884 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); in bnxt_re_create_shadow_qp()
1891 qp->qplib_qp.id); in bnxt_re_create_shadow_qp()
1892 spin_lock_init(&qp->sq_lock); in bnxt_re_create_shadow_qp()
1893 INIT_LIST_HEAD(&qp->list); in bnxt_re_create_shadow_qp()
1894 mutex_lock(&rdev->qp_lock); in bnxt_re_create_shadow_qp()
1895 list_add_tail(&qp->list, &rdev->qp_list); in bnxt_re_create_shadow_qp()
1896 atomic_inc(&rdev->stats.rsors.qp_count); in bnxt_re_create_shadow_qp()
1897 mutex_unlock(&rdev->qp_lock); in bnxt_re_create_shadow_qp()
1900 bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); in bnxt_re_create_shadow_qp()
1915 rdev = qp->rdev; in bnxt_re_init_rq_attr()
1916 qplqp = &qp->qplib_qp; in bnxt_re_init_rq_attr()
1917 rq = &qplqp->rq; in bnxt_re_init_rq_attr()
1918 dev_attr = rdev->dev_attr; in bnxt_re_init_rq_attr()
1920 if (init_attr->srq) { in bnxt_re_init_rq_attr()
1923 srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); in bnxt_re_init_rq_attr()
1926 return -EINVAL; in bnxt_re_init_rq_attr()
1928 qplqp->srq = &srq->qplib_srq; in bnxt_re_init_rq_attr()
1929 rq->max_wqe = 0; in bnxt_re_init_rq_attr()
1931 rq->max_sge = init_attr->cap.max_recv_sge; in bnxt_re_init_rq_attr()
1932 if (rq->max_sge > dev_attr->max_qp_sges) in bnxt_re_init_rq_attr()
1933 rq->max_sge = dev_attr->max_qp_sges; in bnxt_re_init_rq_attr()
1934 init_attr->cap.max_recv_sge = rq->max_sge; in bnxt_re_init_rq_attr()
1935 rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, in bnxt_re_init_rq_attr()
1936 dev_attr->max_qp_sges); in bnxt_re_init_rq_attr()
1940 entries = init_attr->cap.max_recv_wr + 1; in bnxt_re_init_rq_attr()
1942 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); in bnxt_re_init_rq_attr()
1943 rq->q_full_delta = 0; in bnxt_re_init_rq_attr()
1944 rq->sginfo.pgsize = PAGE_SIZE; in bnxt_re_init_rq_attr()
1945 rq->sginfo.pgshft = PAGE_SHIFT; in bnxt_re_init_rq_attr()
1957 rdev = qp->rdev; in bnxt_re_adjust_gsi_rq_attr()
1958 qplqp = &qp->qplib_qp; in bnxt_re_adjust_gsi_rq_attr()
1959 dev_attr = rdev->dev_attr; in bnxt_re_adjust_gsi_rq_attr()
1961 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) in bnxt_re_adjust_gsi_rq_attr()
1962 qplqp->rq.max_sge = dev_attr->max_qp_sges; in bnxt_re_adjust_gsi_rq_attr()
1977 rdev = qp->rdev; in bnxt_re_init_sq_attr()
1978 qplqp = &qp->qplib_qp; in bnxt_re_init_sq_attr()
1979 sq = &qplqp->sq; in bnxt_re_init_sq_attr()
1980 dev_attr = rdev->dev_attr; in bnxt_re_init_sq_attr()
1982 sq->max_sge = init_attr->cap.max_send_sge; in bnxt_re_init_sq_attr()
1983 if (sq->max_sge > dev_attr->max_qp_sges) { in bnxt_re_init_sq_attr()
1984 sq->max_sge = dev_attr->max_qp_sges; in bnxt_re_init_sq_attr()
1985 init_attr->cap.max_send_sge = sq->max_sge; in bnxt_re_init_sq_attr()
1996 entries = init_attr->cap.max_send_wr; in bnxt_re_init_sq_attr()
1997 if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { in bnxt_re_init_sq_attr()
2003 if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) in bnxt_re_init_sq_attr()
2004 entries = rdev->min_tx_depth; in bnxt_re_init_sq_attr()
2008 diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); in bnxt_re_init_sq_attr()
2010 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); in bnxt_re_init_sq_attr()
2011 sq->q_full_delta = diff + 1; in bnxt_re_init_sq_attr()
2017 sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ in bnxt_re_init_sq_attr()
2018 sq->sginfo.pgsize = PAGE_SIZE; in bnxt_re_init_sq_attr()
2019 sq->sginfo.pgshft = PAGE_SHIFT; in bnxt_re_init_sq_attr()
2032 rdev = qp->rdev; in bnxt_re_adjust_gsi_sq_attr()
2033 qplqp = &qp->qplib_qp; in bnxt_re_adjust_gsi_sq_attr()
2034 dev_attr = rdev->dev_attr; in bnxt_re_adjust_gsi_sq_attr()
2036 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { in bnxt_re_adjust_gsi_sq_attr()
2037 entries = init_attr->cap.max_send_wr + 1; in bnxt_re_adjust_gsi_sq_attr()
2039 qplqp->sq.max_wqe = min_t(u32, entries, in bnxt_re_adjust_gsi_sq_attr()
2040 dev_attr->max_qp_wqes + 1); in bnxt_re_adjust_gsi_sq_attr()
2041 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - in bnxt_re_adjust_gsi_sq_attr()
2042 init_attr->cap.max_send_wr; in bnxt_re_adjust_gsi_sq_attr()
2043 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ in bnxt_re_adjust_gsi_sq_attr()
2044 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) in bnxt_re_adjust_gsi_sq_attr()
2045 qplqp->sq.max_sge = dev_attr->max_qp_sges; in bnxt_re_adjust_gsi_sq_attr()
2056 chip_ctx = rdev->chip_ctx; in bnxt_re_init_qp_type()
2057 gsi_ctx = &rdev->gsi_ctx; in bnxt_re_init_qp_type()
2059 qptype = __from_ib_qp_type(init_attr->qp_type); in bnxt_re_init_qp_type()
2063 qptype = -EINVAL; in bnxt_re_init_qp_type()
2067 if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { in bnxt_re_init_qp_type()
2070 gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; in bnxt_re_init_qp_type()
2078 return rdev->chip_ctx->modes.wqe_mode; in bnxt_re_init_qp_wqe_mode()
2093 rdev = qp->rdev; in bnxt_re_init_qp_attr()
2094 qplqp = &qp->qplib_qp; in bnxt_re_init_qp_attr()
2095 dev_attr = rdev->dev_attr; in bnxt_re_init_qp_attr()
2098 context = pd->ibpd.uobject->context; in bnxt_re_init_qp_attr()
2103 qplqp->is_user = false; in bnxt_re_init_qp_attr()
2104 qplqp->pd = &pd->qplib_pd; in bnxt_re_init_qp_attr()
2105 qplqp->qp_handle = (u64)qplqp; in bnxt_re_init_qp_attr()
2106 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? in bnxt_re_init_qp_attr()
2113 qplqp->type = (u8)qptype; in bnxt_re_init_qp_attr()
2114 qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); in bnxt_re_init_qp_attr()
2115 ether_addr_copy(qplqp->smac, rdev->dev_addr); in bnxt_re_init_qp_attr()
2117 if (init_attr->qp_type == IB_QPT_RC) { in bnxt_re_init_qp_attr()
2118 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; in bnxt_re_init_qp_attr()
2119 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; in bnxt_re_init_qp_attr()
2121 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(if_getmtu(rdev->netdev))); in bnxt_re_init_qp_attr()
2122 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ in bnxt_re_init_qp_attr()
2123 if (init_attr->create_flags) { in bnxt_re_init_qp_attr()
2126 init_attr->create_flags); in bnxt_re_init_qp_attr()
2127 return -EOPNOTSUPP; in bnxt_re_init_qp_attr()
2131 if (init_attr->send_cq) { in bnxt_re_init_qp_attr()
2132 cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); in bnxt_re_init_qp_attr()
2135 rc = -EINVAL; in bnxt_re_init_qp_attr()
2138 qplqp->scq = &cq->qplib_cq; in bnxt_re_init_qp_attr()
2139 qp->scq = cq; in bnxt_re_init_qp_attr()
2142 if (init_attr->recv_cq) { in bnxt_re_init_qp_attr()
2143 cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); in bnxt_re_init_qp_attr()
2146 rc = -EINVAL; in bnxt_re_init_qp_attr()
2149 qplqp->rcq = &cq->qplib_cq; in bnxt_re_init_qp_attr()
2150 qp->rcq = cq; in bnxt_re_init_qp_attr()
2157 if (init_attr->qp_type == IB_QPT_GSI) in bnxt_re_init_qp_attr()
2164 if (init_attr->qp_type == IB_QPT_GSI) in bnxt_re_init_qp_attr()
2182 rdev = qp->rdev; in bnxt_re_create_shadow_gsi()
2187 return -ENOMEM; in bnxt_re_create_shadow_gsi()
2188 rdev->gsi_ctx.sqp_tbl = sqp_tbl; in bnxt_re_create_shadow_gsi()
2190 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); in bnxt_re_create_shadow_gsi()
2192 rc = -ENODEV; in bnxt_re_create_shadow_gsi()
2197 rdev->gsi_ctx.gsi_sqp = sqp; in bnxt_re_create_shadow_gsi()
2199 sqp->rcq = qp->rcq; in bnxt_re_create_shadow_gsi()
2200 sqp->scq = qp->scq; in bnxt_re_create_shadow_gsi()
2201 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, in bnxt_re_create_shadow_gsi()
2202 &qp->qplib_qp); in bnxt_re_create_shadow_gsi()
2204 bnxt_qplib_destroy_qp(&rdev->qplib_res, in bnxt_re_create_shadow_gsi()
2205 &sqp->qplib_qp); in bnxt_re_create_shadow_gsi()
2206 rc = -ENODEV; in bnxt_re_create_shadow_gsi()
2211 rdev->gsi_ctx.gsi_sah = sah; in bnxt_re_create_shadow_gsi()
2242 rdev = qp->rdev; in bnxt_re_create_gsi_qp()
2243 qplqp = &qp->qplib_qp; in bnxt_re_create_gsi_qp()
2244 res = &rdev->qplib_res; in bnxt_re_create_gsi_qp()
2245 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; in bnxt_re_create_gsi_qp()
2272 ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); in bnxt_re_test_qp_limits()
2273 if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || in bnxt_re_test_qp_limits()
2274 (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || in bnxt_re_test_qp_limits()
2275 (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || in bnxt_re_test_qp_limits()
2276 (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || in bnxt_re_test_qp_limits()
2277 (ilsize > dev_attr->max_inline_data)) { in bnxt_re_test_qp_limits()
2278 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " in bnxt_re_test_qp_limits()
2281 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, in bnxt_re_test_qp_limits()
2282 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, in bnxt_re_test_qp_limits()
2283 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, in bnxt_re_test_qp_limits()
2284 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, in bnxt_re_test_qp_limits()
2285 init_attr->cap.max_inline_data, in bnxt_re_test_qp_limits()
2286 dev_attr->max_inline_data); in bnxt_re_test_qp_limits()
2317 rdev = pd->rdev; in bnxt_re_create_qp()
2318 dev_attr = rdev->dev_attr; in bnxt_re_create_qp()
2319 if (rdev->mod_exit) { in bnxt_re_create_qp()
2320 rc = -EIO; in bnxt_re_create_qp()
2325 if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { in bnxt_re_create_qp()
2326 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", in bnxt_re_create_qp()
2327 atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); in bnxt_re_create_qp()
2328 rc = -EINVAL; in bnxt_re_create_qp()
2334 rc = -EINVAL; in bnxt_re_create_qp()
2339 rc = -ENOMEM; in bnxt_re_create_qp()
2342 qp->rdev = rdev; in bnxt_re_create_qp()
2348 if (qp_init_attr->qp_type == IB_QPT_GSI && in bnxt_re_create_qp()
2349 !_is_chip_gen_p5_p7(rdev->chip_ctx)) { in bnxt_re_create_qp()
2351 if (rc == -ENODEV) in bnxt_re_create_qp()
2356 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_create_qp()
2365 resp.qpid = qp->qplib_qp.id; in bnxt_re_create_qp()
2367 min(udata->outlen, sizeof(resp)), in bnxt_re_create_qp()
2374 qp->ib_qp.qp_num = qp->qplib_qp.id; in bnxt_re_create_qp()
2375 if (qp_init_attr->qp_type == IB_QPT_GSI) in bnxt_re_create_qp()
2376 rdev->gsi_ctx.gsi_qp = qp; in bnxt_re_create_qp()
2377 spin_lock_init(&qp->sq_lock); in bnxt_re_create_qp()
2378 spin_lock_init(&qp->rq_lock); in bnxt_re_create_qp()
2379 INIT_LIST_HEAD(&qp->list); in bnxt_re_create_qp()
2380 mutex_lock(&rdev->qp_lock); in bnxt_re_create_qp()
2381 list_add_tail(&qp->list, &rdev->qp_list); in bnxt_re_create_qp()
2382 mutex_unlock(&rdev->qp_lock); in bnxt_re_create_qp()
2383 atomic_inc(&rdev->stats.rsors.qp_count); in bnxt_re_create_qp()
2384 active_qps = atomic_read(&rdev->stats.rsors.qp_count); in bnxt_re_create_qp()
2385 if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) in bnxt_re_create_qp()
2386 atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); in bnxt_re_create_qp()
2391 if (qp_init_attr->qp_type == IB_QPT_RC) { in bnxt_re_create_qp()
2392 tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); in bnxt_re_create_qp()
2393 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) in bnxt_re_create_qp()
2394 atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); in bnxt_re_create_qp()
2395 } else if (qp_init_attr->qp_type == IB_QPT_UD) { in bnxt_re_create_qp()
2396 tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); in bnxt_re_create_qp()
2397 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) in bnxt_re_create_qp()
2398 atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); in bnxt_re_create_qp()
2401 return &qp->ib_qp; in bnxt_re_create_qp()
2404 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_create_qp()
2407 if (qp->rumem && !IS_ERR(qp->rumem)) in bnxt_re_create_qp()
2408 ib_umem_release(qp->rumem); in bnxt_re_create_qp()
2409 if (qp->sumem && !IS_ERR(qp->sumem)) in bnxt_re_create_qp()
2410 ib_umem_release(qp->sumem); in bnxt_re_create_qp()
2422 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; in bnxt_re_modify_shadow_qp()
2426 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; in bnxt_re_modify_shadow_qp()
2427 qp->qplib_qp.state = qp1_qp->qplib_qp.state; in bnxt_re_modify_shadow_qp()
2430 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; in bnxt_re_modify_shadow_qp()
2431 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; in bnxt_re_modify_shadow_qp()
2435 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; in bnxt_re_modify_shadow_qp()
2437 qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; in bnxt_re_modify_shadow_qp()
2440 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; in bnxt_re_modify_shadow_qp()
2441 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; in bnxt_re_modify_shadow_qp()
2444 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_modify_shadow_qp()
2463 if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { in get_source_port()
2471 memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); in get_source_port()
2474 memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); in get_source_port()
2480 memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); in get_source_port()
2483 memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); in get_source_port()
2486 qpn = htonl(qp->qplib_qp.dest_qpn); in get_source_port()
2500 type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); in bnxt_re_update_qp_info()
2502 /* User-space can extract ip address with sgid_index. */ in bnxt_re_update_qp_info()
2503 if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { in bnxt_re_update_qp_info()
2504 qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); in bnxt_re_update_qp_info()
2505 qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); in bnxt_re_update_qp_info()
2507 memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, in bnxt_re_update_qp_info()
2508 sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); in bnxt_re_update_qp_info()
2509 memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, in bnxt_re_update_qp_info()
2510 sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); in bnxt_re_update_qp_info()
2514 (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || in bnxt_re_update_qp_info()
2515 qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { in bnxt_re_update_qp_info()
2516 qp->qp_info_entry.s_port = get_source_port(rdev, qp); in bnxt_re_update_qp_info()
2518 qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; in bnxt_re_update_qp_info()
2526 if (qp->sumem) in bnxt_qplib_manage_flush_qp()
2529 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { in bnxt_qplib_manage_flush_qp()
2530 rq = &qp->qplib_qp.rq; in bnxt_qplib_manage_flush_qp()
2531 sq = &qp->qplib_qp.sq; in bnxt_qplib_manage_flush_qp()
2533 dev_dbg(rdev_to_dev(qp->rdev), in bnxt_qplib_manage_flush_qp()
2536 bnxt_qplib_add_flush_qp(&qp->qplib_qp); in bnxt_qplib_manage_flush_qp()
2539 if (sq->hwq.prod != sq->hwq.cons) in bnxt_qplib_manage_flush_qp()
2540 bnxt_re_handle_cqn(&qp->scq->qplib_cq); in bnxt_qplib_manage_flush_qp()
2542 if (qp->rcq && (qp->rcq != qp->scq) && in bnxt_qplib_manage_flush_qp()
2543 (rq->hwq.prod != rq->hwq.cons)) in bnxt_qplib_manage_flush_qp()
2544 bnxt_re_handle_cqn(&qp->rcq->qplib_cq); in bnxt_qplib_manage_flush_qp()
2547 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { in bnxt_qplib_manage_flush_qp()
2548 dev_dbg(rdev_to_dev(qp->rdev), in bnxt_qplib_manage_flush_qp()
2551 bnxt_qplib_clean_qp(&qp->qplib_qp); in bnxt_qplib_manage_flush_qp()
2584 rdev = qp->rdev; in bnxt_re_modify_qp()
2585 dev_attr = rdev->dev_attr; in bnxt_re_modify_qp()
2587 qp->qplib_qp.modify_flags = 0; in bnxt_re_modify_qp()
2588 ppp = &qp->qplib_qp.ppp; in bnxt_re_modify_qp()
2590 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); in bnxt_re_modify_qp()
2591 new_qp_state = qp_attr->qp_state; in bnxt_re_modify_qp()
2593 ib_qp->qp_type, qp_attr_mask)) { in bnxt_re_modify_qp()
2597 qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, in bnxt_re_modify_qp()
2599 return -EINVAL; in bnxt_re_modify_qp()
2603 __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, in bnxt_re_modify_qp()
2604 ib_qp->qp_type, curr_qp_state, new_qp_state); in bnxt_re_modify_qp()
2605 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; in bnxt_re_modify_qp()
2606 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); in bnxt_re_modify_qp()
2613 ppp->req = BNXT_QPLIB_PPP_REQ; in bnxt_re_modify_qp()
2614 ppp->dpi = ureq.dpi; in bnxt_re_modify_qp()
2620 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2622 qp->qplib_qp.en_sqd_async_notify = true; in bnxt_re_modify_qp()
2625 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; in bnxt_re_modify_qp()
2626 qp->qplib_qp.access = in bnxt_re_modify_qp()
2627 __from_ib_access_flags(qp_attr->qp_access_flags); in bnxt_re_modify_qp()
2629 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; in bnxt_re_modify_qp()
2630 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; in bnxt_re_modify_qp()
2631 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; in bnxt_re_modify_qp()
2634 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; in bnxt_re_modify_qp()
2635 qp->qplib_qp.pkey_index = qp_attr->pkey_index; in bnxt_re_modify_qp()
2638 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; in bnxt_re_modify_qp()
2639 qp->qplib_qp.qkey = qp_attr->qkey; in bnxt_re_modify_qp()
2642 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | in bnxt_re_modify_qp()
2649 memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, in bnxt_re_modify_qp()
2650 sizeof(qp->qplib_qp.ah.dgid.data)); in bnxt_re_modify_qp()
2651 qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; in bnxt_re_modify_qp()
2652 qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, in bnxt_re_modify_qp()
2653 qp_attr->ah_attr.grh.sgid_index); in bnxt_re_modify_qp()
2654 qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index; in bnxt_re_modify_qp()
2655 qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; in bnxt_re_modify_qp()
2656 qp->qplib_qp.ah.traffic_class = in bnxt_re_modify_qp()
2657 qp_attr->ah_attr.grh.traffic_class; in bnxt_re_modify_qp()
2658 qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; in bnxt_re_modify_qp()
2659 ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); in bnxt_re_modify_qp()
2661 status = bnxt_re_get_cached_gid(&rdev->ibdev, 1, in bnxt_re_modify_qp()
2662 qp_attr->ah_attr.grh.sgid_index, in bnxt_re_modify_qp()
2664 &qp_attr->ah_attr.grh, NULL); in bnxt_re_modify_qp()
2666 if_rele(sgid_attr->ndev); in bnxt_re_modify_qp()
2668 if (sgid_attr->ndev) { in bnxt_re_modify_qp()
2669 memcpy(qp->qplib_qp.smac, rdev->dev_addr, in bnxt_re_modify_qp()
2676 qp->qplib_qp.nw_type = in bnxt_re_modify_qp()
2680 qp->qplib_qp.nw_type = in bnxt_re_modify_qp()
2684 qp->qplib_qp.nw_type = in bnxt_re_modify_qp()
2689 memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); in bnxt_re_modify_qp()
2692 /* MTU settings allowed only during INIT -> RTR */ in bnxt_re_modify_qp()
2693 if (qp_attr->qp_state == IB_QPS_RTR) { in bnxt_re_modify_qp()
2694 bnxt_re_init_qpmtu(qp, if_getmtu(rdev->netdev), qp_attr_mask, qp_attr, in bnxt_re_modify_qp()
2699 resp.path_mtu = qp->qplib_qp.mtu; in bnxt_re_modify_qp()
2703 qp->qplib_qp.id); in bnxt_re_modify_qp()
2704 return -EINVAL; in bnxt_re_modify_qp()
2710 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; in bnxt_re_modify_qp()
2711 qp->qplib_qp.timeout = qp_attr->timeout; in bnxt_re_modify_qp()
2714 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2716 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; in bnxt_re_modify_qp()
2719 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2721 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; in bnxt_re_modify_qp()
2724 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2726 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; in bnxt_re_modify_qp()
2729 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; in bnxt_re_modify_qp()
2730 qp->qplib_qp.rq.psn = qp_attr->rq_psn; in bnxt_re_modify_qp()
2733 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2736 if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) in bnxt_re_modify_qp()
2739 qp_attr->max_rd_atomic, in bnxt_re_modify_qp()
2740 dev_attr->max_qp_rd_atom); in bnxt_re_modify_qp()
2741 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, in bnxt_re_modify_qp()
2742 dev_attr->max_qp_rd_atom); in bnxt_re_modify_qp()
2745 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; in bnxt_re_modify_qp()
2746 qp->qplib_qp.sq.psn = qp_attr->sq_psn; in bnxt_re_modify_qp()
2749 if (qp_attr->max_dest_rd_atomic > in bnxt_re_modify_qp()
2750 dev_attr->max_qp_init_rd_atom) { in bnxt_re_modify_qp()
2753 qp_attr->max_dest_rd_atomic, in bnxt_re_modify_qp()
2754 dev_attr->max_qp_init_rd_atom); in bnxt_re_modify_qp()
2755 return -EINVAL; in bnxt_re_modify_qp()
2757 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2759 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; in bnxt_re_modify_qp()
2762 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2768 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || in bnxt_re_modify_qp()
2769 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || in bnxt_re_modify_qp()
2770 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || in bnxt_re_modify_qp()
2771 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || in bnxt_re_modify_qp()
2772 (qp_attr->cap.max_inline_data >= in bnxt_re_modify_qp()
2773 dev_attr->max_inline_data)) { in bnxt_re_modify_qp()
2775 "Create QP failed - max exceeded\n"); in bnxt_re_modify_qp()
2776 return -EINVAL; in bnxt_re_modify_qp()
2778 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); in bnxt_re_modify_qp()
2779 if (entries > dev_attr->max_qp_wqes) in bnxt_re_modify_qp()
2780 entries = dev_attr->max_qp_wqes; in bnxt_re_modify_qp()
2781 entries = min_t(u32, entries, dev_attr->max_qp_wqes); in bnxt_re_modify_qp()
2782 qp->qplib_qp.sq.max_wqe = entries; in bnxt_re_modify_qp()
2783 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - in bnxt_re_modify_qp()
2784 qp_attr->cap.max_send_wr; in bnxt_re_modify_qp()
2790 qp->qplib_qp.sq.q_full_delta -= 1; in bnxt_re_modify_qp()
2791 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; in bnxt_re_modify_qp()
2792 if (qp->qplib_qp.rq.max_wqe) { in bnxt_re_modify_qp()
2793 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); in bnxt_re_modify_qp()
2794 if (entries > dev_attr->max_qp_wqes) in bnxt_re_modify_qp()
2795 entries = dev_attr->max_qp_wqes; in bnxt_re_modify_qp()
2796 qp->qplib_qp.rq.max_wqe = entries; in bnxt_re_modify_qp()
2797 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - in bnxt_re_modify_qp()
2798 qp_attr->cap.max_recv_wr; in bnxt_re_modify_qp()
2799 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; in bnxt_re_modify_qp()
2805 qp->qplib_qp.modify_flags |= in bnxt_re_modify_qp()
2807 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; in bnxt_re_modify_qp()
2810 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); in bnxt_re_modify_qp()
2818 ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { in bnxt_re_modify_qp()
2820 resp.ppp_st_idx = ppp->st_idx_en >> in bnxt_re_modify_qp()
2827 min(udata->outlen, sizeof(resp)), in bnxt_re_modify_qp()
2833 if (ib_qp->qp_type == IB_QPT_GSI && in bnxt_re_modify_qp()
2834 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && in bnxt_re_modify_qp()
2835 rdev->gsi_ctx.gsi_sqp) in bnxt_re_modify_qp()
2848 struct bnxt_re_dev *rdev = qp->rdev; in bnxt_re_query_qp()
2854 return -ENOMEM; in bnxt_re_query_qp()
2856 qplib_qp->id = qp->qplib_qp.id; in bnxt_re_query_qp()
2857 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; in bnxt_re_query_qp()
2859 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); in bnxt_re_query_qp()
2862 qplib_qp->id, rc); in bnxt_re_query_qp()
2865 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); in bnxt_re_query_qp()
2866 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); in bnxt_re_query_qp()
2867 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; in bnxt_re_query_qp()
2868 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); in bnxt_re_query_qp()
2869 qp_attr->pkey_index = qplib_qp->pkey_index; in bnxt_re_query_qp()
2870 qp_attr->qkey = qplib_qp->qkey; in bnxt_re_query_qp()
2871 memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data, in bnxt_re_query_qp()
2872 sizeof(qplib_qp->ah.dgid.data)); in bnxt_re_query_qp()
2873 qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label; in bnxt_re_query_qp()
2874 qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index; in bnxt_re_query_qp()
2875 qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit; in bnxt_re_query_qp()
2876 qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class; in bnxt_re_query_qp()
2877 qp_attr->ah_attr.sl = qplib_qp->ah.sl; in bnxt_re_query_qp()
2878 ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); in bnxt_re_query_qp()
2879 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); in bnxt_re_query_qp()
2880 qp_attr->timeout = qplib_qp->timeout; in bnxt_re_query_qp()
2881 qp_attr->retry_cnt = qplib_qp->retry_cnt; in bnxt_re_query_qp()
2882 qp_attr->rnr_retry = qplib_qp->rnr_retry; in bnxt_re_query_qp()
2883 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; in bnxt_re_query_qp()
2884 qp_attr->rq_psn = qplib_qp->rq.psn; in bnxt_re_query_qp()
2885 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; in bnxt_re_query_qp()
2886 qp_attr->sq_psn = qplib_qp->sq.psn; in bnxt_re_query_qp()
2887 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; in bnxt_re_query_qp()
2888 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : in bnxt_re_query_qp()
2890 qp_attr->dest_qp_num = qplib_qp->dest_qpn; in bnxt_re_query_qp()
2892 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; in bnxt_re_query_qp()
2893 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; in bnxt_re_query_qp()
2894 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; in bnxt_re_query_qp()
2895 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; in bnxt_re_query_qp()
2896 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; in bnxt_re_query_qp()
2897 qp_init_attr->cap = qp_attr->cap; in bnxt_re_query_qp()
2910 switch (wr->send_flags) { in bnxt_re_build_raw_send()
2912 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; in bnxt_re_build_raw_send()
2916 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; in bnxt_re_build_raw_send()
2927 struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, in bnxt_re_build_qp1_send()
2929 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; in bnxt_re_build_qp1_send()
2937 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); in bnxt_re_build_qp1_send()
2940 rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); in bnxt_re_build_qp1_send()
2945 qp->qp1_hdr.eth_present = 1; in bnxt_re_build_qp1_send()
2946 ptmac = ah->qplib_ah.dmac; in bnxt_re_build_qp1_send()
2947 memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); in bnxt_re_build_qp1_send()
2949 memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); in bnxt_re_build_qp1_send()
2951 ptmac = qp->qplib_qp.smac; in bnxt_re_build_qp1_send()
2952 memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); in bnxt_re_build_qp1_send()
2954 memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); in bnxt_re_build_qp1_send()
2956 qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); in bnxt_re_build_qp1_send()
2961 qp->qp1_hdr.vlan_present = 1; in bnxt_re_build_qp1_send()
2962 qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); in bnxt_re_build_qp1_send()
2965 qp->qp1_hdr.grh_present = 1; in bnxt_re_build_qp1_send()
2966 qp->qp1_hdr.grh.ip_version = 6; in bnxt_re_build_qp1_send()
2967 qp->qp1_hdr.grh.payload_length = in bnxt_re_build_qp1_send()
2970 qp->qp1_hdr.grh.next_header = 0x1b; in bnxt_re_build_qp1_send()
2971 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); in bnxt_re_build_qp1_send()
2972 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, in bnxt_re_build_qp1_send()
2976 if (wr->opcode == IB_WR_SEND_WITH_IMM) { in bnxt_re_build_qp1_send()
2977 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; in bnxt_re_build_qp1_send()
2978 qp->qp1_hdr.immediate_present = 1; in bnxt_re_build_qp1_send()
2980 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; in bnxt_re_build_qp1_send()
2982 if (wr->send_flags & IB_SEND_SOLICITED) in bnxt_re_build_qp1_send()
2983 qp->qp1_hdr.bth.solicited_event = 1; in bnxt_re_build_qp1_send()
2984 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; in bnxt_re_build_qp1_send()
2986 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); in bnxt_re_build_qp1_send()
2987 qp->qp1_hdr.bth.destination_qpn = IB_QP1; in bnxt_re_build_qp1_send()
2988 qp->qp1_hdr.bth.ack_req = 0; in bnxt_re_build_qp1_send()
2989 qp->send_psn++; in bnxt_re_build_qp1_send()
2990 qp->send_psn &= BTH_PSN_MASK; in bnxt_re_build_qp1_send()
2991 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); in bnxt_re_build_qp1_send()
2994 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); in bnxt_re_build_qp1_send()
2995 qp->qp1_hdr.deth.source_qpn = IB_QP1; in bnxt_re_build_qp1_send()
2998 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); in bnxt_re_build_qp1_send()
3000 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); in bnxt_re_build_qp1_send()
3001 rc = -ENOMEM; in bnxt_re_build_qp1_send()
3003 for (i = wqe->num_sge; i; i--) { in bnxt_re_build_qp1_send()
3004 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; in bnxt_re_build_qp1_send()
3005 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; in bnxt_re_build_qp1_send()
3006 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; in bnxt_re_build_qp1_send()
3008 wqe->sg_list[0].addr = sge.addr; in bnxt_re_build_qp1_send()
3009 wqe->sg_list[0].lkey = sge.lkey; in bnxt_re_build_qp1_send()
3010 wqe->sg_list[0].size = sge.size; in bnxt_re_build_qp1_send()
3011 wqe->num_sge++; in bnxt_re_build_qp1_send()
3023 rdev = qp->rdev; in bnxt_re_build_gsi_send()
3026 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) in bnxt_re_build_gsi_send()
3029 for (indx = 0; indx < wr->num_sge; indx++) { in bnxt_re_build_gsi_send()
3030 wqe->sg_list[indx].addr = wr->sg_list[indx].addr; in bnxt_re_build_gsi_send()
3031 wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; in bnxt_re_build_gsi_send()
3032 wqe->sg_list[indx].size = wr->sg_list[indx].length; in bnxt_re_build_gsi_send()
3033 len += wr->sg_list[indx].length; in bnxt_re_build_gsi_send()
3036 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; in bnxt_re_build_gsi_send()
3052 struct bnxt_re_dev *rdev = qp->rdev; in bnxt_re_build_qp1_recv()
3059 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { in bnxt_re_build_qp1_recv()
3062 * ib_grh (40) - as provided from the wr in bnxt_re_build_qp1_recv()
3064 * MAD (256) - as provided from the wr in bnxt_re_build_qp1_recv()
3069 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) in bnxt_re_build_qp1_recv()
3071 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) in bnxt_re_build_qp1_recv()
3075 ref.addr = wr->sg_list[0].addr; in bnxt_re_build_qp1_recv()
3076 ref.lkey = wr->sg_list[0].lkey; in bnxt_re_build_qp1_recv()
3077 ref.size = wr->sg_list[0].length; in bnxt_re_build_qp1_recv()
3081 wqe->sg_list[0].addr = sge.addr; in bnxt_re_build_qp1_recv()
3082 wqe->sg_list[0].lkey = sge.lkey; in bnxt_re_build_qp1_recv()
3083 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; in bnxt_re_build_qp1_recv()
3084 size -= wqe->sg_list[0].size; in bnxt_re_build_qp1_recv()
3086 dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); in bnxt_re_build_qp1_recv()
3087 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3091 sge.addr += wqe->sg_list[0].size; in bnxt_re_build_qp1_recv()
3095 wqe->sg_list[1].addr = ref.addr + ip_hdr_size; in bnxt_re_build_qp1_recv()
3096 wqe->sg_list[1].lkey = ref.lkey; in bnxt_re_build_qp1_recv()
3097 wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; in bnxt_re_build_qp1_recv()
3098 ref.size -= wqe->sg_list[1].size; in bnxt_re_build_qp1_recv()
3100 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_build_qp1_recv()
3102 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3105 ref.addr += wqe->sg_list[1].size + ip_hdr_size; in bnxt_re_build_qp1_recv()
3108 wqe->sg_list[2].addr = sge.addr; in bnxt_re_build_qp1_recv()
3109 wqe->sg_list[2].lkey = sge.lkey; in bnxt_re_build_qp1_recv()
3110 wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + in bnxt_re_build_qp1_recv()
3112 size -= wqe->sg_list[2].size; in bnxt_re_build_qp1_recv()
3114 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_build_qp1_recv()
3116 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3120 sge.addr += wqe->sg_list[2].size; in bnxt_re_build_qp1_recv()
3123 wqe->sg_list[3].addr = ref.addr; in bnxt_re_build_qp1_recv()
3124 wqe->sg_list[3].lkey = ref.lkey; in bnxt_re_build_qp1_recv()
3125 wqe->sg_list[3].size = ref.size; in bnxt_re_build_qp1_recv()
3126 ref.size -= wqe->sg_list[3].size; in bnxt_re_build_qp1_recv()
3128 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_build_qp1_recv()
3130 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3134 wqe->sg_list[4].addr = sge.addr; in bnxt_re_build_qp1_recv()
3135 wqe->sg_list[4].lkey = sge.lkey; in bnxt_re_build_qp1_recv()
3136 wqe->sg_list[4].size = sge.size; in bnxt_re_build_qp1_recv()
3137 size -= wqe->sg_list[4].size; in bnxt_re_build_qp1_recv()
3139 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_build_qp1_recv()
3141 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3145 wqe->num_sge = 5; in bnxt_re_build_qp1_recv()
3147 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); in bnxt_re_build_qp1_recv()
3148 rc = -ENOMEM; in bnxt_re_build_qp1_recv()
3164 rdev = qp->rdev; in bnxt_re_build_qp1_shadow_qp_recv()
3166 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); in bnxt_re_build_qp1_shadow_qp_recv()
3168 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { in bnxt_re_build_qp1_shadow_qp_recv()
3173 wqe->sg_list[0].addr = sge.addr; in bnxt_re_build_qp1_shadow_qp_recv()
3175 wqe->sg_list[0].lkey = sge.lkey; in bnxt_re_build_qp1_shadow_qp_recv()
3176 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; in bnxt_re_build_qp1_shadow_qp_recv()
3177 if (sge.size < wqe->sg_list[0].size) { in bnxt_re_build_qp1_shadow_qp_recv()
3178 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_build_qp1_shadow_qp_recv()
3180 rc = -ENOMEM; in bnxt_re_build_qp1_shadow_qp_recv()
3184 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; in bnxt_re_build_qp1_shadow_qp_recv()
3185 sqp_entry->sge.addr = wr->sg_list[0].addr; in bnxt_re_build_qp1_shadow_qp_recv()
3186 sqp_entry->sge.lkey = wr->sg_list[0].lkey; in bnxt_re_build_qp1_shadow_qp_recv()
3187 sqp_entry->sge.size = wr->sg_list[0].length; in bnxt_re_build_qp1_shadow_qp_recv()
3189 sqp_entry->wrid = wqe->wr_id; in bnxt_re_build_qp1_shadow_qp_recv()
3190 /* change the wqe->wrid to table index */ in bnxt_re_build_qp1_shadow_qp_recv()
3191 wqe->wr_id = rq_prod_index; in bnxt_re_build_qp1_shadow_qp_recv()
3199 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || in is_ud_qp()
3200 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); in is_ud_qp()
3210 ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); in bnxt_re_build_send_wqe()
3211 wqe->send.q_key = ud_wr(wr)->remote_qkey; in bnxt_re_build_send_wqe()
3212 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; in bnxt_re_build_send_wqe()
3213 wqe->send.avid = ah->qplib_ah.id; in bnxt_re_build_send_wqe()
3215 switch (wr->opcode) { in bnxt_re_build_send_wqe()
3217 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; in bnxt_re_build_send_wqe()
3220 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; in bnxt_re_build_send_wqe()
3221 wqe->send.imm_data = wr->ex.imm_data; in bnxt_re_build_send_wqe()
3224 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; in bnxt_re_build_send_wqe()
3225 wqe->send.inv_key = wr->ex.invalidate_rkey; in bnxt_re_build_send_wqe()
3228 dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", in bnxt_re_build_send_wqe()
3229 __func__, wr->opcode); in bnxt_re_build_send_wqe()
3230 return -EINVAL; in bnxt_re_build_send_wqe()
3232 if (wr->send_flags & IB_SEND_SIGNALED) in bnxt_re_build_send_wqe()
3233 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_build_send_wqe()
3234 if (wr->send_flags & IB_SEND_FENCE) in bnxt_re_build_send_wqe()
3235 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_build_send_wqe()
3236 if (wr->send_flags & IB_SEND_SOLICITED) in bnxt_re_build_send_wqe()
3237 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; in bnxt_re_build_send_wqe()
3238 if (wr->send_flags & IB_SEND_INLINE) in bnxt_re_build_send_wqe()
3239 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; in bnxt_re_build_send_wqe()
3247 switch (wr->opcode) { in bnxt_re_build_rdma_wqe()
3249 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; in bnxt_re_build_rdma_wqe()
3252 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; in bnxt_re_build_rdma_wqe()
3253 wqe->rdma.imm_data = wr->ex.imm_data; in bnxt_re_build_rdma_wqe()
3256 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; in bnxt_re_build_rdma_wqe()
3257 wqe->rdma.inv_key = wr->ex.invalidate_rkey; in bnxt_re_build_rdma_wqe()
3260 return -EINVAL; in bnxt_re_build_rdma_wqe()
3262 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; in bnxt_re_build_rdma_wqe()
3263 wqe->rdma.r_key = rdma_wr(wr)->rkey; in bnxt_re_build_rdma_wqe()
3264 if (wr->send_flags & IB_SEND_SIGNALED) in bnxt_re_build_rdma_wqe()
3265 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_build_rdma_wqe()
3266 if (wr->send_flags & IB_SEND_FENCE) in bnxt_re_build_rdma_wqe()
3267 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_build_rdma_wqe()
3268 if (wr->send_flags & IB_SEND_SOLICITED) in bnxt_re_build_rdma_wqe()
3269 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; in bnxt_re_build_rdma_wqe()
3270 if (wr->send_flags & IB_SEND_INLINE) in bnxt_re_build_rdma_wqe()
3271 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; in bnxt_re_build_rdma_wqe()
3279 switch (wr->opcode) { in bnxt_re_build_atomic_wqe()
3281 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; in bnxt_re_build_atomic_wqe()
3282 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; in bnxt_re_build_atomic_wqe()
3283 wqe->atomic.swap_data = atomic_wr(wr)->swap; in bnxt_re_build_atomic_wqe()
3286 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; in bnxt_re_build_atomic_wqe()
3287 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; in bnxt_re_build_atomic_wqe()
3290 return -EINVAL; in bnxt_re_build_atomic_wqe()
3292 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; in bnxt_re_build_atomic_wqe()
3293 wqe->atomic.r_key = atomic_wr(wr)->rkey; in bnxt_re_build_atomic_wqe()
3294 if (wr->send_flags & IB_SEND_SIGNALED) in bnxt_re_build_atomic_wqe()
3295 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_build_atomic_wqe()
3296 if (wr->send_flags & IB_SEND_FENCE) in bnxt_re_build_atomic_wqe()
3297 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_build_atomic_wqe()
3298 if (wr->send_flags & IB_SEND_SOLICITED) in bnxt_re_build_atomic_wqe()
3299 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; in bnxt_re_build_atomic_wqe()
3306 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; in bnxt_re_build_inv_wqe()
3307 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; in bnxt_re_build_inv_wqe()
3308 if (wr->send_flags & IB_SEND_SIGNALED) in bnxt_re_build_inv_wqe()
3309 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_build_inv_wqe()
3310 if (wr->send_flags & IB_SEND_FENCE) in bnxt_re_build_inv_wqe()
3311 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_build_inv_wqe()
3312 if (wr->send_flags & IB_SEND_SOLICITED) in bnxt_re_build_inv_wqe()
3313 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; in bnxt_re_build_inv_wqe()
3321 struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); in bnxt_re_build_reg_wqe()
3322 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; in bnxt_re_build_reg_wqe()
3323 int reg_len, i, access = wr->access; in bnxt_re_build_reg_wqe()
3325 if (mr->npages > qplib_frpl->max_pg_ptrs) { in bnxt_re_build_reg_wqe()
3326 dev_err_ratelimited(rdev_to_dev(mr->rdev), in bnxt_re_build_reg_wqe()
3328 mr->npages, qplib_frpl->max_pg_ptrs); in bnxt_re_build_reg_wqe()
3329 return -EINVAL; in bnxt_re_build_reg_wqe()
3332 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; in bnxt_re_build_reg_wqe()
3333 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; in bnxt_re_build_reg_wqe()
3334 wqe->frmr.levels = qplib_frpl->hwq.level; in bnxt_re_build_reg_wqe()
3335 wqe->frmr.page_list = mr->pages; in bnxt_re_build_reg_wqe()
3336 wqe->frmr.page_list_len = mr->npages; in bnxt_re_build_reg_wqe()
3337 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; in bnxt_re_build_reg_wqe()
3339 if (wr->wr.send_flags & IB_SEND_SIGNALED) in bnxt_re_build_reg_wqe()
3340 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; in bnxt_re_build_reg_wqe()
3342 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; in bnxt_re_build_reg_wqe()
3344 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; in bnxt_re_build_reg_wqe()
3346 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; in bnxt_re_build_reg_wqe()
3348 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; in bnxt_re_build_reg_wqe()
3350 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; in bnxt_re_build_reg_wqe()
3353 wqe->frmr.l_key = wr->key; in bnxt_re_build_reg_wqe()
3354 wqe->frmr.length = wr->mr->length; in bnxt_re_build_reg_wqe()
3355 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); in bnxt_re_build_reg_wqe()
3356 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); in bnxt_re_build_reg_wqe()
3357 wqe->frmr.va = wr->mr->iova; in bnxt_re_build_reg_wqe()
3358 reg_len = wqe->frmr.page_list_len * wr->mr->page_size; in bnxt_re_build_reg_wqe()
3360 if (wqe->frmr.length > reg_len) { in bnxt_re_build_reg_wqe()
3361 dev_err_ratelimited(rdev_to_dev(mr->rdev), in bnxt_re_build_reg_wqe()
3362 "%s: bnxt_re_mr 0x%px len (%d > %d)\n", in bnxt_re_build_reg_wqe()
3363 __func__, (void *)mr, wqe->frmr.length, in bnxt_re_build_reg_wqe()
3366 for (i = 0; i < mr->npages; i++) in bnxt_re_build_reg_wqe()
3367 dev_dbg(rdev_to_dev(mr->rdev), in bnxt_re_build_reg_wqe()
3369 __func__, i, mr->pages[i]); in bnxt_re_build_reg_wqe()
3371 return -EINVAL; in bnxt_re_build_reg_wqe()
3380 wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; in bnxt_re_set_sg_list()
3381 wqe->num_sge = wr->num_sge; in bnxt_re_set_sg_list()
3386 if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || in bnxt_ud_qp_hw_stall_workaround()
3387 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && in bnxt_ud_qp_hw_stall_workaround()
3388 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { in bnxt_ud_qp_hw_stall_workaround()
3394 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); in bnxt_ud_qp_hw_stall_workaround()
3395 qp->qplib_qp.wqe_cnt = 0; in bnxt_ud_qp_hw_stall_workaround()
3407 spin_lock_irqsave(&qp->sq_lock, flags); in bnxt_re_post_send_shadow_qp()
3412 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { in bnxt_re_post_send_shadow_qp()
3415 rc = -EINVAL; in bnxt_re_post_send_shadow_qp()
3420 wqe.wr_id = wr->wr_id; in bnxt_re_post_send_shadow_qp()
3426 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); in bnxt_re_post_send_shadow_qp()
3430 wr->opcode, rc); in bnxt_re_post_send_shadow_qp()
3433 wr = wr->next; in bnxt_re_post_send_shadow_qp()
3435 bnxt_qplib_post_send_db(&qp->qplib_qp); in bnxt_re_post_send_shadow_qp()
3437 spin_unlock_irqrestore(&qp->sq_lock, flags); in bnxt_re_post_send_shadow_qp()
3443 /* Need unconditional fence for non-wire memory opcode in bnxt_re_legacy_set_uc_fence()
3446 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || in bnxt_re_legacy_set_uc_fence()
3447 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || in bnxt_re_legacy_set_uc_fence()
3448 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || in bnxt_re_legacy_set_uc_fence()
3449 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) in bnxt_re_legacy_set_uc_fence()
3450 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; in bnxt_re_legacy_set_uc_fence()
3463 rdev = qp->rdev; in bnxt_re_post_send()
3464 spin_lock_irqsave(&qp->sq_lock, flags); in bnxt_re_post_send()
3469 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { in bnxt_re_post_send()
3472 rc = -EINVAL; in bnxt_re_post_send()
3477 wqe.wr_id = wr->wr_id; in bnxt_re_post_send()
3479 switch (wr->opcode) { in bnxt_re_post_send()
3482 if (ib_qp->qp_type == IB_QPT_GSI && in bnxt_re_post_send()
3483 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { in bnxt_re_post_send()
3489 } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { in bnxt_re_post_send()
3492 switch (wr->send_flags) { in bnxt_re_post_send()
3516 rc = -EINVAL; in bnxt_re_post_send()
3527 "WR (0x%x) is not supported\n", wr->opcode); in bnxt_re_post_send()
3528 rc = -EINVAL; in bnxt_re_post_send()
3533 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) in bnxt_re_post_send()
3535 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); in bnxt_re_post_send()
3540 "bad_wr seen with opcode = 0x%x\n", wr->opcode); in bnxt_re_post_send()
3544 wr = wr->next; in bnxt_re_post_send()
3546 bnxt_qplib_post_send_db(&qp->qplib_qp); in bnxt_re_post_send()
3547 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) in bnxt_re_post_send()
3549 spin_unlock_irqrestore(&qp->sq_lock, flags); in bnxt_re_post_send()
3566 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { in bnxt_re_post_recv_shadow_qp()
3569 rc = -EINVAL; in bnxt_re_post_recv_shadow_qp()
3573 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; in bnxt_re_post_recv_shadow_qp()
3574 wqe.num_sge = wr->num_sge; in bnxt_re_post_recv_shadow_qp()
3575 wqe.wr_id = wr->wr_id; in bnxt_re_post_recv_shadow_qp()
3577 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); in bnxt_re_post_recv_shadow_qp()
3584 wr = wr->next; in bnxt_re_post_recv_shadow_qp()
3586 bnxt_qplib_post_recv_db(&qp->qplib_qp); in bnxt_re_post_recv_shadow_qp()
3594 struct bnxt_re_dev *rdev = qp->rdev; in bnxt_re_build_gsi_recv()
3597 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) in bnxt_re_build_gsi_recv()
3615 spin_lock_irqsave(&qp->rq_lock, flags); in bnxt_re_post_recv()
3618 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { in bnxt_re_post_recv()
3619 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_post_recv()
3621 rc = -EINVAL; in bnxt_re_post_recv()
3624 wqe.num_sge = wr->num_sge; in bnxt_re_post_recv()
3625 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; in bnxt_re_post_recv()
3626 wqe.wr_id = wr->wr_id; in bnxt_re_post_recv()
3629 if (ib_qp->qp_type == IB_QPT_GSI && in bnxt_re_post_recv()
3630 qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { in bnxt_re_post_recv()
3637 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); in bnxt_re_post_recv()
3640 dev_err(rdev_to_dev(qp->rdev), in bnxt_re_post_recv()
3647 bnxt_qplib_post_recv_db(&qp->qplib_qp); in bnxt_re_post_recv()
3650 wr = wr->next; in bnxt_re_post_recv()
3654 bnxt_qplib_post_recv_db(&qp->qplib_qp); in bnxt_re_post_recv()
3655 spin_unlock_irqrestore(&qp->rq_lock, flags); in bnxt_re_post_recv()
3664 struct bnxt_re_dev *rdev = cq->rdev; in bnxt_re_destroy_cq()
3667 if (cq->uctx_cq_page) { in bnxt_re_destroy_cq()
3668 BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); in bnxt_re_destroy_cq()
3669 free_page((u64)cq->uctx_cq_page); in bnxt_re_destroy_cq()
3670 cq->uctx_cq_page = NULL; in bnxt_re_destroy_cq()
3672 if (cq->is_dbr_soft_cq && cq->uctx) { in bnxt_re_destroy_cq()
3675 if (cq->uctx->dbr_recov_cq) { in bnxt_re_destroy_cq()
3676 dbr_page = cq->uctx->dbr_recov_cq_page; in bnxt_re_destroy_cq()
3677 cq->uctx->dbr_recov_cq_page = NULL; in bnxt_re_destroy_cq()
3678 cq->uctx->dbr_recov_cq = NULL; in bnxt_re_destroy_cq()
3684 spin_lock_bh(&cq->qplib_cq.compl_lock); in bnxt_re_destroy_cq()
3685 cq->qplib_cq.destroyed = true; in bnxt_re_destroy_cq()
3686 spin_unlock_bh(&cq->qplib_cq.compl_lock); in bnxt_re_destroy_cq()
3687 if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || in bnxt_re_destroy_cq()
3688 ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) in bnxt_re_destroy_cq()
3689 cancel_work_sync(&ib_cq->work); in bnxt_re_destroy_cq()
3691 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); in bnxt_re_destroy_cq()
3695 __func__, cq->qplib_cq.id, rc); in bnxt_re_destroy_cq()
3697 bnxt_re_put_nq(rdev, cq->qplib_cq.nq); in bnxt_re_destroy_cq()
3698 if (cq->umem && !IS_ERR(cq->umem)) in bnxt_re_destroy_cq()
3699 ib_umem_release(cq->umem); in bnxt_re_destroy_cq()
3701 kfree(cq->cql); in bnxt_re_destroy_cq()
3702 atomic_dec(&rdev->stats.rsors.cq_count); in bnxt_re_destroy_cq()
3729 int cqe = attr->cqe; in bnxt_re_create_cq()
3731 if (attr->flags) in bnxt_re_create_cq()
3732 return -EOPNOTSUPP; in bnxt_re_create_cq()
3735 if (rdev->mod_exit) { in bnxt_re_create_cq()
3736 rc = -EIO; in bnxt_re_create_cq()
3744 context = &uctx->ibucontext; in bnxt_re_create_cq()
3746 dev_attr = rdev->dev_attr; in bnxt_re_create_cq()
3748 if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { in bnxt_re_create_cq()
3749 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); in bnxt_re_create_cq()
3750 rc = -EINVAL; in bnxt_re_create_cq()
3754 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { in bnxt_re_create_cq()
3755 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); in bnxt_re_create_cq()
3756 rc = -EINVAL; in bnxt_re_create_cq()
3762 rc = -ENOMEM; in bnxt_re_create_cq()
3765 cq->rdev = rdev; in bnxt_re_create_cq()
3766 cq->uctx = uctx; in bnxt_re_create_cq()
3767 qplcq = &cq->qplib_cq; in bnxt_re_create_cq()
3768 qplcq->cq_handle = (u64)qplcq; in bnxt_re_create_cq()
3775 if (!udata && !rdev->gsi_ctx.first_cq_created && in bnxt_re_create_cq()
3776 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { in bnxt_re_create_cq()
3777 rdev->gsi_ctx.first_cq_created = true; in bnxt_re_create_cq()
3787 if (entries > dev_attr->max_cq_wqes + 1) in bnxt_re_create_cq()
3788 entries = dev_attr->max_cq_wqes + 1; in bnxt_re_create_cq()
3790 qplcq->sginfo.pgshft = PAGE_SHIFT; in bnxt_re_create_cq()
3791 qplcq->sginfo.pgsize = PAGE_SIZE; in bnxt_re_create_cq()
3793 if (udata->inlen < sizeof(ureq)) in bnxt_re_create_cq()
3796 (unsigned int)udata->inlen, in bnxt_re_create_cq()
3800 min(udata->inlen, sizeof(ureq))); in bnxt_re_create_cq()
3805 cq->is_dbr_soft_cq = true; in bnxt_re_create_cq()
3817 rc = -ENOMEM; in bnxt_re_create_cq()
3826 uctx->dbr_recov_cq = cq; in bnxt_re_create_cq()
3827 uctx->dbr_recov_cq_page = dbr_page; in bnxt_re_create_cq()
3829 cq->is_dbr_soft_cq = true; in bnxt_re_create_cq()
3833 cq->umem = ib_umem_get_compat in bnxt_re_create_cq()
3837 if (IS_ERR(cq->umem)) { in bnxt_re_create_cq()
3838 rc = PTR_ERR(cq->umem); in bnxt_re_create_cq()
3844 qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, in bnxt_re_create_cq()
3845 &qplcq->sginfo.nmap); in bnxt_re_create_cq()
3846 qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); in bnxt_re_create_cq()
3847 if (!uctx->dpi.dbr) { in bnxt_re_create_cq()
3852 qplcq->dpi = &uctx->dpi; in bnxt_re_create_cq()
3854 cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; in bnxt_re_create_cq()
3855 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), in bnxt_re_create_cq()
3857 if (!cq->cql) { in bnxt_re_create_cq()
3859 "Allocate CQL for %d failed!\n", cq->max_cql); in bnxt_re_create_cq()
3860 rc = -ENOMEM; in bnxt_re_create_cq()
3863 qplcq->dpi = &rdev->dpi_privileged; in bnxt_re_create_cq()
3869 qplcq->max_wqe = entries; in bnxt_re_create_cq()
3870 qplcq->nq = bnxt_re_get_nq(rdev); in bnxt_re_create_cq()
3871 qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; in bnxt_re_create_cq()
3873 rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); in bnxt_re_create_cq()
3879 INIT_LIST_HEAD(&cq->cq_list); in bnxt_re_create_cq()
3880 cq->ibcq.cqe = entries; in bnxt_re_create_cq()
3881 cq->cq_period = qplcq->period; in bnxt_re_create_cq()
3883 atomic_inc(&rdev->stats.rsors.cq_count); in bnxt_re_create_cq()
3884 max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); in bnxt_re_create_cq()
3885 if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) in bnxt_re_create_cq()
3886 atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); in bnxt_re_create_cq()
3887 spin_lock_init(&cq->cq_lock); in bnxt_re_create_cq()
3892 resp.cqid = qplcq->id; in bnxt_re_create_cq()
3893 resp.tail = qplcq->hwq.cons; in bnxt_re_create_cq()
3894 resp.phase = qplcq->period; in bnxt_re_create_cq()
3896 resp.dbr = (u64)uctx->dpi.umdbr; in bnxt_re_create_cq()
3897 resp.dpi = uctx->dpi.dpi; in bnxt_re_create_cq()
3900 if (uctx->wcdpi.dpi) { in bnxt_re_create_cq()
3901 resp.wcdpi = uctx->wcdpi.dpi; in bnxt_re_create_cq()
3905 if (_is_chip_p7(rdev->chip_ctx)) { in bnxt_re_create_cq()
3906 cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); in bnxt_re_create_cq()
3908 if (!cq->uctx_cq_page) { in bnxt_re_create_cq()
3911 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); in bnxt_re_create_cq()
3912 rc = -ENOMEM; in bnxt_re_create_cq()
3916 resp.uctx_cq_page = (u64)cq->uctx_cq_page; in bnxt_re_create_cq()
3921 min(udata->outlen, sizeof(resp)), in bnxt_re_create_cq()
3924 free_page((u64)cq->uctx_cq_page); in bnxt_re_create_cq()
3925 cq->uctx_cq_page = NULL; in bnxt_re_create_cq()
3926 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); in bnxt_re_create_cq()
3930 if (cq->uctx_cq_page) in bnxt_re_create_cq()
3937 if (udata && cq->umem && !IS_ERR(cq->umem)) in bnxt_re_create_cq()
3938 ib_umem_release(cq->umem); in bnxt_re_create_cq()
3941 if (cq->cql) in bnxt_re_create_cq()
3942 kfree(cq->cql); in bnxt_re_create_cq()
3951 struct bnxt_re_dev *rdev = cq->rdev; in bnxt_re_modify_cq()
3954 if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { in bnxt_re_modify_cq()
3955 cq->qplib_cq.count = cq_count; in bnxt_re_modify_cq()
3956 cq->qplib_cq.period = cq_period; in bnxt_re_modify_cq()
3957 rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); in bnxt_re_modify_cq()
3960 cq->qplib_cq.id); in bnxt_re_modify_cq()
3964 cq->cq_count = cq_count; in bnxt_re_modify_cq()
3965 cq->cq_period = cq_period; in bnxt_re_modify_cq()
3972 struct bnxt_re_dev *rdev = cq->rdev; in bnxt_re_resize_cq_complete()
3974 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); in bnxt_re_resize_cq_complete()
3976 cq->qplib_cq.max_wqe = cq->resize_cqe; in bnxt_re_resize_cq_complete()
3977 if (cq->resize_umem) { in bnxt_re_resize_cq_complete()
3978 ib_umem_release(cq->umem); in bnxt_re_resize_cq_complete()
3979 cq->umem = cq->resize_umem; in bnxt_re_resize_cq_complete()
3980 cq->resize_umem = NULL; in bnxt_re_resize_cq_complete()
3981 cq->resize_cqe = 0; in bnxt_re_resize_cq_complete()
4001 rdev = cq->rdev; in bnxt_re_resize_cq()
4002 dev_attr = rdev->dev_attr; in bnxt_re_resize_cq()
4003 if (ib_cq->uobject) { in bnxt_re_resize_cq()
4007 context = &uctx->ibucontext; in bnxt_re_resize_cq()
4010 if (cq->resize_umem) { in bnxt_re_resize_cq()
4011 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", in bnxt_re_resize_cq()
4012 cq->qplib_cq.id); in bnxt_re_resize_cq()
4013 return -EBUSY; in bnxt_re_resize_cq()
4017 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { in bnxt_re_resize_cq()
4018 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", in bnxt_re_resize_cq()
4019 cq->qplib_cq.id); in bnxt_re_resize_cq()
4020 return -EINVAL; in bnxt_re_resize_cq()
4024 entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); in bnxt_re_resize_cq()
4029 if (entries == cq->ibcq.cqe) { in bnxt_re_resize_cq()
4034 if (ib_cq->uobject && udata) { in bnxt_re_resize_cq()
4035 if (udata->inlen < sizeof(ureq)) in bnxt_re_resize_cq()
4038 (unsigned int)udata->inlen, in bnxt_re_resize_cq()
4042 min(udata->inlen, sizeof(ureq))); in bnxt_re_resize_cq()
4048 cq->resize_umem = ib_umem_get_compat in bnxt_re_resize_cq()
4053 if (IS_ERR(cq->resize_umem)) { in bnxt_re_resize_cq()
4054 rc = PTR_ERR(cq->resize_umem); in bnxt_re_resize_cq()
4055 cq->resize_umem = NULL; in bnxt_re_resize_cq()
4060 cq->resize_cqe = entries; in bnxt_re_resize_cq()
4063 memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); in bnxt_re_resize_cq()
4064 orig_dpi = cq->qplib_cq.dpi; in bnxt_re_resize_cq()
4066 cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, in bnxt_re_resize_cq()
4067 &cq->qplib_cq.sginfo.nmap); in bnxt_re_resize_cq()
4068 cq->qplib_cq.sginfo.npages = in bnxt_re_resize_cq()
4069 ib_umem_num_pages_compat(cq->resize_umem); in bnxt_re_resize_cq()
4070 cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; in bnxt_re_resize_cq()
4071 cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; in bnxt_re_resize_cq()
4072 cq->qplib_cq.dpi = &uctx->dpi; in bnxt_re_resize_cq()
4077 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); in bnxt_re_resize_cq()
4080 cq->qplib_cq.id); in bnxt_re_resize_cq()
4084 cq->ibcq.cqe = cq->resize_cqe; in bnxt_re_resize_cq()
4088 if (!cq->resize_umem) in bnxt_re_resize_cq()
4089 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); in bnxt_re_resize_cq()
4091 atomic_inc(&rdev->stats.rsors.resize_count); in bnxt_re_resize_cq()
4095 if (cq->resize_umem) { in bnxt_re_resize_cq()
4096 ib_umem_release(cq->resize_umem); in bnxt_re_resize_cq()
4097 cq->resize_umem = NULL; in bnxt_re_resize_cq()
4098 cq->resize_cqe = 0; in bnxt_re_resize_cq()
4099 memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); in bnxt_re_resize_cq()
4100 cq->qplib_cq.dpi = orig_dpi; in bnxt_re_resize_cq()
4190 switch (cqe->type) { in bnxt_re_process_req_wc()
4192 wc->opcode = IB_WC_SEND; in bnxt_re_process_req_wc()
4195 wc->opcode = IB_WC_SEND; in bnxt_re_process_req_wc()
4196 wc->wc_flags |= IB_WC_WITH_IMM; in bnxt_re_process_req_wc()
4199 wc->opcode = IB_WC_SEND; in bnxt_re_process_req_wc()
4200 wc->wc_flags |= IB_WC_WITH_INVALIDATE; in bnxt_re_process_req_wc()
4203 wc->opcode = IB_WC_RDMA_WRITE; in bnxt_re_process_req_wc()
4206 wc->opcode = IB_WC_RDMA_WRITE; in bnxt_re_process_req_wc()
4207 wc->wc_flags |= IB_WC_WITH_IMM; in bnxt_re_process_req_wc()
4210 wc->opcode = IB_WC_RDMA_READ; in bnxt_re_process_req_wc()
4213 wc->opcode = IB_WC_COMP_SWAP; in bnxt_re_process_req_wc()
4216 wc->opcode = IB_WC_FETCH_ADD; in bnxt_re_process_req_wc()
4219 wc->opcode = IB_WC_LOCAL_INV; in bnxt_re_process_req_wc()
4222 wc->opcode = IB_WC_REG_MR; in bnxt_re_process_req_wc()
4225 wc->opcode = IB_WC_SEND; in bnxt_re_process_req_wc()
4229 wc->status = __req_to_ib_wc_status(cqe->status); in bnxt_re_process_req_wc()
4236 /* raweth_qp1_flags Bit 9-6 indicates itype */ in bnxt_re_check_packet_type()
4240 return -1; in bnxt_re_check_packet_type()
4246 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ in bnxt_re_check_packet_type()
4272 if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { in bnxt_re_is_loopback_packet()
4276 eth_type = ntohs(eth_hdr->h_proto); in bnxt_re_is_loopback_packet()
4301 eth_type = ntohs(eth_hdr->h_proto); in bnxt_re_is_vlan_in_packet()
4305 cqe->raweth_qp1_metadata = in bnxt_re_is_vlan_in_packet()
4306 ntohs(vlan_hdr->h_vlan_TCI) | in bnxt_re_is_vlan_in_packet()
4309 cqe->raweth_qp1_flags2 |= in bnxt_re_is_vlan_in_packet()
4340 rdev = gsi_qp->rdev; in bnxt_re_process_raw_qp_packet_receive()
4341 gsi_sqp = rdev->gsi_ctx.gsi_sqp; in bnxt_re_process_raw_qp_packet_receive()
4342 tbl_idx = cqe->wr_id; in bnxt_re_process_raw_qp_packet_receive()
4344 hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; in bnxt_re_process_raw_qp_packet_receive()
4345 rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; in bnxt_re_process_raw_qp_packet_receive()
4346 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, in bnxt_re_process_raw_qp_packet_receive()
4349 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, in bnxt_re_process_raw_qp_packet_receive()
4351 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; in bnxt_re_process_raw_qp_packet_receive()
4354 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, in bnxt_re_process_raw_qp_packet_receive()
4355 cqe->raweth_qp1_flags2); in bnxt_re_process_raw_qp_packet_receive()
4358 return -EINVAL; in bnxt_re_process_raw_qp_packet_receive()
4377 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); in bnxt_re_process_raw_qp_packet_receive()
4378 sqp_entry->qp1_qp = gsi_qp; in bnxt_re_process_raw_qp_packet_receive()
4400 r_sge[1].addr = sqp_entry->sge.addr + offset; in bnxt_re_process_raw_qp_packet_receive()
4401 r_sge[1].lkey = sqp_entry->sge.lkey; in bnxt_re_process_raw_qp_packet_receive()
4402 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; in bnxt_re_process_raw_qp_packet_receive()
4414 return -ENOMEM; in bnxt_re_process_raw_qp_packet_receive()
4417 swr->num_sge = 2; in bnxt_re_process_raw_qp_packet_receive()
4418 swr->sg_list = s_sge; in bnxt_re_process_raw_qp_packet_receive()
4419 swr->wr_id = tbl_idx; in bnxt_re_process_raw_qp_packet_receive()
4420 swr->opcode = IB_WR_SEND; in bnxt_re_process_raw_qp_packet_receive()
4421 swr->next = NULL; in bnxt_re_process_raw_qp_packet_receive()
4423 gsi_sah = rdev->gsi_ctx.gsi_sah; in bnxt_re_process_raw_qp_packet_receive()
4424 udwr.ah = &gsi_sah->ibah; in bnxt_re_process_raw_qp_packet_receive()
4425 udwr.remote_qpn = gsi_sqp->qplib_qp.id; in bnxt_re_process_raw_qp_packet_receive()
4426 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; in bnxt_re_process_raw_qp_packet_receive()
4436 wc->opcode = IB_WC_RECV; in bnxt_re_process_res_rawqp1_wc()
4437 wc->status = __rawqp1_to_ib_wc_status(cqe->status); in bnxt_re_process_res_rawqp1_wc()
4438 wc->wc_flags |= IB_WC_GRH; in bnxt_re_process_res_rawqp1_wc()
4444 wc->opcode = IB_WC_RECV; in bnxt_re_process_res_rc_wc()
4445 wc->status = __rc_to_ib_wc_status(cqe->status); in bnxt_re_process_res_rc_wc()
4447 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) in bnxt_re_process_res_rc_wc()
4448 wc->wc_flags |= IB_WC_WITH_IMM; in bnxt_re_process_res_rc_wc()
4449 if (cqe->flags & CQ_RES_RC_FLAGS_INV) in bnxt_re_process_res_rc_wc()
4450 wc->wc_flags |= IB_WC_WITH_INVALIDATE; in bnxt_re_process_res_rc_wc()
4451 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == in bnxt_re_process_res_rc_wc()
4453 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; in bnxt_re_process_res_rc_wc()
4456 /* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */
4463 metadata = orig_cqe->raweth_qp1_metadata; in bnxt_re_is_nonzero_vlanid_pkt()
4464 if (orig_cqe->raweth_qp1_flags2 & in bnxt_re_is_nonzero_vlanid_pkt()
4487 struct bnxt_re_dev *rdev = gsi_sqp->rdev; in bnxt_re_process_res_shadow_qp_wc()
4495 tbl_idx = cqe->wr_id; in bnxt_re_process_res_shadow_qp_wc()
4497 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; in bnxt_re_process_res_shadow_qp_wc()
4498 gsi_qp = sqp_entry->qp1_qp; in bnxt_re_process_res_shadow_qp_wc()
4499 orig_cqe = &sqp_entry->cqe; in bnxt_re_process_res_shadow_qp_wc()
4501 wc->wr_id = sqp_entry->wrid; in bnxt_re_process_res_shadow_qp_wc()
4502 wc->byte_len = orig_cqe->length; in bnxt_re_process_res_shadow_qp_wc()
4503 wc->qp = &gsi_qp->ib_qp; in bnxt_re_process_res_shadow_qp_wc()
4505 wc->ex.imm_data = orig_cqe->immdata; in bnxt_re_process_res_shadow_qp_wc()
4506 wc->src_qp = orig_cqe->src_qp; in bnxt_re_process_res_shadow_qp_wc()
4507 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); in bnxt_re_process_res_shadow_qp_wc()
4510 wc->sl = sl; in bnxt_re_process_res_shadow_qp_wc()
4511 wc->vlan_id = vlan_id; in bnxt_re_process_res_shadow_qp_wc()
4512 wc->wc_flags |= IB_WC_WITH_VLAN; in bnxt_re_process_res_shadow_qp_wc()
4515 wc->port_num = 1; in bnxt_re_process_res_shadow_qp_wc()
4516 wc->vendor_err = orig_cqe->status; in bnxt_re_process_res_shadow_qp_wc()
4518 wc->opcode = IB_WC_RECV; in bnxt_re_process_res_shadow_qp_wc()
4519 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); in bnxt_re_process_res_shadow_qp_wc()
4520 wc->wc_flags |= IB_WC_GRH; in bnxt_re_process_res_shadow_qp_wc()
4522 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, in bnxt_re_process_res_shadow_qp_wc()
4523 orig_cqe->raweth_qp1_flags2); in bnxt_re_process_res_shadow_qp_wc()
4534 wc->opcode = IB_WC_RECV; in bnxt_re_process_res_ud_wc()
4535 wc->status = __rc_to_ib_wc_status(cqe->status); in bnxt_re_process_res_ud_wc()
4536 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) in bnxt_re_process_res_ud_wc()
4537 wc->wc_flags |= IB_WC_WITH_IMM; in bnxt_re_process_res_ud_wc()
4538 if (cqe->flags & CQ_RES_RC_FLAGS_INV) in bnxt_re_process_res_ud_wc()
4539 wc->wc_flags |= IB_WC_WITH_INVALIDATE; in bnxt_re_process_res_ud_wc()
4541 if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && in bnxt_re_process_res_ud_wc()
4542 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { in bnxt_re_process_res_ud_wc()
4543 wc->wc_flags |= IB_WC_GRH; in bnxt_re_process_res_ud_wc()
4544 memcpy(wc->smac, cqe->smac, ETH_ALEN); in bnxt_re_process_res_ud_wc()
4545 wc->wc_flags |= IB_WC_WITH_SMAC; in bnxt_re_process_res_ud_wc()
4546 if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { in bnxt_re_process_res_ud_wc()
4547 if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { in bnxt_re_process_res_ud_wc()
4548 if (cqe->cfa_meta & in bnxt_re_process_res_ud_wc()
4550 vlan_id = (cqe->cfa_meta & 0xFFF); in bnxt_re_process_res_ud_wc()
4552 } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { in bnxt_re_process_res_ud_wc()
4553 vlan_id = (cqe->cfa_meta & 0xFFF); in bnxt_re_process_res_ud_wc()
4557 wc->vlan_id = vlan_id; in bnxt_re_process_res_ud_wc()
4558 wc->wc_flags |= IB_WC_WITH_VLAN; in bnxt_re_process_res_ud_wc()
4565 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; in bnxt_re_legacy_send_phantom_wqe()
4569 spin_lock_irqsave(&qp->sq_lock, flags); in bnxt_re_legacy_send_phantom_wqe()
4573 lib_qp->sq.phantom_wqe_cnt++; in bnxt_re_legacy_send_phantom_wqe()
4574 dev_dbg(&lib_qp->sq.hwq.pdev->dev, in bnxt_re_legacy_send_phantom_wqe()
4575 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", in bnxt_re_legacy_send_phantom_wqe()
4576 lib_qp->id, lib_qp->sq.hwq.prod, in bnxt_re_legacy_send_phantom_wqe()
4577 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), in bnxt_re_legacy_send_phantom_wqe()
4578 lib_qp->sq.phantom_wqe_cnt); in bnxt_re_legacy_send_phantom_wqe()
4581 spin_unlock_irqrestore(&qp->sq_lock, flags); in bnxt_re_legacy_send_phantom_wqe()
4588 struct bnxt_re_dev *rdev = cq->rdev; in bnxt_re_poll_cq()
4603 if (cq->is_dbr_soft_cq) { in bnxt_re_poll_cq()
4611 if (cq->umem) { in bnxt_re_poll_cq()
4612 if (cq->resize_umem) in bnxt_re_poll_cq()
4617 spin_lock_irqsave(&cq->cq_lock, flags); in bnxt_re_poll_cq()
4619 budget = min_t(u32, num_entries, cq->max_cql); in bnxt_re_poll_cq()
4621 if (!cq->cql) { in bnxt_re_poll_cq()
4625 cqe = &cq->cql[0]; in bnxt_re_poll_cq()
4626 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; in bnxt_re_poll_cq()
4629 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); in bnxt_re_poll_cq()
4631 sq = &lib_qp->sq; in bnxt_re_poll_cq()
4632 if (sq->legacy_send_phantom == true) { in bnxt_re_poll_cq()
4634 if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) in bnxt_re_poll_cq()
4638 sq->legacy_send_phantom = false; in bnxt_re_poll_cq()
4642 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, in bnxt_re_poll_cq()
4644 budget - ncqe); in bnxt_re_poll_cq()
4653 wc->wr_id = cqe->wr_id; in bnxt_re_poll_cq()
4654 wc->byte_len = cqe->length; in bnxt_re_poll_cq()
4655 qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, in bnxt_re_poll_cq()
4662 wc->qp = &qp->ib_qp; in bnxt_re_poll_cq()
4663 wc->ex.imm_data = cqe->immdata; in bnxt_re_poll_cq()
4664 wc->src_qp = cqe->src_qp; in bnxt_re_poll_cq()
4665 memcpy(wc->smac, cqe->smac, ETH_ALEN); in bnxt_re_poll_cq()
4666 wc->port_num = 1; in bnxt_re_poll_cq()
4667 wc->vendor_err = cqe->status; in bnxt_re_poll_cq()
4669 switch(cqe->opcode) { in bnxt_re_poll_cq()
4672 qp->qplib_qp.id == in bnxt_re_poll_cq()
4673 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { in bnxt_re_poll_cq()
4685 if (!cqe->status) { in bnxt_re_poll_cq()
4693 cqe->status = -1; in bnxt_re_poll_cq()
4699 tbl_idx = cqe->wr_id; in bnxt_re_poll_cq()
4700 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; in bnxt_re_poll_cq()
4701 wc->wr_id = sqp_entry->wrid; in bnxt_re_poll_cq()
4711 qp->qplib_qp.id == in bnxt_re_poll_cq()
4712 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { in bnxt_re_poll_cq()
4718 if (cqe->status) { in bnxt_re_poll_cq()
4731 dev_err(rdev_to_dev(cq->rdev), in bnxt_re_poll_cq()
4733 cqe->opcode); in bnxt_re_poll_cq()
4737 budget--; in bnxt_re_poll_cq()
4741 spin_unlock_irqrestore(&cq->cq_lock, flags); in bnxt_re_poll_cq()
4742 return init_budget - budget; in bnxt_re_poll_cq()
4752 spin_lock_irqsave(&cq->cq_lock, flags); in bnxt_re_req_notify_cq()
4760 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); in bnxt_re_req_notify_cq()
4764 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) in bnxt_re_req_notify_cq()
4767 spin_unlock_irqrestore(&cq->cq_lock, flags); in bnxt_re_req_notify_cq()
4785 rdev = pd->rdev; in bnxt_re_get_dma_mr()
4791 return ERR_PTR(-ENOMEM); in bnxt_re_get_dma_mr()
4793 mr->rdev = rdev; in bnxt_re_get_dma_mr()
4794 mr->qplib_mr.pd = &pd->qplib_pd; in bnxt_re_get_dma_mr()
4795 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); in bnxt_re_get_dma_mr()
4796 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; in bnxt_re_get_dma_mr()
4799 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_get_dma_mr()
4804 mr->qplib_mr.total_size = -1; /* Infinite length */ in bnxt_re_get_dma_mr()
4810 mrinfo.mrw = &mr->qplib_mr; in bnxt_re_get_dma_mr()
4812 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); in bnxt_re_get_dma_mr()
4817 mr->ib_mr.lkey = mr->qplib_mr.lkey; in bnxt_re_get_dma_mr()
4820 mr->ib_mr.rkey = mr->ib_mr.lkey; in bnxt_re_get_dma_mr()
4821 atomic_inc(&rdev->stats.rsors.mr_count); in bnxt_re_get_dma_mr()
4822 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); in bnxt_re_get_dma_mr()
4823 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) in bnxt_re_get_dma_mr()
4824 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); in bnxt_re_get_dma_mr()
4826 return &mr->ib_mr; in bnxt_re_get_dma_mr()
4829 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_get_dma_mr()
4838 struct bnxt_re_dev *rdev = mr->rdev; in bnxt_re_dereg_mr()
4841 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_dereg_mr()
4843 dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", in bnxt_re_dereg_mr()
4844 mr->qplib_mr.lkey, rc); in bnxt_re_dereg_mr()
4846 if (mr->pages) { in bnxt_re_dereg_mr()
4847 bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, in bnxt_re_dereg_mr()
4848 &mr->qplib_frpl); in bnxt_re_dereg_mr()
4849 kfree(mr->pages); in bnxt_re_dereg_mr()
4850 mr->npages = 0; in bnxt_re_dereg_mr()
4851 mr->pages = NULL; in bnxt_re_dereg_mr()
4853 if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { in bnxt_re_dereg_mr()
4854 mr->is_invalcb_active = false; in bnxt_re_dereg_mr()
4855 bnxt_re_peer_mem_release(mr->ib_umem); in bnxt_re_dereg_mr()
4858 atomic_dec(&rdev->stats.rsors.mr_count); in bnxt_re_dereg_mr()
4866 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) in bnxt_re_set_page()
4867 return -ENOMEM; in bnxt_re_set_page()
4869 mr->pages[mr->npages++] = addr; in bnxt_re_set_page()
4871 ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, in bnxt_re_set_page()
4872 mr->pages[mr->npages - 1]); in bnxt_re_set_page()
4881 mr->npages = 0; in bnxt_re_map_mr_sg()
4890 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_alloc_mr()
4898 return ERR_PTR(-EINVAL); in bnxt_re_alloc_mr()
4902 return ERR_PTR(-EINVAL); in bnxt_re_alloc_mr()
4907 return ERR_PTR(-ENOMEM); in bnxt_re_alloc_mr()
4909 mr->rdev = rdev; in bnxt_re_alloc_mr()
4910 mr->qplib_mr.pd = &pd->qplib_pd; in bnxt_re_alloc_mr()
4911 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; in bnxt_re_alloc_mr()
4912 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; in bnxt_re_alloc_mr()
4914 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_alloc_mr()
4919 mr->ib_mr.lkey = mr->qplib_mr.lkey; in bnxt_re_alloc_mr()
4920 mr->ib_mr.rkey = mr->ib_mr.lkey; in bnxt_re_alloc_mr()
4921 mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); in bnxt_re_alloc_mr()
4922 if (!mr->pages) { in bnxt_re_alloc_mr()
4925 rc = -ENOMEM; in bnxt_re_alloc_mr()
4928 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, in bnxt_re_alloc_mr()
4929 &mr->qplib_frpl, max_num_sg); in bnxt_re_alloc_mr()
4935 dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); in bnxt_re_alloc_mr()
4937 atomic_inc(&rdev->stats.rsors.mr_count); in bnxt_re_alloc_mr()
4938 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); in bnxt_re_alloc_mr()
4939 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) in bnxt_re_alloc_mr()
4940 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); in bnxt_re_alloc_mr()
4941 return &mr->ib_mr; in bnxt_re_alloc_mr()
4944 kfree(mr->pages); in bnxt_re_alloc_mr()
4946 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_alloc_mr()
4957 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_alloc_mw()
4965 rc = -ENOMEM; in bnxt_re_alloc_mw()
4968 mw->rdev = rdev; in bnxt_re_alloc_mw()
4969 mw->qplib_mw.pd = &pd->qplib_pd; in bnxt_re_alloc_mw()
4971 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? in bnxt_re_alloc_mw()
4974 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); in bnxt_re_alloc_mw()
4979 mw->ib_mw.rkey = mw->qplib_mw.rkey; in bnxt_re_alloc_mw()
4980 atomic_inc(&rdev->stats.rsors.mw_count); in bnxt_re_alloc_mw()
4981 max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); in bnxt_re_alloc_mw()
4982 if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) in bnxt_re_alloc_mw()
4983 atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); in bnxt_re_alloc_mw()
4985 return &mw->ib_mw; in bnxt_re_alloc_mw()
4995 struct bnxt_re_dev *rdev = mw->rdev; in bnxt_re_dealloc_mw()
4998 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); in bnxt_re_dealloc_mw()
5005 atomic_dec(&rdev->stats.rsors.mw_count); in bnxt_re_dealloc_mw()
5032 pgshft = ilog2(umem->page_size); in bnxt_re_get_page_shift()
5057 struct bnxt_re_dev *rdev = pd->rdev; in bnxt_re_reg_user_mr()
5067 if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) in bnxt_re_reg_user_mr()
5068 return ERR_PTR(-ENOMEM); in bnxt_re_reg_user_mr()
5070 if (rdev->mod_exit) { in bnxt_re_reg_user_mr()
5072 return ERR_PTR(-EIO); in bnxt_re_reg_user_mr()
5078 return ERR_PTR(-ENOMEM); in bnxt_re_reg_user_mr()
5083 return ERR_PTR (-ENOMEM); in bnxt_re_reg_user_mr()
5085 mr->rdev = rdev; in bnxt_re_reg_user_mr()
5086 mr->qplib_mr.pd = &pd->qplib_pd; in bnxt_re_reg_user_mr()
5087 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); in bnxt_re_reg_user_mr()
5088 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; in bnxt_re_reg_user_mr()
5090 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { in bnxt_re_reg_user_mr()
5091 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_reg_user_mr()
5097 mr->ib_mr.rkey = mr->qplib_mr.rkey; in bnxt_re_reg_user_mr()
5100 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, in bnxt_re_reg_user_mr()
5109 mr->ib_umem = umem; in bnxt_re_reg_user_mr()
5111 mr->qplib_mr.va = virt_addr; in bnxt_re_reg_user_mr()
5115 rc = -EINVAL; in bnxt_re_reg_user_mr()
5118 mr->qplib_mr.total_size = length; in bnxt_re_reg_user_mr()
5120 rdev->dev_attr->page_size_cap); in bnxt_re_reg_user_mr()
5123 rc = -EFAULT; in bnxt_re_reg_user_mr()
5134 mrinfo.mrw = &mr->qplib_mr; in bnxt_re_reg_user_mr()
5136 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); in bnxt_re_reg_user_mr()
5142 mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; in bnxt_re_reg_user_mr()
5143 atomic_inc(&rdev->stats.rsors.mr_count); in bnxt_re_reg_user_mr()
5144 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); in bnxt_re_reg_user_mr()
5145 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) in bnxt_re_reg_user_mr()
5146 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); in bnxt_re_reg_user_mr()
5148 return &mr->ib_mr; in bnxt_re_reg_user_mr()
5151 bnxt_re_peer_mem_release(mr->ib_umem); in bnxt_re_reg_user_mr()
5153 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) in bnxt_re_reg_user_mr()
5154 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); in bnxt_re_reg_user_mr()
5168 struct bnxt_re_dev *rdev = mr->rdev; in bnxt_re_rereg_user_mr()
5176 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, in bnxt_re_rereg_user_mr()
5186 mr->ib_umem = umem; in bnxt_re_rereg_user_mr()
5188 mr->qplib_mr.va = virt_addr; in bnxt_re_rereg_user_mr()
5192 rc = -EINVAL; in bnxt_re_rereg_user_mr()
5195 mr->qplib_mr.total_size = length; in bnxt_re_rereg_user_mr()
5197 rdev->dev_attr->page_size_cap); in bnxt_re_rereg_user_mr()
5201 rc = -EFAULT; in bnxt_re_rereg_user_mr()
5212 mrinfo.mrw = &mr->qplib_mr; in bnxt_re_rereg_user_mr()
5214 mr->qplib_mr.pd = &pd->qplib_pd; in bnxt_re_rereg_user_mr()
5217 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); in bnxt_re_rereg_user_mr()
5219 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); in bnxt_re_rereg_user_mr()
5224 mr->ib_mr.rkey = mr->qplib_mr.rkey; in bnxt_re_rereg_user_mr()
5229 bnxt_re_peer_mem_release(mr->ib_umem); in bnxt_re_rereg_user_mr()
5236 struct ib_device *ibdev = &rdev->ibdev; in bnxt_re_check_abi_version()
5245 return -EPERM; in bnxt_re_check_abi_version()
5254 struct ib_device *ibdev = ctx->device; in bnxt_re_alloc_ucontext()
5259 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; in bnxt_re_alloc_ucontext()
5267 cctx = rdev->chip_ctx; in bnxt_re_alloc_ucontext()
5272 uctx->rdev = rdev; in bnxt_re_alloc_ucontext()
5273 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); in bnxt_re_alloc_ucontext()
5274 if (!uctx->shpg) { in bnxt_re_alloc_ucontext()
5276 rc = -ENOMEM; in bnxt_re_alloc_ucontext()
5279 spin_lock_init(&uctx->sh_lock); in bnxt_re_alloc_ucontext()
5281 chip_met_rev_num = cctx->chip_num; in bnxt_re_alloc_ucontext()
5282 chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << in bnxt_re_alloc_ucontext()
5284 chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << in bnxt_re_alloc_ucontext()
5292 resp.dev_id = rdev->en_dev->pdev->devfn; in bnxt_re_alloc_ucontext()
5293 resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; in bnxt_re_alloc_ucontext()
5298 resp.modes = genp5 ? cctx->modes.wqe_mode : 0; in bnxt_re_alloc_ucontext()
5299 if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) in bnxt_re_alloc_ucontext()
5305 resp.max_cqd = dev_attr->max_cq_wqes; in bnxt_re_alloc_ucontext()
5306 if (genp5 && cctx->modes.db_push) { in bnxt_re_alloc_ucontext()
5309 !(dev_attr->dev_cap_flags & in bnxt_re_alloc_ucontext()
5317 if (rdev->dbr_pacing) in bnxt_re_alloc_ucontext()
5320 if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) in bnxt_re_alloc_ucontext()
5323 if (udata->inlen >= sizeof(ureq)) { in bnxt_re_alloc_ucontext()
5325 min(udata->inlen, sizeof(ureq))); in bnxt_re_alloc_ucontext()
5344 uctx->cmask = (uint64_t)resp.comp_mask; in bnxt_re_alloc_ucontext()
5346 min(udata->outlen, sizeof(resp)), in bnxt_re_alloc_ucontext()
5351 INIT_LIST_HEAD(&uctx->cq_list); in bnxt_re_alloc_ucontext()
5352 mutex_init(&uctx->cq_lock); in bnxt_re_alloc_ucontext()
5356 free_page((u64)uctx->shpg); in bnxt_re_alloc_ucontext()
5357 uctx->shpg = NULL; in bnxt_re_alloc_ucontext()
5367 struct bnxt_re_dev *rdev = uctx->rdev; in bnxt_re_dealloc_ucontext()
5370 if (uctx->shpg) in bnxt_re_dealloc_ucontext()
5371 free_page((u64)uctx->shpg); in bnxt_re_dealloc_ucontext()
5373 if (uctx->dpi.dbr) { in bnxt_re_dealloc_ucontext()
5377 if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { in bnxt_re_dealloc_ucontext()
5378 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, in bnxt_re_dealloc_ucontext()
5379 &uctx->wcdpi); in bnxt_re_dealloc_ucontext()
5383 uctx->wcdpi.dbr = NULL; in bnxt_re_dealloc_ucontext()
5386 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, in bnxt_re_dealloc_ucontext()
5387 &uctx->dpi); in bnxt_re_dealloc_ucontext()
5391 uctx->dpi.dbr = NULL; in bnxt_re_dealloc_ucontext()
5401 if (!_is_chip_p7(uctx->rdev->chip_ctx)) in is_bnxt_re_cq_page()
5404 mutex_lock(&uctx->cq_lock); in is_bnxt_re_cq_page()
5405 list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { in is_bnxt_re_cq_page()
5406 if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { in is_bnxt_re_cq_page()
5411 mutex_unlock(&uctx->cq_lock); in is_bnxt_re_cq_page()
5421 struct bnxt_re_dev *rdev = uctx->rdev; in bnxt_re_mmap()
5426 switch (vma->vm_pgoff) { in bnxt_re_mmap()
5428 pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; in bnxt_re_mmap()
5429 return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); in bnxt_re_mmap()
5430 dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", in bnxt_re_mmap()
5431 __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); in bnxt_re_mmap()
5434 rc = -EAGAIN; in bnxt_re_mmap()
5438 vma->vm_page_prot = in bnxt_re_mmap()
5439 pgprot_writecombine(vma->vm_page_prot); in bnxt_re_mmap()
5440 pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); in bnxt_re_mmap()
5442 return -EFAULT; in bnxt_re_mmap()
5446 if (vma->vm_flags & VM_WRITE) in bnxt_re_mmap()
5447 return -EFAULT; in bnxt_re_mmap()
5449 pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; in bnxt_re_mmap()
5451 return -EFAULT; in bnxt_re_mmap()
5454 pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; in bnxt_re_mmap()
5456 return -EFAULT; in bnxt_re_mmap()
5459 cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); in bnxt_re_mmap()
5461 pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; in bnxt_re_mmap()
5462 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); in bnxt_re_mmap()
5466 rc = -EAGAIN; in bnxt_re_mmap()
5470 vma->vm_page_prot = in bnxt_re_mmap()
5471 pgprot_noncached(vma->vm_page_prot); in bnxt_re_mmap()
5472 pfn = vma->vm_pgoff; in bnxt_re_mmap()
5477 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); in bnxt_re_mmap()
5480 return -EAGAIN; in bnxt_re_mmap()