1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2024, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 #define MAX_WR_SGL_NUM (2) 9 10 static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) 11 { 12 struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0}; 13 struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); 14 struct gdma_posted_wqe_info wqe_info = {0}; 15 struct gdma_queue *queue = ib_rq->kmem; 16 struct gdma_wqe_request wqe_req = {0}; 17 struct shadow_wqe_header *shadow_wqe; 18 int err; 19 20 if (shadow_queue_full(&qp->shadow_rq)) 21 return -EINVAL; 22 23 if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES) 24 return -EINVAL; 25 26 wqe_req.num_sge = wr->num_sge; 27 wqe_req.ib_sgl = wr->sg_list; 28 wqe_req.flags = GDMA_WR_IB_SGL; 29 30 if (wr->num_sge == 0) { 31 wqe_req.ib_sgl = &mana_ib_dummy_sge; 32 wqe_req.num_sge = 1; 33 } 34 35 err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info); 36 if (err) 37 return err; 38 39 shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq); 40 memset(shadow_wqe, 0, sizeof(*shadow_wqe)); 41 shadow_wqe->wr_id = wr->wr_id; 42 shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; 43 shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; 44 shadow_queue_advance_producer(&qp->shadow_rq); 45 46 return 0; 47 } 48 49 static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr, 50 const struct ib_recv_wr **bad_wr) 51 { 52 struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 53 struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); 54 struct gdma_queue *rq = ib_rq->kmem; 55 bool ring_rq = false; 56 int err = 0; 57 58 for (; wr; wr = wr->next) { 59 err = mana_ib_post_rq(qp, wr); 60 if (unlikely(err)) { 61 *bad_wr = wr; 62 break; 63 } 64 ring_rq = true; 65 } 66 67 if (ring_rq) 68 mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq); 69 70 return err; 71 } 72 73 int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 74 const struct ib_recv_wr **bad_wr) 75 { 76 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 77 78 switch (ibqp->qp_type) { 79 case IB_QPT_UD: 80 case IB_QPT_GSI: 81 return mana_ib_post_recv_ud(qp, wr, bad_wr); 82 default: 83 /* Unsupported QP type */ 84 *bad_wr = wr; 85 return -EINVAL; 86 } 87 } 88 89 static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) 90 { 91 struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 92 struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah); 93 struct net_device *ndev = mana_ib_get_netdev(&mdev->ib_dev, qp->port); 94 struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 95 struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM + 1]; 96 struct gdma_posted_wqe_info wqe_info = {0}; 97 struct gdma_wqe_request wqe_req = {0}; 98 struct rdma_send_oob send_oob = {0}; 99 struct shadow_wqe_header *shadow_wqe; 100 bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED); 101 int err, i; 102 103 if (!ndev) 104 return -EINVAL; 105 106 if (wr->wr.opcode != IB_WR_SEND) 107 return -EINVAL; 108 109 if (shadow_queue_full(&qp->shadow_sq)) 110 return -EINVAL; 111 112 if (wr->wr.num_sge > MAX_WR_SGL_NUM) 113 return -EINVAL; 114 115 gdma_sgl[0].address = ah->dma_handle; 116 gdma_sgl[0].mem_key = qp->ibqp.pd->local_dma_lkey; 117 gdma_sgl[0].size = sizeof(struct mana_ib_av); 118 for (i = 0; i < wr->wr.num_sge; ++i) { 119 gdma_sgl[i + 1].address = wr->wr.sg_list[i].addr; 120 gdma_sgl[i + 1].mem_key = wr->wr.sg_list[i].lkey; 121 gdma_sgl[i + 1].size = wr->wr.sg_list[i].length; 122 } 123 124 wqe_req.num_sge = wr->wr.num_sge + 1; 125 wqe_req.sgl = gdma_sgl; 126 wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE; 127 wqe_req.inline_oob_data = &send_oob; 128 wqe_req.flags = GDMA_WR_OOB_IN_SGL; 129 wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu)); 130 131 send_oob.wqe_type = WQE_TYPE_UD_SEND; 132 send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE); 133 send_oob.signaled = signaled; 134 send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED); 135 send_oob.psn = qp->sq_psn; 136 send_oob.ssn_or_rqpn = wr->remote_qpn; 137 send_oob.ud_send.remote_qkey = 138 qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey; 139 140 err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info); 141 if (err) 142 return err; 143 144 qp->sq_psn++; 145 shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq); 146 memset(shadow_wqe, 0, sizeof(*shadow_wqe)); 147 shadow_wqe->wr_id = wr->wr.wr_id; 148 shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC; 149 shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; 150 shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; 151 shadow_queue_advance_producer(&qp->shadow_sq); 152 153 return 0; 154 } 155 156 static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr, 157 const struct ib_send_wr **bad_wr) 158 { 159 struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 160 struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 161 struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 162 unsigned long flags; 163 bool ring_sq = false; 164 bool flush_send; 165 int err = 0; 166 167 for (; wr; wr = wr->next) { 168 err = mana_ib_post_send_ud_one(qp, ud_wr(wr)); 169 if (unlikely(err)) { 170 *bad_wr = wr; 171 break; 172 } 173 ring_sq = true; 174 } 175 176 if (ring_sq) { 177 mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); 178 179 spin_lock_irqsave(&cq->cq_lock, flags); 180 flush_send = !list_empty(&qp->send_err_node); 181 spin_unlock_irqrestore(&cq->cq_lock, flags); 182 183 if (flush_send && cq->ibcq.comp_handler) 184 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 185 } 186 187 return err; 188 } 189 190 int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 191 const struct ib_send_wr **bad_wr) 192 { 193 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 194 195 switch (ibqp->qp_type) { 196 case IB_QPT_UD: 197 case IB_QPT_GSI: 198 return mana_ib_post_send_ud(qp, wr, bad_wr); 199 default: 200 /* Unsupported QP type */ 201 *bad_wr = wr; 202 return -EINVAL; 203 } 204 } 205