1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 9 struct uverbs_attr_bundle *attrs) 10 { 11 struct ib_udata *udata = &attrs->driver_udata; 12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 13 struct mana_ib_create_cq_resp resp = {}; 14 struct mana_ib_ucontext *mana_ucontext; 15 struct ib_device *ibdev = ibcq->device; 16 struct mana_ib_create_cq ucmd; 17 struct mana_ib_dev *mdev; 18 bool is_rnic_cq; 19 u32 doorbell; 20 u32 buf_size; 21 int err; 22 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 24 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 26 cq->cq_handle = INVALID_MANA_HANDLE; 27 is_rnic_cq = mana_ib_is_rnic(mdev); 28 29 if (udata) { 30 err = ib_copy_validate_udata_in(udata, ucmd, buf_addr); 31 if (err) 32 return err; 33 34 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || 35 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { 36 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 37 return -EINVAL; 38 } 39 40 cq->cqe = attr->cqe; 41 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, 42 &cq->queue); 43 if (err) { 44 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); 45 return err; 46 } 47 48 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 49 ibucontext); 50 doorbell = mana_ucontext->doorbell; 51 } else { 52 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { 53 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 54 return -EINVAL; 55 } 56 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); 57 cq->cqe = buf_size / COMP_ENTRY_SIZE; 58 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); 59 if (err) { 60 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 61 return err; 62 } 63 doorbell = mdev->gdma_dev->doorbell; 64 } 65 66 if (is_rnic_cq) { 67 err = mana_ib_gd_create_cq(mdev, cq, doorbell); 68 if (err) { 69 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); 70 goto err_destroy_queue; 71 } 72 73 err = mana_ib_install_cq_cb(mdev, cq); 74 if (err) { 75 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); 76 goto err_destroy_rnic_cq; 77 } 78 } 79 80 if (udata) { 81 resp.cqid = cq->queue.id; 82 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 83 if (err) { 84 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 85 goto err_remove_cq_cb; 86 } 87 } 88 89 spin_lock_init(&cq->cq_lock); 90 INIT_LIST_HEAD(&cq->list_send_qp); 91 INIT_LIST_HEAD(&cq->list_recv_qp); 92 93 return 0; 94 95 err_remove_cq_cb: 96 mana_ib_remove_cq_cb(mdev, cq); 97 err_destroy_rnic_cq: 98 mana_ib_gd_destroy_cq(mdev, cq); 99 err_destroy_queue: 100 mana_ib_destroy_queue(mdev, &cq->queue); 101 102 return err; 103 } 104 105 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 106 { 107 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 108 struct ib_device *ibdev = ibcq->device; 109 struct mana_ib_dev *mdev; 110 111 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 112 113 mana_ib_remove_cq_cb(mdev, cq); 114 115 /* Ignore return code as there is not much we can do about it. 116 * The error message is printed inside. 117 */ 118 mana_ib_gd_destroy_cq(mdev, cq); 119 120 mana_ib_destroy_queue(mdev, &cq->queue); 121 122 return 0; 123 } 124 125 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq) 126 { 127 struct mana_ib_cq *cq = ctx; 128 129 if (cq->ibcq.comp_handler) 130 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 131 } 132 133 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 134 { 135 struct gdma_context *gc = mdev_to_gc(mdev); 136 struct gdma_queue *gdma_cq; 137 138 if (cq->queue.id >= gc->max_num_cqs) 139 return -EINVAL; 140 /* Create CQ table entry */ 141 WARN_ON(gc->cq_table[cq->queue.id]); 142 if (cq->queue.kmem) 143 gdma_cq = cq->queue.kmem; 144 else 145 gdma_cq = kzalloc_obj(*gdma_cq); 146 if (!gdma_cq) 147 return -ENOMEM; 148 149 gdma_cq->cq.context = cq; 150 gdma_cq->type = GDMA_CQ; 151 gdma_cq->cq.callback = mana_ib_cq_handler; 152 gdma_cq->id = cq->queue.id; 153 gc->cq_table[cq->queue.id] = gdma_cq; 154 return 0; 155 } 156 157 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 158 { 159 struct gdma_context *gc = mdev_to_gc(mdev); 160 161 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) 162 return; 163 164 if (cq->queue.kmem) 165 /* Then it will be cleaned and removed by the mana */ 166 return; 167 168 kfree(gc->cq_table[cq->queue.id]); 169 gc->cq_table[cq->queue.id] = NULL; 170 } 171 172 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 173 { 174 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 175 struct gdma_queue *gdma_cq = cq->queue.kmem; 176 177 if (!gdma_cq) 178 return -EINVAL; 179 180 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); 181 return 0; 182 } 183 184 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 185 { 186 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 187 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 188 struct ud_sq_shadow_wqe *shadow_wqe; 189 190 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); 191 if (!shadow_wqe) 192 return; 193 194 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; 195 196 wq->tail += shadow_wqe->header.posted_wqe_size; 197 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 198 } 199 200 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 201 { 202 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 203 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; 204 struct ud_rq_shadow_wqe *shadow_wqe; 205 206 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); 207 if (!shadow_wqe) 208 return; 209 210 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; 211 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; 212 shadow_wqe->header.error_code = IB_WC_SUCCESS; 213 214 wq->tail += shadow_wqe->header.posted_wqe_size; 215 shadow_queue_advance_next_to_complete(&qp->shadow_rq); 216 } 217 218 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) 219 { 220 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); 221 222 if (!qp) 223 return; 224 225 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { 226 if (cqe->is_sq) 227 handle_ud_sq_cqe(qp, cqe); 228 else 229 handle_ud_rq_cqe(qp, cqe); 230 } 231 232 mana_put_qp_ref(qp); 233 } 234 235 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, 236 const struct shadow_wqe_header *shadow_wqe) 237 { 238 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; 239 240 wc->wr_id = shadow_wqe->wr_id; 241 wc->status = shadow_wqe->error_code; 242 wc->opcode = shadow_wqe->opcode; 243 wc->vendor_err = shadow_wqe->error_code; 244 wc->wc_flags = 0; 245 wc->qp = &qp->ibqp; 246 wc->pkey_index = 0; 247 248 if (shadow_wqe->opcode == IB_WC_RECV) { 249 wc->byte_len = ud_wqe->byte_len; 250 wc->src_qp = ud_wqe->src_qpn; 251 wc->wc_flags |= IB_WC_GRH; 252 } 253 } 254 255 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) 256 { 257 struct shadow_wqe_header *shadow_wqe; 258 struct mana_ib_qp *qp; 259 int wc_index = 0; 260 261 /* process send shadow queue completions */ 262 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 263 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) 264 != NULL) { 265 if (wc_index >= nwc) 266 goto out; 267 268 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 269 shadow_queue_advance_consumer(&qp->shadow_sq); 270 wc_index++; 271 } 272 } 273 274 /* process recv shadow queue completions */ 275 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 276 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) 277 != NULL) { 278 if (wc_index >= nwc) 279 goto out; 280 281 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 282 shadow_queue_advance_consumer(&qp->shadow_rq); 283 wc_index++; 284 } 285 } 286 287 out: 288 return wc_index; 289 } 290 291 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) 292 { 293 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); 294 struct ud_sq_shadow_wqe *shadow_wqe; 295 struct mana_ib_cq *cq; 296 unsigned long flags; 297 298 if (!qp) 299 return; 300 301 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 302 303 spin_lock_irqsave(&cq->cq_lock, flags); 304 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) 305 != NULL) { 306 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; 307 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 308 } 309 spin_unlock_irqrestore(&cq->cq_lock, flags); 310 311 if (cq->ibcq.comp_handler) 312 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 313 314 mana_put_qp_ref(qp); 315 } 316 317 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 318 { 319 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 320 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); 321 struct gdma_queue *queue = cq->queue.kmem; 322 struct gdma_comp gdma_cqe; 323 unsigned long flags; 324 int num_polled = 0; 325 int comp_read, i; 326 327 spin_lock_irqsave(&cq->cq_lock, flags); 328 for (i = 0; i < num_entries; i++) { 329 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); 330 if (comp_read < 1) 331 break; 332 mana_handle_cqe(mdev, &gdma_cqe); 333 } 334 335 num_polled = mana_process_completions(cq, num_entries, wc); 336 spin_unlock_irqrestore(&cq->cq_lock, flags); 337 338 return num_polled; 339 } 340