1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 9 struct uverbs_attr_bundle *attrs) 10 { 11 struct ib_udata *udata = &attrs->driver_udata; 12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 13 struct mana_ib_create_cq_resp resp = {}; 14 struct mana_ib_ucontext *mana_ucontext; 15 struct ib_device *ibdev = ibcq->device; 16 struct mana_ib_create_cq ucmd = {}; 17 struct mana_ib_dev *mdev; 18 bool is_rnic_cq; 19 u32 doorbell; 20 u32 buf_size; 21 int err; 22 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 24 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 26 cq->cq_handle = INVALID_MANA_HANDLE; 27 28 if (udata) { 29 if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) 30 return -EINVAL; 31 32 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 33 if (err) { 34 ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err); 35 return err; 36 } 37 38 is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ); 39 40 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || 41 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { 42 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 43 return -EINVAL; 44 } 45 46 cq->cqe = attr->cqe; 47 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, 48 &cq->queue); 49 if (err) { 50 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); 51 return err; 52 } 53 54 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 55 ibucontext); 56 doorbell = mana_ucontext->doorbell; 57 } else { 58 is_rnic_cq = true; 59 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { 60 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 61 return -EINVAL; 62 } 63 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); 64 cq->cqe = buf_size / COMP_ENTRY_SIZE; 65 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); 66 if (err) { 67 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 68 return err; 69 } 70 doorbell = mdev->gdma_dev->doorbell; 71 } 72 73 if (is_rnic_cq) { 74 err = mana_ib_gd_create_cq(mdev, cq, doorbell); 75 if (err) { 76 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); 77 goto err_destroy_queue; 78 } 79 80 err = mana_ib_install_cq_cb(mdev, cq); 81 if (err) { 82 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); 83 goto err_destroy_rnic_cq; 84 } 85 } 86 87 if (udata) { 88 resp.cqid = cq->queue.id; 89 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 90 if (err) { 91 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 92 goto err_remove_cq_cb; 93 } 94 } 95 96 spin_lock_init(&cq->cq_lock); 97 INIT_LIST_HEAD(&cq->list_send_qp); 98 INIT_LIST_HEAD(&cq->list_recv_qp); 99 100 return 0; 101 102 err_remove_cq_cb: 103 mana_ib_remove_cq_cb(mdev, cq); 104 err_destroy_rnic_cq: 105 mana_ib_gd_destroy_cq(mdev, cq); 106 err_destroy_queue: 107 mana_ib_destroy_queue(mdev, &cq->queue); 108 109 return err; 110 } 111 112 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 113 { 114 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 115 struct ib_device *ibdev = ibcq->device; 116 struct mana_ib_dev *mdev; 117 118 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 119 120 mana_ib_remove_cq_cb(mdev, cq); 121 122 /* Ignore return code as there is not much we can do about it. 123 * The error message is printed inside. 124 */ 125 mana_ib_gd_destroy_cq(mdev, cq); 126 127 mana_ib_destroy_queue(mdev, &cq->queue); 128 129 return 0; 130 } 131 132 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq) 133 { 134 struct mana_ib_cq *cq = ctx; 135 136 if (cq->ibcq.comp_handler) 137 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 138 } 139 140 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 141 { 142 struct gdma_context *gc = mdev_to_gc(mdev); 143 struct gdma_queue *gdma_cq; 144 145 if (cq->queue.id >= gc->max_num_cqs) 146 return -EINVAL; 147 /* Create CQ table entry */ 148 WARN_ON(gc->cq_table[cq->queue.id]); 149 if (cq->queue.kmem) 150 gdma_cq = cq->queue.kmem; 151 else 152 gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL); 153 if (!gdma_cq) 154 return -ENOMEM; 155 156 gdma_cq->cq.context = cq; 157 gdma_cq->type = GDMA_CQ; 158 gdma_cq->cq.callback = mana_ib_cq_handler; 159 gdma_cq->id = cq->queue.id; 160 gc->cq_table[cq->queue.id] = gdma_cq; 161 return 0; 162 } 163 164 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 165 { 166 struct gdma_context *gc = mdev_to_gc(mdev); 167 168 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) 169 return; 170 171 if (cq->queue.kmem) 172 /* Then it will be cleaned and removed by the mana */ 173 return; 174 175 kfree(gc->cq_table[cq->queue.id]); 176 gc->cq_table[cq->queue.id] = NULL; 177 } 178 179 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 180 { 181 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 182 struct gdma_queue *gdma_cq = cq->queue.kmem; 183 184 if (!gdma_cq) 185 return -EINVAL; 186 187 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); 188 return 0; 189 } 190 191 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 192 { 193 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 194 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 195 struct ud_sq_shadow_wqe *shadow_wqe; 196 197 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); 198 if (!shadow_wqe) 199 return; 200 201 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; 202 203 wq->tail += shadow_wqe->header.posted_wqe_size; 204 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 205 } 206 207 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 208 { 209 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 210 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; 211 struct ud_rq_shadow_wqe *shadow_wqe; 212 213 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); 214 if (!shadow_wqe) 215 return; 216 217 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; 218 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; 219 shadow_wqe->header.error_code = IB_WC_SUCCESS; 220 221 wq->tail += shadow_wqe->header.posted_wqe_size; 222 shadow_queue_advance_next_to_complete(&qp->shadow_rq); 223 } 224 225 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) 226 { 227 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); 228 229 if (!qp) 230 return; 231 232 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { 233 if (cqe->is_sq) 234 handle_ud_sq_cqe(qp, cqe); 235 else 236 handle_ud_rq_cqe(qp, cqe); 237 } 238 239 mana_put_qp_ref(qp); 240 } 241 242 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, 243 const struct shadow_wqe_header *shadow_wqe) 244 { 245 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; 246 247 wc->wr_id = shadow_wqe->wr_id; 248 wc->status = shadow_wqe->error_code; 249 wc->opcode = shadow_wqe->opcode; 250 wc->vendor_err = shadow_wqe->error_code; 251 wc->wc_flags = 0; 252 wc->qp = &qp->ibqp; 253 wc->pkey_index = 0; 254 255 if (shadow_wqe->opcode == IB_WC_RECV) { 256 wc->byte_len = ud_wqe->byte_len; 257 wc->src_qp = ud_wqe->src_qpn; 258 wc->wc_flags |= IB_WC_GRH; 259 } 260 } 261 262 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) 263 { 264 struct shadow_wqe_header *shadow_wqe; 265 struct mana_ib_qp *qp; 266 int wc_index = 0; 267 268 /* process send shadow queue completions */ 269 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 270 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) 271 != NULL) { 272 if (wc_index >= nwc) 273 goto out; 274 275 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 276 shadow_queue_advance_consumer(&qp->shadow_sq); 277 wc_index++; 278 } 279 } 280 281 /* process recv shadow queue completions */ 282 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 283 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) 284 != NULL) { 285 if (wc_index >= nwc) 286 goto out; 287 288 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 289 shadow_queue_advance_consumer(&qp->shadow_rq); 290 wc_index++; 291 } 292 } 293 294 out: 295 return wc_index; 296 } 297 298 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) 299 { 300 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); 301 struct ud_sq_shadow_wqe *shadow_wqe; 302 struct mana_ib_cq *cq; 303 unsigned long flags; 304 305 if (!qp) 306 return; 307 308 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 309 310 spin_lock_irqsave(&cq->cq_lock, flags); 311 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) 312 != NULL) { 313 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; 314 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 315 } 316 spin_unlock_irqrestore(&cq->cq_lock, flags); 317 318 if (cq->ibcq.comp_handler) 319 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 320 321 mana_put_qp_ref(qp); 322 } 323 324 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 325 { 326 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 327 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); 328 struct gdma_queue *queue = cq->queue.kmem; 329 struct gdma_comp gdma_cqe; 330 unsigned long flags; 331 int num_polled = 0; 332 int comp_read, i; 333 334 spin_lock_irqsave(&cq->cq_lock, flags); 335 for (i = 0; i < num_entries; i++) { 336 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); 337 if (comp_read < 1) 338 break; 339 mana_handle_cqe(mdev, &gdma_cqe); 340 } 341 342 num_polled = mana_process_completions(cq, num_entries, wc); 343 spin_unlock_irqrestore(&cq->cq_lock, flags); 344 345 return num_polled; 346 } 347