1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 9 struct uverbs_attr_bundle *attrs) 10 { 11 struct ib_udata *udata = &attrs->driver_udata; 12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 13 struct mana_ib_create_cq_resp resp = {}; 14 struct mana_ib_ucontext *mana_ucontext; 15 struct ib_device *ibdev = ibcq->device; 16 struct mana_ib_create_cq ucmd; 17 struct mana_ib_dev *mdev; 18 bool is_rnic_cq; 19 u32 doorbell; 20 u32 buf_size; 21 int err; 22 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 24 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 26 cq->cq_handle = INVALID_MANA_HANDLE; 27 is_rnic_cq = mana_ib_is_rnic(mdev); 28 29 if (udata) { 30 err = ib_copy_validate_udata_in(udata, ucmd, buf_addr); 31 if (err) 32 return err; 33 34 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || 35 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { 36 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 37 return -EINVAL; 38 } 39 40 cq->cqe = attr->cqe; 41 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, 42 &cq->queue); 43 if (err) { 44 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); 45 return err; 46 } 47 48 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 49 ibucontext); 50 doorbell = mana_ucontext->doorbell; 51 } else { 52 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { 53 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 54 return -EINVAL; 55 } 56 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); 57 cq->cqe = buf_size / COMP_ENTRY_SIZE; 58 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); 59 if (err) { 60 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 61 return err; 62 } 63 doorbell = mdev->gdma_dev->doorbell; 64 } 65 66 if (is_rnic_cq) { 67 err = mana_ib_gd_create_cq(mdev, cq, doorbell); 68 if (err) { 69 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); 70 goto err_destroy_queue; 71 } 72 73 err = mana_ib_install_cq_cb(mdev, cq); 74 if (err) { 75 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); 76 goto err_destroy_rnic_cq; 77 } 78 } 79 80 if (udata) { 81 resp.cqid = cq->queue.id; 82 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 83 if (err) { 84 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 85 goto err_remove_cq_cb; 86 } 87 } 88 89 spin_lock_init(&cq->cq_lock); 90 INIT_LIST_HEAD(&cq->list_send_qp); 91 INIT_LIST_HEAD(&cq->list_recv_qp); 92 93 return 0; 94 95 err_remove_cq_cb: 96 mana_ib_remove_cq_cb(mdev, cq); 97 err_destroy_rnic_cq: 98 mana_ib_gd_destroy_cq(mdev, cq); 99 err_destroy_queue: 100 mana_ib_destroy_queue(mdev, &cq->queue); 101 102 return err; 103 } 104 105 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 106 { 107 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 108 struct ib_device *ibdev = ibcq->device; 109 struct mana_ib_dev *mdev; 110 111 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 112 113 mana_ib_remove_cq_cb(mdev, cq); 114 115 /* Ignore return code as there is not much we can do about it. 116 * The error message is printed inside. 117 */ 118 mana_ib_gd_destroy_cq(mdev, cq); 119 120 mana_ib_destroy_queue(mdev, &cq->queue); 121 122 return 0; 123 } 124 125 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq) 126 { 127 struct mana_ib_cq *cq = ctx; 128 129 if (cq->ibcq.comp_handler) 130 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 131 } 132 133 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 134 { 135 struct gdma_context *gc = mdev_to_gc(mdev); 136 struct gdma_queue *gdma_cq; 137 138 if (cq->queue.id >= gc->max_num_cqs) 139 return -EINVAL; 140 /* Create CQ table entry, sharing a CQ between WQs is not supported */ 141 if (gc->cq_table[cq->queue.id]) 142 return -EINVAL; 143 if (cq->queue.kmem) 144 gdma_cq = cq->queue.kmem; 145 else 146 gdma_cq = kzalloc_obj(*gdma_cq); 147 if (!gdma_cq) 148 return -ENOMEM; 149 150 gdma_cq->cq.context = cq; 151 gdma_cq->type = GDMA_CQ; 152 gdma_cq->cq.callback = mana_ib_cq_handler; 153 gdma_cq->id = cq->queue.id; 154 gc->cq_table[cq->queue.id] = gdma_cq; 155 return 0; 156 } 157 158 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 159 { 160 struct gdma_context *gc = mdev_to_gc(mdev); 161 162 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) 163 return; 164 165 if (cq->queue.kmem) 166 /* Then it will be cleaned and removed by the mana */ 167 return; 168 169 kfree(gc->cq_table[cq->queue.id]); 170 gc->cq_table[cq->queue.id] = NULL; 171 } 172 173 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 174 { 175 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 176 struct gdma_queue *gdma_cq = cq->queue.kmem; 177 178 if (!gdma_cq) 179 return -EINVAL; 180 181 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); 182 return 0; 183 } 184 185 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 186 { 187 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 188 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 189 struct ud_sq_shadow_wqe *shadow_wqe; 190 191 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); 192 if (!shadow_wqe) 193 return; 194 195 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; 196 197 wq->tail += shadow_wqe->header.posted_wqe_size; 198 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 199 } 200 201 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 202 { 203 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 204 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; 205 struct ud_rq_shadow_wqe *shadow_wqe; 206 207 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); 208 if (!shadow_wqe) 209 return; 210 211 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; 212 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; 213 shadow_wqe->header.error_code = IB_WC_SUCCESS; 214 215 wq->tail += shadow_wqe->header.posted_wqe_size; 216 shadow_queue_advance_next_to_complete(&qp->shadow_rq); 217 } 218 219 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) 220 { 221 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); 222 223 if (!qp) 224 return; 225 226 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { 227 if (cqe->is_sq) 228 handle_ud_sq_cqe(qp, cqe); 229 else 230 handle_ud_rq_cqe(qp, cqe); 231 } 232 233 mana_put_qp_ref(qp); 234 } 235 236 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, 237 const struct shadow_wqe_header *shadow_wqe) 238 { 239 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; 240 241 wc->wr_id = shadow_wqe->wr_id; 242 wc->status = shadow_wqe->error_code; 243 wc->opcode = shadow_wqe->opcode; 244 wc->vendor_err = shadow_wqe->error_code; 245 wc->wc_flags = 0; 246 wc->qp = &qp->ibqp; 247 wc->pkey_index = 0; 248 249 if (shadow_wqe->opcode == IB_WC_RECV) { 250 wc->byte_len = ud_wqe->byte_len; 251 wc->src_qp = ud_wqe->src_qpn; 252 wc->wc_flags |= IB_WC_GRH; 253 } 254 } 255 256 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) 257 { 258 struct shadow_wqe_header *shadow_wqe; 259 struct mana_ib_qp *qp; 260 int wc_index = 0; 261 262 /* process send shadow queue completions */ 263 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 264 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) 265 != NULL) { 266 if (wc_index >= nwc) 267 goto out; 268 269 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 270 shadow_queue_advance_consumer(&qp->shadow_sq); 271 wc_index++; 272 } 273 } 274 275 /* process recv shadow queue completions */ 276 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 277 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) 278 != NULL) { 279 if (wc_index >= nwc) 280 goto out; 281 282 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 283 shadow_queue_advance_consumer(&qp->shadow_rq); 284 wc_index++; 285 } 286 } 287 288 out: 289 return wc_index; 290 } 291 292 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) 293 { 294 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); 295 struct ud_sq_shadow_wqe *shadow_wqe; 296 struct mana_ib_cq *cq; 297 unsigned long flags; 298 299 if (!qp) 300 return; 301 302 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 303 304 spin_lock_irqsave(&cq->cq_lock, flags); 305 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) 306 != NULL) { 307 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; 308 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 309 } 310 spin_unlock_irqrestore(&cq->cq_lock, flags); 311 312 if (cq->ibcq.comp_handler) 313 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 314 315 mana_put_qp_ref(qp); 316 } 317 318 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 319 { 320 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 321 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); 322 struct gdma_queue *queue = cq->queue.kmem; 323 struct gdma_comp gdma_cqe; 324 unsigned long flags; 325 int num_polled = 0; 326 int comp_read, i; 327 328 spin_lock_irqsave(&cq->cq_lock, flags); 329 for (i = 0; i < num_entries; i++) { 330 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); 331 if (comp_read < 1) 332 break; 333 mana_handle_cqe(mdev, &gdma_cqe); 334 } 335 336 num_polled = mana_process_completions(cq, num_entries, wc); 337 spin_unlock_irqrestore(&cq->cq_lock, flags); 338 339 return num_polled; 340 } 341