1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 9 struct uverbs_attr_bundle *attrs) 10 { 11 struct ib_udata *udata = &attrs->driver_udata; 12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 13 struct mana_ib_create_cq_resp resp = {}; 14 struct mana_ib_ucontext *mana_ucontext; 15 struct ib_device *ibdev = ibcq->device; 16 struct mana_ib_create_cq ucmd = {}; 17 struct mana_ib_dev *mdev; 18 bool is_rnic_cq; 19 u32 doorbell; 20 u32 buf_size; 21 int err; 22 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 24 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 26 cq->cq_handle = INVALID_MANA_HANDLE; 27 is_rnic_cq = mana_ib_is_rnic(mdev); 28 29 if (udata) { 30 if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) 31 return -EINVAL; 32 33 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 34 if (err) { 35 ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err); 36 return err; 37 } 38 39 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || 40 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { 41 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 42 return -EINVAL; 43 } 44 45 cq->cqe = attr->cqe; 46 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, 47 &cq->queue); 48 if (err) { 49 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); 50 return err; 51 } 52 53 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 54 ibucontext); 55 doorbell = mana_ucontext->doorbell; 56 } else { 57 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { 58 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 59 return -EINVAL; 60 } 61 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); 62 cq->cqe = buf_size / COMP_ENTRY_SIZE; 63 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); 64 if (err) { 65 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 66 return err; 67 } 68 doorbell = mdev->gdma_dev->doorbell; 69 } 70 71 if (is_rnic_cq) { 72 err = mana_ib_gd_create_cq(mdev, cq, doorbell); 73 if (err) { 74 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); 75 goto err_destroy_queue; 76 } 77 78 err = mana_ib_install_cq_cb(mdev, cq); 79 if (err) { 80 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); 81 goto err_destroy_rnic_cq; 82 } 83 } 84 85 if (udata) { 86 resp.cqid = cq->queue.id; 87 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 88 if (err) { 89 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 90 goto err_remove_cq_cb; 91 } 92 } 93 94 spin_lock_init(&cq->cq_lock); 95 INIT_LIST_HEAD(&cq->list_send_qp); 96 INIT_LIST_HEAD(&cq->list_recv_qp); 97 98 return 0; 99 100 err_remove_cq_cb: 101 mana_ib_remove_cq_cb(mdev, cq); 102 err_destroy_rnic_cq: 103 mana_ib_gd_destroy_cq(mdev, cq); 104 err_destroy_queue: 105 mana_ib_destroy_queue(mdev, &cq->queue); 106 107 return err; 108 } 109 110 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 111 { 112 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 113 struct ib_device *ibdev = ibcq->device; 114 struct mana_ib_dev *mdev; 115 116 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 117 118 mana_ib_remove_cq_cb(mdev, cq); 119 120 /* Ignore return code as there is not much we can do about it. 121 * The error message is printed inside. 122 */ 123 mana_ib_gd_destroy_cq(mdev, cq); 124 125 mana_ib_destroy_queue(mdev, &cq->queue); 126 127 return 0; 128 } 129 130 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq) 131 { 132 struct mana_ib_cq *cq = ctx; 133 134 if (cq->ibcq.comp_handler) 135 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 136 } 137 138 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 139 { 140 struct gdma_context *gc = mdev_to_gc(mdev); 141 struct gdma_queue *gdma_cq; 142 143 if (cq->queue.id >= gc->max_num_cqs) 144 return -EINVAL; 145 /* Create CQ table entry */ 146 WARN_ON(gc->cq_table[cq->queue.id]); 147 if (cq->queue.kmem) 148 gdma_cq = cq->queue.kmem; 149 else 150 gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL); 151 if (!gdma_cq) 152 return -ENOMEM; 153 154 gdma_cq->cq.context = cq; 155 gdma_cq->type = GDMA_CQ; 156 gdma_cq->cq.callback = mana_ib_cq_handler; 157 gdma_cq->id = cq->queue.id; 158 gc->cq_table[cq->queue.id] = gdma_cq; 159 return 0; 160 } 161 162 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 163 { 164 struct gdma_context *gc = mdev_to_gc(mdev); 165 166 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) 167 return; 168 169 if (cq->queue.kmem) 170 /* Then it will be cleaned and removed by the mana */ 171 return; 172 173 kfree(gc->cq_table[cq->queue.id]); 174 gc->cq_table[cq->queue.id] = NULL; 175 } 176 177 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 178 { 179 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 180 struct gdma_queue *gdma_cq = cq->queue.kmem; 181 182 if (!gdma_cq) 183 return -EINVAL; 184 185 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); 186 return 0; 187 } 188 189 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 190 { 191 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 192 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 193 struct ud_sq_shadow_wqe *shadow_wqe; 194 195 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); 196 if (!shadow_wqe) 197 return; 198 199 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; 200 201 wq->tail += shadow_wqe->header.posted_wqe_size; 202 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 203 } 204 205 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 206 { 207 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 208 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; 209 struct ud_rq_shadow_wqe *shadow_wqe; 210 211 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); 212 if (!shadow_wqe) 213 return; 214 215 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; 216 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; 217 shadow_wqe->header.error_code = IB_WC_SUCCESS; 218 219 wq->tail += shadow_wqe->header.posted_wqe_size; 220 shadow_queue_advance_next_to_complete(&qp->shadow_rq); 221 } 222 223 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) 224 { 225 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); 226 227 if (!qp) 228 return; 229 230 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { 231 if (cqe->is_sq) 232 handle_ud_sq_cqe(qp, cqe); 233 else 234 handle_ud_rq_cqe(qp, cqe); 235 } 236 237 mana_put_qp_ref(qp); 238 } 239 240 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, 241 const struct shadow_wqe_header *shadow_wqe) 242 { 243 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; 244 245 wc->wr_id = shadow_wqe->wr_id; 246 wc->status = shadow_wqe->error_code; 247 wc->opcode = shadow_wqe->opcode; 248 wc->vendor_err = shadow_wqe->error_code; 249 wc->wc_flags = 0; 250 wc->qp = &qp->ibqp; 251 wc->pkey_index = 0; 252 253 if (shadow_wqe->opcode == IB_WC_RECV) { 254 wc->byte_len = ud_wqe->byte_len; 255 wc->src_qp = ud_wqe->src_qpn; 256 wc->wc_flags |= IB_WC_GRH; 257 } 258 } 259 260 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) 261 { 262 struct shadow_wqe_header *shadow_wqe; 263 struct mana_ib_qp *qp; 264 int wc_index = 0; 265 266 /* process send shadow queue completions */ 267 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 268 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) 269 != NULL) { 270 if (wc_index >= nwc) 271 goto out; 272 273 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 274 shadow_queue_advance_consumer(&qp->shadow_sq); 275 wc_index++; 276 } 277 } 278 279 /* process recv shadow queue completions */ 280 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 281 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) 282 != NULL) { 283 if (wc_index >= nwc) 284 goto out; 285 286 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 287 shadow_queue_advance_consumer(&qp->shadow_rq); 288 wc_index++; 289 } 290 } 291 292 out: 293 return wc_index; 294 } 295 296 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) 297 { 298 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); 299 struct ud_sq_shadow_wqe *shadow_wqe; 300 struct mana_ib_cq *cq; 301 unsigned long flags; 302 303 if (!qp) 304 return; 305 306 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 307 308 spin_lock_irqsave(&cq->cq_lock, flags); 309 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) 310 != NULL) { 311 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; 312 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 313 } 314 spin_unlock_irqrestore(&cq->cq_lock, flags); 315 316 if (cq->ibcq.comp_handler) 317 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 318 319 mana_put_qp_ref(qp); 320 } 321 322 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 323 { 324 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 325 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); 326 struct gdma_queue *queue = cq->queue.kmem; 327 struct gdma_comp gdma_cqe; 328 unsigned long flags; 329 int num_polled = 0; 330 int comp_read, i; 331 332 spin_lock_irqsave(&cq->cq_lock, flags); 333 for (i = 0; i < num_entries; i++) { 334 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); 335 if (comp_read < 1) 336 break; 337 mana_handle_cqe(mdev, &gdma_cqe); 338 } 339 340 num_polled = mana_process_completions(cq, num_entries, wc); 341 spin_unlock_irqrestore(&cq->cq_lock, flags); 342 343 return num_polled; 344 } 345