1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 9 struct uverbs_attr_bundle *attrs) 10 { 11 struct ib_udata *udata = &attrs->driver_udata; 12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 13 struct mana_ib_create_cq_resp resp = {}; 14 struct mana_ib_ucontext *mana_ucontext; 15 struct ib_device *ibdev = ibcq->device; 16 struct mana_ib_create_cq ucmd; 17 struct mana_ib_dev *mdev; 18 bool is_rnic_cq; 19 u32 doorbell; 20 u32 buf_size; 21 int err; 22 23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 24 25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; 26 cq->cq_handle = INVALID_MANA_HANDLE; 27 is_rnic_cq = mana_ib_is_rnic(mdev); 28 29 if (udata) { 30 err = ib_copy_validate_udata_in(udata, ucmd, buf_addr); 31 if (err) 32 return err; 33 34 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || 35 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { 36 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 37 return -EINVAL; 38 } 39 40 cq->cqe = attr->cqe; 41 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, 42 &cq->queue); 43 if (err) { 44 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); 45 return err; 46 } 47 48 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 49 ibucontext); 50 doorbell = mana_ucontext->doorbell; 51 } else { 52 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) { 53 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); 54 return -EINVAL; 55 } 56 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); 57 cq->cqe = buf_size / COMP_ENTRY_SIZE; 58 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); 59 if (err) { 60 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); 61 return err; 62 } 63 doorbell = mdev->gdma_dev->doorbell; 64 } 65 66 if (is_rnic_cq) { 67 err = mana_ib_gd_create_cq(mdev, cq, doorbell); 68 if (err) { 69 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); 70 goto err_destroy_queue; 71 } 72 73 err = mana_ib_install_cq_cb(mdev, cq); 74 if (err) { 75 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); 76 goto err_destroy_rnic_cq; 77 } 78 } 79 80 if (udata) { 81 resp.cqid = cq->queue.id; 82 err = ib_respond_udata(udata, resp); 83 if (err) 84 goto err_remove_cq_cb; 85 } 86 87 spin_lock_init(&cq->cq_lock); 88 INIT_LIST_HEAD(&cq->list_send_qp); 89 INIT_LIST_HEAD(&cq->list_recv_qp); 90 91 return 0; 92 93 err_remove_cq_cb: 94 mana_ib_remove_cq_cb(mdev, cq); 95 err_destroy_rnic_cq: 96 mana_ib_gd_destroy_cq(mdev, cq); 97 err_destroy_queue: 98 mana_ib_destroy_queue(mdev, &cq->queue); 99 100 return err; 101 } 102 103 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 104 { 105 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 106 struct ib_device *ibdev = ibcq->device; 107 struct mana_ib_dev *mdev; 108 109 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 110 111 mana_ib_remove_cq_cb(mdev, cq); 112 113 /* Ignore return code as there is not much we can do about it. 114 * The error message is printed inside. 115 */ 116 mana_ib_gd_destroy_cq(mdev, cq); 117 118 mana_ib_destroy_queue(mdev, &cq->queue); 119 120 return 0; 121 } 122 123 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq) 124 { 125 struct mana_ib_cq *cq = ctx; 126 127 if (cq->ibcq.comp_handler) 128 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 129 } 130 131 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 132 { 133 struct gdma_context *gc = mdev_to_gc(mdev); 134 struct gdma_queue *gdma_cq; 135 136 if (cq->queue.id >= gc->max_num_cqs) 137 return -EINVAL; 138 /* Create CQ table entry, sharing a CQ between WQs is not supported */ 139 if (gc->cq_table[cq->queue.id]) 140 return -EINVAL; 141 if (cq->queue.kmem) 142 gdma_cq = cq->queue.kmem; 143 else 144 gdma_cq = kzalloc_obj(*gdma_cq); 145 if (!gdma_cq) 146 return -ENOMEM; 147 148 gdma_cq->cq.context = cq; 149 gdma_cq->type = GDMA_CQ; 150 gdma_cq->cq.callback = mana_ib_cq_handler; 151 gdma_cq->id = cq->queue.id; 152 gc->cq_table[cq->queue.id] = gdma_cq; 153 return 0; 154 } 155 156 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 157 { 158 struct gdma_context *gc = mdev_to_gc(mdev); 159 160 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) 161 return; 162 163 if (cq->queue.kmem) 164 /* Then it will be cleaned and removed by the mana */ 165 return; 166 167 kfree(gc->cq_table[cq->queue.id]); 168 gc->cq_table[cq->queue.id] = NULL; 169 } 170 171 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 172 { 173 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 174 struct gdma_queue *gdma_cq = cq->queue.kmem; 175 176 if (!gdma_cq) 177 return -EINVAL; 178 179 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); 180 return 0; 181 } 182 183 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 184 { 185 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 186 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; 187 struct ud_sq_shadow_wqe *shadow_wqe; 188 189 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); 190 if (!shadow_wqe) 191 return; 192 193 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; 194 195 wq->tail += shadow_wqe->header.posted_wqe_size; 196 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 197 } 198 199 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) 200 { 201 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; 202 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; 203 struct ud_rq_shadow_wqe *shadow_wqe; 204 205 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); 206 if (!shadow_wqe) 207 return; 208 209 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; 210 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; 211 shadow_wqe->header.error_code = IB_WC_SUCCESS; 212 213 wq->tail += shadow_wqe->header.posted_wqe_size; 214 shadow_queue_advance_next_to_complete(&qp->shadow_rq); 215 } 216 217 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) 218 { 219 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); 220 221 if (!qp) 222 return; 223 224 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { 225 if (cqe->is_sq) 226 handle_ud_sq_cqe(qp, cqe); 227 else 228 handle_ud_rq_cqe(qp, cqe); 229 } 230 231 mana_put_qp_ref(qp); 232 } 233 234 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, 235 const struct shadow_wqe_header *shadow_wqe) 236 { 237 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; 238 239 wc->wr_id = shadow_wqe->wr_id; 240 wc->status = shadow_wqe->error_code; 241 wc->opcode = shadow_wqe->opcode; 242 wc->vendor_err = shadow_wqe->error_code; 243 wc->wc_flags = 0; 244 wc->qp = &qp->ibqp; 245 wc->pkey_index = 0; 246 247 if (shadow_wqe->opcode == IB_WC_RECV) { 248 wc->byte_len = ud_wqe->byte_len; 249 wc->src_qp = ud_wqe->src_qpn; 250 wc->wc_flags |= IB_WC_GRH; 251 } 252 } 253 254 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) 255 { 256 struct shadow_wqe_header *shadow_wqe; 257 struct mana_ib_qp *qp; 258 int wc_index = 0; 259 260 /* process send shadow queue completions */ 261 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { 262 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) 263 != NULL) { 264 if (wc_index >= nwc) 265 goto out; 266 267 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 268 shadow_queue_advance_consumer(&qp->shadow_sq); 269 wc_index++; 270 } 271 } 272 273 /* process recv shadow queue completions */ 274 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { 275 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) 276 != NULL) { 277 if (wc_index >= nwc) 278 goto out; 279 280 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); 281 shadow_queue_advance_consumer(&qp->shadow_rq); 282 wc_index++; 283 } 284 } 285 286 out: 287 return wc_index; 288 } 289 290 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) 291 { 292 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); 293 struct ud_sq_shadow_wqe *shadow_wqe; 294 struct mana_ib_cq *cq; 295 unsigned long flags; 296 297 if (!qp) 298 return; 299 300 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 301 302 spin_lock_irqsave(&cq->cq_lock, flags); 303 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) 304 != NULL) { 305 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; 306 shadow_queue_advance_next_to_complete(&qp->shadow_sq); 307 } 308 spin_unlock_irqrestore(&cq->cq_lock, flags); 309 310 if (cq->ibcq.comp_handler) 311 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 312 313 mana_put_qp_ref(qp); 314 } 315 316 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 317 { 318 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); 319 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); 320 struct gdma_queue *queue = cq->queue.kmem; 321 struct gdma_comp gdma_cqe; 322 unsigned long flags; 323 int num_polled = 0; 324 int comp_read, i; 325 326 spin_lock_irqsave(&cq->cq_lock, flags); 327 for (i = 0; i < num_entries; i++) { 328 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); 329 if (comp_read < 1) 330 break; 331 mana_handle_cqe(mdev, &gdma_cqe); 332 } 333 334 num_polled = mana_process_completions(cq, num_entries, wc); 335 spin_unlock_irqrestore(&cq->cq_lock, flags); 336 337 return num_polled; 338 } 339