1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6 #include "mana_ib.h"
7
mana_ib_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
9 struct uverbs_attr_bundle *attrs)
10 {
11 struct ib_udata *udata = &attrs->driver_udata;
12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
13 struct mana_ib_create_cq_resp resp = {};
14 struct mana_ib_ucontext *mana_ucontext;
15 struct ib_device *ibdev = ibcq->device;
16 struct mana_ib_create_cq ucmd = {};
17 struct mana_ib_dev *mdev;
18 bool is_rnic_cq;
19 u32 doorbell;
20 u32 buf_size;
21 int err;
22
23 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
24
25 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors;
26 cq->cq_handle = INVALID_MANA_HANDLE;
27
28 if (udata) {
29 if (udata->inlen < offsetof(struct mana_ib_create_cq, flags))
30 return -EINVAL;
31
32 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
33 if (err) {
34 ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err);
35 return err;
36 }
37
38 is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ);
39
40 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) ||
41 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) {
42 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
43 return -EINVAL;
44 }
45
46 cq->cqe = attr->cqe;
47 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
48 &cq->queue);
49 if (err) {
50 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
51 return err;
52 }
53
54 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
55 ibucontext);
56 doorbell = mana_ucontext->doorbell;
57 } else {
58 is_rnic_cq = true;
59 if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) {
60 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
61 return -EINVAL;
62 }
63 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE));
64 cq->cqe = buf_size / COMP_ENTRY_SIZE;
65 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue);
66 if (err) {
67 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err);
68 return err;
69 }
70 doorbell = mdev->gdma_dev->doorbell;
71 }
72
73 if (is_rnic_cq) {
74 err = mana_ib_gd_create_cq(mdev, cq, doorbell);
75 if (err) {
76 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err);
77 goto err_destroy_queue;
78 }
79
80 err = mana_ib_install_cq_cb(mdev, cq);
81 if (err) {
82 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err);
83 goto err_destroy_rnic_cq;
84 }
85 }
86
87 if (udata) {
88 resp.cqid = cq->queue.id;
89 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
90 if (err) {
91 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
92 goto err_remove_cq_cb;
93 }
94 }
95
96 spin_lock_init(&cq->cq_lock);
97 INIT_LIST_HEAD(&cq->list_send_qp);
98 INIT_LIST_HEAD(&cq->list_recv_qp);
99
100 return 0;
101
102 err_remove_cq_cb:
103 mana_ib_remove_cq_cb(mdev, cq);
104 err_destroy_rnic_cq:
105 mana_ib_gd_destroy_cq(mdev, cq);
106 err_destroy_queue:
107 mana_ib_destroy_queue(mdev, &cq->queue);
108
109 return err;
110 }
111
mana_ib_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)112 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
113 {
114 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
115 struct ib_device *ibdev = ibcq->device;
116 struct mana_ib_dev *mdev;
117
118 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
119
120 mana_ib_remove_cq_cb(mdev, cq);
121
122 /* Ignore return code as there is not much we can do about it.
123 * The error message is printed inside.
124 */
125 mana_ib_gd_destroy_cq(mdev, cq);
126
127 mana_ib_destroy_queue(mdev, &cq->queue);
128
129 return 0;
130 }
131
mana_ib_cq_handler(void * ctx,struct gdma_queue * gdma_cq)132 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
133 {
134 struct mana_ib_cq *cq = ctx;
135
136 if (cq->ibcq.comp_handler)
137 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
138 }
139
mana_ib_install_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)140 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
141 {
142 struct gdma_context *gc = mdev_to_gc(mdev);
143 struct gdma_queue *gdma_cq;
144
145 if (cq->queue.id >= gc->max_num_cqs)
146 return -EINVAL;
147 /* Create CQ table entry */
148 WARN_ON(gc->cq_table[cq->queue.id]);
149 if (cq->queue.kmem)
150 gdma_cq = cq->queue.kmem;
151 else
152 gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
153 if (!gdma_cq)
154 return -ENOMEM;
155
156 gdma_cq->cq.context = cq;
157 gdma_cq->type = GDMA_CQ;
158 gdma_cq->cq.callback = mana_ib_cq_handler;
159 gdma_cq->id = cq->queue.id;
160 gc->cq_table[cq->queue.id] = gdma_cq;
161 return 0;
162 }
163
mana_ib_remove_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)164 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
165 {
166 struct gdma_context *gc = mdev_to_gc(mdev);
167
168 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID)
169 return;
170
171 if (cq->queue.kmem)
172 /* Then it will be cleaned and removed by the mana */
173 return;
174
175 kfree(gc->cq_table[cq->queue.id]);
176 gc->cq_table[cq->queue.id] = NULL;
177 }
178
mana_ib_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)179 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
180 {
181 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
182 struct gdma_queue *gdma_cq = cq->queue.kmem;
183
184 if (!gdma_cq)
185 return -EINVAL;
186
187 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
188 return 0;
189 }
190
handle_ud_sq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)191 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
192 {
193 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
194 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
195 struct ud_sq_shadow_wqe *shadow_wqe;
196
197 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
198 if (!shadow_wqe)
199 return;
200
201 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
202
203 wq->tail += shadow_wqe->header.posted_wqe_size;
204 shadow_queue_advance_next_to_complete(&qp->shadow_sq);
205 }
206
handle_ud_rq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)207 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
208 {
209 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
210 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
211 struct ud_rq_shadow_wqe *shadow_wqe;
212
213 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
214 if (!shadow_wqe)
215 return;
216
217 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
218 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
219 shadow_wqe->header.error_code = IB_WC_SUCCESS;
220
221 wq->tail += shadow_wqe->header.posted_wqe_size;
222 shadow_queue_advance_next_to_complete(&qp->shadow_rq);
223 }
224
mana_handle_cqe(struct mana_ib_dev * mdev,struct gdma_comp * cqe)225 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
226 {
227 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
228
229 if (!qp)
230 return;
231
232 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
233 if (cqe->is_sq)
234 handle_ud_sq_cqe(qp, cqe);
235 else
236 handle_ud_rq_cqe(qp, cqe);
237 }
238
239 mana_put_qp_ref(qp);
240 }
241
fill_verbs_from_shadow_wqe(struct mana_ib_qp * qp,struct ib_wc * wc,const struct shadow_wqe_header * shadow_wqe)242 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
243 const struct shadow_wqe_header *shadow_wqe)
244 {
245 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
246
247 wc->wr_id = shadow_wqe->wr_id;
248 wc->status = shadow_wqe->error_code;
249 wc->opcode = shadow_wqe->opcode;
250 wc->vendor_err = shadow_wqe->error_code;
251 wc->wc_flags = 0;
252 wc->qp = &qp->ibqp;
253 wc->pkey_index = 0;
254
255 if (shadow_wqe->opcode == IB_WC_RECV) {
256 wc->byte_len = ud_wqe->byte_len;
257 wc->src_qp = ud_wqe->src_qpn;
258 wc->wc_flags |= IB_WC_GRH;
259 }
260 }
261
mana_process_completions(struct mana_ib_cq * cq,int nwc,struct ib_wc * wc)262 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
263 {
264 struct shadow_wqe_header *shadow_wqe;
265 struct mana_ib_qp *qp;
266 int wc_index = 0;
267
268 /* process send shadow queue completions */
269 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
270 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
271 != NULL) {
272 if (wc_index >= nwc)
273 goto out;
274
275 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
276 shadow_queue_advance_consumer(&qp->shadow_sq);
277 wc_index++;
278 }
279 }
280
281 /* process recv shadow queue completions */
282 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
283 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
284 != NULL) {
285 if (wc_index >= nwc)
286 goto out;
287
288 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
289 shadow_queue_advance_consumer(&qp->shadow_rq);
290 wc_index++;
291 }
292 }
293
294 out:
295 return wc_index;
296 }
297
mana_drain_gsi_sqs(struct mana_ib_dev * mdev)298 void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
299 {
300 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);
301 struct ud_sq_shadow_wqe *shadow_wqe;
302 struct mana_ib_cq *cq;
303 unsigned long flags;
304
305 if (!qp)
306 return;
307
308 cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
309
310 spin_lock_irqsave(&cq->cq_lock, flags);
311 while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
312 != NULL) {
313 shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
314 shadow_queue_advance_next_to_complete(&qp->shadow_sq);
315 }
316 spin_unlock_irqrestore(&cq->cq_lock, flags);
317
318 if (cq->ibcq.comp_handler)
319 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
320
321 mana_put_qp_ref(qp);
322 }
323
mana_ib_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)324 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
325 {
326 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
327 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
328 struct gdma_queue *queue = cq->queue.kmem;
329 struct gdma_comp gdma_cqe;
330 unsigned long flags;
331 int num_polled = 0;
332 int comp_read, i;
333
334 spin_lock_irqsave(&cq->cq_lock, flags);
335 for (i = 0; i < num_entries; i++) {
336 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
337 if (comp_read < 1)
338 break;
339 mana_handle_cqe(mdev, &gdma_cqe);
340 }
341
342 num_polled = mana_process_completions(cq, num_entries, wc);
343 spin_unlock_irqrestore(&cq->cq_lock, flags);
344
345 return num_polled;
346 }
347