1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Microsoft Corp.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/types.h>
34 #include <sys/kernel.h>
35 #include <sys/kthread.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mutex.h>
39 #include <sys/bus.h>
40 #include <machine/bus.h>
41
42 #include "mana.h"
43 #include "hw_channel.h"
44
45 static int
mana_hwc_get_msg_index(struct hw_channel_context * hwc,uint16_t * msg_id)46 mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id)
47 {
48 struct gdma_resource *r = &hwc->inflight_msg_res;
49 uint32_t index;
50
51 sema_wait(&hwc->sema);
52
53 mtx_lock_spin(&r->lock_spin);
54
55 index = find_first_zero_bit(hwc->inflight_msg_res.map,
56 hwc->inflight_msg_res.size);
57
58 bitmap_set(hwc->inflight_msg_res.map, index, 1);
59
60 mtx_unlock_spin(&r->lock_spin);
61
62 *msg_id = index;
63
64 return 0;
65 }
66
67 static void
mana_hwc_put_msg_index(struct hw_channel_context * hwc,uint16_t msg_id)68 mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id)
69 {
70 struct gdma_resource *r = &hwc->inflight_msg_res;
71
72 mtx_lock_spin(&r->lock_spin);
73 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1);
74 mtx_unlock_spin(&r->lock_spin);
75
76 sema_post(&hwc->sema);
77 }
78
79 static int
mana_hwc_verify_resp_msg(const struct hwc_caller_ctx * caller_ctx,const struct gdma_resp_hdr * resp_msg,uint32_t resp_len)80 mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
81 const struct gdma_resp_hdr *resp_msg,
82 uint32_t resp_len)
83 {
84 if (resp_len < sizeof(*resp_msg))
85 return EPROTO;
86
87 if (resp_len > caller_ctx->output_buflen)
88 return EPROTO;
89
90 return 0;
91 }
92
93 static void
mana_hwc_handle_resp(struct hw_channel_context * hwc,uint32_t resp_len,const struct gdma_resp_hdr * resp_msg)94 mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len,
95 const struct gdma_resp_hdr *resp_msg)
96 {
97 struct hwc_caller_ctx *ctx;
98 int err;
99
100 if (!test_bit(resp_msg->response.hwc_msg_id,
101 hwc->inflight_msg_res.map)) {
102 device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
103 resp_msg->response.hwc_msg_id);
104 return;
105 }
106
107 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;
108 err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
109 if (err)
110 goto out;
111
112 ctx->status_code = resp_msg->status;
113
114 memcpy(ctx->output_buf, resp_msg, resp_len);
115 out:
116 ctx->error = err;
117 complete(&ctx->comp_event);
118 }
119
120 static int
mana_hwc_post_rx_wqe(const struct hwc_wq * hwc_rxq,struct hwc_work_request * req)121 mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
122 struct hwc_work_request *req)
123 {
124 device_t dev = hwc_rxq->hwc->dev;
125 struct gdma_sge *sge;
126 int err;
127
128 sge = &req->sge;
129 sge->address = (uintptr_t)req->buf_sge_addr;
130 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
131 sge->size = req->buf_len;
132
133 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
134 req->wqe_req.sgl = sge;
135 req->wqe_req.num_sge = 1;
136 req->wqe_req.client_data_unit = 0;
137
138 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
139 if (err)
140 device_printf(dev,
141 "Failed to post WQE on HWC RQ: %d\n", err);
142 return err;
143 }
144
145 static void
mana_hwc_init_event_handler(void * ctx,struct gdma_queue * q_self,struct gdma_event * event)146 mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
147 struct gdma_event *event)
148 {
149 struct hw_channel_context *hwc = ctx;
150 struct gdma_dev *gd = hwc->gdma_dev;
151 union hwc_init_type_data type_data;
152 union hwc_init_eq_id_db eq_db;
153 uint32_t type, val;
154
155 switch (event->type) {
156 case GDMA_EQE_HWC_INIT_EQ_ID_DB:
157 eq_db.as_uint32 = event->details[0];
158 hwc->cq->gdma_eq->id = eq_db.eq_id;
159 gd->doorbell = eq_db.doorbell;
160 break;
161
162 case GDMA_EQE_HWC_INIT_DATA:
163 type_data.as_uint32 = event->details[0];
164 type = type_data.type;
165 val = type_data.value;
166
167 switch (type) {
168 case HWC_INIT_DATA_CQID:
169 hwc->cq->gdma_cq->id = val;
170 break;
171
172 case HWC_INIT_DATA_RQID:
173 hwc->rxq->gdma_wq->id = val;
174 break;
175
176 case HWC_INIT_DATA_SQID:
177 hwc->txq->gdma_wq->id = val;
178 break;
179
180 case HWC_INIT_DATA_QUEUE_DEPTH:
181 hwc->hwc_init_q_depth_max = (uint16_t)val;
182 break;
183
184 case HWC_INIT_DATA_MAX_REQUEST:
185 hwc->hwc_init_max_req_msg_size = val;
186 break;
187
188 case HWC_INIT_DATA_MAX_RESPONSE:
189 hwc->hwc_init_max_resp_msg_size = val;
190 break;
191
192 case HWC_INIT_DATA_MAX_NUM_CQS:
193 gd->gdma_context->max_num_cqs = val;
194 break;
195
196 case HWC_INIT_DATA_PDID:
197 hwc->gdma_dev->pdid = val;
198 break;
199
200 case HWC_INIT_DATA_GPA_MKEY:
201 hwc->rxq->msg_buf->gpa_mkey = val;
202 hwc->txq->msg_buf->gpa_mkey = val;
203 break;
204 }
205
206 break;
207
208 case GDMA_EQE_HWC_INIT_DONE:
209 complete(&hwc->hwc_init_eqe_comp);
210 break;
211
212 default:
213 /* Ignore unknown events, which should never happen. */
214 break;
215 }
216 }
217
218 static void
mana_hwc_rx_event_handler(void * ctx,uint32_t gdma_rxq_id,const struct hwc_rx_oob * rx_oob)219 mana_hwc_rx_event_handler(void *ctx, uint32_t gdma_rxq_id,
220 const struct hwc_rx_oob *rx_oob)
221 {
222 struct hw_channel_context *hwc = ctx;
223 struct hwc_wq *hwc_rxq = hwc->rxq;
224 struct hwc_work_request *rx_req;
225 struct gdma_resp_hdr *resp;
226 struct gdma_wqe *dma_oob;
227 struct gdma_queue *rq;
228 struct gdma_sge *sge;
229 uint64_t rq_base_addr;
230 uint64_t rx_req_idx;
231 uint8_t *wqe;
232
233 if (hwc_rxq->gdma_wq->id != gdma_rxq_id) {
234 mana_warn(NULL, "unmatched rx queue %u != %u\n",
235 hwc_rxq->gdma_wq->id, gdma_rxq_id);
236 return;
237 }
238
239
240 rq = hwc_rxq->gdma_wq;
241 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE);
242 dma_oob = (struct gdma_wqe *)wqe;
243
244 bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map,
245 BUS_DMASYNC_POSTREAD);
246
247 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4);
248
249 /* Select the RX work request for virtual address and for reposting. */
250 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
251 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;
252
253 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag,
254 hwc_rxq->msg_buf->mem_info.dma_map,
255 BUS_DMASYNC_POSTREAD);
256
257 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
258 resp = (struct gdma_resp_hdr *)rx_req->buf_va;
259
260 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {
261 device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n",
262 resp->response.hwc_msg_id);
263 return;
264 }
265
266 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp);
267
268 /* Do no longer use 'resp', because the buffer is posted to the HW
269 * in the below mana_hwc_post_rx_wqe().
270 */
271 resp = NULL;
272
273 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag,
274 hwc_rxq->msg_buf->mem_info.dma_map,
275 BUS_DMASYNC_PREREAD);
276
277 mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
278 }
279
280 static void
mana_hwc_tx_event_handler(void * ctx,uint32_t gdma_txq_id,const struct hwc_rx_oob * rx_oob)281 mana_hwc_tx_event_handler(void *ctx, uint32_t gdma_txq_id,
282 const struct hwc_rx_oob *rx_oob)
283 {
284 struct hw_channel_context *hwc = ctx;
285 struct hwc_wq *hwc_txq = hwc->txq;
286
287 if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) {
288 mana_warn(NULL, "unmatched tx queue %u != %u\n",
289 hwc_txq->gdma_wq->id, gdma_txq_id);
290 }
291
292 bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag,
293 hwc_txq->gdma_wq->mem_info.dma_map,
294 BUS_DMASYNC_POSTWRITE);
295 }
296
297 static int
mana_hwc_create_gdma_wq(struct hw_channel_context * hwc,enum gdma_queue_type type,uint64_t queue_size,struct gdma_queue ** queue)298 mana_hwc_create_gdma_wq(struct hw_channel_context *hwc,
299 enum gdma_queue_type type, uint64_t queue_size,
300 struct gdma_queue **queue)
301 {
302 struct gdma_queue_spec spec = {};
303
304 if (type != GDMA_SQ && type != GDMA_RQ)
305 return EINVAL;
306
307 spec.type = type;
308 spec.monitor_avl_buf = false;
309 spec.queue_size = queue_size;
310
311 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
312 }
313
314 static int
mana_hwc_create_gdma_cq(struct hw_channel_context * hwc,uint64_t queue_size,void * ctx,gdma_cq_callback * cb,struct gdma_queue * parent_eq,struct gdma_queue ** queue)315 mana_hwc_create_gdma_cq(struct hw_channel_context *hwc,
316 uint64_t queue_size,
317 void *ctx, gdma_cq_callback *cb,
318 struct gdma_queue *parent_eq,
319 struct gdma_queue **queue)
320 {
321 struct gdma_queue_spec spec = {};
322
323 spec.type = GDMA_CQ;
324 spec.monitor_avl_buf = false;
325 spec.queue_size = queue_size;
326 spec.cq.context = ctx;
327 spec.cq.callback = cb;
328 spec.cq.parent_eq = parent_eq;
329
330 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
331 }
332
333 static int
mana_hwc_create_gdma_eq(struct hw_channel_context * hwc,uint64_t queue_size,void * ctx,gdma_eq_callback * cb,struct gdma_queue ** queue)334 mana_hwc_create_gdma_eq(struct hw_channel_context *hwc,
335 uint64_t queue_size,
336 void *ctx, gdma_eq_callback *cb,
337 struct gdma_queue **queue)
338 {
339 struct gdma_queue_spec spec = {};
340
341 spec.type = GDMA_EQ;
342 spec.monitor_avl_buf = false;
343 spec.queue_size = queue_size;
344 spec.eq.context = ctx;
345 spec.eq.callback = cb;
346 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ;
347
348 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
349 }
350
351 static void
mana_hwc_comp_event(void * ctx,struct gdma_queue * q_self)352 mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self)
353 {
354 struct hwc_rx_oob comp_data = {};
355 struct gdma_comp *completions;
356 struct hwc_cq *hwc_cq = ctx;
357 int comp_read, i;
358
359 completions = hwc_cq->comp_buf;
360 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth);
361
362 for (i = 0; i < comp_read; ++i) {
363 comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data;
364
365 if (completions[i].is_sq)
366 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx,
367 completions[i].wq_num,
368 &comp_data);
369 else
370 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx,
371 completions[i].wq_num,
372 &comp_data);
373 }
374
375 bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map,
376 BUS_DMASYNC_POSTREAD);
377
378 mana_gd_ring_cq(q_self, SET_ARM_BIT);
379 }
380
381 static void
mana_hwc_destroy_cq(struct gdma_context * gc,struct hwc_cq * hwc_cq)382 mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq)
383 {
384 if (hwc_cq->comp_buf)
385 free(hwc_cq->comp_buf, M_DEVBUF);
386
387 if (hwc_cq->gdma_cq)
388 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq);
389
390 if (hwc_cq->gdma_eq)
391 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq);
392
393 free(hwc_cq, M_DEVBUF);
394 }
395
396 static int
mana_hwc_create_cq(struct hw_channel_context * hwc,uint16_t q_depth,gdma_eq_callback * callback,void * ctx,hwc_rx_event_handler_t * rx_ev_hdlr,void * rx_ev_ctx,hwc_tx_event_handler_t * tx_ev_hdlr,void * tx_ev_ctx,struct hwc_cq ** hwc_cq_ptr)397 mana_hwc_create_cq(struct hw_channel_context *hwc,
398 uint16_t q_depth,
399 gdma_eq_callback *callback, void *ctx,
400 hwc_rx_event_handler_t *rx_ev_hdlr, void *rx_ev_ctx,
401 hwc_tx_event_handler_t *tx_ev_hdlr, void *tx_ev_ctx,
402 struct hwc_cq **hwc_cq_ptr)
403 {
404 struct gdma_queue *eq, *cq;
405 struct gdma_comp *comp_buf;
406 struct hwc_cq *hwc_cq;
407 uint32_t eq_size, cq_size;
408 int err;
409
410 eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth);
411 if (eq_size < MINIMUM_SUPPORTED_PAGE_SIZE)
412 eq_size = MINIMUM_SUPPORTED_PAGE_SIZE;
413
414 cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth);
415 if (cq_size < MINIMUM_SUPPORTED_PAGE_SIZE)
416 cq_size = MINIMUM_SUPPORTED_PAGE_SIZE;
417
418 hwc_cq = malloc(sizeof(*hwc_cq), M_DEVBUF, M_WAITOK | M_ZERO);
419
420 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq);
421 if (err) {
422 device_printf(hwc->dev,
423 "Failed to create HWC EQ for RQ: %d\n", err);
424 goto out;
425 }
426 hwc_cq->gdma_eq = eq;
427
428 err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq,
429 mana_hwc_comp_event, eq, &cq);
430 if (err) {
431 device_printf(hwc->dev,
432 "Failed to create HWC CQ for RQ: %d\n", err);
433 goto out;
434 }
435 hwc_cq->gdma_cq = cq;
436
437 comp_buf = mallocarray(q_depth, sizeof(struct gdma_comp),
438 M_DEVBUF, M_WAITOK | M_ZERO);
439
440 hwc_cq->hwc = hwc;
441 hwc_cq->comp_buf = comp_buf;
442 hwc_cq->queue_depth = q_depth;
443 hwc_cq->rx_event_handler = rx_ev_hdlr;
444 hwc_cq->rx_event_ctx = rx_ev_ctx;
445 hwc_cq->tx_event_handler = tx_ev_hdlr;
446 hwc_cq->tx_event_ctx = tx_ev_ctx;
447
448 *hwc_cq_ptr = hwc_cq;
449 return 0;
450 out:
451 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq);
452 return err;
453 }
454
455 static int
mana_hwc_alloc_dma_buf(struct hw_channel_context * hwc,uint16_t q_depth,uint32_t max_msg_size,struct hwc_dma_buf ** dma_buf_ptr)456 mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth,
457 uint32_t max_msg_size,
458 struct hwc_dma_buf **dma_buf_ptr)
459 {
460 struct gdma_context *gc = hwc->gdma_dev->gdma_context;
461 struct hwc_work_request *hwc_wr;
462 struct hwc_dma_buf *dma_buf;
463 struct gdma_mem_info *gmi;
464 uint32_t buf_size;
465 uint8_t *base_pa;
466 void *virt_addr;
467 uint16_t i;
468 int err;
469
470 dma_buf = malloc(sizeof(*dma_buf) +
471 q_depth * sizeof(struct hwc_work_request),
472 M_DEVBUF, M_WAITOK | M_ZERO);
473
474 dma_buf->num_reqs = q_depth;
475
476 buf_size = ALIGN(q_depth * max_msg_size, PAGE_SIZE);
477
478 gmi = &dma_buf->mem_info;
479 err = mana_gd_alloc_memory(gc, buf_size, gmi);
480 if (err) {
481 device_printf(hwc->dev,
482 "Failed to allocate DMA buffer: %d\n", err);
483 goto out;
484 }
485
486 virt_addr = dma_buf->mem_info.virt_addr;
487 base_pa = (uint8_t *)dma_buf->mem_info.dma_handle;
488
489 for (i = 0; i < q_depth; i++) {
490 hwc_wr = &dma_buf->reqs[i];
491
492 hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size;
493 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size;
494
495 hwc_wr->buf_len = max_msg_size;
496 }
497
498 *dma_buf_ptr = dma_buf;
499 return 0;
500 out:
501 free(dma_buf, M_DEVBUF);
502 return err;
503 }
504
505 static void
mana_hwc_dealloc_dma_buf(struct hw_channel_context * hwc,struct hwc_dma_buf * dma_buf)506 mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc,
507 struct hwc_dma_buf *dma_buf)
508 {
509 if (!dma_buf)
510 return;
511
512 mana_gd_free_memory(&dma_buf->mem_info);
513
514 free(dma_buf, M_DEVBUF);
515 }
516
517 static void
mana_hwc_destroy_wq(struct hw_channel_context * hwc,struct hwc_wq * hwc_wq)518 mana_hwc_destroy_wq(struct hw_channel_context *hwc,
519 struct hwc_wq *hwc_wq)
520 {
521 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf);
522
523 if (hwc_wq->gdma_wq)
524 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context,
525 hwc_wq->gdma_wq);
526
527 free(hwc_wq, M_DEVBUF);
528 }
529
530 static int
mana_hwc_create_wq(struct hw_channel_context * hwc,enum gdma_queue_type q_type,uint16_t q_depth,uint32_t max_msg_size,struct hwc_cq * hwc_cq,struct hwc_wq ** hwc_wq_ptr)531 mana_hwc_create_wq(struct hw_channel_context *hwc,
532 enum gdma_queue_type q_type, uint16_t q_depth,
533 uint32_t max_msg_size, struct hwc_cq *hwc_cq,
534 struct hwc_wq **hwc_wq_ptr)
535 {
536 struct gdma_queue *queue;
537 struct hwc_wq *hwc_wq;
538 uint32_t queue_size;
539 int err;
540
541 if (q_type != GDMA_SQ && q_type != GDMA_RQ) {
542 /* XXX should fail and return error? */
543 mana_warn(NULL, "Invalid q_type %u\n", q_type);
544 }
545
546 if (q_type == GDMA_RQ)
547 queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth);
548 else
549 queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth);
550
551 if (queue_size < MINIMUM_SUPPORTED_PAGE_SIZE)
552 queue_size = MINIMUM_SUPPORTED_PAGE_SIZE;
553
554 hwc_wq = malloc(sizeof(*hwc_wq), M_DEVBUF, M_WAITOK | M_ZERO);
555
556 err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue);
557 if (err)
558 goto out;
559
560 hwc_wq->hwc = hwc;
561 hwc_wq->gdma_wq = queue;
562 hwc_wq->queue_depth = q_depth;
563 hwc_wq->hwc_cq = hwc_cq;
564
565 err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,
566 &hwc_wq->msg_buf);
567 if (err)
568 goto out;
569
570 *hwc_wq_ptr = hwc_wq;
571 return 0;
572 out:
573 if (err)
574 mana_hwc_destroy_wq(hwc, hwc_wq);
575 return err;
576 }
577
578 static int
mana_hwc_post_tx_wqe(const struct hwc_wq * hwc_txq,struct hwc_work_request * req,uint32_t dest_virt_rq_id,uint32_t dest_virt_rcq_id,bool dest_pf)579 mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,
580 struct hwc_work_request *req,
581 uint32_t dest_virt_rq_id, uint32_t dest_virt_rcq_id,
582 bool dest_pf)
583 {
584 device_t dev = hwc_txq->hwc->dev;
585 struct hwc_tx_oob *tx_oob;
586 struct gdma_sge *sge;
587 int err;
588
589 if (req->msg_size == 0 || req->msg_size > req->buf_len) {
590 device_printf(dev, "wrong msg_size: %u, buf_len: %u\n",
591 req->msg_size, req->buf_len);
592 return EINVAL;
593 }
594
595 tx_oob = &req->tx_oob;
596
597 tx_oob->vrq_id = dest_virt_rq_id;
598 tx_oob->dest_vfid = 0;
599 tx_oob->vrcq_id = dest_virt_rcq_id;
600 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id;
601 tx_oob->loopback = false;
602 tx_oob->lso_override = false;
603 tx_oob->dest_pf = dest_pf;
604 tx_oob->vsq_id = hwc_txq->gdma_wq->id;
605
606 sge = &req->sge;
607 sge->address = (uintptr_t)req->buf_sge_addr;
608 sge->mem_key = hwc_txq->msg_buf->gpa_mkey;
609 sge->size = req->msg_size;
610
611 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
612 req->wqe_req.sgl = sge;
613 req->wqe_req.num_sge = 1;
614 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob);
615 req->wqe_req.inline_oob_data = tx_oob;
616 req->wqe_req.client_data_unit = 0;
617
618 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL);
619 if (err)
620 device_printf(dev,
621 "Failed to post WQE on HWC SQ: %d\n", err);
622 return err;
623 }
624
625 static int
mana_hwc_init_inflight_msg(struct hw_channel_context * hwc,uint16_t num_msg)626 mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg)
627 {
628 int err;
629
630 sema_init(&hwc->sema, num_msg, "gdma hwc sema");
631
632 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res,
633 "gdma hwc res lock");
634 if (err)
635 device_printf(hwc->dev,
636 "Failed to init inflight_msg_res: %d\n", err);
637
638 return (err);
639 }
640
641 static int
mana_hwc_test_channel(struct hw_channel_context * hwc,uint16_t q_depth,uint32_t max_req_msg_size,uint32_t max_resp_msg_size)642 mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth,
643 uint32_t max_req_msg_size, uint32_t max_resp_msg_size)
644 {
645 struct gdma_context *gc = hwc->gdma_dev->gdma_context;
646 struct hwc_wq *hwc_rxq = hwc->rxq;
647 struct hwc_work_request *req;
648 struct hwc_caller_ctx *ctx;
649 int err;
650 int i;
651
652 /* Post all WQEs on the RQ */
653 for (i = 0; i < q_depth; i++) {
654 req = &hwc_rxq->msg_buf->reqs[i];
655 err = mana_hwc_post_rx_wqe(hwc_rxq, req);
656 if (err)
657 return err;
658 }
659
660 ctx = malloc(q_depth * sizeof(struct hwc_caller_ctx),
661 M_DEVBUF, M_WAITOK | M_ZERO);
662
663 for (i = 0; i < q_depth; ++i)
664 init_completion(&ctx[i].comp_event);
665
666 hwc->caller_ctx = ctx;
667
668 return mana_gd_test_eq(gc, hwc->cq->gdma_eq);
669 }
670
671 static int
mana_hwc_establish_channel(struct gdma_context * gc,uint16_t * q_depth,uint32_t * max_req_msg_size,uint32_t * max_resp_msg_size)672 mana_hwc_establish_channel(struct gdma_context *gc, uint16_t *q_depth,
673 uint32_t *max_req_msg_size,
674 uint32_t *max_resp_msg_size)
675 {
676 struct hw_channel_context *hwc = gc->hwc.driver_data;
677 struct gdma_queue *rq = hwc->rxq->gdma_wq;
678 struct gdma_queue *sq = hwc->txq->gdma_wq;
679 struct gdma_queue *eq = hwc->cq->gdma_eq;
680 struct gdma_queue *cq = hwc->cq->gdma_cq;
681 int err;
682
683 init_completion(&hwc->hwc_init_eqe_comp);
684
685 err = mana_smc_setup_hwc(&gc->shm_channel, false,
686 eq->mem_info.dma_handle,
687 cq->mem_info.dma_handle,
688 rq->mem_info.dma_handle,
689 sq->mem_info.dma_handle,
690 eq->eq.msix_index);
691 if (err)
692 return err;
693
694 if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz))
695 return ETIMEDOUT;
696
697 *q_depth = hwc->hwc_init_q_depth_max;
698 *max_req_msg_size = hwc->hwc_init_max_req_msg_size;
699 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size;
700
701 /* Both were set in mana_hwc_init_event_handler(). */
702 if (cq->id >= gc->max_num_cqs) {
703 mana_warn(NULL, "invalid cq id %u > %u\n",
704 cq->id, gc->max_num_cqs);
705 return EPROTO;
706 }
707
708 gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *),
709 M_DEVBUF, M_WAITOK | M_ZERO);
710 gc->cq_table[cq->id] = cq;
711
712 return 0;
713 }
714
715 static int
mana_hwc_init_queues(struct hw_channel_context * hwc,uint16_t q_depth,uint32_t max_req_msg_size,uint32_t max_resp_msg_size)716 mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth,
717 uint32_t max_req_msg_size, uint32_t max_resp_msg_size)
718 {
719 int err;
720
721 err = mana_hwc_init_inflight_msg(hwc, q_depth);
722 if (err)
723 return err;
724
725 /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ
726 * queue depth and RQ queue depth.
727 */
728 err = mana_hwc_create_cq(hwc, q_depth * 2,
729 mana_hwc_init_event_handler, hwc,
730 mana_hwc_rx_event_handler, hwc,
731 mana_hwc_tx_event_handler, hwc, &hwc->cq);
732 if (err) {
733 device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err);
734 goto out;
735 }
736
737 err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,
738 hwc->cq, &hwc->rxq);
739 if (err) {
740 device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err);
741 goto out;
742 }
743
744 err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,
745 hwc->cq, &hwc->txq);
746 if (err) {
747 device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err);
748 goto out;
749 }
750
751 hwc->num_inflight_msg = q_depth;
752 hwc->max_req_msg_size = max_req_msg_size;
753
754 return 0;
755 out:
756 /* mana_hwc_create_channel() will do the cleanup.*/
757 return err;
758 }
759
760 int
mana_hwc_create_channel(struct gdma_context * gc)761 mana_hwc_create_channel(struct gdma_context *gc)
762 {
763 uint32_t max_req_msg_size, max_resp_msg_size;
764 struct gdma_dev *gd = &gc->hwc;
765 struct hw_channel_context *hwc;
766 uint16_t q_depth_max;
767 int err;
768
769 hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO);
770
771 gd->gdma_context = gc;
772 gd->driver_data = hwc;
773 hwc->gdma_dev = gd;
774 hwc->dev = gc->dev;
775
776 /* HWC's instance number is always 0. */
777 gd->dev_id.as_uint32 = 0;
778 gd->dev_id.type = GDMA_DEVICE_HWC;
779
780 gd->pdid = INVALID_PDID;
781 gd->doorbell = INVALID_DOORBELL;
782
783 /*
784 * mana_hwc_init_queues() only creates the required data structures,
785 * and doesn't touch the HWC device.
786 */
787 err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
788 HW_CHANNEL_MAX_REQUEST_SIZE,
789 HW_CHANNEL_MAX_RESPONSE_SIZE);
790 if (err) {
791 device_printf(hwc->dev, "Failed to initialize HWC: %d\n",
792 err);
793 goto out;
794 }
795
796 err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size,
797 &max_resp_msg_size);
798 if (err) {
799 device_printf(hwc->dev, "Failed to establish HWC: %d\n", err);
800 goto out;
801 }
802
803 err = mana_hwc_test_channel(gc->hwc.driver_data,
804 HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
805 max_req_msg_size, max_resp_msg_size);
806 if (err) {
807 /* Test failed, but the channel has been established */
808 device_printf(hwc->dev, "Failed to test HWC: %d\n", err);
809 return EIO;
810 }
811
812 return 0;
813 out:
814 mana_hwc_destroy_channel(gc);
815 return (err);
816 }
817
818 void
mana_hwc_destroy_channel(struct gdma_context * gc)819 mana_hwc_destroy_channel(struct gdma_context *gc)
820 {
821 struct hw_channel_context *hwc = gc->hwc.driver_data;
822
823 if (!hwc)
824 return;
825
826 /*
827 * gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's
828 * non-zero, the HWC worked and we should tear down the HWC here.
829 */
830 if (gc->max_num_cqs > 0) {
831 mana_smc_teardown_hwc(&gc->shm_channel, false);
832 gc->max_num_cqs = 0;
833 }
834
835 free(hwc->caller_ctx, M_DEVBUF);
836 hwc->caller_ctx = NULL;
837
838 if (hwc->txq)
839 mana_hwc_destroy_wq(hwc, hwc->txq);
840
841 if (hwc->rxq)
842 mana_hwc_destroy_wq(hwc, hwc->rxq);
843
844 if (hwc->cq)
845 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
846
847 mana_gd_free_res_map(&hwc->inflight_msg_res);
848
849 hwc->num_inflight_msg = 0;
850
851 hwc->gdma_dev->doorbell = INVALID_DOORBELL;
852 hwc->gdma_dev->pdid = INVALID_PDID;
853
854 free(hwc, M_DEVBUF);
855 gc->hwc.driver_data = NULL;
856 gc->hwc.gdma_context = NULL;
857
858 free(gc->cq_table, M_DEVBUF);
859 gc->cq_table = NULL;
860 }
861
862 int
mana_hwc_send_request(struct hw_channel_context * hwc,uint32_t req_len,const void * req,uint32_t resp_len,void * resp)863 mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len,
864 const void *req, uint32_t resp_len, void *resp)
865 {
866 struct hwc_work_request *tx_wr;
867 struct hwc_wq *txq = hwc->txq;
868 struct gdma_req_hdr *req_msg;
869 struct hwc_caller_ctx *ctx;
870 uint16_t msg_id;
871 int err;
872
873 mana_hwc_get_msg_index(hwc, &msg_id);
874
875 tx_wr = &txq->msg_buf->reqs[msg_id];
876
877 if (req_len > tx_wr->buf_len) {
878 device_printf(hwc->dev,
879 "HWC: req msg size: %d > %d\n", req_len,
880 tx_wr->buf_len);
881 err = EINVAL;
882 goto out;
883 }
884
885 ctx = hwc->caller_ctx + msg_id;
886 ctx->output_buf = resp;
887 ctx->output_buflen = resp_len;
888
889 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va;
890 if (req)
891 memcpy(req_msg, req, req_len);
892
893 req_msg->req.hwc_msg_id = msg_id;
894
895 tx_wr->msg_size = req_len;
896
897 err = mana_hwc_post_tx_wqe(txq, tx_wr, 0, 0, false);
898 if (err) {
899 device_printf(hwc->dev,
900 "HWC: Failed to post send WQE: %d\n", err);
901 goto out;
902 }
903
904 if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) {
905 device_printf(hwc->dev, "HWC: Request timed out!\n");
906 err = ETIMEDOUT;
907 goto out;
908 }
909
910 if (ctx->error) {
911 err = ctx->error;
912 goto out;
913 }
914
915 if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) {
916 device_printf(hwc->dev,
917 "HWC: Failed hw_channel req: 0x%x\n", ctx->status_code);
918 err = EPROTO;
919 goto out;
920 }
921 out:
922 mana_hwc_put_msg_index(hwc, msg_id);
923 return err;
924 }
925