1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Microsoft Corp. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/types.h> 34 #include <sys/kernel.h> 35 #include <sys/kthread.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/bus.h> 40 #include <machine/bus.h> 41 42 #include "mana.h" 43 #include "hw_channel.h" 44 45 static int 46 mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id) 47 { 48 struct gdma_resource *r = &hwc->inflight_msg_res; 49 uint32_t index; 50 51 sema_wait(&hwc->sema); 52 53 mtx_lock_spin(&r->lock_spin); 54 55 index = find_first_zero_bit(hwc->inflight_msg_res.map, 56 hwc->inflight_msg_res.size); 57 58 bitmap_set(hwc->inflight_msg_res.map, index, 1); 59 60 mtx_unlock_spin(&r->lock_spin); 61 62 *msg_id = index; 63 64 return 0; 65 } 66 67 static void 68 mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id) 69 { 70 struct gdma_resource *r = &hwc->inflight_msg_res; 71 72 mtx_lock_spin(&r->lock_spin); 73 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); 74 mtx_unlock_spin(&r->lock_spin); 75 76 sema_post(&hwc->sema); 77 } 78 79 static int 80 mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, 81 const struct gdma_resp_hdr *resp_msg, 82 uint32_t resp_len) 83 { 84 if (resp_len < sizeof(*resp_msg)) 85 return EPROTO; 86 87 if (resp_len > caller_ctx->output_buflen) 88 return EPROTO; 89 90 return 0; 91 } 92 93 static void 94 mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len, 95 const struct gdma_resp_hdr *resp_msg) 96 { 97 struct hwc_caller_ctx *ctx; 98 int err; 99 100 if (!test_bit(resp_msg->response.hwc_msg_id, 101 hwc->inflight_msg_res.map)) { 102 device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n", 103 resp_msg->response.hwc_msg_id); 104 return; 105 } 106 107 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; 108 err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); 109 if (err) 110 goto out; 111 112 ctx->status_code = resp_msg->status; 113 114 memcpy(ctx->output_buf, resp_msg, resp_len); 115 out: 116 ctx->error = err; 117 complete(&ctx->comp_event); 118 } 119 120 static int 121 mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq, 122 struct hwc_work_request *req) 123 { 124 device_t dev = hwc_rxq->hwc->dev; 125 struct gdma_sge *sge; 126 int err; 127 128 sge = &req->sge; 129 sge->address = (uintptr_t)req->buf_sge_addr; 130 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; 131 sge->size = req->buf_len; 132 133 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 134 req->wqe_req.sgl = sge; 135 req->wqe_req.num_sge = 1; 136 req->wqe_req.client_data_unit = 0; 137 138 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); 139 if (err) 140 device_printf(dev, 141 "Failed to post WQE on HWC RQ: %d\n", err); 142 return err; 143 } 144 145 static void 146 mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, 147 struct gdma_event *event) 148 { 149 struct hw_channel_context *hwc = ctx; 150 struct gdma_dev *gd = hwc->gdma_dev; 151 union hwc_init_type_data type_data; 152 union hwc_init_eq_id_db eq_db; 153 uint32_t type, val; 154 155 switch (event->type) { 156 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 157 eq_db.as_uint32 = event->details[0]; 158 hwc->cq->gdma_eq->id = eq_db.eq_id; 159 gd->doorbell = eq_db.doorbell; 160 break; 161 162 case GDMA_EQE_HWC_INIT_DATA: 163 type_data.as_uint32 = event->details[0]; 164 type = type_data.type; 165 val = type_data.value; 166 167 switch (type) { 168 case HWC_INIT_DATA_CQID: 169 hwc->cq->gdma_cq->id = val; 170 break; 171 172 case HWC_INIT_DATA_RQID: 173 hwc->rxq->gdma_wq->id = val; 174 break; 175 176 case HWC_INIT_DATA_SQID: 177 hwc->txq->gdma_wq->id = val; 178 break; 179 180 case HWC_INIT_DATA_QUEUE_DEPTH: 181 hwc->hwc_init_q_depth_max = (uint16_t)val; 182 break; 183 184 case HWC_INIT_DATA_MAX_REQUEST: 185 hwc->hwc_init_max_req_msg_size = val; 186 break; 187 188 case HWC_INIT_DATA_MAX_RESPONSE: 189 hwc->hwc_init_max_resp_msg_size = val; 190 break; 191 192 case HWC_INIT_DATA_MAX_NUM_CQS: 193 gd->gdma_context->max_num_cqs = val; 194 break; 195 196 case HWC_INIT_DATA_PDID: 197 hwc->gdma_dev->pdid = val; 198 break; 199 200 case HWC_INIT_DATA_GPA_MKEY: 201 hwc->rxq->msg_buf->gpa_mkey = val; 202 hwc->txq->msg_buf->gpa_mkey = val; 203 break; 204 } 205 206 break; 207 208 case GDMA_EQE_HWC_INIT_DONE: 209 complete(&hwc->hwc_init_eqe_comp); 210 break; 211 212 default: 213 /* Ignore unknown events, which should never happen. */ 214 break; 215 } 216 } 217 218 static void 219 mana_hwc_rx_event_handler(void *ctx, uint32_t gdma_rxq_id, 220 const struct hwc_rx_oob *rx_oob) 221 { 222 struct hw_channel_context *hwc = ctx; 223 struct hwc_wq *hwc_rxq = hwc->rxq; 224 struct hwc_work_request *rx_req; 225 struct gdma_resp_hdr *resp; 226 struct gdma_wqe *dma_oob; 227 struct gdma_queue *rq; 228 struct gdma_sge *sge; 229 uint64_t rq_base_addr; 230 uint64_t rx_req_idx; 231 uint8_t *wqe; 232 233 if (hwc_rxq->gdma_wq->id != gdma_rxq_id) { 234 mana_warn(NULL, "unmatched rx queue %u != %u\n", 235 hwc_rxq->gdma_wq->id, gdma_rxq_id); 236 return; 237 } 238 239 240 rq = hwc_rxq->gdma_wq; 241 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); 242 dma_oob = (struct gdma_wqe *)wqe; 243 244 bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map, 245 BUS_DMASYNC_POSTREAD); 246 247 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); 248 249 /* Select the RX work request for virtual address and for reposting. */ 250 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; 251 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; 252 253 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 254 hwc_rxq->msg_buf->mem_info.dma_map, 255 BUS_DMASYNC_POSTREAD); 256 257 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; 258 resp = (struct gdma_resp_hdr *)rx_req->buf_va; 259 260 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { 261 device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n", 262 resp->response.hwc_msg_id); 263 return; 264 } 265 266 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp); 267 268 /* Do no longer use 'resp', because the buffer is posted to the HW 269 * in the below mana_hwc_post_rx_wqe(). 270 */ 271 resp = NULL; 272 273 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 274 hwc_rxq->msg_buf->mem_info.dma_map, 275 BUS_DMASYNC_PREREAD); 276 277 mana_hwc_post_rx_wqe(hwc_rxq, rx_req); 278 } 279 280 static void 281 mana_hwc_tx_event_handler(void *ctx, uint32_t gdma_txq_id, 282 const struct hwc_rx_oob *rx_oob) 283 { 284 struct hw_channel_context *hwc = ctx; 285 struct hwc_wq *hwc_txq = hwc->txq; 286 287 if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) { 288 mana_warn(NULL, "unmatched tx queue %u != %u\n", 289 hwc_txq->gdma_wq->id, gdma_txq_id); 290 } 291 292 bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag, 293 hwc_txq->gdma_wq->mem_info.dma_map, 294 BUS_DMASYNC_POSTWRITE); 295 } 296 297 static int 298 mana_hwc_create_gdma_wq(struct hw_channel_context *hwc, 299 enum gdma_queue_type type, uint64_t queue_size, 300 struct gdma_queue **queue) 301 { 302 struct gdma_queue_spec spec = {}; 303 304 if (type != GDMA_SQ && type != GDMA_RQ) 305 return EINVAL; 306 307 spec.type = type; 308 spec.monitor_avl_buf = false; 309 spec.queue_size = queue_size; 310 311 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 312 } 313 314 static int 315 mana_hwc_create_gdma_cq(struct hw_channel_context *hwc, 316 uint64_t queue_size, 317 void *ctx, gdma_cq_callback *cb, 318 struct gdma_queue *parent_eq, 319 struct gdma_queue **queue) 320 { 321 struct gdma_queue_spec spec = {}; 322 323 spec.type = GDMA_CQ; 324 spec.monitor_avl_buf = false; 325 spec.queue_size = queue_size; 326 spec.cq.context = ctx; 327 spec.cq.callback = cb; 328 spec.cq.parent_eq = parent_eq; 329 330 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 331 } 332 333 static int 334 mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, 335 uint64_t queue_size, 336 void *ctx, gdma_eq_callback *cb, 337 struct gdma_queue **queue) 338 { 339 struct gdma_queue_spec spec = {}; 340 341 spec.type = GDMA_EQ; 342 spec.monitor_avl_buf = false; 343 spec.queue_size = queue_size; 344 spec.eq.context = ctx; 345 spec.eq.callback = cb; 346 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; 347 348 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 349 } 350 351 static void 352 mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self) 353 { 354 struct hwc_rx_oob comp_data = {}; 355 struct gdma_comp *completions; 356 struct hwc_cq *hwc_cq = ctx; 357 int comp_read, i; 358 359 completions = hwc_cq->comp_buf; 360 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); 361 362 for (i = 0; i < comp_read; ++i) { 363 comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data; 364 365 if (completions[i].is_sq) 366 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, 367 completions[i].wq_num, 368 &comp_data); 369 else 370 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, 371 completions[i].wq_num, 372 &comp_data); 373 } 374 375 bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map, 376 BUS_DMASYNC_POSTREAD); 377 378 mana_gd_ring_cq(q_self, SET_ARM_BIT); 379 } 380 381 static void 382 mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq) 383 { 384 if (hwc_cq->comp_buf) 385 free(hwc_cq->comp_buf, M_DEVBUF); 386 387 if (hwc_cq->gdma_cq) 388 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); 389 390 if (hwc_cq->gdma_eq) 391 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); 392 393 free(hwc_cq, M_DEVBUF); 394 } 395 396 static int 397 mana_hwc_create_cq(struct hw_channel_context *hwc, 398 uint16_t q_depth, 399 gdma_eq_callback *callback, void *ctx, 400 hwc_rx_event_handler_t *rx_ev_hdlr, void *rx_ev_ctx, 401 hwc_tx_event_handler_t *tx_ev_hdlr, void *tx_ev_ctx, 402 struct hwc_cq **hwc_cq_ptr) 403 { 404 struct gdma_queue *eq, *cq; 405 struct gdma_comp *comp_buf; 406 struct hwc_cq *hwc_cq; 407 uint32_t eq_size, cq_size; 408 int err; 409 410 eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth); 411 if (eq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 412 eq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 413 414 cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth); 415 if (cq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 416 cq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 417 418 hwc_cq = malloc(sizeof(*hwc_cq), M_DEVBUF, M_WAITOK | M_ZERO); 419 if (!hwc_cq) 420 return ENOMEM; 421 422 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); 423 if (err) { 424 device_printf(hwc->dev, 425 "Failed to create HWC EQ for RQ: %d\n", err); 426 goto out; 427 } 428 hwc_cq->gdma_eq = eq; 429 430 err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, 431 mana_hwc_comp_event, eq, &cq); 432 if (err) { 433 device_printf(hwc->dev, 434 "Failed to create HWC CQ for RQ: %d\n", err); 435 goto out; 436 } 437 hwc_cq->gdma_cq = cq; 438 439 comp_buf = mallocarray(q_depth, sizeof(struct gdma_comp), 440 M_DEVBUF, M_WAITOK | M_ZERO); 441 if (!comp_buf) { 442 err = ENOMEM; 443 goto out; 444 } 445 446 hwc_cq->hwc = hwc; 447 hwc_cq->comp_buf = comp_buf; 448 hwc_cq->queue_depth = q_depth; 449 hwc_cq->rx_event_handler = rx_ev_hdlr; 450 hwc_cq->rx_event_ctx = rx_ev_ctx; 451 hwc_cq->tx_event_handler = tx_ev_hdlr; 452 hwc_cq->tx_event_ctx = tx_ev_ctx; 453 454 *hwc_cq_ptr = hwc_cq; 455 return 0; 456 out: 457 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); 458 return err; 459 } 460 461 static int 462 mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth, 463 uint32_t max_msg_size, 464 struct hwc_dma_buf **dma_buf_ptr) 465 { 466 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 467 struct hwc_work_request *hwc_wr; 468 struct hwc_dma_buf *dma_buf; 469 struct gdma_mem_info *gmi; 470 uint32_t buf_size; 471 uint8_t *base_pa; 472 void *virt_addr; 473 uint16_t i; 474 int err; 475 476 dma_buf = malloc(sizeof(*dma_buf) + 477 q_depth * sizeof(struct hwc_work_request), 478 M_DEVBUF, M_WAITOK | M_ZERO); 479 if (!dma_buf) 480 return ENOMEM; 481 482 dma_buf->num_reqs = q_depth; 483 484 buf_size = ALIGN(q_depth * max_msg_size, PAGE_SIZE); 485 486 gmi = &dma_buf->mem_info; 487 err = mana_gd_alloc_memory(gc, buf_size, gmi); 488 if (err) { 489 device_printf(hwc->dev, 490 "Failed to allocate DMA buffer: %d\n", err); 491 goto out; 492 } 493 494 virt_addr = dma_buf->mem_info.virt_addr; 495 base_pa = (uint8_t *)dma_buf->mem_info.dma_handle; 496 497 for (i = 0; i < q_depth; i++) { 498 hwc_wr = &dma_buf->reqs[i]; 499 500 hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size; 501 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; 502 503 hwc_wr->buf_len = max_msg_size; 504 } 505 506 *dma_buf_ptr = dma_buf; 507 return 0; 508 out: 509 free(dma_buf, M_DEVBUF); 510 return err; 511 } 512 513 static void 514 mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc, 515 struct hwc_dma_buf *dma_buf) 516 { 517 if (!dma_buf) 518 return; 519 520 mana_gd_free_memory(&dma_buf->mem_info); 521 522 free(dma_buf, M_DEVBUF); 523 } 524 525 static void 526 mana_hwc_destroy_wq(struct hw_channel_context *hwc, 527 struct hwc_wq *hwc_wq) 528 { 529 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); 530 531 if (hwc_wq->gdma_wq) 532 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, 533 hwc_wq->gdma_wq); 534 535 free(hwc_wq, M_DEVBUF); 536 } 537 538 static int 539 mana_hwc_create_wq(struct hw_channel_context *hwc, 540 enum gdma_queue_type q_type, uint16_t q_depth, 541 uint32_t max_msg_size, struct hwc_cq *hwc_cq, 542 struct hwc_wq **hwc_wq_ptr) 543 { 544 struct gdma_queue *queue; 545 struct hwc_wq *hwc_wq; 546 uint32_t queue_size; 547 int err; 548 549 if (q_type != GDMA_SQ && q_type != GDMA_RQ) { 550 /* XXX should fail and return error? */ 551 mana_warn(NULL, "Invalid q_type %u\n", q_type); 552 } 553 554 if (q_type == GDMA_RQ) 555 queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth); 556 else 557 queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth); 558 559 if (queue_size < MINIMUM_SUPPORTED_PAGE_SIZE) 560 queue_size = MINIMUM_SUPPORTED_PAGE_SIZE; 561 562 hwc_wq = malloc(sizeof(*hwc_wq), M_DEVBUF, M_WAITOK | M_ZERO); 563 if (!hwc_wq) 564 return ENOMEM; 565 566 err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue); 567 if (err) 568 goto out; 569 570 hwc_wq->hwc = hwc; 571 hwc_wq->gdma_wq = queue; 572 hwc_wq->queue_depth = q_depth; 573 hwc_wq->hwc_cq = hwc_cq; 574 575 err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, 576 &hwc_wq->msg_buf); 577 if (err) 578 goto out; 579 580 *hwc_wq_ptr = hwc_wq; 581 return 0; 582 out: 583 if (err) 584 mana_hwc_destroy_wq(hwc, hwc_wq); 585 return err; 586 } 587 588 static int 589 mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq, 590 struct hwc_work_request *req, 591 uint32_t dest_virt_rq_id, uint32_t dest_virt_rcq_id, 592 bool dest_pf) 593 { 594 device_t dev = hwc_txq->hwc->dev; 595 struct hwc_tx_oob *tx_oob; 596 struct gdma_sge *sge; 597 int err; 598 599 if (req->msg_size == 0 || req->msg_size > req->buf_len) { 600 device_printf(dev, "wrong msg_size: %u, buf_len: %u\n", 601 req->msg_size, req->buf_len); 602 return EINVAL; 603 } 604 605 tx_oob = &req->tx_oob; 606 607 tx_oob->vrq_id = dest_virt_rq_id; 608 tx_oob->dest_vfid = 0; 609 tx_oob->vrcq_id = dest_virt_rcq_id; 610 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; 611 tx_oob->loopback = false; 612 tx_oob->lso_override = false; 613 tx_oob->dest_pf = dest_pf; 614 tx_oob->vsq_id = hwc_txq->gdma_wq->id; 615 616 sge = &req->sge; 617 sge->address = (uintptr_t)req->buf_sge_addr; 618 sge->mem_key = hwc_txq->msg_buf->gpa_mkey; 619 sge->size = req->msg_size; 620 621 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 622 req->wqe_req.sgl = sge; 623 req->wqe_req.num_sge = 1; 624 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); 625 req->wqe_req.inline_oob_data = tx_oob; 626 req->wqe_req.client_data_unit = 0; 627 628 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); 629 if (err) 630 device_printf(dev, 631 "Failed to post WQE on HWC SQ: %d\n", err); 632 return err; 633 } 634 635 static int 636 mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg) 637 { 638 int err; 639 640 sema_init(&hwc->sema, num_msg, "gdma hwc sema"); 641 642 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res, 643 "gdma hwc res lock"); 644 if (err) 645 device_printf(hwc->dev, 646 "Failed to init inflight_msg_res: %d\n", err); 647 648 return (err); 649 } 650 651 static int 652 mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth, 653 uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 654 { 655 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 656 struct hwc_wq *hwc_rxq = hwc->rxq; 657 struct hwc_work_request *req; 658 struct hwc_caller_ctx *ctx; 659 int err; 660 int i; 661 662 /* Post all WQEs on the RQ */ 663 for (i = 0; i < q_depth; i++) { 664 req = &hwc_rxq->msg_buf->reqs[i]; 665 err = mana_hwc_post_rx_wqe(hwc_rxq, req); 666 if (err) 667 return err; 668 } 669 670 ctx = malloc(q_depth * sizeof(struct hwc_caller_ctx), 671 M_DEVBUF, M_WAITOK | M_ZERO); 672 if (!ctx) 673 return ENOMEM; 674 675 for (i = 0; i < q_depth; ++i) 676 init_completion(&ctx[i].comp_event); 677 678 hwc->caller_ctx = ctx; 679 680 return mana_gd_test_eq(gc, hwc->cq->gdma_eq); 681 } 682 683 static int 684 mana_hwc_establish_channel(struct gdma_context *gc, uint16_t *q_depth, 685 uint32_t *max_req_msg_size, 686 uint32_t *max_resp_msg_size) 687 { 688 struct hw_channel_context *hwc = gc->hwc.driver_data; 689 struct gdma_queue *rq = hwc->rxq->gdma_wq; 690 struct gdma_queue *sq = hwc->txq->gdma_wq; 691 struct gdma_queue *eq = hwc->cq->gdma_eq; 692 struct gdma_queue *cq = hwc->cq->gdma_cq; 693 int err; 694 695 init_completion(&hwc->hwc_init_eqe_comp); 696 697 err = mana_smc_setup_hwc(&gc->shm_channel, false, 698 eq->mem_info.dma_handle, 699 cq->mem_info.dma_handle, 700 rq->mem_info.dma_handle, 701 sq->mem_info.dma_handle, 702 eq->eq.msix_index); 703 if (err) 704 return err; 705 706 if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz)) 707 return ETIMEDOUT; 708 709 *q_depth = hwc->hwc_init_q_depth_max; 710 *max_req_msg_size = hwc->hwc_init_max_req_msg_size; 711 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; 712 713 /* Both were set in mana_hwc_init_event_handler(). */ 714 if (cq->id >= gc->max_num_cqs) { 715 mana_warn(NULL, "invalid cq id %u > %u\n", 716 cq->id, gc->max_num_cqs); 717 return EPROTO; 718 } 719 720 gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *), 721 M_DEVBUF, M_WAITOK | M_ZERO); 722 if (!gc->cq_table) 723 return ENOMEM; 724 725 gc->cq_table[cq->id] = cq; 726 727 return 0; 728 } 729 730 static int 731 mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth, 732 uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 733 { 734 int err; 735 736 err = mana_hwc_init_inflight_msg(hwc, q_depth); 737 if (err) 738 return err; 739 740 /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ 741 * queue depth and RQ queue depth. 742 */ 743 err = mana_hwc_create_cq(hwc, q_depth * 2, 744 mana_hwc_init_event_handler, hwc, 745 mana_hwc_rx_event_handler, hwc, 746 mana_hwc_tx_event_handler, hwc, &hwc->cq); 747 if (err) { 748 device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err); 749 goto out; 750 } 751 752 err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, 753 hwc->cq, &hwc->rxq); 754 if (err) { 755 device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err); 756 goto out; 757 } 758 759 err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, 760 hwc->cq, &hwc->txq); 761 if (err) { 762 device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err); 763 goto out; 764 } 765 766 hwc->num_inflight_msg = q_depth; 767 hwc->max_req_msg_size = max_req_msg_size; 768 769 return 0; 770 out: 771 /* mana_hwc_create_channel() will do the cleanup.*/ 772 return err; 773 } 774 775 int 776 mana_hwc_create_channel(struct gdma_context *gc) 777 { 778 uint32_t max_req_msg_size, max_resp_msg_size; 779 struct gdma_dev *gd = &gc->hwc; 780 struct hw_channel_context *hwc; 781 uint16_t q_depth_max; 782 int err; 783 784 hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO); 785 if (!hwc) 786 return ENOMEM; 787 788 gd->gdma_context = gc; 789 gd->driver_data = hwc; 790 hwc->gdma_dev = gd; 791 hwc->dev = gc->dev; 792 793 /* HWC's instance number is always 0. */ 794 gd->dev_id.as_uint32 = 0; 795 gd->dev_id.type = GDMA_DEVICE_HWC; 796 797 gd->pdid = INVALID_PDID; 798 gd->doorbell = INVALID_DOORBELL; 799 800 /* 801 * mana_hwc_init_queues() only creates the required data structures, 802 * and doesn't touch the HWC device. 803 */ 804 err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 805 HW_CHANNEL_MAX_REQUEST_SIZE, 806 HW_CHANNEL_MAX_RESPONSE_SIZE); 807 if (err) { 808 device_printf(hwc->dev, "Failed to initialize HWC: %d\n", 809 err); 810 goto out; 811 } 812 813 err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, 814 &max_resp_msg_size); 815 if (err) { 816 device_printf(hwc->dev, "Failed to establish HWC: %d\n", err); 817 goto out; 818 } 819 820 err = mana_hwc_test_channel(gc->hwc.driver_data, 821 HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 822 max_req_msg_size, max_resp_msg_size); 823 if (err) { 824 /* Test failed, but the channel has been established */ 825 device_printf(hwc->dev, "Failed to test HWC: %d\n", err); 826 return EIO; 827 } 828 829 return 0; 830 out: 831 mana_hwc_destroy_channel(gc); 832 return (err); 833 } 834 835 void 836 mana_hwc_destroy_channel(struct gdma_context *gc) 837 { 838 struct hw_channel_context *hwc = gc->hwc.driver_data; 839 840 if (!hwc) 841 return; 842 843 /* 844 * gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's 845 * non-zero, the HWC worked and we should tear down the HWC here. 846 */ 847 if (gc->max_num_cqs > 0) { 848 mana_smc_teardown_hwc(&gc->shm_channel, false); 849 gc->max_num_cqs = 0; 850 } 851 852 free(hwc->caller_ctx, M_DEVBUF); 853 hwc->caller_ctx = NULL; 854 855 if (hwc->txq) 856 mana_hwc_destroy_wq(hwc, hwc->txq); 857 858 if (hwc->rxq) 859 mana_hwc_destroy_wq(hwc, hwc->rxq); 860 861 if (hwc->cq) 862 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); 863 864 mana_gd_free_res_map(&hwc->inflight_msg_res); 865 866 hwc->num_inflight_msg = 0; 867 868 hwc->gdma_dev->doorbell = INVALID_DOORBELL; 869 hwc->gdma_dev->pdid = INVALID_PDID; 870 871 free(hwc, M_DEVBUF); 872 gc->hwc.driver_data = NULL; 873 gc->hwc.gdma_context = NULL; 874 875 free(gc->cq_table, M_DEVBUF); 876 gc->cq_table = NULL; 877 } 878 879 int 880 mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len, 881 const void *req, uint32_t resp_len, void *resp) 882 { 883 struct hwc_work_request *tx_wr; 884 struct hwc_wq *txq = hwc->txq; 885 struct gdma_req_hdr *req_msg; 886 struct hwc_caller_ctx *ctx; 887 uint16_t msg_id; 888 int err; 889 890 mana_hwc_get_msg_index(hwc, &msg_id); 891 892 tx_wr = &txq->msg_buf->reqs[msg_id]; 893 894 if (req_len > tx_wr->buf_len) { 895 device_printf(hwc->dev, 896 "HWC: req msg size: %d > %d\n", req_len, 897 tx_wr->buf_len); 898 err = EINVAL; 899 goto out; 900 } 901 902 ctx = hwc->caller_ctx + msg_id; 903 ctx->output_buf = resp; 904 ctx->output_buflen = resp_len; 905 906 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; 907 if (req) 908 memcpy(req_msg, req, req_len); 909 910 req_msg->req.hwc_msg_id = msg_id; 911 912 tx_wr->msg_size = req_len; 913 914 err = mana_hwc_post_tx_wqe(txq, tx_wr, 0, 0, false); 915 if (err) { 916 device_printf(hwc->dev, 917 "HWC: Failed to post send WQE: %d\n", err); 918 goto out; 919 } 920 921 if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) { 922 device_printf(hwc->dev, "HWC: Request timed out!\n"); 923 err = ETIMEDOUT; 924 goto out; 925 } 926 927 if (ctx->error) { 928 err = ctx->error; 929 goto out; 930 } 931 932 if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { 933 device_printf(hwc->dev, 934 "HWC: Failed hw_channel req: 0x%x\n", ctx->status_code); 935 err = EPROTO; 936 goto out; 937 } 938 out: 939 mana_hwc_put_msg_index(hwc, msg_id); 940 return err; 941 } 942