1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Microsoft Corp. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/types.h> 34 #include <sys/kernel.h> 35 #include <sys/kthread.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/mutex.h> 39 #include <sys/bus.h> 40 #include <machine/bus.h> 41 42 #include "mana.h" 43 #include "hw_channel.h" 44 45 static int 46 mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id) 47 { 48 struct gdma_resource *r = &hwc->inflight_msg_res; 49 uint32_t index; 50 51 sema_wait(&hwc->sema); 52 53 mtx_lock_spin(&r->lock_spin); 54 55 index = find_first_zero_bit(hwc->inflight_msg_res.map, 56 hwc->inflight_msg_res.size); 57 58 bitmap_set(hwc->inflight_msg_res.map, index, 1); 59 60 mtx_unlock_spin(&r->lock_spin); 61 62 *msg_id = index; 63 64 return 0; 65 } 66 67 static void 68 mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id) 69 { 70 struct gdma_resource *r = &hwc->inflight_msg_res; 71 72 mtx_lock_spin(&r->lock_spin); 73 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); 74 mtx_unlock_spin(&r->lock_spin); 75 76 sema_post(&hwc->sema); 77 } 78 79 static int 80 mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, 81 const struct gdma_resp_hdr *resp_msg, 82 uint32_t resp_len) 83 { 84 if (resp_len < sizeof(*resp_msg)) 85 return EPROTO; 86 87 if (resp_len > caller_ctx->output_buflen) 88 return EPROTO; 89 90 return 0; 91 } 92 93 static void 94 mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len, 95 const struct gdma_resp_hdr *resp_msg) 96 { 97 struct hwc_caller_ctx *ctx; 98 int err; 99 100 if (!test_bit(resp_msg->response.hwc_msg_id, 101 hwc->inflight_msg_res.map)) { 102 device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n", 103 resp_msg->response.hwc_msg_id); 104 return; 105 } 106 107 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; 108 err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); 109 if (err) 110 goto out; 111 112 ctx->status_code = resp_msg->status; 113 114 memcpy(ctx->output_buf, resp_msg, resp_len); 115 out: 116 ctx->error = err; 117 complete(&ctx->comp_event); 118 } 119 120 static int 121 mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq, 122 struct hwc_work_request *req) 123 { 124 device_t dev = hwc_rxq->hwc->dev; 125 struct gdma_sge *sge; 126 int err; 127 128 sge = &req->sge; 129 sge->address = (uintptr_t)req->buf_sge_addr; 130 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; 131 sge->size = req->buf_len; 132 133 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 134 req->wqe_req.sgl = sge; 135 req->wqe_req.num_sge = 1; 136 req->wqe_req.client_data_unit = 0; 137 138 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); 139 if (err) 140 device_printf(dev, 141 "Failed to post WQE on HWC RQ: %d\n", err); 142 return err; 143 } 144 145 static void 146 mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, 147 struct gdma_event *event) 148 { 149 struct hw_channel_context *hwc = ctx; 150 struct gdma_dev *gd = hwc->gdma_dev; 151 union hwc_init_type_data type_data; 152 union hwc_init_eq_id_db eq_db; 153 uint32_t type, val; 154 155 switch (event->type) { 156 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 157 eq_db.as_uint32 = event->details[0]; 158 hwc->cq->gdma_eq->id = eq_db.eq_id; 159 gd->doorbell = eq_db.doorbell; 160 break; 161 162 case GDMA_EQE_HWC_INIT_DATA: 163 type_data.as_uint32 = event->details[0]; 164 type = type_data.type; 165 val = type_data.value; 166 167 switch (type) { 168 case HWC_INIT_DATA_CQID: 169 hwc->cq->gdma_cq->id = val; 170 break; 171 172 case HWC_INIT_DATA_RQID: 173 hwc->rxq->gdma_wq->id = val; 174 break; 175 176 case HWC_INIT_DATA_SQID: 177 hwc->txq->gdma_wq->id = val; 178 break; 179 180 case HWC_INIT_DATA_QUEUE_DEPTH: 181 hwc->hwc_init_q_depth_max = (uint16_t)val; 182 break; 183 184 case HWC_INIT_DATA_MAX_REQUEST: 185 hwc->hwc_init_max_req_msg_size = val; 186 break; 187 188 case HWC_INIT_DATA_MAX_RESPONSE: 189 hwc->hwc_init_max_resp_msg_size = val; 190 break; 191 192 case HWC_INIT_DATA_MAX_NUM_CQS: 193 gd->gdma_context->max_num_cqs = val; 194 break; 195 196 case HWC_INIT_DATA_PDID: 197 hwc->gdma_dev->pdid = val; 198 break; 199 200 case HWC_INIT_DATA_GPA_MKEY: 201 hwc->rxq->msg_buf->gpa_mkey = val; 202 hwc->txq->msg_buf->gpa_mkey = val; 203 break; 204 } 205 206 break; 207 208 case GDMA_EQE_HWC_INIT_DONE: 209 complete(&hwc->hwc_init_eqe_comp); 210 break; 211 212 default: 213 /* Ignore unknown events, which should never happen. */ 214 break; 215 } 216 } 217 218 static void 219 mana_hwc_rx_event_handler(void *ctx, uint32_t gdma_rxq_id, 220 const struct hwc_rx_oob *rx_oob) 221 { 222 struct hw_channel_context *hwc = ctx; 223 struct hwc_wq *hwc_rxq = hwc->rxq; 224 struct hwc_work_request *rx_req; 225 struct gdma_resp_hdr *resp; 226 struct gdma_wqe *dma_oob; 227 struct gdma_queue *rq; 228 struct gdma_sge *sge; 229 uint64_t rq_base_addr; 230 uint64_t rx_req_idx; 231 uint8_t *wqe; 232 233 if (hwc_rxq->gdma_wq->id != gdma_rxq_id) { 234 mana_warn(NULL, "unmatched rx queue %u != %u\n", 235 hwc_rxq->gdma_wq->id, gdma_rxq_id); 236 return; 237 } 238 239 240 rq = hwc_rxq->gdma_wq; 241 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); 242 dma_oob = (struct gdma_wqe *)wqe; 243 244 bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map, 245 BUS_DMASYNC_POSTREAD); 246 247 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); 248 249 /* Select the RX work request for virtual address and for reposting. */ 250 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; 251 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; 252 253 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 254 hwc_rxq->msg_buf->mem_info.dma_map, 255 BUS_DMASYNC_POSTREAD); 256 257 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; 258 resp = (struct gdma_resp_hdr *)rx_req->buf_va; 259 260 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { 261 device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n", 262 resp->response.hwc_msg_id); 263 return; 264 } 265 266 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp); 267 268 /* Do no longer use 'resp', because the buffer is posted to the HW 269 * in the below mana_hwc_post_rx_wqe(). 270 */ 271 resp = NULL; 272 273 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 274 hwc_rxq->msg_buf->mem_info.dma_map, 275 BUS_DMASYNC_PREREAD); 276 277 mana_hwc_post_rx_wqe(hwc_rxq, rx_req); 278 } 279 280 static void 281 mana_hwc_tx_event_handler(void *ctx, uint32_t gdma_txq_id, 282 const struct hwc_rx_oob *rx_oob) 283 { 284 struct hw_channel_context *hwc = ctx; 285 struct hwc_wq *hwc_txq = hwc->txq; 286 287 if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) { 288 mana_warn(NULL, "unmatched tx queue %u != %u\n", 289 hwc_txq->gdma_wq->id, gdma_txq_id); 290 } 291 292 bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag, 293 hwc_txq->gdma_wq->mem_info.dma_map, 294 BUS_DMASYNC_POSTWRITE); 295 } 296 297 static int 298 mana_hwc_create_gdma_wq(struct hw_channel_context *hwc, 299 enum gdma_queue_type type, uint64_t queue_size, 300 struct gdma_queue **queue) 301 { 302 struct gdma_queue_spec spec = {}; 303 304 if (type != GDMA_SQ && type != GDMA_RQ) 305 return EINVAL; 306 307 spec.type = type; 308 spec.monitor_avl_buf = false; 309 spec.queue_size = queue_size; 310 311 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 312 } 313 314 static int 315 mana_hwc_create_gdma_cq(struct hw_channel_context *hwc, 316 uint64_t queue_size, 317 void *ctx, gdma_cq_callback *cb, 318 struct gdma_queue *parent_eq, 319 struct gdma_queue **queue) 320 { 321 struct gdma_queue_spec spec = {}; 322 323 spec.type = GDMA_CQ; 324 spec.monitor_avl_buf = false; 325 spec.queue_size = queue_size; 326 spec.cq.context = ctx; 327 spec.cq.callback = cb; 328 spec.cq.parent_eq = parent_eq; 329 330 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 331 } 332 333 static int 334 mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, 335 uint64_t queue_size, 336 void *ctx, gdma_eq_callback *cb, 337 struct gdma_queue **queue) 338 { 339 struct gdma_queue_spec spec = {}; 340 341 spec.type = GDMA_EQ; 342 spec.monitor_avl_buf = false; 343 spec.queue_size = queue_size; 344 spec.eq.context = ctx; 345 spec.eq.callback = cb; 346 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; 347 348 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 349 } 350 351 static void 352 mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self) 353 { 354 struct hwc_rx_oob comp_data = {}; 355 struct gdma_comp *completions; 356 struct hwc_cq *hwc_cq = ctx; 357 int comp_read, i; 358 359 completions = hwc_cq->comp_buf; 360 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); 361 362 for (i = 0; i < comp_read; ++i) { 363 comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data; 364 365 if (completions[i].is_sq) 366 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, 367 completions[i].wq_num, 368 &comp_data); 369 else 370 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, 371 completions[i].wq_num, 372 &comp_data); 373 } 374 375 bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map, 376 BUS_DMASYNC_POSTREAD); 377 378 mana_gd_ring_cq(q_self, SET_ARM_BIT); 379 } 380 381 static void 382 mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq) 383 { 384 if (hwc_cq->comp_buf) 385 free(hwc_cq->comp_buf, M_DEVBUF); 386 387 if (hwc_cq->gdma_cq) 388 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); 389 390 if (hwc_cq->gdma_eq) 391 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); 392 393 free(hwc_cq, M_DEVBUF); 394 } 395 396 static int 397 mana_hwc_create_cq(struct hw_channel_context *hwc, 398 uint16_t q_depth, 399 gdma_eq_callback *callback, void *ctx, 400 hwc_rx_event_handler_t *rx_ev_hdlr, void *rx_ev_ctx, 401 hwc_tx_event_handler_t *tx_ev_hdlr, void *tx_ev_ctx, 402 struct hwc_cq **hwc_cq_ptr) 403 { 404 struct gdma_queue *eq, *cq; 405 struct gdma_comp *comp_buf; 406 struct hwc_cq *hwc_cq; 407 uint32_t eq_size, cq_size; 408 int err; 409 410 eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth); 411 if (eq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 412 eq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 413 414 cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth); 415 if (cq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 416 cq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 417 418 hwc_cq = malloc(sizeof(*hwc_cq), M_DEVBUF, M_WAITOK | M_ZERO); 419 420 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); 421 if (err) { 422 device_printf(hwc->dev, 423 "Failed to create HWC EQ for RQ: %d\n", err); 424 goto out; 425 } 426 hwc_cq->gdma_eq = eq; 427 428 err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, 429 mana_hwc_comp_event, eq, &cq); 430 if (err) { 431 device_printf(hwc->dev, 432 "Failed to create HWC CQ for RQ: %d\n", err); 433 goto out; 434 } 435 hwc_cq->gdma_cq = cq; 436 437 comp_buf = mallocarray(q_depth, sizeof(struct gdma_comp), 438 M_DEVBUF, M_WAITOK | M_ZERO); 439 440 hwc_cq->hwc = hwc; 441 hwc_cq->comp_buf = comp_buf; 442 hwc_cq->queue_depth = q_depth; 443 hwc_cq->rx_event_handler = rx_ev_hdlr; 444 hwc_cq->rx_event_ctx = rx_ev_ctx; 445 hwc_cq->tx_event_handler = tx_ev_hdlr; 446 hwc_cq->tx_event_ctx = tx_ev_ctx; 447 448 *hwc_cq_ptr = hwc_cq; 449 return 0; 450 out: 451 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); 452 return err; 453 } 454 455 static int 456 mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth, 457 uint32_t max_msg_size, 458 struct hwc_dma_buf **dma_buf_ptr) 459 { 460 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 461 struct hwc_work_request *hwc_wr; 462 struct hwc_dma_buf *dma_buf; 463 struct gdma_mem_info *gmi; 464 uint32_t buf_size; 465 uint8_t *base_pa; 466 void *virt_addr; 467 uint16_t i; 468 int err; 469 470 dma_buf = malloc(sizeof(*dma_buf) + 471 q_depth * sizeof(struct hwc_work_request), 472 M_DEVBUF, M_WAITOK | M_ZERO); 473 474 dma_buf->num_reqs = q_depth; 475 476 buf_size = ALIGN(q_depth * max_msg_size, PAGE_SIZE); 477 478 gmi = &dma_buf->mem_info; 479 err = mana_gd_alloc_memory(gc, buf_size, gmi); 480 if (err) { 481 device_printf(hwc->dev, 482 "Failed to allocate DMA buffer: %d\n", err); 483 goto out; 484 } 485 486 virt_addr = dma_buf->mem_info.virt_addr; 487 base_pa = (uint8_t *)dma_buf->mem_info.dma_handle; 488 489 for (i = 0; i < q_depth; i++) { 490 hwc_wr = &dma_buf->reqs[i]; 491 492 hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size; 493 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; 494 495 hwc_wr->buf_len = max_msg_size; 496 } 497 498 *dma_buf_ptr = dma_buf; 499 return 0; 500 out: 501 free(dma_buf, M_DEVBUF); 502 return err; 503 } 504 505 static void 506 mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc, 507 struct hwc_dma_buf *dma_buf) 508 { 509 if (!dma_buf) 510 return; 511 512 mana_gd_free_memory(&dma_buf->mem_info); 513 514 free(dma_buf, M_DEVBUF); 515 } 516 517 static void 518 mana_hwc_destroy_wq(struct hw_channel_context *hwc, 519 struct hwc_wq *hwc_wq) 520 { 521 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); 522 523 if (hwc_wq->gdma_wq) 524 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, 525 hwc_wq->gdma_wq); 526 527 free(hwc_wq, M_DEVBUF); 528 } 529 530 static int 531 mana_hwc_create_wq(struct hw_channel_context *hwc, 532 enum gdma_queue_type q_type, uint16_t q_depth, 533 uint32_t max_msg_size, struct hwc_cq *hwc_cq, 534 struct hwc_wq **hwc_wq_ptr) 535 { 536 struct gdma_queue *queue; 537 struct hwc_wq *hwc_wq; 538 uint32_t queue_size; 539 int err; 540 541 if (q_type != GDMA_SQ && q_type != GDMA_RQ) { 542 /* XXX should fail and return error? */ 543 mana_warn(NULL, "Invalid q_type %u\n", q_type); 544 } 545 546 if (q_type == GDMA_RQ) 547 queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth); 548 else 549 queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth); 550 551 if (queue_size < MINIMUM_SUPPORTED_PAGE_SIZE) 552 queue_size = MINIMUM_SUPPORTED_PAGE_SIZE; 553 554 hwc_wq = malloc(sizeof(*hwc_wq), M_DEVBUF, M_WAITOK | M_ZERO); 555 556 err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue); 557 if (err) 558 goto out; 559 560 hwc_wq->hwc = hwc; 561 hwc_wq->gdma_wq = queue; 562 hwc_wq->queue_depth = q_depth; 563 hwc_wq->hwc_cq = hwc_cq; 564 565 err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, 566 &hwc_wq->msg_buf); 567 if (err) 568 goto out; 569 570 *hwc_wq_ptr = hwc_wq; 571 return 0; 572 out: 573 if (err) 574 mana_hwc_destroy_wq(hwc, hwc_wq); 575 return err; 576 } 577 578 static int 579 mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq, 580 struct hwc_work_request *req, 581 uint32_t dest_virt_rq_id, uint32_t dest_virt_rcq_id, 582 bool dest_pf) 583 { 584 device_t dev = hwc_txq->hwc->dev; 585 struct hwc_tx_oob *tx_oob; 586 struct gdma_sge *sge; 587 int err; 588 589 if (req->msg_size == 0 || req->msg_size > req->buf_len) { 590 device_printf(dev, "wrong msg_size: %u, buf_len: %u\n", 591 req->msg_size, req->buf_len); 592 return EINVAL; 593 } 594 595 tx_oob = &req->tx_oob; 596 597 tx_oob->vrq_id = dest_virt_rq_id; 598 tx_oob->dest_vfid = 0; 599 tx_oob->vrcq_id = dest_virt_rcq_id; 600 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; 601 tx_oob->loopback = false; 602 tx_oob->lso_override = false; 603 tx_oob->dest_pf = dest_pf; 604 tx_oob->vsq_id = hwc_txq->gdma_wq->id; 605 606 sge = &req->sge; 607 sge->address = (uintptr_t)req->buf_sge_addr; 608 sge->mem_key = hwc_txq->msg_buf->gpa_mkey; 609 sge->size = req->msg_size; 610 611 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 612 req->wqe_req.sgl = sge; 613 req->wqe_req.num_sge = 1; 614 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); 615 req->wqe_req.inline_oob_data = tx_oob; 616 req->wqe_req.client_data_unit = 0; 617 618 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); 619 if (err) 620 device_printf(dev, 621 "Failed to post WQE on HWC SQ: %d\n", err); 622 return err; 623 } 624 625 static int 626 mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg) 627 { 628 int err; 629 630 sema_init(&hwc->sema, num_msg, "gdma hwc sema"); 631 632 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res, 633 "gdma hwc res lock"); 634 if (err) 635 device_printf(hwc->dev, 636 "Failed to init inflight_msg_res: %d\n", err); 637 638 return (err); 639 } 640 641 static int 642 mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth, 643 uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 644 { 645 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 646 struct hwc_wq *hwc_rxq = hwc->rxq; 647 struct hwc_work_request *req; 648 struct hwc_caller_ctx *ctx; 649 int err; 650 int i; 651 652 /* Post all WQEs on the RQ */ 653 for (i = 0; i < q_depth; i++) { 654 req = &hwc_rxq->msg_buf->reqs[i]; 655 err = mana_hwc_post_rx_wqe(hwc_rxq, req); 656 if (err) 657 return err; 658 } 659 660 ctx = malloc(q_depth * sizeof(struct hwc_caller_ctx), 661 M_DEVBUF, M_WAITOK | M_ZERO); 662 663 for (i = 0; i < q_depth; ++i) 664 init_completion(&ctx[i].comp_event); 665 666 hwc->caller_ctx = ctx; 667 668 return mana_gd_test_eq(gc, hwc->cq->gdma_eq); 669 } 670 671 static int 672 mana_hwc_establish_channel(struct gdma_context *gc, uint16_t *q_depth, 673 uint32_t *max_req_msg_size, 674 uint32_t *max_resp_msg_size) 675 { 676 struct hw_channel_context *hwc = gc->hwc.driver_data; 677 struct gdma_queue *rq = hwc->rxq->gdma_wq; 678 struct gdma_queue *sq = hwc->txq->gdma_wq; 679 struct gdma_queue *eq = hwc->cq->gdma_eq; 680 struct gdma_queue *cq = hwc->cq->gdma_cq; 681 int err; 682 683 init_completion(&hwc->hwc_init_eqe_comp); 684 685 err = mana_smc_setup_hwc(&gc->shm_channel, false, 686 eq->mem_info.dma_handle, 687 cq->mem_info.dma_handle, 688 rq->mem_info.dma_handle, 689 sq->mem_info.dma_handle, 690 eq->eq.msix_index); 691 if (err) 692 return err; 693 694 if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz)) 695 return ETIMEDOUT; 696 697 *q_depth = hwc->hwc_init_q_depth_max; 698 *max_req_msg_size = hwc->hwc_init_max_req_msg_size; 699 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; 700 701 /* Both were set in mana_hwc_init_event_handler(). */ 702 if (cq->id >= gc->max_num_cqs) { 703 mana_warn(NULL, "invalid cq id %u > %u\n", 704 cq->id, gc->max_num_cqs); 705 return EPROTO; 706 } 707 708 gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *), 709 M_DEVBUF, M_WAITOK | M_ZERO); 710 gc->cq_table[cq->id] = cq; 711 712 return 0; 713 } 714 715 static int 716 mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth, 717 uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 718 { 719 int err; 720 721 err = mana_hwc_init_inflight_msg(hwc, q_depth); 722 if (err) 723 return err; 724 725 /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ 726 * queue depth and RQ queue depth. 727 */ 728 err = mana_hwc_create_cq(hwc, q_depth * 2, 729 mana_hwc_init_event_handler, hwc, 730 mana_hwc_rx_event_handler, hwc, 731 mana_hwc_tx_event_handler, hwc, &hwc->cq); 732 if (err) { 733 device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err); 734 goto out; 735 } 736 737 err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, 738 hwc->cq, &hwc->rxq); 739 if (err) { 740 device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err); 741 goto out; 742 } 743 744 err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, 745 hwc->cq, &hwc->txq); 746 if (err) { 747 device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err); 748 goto out; 749 } 750 751 hwc->num_inflight_msg = q_depth; 752 hwc->max_req_msg_size = max_req_msg_size; 753 754 return 0; 755 out: 756 /* mana_hwc_create_channel() will do the cleanup.*/ 757 return err; 758 } 759 760 int 761 mana_hwc_create_channel(struct gdma_context *gc) 762 { 763 uint32_t max_req_msg_size, max_resp_msg_size; 764 struct gdma_dev *gd = &gc->hwc; 765 struct hw_channel_context *hwc; 766 uint16_t q_depth_max; 767 int err; 768 769 hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO); 770 771 gd->gdma_context = gc; 772 gd->driver_data = hwc; 773 hwc->gdma_dev = gd; 774 hwc->dev = gc->dev; 775 776 /* HWC's instance number is always 0. */ 777 gd->dev_id.as_uint32 = 0; 778 gd->dev_id.type = GDMA_DEVICE_HWC; 779 780 gd->pdid = INVALID_PDID; 781 gd->doorbell = INVALID_DOORBELL; 782 783 /* 784 * mana_hwc_init_queues() only creates the required data structures, 785 * and doesn't touch the HWC device. 786 */ 787 err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 788 HW_CHANNEL_MAX_REQUEST_SIZE, 789 HW_CHANNEL_MAX_RESPONSE_SIZE); 790 if (err) { 791 device_printf(hwc->dev, "Failed to initialize HWC: %d\n", 792 err); 793 goto out; 794 } 795 796 err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, 797 &max_resp_msg_size); 798 if (err) { 799 device_printf(hwc->dev, "Failed to establish HWC: %d\n", err); 800 goto out; 801 } 802 803 err = mana_hwc_test_channel(gc->hwc.driver_data, 804 HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 805 max_req_msg_size, max_resp_msg_size); 806 if (err) { 807 /* Test failed, but the channel has been established */ 808 device_printf(hwc->dev, "Failed to test HWC: %d\n", err); 809 return EIO; 810 } 811 812 return 0; 813 out: 814 mana_hwc_destroy_channel(gc); 815 return (err); 816 } 817 818 void 819 mana_hwc_destroy_channel(struct gdma_context *gc) 820 { 821 struct hw_channel_context *hwc = gc->hwc.driver_data; 822 823 if (!hwc) 824 return; 825 826 /* 827 * gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's 828 * non-zero, the HWC worked and we should tear down the HWC here. 829 */ 830 if (gc->max_num_cqs > 0) { 831 mana_smc_teardown_hwc(&gc->shm_channel, false); 832 gc->max_num_cqs = 0; 833 } 834 835 free(hwc->caller_ctx, M_DEVBUF); 836 hwc->caller_ctx = NULL; 837 838 if (hwc->txq) 839 mana_hwc_destroy_wq(hwc, hwc->txq); 840 841 if (hwc->rxq) 842 mana_hwc_destroy_wq(hwc, hwc->rxq); 843 844 if (hwc->cq) 845 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); 846 847 mana_gd_free_res_map(&hwc->inflight_msg_res); 848 849 hwc->num_inflight_msg = 0; 850 851 hwc->gdma_dev->doorbell = INVALID_DOORBELL; 852 hwc->gdma_dev->pdid = INVALID_PDID; 853 854 free(hwc, M_DEVBUF); 855 gc->hwc.driver_data = NULL; 856 gc->hwc.gdma_context = NULL; 857 858 free(gc->cq_table, M_DEVBUF); 859 gc->cq_table = NULL; 860 } 861 862 int 863 mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len, 864 const void *req, uint32_t resp_len, void *resp) 865 { 866 struct hwc_work_request *tx_wr; 867 struct hwc_wq *txq = hwc->txq; 868 struct gdma_req_hdr *req_msg; 869 struct hwc_caller_ctx *ctx; 870 uint16_t msg_id; 871 int err; 872 873 mana_hwc_get_msg_index(hwc, &msg_id); 874 875 tx_wr = &txq->msg_buf->reqs[msg_id]; 876 877 if (req_len > tx_wr->buf_len) { 878 device_printf(hwc->dev, 879 "HWC: req msg size: %d > %d\n", req_len, 880 tx_wr->buf_len); 881 err = EINVAL; 882 goto out; 883 } 884 885 ctx = hwc->caller_ctx + msg_id; 886 ctx->output_buf = resp; 887 ctx->output_buflen = resp_len; 888 889 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; 890 if (req) 891 memcpy(req_msg, req, req_len); 892 893 req_msg->req.hwc_msg_id = msg_id; 894 895 tx_wr->msg_size = req_len; 896 897 err = mana_hwc_post_tx_wqe(txq, tx_wr, 0, 0, false); 898 if (err) { 899 device_printf(hwc->dev, 900 "HWC: Failed to post send WQE: %d\n", err); 901 goto out; 902 } 903 904 if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) { 905 device_printf(hwc->dev, "HWC: Request timed out!\n"); 906 err = ETIMEDOUT; 907 goto out; 908 } 909 910 if (ctx->error) { 911 err = ctx->error; 912 goto out; 913 } 914 915 if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { 916 device_printf(hwc->dev, 917 "HWC: Failed hw_channel req: 0x%x\n", ctx->status_code); 918 err = EPROTO; 919 goto out; 920 } 921 out: 922 mana_hwc_put_msg_index(hwc, msg_id); 923 return err; 924 } 925