1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <net/mana/gdma.h> 5 #include <net/mana/hw_channel.h> 6 #include <linux/vmalloc.h> 7 8 static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id) 9 { 10 struct gdma_resource *r = &hwc->inflight_msg_res; 11 unsigned long flags; 12 u32 index; 13 14 down(&hwc->sema); 15 16 spin_lock_irqsave(&r->lock, flags); 17 18 index = find_first_zero_bit(hwc->inflight_msg_res.map, 19 hwc->inflight_msg_res.size); 20 21 bitmap_set(hwc->inflight_msg_res.map, index, 1); 22 23 spin_unlock_irqrestore(&r->lock, flags); 24 25 *msg_id = index; 26 27 return 0; 28 } 29 30 static void mana_hwc_put_msg_index(struct hw_channel_context *hwc, u16 msg_id) 31 { 32 struct gdma_resource *r = &hwc->inflight_msg_res; 33 unsigned long flags; 34 35 spin_lock_irqsave(&r->lock, flags); 36 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); 37 spin_unlock_irqrestore(&r->lock, flags); 38 39 up(&hwc->sema); 40 } 41 42 static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, 43 const struct gdma_resp_hdr *resp_msg, 44 u32 resp_len) 45 { 46 if (resp_len < sizeof(*resp_msg)) 47 return -EPROTO; 48 49 if (resp_len > caller_ctx->output_buflen) 50 return -EPROTO; 51 52 return 0; 53 } 54 55 static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq, 56 struct hwc_work_request *req) 57 { 58 struct device *dev = hwc_rxq->hwc->dev; 59 struct gdma_sge *sge; 60 int err; 61 62 sge = &req->sge; 63 sge->address = (u64)req->buf_sge_addr; 64 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; 65 sge->size = req->buf_len; 66 67 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 68 req->wqe_req.sgl = sge; 69 req->wqe_req.num_sge = 1; 70 req->wqe_req.client_data_unit = 0; 71 72 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); 73 if (err) 74 dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err); 75 return err; 76 } 77 78 static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len, 79 struct hwc_work_request *rx_req) 80 { 81 const struct gdma_resp_hdr *resp_msg = rx_req->buf_va; 82 struct hwc_caller_ctx *ctx; 83 int err; 84 85 if (!test_bit(resp_msg->response.hwc_msg_id, 86 hwc->inflight_msg_res.map)) { 87 dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", 88 resp_msg->response.hwc_msg_id); 89 mana_hwc_post_rx_wqe(hwc->rxq, rx_req); 90 return; 91 } 92 93 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; 94 err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); 95 if (err) 96 goto out; 97 98 ctx->status_code = resp_msg->status; 99 100 memcpy(ctx->output_buf, resp_msg, resp_len); 101 out: 102 ctx->error = err; 103 104 /* Must post rx wqe before complete(), otherwise the next rx may 105 * hit no_wqe error. 106 */ 107 mana_hwc_post_rx_wqe(hwc->rxq, rx_req); 108 109 complete(&ctx->comp_event); 110 } 111 112 static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, 113 struct gdma_event *event) 114 { 115 struct hw_channel_context *hwc = ctx; 116 struct gdma_dev *gd = hwc->gdma_dev; 117 union hwc_init_type_data type_data; 118 union hwc_init_eq_id_db eq_db; 119 u32 type, val; 120 121 switch (event->type) { 122 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 123 eq_db.as_uint32 = event->details[0]; 124 hwc->cq->gdma_eq->id = eq_db.eq_id; 125 gd->doorbell = eq_db.doorbell; 126 break; 127 128 case GDMA_EQE_HWC_INIT_DATA: 129 type_data.as_uint32 = event->details[0]; 130 type = type_data.type; 131 val = type_data.value; 132 133 switch (type) { 134 case HWC_INIT_DATA_CQID: 135 hwc->cq->gdma_cq->id = val; 136 break; 137 138 case HWC_INIT_DATA_RQID: 139 hwc->rxq->gdma_wq->id = val; 140 break; 141 142 case HWC_INIT_DATA_SQID: 143 hwc->txq->gdma_wq->id = val; 144 break; 145 146 case HWC_INIT_DATA_QUEUE_DEPTH: 147 hwc->hwc_init_q_depth_max = (u16)val; 148 break; 149 150 case HWC_INIT_DATA_MAX_REQUEST: 151 hwc->hwc_init_max_req_msg_size = val; 152 break; 153 154 case HWC_INIT_DATA_MAX_RESPONSE: 155 hwc->hwc_init_max_resp_msg_size = val; 156 break; 157 158 case HWC_INIT_DATA_MAX_NUM_CQS: 159 gd->gdma_context->max_num_cqs = val; 160 break; 161 162 case HWC_INIT_DATA_PDID: 163 hwc->gdma_dev->pdid = val; 164 break; 165 166 case HWC_INIT_DATA_GPA_MKEY: 167 hwc->rxq->msg_buf->gpa_mkey = val; 168 hwc->txq->msg_buf->gpa_mkey = val; 169 break; 170 171 case HWC_INIT_DATA_PF_DEST_RQ_ID: 172 hwc->pf_dest_vrq_id = val; 173 break; 174 175 case HWC_INIT_DATA_PF_DEST_CQ_ID: 176 hwc->pf_dest_vrcq_id = val; 177 break; 178 } 179 180 break; 181 182 case GDMA_EQE_HWC_INIT_DONE: 183 complete(&hwc->hwc_init_eqe_comp); 184 break; 185 186 case GDMA_EQE_HWC_SOC_RECONFIG_DATA: 187 type_data.as_uint32 = event->details[0]; 188 type = type_data.type; 189 val = type_data.value; 190 191 switch (type) { 192 case HWC_DATA_CFG_HWC_TIMEOUT: 193 hwc->hwc_timeout = val; 194 break; 195 196 default: 197 dev_warn(hwc->dev, "Received unknown reconfig type %u\n", type); 198 break; 199 } 200 201 break; 202 203 default: 204 dev_warn(hwc->dev, "Received unknown gdma event %u\n", event->type); 205 /* Ignore unknown events, which should never happen. */ 206 break; 207 } 208 } 209 210 static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id, 211 const struct hwc_rx_oob *rx_oob) 212 { 213 struct hw_channel_context *hwc = ctx; 214 struct hwc_wq *hwc_rxq = hwc->rxq; 215 struct hwc_work_request *rx_req; 216 struct gdma_resp_hdr *resp; 217 struct gdma_wqe *dma_oob; 218 struct gdma_queue *rq; 219 struct gdma_sge *sge; 220 u64 rq_base_addr; 221 u64 rx_req_idx; 222 u8 *wqe; 223 224 if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id)) 225 return; 226 227 rq = hwc_rxq->gdma_wq; 228 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); 229 dma_oob = (struct gdma_wqe *)wqe; 230 231 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); 232 233 /* Select the RX work request for virtual address and for reposting. */ 234 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; 235 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; 236 237 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; 238 resp = (struct gdma_resp_hdr *)rx_req->buf_va; 239 240 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { 241 dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", 242 resp->response.hwc_msg_id); 243 return; 244 } 245 246 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req); 247 248 /* Can no longer use 'resp', because the buffer is posted to the HW 249 * in mana_hwc_handle_resp() above. 250 */ 251 resp = NULL; 252 } 253 254 static void mana_hwc_tx_event_handler(void *ctx, u32 gdma_txq_id, 255 const struct hwc_rx_oob *rx_oob) 256 { 257 struct hw_channel_context *hwc = ctx; 258 struct hwc_wq *hwc_txq = hwc->txq; 259 260 WARN_ON_ONCE(!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id); 261 } 262 263 static int mana_hwc_create_gdma_wq(struct hw_channel_context *hwc, 264 enum gdma_queue_type type, u64 queue_size, 265 struct gdma_queue **queue) 266 { 267 struct gdma_queue_spec spec = {}; 268 269 if (type != GDMA_SQ && type != GDMA_RQ) 270 return -EINVAL; 271 272 spec.type = type; 273 spec.monitor_avl_buf = false; 274 spec.queue_size = queue_size; 275 276 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 277 } 278 279 static int mana_hwc_create_gdma_cq(struct hw_channel_context *hwc, 280 u64 queue_size, 281 void *ctx, gdma_cq_callback *cb, 282 struct gdma_queue *parent_eq, 283 struct gdma_queue **queue) 284 { 285 struct gdma_queue_spec spec = {}; 286 287 spec.type = GDMA_CQ; 288 spec.monitor_avl_buf = false; 289 spec.queue_size = queue_size; 290 spec.cq.context = ctx; 291 spec.cq.callback = cb; 292 spec.cq.parent_eq = parent_eq; 293 294 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 295 } 296 297 static int mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, 298 u64 queue_size, 299 void *ctx, gdma_eq_callback *cb, 300 struct gdma_queue **queue) 301 { 302 struct gdma_queue_spec spec = {}; 303 304 spec.type = GDMA_EQ; 305 spec.monitor_avl_buf = false; 306 spec.queue_size = queue_size; 307 spec.eq.context = ctx; 308 spec.eq.callback = cb; 309 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; 310 spec.eq.msix_index = 0; 311 312 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 313 } 314 315 static void mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self) 316 { 317 struct hwc_rx_oob comp_data = {}; 318 struct gdma_comp *completions; 319 struct hwc_cq *hwc_cq = ctx; 320 int comp_read, i; 321 322 WARN_ON_ONCE(hwc_cq->gdma_cq != q_self); 323 324 completions = hwc_cq->comp_buf; 325 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); 326 WARN_ON_ONCE(comp_read <= 0 || comp_read > hwc_cq->queue_depth); 327 328 for (i = 0; i < comp_read; ++i) { 329 comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data; 330 331 if (completions[i].is_sq) 332 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, 333 completions[i].wq_num, 334 &comp_data); 335 else 336 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, 337 completions[i].wq_num, 338 &comp_data); 339 } 340 341 mana_gd_ring_cq(q_self, SET_ARM_BIT); 342 } 343 344 static void mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq) 345 { 346 kfree(hwc_cq->comp_buf); 347 348 if (hwc_cq->gdma_cq) 349 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); 350 351 if (hwc_cq->gdma_eq) 352 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); 353 354 kfree(hwc_cq); 355 } 356 357 static int mana_hwc_create_cq(struct hw_channel_context *hwc, u16 q_depth, 358 gdma_eq_callback *callback, void *ctx, 359 hwc_rx_event_handler_t *rx_ev_hdlr, 360 void *rx_ev_ctx, 361 hwc_tx_event_handler_t *tx_ev_hdlr, 362 void *tx_ev_ctx, struct hwc_cq **hwc_cq_ptr) 363 { 364 struct gdma_queue *eq, *cq; 365 struct gdma_comp *comp_buf; 366 struct hwc_cq *hwc_cq; 367 u32 eq_size, cq_size; 368 int err; 369 370 eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth); 371 if (eq_size < MANA_MIN_QSIZE) 372 eq_size = MANA_MIN_QSIZE; 373 374 cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth); 375 if (cq_size < MANA_MIN_QSIZE) 376 cq_size = MANA_MIN_QSIZE; 377 378 hwc_cq = kzalloc(sizeof(*hwc_cq), GFP_KERNEL); 379 if (!hwc_cq) 380 return -ENOMEM; 381 382 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); 383 if (err) { 384 dev_err(hwc->dev, "Failed to create HWC EQ for RQ: %d\n", err); 385 goto out; 386 } 387 hwc_cq->gdma_eq = eq; 388 389 err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, mana_hwc_comp_event, 390 eq, &cq); 391 if (err) { 392 dev_err(hwc->dev, "Failed to create HWC CQ for RQ: %d\n", err); 393 goto out; 394 } 395 hwc_cq->gdma_cq = cq; 396 397 comp_buf = kcalloc(q_depth, sizeof(*comp_buf), GFP_KERNEL); 398 if (!comp_buf) { 399 err = -ENOMEM; 400 goto out; 401 } 402 403 hwc_cq->hwc = hwc; 404 hwc_cq->comp_buf = comp_buf; 405 hwc_cq->queue_depth = q_depth; 406 hwc_cq->rx_event_handler = rx_ev_hdlr; 407 hwc_cq->rx_event_ctx = rx_ev_ctx; 408 hwc_cq->tx_event_handler = tx_ev_hdlr; 409 hwc_cq->tx_event_ctx = tx_ev_ctx; 410 411 *hwc_cq_ptr = hwc_cq; 412 return 0; 413 out: 414 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); 415 return err; 416 } 417 418 static int mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, u16 q_depth, 419 u32 max_msg_size, 420 struct hwc_dma_buf **dma_buf_ptr) 421 { 422 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 423 struct hwc_work_request *hwc_wr; 424 struct hwc_dma_buf *dma_buf; 425 struct gdma_mem_info *gmi; 426 void *virt_addr; 427 u32 buf_size; 428 u8 *base_pa; 429 int err; 430 u16 i; 431 432 dma_buf = kzalloc(struct_size(dma_buf, reqs, q_depth), GFP_KERNEL); 433 if (!dma_buf) 434 return -ENOMEM; 435 436 dma_buf->num_reqs = q_depth; 437 438 buf_size = MANA_PAGE_ALIGN(q_depth * max_msg_size); 439 440 gmi = &dma_buf->mem_info; 441 err = mana_gd_alloc_memory(gc, buf_size, gmi); 442 if (err) { 443 dev_err(hwc->dev, "Failed to allocate DMA buffer: %d\n", err); 444 goto out; 445 } 446 447 virt_addr = dma_buf->mem_info.virt_addr; 448 base_pa = (u8 *)dma_buf->mem_info.dma_handle; 449 450 for (i = 0; i < q_depth; i++) { 451 hwc_wr = &dma_buf->reqs[i]; 452 453 hwc_wr->buf_va = virt_addr + i * max_msg_size; 454 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; 455 456 hwc_wr->buf_len = max_msg_size; 457 } 458 459 *dma_buf_ptr = dma_buf; 460 return 0; 461 out: 462 kfree(dma_buf); 463 return err; 464 } 465 466 static void mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc, 467 struct hwc_dma_buf *dma_buf) 468 { 469 if (!dma_buf) 470 return; 471 472 mana_gd_free_memory(&dma_buf->mem_info); 473 474 kfree(dma_buf); 475 } 476 477 static void mana_hwc_destroy_wq(struct hw_channel_context *hwc, 478 struct hwc_wq *hwc_wq) 479 { 480 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); 481 482 if (hwc_wq->gdma_wq) 483 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, 484 hwc_wq->gdma_wq); 485 486 kfree(hwc_wq); 487 } 488 489 static int mana_hwc_create_wq(struct hw_channel_context *hwc, 490 enum gdma_queue_type q_type, u16 q_depth, 491 u32 max_msg_size, struct hwc_cq *hwc_cq, 492 struct hwc_wq **hwc_wq_ptr) 493 { 494 struct gdma_queue *queue; 495 struct hwc_wq *hwc_wq; 496 u32 queue_size; 497 int err; 498 499 WARN_ON(q_type != GDMA_SQ && q_type != GDMA_RQ); 500 501 if (q_type == GDMA_RQ) 502 queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth); 503 else 504 queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth); 505 506 if (queue_size < MANA_MIN_QSIZE) 507 queue_size = MANA_MIN_QSIZE; 508 509 hwc_wq = kzalloc(sizeof(*hwc_wq), GFP_KERNEL); 510 if (!hwc_wq) 511 return -ENOMEM; 512 513 err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue); 514 if (err) 515 goto out; 516 517 hwc_wq->hwc = hwc; 518 hwc_wq->gdma_wq = queue; 519 hwc_wq->queue_depth = q_depth; 520 hwc_wq->hwc_cq = hwc_cq; 521 522 err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, 523 &hwc_wq->msg_buf); 524 if (err) 525 goto out; 526 527 *hwc_wq_ptr = hwc_wq; 528 return 0; 529 out: 530 if (err) 531 mana_hwc_destroy_wq(hwc, hwc_wq); 532 return err; 533 } 534 535 static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq, 536 struct hwc_work_request *req, 537 u32 dest_virt_rq_id, u32 dest_virt_rcq_id, 538 bool dest_pf) 539 { 540 struct device *dev = hwc_txq->hwc->dev; 541 struct hwc_tx_oob *tx_oob; 542 struct gdma_sge *sge; 543 int err; 544 545 if (req->msg_size == 0 || req->msg_size > req->buf_len) { 546 dev_err(dev, "wrong msg_size: %u, buf_len: %u\n", 547 req->msg_size, req->buf_len); 548 return -EINVAL; 549 } 550 551 tx_oob = &req->tx_oob; 552 553 tx_oob->vrq_id = dest_virt_rq_id; 554 tx_oob->dest_vfid = 0; 555 tx_oob->vrcq_id = dest_virt_rcq_id; 556 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; 557 tx_oob->loopback = false; 558 tx_oob->lso_override = false; 559 tx_oob->dest_pf = dest_pf; 560 tx_oob->vsq_id = hwc_txq->gdma_wq->id; 561 562 sge = &req->sge; 563 sge->address = (u64)req->buf_sge_addr; 564 sge->mem_key = hwc_txq->msg_buf->gpa_mkey; 565 sge->size = req->msg_size; 566 567 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 568 req->wqe_req.sgl = sge; 569 req->wqe_req.num_sge = 1; 570 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); 571 req->wqe_req.inline_oob_data = tx_oob; 572 req->wqe_req.client_data_unit = 0; 573 574 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); 575 if (err) 576 dev_err(dev, "Failed to post WQE on HWC SQ: %d\n", err); 577 return err; 578 } 579 580 static int mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, 581 u16 num_msg) 582 { 583 int err; 584 585 sema_init(&hwc->sema, num_msg); 586 587 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res); 588 if (err) 589 dev_err(hwc->dev, "Failed to init inflight_msg_res: %d\n", err); 590 return err; 591 } 592 593 static int mana_hwc_test_channel(struct hw_channel_context *hwc, u16 q_depth, 594 u32 max_req_msg_size, u32 max_resp_msg_size) 595 { 596 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 597 struct hwc_wq *hwc_rxq = hwc->rxq; 598 struct hwc_work_request *req; 599 struct hwc_caller_ctx *ctx; 600 int err; 601 int i; 602 603 /* Post all WQEs on the RQ */ 604 for (i = 0; i < q_depth; i++) { 605 req = &hwc_rxq->msg_buf->reqs[i]; 606 err = mana_hwc_post_rx_wqe(hwc_rxq, req); 607 if (err) 608 return err; 609 } 610 611 ctx = kcalloc(q_depth, sizeof(*ctx), GFP_KERNEL); 612 if (!ctx) 613 return -ENOMEM; 614 615 for (i = 0; i < q_depth; ++i) 616 init_completion(&ctx[i].comp_event); 617 618 hwc->caller_ctx = ctx; 619 620 return mana_gd_test_eq(gc, hwc->cq->gdma_eq); 621 } 622 623 static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth, 624 u32 *max_req_msg_size, 625 u32 *max_resp_msg_size) 626 { 627 struct hw_channel_context *hwc = gc->hwc.driver_data; 628 struct gdma_queue *rq = hwc->rxq->gdma_wq; 629 struct gdma_queue *sq = hwc->txq->gdma_wq; 630 struct gdma_queue *eq = hwc->cq->gdma_eq; 631 struct gdma_queue *cq = hwc->cq->gdma_cq; 632 int err; 633 634 init_completion(&hwc->hwc_init_eqe_comp); 635 636 err = mana_smc_setup_hwc(&gc->shm_channel, false, 637 eq->mem_info.dma_handle, 638 cq->mem_info.dma_handle, 639 rq->mem_info.dma_handle, 640 sq->mem_info.dma_handle, 641 eq->eq.msix_index); 642 if (err) 643 return err; 644 645 if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ)) 646 return -ETIMEDOUT; 647 648 *q_depth = hwc->hwc_init_q_depth_max; 649 *max_req_msg_size = hwc->hwc_init_max_req_msg_size; 650 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; 651 652 /* Both were set in mana_hwc_init_event_handler(). */ 653 if (WARN_ON(cq->id >= gc->max_num_cqs)) 654 return -EPROTO; 655 656 gc->cq_table = vcalloc(gc->max_num_cqs, sizeof(struct gdma_queue *)); 657 if (!gc->cq_table) 658 return -ENOMEM; 659 660 gc->cq_table[cq->id] = cq; 661 662 return 0; 663 } 664 665 static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth, 666 u32 max_req_msg_size, u32 max_resp_msg_size) 667 { 668 int err; 669 670 err = mana_hwc_init_inflight_msg(hwc, q_depth); 671 if (err) 672 return err; 673 674 /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ 675 * queue depth and RQ queue depth. 676 */ 677 err = mana_hwc_create_cq(hwc, q_depth * 2, 678 mana_hwc_init_event_handler, hwc, 679 mana_hwc_rx_event_handler, hwc, 680 mana_hwc_tx_event_handler, hwc, &hwc->cq); 681 if (err) { 682 dev_err(hwc->dev, "Failed to create HWC CQ: %d\n", err); 683 goto out; 684 } 685 686 err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, 687 hwc->cq, &hwc->rxq); 688 if (err) { 689 dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err); 690 goto out; 691 } 692 693 err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, 694 hwc->cq, &hwc->txq); 695 if (err) { 696 dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err); 697 goto out; 698 } 699 700 hwc->num_inflight_msg = q_depth; 701 hwc->max_req_msg_size = max_req_msg_size; 702 703 return 0; 704 out: 705 /* mana_hwc_create_channel() will do the cleanup.*/ 706 return err; 707 } 708 709 int mana_hwc_create_channel(struct gdma_context *gc) 710 { 711 u32 max_req_msg_size, max_resp_msg_size; 712 struct gdma_dev *gd = &gc->hwc; 713 struct hw_channel_context *hwc; 714 u16 q_depth_max; 715 int err; 716 717 hwc = kzalloc(sizeof(*hwc), GFP_KERNEL); 718 if (!hwc) 719 return -ENOMEM; 720 721 gd->gdma_context = gc; 722 gd->driver_data = hwc; 723 hwc->gdma_dev = gd; 724 hwc->dev = gc->dev; 725 hwc->hwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS; 726 727 /* HWC's instance number is always 0. */ 728 gd->dev_id.as_uint32 = 0; 729 gd->dev_id.type = GDMA_DEVICE_HWC; 730 731 gd->pdid = INVALID_PDID; 732 gd->doorbell = INVALID_DOORBELL; 733 734 /* mana_hwc_init_queues() only creates the required data structures, 735 * and doesn't touch the HWC device. 736 */ 737 err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 738 HW_CHANNEL_MAX_REQUEST_SIZE, 739 HW_CHANNEL_MAX_RESPONSE_SIZE); 740 if (err) { 741 dev_err(hwc->dev, "Failed to initialize HWC: %d\n", err); 742 goto out; 743 } 744 745 err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, 746 &max_resp_msg_size); 747 if (err) { 748 dev_err(hwc->dev, "Failed to establish HWC: %d\n", err); 749 goto out; 750 } 751 752 err = mana_hwc_test_channel(gc->hwc.driver_data, 753 HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 754 max_req_msg_size, max_resp_msg_size); 755 if (err) { 756 dev_err(hwc->dev, "Failed to test HWC: %d\n", err); 757 goto out; 758 } 759 760 return 0; 761 out: 762 mana_hwc_destroy_channel(gc); 763 return err; 764 } 765 766 void mana_hwc_destroy_channel(struct gdma_context *gc) 767 { 768 struct hw_channel_context *hwc = gc->hwc.driver_data; 769 770 if (!hwc) 771 return; 772 773 /* gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's 774 * non-zero, the HWC worked and we should tear down the HWC here. 775 */ 776 if (gc->max_num_cqs > 0) { 777 mana_smc_teardown_hwc(&gc->shm_channel, false); 778 gc->max_num_cqs = 0; 779 } 780 781 kfree(hwc->caller_ctx); 782 hwc->caller_ctx = NULL; 783 784 if (hwc->txq) 785 mana_hwc_destroy_wq(hwc, hwc->txq); 786 787 if (hwc->rxq) 788 mana_hwc_destroy_wq(hwc, hwc->rxq); 789 790 if (hwc->cq) 791 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); 792 793 mana_gd_free_res_map(&hwc->inflight_msg_res); 794 795 hwc->num_inflight_msg = 0; 796 797 hwc->gdma_dev->doorbell = INVALID_DOORBELL; 798 hwc->gdma_dev->pdid = INVALID_PDID; 799 800 hwc->hwc_timeout = 0; 801 802 kfree(hwc); 803 gc->hwc.driver_data = NULL; 804 gc->hwc.gdma_context = NULL; 805 806 vfree(gc->cq_table); 807 gc->cq_table = NULL; 808 } 809 810 int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, 811 const void *req, u32 resp_len, void *resp) 812 { 813 struct gdma_context *gc = hwc->gdma_dev->gdma_context; 814 struct hwc_work_request *tx_wr; 815 struct hwc_wq *txq = hwc->txq; 816 struct gdma_req_hdr *req_msg; 817 struct hwc_caller_ctx *ctx; 818 u32 dest_vrcq = 0; 819 u32 dest_vrq = 0; 820 u16 msg_id; 821 int err; 822 823 mana_hwc_get_msg_index(hwc, &msg_id); 824 825 tx_wr = &txq->msg_buf->reqs[msg_id]; 826 827 if (req_len > tx_wr->buf_len) { 828 dev_err(hwc->dev, "HWC: req msg size: %d > %d\n", req_len, 829 tx_wr->buf_len); 830 err = -EINVAL; 831 goto out; 832 } 833 834 ctx = hwc->caller_ctx + msg_id; 835 ctx->output_buf = resp; 836 ctx->output_buflen = resp_len; 837 838 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; 839 if (req) 840 memcpy(req_msg, req, req_len); 841 842 req_msg->req.hwc_msg_id = msg_id; 843 844 tx_wr->msg_size = req_len; 845 846 if (gc->is_pf) { 847 dest_vrq = hwc->pf_dest_vrq_id; 848 dest_vrcq = hwc->pf_dest_vrcq_id; 849 } 850 851 err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false); 852 if (err) { 853 dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err); 854 goto out; 855 } 856 857 if (!wait_for_completion_timeout(&ctx->comp_event, 858 (msecs_to_jiffies(hwc->hwc_timeout)))) { 859 dev_err(hwc->dev, "HWC: Request timed out!\n"); 860 err = -ETIMEDOUT; 861 goto out; 862 } 863 864 if (ctx->error) { 865 err = ctx->error; 866 goto out; 867 } 868 869 if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { 870 dev_err(hwc->dev, "HWC: Failed hw_channel req: 0x%x\n", 871 ctx->status_code); 872 err = -EPROTO; 873 goto out; 874 } 875 out: 876 mana_hwc_put_msg_index(hwc, msg_id); 877 return err; 878 } 879