1ce110ea1SWei Hu /*- 2ce110ea1SWei Hu * SPDX-License-Identifier: BSD-2-Clause 3ce110ea1SWei Hu * 4ce110ea1SWei Hu * Copyright (c) 2021 Microsoft Corp. 5ce110ea1SWei Hu * All rights reserved. 6ce110ea1SWei Hu * 7ce110ea1SWei Hu * Redistribution and use in source and binary forms, with or without 8ce110ea1SWei Hu * modification, are permitted provided that the following conditions 9ce110ea1SWei Hu * are met: 10ce110ea1SWei Hu * 11ce110ea1SWei Hu * 1. Redistributions of source code must retain the above copyright 12ce110ea1SWei Hu * notice, this list of conditions and the following disclaimer. 13ce110ea1SWei Hu * 14ce110ea1SWei Hu * 2. Redistributions in binary form must reproduce the above copyright 15ce110ea1SWei Hu * notice, this list of conditions and the following disclaimer in the 16ce110ea1SWei Hu * documentation and/or other materials provided with the distribution. 17ce110ea1SWei Hu * 18ce110ea1SWei Hu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19ce110ea1SWei Hu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20ce110ea1SWei Hu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21ce110ea1SWei Hu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22ce110ea1SWei Hu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23ce110ea1SWei Hu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24ce110ea1SWei Hu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25ce110ea1SWei Hu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26ce110ea1SWei Hu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27ce110ea1SWei Hu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28ce110ea1SWei Hu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29ce110ea1SWei Hu */ 30ce110ea1SWei Hu #include <sys/cdefs.h> 31ce110ea1SWei Hu __FBSDID("$FreeBSD$"); 32ce110ea1SWei Hu 33ce110ea1SWei Hu #include <sys/param.h> 34ce110ea1SWei Hu #include <sys/systm.h> 35ce110ea1SWei Hu #include <sys/types.h> 36ce110ea1SWei Hu #include <sys/kernel.h> 37ce110ea1SWei Hu #include <sys/kthread.h> 38ce110ea1SWei Hu #include <sys/lock.h> 39ce110ea1SWei Hu #include <sys/malloc.h> 40ce110ea1SWei Hu #include <sys/mutex.h> 41ce110ea1SWei Hu #include <sys/bus.h> 42ce110ea1SWei Hu #include <machine/bus.h> 43ce110ea1SWei Hu 44ce110ea1SWei Hu #include "mana.h" 45ce110ea1SWei Hu #include "hw_channel.h" 46ce110ea1SWei Hu 47ce110ea1SWei Hu static int 48ce110ea1SWei Hu mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id) 49ce110ea1SWei Hu { 50ce110ea1SWei Hu struct gdma_resource *r = &hwc->inflight_msg_res; 51ce110ea1SWei Hu uint32_t index; 52ce110ea1SWei Hu 53ce110ea1SWei Hu sema_wait(&hwc->sema); 54ce110ea1SWei Hu 55ce110ea1SWei Hu mtx_lock_spin(&r->lock_spin); 56ce110ea1SWei Hu 57ce110ea1SWei Hu index = find_first_zero_bit(hwc->inflight_msg_res.map, 58ce110ea1SWei Hu hwc->inflight_msg_res.size); 59ce110ea1SWei Hu 60ce110ea1SWei Hu bitmap_set(hwc->inflight_msg_res.map, index, 1); 61ce110ea1SWei Hu 62ce110ea1SWei Hu mtx_unlock_spin(&r->lock_spin); 63ce110ea1SWei Hu 64ce110ea1SWei Hu *msg_id = index; 65ce110ea1SWei Hu 66ce110ea1SWei Hu return 0; 67ce110ea1SWei Hu } 68ce110ea1SWei Hu 69ce110ea1SWei Hu static void 70ce110ea1SWei Hu mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id) 71ce110ea1SWei Hu { 72ce110ea1SWei Hu struct gdma_resource *r = &hwc->inflight_msg_res; 73ce110ea1SWei Hu 74ce110ea1SWei Hu mtx_lock_spin(&r->lock_spin); 75ce110ea1SWei Hu bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); 76ce110ea1SWei Hu mtx_unlock_spin(&r->lock_spin); 77ce110ea1SWei Hu 78ce110ea1SWei Hu sema_post(&hwc->sema); 79ce110ea1SWei Hu } 80ce110ea1SWei Hu 81ce110ea1SWei Hu static int 82ce110ea1SWei Hu mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, 83ce110ea1SWei Hu const struct gdma_resp_hdr *resp_msg, 84ce110ea1SWei Hu uint32_t resp_len) 85ce110ea1SWei Hu { 86ce110ea1SWei Hu if (resp_len < sizeof(*resp_msg)) 87ce110ea1SWei Hu return EPROTO; 88ce110ea1SWei Hu 89ce110ea1SWei Hu if (resp_len > caller_ctx->output_buflen) 90ce110ea1SWei Hu return EPROTO; 91ce110ea1SWei Hu 92ce110ea1SWei Hu return 0; 93ce110ea1SWei Hu } 94ce110ea1SWei Hu 95ce110ea1SWei Hu static void 96ce110ea1SWei Hu mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len, 97ce110ea1SWei Hu const struct gdma_resp_hdr *resp_msg) 98ce110ea1SWei Hu { 99ce110ea1SWei Hu struct hwc_caller_ctx *ctx; 100ce110ea1SWei Hu int err; 101ce110ea1SWei Hu 102ce110ea1SWei Hu if (!test_bit(resp_msg->response.hwc_msg_id, 103ce110ea1SWei Hu hwc->inflight_msg_res.map)) { 104ce110ea1SWei Hu device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n", 105ce110ea1SWei Hu resp_msg->response.hwc_msg_id); 106ce110ea1SWei Hu return; 107ce110ea1SWei Hu } 108ce110ea1SWei Hu 109ce110ea1SWei Hu ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; 110ce110ea1SWei Hu err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); 111ce110ea1SWei Hu if (err) 112ce110ea1SWei Hu goto out; 113ce110ea1SWei Hu 114ce110ea1SWei Hu ctx->status_code = resp_msg->status; 115ce110ea1SWei Hu 116ce110ea1SWei Hu memcpy(ctx->output_buf, resp_msg, resp_len); 117ce110ea1SWei Hu out: 118ce110ea1SWei Hu ctx->error = err; 119ce110ea1SWei Hu complete(&ctx->comp_event); 120ce110ea1SWei Hu } 121ce110ea1SWei Hu 122ce110ea1SWei Hu static int 123ce110ea1SWei Hu mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq, 124ce110ea1SWei Hu struct hwc_work_request *req) 125ce110ea1SWei Hu { 126ce110ea1SWei Hu device_t dev = hwc_rxq->hwc->dev; 127ce110ea1SWei Hu struct gdma_sge *sge; 128ce110ea1SWei Hu int err; 129ce110ea1SWei Hu 130ce110ea1SWei Hu sge = &req->sge; 131ce110ea1SWei Hu sge->address = (uint64_t)req->buf_sge_addr; 132ce110ea1SWei Hu sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; 133ce110ea1SWei Hu sge->size = req->buf_len; 134ce110ea1SWei Hu 135ce110ea1SWei Hu memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 136ce110ea1SWei Hu req->wqe_req.sgl = sge; 137ce110ea1SWei Hu req->wqe_req.num_sge = 1; 138ce110ea1SWei Hu req->wqe_req.client_data_unit = 0; 139ce110ea1SWei Hu 140ce110ea1SWei Hu err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); 141ce110ea1SWei Hu if (err) 142ce110ea1SWei Hu device_printf(dev, 143ce110ea1SWei Hu "Failed to post WQE on HWC RQ: %d\n", err); 144ce110ea1SWei Hu return err; 145ce110ea1SWei Hu } 146ce110ea1SWei Hu 147ce110ea1SWei Hu static void 148ce110ea1SWei Hu mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, 149ce110ea1SWei Hu struct gdma_event *event) 150ce110ea1SWei Hu { 151ce110ea1SWei Hu struct hw_channel_context *hwc = ctx; 152ce110ea1SWei Hu struct gdma_dev *gd = hwc->gdma_dev; 153ce110ea1SWei Hu union hwc_init_type_data type_data; 154ce110ea1SWei Hu union hwc_init_eq_id_db eq_db; 155ce110ea1SWei Hu uint32_t type, val; 156ce110ea1SWei Hu 157ce110ea1SWei Hu switch (event->type) { 158ce110ea1SWei Hu case GDMA_EQE_HWC_INIT_EQ_ID_DB: 159ce110ea1SWei Hu eq_db.as_uint32 = event->details[0]; 160ce110ea1SWei Hu hwc->cq->gdma_eq->id = eq_db.eq_id; 161ce110ea1SWei Hu gd->doorbell = eq_db.doorbell; 162ce110ea1SWei Hu break; 163ce110ea1SWei Hu 164ce110ea1SWei Hu case GDMA_EQE_HWC_INIT_DATA: 165ce110ea1SWei Hu type_data.as_uint32 = event->details[0]; 166ce110ea1SWei Hu type = type_data.type; 167ce110ea1SWei Hu val = type_data.value; 168ce110ea1SWei Hu 169ce110ea1SWei Hu switch (type) { 170ce110ea1SWei Hu case HWC_INIT_DATA_CQID: 171ce110ea1SWei Hu hwc->cq->gdma_cq->id = val; 172ce110ea1SWei Hu break; 173ce110ea1SWei Hu 174ce110ea1SWei Hu case HWC_INIT_DATA_RQID: 175ce110ea1SWei Hu hwc->rxq->gdma_wq->id = val; 176ce110ea1SWei Hu break; 177ce110ea1SWei Hu 178ce110ea1SWei Hu case HWC_INIT_DATA_SQID: 179ce110ea1SWei Hu hwc->txq->gdma_wq->id = val; 180ce110ea1SWei Hu break; 181ce110ea1SWei Hu 182ce110ea1SWei Hu case HWC_INIT_DATA_QUEUE_DEPTH: 183ce110ea1SWei Hu hwc->hwc_init_q_depth_max = (uint16_t)val; 184ce110ea1SWei Hu break; 185ce110ea1SWei Hu 186ce110ea1SWei Hu case HWC_INIT_DATA_MAX_REQUEST: 187ce110ea1SWei Hu hwc->hwc_init_max_req_msg_size = val; 188ce110ea1SWei Hu break; 189ce110ea1SWei Hu 190ce110ea1SWei Hu case HWC_INIT_DATA_MAX_RESPONSE: 191ce110ea1SWei Hu hwc->hwc_init_max_resp_msg_size = val; 192ce110ea1SWei Hu break; 193ce110ea1SWei Hu 194ce110ea1SWei Hu case HWC_INIT_DATA_MAX_NUM_CQS: 195ce110ea1SWei Hu gd->gdma_context->max_num_cqs = val; 196ce110ea1SWei Hu break; 197ce110ea1SWei Hu 198ce110ea1SWei Hu case HWC_INIT_DATA_PDID: 199ce110ea1SWei Hu hwc->gdma_dev->pdid = val; 200ce110ea1SWei Hu break; 201ce110ea1SWei Hu 202ce110ea1SWei Hu case HWC_INIT_DATA_GPA_MKEY: 203ce110ea1SWei Hu hwc->rxq->msg_buf->gpa_mkey = val; 204ce110ea1SWei Hu hwc->txq->msg_buf->gpa_mkey = val; 205ce110ea1SWei Hu break; 206ce110ea1SWei Hu } 207ce110ea1SWei Hu 208ce110ea1SWei Hu break; 209ce110ea1SWei Hu 210ce110ea1SWei Hu case GDMA_EQE_HWC_INIT_DONE: 211ce110ea1SWei Hu complete(&hwc->hwc_init_eqe_comp); 212ce110ea1SWei Hu break; 213ce110ea1SWei Hu 214ce110ea1SWei Hu default: 215ce110ea1SWei Hu /* Ignore unknown events, which should never happen. */ 216ce110ea1SWei Hu break; 217ce110ea1SWei Hu } 218ce110ea1SWei Hu } 219ce110ea1SWei Hu 220ce110ea1SWei Hu static void 221ce110ea1SWei Hu mana_hwc_rx_event_handler(void *ctx, uint32_t gdma_rxq_id, 222ce110ea1SWei Hu const struct hwc_rx_oob *rx_oob) 223ce110ea1SWei Hu { 224ce110ea1SWei Hu struct hw_channel_context *hwc = ctx; 225ce110ea1SWei Hu struct hwc_wq *hwc_rxq = hwc->rxq; 226ce110ea1SWei Hu struct hwc_work_request *rx_req; 227ce110ea1SWei Hu struct gdma_resp_hdr *resp; 228ce110ea1SWei Hu struct gdma_wqe *dma_oob; 229ce110ea1SWei Hu struct gdma_queue *rq; 230ce110ea1SWei Hu struct gdma_sge *sge; 231ce110ea1SWei Hu uint64_t rq_base_addr; 232ce110ea1SWei Hu uint64_t rx_req_idx; 233ce110ea1SWei Hu uint8_t *wqe; 234ce110ea1SWei Hu 235ce110ea1SWei Hu if (hwc_rxq->gdma_wq->id != gdma_rxq_id) { 236ce110ea1SWei Hu mana_warn(NULL, "unmatched rx queue %u != %u\n", 237ce110ea1SWei Hu hwc_rxq->gdma_wq->id, gdma_rxq_id); 238ce110ea1SWei Hu return; 239ce110ea1SWei Hu } 240ce110ea1SWei Hu 241ce110ea1SWei Hu 242ce110ea1SWei Hu rq = hwc_rxq->gdma_wq; 243ce110ea1SWei Hu wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); 244ce110ea1SWei Hu dma_oob = (struct gdma_wqe *)wqe; 245ce110ea1SWei Hu 246ce110ea1SWei Hu bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map, 247ce110ea1SWei Hu BUS_DMASYNC_POSTREAD); 248ce110ea1SWei Hu 249ce110ea1SWei Hu sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); 250ce110ea1SWei Hu 251ce110ea1SWei Hu /* Select the RX work request for virtual address and for reposting. */ 252ce110ea1SWei Hu rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; 253ce110ea1SWei Hu rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; 254ce110ea1SWei Hu 255ce110ea1SWei Hu bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 256ce110ea1SWei Hu hwc_rxq->msg_buf->mem_info.dma_map, 257ce110ea1SWei Hu BUS_DMASYNC_POSTREAD); 258ce110ea1SWei Hu 259ce110ea1SWei Hu rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; 260ce110ea1SWei Hu resp = (struct gdma_resp_hdr *)rx_req->buf_va; 261ce110ea1SWei Hu 262ce110ea1SWei Hu if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { 263ce110ea1SWei Hu device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n", 264ce110ea1SWei Hu resp->response.hwc_msg_id); 265ce110ea1SWei Hu return; 266ce110ea1SWei Hu } 267ce110ea1SWei Hu 268ce110ea1SWei Hu mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp); 269ce110ea1SWei Hu 270ce110ea1SWei Hu /* Do no longer use 'resp', because the buffer is posted to the HW 271ce110ea1SWei Hu * in the below mana_hwc_post_rx_wqe(). 272ce110ea1SWei Hu */ 273ce110ea1SWei Hu resp = NULL; 274ce110ea1SWei Hu 275ce110ea1SWei Hu bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, 276ce110ea1SWei Hu hwc_rxq->msg_buf->mem_info.dma_map, 277ce110ea1SWei Hu BUS_DMASYNC_PREREAD); 278ce110ea1SWei Hu 279ce110ea1SWei Hu mana_hwc_post_rx_wqe(hwc_rxq, rx_req); 280ce110ea1SWei Hu } 281ce110ea1SWei Hu 282ce110ea1SWei Hu static void 283ce110ea1SWei Hu mana_hwc_tx_event_handler(void *ctx, uint32_t gdma_txq_id, 284ce110ea1SWei Hu const struct hwc_rx_oob *rx_oob) 285ce110ea1SWei Hu { 286ce110ea1SWei Hu struct hw_channel_context *hwc = ctx; 287ce110ea1SWei Hu struct hwc_wq *hwc_txq = hwc->txq; 288ce110ea1SWei Hu 289ce110ea1SWei Hu if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) { 290ce110ea1SWei Hu mana_warn(NULL, "unmatched tx queue %u != %u\n", 291ce110ea1SWei Hu hwc_txq->gdma_wq->id, gdma_txq_id); 292ce110ea1SWei Hu } 293ce110ea1SWei Hu 294ce110ea1SWei Hu bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag, 295ce110ea1SWei Hu hwc_txq->gdma_wq->mem_info.dma_map, 296ce110ea1SWei Hu BUS_DMASYNC_POSTWRITE); 297ce110ea1SWei Hu } 298ce110ea1SWei Hu 299ce110ea1SWei Hu static int 300ce110ea1SWei Hu mana_hwc_create_gdma_wq(struct hw_channel_context *hwc, 301ce110ea1SWei Hu enum gdma_queue_type type, uint64_t queue_size, 302ce110ea1SWei Hu struct gdma_queue **queue) 303ce110ea1SWei Hu { 304ce110ea1SWei Hu struct gdma_queue_spec spec = {}; 305ce110ea1SWei Hu 306ce110ea1SWei Hu if (type != GDMA_SQ && type != GDMA_RQ) 307ce110ea1SWei Hu return EINVAL; 308ce110ea1SWei Hu 309ce110ea1SWei Hu spec.type = type; 310ce110ea1SWei Hu spec.monitor_avl_buf = false; 311ce110ea1SWei Hu spec.queue_size = queue_size; 312ce110ea1SWei Hu 313ce110ea1SWei Hu return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 314ce110ea1SWei Hu } 315ce110ea1SWei Hu 316ce110ea1SWei Hu static int 317ce110ea1SWei Hu mana_hwc_create_gdma_cq(struct hw_channel_context *hwc, 318ce110ea1SWei Hu uint64_t queue_size, 319ce110ea1SWei Hu void *ctx, gdma_cq_callback *cb, 320ce110ea1SWei Hu struct gdma_queue *parent_eq, 321ce110ea1SWei Hu struct gdma_queue **queue) 322ce110ea1SWei Hu { 323ce110ea1SWei Hu struct gdma_queue_spec spec = {}; 324ce110ea1SWei Hu 325ce110ea1SWei Hu spec.type = GDMA_CQ; 326ce110ea1SWei Hu spec.monitor_avl_buf = false; 327ce110ea1SWei Hu spec.queue_size = queue_size; 328ce110ea1SWei Hu spec.cq.context = ctx; 329ce110ea1SWei Hu spec.cq.callback = cb; 330ce110ea1SWei Hu spec.cq.parent_eq = parent_eq; 331ce110ea1SWei Hu 332ce110ea1SWei Hu return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 333ce110ea1SWei Hu } 334ce110ea1SWei Hu 335ce110ea1SWei Hu static int 336ce110ea1SWei Hu mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, 337ce110ea1SWei Hu uint64_t queue_size, 338ce110ea1SWei Hu void *ctx, gdma_eq_callback *cb, 339ce110ea1SWei Hu struct gdma_queue **queue) 340ce110ea1SWei Hu { 341ce110ea1SWei Hu struct gdma_queue_spec spec = {}; 342ce110ea1SWei Hu 343ce110ea1SWei Hu spec.type = GDMA_EQ; 344ce110ea1SWei Hu spec.monitor_avl_buf = false; 345ce110ea1SWei Hu spec.queue_size = queue_size; 346ce110ea1SWei Hu spec.eq.context = ctx; 347ce110ea1SWei Hu spec.eq.callback = cb; 348ce110ea1SWei Hu spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; 349ce110ea1SWei Hu 350ce110ea1SWei Hu return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); 351ce110ea1SWei Hu } 352ce110ea1SWei Hu 353ce110ea1SWei Hu static void 354ce110ea1SWei Hu mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self) 355ce110ea1SWei Hu { 356ce110ea1SWei Hu struct hwc_rx_oob comp_data = {}; 357ce110ea1SWei Hu struct gdma_comp *completions; 358ce110ea1SWei Hu struct hwc_cq *hwc_cq = ctx; 359ce110ea1SWei Hu int comp_read, i; 360ce110ea1SWei Hu 361ce110ea1SWei Hu completions = hwc_cq->comp_buf; 362ce110ea1SWei Hu comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); 363ce110ea1SWei Hu 364ce110ea1SWei Hu for (i = 0; i < comp_read; ++i) { 365ce110ea1SWei Hu comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data; 366ce110ea1SWei Hu 367ce110ea1SWei Hu if (completions[i].is_sq) 368ce110ea1SWei Hu hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, 369ce110ea1SWei Hu completions[i].wq_num, 370ce110ea1SWei Hu &comp_data); 371ce110ea1SWei Hu else 372ce110ea1SWei Hu hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, 373ce110ea1SWei Hu completions[i].wq_num, 374ce110ea1SWei Hu &comp_data); 375ce110ea1SWei Hu } 376ce110ea1SWei Hu 377ce110ea1SWei Hu bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map, 378ce110ea1SWei Hu BUS_DMASYNC_POSTREAD); 379ce110ea1SWei Hu 380*1833cf13SWei Hu mana_gd_ring_cq(q_self, SET_ARM_BIT); 381ce110ea1SWei Hu } 382ce110ea1SWei Hu 383ce110ea1SWei Hu static void 384ce110ea1SWei Hu mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq) 385ce110ea1SWei Hu { 386ce110ea1SWei Hu if (!hwc_cq) 387ce110ea1SWei Hu return; 388ce110ea1SWei Hu 389ce110ea1SWei Hu if (hwc_cq->comp_buf) 390ce110ea1SWei Hu free(hwc_cq->comp_buf, M_DEVBUF); 391ce110ea1SWei Hu 392ce110ea1SWei Hu if (hwc_cq->gdma_cq) 393ce110ea1SWei Hu mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); 394ce110ea1SWei Hu 395ce110ea1SWei Hu if (hwc_cq->gdma_eq) 396ce110ea1SWei Hu mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); 397ce110ea1SWei Hu 398ce110ea1SWei Hu free(hwc_cq, M_DEVBUF); 399ce110ea1SWei Hu } 400ce110ea1SWei Hu 401ce110ea1SWei Hu static int 402ce110ea1SWei Hu mana_hwc_create_cq(struct hw_channel_context *hwc, 403ce110ea1SWei Hu uint16_t q_depth, 404ce110ea1SWei Hu gdma_eq_callback *callback, void *ctx, 405ce110ea1SWei Hu hwc_rx_event_handler_t *rx_ev_hdlr, void *rx_ev_ctx, 406ce110ea1SWei Hu hwc_tx_event_handler_t *tx_ev_hdlr, void *tx_ev_ctx, 407ce110ea1SWei Hu struct hwc_cq **hwc_cq_ptr) 408ce110ea1SWei Hu { 409ce110ea1SWei Hu struct gdma_queue *eq, *cq; 410ce110ea1SWei Hu struct gdma_comp *comp_buf; 411ce110ea1SWei Hu struct hwc_cq *hwc_cq; 412ce110ea1SWei Hu uint32_t eq_size, cq_size; 413ce110ea1SWei Hu int err; 414ce110ea1SWei Hu 415ce110ea1SWei Hu eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth); 416ce110ea1SWei Hu if (eq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 417ce110ea1SWei Hu eq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 418ce110ea1SWei Hu 419ce110ea1SWei Hu cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth); 420ce110ea1SWei Hu if (cq_size < MINIMUM_SUPPORTED_PAGE_SIZE) 421ce110ea1SWei Hu cq_size = MINIMUM_SUPPORTED_PAGE_SIZE; 422ce110ea1SWei Hu 423ce110ea1SWei Hu hwc_cq = malloc(sizeof(*hwc_cq), M_DEVBUF, M_WAITOK | M_ZERO); 424ce110ea1SWei Hu if (!hwc_cq) 425ce110ea1SWei Hu return ENOMEM; 426ce110ea1SWei Hu 427ce110ea1SWei Hu err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); 428ce110ea1SWei Hu if (err) { 429ce110ea1SWei Hu device_printf(hwc->dev, 430ce110ea1SWei Hu "Failed to create HWC EQ for RQ: %d\n", err); 431ce110ea1SWei Hu goto out; 432ce110ea1SWei Hu } 433ce110ea1SWei Hu hwc_cq->gdma_eq = eq; 434ce110ea1SWei Hu 435ce110ea1SWei Hu err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, 436ce110ea1SWei Hu mana_hwc_comp_event, eq, &cq); 437ce110ea1SWei Hu if (err) { 438ce110ea1SWei Hu device_printf(hwc->dev, 439ce110ea1SWei Hu "Failed to create HWC CQ for RQ: %d\n", err); 440ce110ea1SWei Hu goto out; 441ce110ea1SWei Hu } 442ce110ea1SWei Hu hwc_cq->gdma_cq = cq; 443ce110ea1SWei Hu 444ce110ea1SWei Hu comp_buf = mallocarray(q_depth, sizeof(struct gdma_comp), 445ce110ea1SWei Hu M_DEVBUF, M_WAITOK | M_ZERO); 446ce110ea1SWei Hu if (!comp_buf) { 447ce110ea1SWei Hu err = ENOMEM; 448ce110ea1SWei Hu goto out; 449ce110ea1SWei Hu } 450ce110ea1SWei Hu 451ce110ea1SWei Hu hwc_cq->hwc = hwc; 452ce110ea1SWei Hu hwc_cq->comp_buf = comp_buf; 453ce110ea1SWei Hu hwc_cq->queue_depth = q_depth; 454ce110ea1SWei Hu hwc_cq->rx_event_handler = rx_ev_hdlr; 455ce110ea1SWei Hu hwc_cq->rx_event_ctx = rx_ev_ctx; 456ce110ea1SWei Hu hwc_cq->tx_event_handler = tx_ev_hdlr; 457ce110ea1SWei Hu hwc_cq->tx_event_ctx = tx_ev_ctx; 458ce110ea1SWei Hu 459ce110ea1SWei Hu *hwc_cq_ptr = hwc_cq; 460ce110ea1SWei Hu return 0; 461ce110ea1SWei Hu out: 462ce110ea1SWei Hu mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); 463ce110ea1SWei Hu return err; 464ce110ea1SWei Hu } 465ce110ea1SWei Hu 466ce110ea1SWei Hu static int 467ce110ea1SWei Hu mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth, 468ce110ea1SWei Hu uint32_t max_msg_size, 469ce110ea1SWei Hu struct hwc_dma_buf **dma_buf_ptr) 470ce110ea1SWei Hu { 471ce110ea1SWei Hu struct gdma_context *gc = hwc->gdma_dev->gdma_context; 472ce110ea1SWei Hu struct hwc_work_request *hwc_wr; 473ce110ea1SWei Hu struct hwc_dma_buf *dma_buf; 474ce110ea1SWei Hu struct gdma_mem_info *gmi; 475ce110ea1SWei Hu uint32_t buf_size; 476ce110ea1SWei Hu uint8_t *base_pa; 477ce110ea1SWei Hu void *virt_addr; 478ce110ea1SWei Hu uint16_t i; 479ce110ea1SWei Hu int err; 480ce110ea1SWei Hu 481ce110ea1SWei Hu dma_buf = malloc(sizeof(*dma_buf) + 482ce110ea1SWei Hu q_depth * sizeof(struct hwc_work_request), 483ce110ea1SWei Hu M_DEVBUF, M_WAITOK | M_ZERO); 484ce110ea1SWei Hu if (!dma_buf) 485ce110ea1SWei Hu return ENOMEM; 486ce110ea1SWei Hu 487ce110ea1SWei Hu dma_buf->num_reqs = q_depth; 488ce110ea1SWei Hu 489ce110ea1SWei Hu buf_size = ALIGN(q_depth * max_msg_size, PAGE_SIZE); 490ce110ea1SWei Hu 491ce110ea1SWei Hu gmi = &dma_buf->mem_info; 492ce110ea1SWei Hu err = mana_gd_alloc_memory(gc, buf_size, gmi); 493ce110ea1SWei Hu if (err) { 494ce110ea1SWei Hu device_printf(hwc->dev, 495ce110ea1SWei Hu "Failed to allocate DMA buffer: %d\n", err); 496ce110ea1SWei Hu goto out; 497ce110ea1SWei Hu } 498ce110ea1SWei Hu 499ce110ea1SWei Hu virt_addr = dma_buf->mem_info.virt_addr; 500ce110ea1SWei Hu base_pa = (uint8_t *)dma_buf->mem_info.dma_handle; 501ce110ea1SWei Hu 502ce110ea1SWei Hu for (i = 0; i < q_depth; i++) { 503ce110ea1SWei Hu hwc_wr = &dma_buf->reqs[i]; 504ce110ea1SWei Hu 505ce110ea1SWei Hu hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size; 506ce110ea1SWei Hu hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; 507ce110ea1SWei Hu 508ce110ea1SWei Hu hwc_wr->buf_len = max_msg_size; 509ce110ea1SWei Hu } 510ce110ea1SWei Hu 511ce110ea1SWei Hu *dma_buf_ptr = dma_buf; 512ce110ea1SWei Hu return 0; 513ce110ea1SWei Hu out: 514ce110ea1SWei Hu free(dma_buf, M_DEVBUF); 515ce110ea1SWei Hu return err; 516ce110ea1SWei Hu } 517ce110ea1SWei Hu 518ce110ea1SWei Hu static void 519ce110ea1SWei Hu mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc, 520ce110ea1SWei Hu struct hwc_dma_buf *dma_buf) 521ce110ea1SWei Hu { 522ce110ea1SWei Hu if (!dma_buf) 523ce110ea1SWei Hu return; 524ce110ea1SWei Hu 525ce110ea1SWei Hu mana_gd_free_memory(&dma_buf->mem_info); 526ce110ea1SWei Hu 527ce110ea1SWei Hu free(dma_buf, M_DEVBUF); 528ce110ea1SWei Hu } 529ce110ea1SWei Hu 530ce110ea1SWei Hu static void 531ce110ea1SWei Hu mana_hwc_destroy_wq(struct hw_channel_context *hwc, 532ce110ea1SWei Hu struct hwc_wq *hwc_wq) 533ce110ea1SWei Hu { 534ce110ea1SWei Hu if (!hwc_wq) 535ce110ea1SWei Hu return; 536ce110ea1SWei Hu 537ce110ea1SWei Hu mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); 538ce110ea1SWei Hu 539ce110ea1SWei Hu if (hwc_wq->gdma_wq) 540ce110ea1SWei Hu mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, 541ce110ea1SWei Hu hwc_wq->gdma_wq); 542ce110ea1SWei Hu 543ce110ea1SWei Hu free(hwc_wq, M_DEVBUF); 544ce110ea1SWei Hu } 545ce110ea1SWei Hu 546ce110ea1SWei Hu static int 547ce110ea1SWei Hu mana_hwc_create_wq(struct hw_channel_context *hwc, 548ce110ea1SWei Hu enum gdma_queue_type q_type, uint16_t q_depth, 549ce110ea1SWei Hu uint32_t max_msg_size, struct hwc_cq *hwc_cq, 550ce110ea1SWei Hu struct hwc_wq **hwc_wq_ptr) 551ce110ea1SWei Hu { 552ce110ea1SWei Hu struct gdma_queue *queue; 553ce110ea1SWei Hu struct hwc_wq *hwc_wq; 554ce110ea1SWei Hu uint32_t queue_size; 555ce110ea1SWei Hu int err; 556ce110ea1SWei Hu 557ce110ea1SWei Hu if (q_type != GDMA_SQ && q_type != GDMA_RQ) { 558ce110ea1SWei Hu /* XXX should fail and return error? */ 559ce110ea1SWei Hu mana_warn(NULL, "Invalid q_type %u\n", q_type); 560ce110ea1SWei Hu } 561ce110ea1SWei Hu 562ce110ea1SWei Hu if (q_type == GDMA_RQ) 563ce110ea1SWei Hu queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth); 564ce110ea1SWei Hu else 565ce110ea1SWei Hu queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth); 566ce110ea1SWei Hu 567ce110ea1SWei Hu if (queue_size < MINIMUM_SUPPORTED_PAGE_SIZE) 568ce110ea1SWei Hu queue_size = MINIMUM_SUPPORTED_PAGE_SIZE; 569ce110ea1SWei Hu 570ce110ea1SWei Hu hwc_wq = malloc(sizeof(*hwc_wq), M_DEVBUF, M_WAITOK | M_ZERO); 571ce110ea1SWei Hu if (!hwc_wq) 572ce110ea1SWei Hu return ENOMEM; 573ce110ea1SWei Hu 574ce110ea1SWei Hu err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue); 575ce110ea1SWei Hu if (err) 576ce110ea1SWei Hu goto out; 577ce110ea1SWei Hu 578ce110ea1SWei Hu err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, 579ce110ea1SWei Hu &hwc_wq->msg_buf); 580ce110ea1SWei Hu if (err) 581ce110ea1SWei Hu goto out; 582ce110ea1SWei Hu 583ce110ea1SWei Hu hwc_wq->hwc = hwc; 584ce110ea1SWei Hu hwc_wq->gdma_wq = queue; 585ce110ea1SWei Hu hwc_wq->queue_depth = q_depth; 586ce110ea1SWei Hu hwc_wq->hwc_cq = hwc_cq; 587ce110ea1SWei Hu 588ce110ea1SWei Hu *hwc_wq_ptr = hwc_wq; 589ce110ea1SWei Hu return 0; 590ce110ea1SWei Hu out: 591ce110ea1SWei Hu if (err) 592ce110ea1SWei Hu mana_hwc_destroy_wq(hwc, hwc_wq); 593ce110ea1SWei Hu return err; 594ce110ea1SWei Hu } 595ce110ea1SWei Hu 596ce110ea1SWei Hu static int 597ce110ea1SWei Hu mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq, 598ce110ea1SWei Hu struct hwc_work_request *req, 599ce110ea1SWei Hu uint32_t dest_virt_rq_id, uint32_t dest_virt_rcq_id, 600ce110ea1SWei Hu bool dest_pf) 601ce110ea1SWei Hu { 602ce110ea1SWei Hu device_t dev = hwc_txq->hwc->dev; 603ce110ea1SWei Hu struct hwc_tx_oob *tx_oob; 604ce110ea1SWei Hu struct gdma_sge *sge; 605ce110ea1SWei Hu int err; 606ce110ea1SWei Hu 607ce110ea1SWei Hu if (req->msg_size == 0 || req->msg_size > req->buf_len) { 608ce110ea1SWei Hu device_printf(dev, "wrong msg_size: %u, buf_len: %u\n", 609ce110ea1SWei Hu req->msg_size, req->buf_len); 610ce110ea1SWei Hu return EINVAL; 611ce110ea1SWei Hu } 612ce110ea1SWei Hu 613ce110ea1SWei Hu tx_oob = &req->tx_oob; 614ce110ea1SWei Hu 615ce110ea1SWei Hu tx_oob->vrq_id = dest_virt_rq_id; 616ce110ea1SWei Hu tx_oob->dest_vfid = 0; 617ce110ea1SWei Hu tx_oob->vrcq_id = dest_virt_rcq_id; 618ce110ea1SWei Hu tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; 619ce110ea1SWei Hu tx_oob->loopback = false; 620ce110ea1SWei Hu tx_oob->lso_override = false; 621ce110ea1SWei Hu tx_oob->dest_pf = dest_pf; 622ce110ea1SWei Hu tx_oob->vsq_id = hwc_txq->gdma_wq->id; 623ce110ea1SWei Hu 624ce110ea1SWei Hu sge = &req->sge; 625ce110ea1SWei Hu sge->address = (uint64_t)req->buf_sge_addr; 626ce110ea1SWei Hu sge->mem_key = hwc_txq->msg_buf->gpa_mkey; 627ce110ea1SWei Hu sge->size = req->msg_size; 628ce110ea1SWei Hu 629ce110ea1SWei Hu memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); 630ce110ea1SWei Hu req->wqe_req.sgl = sge; 631ce110ea1SWei Hu req->wqe_req.num_sge = 1; 632ce110ea1SWei Hu req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); 633ce110ea1SWei Hu req->wqe_req.inline_oob_data = tx_oob; 634ce110ea1SWei Hu req->wqe_req.client_data_unit = 0; 635ce110ea1SWei Hu 636ce110ea1SWei Hu err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); 637ce110ea1SWei Hu if (err) 638ce110ea1SWei Hu device_printf(dev, 639ce110ea1SWei Hu "Failed to post WQE on HWC SQ: %d\n", err); 640ce110ea1SWei Hu return err; 641ce110ea1SWei Hu } 642ce110ea1SWei Hu 643ce110ea1SWei Hu static int 644ce110ea1SWei Hu mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg) 645ce110ea1SWei Hu { 646ce110ea1SWei Hu int err; 647ce110ea1SWei Hu 648ce110ea1SWei Hu sema_init(&hwc->sema, num_msg, "gdma hwc sema"); 649ce110ea1SWei Hu 650ce110ea1SWei Hu err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res, 651ce110ea1SWei Hu "gdma hwc res lock"); 652ce110ea1SWei Hu if (err) 653ce110ea1SWei Hu device_printf(hwc->dev, 654ce110ea1SWei Hu "Failed to init inflight_msg_res: %d\n", err); 655ce110ea1SWei Hu 656ce110ea1SWei Hu return (err); 657ce110ea1SWei Hu } 658ce110ea1SWei Hu 659ce110ea1SWei Hu static int 660ce110ea1SWei Hu mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth, 661ce110ea1SWei Hu uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 662ce110ea1SWei Hu { 663ce110ea1SWei Hu struct gdma_context *gc = hwc->gdma_dev->gdma_context; 664ce110ea1SWei Hu struct hwc_wq *hwc_rxq = hwc->rxq; 665ce110ea1SWei Hu struct hwc_work_request *req; 666ce110ea1SWei Hu struct hwc_caller_ctx *ctx; 667ce110ea1SWei Hu int err; 668ce110ea1SWei Hu int i; 669ce110ea1SWei Hu 670ce110ea1SWei Hu /* Post all WQEs on the RQ */ 671ce110ea1SWei Hu for (i = 0; i < q_depth; i++) { 672ce110ea1SWei Hu req = &hwc_rxq->msg_buf->reqs[i]; 673ce110ea1SWei Hu err = mana_hwc_post_rx_wqe(hwc_rxq, req); 674ce110ea1SWei Hu if (err) 675ce110ea1SWei Hu return err; 676ce110ea1SWei Hu } 677ce110ea1SWei Hu 678ce110ea1SWei Hu ctx = malloc(q_depth * sizeof(struct hwc_caller_ctx), 679ce110ea1SWei Hu M_DEVBUF, M_WAITOK | M_ZERO); 680ce110ea1SWei Hu if (!ctx) 681ce110ea1SWei Hu return ENOMEM; 682ce110ea1SWei Hu 683ce110ea1SWei Hu for (i = 0; i < q_depth; ++i) 684ce110ea1SWei Hu init_completion(&ctx[i].comp_event); 685ce110ea1SWei Hu 686ce110ea1SWei Hu hwc->caller_ctx = ctx; 687ce110ea1SWei Hu 688ce110ea1SWei Hu return mana_gd_test_eq(gc, hwc->cq->gdma_eq); 689ce110ea1SWei Hu } 690ce110ea1SWei Hu 691ce110ea1SWei Hu static int 692ce110ea1SWei Hu mana_hwc_establish_channel(struct gdma_context *gc, uint16_t *q_depth, 693ce110ea1SWei Hu uint32_t *max_req_msg_size, 694ce110ea1SWei Hu uint32_t *max_resp_msg_size) 695ce110ea1SWei Hu { 696ce110ea1SWei Hu struct hw_channel_context *hwc = gc->hwc.driver_data; 697ce110ea1SWei Hu struct gdma_queue *rq = hwc->rxq->gdma_wq; 698ce110ea1SWei Hu struct gdma_queue *sq = hwc->txq->gdma_wq; 699ce110ea1SWei Hu struct gdma_queue *eq = hwc->cq->gdma_eq; 700ce110ea1SWei Hu struct gdma_queue *cq = hwc->cq->gdma_cq; 701ce110ea1SWei Hu int err; 702ce110ea1SWei Hu 703ce110ea1SWei Hu init_completion(&hwc->hwc_init_eqe_comp); 704ce110ea1SWei Hu 705ce110ea1SWei Hu err = mana_smc_setup_hwc(&gc->shm_channel, false, 706ce110ea1SWei Hu eq->mem_info.dma_handle, 707ce110ea1SWei Hu cq->mem_info.dma_handle, 708ce110ea1SWei Hu rq->mem_info.dma_handle, 709ce110ea1SWei Hu sq->mem_info.dma_handle, 710ce110ea1SWei Hu eq->eq.msix_index); 711ce110ea1SWei Hu if (err) 712ce110ea1SWei Hu return err; 713ce110ea1SWei Hu 714ce110ea1SWei Hu if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz)) 715ce110ea1SWei Hu return ETIMEDOUT; 716ce110ea1SWei Hu 717ce110ea1SWei Hu *q_depth = hwc->hwc_init_q_depth_max; 718ce110ea1SWei Hu *max_req_msg_size = hwc->hwc_init_max_req_msg_size; 719ce110ea1SWei Hu *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; 720ce110ea1SWei Hu 721ce110ea1SWei Hu if (cq->id >= gc->max_num_cqs) { 722ce110ea1SWei Hu mana_warn(NULL, "invalid cq id %u > %u\n", 723ce110ea1SWei Hu cq->id, gc->max_num_cqs); 724ce110ea1SWei Hu return EPROTO; 725ce110ea1SWei Hu } 726ce110ea1SWei Hu 727ce110ea1SWei Hu gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *), 728ce110ea1SWei Hu M_DEVBUF, M_WAITOK | M_ZERO); 729ce110ea1SWei Hu if (!gc->cq_table) 730ce110ea1SWei Hu return ENOMEM; 731ce110ea1SWei Hu 732ce110ea1SWei Hu gc->cq_table[cq->id] = cq; 733ce110ea1SWei Hu 734ce110ea1SWei Hu return 0; 735ce110ea1SWei Hu } 736ce110ea1SWei Hu 737ce110ea1SWei Hu static int 738ce110ea1SWei Hu mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth, 739ce110ea1SWei Hu uint32_t max_req_msg_size, uint32_t max_resp_msg_size) 740ce110ea1SWei Hu { 741ce110ea1SWei Hu struct hwc_wq *hwc_rxq = NULL; 742ce110ea1SWei Hu struct hwc_wq *hwc_txq = NULL; 743ce110ea1SWei Hu struct hwc_cq *hwc_cq = NULL; 744ce110ea1SWei Hu int err; 745ce110ea1SWei Hu 746ce110ea1SWei Hu err = mana_hwc_init_inflight_msg(hwc, q_depth); 747ce110ea1SWei Hu if (err) 748ce110ea1SWei Hu return err; 749ce110ea1SWei Hu 750ce110ea1SWei Hu /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ 751ce110ea1SWei Hu * queue depth and RQ queue depth. 752ce110ea1SWei Hu */ 753ce110ea1SWei Hu err = mana_hwc_create_cq(hwc, q_depth * 2, 754ce110ea1SWei Hu mana_hwc_init_event_handler, hwc, 755ce110ea1SWei Hu mana_hwc_rx_event_handler, hwc, 756ce110ea1SWei Hu mana_hwc_tx_event_handler, hwc, &hwc_cq); 757ce110ea1SWei Hu if (err) { 758ce110ea1SWei Hu device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err); 759ce110ea1SWei Hu goto out; 760ce110ea1SWei Hu } 761ce110ea1SWei Hu hwc->cq = hwc_cq; 762ce110ea1SWei Hu 763ce110ea1SWei Hu err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, 764ce110ea1SWei Hu hwc_cq, &hwc_rxq); 765ce110ea1SWei Hu if (err) { 766ce110ea1SWei Hu device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err); 767ce110ea1SWei Hu goto out; 768ce110ea1SWei Hu } 769ce110ea1SWei Hu hwc->rxq = hwc_rxq; 770ce110ea1SWei Hu 771ce110ea1SWei Hu err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, 772ce110ea1SWei Hu hwc_cq, &hwc_txq); 773ce110ea1SWei Hu if (err) { 774ce110ea1SWei Hu device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err); 775ce110ea1SWei Hu goto out; 776ce110ea1SWei Hu } 777ce110ea1SWei Hu hwc->txq = hwc_txq; 778ce110ea1SWei Hu 779ce110ea1SWei Hu hwc->num_inflight_msg = q_depth; 780ce110ea1SWei Hu hwc->max_req_msg_size = max_req_msg_size; 781ce110ea1SWei Hu 782ce110ea1SWei Hu return 0; 783ce110ea1SWei Hu out: 784ce110ea1SWei Hu if (hwc_txq) 785ce110ea1SWei Hu mana_hwc_destroy_wq(hwc, hwc_txq); 786ce110ea1SWei Hu 787ce110ea1SWei Hu if (hwc_rxq) 788ce110ea1SWei Hu mana_hwc_destroy_wq(hwc, hwc_rxq); 789ce110ea1SWei Hu 790ce110ea1SWei Hu if (hwc_cq) 791ce110ea1SWei Hu mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); 792ce110ea1SWei Hu 793ce110ea1SWei Hu mana_gd_free_res_map(&hwc->inflight_msg_res); 794ce110ea1SWei Hu return err; 795ce110ea1SWei Hu } 796ce110ea1SWei Hu 797ce110ea1SWei Hu int 798ce110ea1SWei Hu mana_hwc_create_channel(struct gdma_context *gc) 799ce110ea1SWei Hu { 800ce110ea1SWei Hu uint32_t max_req_msg_size, max_resp_msg_size; 801ce110ea1SWei Hu struct gdma_dev *gd = &gc->hwc; 802ce110ea1SWei Hu struct hw_channel_context *hwc; 803ce110ea1SWei Hu uint16_t q_depth_max; 804ce110ea1SWei Hu int err; 805ce110ea1SWei Hu 806ce110ea1SWei Hu hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO); 807ce110ea1SWei Hu if (!hwc) 808ce110ea1SWei Hu return ENOMEM; 809ce110ea1SWei Hu 810ce110ea1SWei Hu gd->gdma_context = gc; 811ce110ea1SWei Hu gd->driver_data = hwc; 812ce110ea1SWei Hu hwc->gdma_dev = gd; 813ce110ea1SWei Hu hwc->dev = gc->dev; 814ce110ea1SWei Hu 815ce110ea1SWei Hu /* HWC's instance number is always 0. */ 816ce110ea1SWei Hu gd->dev_id.as_uint32 = 0; 817ce110ea1SWei Hu gd->dev_id.type = GDMA_DEVICE_HWC; 818ce110ea1SWei Hu 819ce110ea1SWei Hu gd->pdid = INVALID_PDID; 820ce110ea1SWei Hu gd->doorbell = INVALID_DOORBELL; 821ce110ea1SWei Hu 822ce110ea1SWei Hu err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 823ce110ea1SWei Hu HW_CHANNEL_MAX_REQUEST_SIZE, 824ce110ea1SWei Hu HW_CHANNEL_MAX_RESPONSE_SIZE); 825ce110ea1SWei Hu if (err) { 826ce110ea1SWei Hu device_printf(hwc->dev, "Failed to initialize HWC: %d\n", 827ce110ea1SWei Hu err); 828ce110ea1SWei Hu goto out; 829ce110ea1SWei Hu } 830ce110ea1SWei Hu 831ce110ea1SWei Hu err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, 832ce110ea1SWei Hu &max_resp_msg_size); 833ce110ea1SWei Hu if (err) { 834ce110ea1SWei Hu device_printf(hwc->dev, "Failed to establish HWC: %d\n", err); 835ce110ea1SWei Hu goto out; 836ce110ea1SWei Hu } 837ce110ea1SWei Hu 838ce110ea1SWei Hu err = mana_hwc_test_channel(gc->hwc.driver_data, 839ce110ea1SWei Hu HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, 840ce110ea1SWei Hu max_req_msg_size, max_resp_msg_size); 841ce110ea1SWei Hu if (err) { 842ce110ea1SWei Hu /* Test failed, but the channel has been established */ 843ce110ea1SWei Hu device_printf(hwc->dev, "Failed to test HWC: %d\n", err); 844ce110ea1SWei Hu return EIO; 845ce110ea1SWei Hu } 846ce110ea1SWei Hu 847ce110ea1SWei Hu return 0; 848ce110ea1SWei Hu out: 849ce110ea1SWei Hu free(hwc, M_DEVBUF); 850ce110ea1SWei Hu return (err); 851ce110ea1SWei Hu } 852ce110ea1SWei Hu 853ce110ea1SWei Hu void 854ce110ea1SWei Hu mana_hwc_destroy_channel(struct gdma_context *gc) 855ce110ea1SWei Hu { 856ce110ea1SWei Hu struct hw_channel_context *hwc = gc->hwc.driver_data; 857ce110ea1SWei Hu struct hwc_caller_ctx *ctx; 858ce110ea1SWei Hu 859ce110ea1SWei Hu mana_smc_teardown_hwc(&gc->shm_channel, false); 860ce110ea1SWei Hu 861ce110ea1SWei Hu ctx = hwc->caller_ctx; 862ce110ea1SWei Hu free(ctx, M_DEVBUF); 863ce110ea1SWei Hu hwc->caller_ctx = NULL; 864ce110ea1SWei Hu 865ce110ea1SWei Hu mana_hwc_destroy_wq(hwc, hwc->txq); 866ce110ea1SWei Hu hwc->txq = NULL; 867ce110ea1SWei Hu 868ce110ea1SWei Hu mana_hwc_destroy_wq(hwc, hwc->rxq); 869ce110ea1SWei Hu hwc->rxq = NULL; 870ce110ea1SWei Hu 871ce110ea1SWei Hu mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); 872ce110ea1SWei Hu hwc->cq = NULL; 873ce110ea1SWei Hu 874ce110ea1SWei Hu mana_gd_free_res_map(&hwc->inflight_msg_res); 875ce110ea1SWei Hu 876ce110ea1SWei Hu hwc->num_inflight_msg = 0; 877ce110ea1SWei Hu 878ce110ea1SWei Hu if (hwc->gdma_dev->pdid != INVALID_PDID) { 879ce110ea1SWei Hu hwc->gdma_dev->doorbell = INVALID_DOORBELL; 880ce110ea1SWei Hu hwc->gdma_dev->pdid = INVALID_PDID; 881ce110ea1SWei Hu } 882ce110ea1SWei Hu 883ce110ea1SWei Hu free(hwc, M_DEVBUF); 884ce110ea1SWei Hu gc->hwc.driver_data = NULL; 885ce110ea1SWei Hu gc->hwc.gdma_context = NULL; 886ce110ea1SWei Hu } 887ce110ea1SWei Hu 888ce110ea1SWei Hu int 889ce110ea1SWei Hu mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len, 890ce110ea1SWei Hu const void *req, uint32_t resp_len, void *resp) 891ce110ea1SWei Hu { 892ce110ea1SWei Hu struct hwc_work_request *tx_wr; 893ce110ea1SWei Hu struct hwc_wq *txq = hwc->txq; 894ce110ea1SWei Hu struct gdma_req_hdr *req_msg; 895ce110ea1SWei Hu struct hwc_caller_ctx *ctx; 896ce110ea1SWei Hu uint16_t msg_id; 897ce110ea1SWei Hu int err; 898ce110ea1SWei Hu 899ce110ea1SWei Hu mana_hwc_get_msg_index(hwc, &msg_id); 900ce110ea1SWei Hu 901ce110ea1SWei Hu tx_wr = &txq->msg_buf->reqs[msg_id]; 902ce110ea1SWei Hu 903ce110ea1SWei Hu if (req_len > tx_wr->buf_len) { 904ce110ea1SWei Hu device_printf(hwc->dev, 905ce110ea1SWei Hu "HWC: req msg size: %d > %d\n", req_len, 906ce110ea1SWei Hu tx_wr->buf_len); 907ce110ea1SWei Hu err = EINVAL; 908ce110ea1SWei Hu goto out; 909ce110ea1SWei Hu } 910ce110ea1SWei Hu 911ce110ea1SWei Hu ctx = hwc->caller_ctx + msg_id; 912ce110ea1SWei Hu ctx->output_buf = resp; 913ce110ea1SWei Hu ctx->output_buflen = resp_len; 914ce110ea1SWei Hu 915ce110ea1SWei Hu req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; 916ce110ea1SWei Hu if (req) 917ce110ea1SWei Hu memcpy(req_msg, req, req_len); 918ce110ea1SWei Hu 919ce110ea1SWei Hu req_msg->req.hwc_msg_id = msg_id; 920ce110ea1SWei Hu 921ce110ea1SWei Hu tx_wr->msg_size = req_len; 922ce110ea1SWei Hu 923ce110ea1SWei Hu err = mana_hwc_post_tx_wqe(txq, tx_wr, 0, 0, false); 924ce110ea1SWei Hu if (err) { 925ce110ea1SWei Hu device_printf(hwc->dev, 926ce110ea1SWei Hu "HWC: Failed to post send WQE: %d\n", err); 927ce110ea1SWei Hu goto out; 928ce110ea1SWei Hu } 929ce110ea1SWei Hu 930ce110ea1SWei Hu if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) { 931ce110ea1SWei Hu device_printf(hwc->dev, "HWC: Request timed out!\n"); 932ce110ea1SWei Hu err = ETIMEDOUT; 933ce110ea1SWei Hu goto out; 934ce110ea1SWei Hu } 935ce110ea1SWei Hu 936ce110ea1SWei Hu if (ctx->error) { 937ce110ea1SWei Hu err = ctx->error; 938ce110ea1SWei Hu goto out; 939ce110ea1SWei Hu } 940ce110ea1SWei Hu 941ce110ea1SWei Hu if (ctx->status_code) { 942ce110ea1SWei Hu device_printf(hwc->dev, 943ce110ea1SWei Hu "HWC: Failed hw_channel req: 0x%x\n", ctx->status_code); 944ce110ea1SWei Hu err = EPROTO; 945ce110ea1SWei Hu goto out; 946ce110ea1SWei Hu } 947ce110ea1SWei Hu out: 948ce110ea1SWei Hu mana_hwc_put_msg_index(hwc, msg_id); 949ce110ea1SWei Hu return err; 950ce110ea1SWei Hu } 951