Lines Matching +full:rx +full:- +full:eq
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
48 struct gdma_resource *r = &hwc->inflight_msg_res; in mana_hwc_get_msg_index()
51 sema_wait(&hwc->sema); in mana_hwc_get_msg_index()
53 mtx_lock_spin(&r->lock_spin); in mana_hwc_get_msg_index()
55 index = find_first_zero_bit(hwc->inflight_msg_res.map, in mana_hwc_get_msg_index()
56 hwc->inflight_msg_res.size); in mana_hwc_get_msg_index()
58 bitmap_set(hwc->inflight_msg_res.map, index, 1); in mana_hwc_get_msg_index()
60 mtx_unlock_spin(&r->lock_spin); in mana_hwc_get_msg_index()
70 struct gdma_resource *r = &hwc->inflight_msg_res; in mana_hwc_put_msg_index()
72 mtx_lock_spin(&r->lock_spin); in mana_hwc_put_msg_index()
73 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); in mana_hwc_put_msg_index()
74 mtx_unlock_spin(&r->lock_spin); in mana_hwc_put_msg_index()
76 sema_post(&hwc->sema); in mana_hwc_put_msg_index()
87 if (resp_len > caller_ctx->output_buflen) in mana_hwc_verify_resp_msg()
100 if (!test_bit(resp_msg->response.hwc_msg_id, in mana_hwc_handle_resp()
101 hwc->inflight_msg_res.map)) { in mana_hwc_handle_resp()
102 device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n", in mana_hwc_handle_resp()
103 resp_msg->response.hwc_msg_id); in mana_hwc_handle_resp()
107 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; in mana_hwc_handle_resp()
112 ctx->status_code = resp_msg->status; in mana_hwc_handle_resp()
114 memcpy(ctx->output_buf, resp_msg, resp_len); in mana_hwc_handle_resp()
116 ctx->error = err; in mana_hwc_handle_resp()
117 complete(&ctx->comp_event); in mana_hwc_handle_resp()
124 device_t dev = hwc_rxq->hwc->dev; in mana_hwc_post_rx_wqe()
128 sge = &req->sge; in mana_hwc_post_rx_wqe()
129 sge->address = (uintptr_t)req->buf_sge_addr; in mana_hwc_post_rx_wqe()
130 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; in mana_hwc_post_rx_wqe()
131 sge->size = req->buf_len; in mana_hwc_post_rx_wqe()
133 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); in mana_hwc_post_rx_wqe()
134 req->wqe_req.sgl = sge; in mana_hwc_post_rx_wqe()
135 req->wqe_req.num_sge = 1; in mana_hwc_post_rx_wqe()
136 req->wqe_req.client_data_unit = 0; in mana_hwc_post_rx_wqe()
138 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); in mana_hwc_post_rx_wqe()
150 struct gdma_dev *gd = hwc->gdma_dev; in mana_hwc_init_event_handler()
155 switch (event->type) { in mana_hwc_init_event_handler()
157 eq_db.as_uint32 = event->details[0]; in mana_hwc_init_event_handler()
158 hwc->cq->gdma_eq->id = eq_db.eq_id; in mana_hwc_init_event_handler()
159 gd->doorbell = eq_db.doorbell; in mana_hwc_init_event_handler()
163 type_data.as_uint32 = event->details[0]; in mana_hwc_init_event_handler()
169 hwc->cq->gdma_cq->id = val; in mana_hwc_init_event_handler()
173 hwc->rxq->gdma_wq->id = val; in mana_hwc_init_event_handler()
177 hwc->txq->gdma_wq->id = val; in mana_hwc_init_event_handler()
181 hwc->hwc_init_q_depth_max = (uint16_t)val; in mana_hwc_init_event_handler()
185 hwc->hwc_init_max_req_msg_size = val; in mana_hwc_init_event_handler()
189 hwc->hwc_init_max_resp_msg_size = val; in mana_hwc_init_event_handler()
193 gd->gdma_context->max_num_cqs = val; in mana_hwc_init_event_handler()
197 hwc->gdma_dev->pdid = val; in mana_hwc_init_event_handler()
201 hwc->rxq->msg_buf->gpa_mkey = val; in mana_hwc_init_event_handler()
202 hwc->txq->msg_buf->gpa_mkey = val; in mana_hwc_init_event_handler()
209 complete(&hwc->hwc_init_eqe_comp); in mana_hwc_init_event_handler()
223 struct hwc_wq *hwc_rxq = hwc->rxq; in mana_hwc_rx_event_handler()
233 if (hwc_rxq->gdma_wq->id != gdma_rxq_id) { in mana_hwc_rx_event_handler()
234 mana_warn(NULL, "unmatched rx queue %u != %u\n", in mana_hwc_rx_event_handler()
235 hwc_rxq->gdma_wq->id, gdma_rxq_id); in mana_hwc_rx_event_handler()
240 rq = hwc_rxq->gdma_wq; in mana_hwc_rx_event_handler()
241 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); in mana_hwc_rx_event_handler()
244 bus_dmamap_sync(rq->mem_info.dma_tag, rq->mem_info.dma_map, in mana_hwc_rx_event_handler()
247 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); in mana_hwc_rx_event_handler()
249 /* Select the RX work request for virtual address and for reposting. */ in mana_hwc_rx_event_handler()
250 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; in mana_hwc_rx_event_handler()
251 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; in mana_hwc_rx_event_handler()
253 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, in mana_hwc_rx_event_handler()
254 hwc_rxq->msg_buf->mem_info.dma_map, in mana_hwc_rx_event_handler()
257 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; in mana_hwc_rx_event_handler()
258 resp = (struct gdma_resp_hdr *)rx_req->buf_va; in mana_hwc_rx_event_handler()
260 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { in mana_hwc_rx_event_handler()
261 device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n", in mana_hwc_rx_event_handler()
262 resp->response.hwc_msg_id); in mana_hwc_rx_event_handler()
266 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp); in mana_hwc_rx_event_handler()
273 bus_dmamap_sync(hwc_rxq->msg_buf->mem_info.dma_tag, in mana_hwc_rx_event_handler()
274 hwc_rxq->msg_buf->mem_info.dma_map, in mana_hwc_rx_event_handler()
285 struct hwc_wq *hwc_txq = hwc->txq; in mana_hwc_tx_event_handler()
287 if (!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id) { in mana_hwc_tx_event_handler()
289 hwc_txq->gdma_wq->id, gdma_txq_id); in mana_hwc_tx_event_handler()
292 bus_dmamap_sync(hwc_txq->gdma_wq->mem_info.dma_tag, in mana_hwc_tx_event_handler()
293 hwc_txq->gdma_wq->mem_info.dma_map, in mana_hwc_tx_event_handler()
311 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_wq()
330 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_cq()
344 spec.eq.context = ctx; in mana_hwc_create_gdma_eq()
345 spec.eq.callback = cb; in mana_hwc_create_gdma_eq()
346 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; in mana_hwc_create_gdma_eq()
348 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_eq()
359 completions = hwc_cq->comp_buf; in mana_hwc_comp_event()
360 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); in mana_hwc_comp_event()
366 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, in mana_hwc_comp_event()
370 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, in mana_hwc_comp_event()
375 bus_dmamap_sync(q_self->mem_info.dma_tag, q_self->mem_info.dma_map, in mana_hwc_comp_event()
384 if (hwc_cq->comp_buf) in mana_hwc_destroy_cq()
385 free(hwc_cq->comp_buf, M_DEVBUF); in mana_hwc_destroy_cq()
387 if (hwc_cq->gdma_cq) in mana_hwc_destroy_cq()
388 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); in mana_hwc_destroy_cq()
390 if (hwc_cq->gdma_eq) in mana_hwc_destroy_cq()
391 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); in mana_hwc_destroy_cq()
404 struct gdma_queue *eq, *cq; in mana_hwc_create_cq() local
420 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); in mana_hwc_create_cq()
422 device_printf(hwc->dev, in mana_hwc_create_cq()
423 "Failed to create HWC EQ for RQ: %d\n", err); in mana_hwc_create_cq()
426 hwc_cq->gdma_eq = eq; in mana_hwc_create_cq()
429 mana_hwc_comp_event, eq, &cq); in mana_hwc_create_cq()
431 device_printf(hwc->dev, in mana_hwc_create_cq()
435 hwc_cq->gdma_cq = cq; in mana_hwc_create_cq()
440 hwc_cq->hwc = hwc; in mana_hwc_create_cq()
441 hwc_cq->comp_buf = comp_buf; in mana_hwc_create_cq()
442 hwc_cq->queue_depth = q_depth; in mana_hwc_create_cq()
443 hwc_cq->rx_event_handler = rx_ev_hdlr; in mana_hwc_create_cq()
444 hwc_cq->rx_event_ctx = rx_ev_ctx; in mana_hwc_create_cq()
445 hwc_cq->tx_event_handler = tx_ev_hdlr; in mana_hwc_create_cq()
446 hwc_cq->tx_event_ctx = tx_ev_ctx; in mana_hwc_create_cq()
451 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); in mana_hwc_create_cq()
460 struct gdma_context *gc = hwc->gdma_dev->gdma_context; in mana_hwc_alloc_dma_buf()
474 dma_buf->num_reqs = q_depth; in mana_hwc_alloc_dma_buf()
478 gmi = &dma_buf->mem_info; in mana_hwc_alloc_dma_buf()
481 device_printf(hwc->dev, in mana_hwc_alloc_dma_buf()
486 virt_addr = dma_buf->mem_info.virt_addr; in mana_hwc_alloc_dma_buf()
487 base_pa = (uint8_t *)dma_buf->mem_info.dma_handle; in mana_hwc_alloc_dma_buf()
490 hwc_wr = &dma_buf->reqs[i]; in mana_hwc_alloc_dma_buf()
492 hwc_wr->buf_va = (char *)virt_addr + i * max_msg_size; in mana_hwc_alloc_dma_buf()
493 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; in mana_hwc_alloc_dma_buf()
495 hwc_wr->buf_len = max_msg_size; in mana_hwc_alloc_dma_buf()
512 mana_gd_free_memory(&dma_buf->mem_info); in mana_hwc_dealloc_dma_buf()
521 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); in mana_hwc_destroy_wq()
523 if (hwc_wq->gdma_wq) in mana_hwc_destroy_wq()
524 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, in mana_hwc_destroy_wq()
525 hwc_wq->gdma_wq); in mana_hwc_destroy_wq()
560 hwc_wq->hwc = hwc; in mana_hwc_create_wq()
561 hwc_wq->gdma_wq = queue; in mana_hwc_create_wq()
562 hwc_wq->queue_depth = q_depth; in mana_hwc_create_wq()
563 hwc_wq->hwc_cq = hwc_cq; in mana_hwc_create_wq()
566 &hwc_wq->msg_buf); in mana_hwc_create_wq()
584 device_t dev = hwc_txq->hwc->dev; in mana_hwc_post_tx_wqe()
589 if (req->msg_size == 0 || req->msg_size > req->buf_len) { in mana_hwc_post_tx_wqe()
591 req->msg_size, req->buf_len); in mana_hwc_post_tx_wqe()
595 tx_oob = &req->tx_oob; in mana_hwc_post_tx_wqe()
597 tx_oob->vrq_id = dest_virt_rq_id; in mana_hwc_post_tx_wqe()
598 tx_oob->dest_vfid = 0; in mana_hwc_post_tx_wqe()
599 tx_oob->vrcq_id = dest_virt_rcq_id; in mana_hwc_post_tx_wqe()
600 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; in mana_hwc_post_tx_wqe()
601 tx_oob->loopback = false; in mana_hwc_post_tx_wqe()
602 tx_oob->lso_override = false; in mana_hwc_post_tx_wqe()
603 tx_oob->dest_pf = dest_pf; in mana_hwc_post_tx_wqe()
604 tx_oob->vsq_id = hwc_txq->gdma_wq->id; in mana_hwc_post_tx_wqe()
606 sge = &req->sge; in mana_hwc_post_tx_wqe()
607 sge->address = (uintptr_t)req->buf_sge_addr; in mana_hwc_post_tx_wqe()
608 sge->mem_key = hwc_txq->msg_buf->gpa_mkey; in mana_hwc_post_tx_wqe()
609 sge->size = req->msg_size; in mana_hwc_post_tx_wqe()
611 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); in mana_hwc_post_tx_wqe()
612 req->wqe_req.sgl = sge; in mana_hwc_post_tx_wqe()
613 req->wqe_req.num_sge = 1; in mana_hwc_post_tx_wqe()
614 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); in mana_hwc_post_tx_wqe()
615 req->wqe_req.inline_oob_data = tx_oob; in mana_hwc_post_tx_wqe()
616 req->wqe_req.client_data_unit = 0; in mana_hwc_post_tx_wqe()
618 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); in mana_hwc_post_tx_wqe()
630 sema_init(&hwc->sema, num_msg, "gdma hwc sema"); in mana_hwc_init_inflight_msg()
632 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res, in mana_hwc_init_inflight_msg()
635 device_printf(hwc->dev, in mana_hwc_init_inflight_msg()
645 struct gdma_context *gc = hwc->gdma_dev->gdma_context; in mana_hwc_test_channel()
646 struct hwc_wq *hwc_rxq = hwc->rxq; in mana_hwc_test_channel()
654 req = &hwc_rxq->msg_buf->reqs[i]; in mana_hwc_test_channel()
666 hwc->caller_ctx = ctx; in mana_hwc_test_channel()
668 return mana_gd_test_eq(gc, hwc->cq->gdma_eq); in mana_hwc_test_channel()
676 struct hw_channel_context *hwc = gc->hwc.driver_data; in mana_hwc_establish_channel()
677 struct gdma_queue *rq = hwc->rxq->gdma_wq; in mana_hwc_establish_channel()
678 struct gdma_queue *sq = hwc->txq->gdma_wq; in mana_hwc_establish_channel()
679 struct gdma_queue *eq = hwc->cq->gdma_eq; in mana_hwc_establish_channel() local
680 struct gdma_queue *cq = hwc->cq->gdma_cq; in mana_hwc_establish_channel()
683 init_completion(&hwc->hwc_init_eqe_comp); in mana_hwc_establish_channel()
685 err = mana_smc_setup_hwc(&gc->shm_channel, false, in mana_hwc_establish_channel()
686 eq->mem_info.dma_handle, in mana_hwc_establish_channel()
687 cq->mem_info.dma_handle, in mana_hwc_establish_channel()
688 rq->mem_info.dma_handle, in mana_hwc_establish_channel()
689 sq->mem_info.dma_handle, in mana_hwc_establish_channel()
690 eq->eq.msix_index); in mana_hwc_establish_channel()
694 if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz)) in mana_hwc_establish_channel()
697 *q_depth = hwc->hwc_init_q_depth_max; in mana_hwc_establish_channel()
698 *max_req_msg_size = hwc->hwc_init_max_req_msg_size; in mana_hwc_establish_channel()
699 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; in mana_hwc_establish_channel()
702 if (cq->id >= gc->max_num_cqs) { in mana_hwc_establish_channel()
704 cq->id, gc->max_num_cqs); in mana_hwc_establish_channel()
708 gc->cq_table = malloc(gc->max_num_cqs * sizeof(struct gdma_queue *), in mana_hwc_establish_channel()
710 gc->cq_table[cq->id] = cq; in mana_hwc_establish_channel()
731 mana_hwc_tx_event_handler, hwc, &hwc->cq); in mana_hwc_init_queues()
733 device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err); in mana_hwc_init_queues()
738 hwc->cq, &hwc->rxq); in mana_hwc_init_queues()
740 device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err); in mana_hwc_init_queues()
745 hwc->cq, &hwc->txq); in mana_hwc_init_queues()
747 device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err); in mana_hwc_init_queues()
751 hwc->num_inflight_msg = q_depth; in mana_hwc_init_queues()
752 hwc->max_req_msg_size = max_req_msg_size; in mana_hwc_init_queues()
764 struct gdma_dev *gd = &gc->hwc; in mana_hwc_create_channel()
771 gd->gdma_context = gc; in mana_hwc_create_channel()
772 gd->driver_data = hwc; in mana_hwc_create_channel()
773 hwc->gdma_dev = gd; in mana_hwc_create_channel()
774 hwc->dev = gc->dev; in mana_hwc_create_channel()
777 gd->dev_id.as_uint32 = 0; in mana_hwc_create_channel()
778 gd->dev_id.type = GDMA_DEVICE_HWC; in mana_hwc_create_channel()
780 gd->pdid = INVALID_PDID; in mana_hwc_create_channel()
781 gd->doorbell = INVALID_DOORBELL; in mana_hwc_create_channel()
791 device_printf(hwc->dev, "Failed to initialize HWC: %d\n", in mana_hwc_create_channel()
799 device_printf(hwc->dev, "Failed to establish HWC: %d\n", err); in mana_hwc_create_channel()
803 err = mana_hwc_test_channel(gc->hwc.driver_data, in mana_hwc_create_channel()
808 device_printf(hwc->dev, "Failed to test HWC: %d\n", err); in mana_hwc_create_channel()
821 struct hw_channel_context *hwc = gc->hwc.driver_data; in mana_hwc_destroy_channel()
827 * gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's in mana_hwc_destroy_channel()
828 * non-zero, the HWC worked and we should tear down the HWC here. in mana_hwc_destroy_channel()
830 if (gc->max_num_cqs > 0) { in mana_hwc_destroy_channel()
831 mana_smc_teardown_hwc(&gc->shm_channel, false); in mana_hwc_destroy_channel()
832 gc->max_num_cqs = 0; in mana_hwc_destroy_channel()
835 free(hwc->caller_ctx, M_DEVBUF); in mana_hwc_destroy_channel()
836 hwc->caller_ctx = NULL; in mana_hwc_destroy_channel()
838 if (hwc->txq) in mana_hwc_destroy_channel()
839 mana_hwc_destroy_wq(hwc, hwc->txq); in mana_hwc_destroy_channel()
841 if (hwc->rxq) in mana_hwc_destroy_channel()
842 mana_hwc_destroy_wq(hwc, hwc->rxq); in mana_hwc_destroy_channel()
844 if (hwc->cq) in mana_hwc_destroy_channel()
845 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); in mana_hwc_destroy_channel()
847 mana_gd_free_res_map(&hwc->inflight_msg_res); in mana_hwc_destroy_channel()
849 hwc->num_inflight_msg = 0; in mana_hwc_destroy_channel()
851 hwc->gdma_dev->doorbell = INVALID_DOORBELL; in mana_hwc_destroy_channel()
852 hwc->gdma_dev->pdid = INVALID_PDID; in mana_hwc_destroy_channel()
855 gc->hwc.driver_data = NULL; in mana_hwc_destroy_channel()
856 gc->hwc.gdma_context = NULL; in mana_hwc_destroy_channel()
858 free(gc->cq_table, M_DEVBUF); in mana_hwc_destroy_channel()
859 gc->cq_table = NULL; in mana_hwc_destroy_channel()
867 struct hwc_wq *txq = hwc->txq; in mana_hwc_send_request()
875 tx_wr = &txq->msg_buf->reqs[msg_id]; in mana_hwc_send_request()
877 if (req_len > tx_wr->buf_len) { in mana_hwc_send_request()
878 device_printf(hwc->dev, in mana_hwc_send_request()
880 tx_wr->buf_len); in mana_hwc_send_request()
885 ctx = hwc->caller_ctx + msg_id; in mana_hwc_send_request()
886 ctx->output_buf = resp; in mana_hwc_send_request()
887 ctx->output_buflen = resp_len; in mana_hwc_send_request()
889 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; in mana_hwc_send_request()
893 req_msg->req.hwc_msg_id = msg_id; in mana_hwc_send_request()
895 tx_wr->msg_size = req_len; in mana_hwc_send_request()
899 device_printf(hwc->dev, in mana_hwc_send_request()
904 if (wait_for_completion_timeout(&ctx->comp_event, 30 * hz)) { in mana_hwc_send_request()
905 device_printf(hwc->dev, "HWC: Request timed out!\n"); in mana_hwc_send_request()
910 if (ctx->error) { in mana_hwc_send_request()
911 err = ctx->error; in mana_hwc_send_request()
915 if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { in mana_hwc_send_request()
916 device_printf(hwc->dev, in mana_hwc_send_request()
917 "HWC: Failed hw_channel req: 0x%x\n", ctx->status_code); in mana_hwc_send_request()