46 mana_hwc_get_msg_index(struct hw_channel_context *hwc, uint16_t *msg_id)  in mana_hwc_get_msg_index()  argument
48 	struct gdma_resource *r = &hwc->inflight_msg_res;  in mana_hwc_get_msg_index()
51 	sema_wait(&hwc->sema);  in mana_hwc_get_msg_index()
55 	index = find_first_zero_bit(hwc->inflight_msg_res.map,  in mana_hwc_get_msg_index()
56 	    hwc->inflight_msg_res.size);  in mana_hwc_get_msg_index()
58 	bitmap_set(hwc->inflight_msg_res.map, index, 1);  in mana_hwc_get_msg_index()
68 mana_hwc_put_msg_index(struct hw_channel_context *hwc, uint16_t msg_id)  in mana_hwc_put_msg_index()  argument
70 	struct gdma_resource *r = &hwc->inflight_msg_res;  in mana_hwc_put_msg_index()
73 	bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1);  in mana_hwc_put_msg_index()
76 	sema_post(&hwc->sema);  in mana_hwc_put_msg_index()
94 mana_hwc_handle_resp(struct hw_channel_context *hwc, uint32_t resp_len,  in mana_hwc_handle_resp()  argument
101 	    hwc->inflight_msg_res.map)) {  in mana_hwc_handle_resp()
102 		device_printf(hwc->dev, "hwc_rx: invalid msg_id = %u\n",  in mana_hwc_handle_resp()
107 	ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;  in mana_hwc_handle_resp()
124 	device_t dev = hwc_rxq->hwc->dev;  in mana_hwc_post_rx_wqe()
149 	struct hw_channel_context *hwc = ctx;  in mana_hwc_init_event_handler()  local
150 	struct gdma_dev *gd = hwc->gdma_dev;  in mana_hwc_init_event_handler()
158 		hwc->cq->gdma_eq->id = eq_db.eq_id;  in mana_hwc_init_event_handler()
169 			hwc->cq->gdma_cq->id = val;  in mana_hwc_init_event_handler()
173 			hwc->rxq->gdma_wq->id = val;  in mana_hwc_init_event_handler()
177 			hwc->txq->gdma_wq->id = val;  in mana_hwc_init_event_handler()
181 			hwc->hwc_init_q_depth_max = (uint16_t)val;  in mana_hwc_init_event_handler()
185 			hwc->hwc_init_max_req_msg_size = val;  in mana_hwc_init_event_handler()
189 			hwc->hwc_init_max_resp_msg_size = val;  in mana_hwc_init_event_handler()
197 			hwc->gdma_dev->pdid = val;  in mana_hwc_init_event_handler()
201 			hwc->rxq->msg_buf->gpa_mkey = val;  in mana_hwc_init_event_handler()
202 			hwc->txq->msg_buf->gpa_mkey = val;  in mana_hwc_init_event_handler()
209 		complete(&hwc->hwc_init_eqe_comp);  in mana_hwc_init_event_handler()
222 	struct hw_channel_context *hwc = ctx;  in mana_hwc_rx_event_handler()  local
223 	struct hwc_wq *hwc_rxq = hwc->rxq;  in mana_hwc_rx_event_handler()
251 	rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;  in mana_hwc_rx_event_handler()
260 	if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {  in mana_hwc_rx_event_handler()
261 		device_printf(hwc->dev, "HWC RX: wrong msg_id=%u\n",  in mana_hwc_rx_event_handler()
266 	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp);  in mana_hwc_rx_event_handler()
284 	struct hw_channel_context *hwc = ctx;  in mana_hwc_tx_event_handler()  local
285 	struct hwc_wq *hwc_txq = hwc->txq;  in mana_hwc_tx_event_handler()
298 mana_hwc_create_gdma_wq(struct hw_channel_context *hwc,  in mana_hwc_create_gdma_wq()  argument
311 	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);  in mana_hwc_create_gdma_wq()
315 mana_hwc_create_gdma_cq(struct hw_channel_context *hwc,  in mana_hwc_create_gdma_cq()  argument
330 	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);  in mana_hwc_create_gdma_cq()
334 mana_hwc_create_gdma_eq(struct hw_channel_context *hwc,  in mana_hwc_create_gdma_eq()  argument
348 	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);  in mana_hwc_create_gdma_eq()
397 mana_hwc_create_cq(struct hw_channel_context *hwc,  in mana_hwc_create_cq()  argument
420 	err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq);  in mana_hwc_create_cq()
422 		device_printf(hwc->dev,  in mana_hwc_create_cq()
428 	err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq,  in mana_hwc_create_cq()
431 		device_printf(hwc->dev,  in mana_hwc_create_cq()
440 	hwc_cq->hwc = hwc;  in mana_hwc_create_cq()
451 	mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq);  in mana_hwc_create_cq()
456 mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, uint16_t q_depth,  in mana_hwc_alloc_dma_buf()  argument
460 	struct gdma_context *gc = hwc->gdma_dev->gdma_context;  in mana_hwc_alloc_dma_buf()
481 		device_printf(hwc->dev,  in mana_hwc_alloc_dma_buf()
506 mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc,  in mana_hwc_dealloc_dma_buf()  argument
518 mana_hwc_destroy_wq(struct hw_channel_context *hwc,  in mana_hwc_destroy_wq()  argument
521 	mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf);  in mana_hwc_destroy_wq()
524 		mana_gd_destroy_queue(hwc->gdma_dev->gdma_context,  in mana_hwc_destroy_wq()
531 mana_hwc_create_wq(struct hw_channel_context *hwc,  in mana_hwc_create_wq()  argument
556 	err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue);  in mana_hwc_create_wq()
560 	hwc_wq->hwc = hwc;  in mana_hwc_create_wq()
565 	err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,  in mana_hwc_create_wq()
574 		mana_hwc_destroy_wq(hwc, hwc_wq);  in mana_hwc_create_wq()
584 	device_t dev = hwc_txq->hwc->dev;  in mana_hwc_post_tx_wqe()
626 mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, uint16_t num_msg)  in mana_hwc_init_inflight_msg()  argument
630 	sema_init(&hwc->sema, num_msg, "gdma hwc sema");  in mana_hwc_init_inflight_msg()
632 	err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res,  in mana_hwc_init_inflight_msg()
635 		device_printf(hwc->dev,  in mana_hwc_init_inflight_msg()
642 mana_hwc_test_channel(struct hw_channel_context *hwc, uint16_t q_depth,  in mana_hwc_test_channel()  argument
645 	struct gdma_context *gc = hwc->gdma_dev->gdma_context;  in mana_hwc_test_channel()
646 	struct hwc_wq *hwc_rxq = hwc->rxq;  in mana_hwc_test_channel()
666 	hwc->caller_ctx = ctx;  in mana_hwc_test_channel()
668 	return mana_gd_test_eq(gc, hwc->cq->gdma_eq);  in mana_hwc_test_channel()
676 	struct hw_channel_context *hwc = gc->hwc.driver_data;  in mana_hwc_establish_channel()  local
677 	struct gdma_queue *rq = hwc->rxq->gdma_wq;  in mana_hwc_establish_channel()
678 	struct gdma_queue *sq = hwc->txq->gdma_wq;  in mana_hwc_establish_channel()
679 	struct gdma_queue *eq = hwc->cq->gdma_eq;  in mana_hwc_establish_channel()
680 	struct gdma_queue *cq = hwc->cq->gdma_cq;  in mana_hwc_establish_channel()
683 	init_completion(&hwc->hwc_init_eqe_comp);  in mana_hwc_establish_channel()
694 	if (wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * hz))  in mana_hwc_establish_channel()
697 	*q_depth = hwc->hwc_init_q_depth_max;  in mana_hwc_establish_channel()
698 	*max_req_msg_size = hwc->hwc_init_max_req_msg_size;  in mana_hwc_establish_channel()
699 	*max_resp_msg_size = hwc->hwc_init_max_resp_msg_size;  in mana_hwc_establish_channel()
716 mana_hwc_init_queues(struct hw_channel_context *hwc, uint16_t q_depth,  in mana_hwc_init_queues()  argument
721 	err = mana_hwc_init_inflight_msg(hwc, q_depth);  in mana_hwc_init_queues()
728 	err = mana_hwc_create_cq(hwc, q_depth * 2,  in mana_hwc_init_queues()
729 	    mana_hwc_init_event_handler, hwc,  in mana_hwc_init_queues()
730 	    mana_hwc_rx_event_handler, hwc,  in mana_hwc_init_queues()
731 	    mana_hwc_tx_event_handler, hwc, &hwc->cq);  in mana_hwc_init_queues()
733 		device_printf(hwc->dev, "Failed to create HWC CQ: %d\n", err);  in mana_hwc_init_queues()
737 	err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,  in mana_hwc_init_queues()
738 	    hwc->cq, &hwc->rxq);  in mana_hwc_init_queues()
740 		device_printf(hwc->dev, "Failed to create HWC RQ: %d\n", err);  in mana_hwc_init_queues()
744 	err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,  in mana_hwc_init_queues()
745 	    hwc->cq, &hwc->txq);  in mana_hwc_init_queues()
747 		device_printf(hwc->dev, "Failed to create HWC SQ: %d\n", err);  in mana_hwc_init_queues()
751 	hwc->num_inflight_msg = q_depth;  in mana_hwc_init_queues()
752 	hwc->max_req_msg_size = max_req_msg_size;  in mana_hwc_init_queues()
764 	struct gdma_dev *gd = &gc->hwc;  in mana_hwc_create_channel()
765 	struct hw_channel_context *hwc;  in mana_hwc_create_channel()  local
769 	hwc = malloc(sizeof(*hwc), M_DEVBUF, M_WAITOK | M_ZERO);  in mana_hwc_create_channel()
772 	gd->driver_data = hwc;  in mana_hwc_create_channel()
773 	hwc->gdma_dev = gd;  in mana_hwc_create_channel()
774 	hwc->dev = gc->dev;  in mana_hwc_create_channel()
787 	err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,  in mana_hwc_create_channel()
791 		device_printf(hwc->dev, "Failed to initialize HWC: %d\n",  in mana_hwc_create_channel()
799 		device_printf(hwc->dev, "Failed to establish HWC: %d\n", err);  in mana_hwc_create_channel()
803 	err = mana_hwc_test_channel(gc->hwc.driver_data,  in mana_hwc_create_channel()
808 		device_printf(hwc->dev, "Failed to test HWC: %d\n", err);  in mana_hwc_create_channel()
821 	struct hw_channel_context *hwc = gc->hwc.driver_data;  in mana_hwc_destroy_channel()  local
823 	if (!hwc)  in mana_hwc_destroy_channel()
835 	free(hwc->caller_ctx, M_DEVBUF);  in mana_hwc_destroy_channel()
836 	hwc->caller_ctx = NULL;  in mana_hwc_destroy_channel()
838 	if (hwc->txq)  in mana_hwc_destroy_channel()
839 		mana_hwc_destroy_wq(hwc, hwc->txq);  in mana_hwc_destroy_channel()
841 	if (hwc->rxq)  in mana_hwc_destroy_channel()
842 		mana_hwc_destroy_wq(hwc, hwc->rxq);  in mana_hwc_destroy_channel()
844 	if (hwc->cq)  in mana_hwc_destroy_channel()
845 		mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);  in mana_hwc_destroy_channel()
847 	mana_gd_free_res_map(&hwc->inflight_msg_res);  in mana_hwc_destroy_channel()
849 	hwc->num_inflight_msg = 0;  in mana_hwc_destroy_channel()
851 	hwc->gdma_dev->doorbell = INVALID_DOORBELL;  in mana_hwc_destroy_channel()
852 	hwc->gdma_dev->pdid = INVALID_PDID;  in mana_hwc_destroy_channel()
854 	free(hwc, M_DEVBUF);  in mana_hwc_destroy_channel()
855 	gc->hwc.driver_data = NULL;  in mana_hwc_destroy_channel()
856 	gc->hwc.gdma_context = NULL;  in mana_hwc_destroy_channel()
863 mana_hwc_send_request(struct hw_channel_context *hwc, uint32_t req_len,  in mana_hwc_send_request()  argument
867 	struct hwc_wq *txq = hwc->txq;  in mana_hwc_send_request()
873 	mana_hwc_get_msg_index(hwc, &msg_id);  in mana_hwc_send_request()
878 		device_printf(hwc->dev,  in mana_hwc_send_request()
885 	ctx = hwc->caller_ctx + msg_id;  in mana_hwc_send_request()
899 		device_printf(hwc->dev,  in mana_hwc_send_request()
905 		device_printf(hwc->dev, "HWC: Request timed out!\n");  in mana_hwc_send_request()
916 		device_printf(hwc->dev,  in mana_hwc_send_request()
922 	mana_hwc_put_msg_index(hwc, msg_id);  in mana_hwc_send_request()