2356 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,  in mlxcx_cmd_query_rq()  argument
2367 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_query_rq()
2368 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_query_rq()
2369 	ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);  in mlxcx_cmd_query_rq()
2375 	in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_query_rq()
2393 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq,  in mlxcx_cmd_query_sq()  argument
2404 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_query_sq()
2405 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_query_sq()
2406 	ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ);  in mlxcx_cmd_query_sq()
2412 	in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_query_sq()
2501 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_create_rq()  argument
2515 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_create_rq()
2516 	VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);  in mlxcx_cmd_create_rq()
2517 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_create_rq()
2518 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_create_rq()
2529 	ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num);  in mlxcx_cmd_create_rq()
2533 	ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num);  in mlxcx_cmd_create_rq()
2534 	ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift;  in mlxcx_cmd_create_rq()
2537 	c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma);  in mlxcx_cmd_create_rq()
2543 	while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) {  in mlxcx_cmd_create_rq()
2569 		mlwq->mlwq_state |= MLXCX_WQ_CREATED;  in mlxcx_cmd_create_rq()
2570 		mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn);  in mlxcx_cmd_create_rq()
2577 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_start_rq()  argument
2588 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_start_rq()
2589 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_start_rq()
2590 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_start_rq()
2591 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_start_rq()
2597 	ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx));  in mlxcx_cmd_start_rq()
2598 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers));  in mlxcx_cmd_start_rq()
2599 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b));  in mlxcx_cmd_start_rq()
2601 	mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0);  in mlxcx_cmd_start_rq()
2602 	MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV);  in mlxcx_cmd_start_rq()
2603 	ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err,  in mlxcx_cmd_start_rq()
2607 	mlwq->mlwq_pc = 0;  in mlxcx_cmd_start_rq()
2613 	in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_start_rq()
2630 		mlwq->mlwq_state |= MLXCX_WQ_STARTED;  in mlxcx_cmd_start_rq()
2637 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_stop_rq()  argument
2647 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_stop_rq()
2648 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_stop_rq()
2649 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_stop_rq()
2650 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_stop_rq()
2656 	in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_stop_rq()
2673 		mlwq->mlwq_state &= ~MLXCX_WQ_STARTED;  in mlxcx_cmd_stop_rq()
2680 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_destroy_rq()  argument
2690 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_destroy_rq()
2691 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_destroy_rq()
2692 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_destroy_rq()
2693 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_destroy_rq()
2699 	in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_destroy_rq()
2709 		mlwq->mlwq_state |= MLXCX_WQ_DESTROYED;  in mlxcx_cmd_destroy_rq()
3378 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_create_sq()  argument
3392 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_create_sq()
3393 	VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ);  in mlxcx_cmd_create_sq()
3394 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_create_sq()
3395 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_create_sq()
3406 	    mlwq->mlwq_inline_mode);  in mlxcx_cmd_create_sq()
3407 	ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num);  in mlxcx_cmd_create_sq()
3409 	VERIFY(mlwq->mlwq_tis != NULL);  in mlxcx_cmd_create_sq()
3411 	ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num);  in mlxcx_cmd_create_sq()
3415 	ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num);  in mlxcx_cmd_create_sq()
3416 	ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num);  in mlxcx_cmd_create_sq()
3417 	ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift;  in mlxcx_cmd_create_sq()
3420 	c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma);  in mlxcx_cmd_create_sq()
3426 	while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) {  in mlxcx_cmd_create_sq()
3452 		mlwq->mlwq_state |= MLXCX_WQ_CREATED;  in mlxcx_cmd_create_sq()
3453 		mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn);  in mlxcx_cmd_create_sq()
3460 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_start_sq()  argument
3471 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_start_sq()
3472 	ASSERT(mlwq->mlwq_cq != NULL);  in mlxcx_cmd_start_sq()
3474 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_start_sq()
3475 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_start_sq()
3476 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_start_sq()
3482 	ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx));  in mlxcx_cmd_start_sq()
3483 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers));  in mlxcx_cmd_start_sq()
3484 	ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b));  in mlxcx_cmd_start_sq()
3486 	mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0);  in mlxcx_cmd_start_sq()
3487 	MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV);  in mlxcx_cmd_start_sq()
3488 	ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err,  in mlxcx_cmd_start_sq()
3492 	mlwq->mlwq_pc = 0;  in mlxcx_cmd_start_sq()
3498 	in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_start_sq()
3515 		mlwq->mlwq_state |= MLXCX_WQ_STARTED;  in mlxcx_cmd_start_sq()
3522 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_stop_sq()  argument
3532 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_stop_sq()
3533 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_stop_sq()
3534 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_stop_sq()
3535 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_stop_sq()
3541 	in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_stop_sq()
3558 		mlwq->mlwq_state &= ~MLXCX_WQ_STARTED;  in mlxcx_cmd_stop_sq()
3565 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq)  in mlxcx_cmd_destroy_sq()  argument
3575 	ASSERT(mutex_owned(&mlwq->mlwq_mtx));  in mlxcx_cmd_destroy_sq()
3576 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC);  in mlxcx_cmd_destroy_sq()
3577 	VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED);  in mlxcx_cmd_destroy_sq()
3578 	VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED);  in mlxcx_cmd_destroy_sq()
3584 	in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num);  in mlxcx_cmd_destroy_sq()
3594 		mlwq->mlwq_state |= MLXCX_WQ_DESTROYED;  in mlxcx_cmd_destroy_sq()