340 	mlxcx_completion_queue_t *cq;  in mlxcx_cq_setup()  local
342 	cq = kmem_zalloc(sizeof (mlxcx_completion_queue_t), KM_SLEEP);  in mlxcx_cq_setup()
343 	mutex_init(&cq->mlcq_mtx, NULL, MUTEX_DRIVER,  in mlxcx_cq_setup()
345 	mutex_init(&cq->mlcq_arm_mtx, NULL, MUTEX_DRIVER,  in mlxcx_cq_setup()
347 	mutex_init(&cq->mlcq_bufbmtx, NULL, MUTEX_DRIVER,  in mlxcx_cq_setup()
349 	list_create(&cq->mlcq_buffers, sizeof (mlxcx_buffer_t),  in mlxcx_cq_setup()
351 	list_create(&cq->mlcq_buffers_b, sizeof (mlxcx_buffer_t),  in mlxcx_cq_setup()
354 	cq->mlcq_mlx = mlxp;  in mlxcx_cq_setup()
355 	list_insert_tail(&mlxp->mlx_cqs, cq);  in mlxcx_cq_setup()
357 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_cq_setup()
359 	if (!mlxcx_cq_alloc_dma(mlxp, cq, ent_shift)) {  in mlxcx_cq_setup()
360 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_cq_setup()
364 	cq->mlcq_bufhwm = cq->mlcq_nents - MLXCX_CQ_HWM_GAP;  in mlxcx_cq_setup()
365 	cq->mlcq_buflwm = cq->mlcq_nents - MLXCX_CQ_LWM_GAP;  in mlxcx_cq_setup()
367 	cq->mlcq_uar = &mlxp->mlx_uar;  in mlxcx_cq_setup()
368 	cq->mlcq_eq = eq;  in mlxcx_cq_setup()
370 	cq->mlcq_cqemod_period_usec = mlxp->mlx_props.mldp_cqemod_period_usec;  in mlxcx_cq_setup()
371 	cq->mlcq_cqemod_count = mlxp->mlx_props.mldp_cqemod_count;  in mlxcx_cq_setup()
373 	if (!mlxcx_cmd_create_cq(mlxp, cq)) {  in mlxcx_cq_setup()
374 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_cq_setup()
378 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_cq_setup()
381 	mutex_enter(&cq->mlcq_arm_mtx);  in mlxcx_cq_setup()
382 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_cq_setup()
383 	ASSERT0(cq->mlcq_state & MLXCX_CQ_EQAVL);  in mlxcx_cq_setup()
384 	avl_add(&eq->mleq_cqs, cq);  in mlxcx_cq_setup()
385 	atomic_or_uint(&cq->mlcq_state, MLXCX_CQ_EQAVL);  in mlxcx_cq_setup()
386 	mlxcx_arm_cq(mlxp, cq);  in mlxcx_cq_setup()
387 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_cq_setup()
388 	mutex_exit(&cq->mlcq_arm_mtx);  in mlxcx_cq_setup()
391 	*cqp = cq;  in mlxcx_cq_setup()
396 mlxcx_rq_setup(mlxcx_t *mlxp, mlxcx_completion_queue_t *cq,  in mlxcx_rq_setup()  argument
408 	wq->mlwq_cq = cq;  in mlxcx_rq_setup()
429 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_rq_setup()
431 	ASSERT3P(cq->mlcq_wq, ==, NULL);  in mlxcx_rq_setup()
432 	cq->mlcq_wq = wq;  in mlxcx_rq_setup()
434 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_rq_setup()
440 mlxcx_sq_setup(mlxcx_t *mlxp, mlxcx_port_t *port, mlxcx_completion_queue_t *cq,  in mlxcx_sq_setup()  argument
452 	wq->mlwq_cq = cq;  in mlxcx_sq_setup()
478 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_sq_setup()
480 	ASSERT3P(cq->mlcq_wq, ==, NULL);  in mlxcx_sq_setup()
481 	cq->mlcq_wq = wq;  in mlxcx_sq_setup()
483 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_sq_setup()
496 	mlxcx_completion_queue_t *cq;  in mlxcx_quiesce_rx_cqs()  local
504 		cq = wq->mlwq_cq;  in mlxcx_quiesce_rx_cqs()
505 		if (cq != NULL) {  in mlxcx_quiesce_rx_cqs()
508 			atomic_or_uint(&cq->mlcq_state, MLXCX_CQ_TEARDOWN);  in mlxcx_quiesce_rx_cqs()
521 	mlxcx_completion_queue_t *cq;  in mlxcx_teardown_rx_group()  local
640 		cq = wq->mlwq_cq;  in mlxcx_teardown_rx_group()
642 		if (cq != NULL)  in mlxcx_teardown_rx_group()
643 			mlxcx_cq_teardown(mlxp, cq);  in mlxcx_teardown_rx_group()
662 	mlxcx_completion_queue_t *cq;  in mlxcx_teardown_tx_group()  local
671 			cq = wq->mlwq_cq;  in mlxcx_teardown_tx_group()
679 			if (cq != NULL)  in mlxcx_teardown_tx_group()
680 				mlxcx_cq_teardown(mlxp, cq);  in mlxcx_teardown_tx_group()
744 	mlxcx_completion_queue_t *cq;  in mlxcx_rx_group_setup()  local
800 		if (!mlxcx_cq_setup(mlxp, eq, &cq, ent_shift)) {  in mlxcx_rx_group_setup()
805 		cq->mlcq_stats = &g->mlg_port->mlp_stats;  in mlxcx_rx_group_setup()
808 		if (!mlxcx_rq_setup(mlxp, cq, rq)) {  in mlxcx_rx_group_setup()
1184 	mlxcx_completion_queue_t *cq;  in mlxcx_rx_ring_start()  local
1203 	cq = rq->mlwq_cq;  in mlxcx_rx_ring_start()
1204 	ASSERT(cq != NULL);  in mlxcx_rx_ring_start()
1206 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_rx_ring_start()
1211 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_rx_ring_start()
1218 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_rx_ring_start()
1243 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_rx_ring_start()
1324 	mlxcx_completion_queue_t *cq;  in mlxcx_tx_group_setup()  local
1366 		if (!mlxcx_cq_setup(mlxp, eq, &cq,  in mlxcx_tx_group_setup()
1370 		cq->mlcq_stats = &g->mlg_port->mlp_stats;  in mlxcx_tx_group_setup()
1373 		if (!mlxcx_sq_setup(mlxp, g->mlg_port, cq, &g->mlg_tis, sq)) {  in mlxcx_tx_group_setup()
1391 	mlxcx_completion_queue_t *cq;  in mlxcx_tx_ring_start()  local
1395 	cq = sq->mlwq_cq;  in mlxcx_tx_ring_start()
1396 	ASSERT(cq != NULL);  in mlxcx_tx_ring_start()
1398 	mutex_enter(&cq->mlcq_mtx);  in mlxcx_tx_ring_start()
1402 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_tx_ring_start()
1430 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_tx_ring_start()
1439 	mutex_exit(&cq->mlcq_mtx);  in mlxcx_tx_ring_start()
1545 	mlxcx_completion_queue_t *cq;  in mlxcx_sq_add_buffer()  local
1560 	cq = mlwq->mlwq_cq;  in mlxcx_sq_add_buffer()
1701 	mutex_enter(&cq->mlcq_bufbmtx);  in mlxcx_sq_add_buffer()
1703 	list_insert_tail(&cq->mlcq_buffers_b, b0);  in mlxcx_sq_add_buffer()
1705 		atomic_inc_64(&cq->mlcq_bufcnt);  in mlxcx_sq_add_buffer()
1707 		list_remove(&cq->mlcq_buffers_b, b0);  in mlxcx_sq_add_buffer()
1710 	mutex_exit(&cq->mlcq_bufbmtx);  in mlxcx_sq_add_buffer()
1728 	mlxcx_completion_queue_t *cq;  in mlxcx_rq_add_buffers()  local
1736 	cq = mlwq->mlwq_cq;  in mlxcx_rq_add_buffers()
1737 	ASSERT(mutex_owned(&cq->mlcq_mtx));  in mlxcx_rq_add_buffers()
1752 		mutex_enter(&cq->mlcq_bufbmtx);  in mlxcx_rq_add_buffers()
1753 		list_insert_tail(&cq->mlcq_buffers, buf);  in mlxcx_rq_add_buffers()
1754 		atomic_inc_64(&cq->mlcq_bufcnt);  in mlxcx_rq_add_buffers()
1755 		mutex_exit(&cq->mlcq_bufbmtx);  in mlxcx_rq_add_buffers()
1796 	MLXCX_DMA_SYNC(cq->mlcq_doorbell_dma, DDI_DMA_SYNC_FORDEV);  in mlxcx_rq_add_buffers()
1797 	ddi_fm_dma_err_get(cq->mlcq_doorbell_dma.mxdb_dma_handle, &err,  in mlxcx_rq_add_buffers()
1815 	mlxcx_completion_queue_t *cq = wq->mlwq_cq;  in mlxcx_rq_refill_task()  local
1830 		    (cq->mlcq_state & MLXCX_CQ_TEARDOWN) == 0) {  in mlxcx_rq_refill_task()
1837 		mutex_enter(&cq->mlcq_mtx);  in mlxcx_rq_refill_task()
1840 		if (draining || (cq->mlcq_state & MLXCX_CQ_TEARDOWN) != 0) {  in mlxcx_rq_refill_task()
1846 			if (cq->mlcq_bufcnt < MLXCX_RQ_REFILL_STEP) {  in mlxcx_rq_refill_task()
1855 		mutex_exit(&cq->mlcq_mtx);  in mlxcx_rq_refill_task()
1863 	mlxcx_completion_queue_t *cq;  in mlxcx_rq_refill()  local
1869 	cq = mlwq->mlwq_cq;  in mlxcx_rq_refill()
1870 	ASSERT(mutex_owned(&cq->mlcq_mtx));  in mlxcx_rq_refill()
1875 	cq = mlwq->mlwq_cq;  in mlxcx_rq_refill()
1880 	if ((cq->mlcq_state & MLXCX_CQ_TEARDOWN) != 0)  in mlxcx_rq_refill()
1883 	current = cq->mlcq_bufcnt;  in mlxcx_rq_refill()