397     mlxcx_work_queue_t *wq)  in mlxcx_rq_setup()  argument
399 	mutex_init(&wq->mlwq_mtx, NULL, MUTEX_DRIVER,  in mlxcx_rq_setup()
402 	list_insert_tail(&mlxp->mlx_wqs, wq);  in mlxcx_rq_setup()
404 	mutex_enter(&wq->mlwq_mtx);  in mlxcx_rq_setup()
406 	wq->mlwq_mlx = mlxp;  in mlxcx_rq_setup()
407 	wq->mlwq_type = MLXCX_WQ_TYPE_RECVQ;  in mlxcx_rq_setup()
408 	wq->mlwq_cq = cq;  in mlxcx_rq_setup()
409 	wq->mlwq_pd = &mlxp->mlx_pd;  in mlxcx_rq_setup()
410 	wq->mlwq_uar = &mlxp->mlx_uar;  in mlxcx_rq_setup()
412 	wq->mlwq_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_rq_setup()
414 	if (!mlxcx_wq_alloc_dma(mlxp, wq)) {  in mlxcx_rq_setup()
415 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_rq_setup()
419 	if (!mlxcx_cmd_create_rq(mlxp, wq)) {  in mlxcx_rq_setup()
420 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_rq_setup()
424 	wq->mlwq_bufhwm = wq->mlwq_nents - MLXCX_WQ_HWM_GAP;  in mlxcx_rq_setup()
425 	wq->mlwq_buflwm = wq->mlwq_nents - MLXCX_WQ_LWM_GAP;  in mlxcx_rq_setup()
427 	mutex_exit(&wq->mlwq_mtx);  in mlxcx_rq_setup()
430 	mutex_enter(&wq->mlwq_mtx);  in mlxcx_rq_setup()
432 	cq->mlcq_wq = wq;  in mlxcx_rq_setup()
433 	mutex_exit(&wq->mlwq_mtx);  in mlxcx_rq_setup()
441     mlxcx_tis_t *tis, mlxcx_work_queue_t *wq)  in mlxcx_sq_setup()  argument
443 	mutex_init(&wq->mlwq_mtx, NULL, MUTEX_DRIVER,  in mlxcx_sq_setup()
446 	list_insert_tail(&mlxp->mlx_wqs, wq);  in mlxcx_sq_setup()
448 	mutex_enter(&wq->mlwq_mtx);  in mlxcx_sq_setup()
450 	wq->mlwq_mlx = mlxp;  in mlxcx_sq_setup()
451 	wq->mlwq_type = MLXCX_WQ_TYPE_SENDQ;  in mlxcx_sq_setup()
452 	wq->mlwq_cq = cq;  in mlxcx_sq_setup()
453 	wq->mlwq_pd = &mlxp->mlx_pd;  in mlxcx_sq_setup()
454 	wq->mlwq_uar = &mlxp->mlx_uar;  in mlxcx_sq_setup()
455 	wq->mlwq_tis = tis;  in mlxcx_sq_setup()
457 	wq->mlwq_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_sq_setup()
458 	wq->mlwq_foreign_bufs = mlxcx_mlbs_create(mlxp);  in mlxcx_sq_setup()
461 	wq->mlwq_inline_mode = MLXCX_ETH_INLINE_L2;  in mlxcx_sq_setup()
463 	if (!mlxcx_wq_alloc_dma(mlxp, wq)) {  in mlxcx_sq_setup()
464 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_sq_setup()
468 	if (!mlxcx_cmd_create_sq(mlxp, wq)) {  in mlxcx_sq_setup()
469 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_sq_setup()
473 	wq->mlwq_bufhwm = wq->mlwq_nents - MLXCX_WQ_HWM_GAP;  in mlxcx_sq_setup()
474 	wq->mlwq_buflwm = wq->mlwq_nents - MLXCX_WQ_LWM_GAP;  in mlxcx_sq_setup()
476 	mutex_exit(&wq->mlwq_mtx);  in mlxcx_sq_setup()
479 	mutex_enter(&wq->mlwq_mtx);  in mlxcx_sq_setup()
481 	cq->mlcq_wq = wq;  in mlxcx_sq_setup()
482 	mutex_exit(&wq->mlwq_mtx);  in mlxcx_sq_setup()
495 	mlxcx_work_queue_t *wq;  in mlxcx_quiesce_rx_cqs()  local
503 		wq = &g->mlg_wqs[i];  in mlxcx_quiesce_rx_cqs()
504 		cq = wq->mlwq_cq;  in mlxcx_quiesce_rx_cqs()
506 			s = wq->mlwq_bufs;  in mlxcx_quiesce_rx_cqs()
520 	mlxcx_work_queue_t *wq;  in mlxcx_teardown_rx_group()  local
594 			wq = &g->mlg_wqs[i];  in mlxcx_teardown_rx_group()
595 			mutex_enter(&wq->mlwq_mtx);  in mlxcx_teardown_rx_group()
596 			if (wq->mlwq_state & MLXCX_WQ_STARTED &&  in mlxcx_teardown_rx_group()
597 			    !mlxcx_cmd_stop_rq(mlxp, wq)) {  in mlxcx_teardown_rx_group()
599 				    wq->mlwq_num);  in mlxcx_teardown_rx_group()
601 			mutex_exit(&wq->mlwq_mtx);  in mlxcx_teardown_rx_group()
639 		wq = &g->mlg_wqs[i];  in mlxcx_teardown_rx_group()
640 		cq = wq->mlwq_cq;  in mlxcx_teardown_rx_group()
641 		mlxcx_wq_teardown(mlxp, wq);  in mlxcx_teardown_rx_group()
661 	mlxcx_work_queue_t *wq;  in mlxcx_teardown_tx_group()  local
669 			wq = &g->mlg_wqs[i];  in mlxcx_teardown_tx_group()
670 			mutex_enter(&wq->mlwq_mtx);  in mlxcx_teardown_tx_group()
671 			cq = wq->mlwq_cq;  in mlxcx_teardown_tx_group()
672 			if (wq->mlwq_state & MLXCX_WQ_STARTED &&  in mlxcx_teardown_tx_group()
673 			    !mlxcx_cmd_stop_sq(mlxp, wq)) {  in mlxcx_teardown_tx_group()
675 				    wq->mlwq_num);  in mlxcx_teardown_tx_group()
677 			mutex_exit(&wq->mlwq_mtx);  in mlxcx_teardown_tx_group()
678 			mlxcx_wq_teardown(mlxp, wq);  in mlxcx_teardown_tx_group()
1814 	mlxcx_work_queue_t *wq = arg;  in mlxcx_rq_refill_task()  local
1815 	mlxcx_completion_queue_t *cq = wq->mlwq_cq;  in mlxcx_rq_refill_task()
1816 	mlxcx_t *mlxp = wq->mlwq_mlx;  in mlxcx_rq_refill_task()
1817 	mlxcx_buf_shard_t *s = wq->mlwq_bufs;  in mlxcx_rq_refill_task()
1838 		mutex_enter(&wq->mlwq_mtx);  in mlxcx_rq_refill_task()
1842 			wq->mlwq_state &= ~MLXCX_WQ_REFILLING;  in mlxcx_rq_refill_task()
1844 			mlxcx_rq_refill(mlxp, wq);  in mlxcx_rq_refill_task()
1850 				wq->mlwq_state &= ~MLXCX_WQ_REFILLING;  in mlxcx_rq_refill_task()
1854 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_rq_refill_task()
2134 		mlxcx_work_queue_t *wq = mlcq->mlcq_wq;  in mlxcx_rx_completion()  local
2135 		ASSERT(wq != NULL);  in mlxcx_rx_completion()
2136 		mutex_enter(&wq->mlwq_mtx);  in mlxcx_rx_completion()
2137 		if (!(wq->mlwq_state & MLXCX_WQ_TEARDOWN))  in mlxcx_rx_completion()
2138 			mlxcx_rq_refill(mlxp, wq);  in mlxcx_rx_completion()
2139 		mutex_exit(&wq->mlwq_mtx);  in mlxcx_rx_completion()
2216 mlxcx_buf_take_foreign(mlxcx_t *mlxp, mlxcx_work_queue_t *wq)  in mlxcx_buf_take_foreign()  argument
2219 	mlxcx_buf_shard_t *s = wq->mlwq_foreign_bufs;  in mlxcx_buf_take_foreign()
2239 mlxcx_copy_data(mlxcx_t *mlxp, mlxcx_work_queue_t *wq, uint8_t *rptr, size_t sz)  in mlxcx_copy_data()  argument
2246 	if ((b = mlxcx_buf_take(mlxp, wq)) == NULL)  in mlxcx_copy_data()
2270 mlxcx_bind_or_copy_mblk(mlxcx_t *mlxp, mlxcx_work_queue_t *wq,  in mlxcx_bind_or_copy_mblk()  argument
2291 		b = mlxcx_copy_data(mlxp, wq, rptr, sz);  in mlxcx_bind_or_copy_mblk()
2293 		b = mlxcx_buf_take_foreign(mlxp, wq);  in mlxcx_bind_or_copy_mblk()
2303 			b = mlxcx_copy_data(mlxp, wq, rptr, sz);  in mlxcx_bind_or_copy_mblk()
2311 mlxcx_buf_bind_or_copy(mlxcx_t *mlxp, mlxcx_work_queue_t *wq,  in mlxcx_buf_bind_or_copy()  argument
2323 		b = mlxcx_bind_or_copy_mblk(mlxp, wq, mp, offset);  in mlxcx_buf_bind_or_copy()
2354 		DTRACE_PROBE4(pullup, mlxcx_t *, mlxp, mlxcx_work_queue_t *, wq,  in mlxcx_buf_bind_or_copy()
2363 		b0 = mlxcx_bind_or_copy_mblk(mlxp, wq, mp, off);  in mlxcx_buf_bind_or_copy()
2389 mlxcx_buf_take(mlxcx_t *mlxp, mlxcx_work_queue_t *wq)  in mlxcx_buf_take()  argument
2392 	mlxcx_buf_shard_t *s = wq->mlwq_bufs;  in mlxcx_buf_take()
2411 mlxcx_buf_take_n(mlxcx_t *mlxp, mlxcx_work_queue_t *wq, mlxcx_buffer_t **bp,  in mlxcx_buf_take_n()  argument
2418 	s = wq->mlwq_bufs;  in mlxcx_buf_take_n()