219 mlxcx_cq_next(mlxcx_completion_queue_t *mlcq)  in mlxcx_cq_next()  argument
224 	const uint_t swowner = ((mlcq->mlcq_cc >> mlcq->mlcq_entshift) & 1);  in mlxcx_cq_next()
226 	ASSERT(mutex_owned(&mlcq->mlcq_mtx));  in mlxcx_cq_next()
227 	ASSERT(mlcq->mlcq_state & MLXCX_CQ_CREATED);  in mlxcx_cq_next()
228 	ASSERT0(mlcq->mlcq_state & MLXCX_CQ_DESTROYED);  in mlxcx_cq_next()
231 	ci = mlcq->mlcq_cc & (mlcq->mlcq_nents - 1);  in mlxcx_cq_next()
233 	ent = &mlcq->mlcq_ent[ci];  in mlxcx_cq_next()
234 	VERIFY0(ddi_dma_sync(mlcq->mlcq_dma.mxdb_dma_handle,  in mlxcx_cq_next()
235 	    (uintptr_t)ent - (uintptr_t)mlcq->mlcq_ent,  in mlxcx_cq_next()
237 	ddi_fm_dma_err_get(mlcq->mlcq_dma.mxdb_dma_handle, &err,  in mlxcx_cq_next()
242 		++mlcq->mlcq_cc;  in mlxcx_cq_next()
245 	ddi_fm_dma_err_clear(mlcq->mlcq_dma.mxdb_dma_handle, DDI_FME_VERSION);  in mlxcx_cq_next()
251 mlxcx_update_cqci(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)  in mlxcx_update_cqci()  argument
256 	mlcq->mlcq_doorbell->mlcqd_update_ci = to_be24(mlcq->mlcq_cc);  in mlxcx_update_cqci()
259 	MLXCX_DMA_SYNC(mlcq->mlcq_doorbell_dma, DDI_DMA_SYNC_FORDEV);  in mlxcx_update_cqci()
260 	ddi_fm_dma_err_get(mlcq->mlcq_doorbell_dma.mxdb_dma_handle, &err,  in mlxcx_update_cqci()
265 			    mlcq->mlcq_doorbell_dma.mxdb_dma_handle,  in mlxcx_update_cqci()
276 mlxcx_arm_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq)  in mlxcx_arm_cq()  argument
283 	ASSERT(mutex_owned(&mlcq->mlcq_arm_mtx));  in mlxcx_arm_cq()
284 	ASSERT(mutex_owned(&mlcq->mlcq_mtx));  in mlxcx_arm_cq()
285 	ASSERT(mlcq->mlcq_state & MLXCX_CQ_CREATED);  in mlxcx_arm_cq()
286 	ASSERT0(mlcq->mlcq_state & MLXCX_CQ_DESTROYED);  in mlxcx_arm_cq()
288 	if (mlcq->mlcq_state & MLXCX_CQ_ARMED) {  in mlxcx_arm_cq()
289 		ASSERT3U(mlcq->mlcq_ec, >, mlcq->mlcq_ec_armed);  in mlxcx_arm_cq()
292 	if (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)  in mlxcx_arm_cq()
295 	atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_ARMED);  in mlxcx_arm_cq()
296 	mlcq->mlcq_cc_armed = mlcq->mlcq_cc;  in mlxcx_arm_cq()
297 	mlcq->mlcq_ec_armed = mlcq->mlcq_ec;  in mlxcx_arm_cq()
299 	set_bits32(&dbval, MLXCX_CQ_ARM_SEQ, mlcq->mlcq_ec);  in mlxcx_arm_cq()
300 	set_bits32(&dbval, MLXCX_CQ_ARM_CI, mlcq->mlcq_cc);  in mlxcx_arm_cq()
303 	udbval |= mlcq->mlcq_num & 0xffffff;  in mlxcx_arm_cq()
305 	mlcq->mlcq_doorbell->mlcqd_update_ci = to_be24(mlcq->mlcq_cc);  in mlxcx_arm_cq()
306 	mlcq->mlcq_doorbell->mlcqd_arm_ci = dbval;  in mlxcx_arm_cq()
309 	MLXCX_DMA_SYNC(mlcq->mlcq_doorbell_dma, DDI_DMA_SYNC_FORDEV);  in mlxcx_arm_cq()
310 	ddi_fm_dma_err_get(mlcq->mlcq_doorbell_dma.mxdb_dma_handle, &err,  in mlxcx_arm_cq()
315 			    mlcq->mlcq_doorbell_dma.mxdb_dma_handle,  in mlxcx_arm_cq()
323 	mlxcx_uar_put64(mlxp, mlcq->mlcq_uar, MLXCX_UAR_CQ_ARM, udbval);  in mlxcx_arm_cq()
865 mlxcx_process_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, mblk_t **mpp,  in mlxcx_process_cq()  argument
868 	mlxcx_work_queue_t *wq = mlcq->mlcq_wq;  in mlxcx_process_cq()
880 	if (!(mlcq->mlcq_state & MLXCX_CQ_ALLOC) ||  in mlxcx_process_cq()
881 	    !(mlcq->mlcq_state & MLXCX_CQ_CREATED) ||  in mlxcx_process_cq()
882 	    (mlcq->mlcq_state & MLXCX_CQ_DESTROYED) ||  in mlxcx_process_cq()
883 	    (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)) {  in mlxcx_process_cq()
891 	for (cent = mlxcx_cq_next(mlcq); cent != NULL;  in mlxcx_process_cq()
892 	    cent = mlxcx_cq_next(mlcq)) {  in mlxcx_process_cq()
897 		if (mlcq->mlcq_state & MLXCX_CQ_TEARDOWN)  in mlxcx_process_cq()
914 		buf = list_head(&mlcq->mlcq_buffers);  in mlxcx_process_cq()
922 			buf = list_next(&mlcq->mlcq_buffers, buf);  in mlxcx_process_cq()
937 			mutex_enter(&mlcq->mlcq_bufbmtx);  in mlxcx_process_cq()
938 			if (!list_is_empty(&mlcq->mlcq_buffers_b)) {  in mlxcx_process_cq()
939 				list_move_tail(&mlcq->mlcq_buffers,  in mlxcx_process_cq()
940 				    &mlcq->mlcq_buffers_b);  in mlxcx_process_cq()
943 			mutex_exit(&mlcq->mlcq_bufbmtx);  in mlxcx_process_cq()
961 			buf = list_head(&mlcq->mlcq_buffers);  in mlxcx_process_cq()
964 			    "buffer counter = %x)", mlcq->mlcq_num,  in mlxcx_process_cq()
977 		list_remove(&mlcq->mlcq_buffers, buf);  in mlxcx_process_cq()
980 		switch (mlcq->mlcq_wq->mlwq_type) {  in mlxcx_process_cq()
982 			mlxcx_tx_completion(mlxp, mlcq, cent, buf);  in mlxcx_process_cq()
985 			nmp = mlxcx_rx_completion(mlxp, mlcq, cent, buf);  in mlxcx_process_cq()
1009 			mlxcx_update_cqci(mlxp, mlcq);  in mlxcx_process_cq()
1015 			atomic_add_64(&mlcq->mlcq_bufcnt, -bufcnt);  in mlxcx_process_cq()
1028 		mlxcx_update_cqci(mlxp, mlcq);  in mlxcx_process_cq()
1029 		atomic_add_64(&mlcq->mlcq_bufcnt, -bufcnt);  in mlxcx_process_cq()
1039 mlxcx_rx_poll(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, size_t bytelim)  in mlxcx_rx_poll()  argument
1043 	ASSERT(mutex_owned(&mlcq->mlcq_mtx));  in mlxcx_rx_poll()
1045 	ASSERT(mlcq->mlcq_wq != NULL);  in mlxcx_rx_poll()
1046 	ASSERT3U(mlcq->mlcq_wq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ);  in mlxcx_rx_poll()
1048 	(void) mlxcx_process_cq(mlxp, mlcq, &mp, bytelim);  in mlxcx_rx_poll()
1059 	mlxcx_completion_queue_t *mlcq, probe;  in mlxcx_intr_n()  local
1108 		mlcq = avl_find(&mleq->mleq_cqs, &probe, NULL);  in mlxcx_intr_n()
1111 		if (mlcq == NULL)  in mlxcx_intr_n()
1114 		mlwq = mlcq->mlcq_wq;  in mlxcx_intr_n()
1127 		mutex_enter(&mlcq->mlcq_arm_mtx);  in mlxcx_intr_n()
1128 		mlcq->mlcq_ec++;  in mlxcx_intr_n()
1129 		atomic_and_uint(&mlcq->mlcq_state, ~MLXCX_CQ_ARMED);  in mlxcx_intr_n()
1131 		if (mutex_tryenter(&mlcq->mlcq_mtx) == 0) {  in mlxcx_intr_n()
1138 			if ((mlcq->mlcq_state & MLXCX_CQ_POLLING) != 0) {  in mlxcx_intr_n()
1139 				mutex_exit(&mlcq->mlcq_arm_mtx);  in mlxcx_intr_n()
1144 			mutex_enter(&mlcq->mlcq_mtx);  in mlxcx_intr_n()
1147 		if ((mlcq->mlcq_state & MLXCX_CQ_POLLING) == 0 &&  in mlxcx_intr_n()
1148 		    mlxcx_process_cq(mlxp, mlcq, &mp, 0)) {  in mlxcx_intr_n()
1153 			if ((mlcq->mlcq_state & MLXCX_CQ_BLOCKED_MAC) != 0 &&  in mlxcx_intr_n()
1154 			    mlcq->mlcq_bufcnt < mlcq->mlcq_buflwm) {  in mlxcx_intr_n()
1155 				atomic_and_uint(&mlcq->mlcq_state,  in mlxcx_intr_n()
1167 			mlxcx_arm_cq(mlxp, mlcq);  in mlxcx_intr_n()
1169 			mutex_exit(&mlcq->mlcq_mtx);  in mlxcx_intr_n()
1170 			mutex_exit(&mlcq->mlcq_arm_mtx);  in mlxcx_intr_n()
1174 				    mlcq->mlcq_mac_hdl);  in mlxcx_intr_n()
1180 				    mlcq->mlcq_mac_hdl, mp, mlcq->mlcq_mac_gen);  in mlxcx_intr_n()
1183 			mutex_exit(&mlcq->mlcq_mtx);  in mlxcx_intr_n()
1184 			mutex_exit(&mlcq->mlcq_arm_mtx);  in mlxcx_intr_n()