52 		mlxcx_event_queue_t *mleq = &mlxp->mlx_eqs[i];  in mlxcx_intr_disable()  local
54 		mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_disable()
56 		if ((mleq->mleq_state & MLXCX_EQ_INTR_ENABLED) == 0) {  in mlxcx_intr_disable()
57 			mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_disable()
63 		mleq->mleq_state |= MLXCX_EQ_INTR_QUIESCE;  in mlxcx_intr_disable()
64 		while ((mleq->mleq_state & MLXCX_EQ_INTR_ACTIVE) != 0)  in mlxcx_intr_disable()
65 			cv_wait(&mleq->mleq_cv, &mleq->mleq_mtx);  in mlxcx_intr_disable()
67 		mleq->mleq_state &= ~MLXCX_EQ_INTR_ENABLED;  in mlxcx_intr_disable()
69 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_disable()
80 		mlxcx_event_queue_t *mleq = &mlxp->mlx_eqs[i];  in mlxcx_intr_teardown()  local
82 		mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_teardown()
83 		VERIFY0(mleq->mleq_state & MLXCX_EQ_ALLOC);  in mlxcx_intr_teardown()
84 		if (mleq->mleq_state & MLXCX_EQ_CREATED)  in mlxcx_intr_teardown()
85 			VERIFY(mleq->mleq_state & MLXCX_EQ_DESTROYED);  in mlxcx_intr_teardown()
87 			VERIFY(avl_is_empty(&mleq->mleq_cqs));  in mlxcx_intr_teardown()
88 			avl_destroy(&mleq->mleq_cqs);  in mlxcx_intr_teardown()
90 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_teardown()
97 		mutex_destroy(&mleq->mleq_mtx);  in mlxcx_intr_teardown()
98 		cv_destroy(&mleq->mleq_cv);  in mlxcx_intr_teardown()
110 mlxcx_eq_next(mlxcx_event_queue_t *mleq)  in mlxcx_eq_next()  argument
115 	const uint_t swowner = ((mleq->mleq_cc >> mleq->mleq_entshift) & 1);  in mlxcx_eq_next()
122 	ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);  in mlxcx_eq_next()
123 	ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);  in mlxcx_eq_next()
126 	ci = mleq->mleq_cc & (mleq->mleq_nents - 1);  in mlxcx_eq_next()
128 	ent = &mleq->mleq_ent[ci];  in mlxcx_eq_next()
129 	VERIFY0(ddi_dma_sync(mleq->mleq_dma.mxdb_dma_handle,  in mlxcx_eq_next()
130 	    (uintptr_t)ent - (uintptr_t)mleq->mleq_ent,  in mlxcx_eq_next()
132 	ddi_fm_dma_err_get(mleq->mleq_dma.mxdb_dma_handle, &err,  in mlxcx_eq_next()
137 		++mleq->mleq_cc;  in mlxcx_eq_next()
146 	ddi_fm_dma_err_clear(mleq->mleq_dma.mxdb_dma_handle, DDI_FME_VERSION);  in mlxcx_eq_next()
152 mlxcx_arm_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)  in mlxcx_arm_eq()  argument
163 	ASSERT(mutex_owned(&mleq->mleq_mtx) || servicing_interrupt());  in mlxcx_arm_eq()
164 	ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);  in mlxcx_arm_eq()
165 	ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);  in mlxcx_arm_eq()
166 	ASSERT0(mleq->mleq_state & MLXCX_EQ_ARMED);  in mlxcx_arm_eq()
167 	ASSERT0(mleq->mleq_state & MLXCX_EQ_POLLING);  in mlxcx_arm_eq()
169 	mleq->mleq_state |= MLXCX_EQ_ARMED;  in mlxcx_arm_eq()
170 	mleq->mleq_cc_armed = mleq->mleq_cc;  in mlxcx_arm_eq()
172 	set_bits32(&v, MLXCX_EQ_ARM_EQN, mleq->mleq_num);  in mlxcx_arm_eq()
173 	set_bits32(&v, MLXCX_EQ_ARM_CI, mleq->mleq_cc);  in mlxcx_arm_eq()
176 	mlxcx_uar_put32(mlxp, mleq->mleq_uar, MLXCX_UAR_EQ_ARM,  in mlxcx_arm_eq()
190 mlxcx_update_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)  in mlxcx_update_eq()  argument
200 	ASSERT(mleq->mleq_state & MLXCX_EQ_CREATED);  in mlxcx_update_eq()
201 	ASSERT0(mleq->mleq_state & MLXCX_EQ_DESTROYED);  in mlxcx_update_eq()
202 	ASSERT0(mleq->mleq_state & MLXCX_EQ_ARMED);  in mlxcx_update_eq()
204 	set_bits32(&v, MLXCX_EQ_ARM_EQN, mleq->mleq_num);  in mlxcx_update_eq()
205 	set_bits32(&v, MLXCX_EQ_ARM_CI, mleq->mleq_cc);  in mlxcx_update_eq()
207 	mlxcx_uar_put32(mlxp, mleq->mleq_uar, MLXCX_UAR_EQ_NOARM,  in mlxcx_update_eq()
702 mlxcx_intr_ini(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)  in mlxcx_intr_ini()  argument
704 	mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_ini()
706 	if ((mleq->mleq_state & MLXCX_EQ_INTR_ENABLED) == 0) {  in mlxcx_intr_ini()
707 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_ini()
711 	if (!(mleq->mleq_state & MLXCX_EQ_ALLOC) ||  in mlxcx_intr_ini()
712 	    !(mleq->mleq_state & MLXCX_EQ_CREATED) ||  in mlxcx_intr_ini()
713 	    (mleq->mleq_state & MLXCX_EQ_DESTROYED)) {  in mlxcx_intr_ini()
715 		    mleq->mleq_intr_index);  in mlxcx_intr_ini()
716 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_ini()
720 	mleq->mleq_state |= MLXCX_EQ_INTR_ACTIVE;  in mlxcx_intr_ini()
721 	mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_ini()
732 mlxcx_intr_fini(mlxcx_event_queue_t *mleq)  in mlxcx_intr_fini()  argument
734 	mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_fini()
735 	if ((mleq->mleq_state & MLXCX_EQ_INTR_QUIESCE) != 0)  in mlxcx_intr_fini()
736 		cv_signal(&mleq->mleq_cv);  in mlxcx_intr_fini()
738 	mleq->mleq_state &= ~MLXCX_EQ_INTR_ACTIVE;  in mlxcx_intr_fini()
739 	mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_fini()
746 	mlxcx_event_queue_t *mleq = (mlxcx_event_queue_t *)arg2;  in mlxcx_intr_async()  local
752 	if (!mlxcx_intr_ini(mlxp, mleq))  in mlxcx_intr_async()
755 	ent = mlxcx_eq_next(mleq);  in mlxcx_intr_async()
760 	ASSERT(mleq->mleq_state & MLXCX_EQ_ARMED);  in mlxcx_intr_async()
761 	mleq->mleq_state &= ~MLXCX_EQ_ARMED;  in mlxcx_intr_async()
763 	for (; ent != NULL; ent = mlxcx_eq_next(mleq)) {  in mlxcx_intr_async()
812 		mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_async()
813 		if (mleq->mleq_state & MLXCX_EQ_ATTACHING) {  in mlxcx_intr_async()
814 			mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_async()
817 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_async()
853 			    ent->mleqe_event_type, mleq->mleq_intr_index);  in mlxcx_intr_async()
857 	mlxcx_arm_eq(mlxp, mleq);  in mlxcx_intr_async()
860 	mlxcx_intr_fini(mleq);  in mlxcx_intr_async()
1057 	mlxcx_event_queue_t *mleq = (mlxcx_event_queue_t *)arg2;  in mlxcx_intr_n()  local
1064 	if (!mlxcx_intr_ini(mlxp, mleq))  in mlxcx_intr_n()
1067 	ent = mlxcx_eq_next(mleq);  in mlxcx_intr_n()
1069 		if (++mleq->mleq_badintrs > mlxcx_stuck_intr_count) {  in mlxcx_intr_n()
1073 			    mleq->mleq_intr_index]);  in mlxcx_intr_n()
1077 	mleq->mleq_badintrs = 0;  in mlxcx_intr_n()
1079 	mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_n()
1080 	ASSERT(mleq->mleq_state & MLXCX_EQ_ARMED);  in mlxcx_intr_n()
1081 	mleq->mleq_state &= ~MLXCX_EQ_ARMED;  in mlxcx_intr_n()
1089 	if (mleq->mleq_state & MLXCX_EQ_ATTACHING) {  in mlxcx_intr_n()
1090 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_n()
1092 		    "vector", mleq->mleq_intr_index);  in mlxcx_intr_n()
1096 		    mleq->mleq_intr_index]);  in mlxcx_intr_n()
1100 	mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_n()
1102 	for (; ent != NULL; ent = mlxcx_eq_next(mleq)) {  in mlxcx_intr_n()
1107 		mutex_enter(&mleq->mleq_mtx);  in mlxcx_intr_n()
1108 		mlcq = avl_find(&mleq->mleq_cqs, &probe, NULL);  in mlxcx_intr_n()
1109 		mutex_exit(&mleq->mleq_mtx);  in mlxcx_intr_n()
1194 		if ((mleq->mleq_cc & 0x7) == 0)  in mlxcx_intr_n()
1195 			mlxcx_update_eq(mlxp, mleq);  in mlxcx_intr_n()
1198 	mlxcx_arm_eq(mlxp, mleq);  in mlxcx_intr_n()
1201 	mlxcx_intr_fini(mleq);  in mlxcx_intr_n()