137 static void squeue_worker(squeue_t *sqp);
138 static void squeue_polling_thread(squeue_t *sqp);
156 #define	ENQUEUE_CHAIN(sqp, mp, tail, cnt) {			\  argument
160 	ASSERT(MUTEX_HELD(&(sqp)->sq_lock));			\
162 	if ((sqp)->sq_last != NULL)				\
163 		(sqp)->sq_last->b_next = (mp);			\
165 		(sqp)->sq_first = (mp);				\
166 	(sqp)->sq_last = (tail);				\
167 	(sqp)->sq_count += (cnt);				\
168 	ASSERT((sqp)->sq_count > 0);				\
169 	DTRACE_PROBE4(squeue__enqueuechain, squeue_t *, sqp,	\
181 #define	SQS_POLLING_ON(sqp, sq_poll_capable, rx_ring) {		\  argument
182 	ASSERT(MUTEX_HELD(&(sqp)->sq_lock));			\
185 		ASSERT(sqp->sq_state & SQS_POLL_CAPAB);		\
186 		if (!(sqp->sq_state & SQS_POLLING)) {		\
188 				sqp->sq_state |= SQS_POLLING;	\
193 #define	SQS_POLLING_OFF(sqp, sq_poll_capable, rx_ring) {	\  argument
194 	ASSERT(MUTEX_HELD(&(sqp)->sq_lock));			\
197 		ASSERT(sqp->sq_state & SQS_POLL_CAPAB);		\
198 		if (sqp->sq_state & SQS_POLLING) {		\
199 			sqp->sq_state &= ~SQS_POLLING;		\
206 #define	SQS_POLL_RING(sqp) {			\  argument
207 	ASSERT(MUTEX_HELD(&(sqp)->sq_lock));			\
208 	if (sqp->sq_state & SQS_POLLING) {			\
209 		ASSERT(sqp->sq_state & SQS_POLL_CAPAB);		\
210 		if (!(sqp->sq_state & SQS_GET_PKTS)) {		\
211 			sqp->sq_state |= SQS_GET_PKTS;		\
212 			cv_signal(&sqp->sq_poll_cv);		\
218 #define	SQUEUE_DBG_SET(sqp, mp, proc, connp, tag) {		\  argument
219 	(sqp)->sq_curmp = (mp);					\
220 	(sqp)->sq_curproc = (proc);				\
221 	(sqp)->sq_connp = (connp);				\
222 	(mp)->b_tag = (sqp)->sq_tag = (tag);			\
225 #define	SQUEUE_DBG_CLEAR(sqp)	{				\  argument
226 	(sqp)->sq_curmp = NULL;					\
227 	(sqp)->sq_curproc = NULL;				\
228 	(sqp)->sq_connp = NULL;					\
231 #define	SQUEUE_DBG_SET(sqp, mp, proc, connp, tag)  argument
232 #define	SQUEUE_DBG_CLEAR(sqp)  argument
249 	squeue_t *sqp = kmem_cache_alloc(squeue_cache, KM_SLEEP);  in squeue_create()  local
251 	bzero(sqp, sizeof (squeue_t));  in squeue_create()
252 	sqp->sq_bind = PBIND_NONE;  in squeue_create()
253 	sqp->sq_priority = pri;  in squeue_create()
254 	sqp->sq_wait = MSEC_TO_TICK(wait);  in squeue_create()
255 	sqp->sq_worker = thread_create(NULL, 0, squeue_worker,  in squeue_create()
256 	    sqp, 0, &p0, TS_RUN, pri);  in squeue_create()
258 	sqp->sq_poll_thr = thread_create(NULL, 0, squeue_polling_thread,  in squeue_create()
259 	    sqp, 0, &p0, TS_RUN, pri);  in squeue_create()
261 	sqp->sq_enter = squeue_enter;  in squeue_create()
262 	sqp->sq_drain = squeue_drain;  in squeue_create()
264 	return (sqp);  in squeue_create()
275 squeue_bind(squeue_t *sqp, processorid_t bind)  in squeue_bind()  argument
277 	mutex_enter(&sqp->sq_lock);  in squeue_bind()
278 	ASSERT(sqp->sq_bind != PBIND_NONE || bind != PBIND_NONE);  in squeue_bind()
281 	if (sqp->sq_state & SQS_BOUND) {  in squeue_bind()
282 		if (sqp->sq_bind == bind) {  in squeue_bind()
283 			mutex_exit(&sqp->sq_lock);  in squeue_bind()
286 		thread_affinity_clear(sqp->sq_worker);  in squeue_bind()
288 		sqp->sq_state |= SQS_BOUND;  in squeue_bind()
292 		sqp->sq_bind = bind;  in squeue_bind()
294 	thread_affinity_set(sqp->sq_worker, sqp->sq_bind);  in squeue_bind()
295 	mutex_exit(&sqp->sq_lock);  in squeue_bind()
299 squeue_unbind(squeue_t *sqp)  in squeue_unbind()  argument
301 	mutex_enter(&sqp->sq_lock);  in squeue_unbind()
302 	if (!(sqp->sq_state & SQS_BOUND)) {  in squeue_unbind()
303 		mutex_exit(&sqp->sq_lock);  in squeue_unbind()
307 	sqp->sq_state &= ~SQS_BOUND;  in squeue_unbind()
308 	thread_affinity_clear(sqp->sq_worker);  in squeue_unbind()
309 	mutex_exit(&sqp->sq_lock);  in squeue_unbind()
313 squeue_worker_wakeup(squeue_t *sqp)  in squeue_worker_wakeup()  argument
315 	timeout_id_t tid = (sqp)->sq_tid;  in squeue_worker_wakeup()
317 	ASSERT(MUTEX_HELD(&(sqp)->sq_lock));  in squeue_worker_wakeup()
319 	if (sqp->sq_wait == 0) {  in squeue_worker_wakeup()
321 		ASSERT(!(sqp->sq_state & SQS_TMO_PROG));  in squeue_worker_wakeup()
322 		sqp->sq_awaken = ddi_get_lbolt();  in squeue_worker_wakeup()
323 		cv_signal(&sqp->sq_worker_cv);  in squeue_worker_wakeup()
324 		mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
337 		clock_t	waited = now - sqp->sq_awaken;  in squeue_worker_wakeup()
339 		if (TICK_TO_MSEC(waited) >= sqp->sq_wait) {  in squeue_worker_wakeup()
344 			sqp->sq_tid = 0;  in squeue_worker_wakeup()
345 			sqp->sq_awaken = now;  in squeue_worker_wakeup()
346 			cv_signal(&sqp->sq_worker_cv);  in squeue_worker_wakeup()
347 			mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
351 		mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
353 	} else if (sqp->sq_state & SQS_TMO_PROG) {  in squeue_worker_wakeup()
354 		mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
357 		clock_t	wait = sqp->sq_wait;  in squeue_worker_wakeup()
370 		sqp->sq_state |= SQS_TMO_PROG;  in squeue_worker_wakeup()
371 		mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
372 		tid = timeout(squeue_fire, sqp, wait);  in squeue_worker_wakeup()
373 		mutex_enter(&sqp->sq_lock);  in squeue_worker_wakeup()
375 		if (((sqp->sq_state & (SQS_PROC|SQS_TMO_PROG)) ==  in squeue_worker_wakeup()
376 		    SQS_TMO_PROG) && (sqp->sq_tid == 0) &&  in squeue_worker_wakeup()
377 		    (sqp->sq_first != NULL)) {  in squeue_worker_wakeup()
378 				sqp->sq_state &= ~SQS_TMO_PROG;  in squeue_worker_wakeup()
379 				sqp->sq_tid = tid;  in squeue_worker_wakeup()
380 				mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
383 			if (sqp->sq_state & SQS_TMO_PROG) {  in squeue_worker_wakeup()
384 				sqp->sq_state &= ~SQS_TMO_PROG;  in squeue_worker_wakeup()
385 				mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
395 				mutex_exit(&sqp->sq_lock);  in squeue_worker_wakeup()
400 	ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));  in squeue_worker_wakeup()
425 squeue_enter(squeue_t *sqp, mblk_t *mp, mblk_t *tail, uint32_t cnt,  in squeue_enter()  argument
432 	ASSERT(sqp != NULL);  in squeue_enter()
436 	ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));  in squeue_enter()
439 	mutex_enter(&sqp->sq_lock);  in squeue_enter()
446 	if (!(sqp->sq_state & SQS_PROC) && ((process_flag == SQ_PROCESS) ||  in squeue_enter()
447 	    (process_flag == SQ_NODRAIN && sqp->sq_first == NULL))) {  in squeue_enter()
453 		if (sqp->sq_first == NULL && cnt == 1) {  in squeue_enter()
457 			sqp->sq_state |= (SQS_PROC|SQS_FAST);  in squeue_enter()
458 			sqp->sq_run = curthread;  in squeue_enter()
459 			mutex_exit(&sqp->sq_lock);  in squeue_enter()
478 			if (connp->conn_sqp == sqp) {  in squeue_enter()
479 				SQUEUE_DBG_SET(sqp, mp, proc, connp,  in squeue_enter()
483 				    sqp, mblk_t *, mp, conn_t *, connp);  in squeue_enter()
484 				(*proc)(connp, mp, sqp, ira);  in squeue_enter()
486 				    sqp, conn_t *, connp);  in squeue_enter()
488 				SQUEUE_DBG_CLEAR(sqp);  in squeue_enter()
494 			ASSERT(MUTEX_NOT_HELD(&sqp->sq_lock));  in squeue_enter()
495 			mutex_enter(&sqp->sq_lock);  in squeue_enter()
496 			sqp->sq_state &= ~(SQS_PROC|SQS_FAST);  in squeue_enter()
497 			sqp->sq_run = NULL;  in squeue_enter()
498 			if (sqp->sq_first == NULL ||  in squeue_enter()
500 				if (sqp->sq_first != NULL) {  in squeue_enter()
501 					squeue_worker_wakeup(sqp);  in squeue_enter()
510 				if (sqp->sq_state & SQS_WORKER_THR_CONTROL)  in squeue_enter()
511 					cv_signal(&sqp->sq_worker_cv);  in squeue_enter()
512 				mutex_exit(&sqp->sq_lock);  in squeue_enter()
522 					mutex_exit(&sqp->sq_lock);  in squeue_enter()
543 			ENQUEUE_CHAIN(sqp, mp, tail, cnt);  in squeue_enter()
555 		ASSERT(MUTEX_HELD(&sqp->sq_lock));  in squeue_enter()
556 		ASSERT(sqp->sq_first != NULL);  in squeue_enter()
558 		sqp->sq_run = curthread;  in squeue_enter()
559 		sqp->sq_drain(sqp, SQS_ENTER, now + squeue_drain_ns);  in squeue_enter()
567 		sqp->sq_run = NULL;  in squeue_enter()
568 		if (sqp->sq_state & SQS_WORKER_THR_CONTROL)  in squeue_enter()
569 			cv_signal(&sqp->sq_worker_cv);  in squeue_enter()
570 		mutex_exit(&sqp->sq_lock);  in squeue_enter()
587 		if (!(sqp->sq_state & SQS_REENTER) &&  in squeue_enter()
588 		    (process_flag != SQ_FILL) && (sqp->sq_first == NULL) &&  in squeue_enter()
589 		    (sqp->sq_run == curthread) && (cnt == 1) &&  in squeue_enter()
591 			sqp->sq_state |= SQS_REENTER;  in squeue_enter()
592 			mutex_exit(&sqp->sq_lock);  in squeue_enter()
605 			if (connp->conn_sqp == sqp) {  in squeue_enter()
608 				    sqp, mblk_t *, mp, conn_t *, connp);  in squeue_enter()
609 				(*proc)(connp, mp, sqp, ira);  in squeue_enter()
611 				    sqp, conn_t *, connp);  in squeue_enter()
619 			mutex_enter(&sqp->sq_lock);  in squeue_enter()
620 			sqp->sq_state &= ~SQS_REENTER;  in squeue_enter()
621 			mutex_exit(&sqp->sq_lock);  in squeue_enter()
640 				mutex_exit(&sqp->sq_lock);  in squeue_enter()
659 		ENQUEUE_CHAIN(sqp, mp, tail, cnt);  in squeue_enter()
660 		if (!(sqp->sq_state & SQS_PROC)) {  in squeue_enter()
661 			squeue_worker_wakeup(sqp);  in squeue_enter()
668 		if (sqp->sq_state & SQS_WORKER_THR_CONTROL)  in squeue_enter()
669 			cv_signal(&sqp->sq_worker_cv);  in squeue_enter()
670 		mutex_exit(&sqp->sq_lock);  in squeue_enter()
682 	squeue_t	*sqp = arg;  in squeue_fire()  local
685 	mutex_enter(&sqp->sq_lock);  in squeue_fire()
687 	state = sqp->sq_state;  in squeue_fire()
688 	if (sqp->sq_tid == 0 && !(state & SQS_TMO_PROG)) {  in squeue_fire()
689 		mutex_exit(&sqp->sq_lock);  in squeue_fire()
693 	sqp->sq_tid = 0;  in squeue_fire()
701 		sqp->sq_state &= ~SQS_TMO_PROG;  in squeue_fire()
704 		sqp->sq_awaken = ddi_get_lbolt();  in squeue_fire()
705 		cv_signal(&sqp->sq_worker_cv);  in squeue_fire()
707 	mutex_exit(&sqp->sq_lock);  in squeue_fire()
711 squeue_drain(squeue_t *sqp, uint_t proc_type, hrtime_t expire)  in squeue_drain()  argument
718 	ill_rx_ring_t	*sq_rx_ring = sqp->sq_rx_ring;  in squeue_drain()
731 		ASSERT(mutex_owned(&sqp->sq_lock));  in squeue_drain()
732 		sqp->sq_awaken = ddi_get_lbolt();  in squeue_drain()
733 		cv_signal(&sqp->sq_worker_cv);  in squeue_drain()
738 	sq_poll_capable = (sqp->sq_state & SQS_POLL_CAPAB) != 0;  in squeue_drain()
740 	ASSERT(mutex_owned(&sqp->sq_lock));  in squeue_drain()
741 	ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |  in squeue_drain()
744 	head = sqp->sq_first;  in squeue_drain()
745 	sqp->sq_first = NULL;  in squeue_drain()
746 	sqp->sq_last = NULL;  in squeue_drain()
747 	sqp->sq_count = 0;  in squeue_drain()
749 	if ((tid = sqp->sq_tid) != 0)  in squeue_drain()
750 		sqp->sq_tid = 0;  in squeue_drain()
752 	sqp->sq_state |= SQS_PROC | proc_type;  in squeue_drain()
765 	SQS_POLLING_ON(sqp, sq_poll_capable, sq_rx_ring);  in squeue_drain()
766 	mutex_exit(&sqp->sq_lock);  in squeue_drain()
810 		if (connp->conn_sqp == sqp) {  in squeue_drain()
811 			SQUEUE_DBG_SET(sqp, mp, proc, connp,  in squeue_drain()
815 			    sqp, mblk_t *, mp, conn_t *, connp);  in squeue_drain()
816 			(*proc)(connp, mp, sqp, ira);  in squeue_drain()
818 			    sqp, conn_t *, connp);  in squeue_drain()
829 	SQUEUE_DBG_CLEAR(sqp);  in squeue_drain()
831 	mutex_enter(&sqp->sq_lock);  in squeue_drain()
842 	if (sqp->sq_first != NULL) {  in squeue_drain()
865 				SQS_POLL_RING(sqp);  in squeue_drain()
869 			sqp->sq_awaken = ddi_get_lbolt();  in squeue_drain()
870 			cv_signal(&sqp->sq_worker_cv);  in squeue_drain()
879 	if (sqp->sq_state & SQS_GET_PKTS) {  in squeue_drain()
880 		ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |  in squeue_drain()
882 		sqp->sq_state &= ~proc_type;  in squeue_drain()
892 	if ((proc_type == SQS_WORKER) && (sqp->sq_state & SQS_POLLING)) {  in squeue_drain()
912 		ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |  in squeue_drain()
914 		SQS_POLL_RING(sqp);  in squeue_drain()
915 		sqp->sq_state &= ~proc_type;  in squeue_drain()
926 		ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |  in squeue_drain()
928 		SQS_POLLING_OFF(sqp, sq_poll_capable, sq_rx_ring);  in squeue_drain()
929 		sqp->sq_state &= ~(SQS_PROC | proc_type);  in squeue_drain()
930 		if (!did_wakeup && sqp->sq_first != NULL) {  in squeue_drain()
931 			squeue_worker_wakeup(sqp);  in squeue_drain()
932 			mutex_enter(&sqp->sq_lock);  in squeue_drain()
939 		    (sqp->sq_state & SQS_WORKER_THR_CONTROL))  in squeue_drain()
940 			cv_signal(&sqp->sq_worker_cv);  in squeue_drain()
965 squeue_poll_thr_control(squeue_t *sqp)  in squeue_poll_thr_control()  argument
967 	if (sqp->sq_state & SQS_POLL_THR_RESTART) {  in squeue_poll_thr_control()
969 		ASSERT(sqp->sq_state & SQS_POLL_THR_QUIESCED);  in squeue_poll_thr_control()
970 		sqp->sq_state &= ~(SQS_POLL_THR_QUIESCED |  in squeue_poll_thr_control()
972 		sqp->sq_state |= SQS_POLL_CAPAB;  in squeue_poll_thr_control()
973 		cv_signal(&sqp->sq_worker_cv);  in squeue_poll_thr_control()
977 	if (sqp->sq_state & SQS_POLL_THR_QUIESCE) {  in squeue_poll_thr_control()
978 		sqp->sq_state |= SQS_POLL_THR_QUIESCED;  in squeue_poll_thr_control()
979 		sqp->sq_state &= ~SQS_POLL_THR_QUIESCE;  in squeue_poll_thr_control()
980 		cv_signal(&sqp->sq_worker_cv);  in squeue_poll_thr_control()
1031 squeue_polling_thread(squeue_t *sqp)  in squeue_polling_thread()  argument
1033 	kmutex_t *lock = &sqp->sq_lock;  in squeue_polling_thread()
1034 	kcondvar_t *async = &sqp->sq_poll_cv;  in squeue_polling_thread()
1054 		ctl_state = sqp->sq_state & (SQS_POLL_THR_CONTROL |  in squeue_polling_thread()
1068 			squeue_poll_thr_control(sqp);  in squeue_polling_thread()
1072 		if (!(sqp->sq_state & SQS_POLL_CAPAB))  in squeue_polling_thread()
1075 		ASSERT((sqp->sq_state &  in squeue_polling_thread()
1080 		sq_rx_ring = sqp->sq_rx_ring;  in squeue_polling_thread()
1098 			mp = ip_accept(sq_ill, sq_rx_ring, sqp, head,  in squeue_polling_thread()
1107 			ENQUEUE_CHAIN(sqp, mp, tail, cnt);  in squeue_polling_thread()
1109 		ASSERT((sqp->sq_state &  in squeue_polling_thread()
1113 		if (sqp->sq_first != NULL && !(sqp->sq_state & SQS_WORKER)) {  in squeue_polling_thread()
1125 				sqp->sq_run = curthread;  in squeue_polling_thread()
1126 				sqp->sq_drain(sqp, SQS_POLL_PROC, now +  in squeue_polling_thread()
1128 				sqp->sq_run = NULL;  in squeue_polling_thread()
1130 				if (sqp->sq_first == NULL)  in squeue_polling_thread()
1140 			sqp->sq_awaken = ddi_get_lbolt();  in squeue_polling_thread()
1154 			sqp->sq_state |= SQS_PROC_HELD;  in squeue_polling_thread()
1155 			sqp->sq_state &= ~SQS_GET_PKTS;  in squeue_polling_thread()
1156 			cv_signal(&sqp->sq_worker_cv);  in squeue_polling_thread()
1157 		} else if (sqp->sq_first == NULL &&  in squeue_polling_thread()
1158 		    !(sqp->sq_state & SQS_WORKER)) {  in squeue_polling_thread()
1167 			sqp->sq_state &= ~(SQS_PROC|SQS_GET_PKTS);  in squeue_polling_thread()
1169 			SQS_POLLING_OFF(sqp, B_TRUE, sq_rx_ring);  in squeue_polling_thread()
1176 			if (sqp->sq_state & SQS_WORKER_THR_CONTROL)  in squeue_polling_thread()
1177 				cv_signal(&sqp->sq_worker_cv);  in squeue_polling_thread()
1183 			sqp->sq_state &= ~SQS_GET_PKTS;  in squeue_polling_thread()
1185 		if (sqp->sq_state & SQS_POLL_THR_CONTROL) {  in squeue_polling_thread()
1190 			squeue_poll_thr_control(sqp);  in squeue_polling_thread()
1202 squeue_worker_thr_control(squeue_t *sqp)  in squeue_worker_thr_control()  argument
1207 	ASSERT(MUTEX_HELD(&sqp->sq_lock));  in squeue_worker_thr_control()
1209 	if (sqp->sq_state & SQS_POLL_RESTART) {  in squeue_worker_thr_control()
1211 		ASSERT((sqp->sq_state & (SQS_PROC_HELD |  in squeue_worker_thr_control()
1218 		sqp->sq_state &= ~SQS_POLL_QUIESCE_DONE;  in squeue_worker_thr_control()
1219 		sqp->sq_state |= SQS_POLL_THR_RESTART;  in squeue_worker_thr_control()
1220 		cv_signal(&sqp->sq_poll_cv);  in squeue_worker_thr_control()
1221 		while (sqp->sq_state & SQS_POLL_THR_QUIESCED)  in squeue_worker_thr_control()
1222 			cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);  in squeue_worker_thr_control()
1223 		sqp->sq_state &= ~(SQS_POLL_RESTART | SQS_PROC |  in squeue_worker_thr_control()
1229 		sqp->sq_state |= SQS_POLL_RESTART_DONE;  in squeue_worker_thr_control()
1230 		cv_signal(&sqp->sq_ctrlop_done_cv);  in squeue_worker_thr_control()
1234 	if (sqp->sq_state & SQS_PROC_HELD) {  in squeue_worker_thr_control()
1236 		ASSERT(sqp->sq_state & SQS_PROC);  in squeue_worker_thr_control()
1244 	sqp->sq_state |= (SQS_PROC | SQS_WORKER);  in squeue_worker_thr_control()
1247 	if (!(sqp->sq_state & SQS_POLL_THR_QUIESCED)) {  in squeue_worker_thr_control()
1248 		sqp->sq_state |= SQS_POLL_THR_QUIESCE;  in squeue_worker_thr_control()
1249 		cv_signal(&sqp->sq_poll_cv);  in squeue_worker_thr_control()
1250 		while (!(sqp->sq_state & SQS_POLL_THR_QUIESCED))  in squeue_worker_thr_control()
1251 			cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);  in squeue_worker_thr_control()
1254 	rx_ring = sqp->sq_rx_ring;  in squeue_worker_thr_control()
1260 	mutex_exit(&sqp->sq_lock);  in squeue_worker_thr_control()
1262 	mutex_enter(&sqp->sq_lock);  in squeue_worker_thr_control()
1264 	SQS_POLLING_OFF(sqp, (sqp->sq_state & SQS_POLL_CAPAB) != 0,  in squeue_worker_thr_control()
1265 	    sqp->sq_rx_ring);  in squeue_worker_thr_control()
1266 	sqp->sq_state &= ~(SQS_POLL_CAPAB | SQS_GET_PKTS | SQS_PROC_HELD);  in squeue_worker_thr_control()
1267 	if (sqp->sq_state & SQS_POLL_CLEANUP) {  in squeue_worker_thr_control()
1275 		ASSERT(!(sqp->sq_state & SQS_DEFAULT));  in squeue_worker_thr_control()
1277 		sqp->sq_rx_ring = NULL;  in squeue_worker_thr_control()
1280 		sqp->sq_state &= ~(SQS_POLL_CLEANUP | SQS_POLL_THR_QUIESCED |  in squeue_worker_thr_control()
1282 		sqp->sq_ill = NULL;  in squeue_worker_thr_control()
1288 		sqp->sq_state |= SQS_POLL_CLEANUP_DONE;  in squeue_worker_thr_control()
1290 		sqp->sq_state &= ~SQS_POLL_QUIESCE;  in squeue_worker_thr_control()
1291 		sqp->sq_state |= SQS_POLL_QUIESCE_DONE;  in squeue_worker_thr_control()
1298 	cv_signal(&sqp->sq_ctrlop_done_cv);  in squeue_worker_thr_control()
1300 	if (sqp->sq_state & SQS_POLL_CLEANUP_DONE) {  in squeue_worker_thr_control()
1301 		cv_wait(&sqp->sq_worker_cv, &sqp->sq_lock);  in squeue_worker_thr_control()
1302 		sqp->sq_state &= ~(SQS_PROC | SQS_WORKER);  in squeue_worker_thr_control()
1307 squeue_worker(squeue_t *sqp)  in squeue_worker()  argument
1309 	kmutex_t *lock = &sqp->sq_lock;  in squeue_worker()
1310 	kcondvar_t *async = &sqp->sq_worker_cv;  in squeue_worker()
1323 			if (sqp->sq_state & SQS_PROC_HELD)  in squeue_worker()
1331 			if (!(sqp->sq_state & SQS_PROC) &&  in squeue_worker()
1332 			    ((sqp->sq_state & SQS_WORKER_THR_CONTROL) ||  in squeue_worker()
1333 			    (sqp->sq_first != NULL)))  in squeue_worker()
1342 			if ((sqp->sq_state & SQS_WORKER_THR_CONTROL) &&  in squeue_worker()
1343 			    (sqp->sq_state & SQS_WORKER))  in squeue_worker()
1350 		if (sqp->sq_state & SQS_WORKER_THR_CONTROL) {  in squeue_worker()
1351 			squeue_worker_thr_control(sqp);  in squeue_worker()
1354 		ASSERT(!(sqp->sq_state & (SQS_POLL_THR_QUIESCED |  in squeue_worker()
1358 		if (sqp->sq_state & SQS_PROC_HELD)  in squeue_worker()
1359 			sqp->sq_state &= ~SQS_PROC_HELD;  in squeue_worker()
1362 		sqp->sq_run = curthread;  in squeue_worker()
1363 		sqp->sq_drain(sqp, SQS_WORKER, now +  squeue_drain_ns);  in squeue_worker()
1364 		sqp->sq_run = NULL;  in squeue_worker()
1369 squeue_getprivate(squeue_t *sqp, sqprivate_t p)  in squeue_getprivate()  argument
1373 	return (&sqp->sq_private[p]);  in squeue_getprivate()
1381 	squeue_t *sqp = connp->conn_sqp;  in squeue_wakeup_conn()  local
1387 	mutex_enter(&sqp->sq_lock);  in squeue_wakeup_conn()
1388 	sqp->sq_state |= SQS_PAUSE;  in squeue_wakeup_conn()
1402 	while (sqp->sq_state & SQS_PAUSE) {  in squeue_wakeup_conn()
1403 		cv_wait(&sqp->sq_synch_cv, &sqp->sq_lock);  in squeue_wakeup_conn()
1405 	mutex_exit(&sqp->sq_lock);  in squeue_wakeup_conn()
1411 	squeue_t *sqp;  in squeue_synch_enter()  local
1414 	sqp = connp->conn_sqp;  in squeue_synch_enter()
1416 	mutex_enter(&sqp->sq_lock);  in squeue_synch_enter()
1417 	if (sqp->sq_first == NULL && !(sqp->sq_state & SQS_PROC)) {  in squeue_synch_enter()
1425 		ASSERT(sqp->sq_run == NULL);  in squeue_synch_enter()
1427 		sqp->sq_state |= SQS_PROC;  in squeue_synch_enter()
1428 		sqp->sq_run = curthread;  in squeue_synch_enter()
1429 		mutex_exit(&sqp->sq_lock);  in squeue_synch_enter()
1437 		if (sqp != connp->conn_sqp) {  in squeue_synch_enter()
1438 			mutex_enter(&sqp->sq_lock);  in squeue_synch_enter()
1439 			sqp->sq_state &= ~SQS_PROC;  in squeue_synch_enter()
1440 			sqp->sq_run = NULL;  in squeue_synch_enter()
1441 			mutex_exit(&sqp->sq_lock);  in squeue_synch_enter()
1445 		sqp->sq_curmp = NULL;  in squeue_synch_enter()
1446 		sqp->sq_curproc = NULL;  in squeue_synch_enter()
1447 		sqp->sq_connp = connp;  in squeue_synch_enter()
1456 			mutex_exit(&sqp->sq_lock);  in squeue_synch_enter()
1470 		ENQUEUE_CHAIN(sqp, mp, mp, 1);  in squeue_synch_enter()
1472 		ASSERT(sqp->sq_run != curthread);  in squeue_synch_enter()
1476 			cv_wait(&connp->conn_sq_cv, &sqp->sq_lock);  in squeue_synch_enter()
1477 		mutex_exit(&sqp->sq_lock);  in squeue_synch_enter()
1489 	squeue_t *sqp = connp->conn_sqp;  in squeue_synch_exit()  local
1491 	mutex_enter(&sqp->sq_lock);  in squeue_synch_exit()
1492 	if (sqp->sq_run == curthread) {  in squeue_synch_exit()
1493 		ASSERT(sqp->sq_state & SQS_PROC);  in squeue_synch_exit()
1495 		sqp->sq_state &= ~SQS_PROC;  in squeue_synch_exit()
1496 		sqp->sq_run = NULL;  in squeue_synch_exit()
1499 		if (sqp->sq_first == NULL) {  in squeue_synch_exit()
1500 			mutex_exit(&sqp->sq_lock);  in squeue_synch_exit()
1511 			sqp->sq_awaken = ddi_get_lbolt();  in squeue_synch_exit()
1512 			cv_signal(&sqp->sq_worker_cv);  in squeue_synch_exit()
1513 			mutex_exit(&sqp->sq_lock);  in squeue_synch_exit()
1521 		ASSERT(sqp->sq_state & SQS_PAUSE);  in squeue_synch_exit()
1522 		sqp->sq_state &= ~SQS_PAUSE;  in squeue_synch_exit()
1525 		cv_signal(&sqp->sq_synch_cv);  in squeue_synch_exit()
1526 		mutex_exit(&sqp->sq_lock);  in squeue_synch_exit()